2543 files changed, 114578 insertions, 40201 deletions

diff --git a/ChangeLog b/ChangeLog
index e0d7275..c899a86 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,27 @@
+2025-08-01  Luis Machado  <luis.machado.foss@gmail.com>
+
+	* MAINTAINERS: Update my e-mail address.
+
+2025-07-22  Patrick Palka  <ppalka@redhat.com>
+
+	* MAINTAINERS: Add myself as C++ front end reviewer.
+
+2025-07-07  Tamar Christina  <tamar.christina@arm.com>
+
+	* MAINTAINERS: Add myself to AArch64 pot.
+
+2025-07-04  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	* MAINTAINERS: Replace tabs with spaces.
+
+2025-07-04  Alex Coplan  <alex.coplan@arm.com>
+
+	* MAINTAINERS (Reviewers): Add myself for the aarch64 port.
+
+2025-07-03  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	* MAINTAINERS: Add myself as an aarch64 port reviewer.
+
 2025-06-22  Nicolas Boulenguez  <nicolas@debian.org>
 
 	PR ada/120106
diff --git a/MAINTAINERS b/MAINTAINERS
index 6f66374..6148ce0 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -57,6 +57,7 @@ docs, and the testsuite related to that.
 aarch64 ldp/stp         Alex Coplan             <alex.coplan@arm.com>
 aarch64 port            Richard Earnshaw        <richard.earnshaw@arm.com>
 aarch64 port            Richard Sandiford       <richard.sandiford@arm.com>
+aarch64 port            Tamar Christina         <tamar.christina@arm.com>
 aarch64 port            Kyrylo Tkachov          <ktkachov@nvidia.com>
 alpha port              Richard Henderson       <rth@gcc.gnu.org>
 amdgcn port             Julian Brown            <julian@codesourcery.com>
@@ -274,11 +275,14 @@ check in changes outside of the parts of the compiler they maintain.
 
                         Reviewers
 
+aarch64 port            Alex Coplan             <alex.coplan@arm.com>
+aarch64 port            Andrew Pinski           <pinskia@gmail.com>
 arm port (MVE)          Christophe Lyon         <christophe.lyon@arm.com>
 callgraph               Martin Jambor           <mjambor@suse.cz>
 C front end             Marek Polacek           <polacek@redhat.com>
 CTF, BTF                Indu Bhagat             <indu.bhagat@oracle.com>
 CTF, BTF, bpf port      David Faust             <david.faust@oracle.com>
+c++                     Patrick Palka           <ppalka@gcc.gnu.org>
 dataflow                Paolo Bonzini           <bonzini@gnu.org>
 dataflow                Seongbae Park           <seongbae.park@gmail.com>
 dataflow                Kenneth Zadeck          <zadeck@naturalbridge.com>
@@ -649,7 +653,7 @@ Christophe Lyon                 clyon           <christophe.lyon@arm.com>
 Jin Ma                          majin           <jinma@linux.alibaba.com>
 Jun Ma                          junma           <junma@linux.alibaba.com>
 Andrew MacLeod                  -               <amacleod@redhat.com>
-Luis Machado                    luisgpm         <luisgpm@br.ibm.com>
+Luis Machado                    luisgpm         <luis.machado.foss@gmail.com>
 Ziga Mahkovec                   ziga            <ziga.mahkovec@klika.si>
 Vladimir Makarov                vmakarov        <vmakarov@redhat.com>
 David Malcolm                   dmalcolm        <dmalcolm@redhat.com>
diff --git a/contrib/ChangeLog b/contrib/ChangeLog
index ce9dfec..f4f48e5 100644
--- a/contrib/ChangeLog
+++ b/contrib/ChangeLog
@@ -1,3 +1,50 @@
+2025-07-28  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc-changelog/git_commit.py: Add "diagnostics" to bug
+	components.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* filter-clang-warnings.py: Update for move of
+	diagnostic-path-output.cc to diagnostics/paths-output.cc.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.doxy (INPUT): Add gcc/diagnostics subdir.
+
+2025-07-23  Sam James  <sam@gentoo.org>
+
+	PR other/105404
+	* gcc-changelog/git_commit.py (ignored_prefixes): Add zlib/.
+
+2025-07-16  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.doxy (INPUT): Add directory "gcc/text-art".
+
+2025-07-11  Richard Biener  <rguenther@suse.de>
+
+	* gcc-changelog/git_update_version.py: Stop updating gcc-12
+	branch.
+
+2025-07-08  Pierre-Emmanuel Patry  <pierre-emmanuel.patry@embecosm.com>
+
+	* prepare-commit-msg: Force default git prefixes.
+
+2025-07-07  Martin Jambor  <mjambor@suse.cz>
+
+	* filter-clang-warnings.py (skip_warning): Also ignore
+	-Woverloaded-shift-op-parentheses, -Wunused-function,
+	-Wunneeded-internal-declaration, -Wvla-cxx-extension', and
+	-Wunused-command-line-argument everywhere and a warning about
+	m_logical_loc_mgr in diagnostic-path-output.cc.  Adjust gimple-match
+	and generic-match "filenames."  Ignore -Wnontrivial-memcall warnings
+	in wide-int.h, all warnings about unused stuff in files under
+	m2/gm2-compiler-boot, all -Wunused-private-field in rust FE, in
+	analyzer/ana-state-to-diagnostic-state.h and c-family/c-format.cc, all
+	Warnings in avr-mmcu.texi, install.texi and libgccjit.texi and all
+	-Wc23-extensions warnings in libiberty/sha1.c. Ignore
+	-Wunused-parameter in analyzer/sm.cc.  Reorder entries.
+
 2025-06-29  Filip Kastl  <fkastl@suse.cz>
 
 	* mklog.py: In 'main()', specify variable 'root' as global.
diff --git a/contrib/filter-clang-warnings.py b/contrib/filter-clang-warnings.py
index 2ea7c71..20705b2 100755
--- a/contrib/filter-clang-warnings.py
+++ b/contrib/filter-clang-warnings.py
@@ -41,22 +41,43 @@ def skip_warning(filename, message):
                  '-Wignored-attributes', '-Wgnu-zero-variadic-macro-arguments',
                  '-Wformat-security', '-Wundefined-internal',
                  '-Wunknown-warning-option', '-Wc++20-extensions',
-                 '-Wbitwise-instead-of-logical', 'egrep is obsolescent'],
+                 '-Wbitwise-instead-of-logical', 'egrep is obsolescent',
+                 '-Woverloaded-shift-op-parentheses',
+                 '-Wunused-function', '-Wunneeded-internal-declaration',
+                 '-Wvla-cxx-extension', '-Wunused-command-line-argument'],
+
+            'diagnostics/paths-output.cc': ['m_logical_loc_mgr'],
+            'fold-const-call.cc': ['-Wreturn-type'],
+            'gimple-match': ['-Wunused-', '-Wtautological-compare'],
+            'generic-match': ['-Wunused-', '-Wtautological-compare'],
+            'genautomata.cc': ['-Wstring-plus-int'],
+            # Perhaps revisit when ATTR_FNSPEC_DECONST_WATERMARK ifdef case is
+            # made default or removed:
+            'ipa-strub.cc': ['-Wunused-but-set-variable'],
             'insn-modes.cc': ['-Wshift-count-overflow'],
             'insn-emit.cc': ['-Wtautological-compare'],
             'insn-attrtab.cc': ['-Wparentheses-equality'],
-            'gimple-match.cc': ['-Wunused-', '-Wtautological-compare'],
-            'generic-match.cc': ['-Wunused-', '-Wtautological-compare'],
+            'omp-builtins.def': ['-Wc++11-narrowing'],
+            'wide-int.h': ['-Wnontrivial-memcall'],
             'i386.md': ['-Wparentheses-equality', '-Wtautological-compare',
                         '-Wtautological-overlap-compare'],
             'sse.md': ['-Wparentheses-equality', '-Wtautological-compare',
                        '-Wconstant-logical-operand'],
             'mmx.md': ['-Wtautological-compare'],
-            'genautomata.cc': ['-Wstring-plus-int'],
-            'fold-const-call.cc': ['-Wreturn-type'],
-            'gfortran.texi': [''],
-            'libtool': [''],
             'lex.cc': ['-Wc++20-attribute-extensions'],
+            # Perhaps remove once PR 120960 is resolved:
+            'analyzer/ana-state-to-diagnostic-state.h': ['-Wunused-private-field'],
+            'analyzer/sm.cc': ['-Wunused-parameter'],
+            'c-family/c-format.cc': ['-Wunused-private-field'],
+            'm2/gm2-compiler-boot': ['-Wunused-'],
+            # Rust peopel promised to clean these warnings too
+            'rust/': ['-Wunused-private-field'],
+                 'libiberty/sha1.c': ['-Wc23-extensions'],
+            'avr-mmcu.texi': [''],
+            'gfortran.texi': [''],
+            'install.texi': [''],
+            'libgccjit.texi': [''],
+            'libtool': ['']
     }
 
     for name, ignore in ignores.items():
diff --git a/contrib/gcc-changelog/git_commit.py b/contrib/gcc-changelog/git_commit.py
index 5645f80..e0c46be 100755
--- a/contrib/gcc-changelog/git_commit.py
+++ b/contrib/gcc-changelog/git_commit.py
@@ -97,6 +97,7 @@ bug_components = {
     'd',
     'debug',
     'demangler',
+    'diagnostics',
     'driver',
     'fastjar',
     'fortran',
@@ -152,6 +153,7 @@ ignored_prefixes = {
     'libphobos/libdruntime/',
     'libphobos/src/',
     'libsanitizer/',
+    'zlib/',
     }
 
 wildcard_prefixes = {
diff --git a/contrib/gcc-changelog/git_update_version.py b/contrib/gcc-changelog/git_update_version.py
index aa9adee..b3ea33b 100755
--- a/contrib/gcc-changelog/git_update_version.py
+++ b/contrib/gcc-changelog/git_update_version.py
@@ -85,7 +85,7 @@ def prepend_to_changelog_files(repo, folder, git_commit, add_to_git):
             repo.git.add(full_path)
 
 
-active_refs = ['master', 'releases/gcc-12',
+active_refs = ['master',
                'releases/gcc-13', 'releases/gcc-14', 'releases/gcc-15']
 
 parser = argparse.ArgumentParser(description='Update DATESTAMP and generate '
diff --git a/contrib/gcc.doxy b/contrib/gcc.doxy
index 9e0a1af..1595204 100644
--- a/contrib/gcc.doxy
+++ b/contrib/gcc.doxy
@@ -478,7 +478,7 @@ WARN_LOGFILE           =
 # directories like "/usr/src/myproject". Separate the files or directories 
 # with spaces.
 
-INPUT                  = gcc gcc/analyzer
+INPUT                  = gcc gcc/analyzer gcc/diagnostics gcc/text-art
 
 # This tag can be used to specify the character encoding of the source files that 
 # doxygen parses. Internally doxygen uses the UTF-8 encoding, which is also the default 
diff --git a/contrib/prepare-commit-msg b/contrib/prepare-commit-msg
index 1b87877..75d1025 100755
--- a/contrib/prepare-commit-msg
+++ b/contrib/prepare-commit-msg
@@ -78,4 +78,4 @@ else
     tee="cat"
 fi
 
-git $cmd | $tee | git gcc-mklog -c "$COMMIT_MSG_FILE"
+git $cmd --default-prefix | $tee | git gcc-mklog -c "$COMMIT_MSG_FILE"
diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 94cb5ae..fec8c7a 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,4764 @@
+2025-08-02  Gerald Pfeifer  <gerald@pfeifer.com>
+
+	PR target/69374
+	* doc/install.texi (Specific) <windows>: Drop note on 16-bit
+	Windows support. Streamline note on 32-bit support.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121350
+	* tree-vect-stmts.cc (vectorizable_store): Pass down SLP
+	node when costing scalar stores in vect_body.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121349
+	* tree-vect-stmts.cc (check_load_store_for_partial_vectors):
+	Get full SLP mask, reduce to uniform scalar_mask for further
+	processing if possible.
+	(vect_check_scalar_mask): Remove scalar mask output, remove
+	code conditional on slp_mask.
+	(vectorizable_call): Adjust.
+	(check_scan_store): Get and check SLP mask.
+	(vectorizable_store): Eliminate scalar mask variable.
+	(vectorizable_load): Likewise.
+
+2025-08-01  Gerald Pfeifer  <gerald@pfeifer.com>
+
+	* doc/install.texi (Prerequisites): mdocml.bsd.lv is now
+	mandoc.bsd.lv.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (get_group_load_store_type): Remove,
+	inline into ...
+	(get_load_store_type): ... this.  Remove ncopies parameter.
+	(vectorizable_load): Adjust.
+	(vectorizable_store): Likewise.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (get_group_load_store_type): Remove
+	checks performed at SLP build time.
+	(vect_check_store_rhs): Remove scalar RHS output.
+	(vectorizable_store): Remove uses of scalar RHS.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (VMAT_UNINITIALIZED): New
+	vect_memory_access_type.
+	* tree-vect-slp.cc (_slp_tree::_slp_tree): Use it.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121338
+	* tree-ssa-loop-ivopts.cc (avg_loop_niter): Return an
+	unsigned.
+	(adjust_setup_cost): When niters is so large the division
+	result is one or zero avoid it.
+	(create_new_ivs): Adjust.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_simd_clone_data): New.
+	(_slp_tree::simd_clone_info): Remove.
+	(SLP_TREE_SIMD_CLONE_INFO): Likewise.
+	* tree-vect-slp.cc (_slp_tree::_slp_tree): Adjust.
+	(_slp_tree::~_slp_tree): Likewise.
+	* tree-vect-stmts.cc (vectorizable_simd_clone_call): Use
+	tyupe specific data to store SLP_TREE_SIMD_CLONE_INFO.
+
+2025-08-01  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-slp.cc (_slp_tree::_slp_tree): Adjust.
+	(_slp_tree::~_slp_tree): Likewise.
+	* tree-vectorizer.h (vect_data): New base class.
+	(_slp_tree::u): Remove.
+	(_slp_tree::data): Add pointer to vect_data.
+	(_slp_tree::get_data): New helper template.
+
+2025-08-01  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/121322
+	* gimple-ssa-store-merging.cc (find_bswap_or_nop): Return NULL if
+	count is 0.
+
+2025-07-31  Georg-Johann Lay  <avr@gjlay.de>
+
+	* config/avr/avr.opt.urls (-mfuse-move2): Add url.
+
+2025-07-31  Georg-Johann Lay  <avr@gjlay.de>
+
+	* config/avr/avr.cc (avr_output_addr_vec) <labl>: Asm out its .type.
+
+2025-07-31  Georg-Johann Lay  <avr@gjlay.de>
+
+	PR rtl-optimization/121340
+	* config/avr/avr.opt (-mfuse-move2): New option.
+	* config/avr/avr-passes.def (avr_pass_2moves): Insert after combine.
+	* config/avr/avr-passes.cc (make_avr_pass_2moves): New function.
+	(pass_data avr_pass_data_2moves): New static variable.
+	(avr_pass_2moves): New rtl_opt_pass.
+	* config/avr/avr-protos.h (make_avr_pass_2moves): New proto.
+	* common/config/avr/avr-common.cc
+	(default_options avr_option_optimization_table) <-mfuse-move2>:
+	Set for -O1 and higher.
+	* doc/invoke.texi (AVR Options) <-mfuse-move2>: Document.
+
+2025-07-31  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/120805
+	* tree-vect-loop-manip.cc (vect_gen_vector_loop_niters): Skip setting
+	bounds on epilogues.
+
+2025-07-31  Wilco Dijkstra  <wilco.dijkstra@arm.com>
+
+	* common/config/aarch64/cpuinfo.h: Remove unused features, add FEAT_CSSC
+	and FEAT_MOPS.
+	* config/aarch64/aarch64-option-extensions.def: Remove FMV support
+	for RPRES, use PULL rather than AES, add FMV support for CSSC and MOPS.
+
+2025-07-31  Wilco Dijkstra  <wilco.dijkstra@arm.com>
+
+	* config/aarch64/tuning_models/generic_armv9_a.h
+	(generic_armv9_a_addrcost_table): Use zero cost for himode.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (get_group_load_store_type): Properly
+	compare the scalar type of the gather/scatter offset to
+	the offset vector component type.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* gimple-fold.h (fold_stmt_inplace): Add valueization hook
+	argument, defaulted to no_follow_ssa_edges.
+	* gimple-fold.cc (fold_stmt_inplace): Adjust.
+
+2025-07-31  Artemiy Granat  <a.granat@ispras.ru>
+
+	* config/i386/i386-options.cc (ix86_handle_cconv_attribute):
+	Fix typo.
+
+2025-07-31  Artemiy Granat  <a.granat@ispras.ru>
+
+	* config/i386/i386-options.cc (ix86_handle_cconv_attribute):
+	Handle simultaneous use of regparm and thiscall attributes in
+	case when regparm is set before thiscall.
+
+2025-07-31  Artemiy Granat  <a.granat@ispras.ru>
+
+	* config/i386/i386-options.cc (ix86_handle_cconv_attribute):
+	Fix comments which state that combination of stdcall and fastcall
+	attributes is valid but redundant.
+
+2025-07-31  Artemiy Granat  <a.granat@ispras.ru>
+
+	* config/i386/i386-options.cc (ix86_handle_cconv_attribute):
+	Move 64-bit mode check before regparm handling.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121320
+	* tree-ssa-sccvn.cc (ao_ref_init_from_vn_reference): Convert
+	op->off to poly_offset_int before multiplying by
+	BITS_PER_UNIT.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121323
+	* tree-ssa-alias.cc (ao_ref_init_from_ptr_and_range): Check
+	the pointer offset fits in a HWI when represented in bits.
+
+2025-07-31  Yury Khrustalev  <yury.khrustalev@arm.com>
+
+	* config/aarch64/aarch64.cc (build_ifunc_arg_type):
+	Add new fields _hwcap3 and _hwcap4.
+
+2025-07-31  Kishan Parmar  <kishan@linux.ibm.com>
+
+	PR target/118890
+	* config/rs6000/rs6000.cc (can_be_rotated_to_negative_lis): Avoid left
+	shift of negative value and guard shift count.
+	(can_be_built_by_li_and_rldic): Likewise.
+	(rs6000_emit_set_long_const): Likewise.
+	* config/rs6000/rs6000.md (splitter for plus into two 16-bit parts): Fix
+	UB from overflow in addition.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* config/aarch64/aarch64.cc (aarch64_detect_vector_stmt_subtype):
+	Check for node before dereferencing.
+	(aarch64_vector_costs::add_stmt_cost): Likewise.
+
+2025-07-31  Spencer Abson  <spencer.abson@arm.com>
+
+	PR target/121028
+	* config/aarch64/aarch64-sme.md (aarch64_smstart_sm): Use the .inst
+	directive if !TARGET_SME.
+	(aarch64_smstop_sm): Likewise.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (_stmt_vec_info::memory_access_type): Remove.
+	(STMT_VINFO_MEMORY_ACCESS_TYPE): Likewise.
+	(vect_mem_access_type): Likewise.
+	* tree-vect-stmts.cc (vectorizable_store): Do not set
+	STMT_VINFO_MEMORY_ACCESS_TYPE.  Fix SLP_TREE_MEMORY_ACCESS_TYPE
+	usage.
+	* tree-vect-loop.cc (update_epilogue_loop_vinfo): Remove
+	checking of memory access type.
+	* config/riscv/riscv-vector-costs.cc (costs::compute_local_live_ranges):
+	Use SLP_TREE_MEMORY_ACCESS_TYPE.
+	(costs::need_additional_vector_vars_p): Likewise.
+	(segment_loadstore_group_size): Get SLP node as argument,
+	use SLP_TREE_MEMORY_ACCESS_TYPE.
+	(costs::adjust_stmt_cost): Pass down SLP node.
+	* config/aarch64/aarch64.cc (aarch64_ld234_st234_vectors): Use
+	SLP_TREE_MEMORY_ACCESS_TYPE instead of vect_mem_access_type.
+	(aarch64_detect_vector_stmt_subtype): Likewise.
+	(aarch64_vector_costs::count_ops): Likewise.
+	(aarch64_vector_costs::add_stmt_cost): Likewise.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vect_transform_loop): Do not verify DRs
+	have not been modified for epilogue loops.
+	(update_epilogue_loop_vinfo): Do not copy modified DRs to
+	the originals.
+
+2025-07-31  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/121264
+	* machmode.h (get_best_mode): Change type of first 2 arguments
+	from int to HOST_WIDE_INT.
+	* stor-layout.cc (get_best_mode): Likewise.
+
+2025-07-31  Jakub Jelinek  <jakub@redhat.com>
+
+	* gimple-ssa-store-merging.cc (find_bswap_or_nop): Fix comment typos,
+	hanlde -> handle.
+	* config/i386/i386.cc (ix86_gimple_fold_builtin, ix86_rtx_costs):
+	Likewise.
+	* config/i386/i386-features.cc (remove_partial_avx_dependency):
+	Likewise.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (check_scan_store): Remove redundant
+	slp_node check.  Disallow epilogue vectorization.
+
+2025-07-31  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vector_costs::costing_for_scalar): New
+	accessor.
+	(add_stmt_cost): For scalar costing force vectype to NULL.
+	Verify we do not pass in a SLP node.
+
+2025-07-31  Kito Cheng  <kito.cheng@sifive.com>
+
+	PR target/121312
+	* config/riscv/arch-canonicalize: Add H extension to the
+	canonical order.
+
+2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR rtl-optimization/121308
+	* simplify-rtx.cc (simplify_context::simplify_subreg): Handle
+	subreg of `not` with word_mode to make it symmetric with the
+	other bitwise operators.
+
+2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/121236
+	PR tree-optimization/121295
+	* tree-if-conv.cc (factor_out_operators): Change the phi node
+	to the new result and args.
+
+2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	Revert:
+	2025-07-28  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/121236
+	* tree-if-conv.cc (is_cond_scalar_reduction): Instead of phi argument,
+	pass bb and res of the phi.
+	(factor_out_operators): Add iterator for the phi. Remove the phi
+	if this is the first time. Return if we had removed the phi.
+	(predicate_scalar_phi): Add the phi iterator argument.
+	Update call to is_cond_scalar_reduction.
+	Update call to factor_out_operators and set the return value to true
+	when factor_out_operators returns true.
+	(predicate_all_scalar_phis): Don't remove the phi if predicate_scalar_phi
+	already removed it.
+
+2025-07-30  Jan Hubicka  <jh@suse.cz>
+
+	* auto-profile.cc (string_table::read): Check gcov_is_error.
+	(read_profile): Likewise.
+	* gcov-io.cc (gcov_is_error): Export for gcc linkage.
+	* gcov-io.h (gcov_is_error): Declare.
+
+2025-07-30  Richard Biener  <rguenther@suse.de>
+
+	* config/i386/i386.cc (ix86_default_vector_cost): Split
+	out from ...
+	(ix86_builtin_vectorization_cost): ... this and use
+	mode instead of vectype as argument.
+	(ix86_vector_costs::add_stmt_cost): Call
+	ix86_default_vector_cost instead of ix86_builtin_vectorization_cost.
+
+2025-07-30  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	PR target/117015
+	* config/s390/s390-protos.h (s390_expand_int_spaceship): New
+	function.
+	(s390_expand_fp_spaceship): New function.
+	* config/s390/s390.cc (s390_expand_int_spaceship): New function.
+	(s390_expand_fp_spaceship): New function.
+	* config/s390/s390.md (spaceship<mode>4): New expander.
+
+2025-07-30  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* cprop.cc (bypass_block): Extract single set.
+	(bypass_conditional_jumps): Ditto.
+
+2025-07-30  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120427
+	* config/i386/i386.md (peephole2): Transform "movq $-1,reg" to
+	"pushq $-1; popq reg" for -Oz if reg is a legacy integer register.
+
+2025-07-30  Jan Hubicka  <jh@suse.cz>
+
+	* auto-profile.cc (function_instance::match): Disable warning
+	about bogus locations since dwarf does not represent enough
+	info to output them correctly in all cases.
+	(add_scale): Use nonzero_p instead of orig.force_nonzero () == orig.
+	(afdo_adjust_guessed_profile): Add missing newline in dump
+	file.
+
+2025-07-30  Jan Hubicka  <jh@suse.cz>
+
+	* symtab.cc (symbol_table::change_decl_assembler_name): Recompute DECL_RTL
+	in case it is already computed.
+
+2025-07-30  Jan Hubicka  <jh@suse.cz>
+
+	* predict.cc (unlikely_executed_edge_p): Ignore EDGE_EH if profile
+	is reliable.
+	(unlikely_executed_stmt_p): special case builtin_trap/unreachable and
+	ignore other heuristics for reliable profiles.
+	(tree_estimate_probability): Disable unlikely bb detection when
+	doing dry run
+
+2025-07-30  Andrew Stubbs  <ams@baylibre.com>
+	     Julian Brown   <julian@codesourcery.com>
+
+	* doc/tm.texi.in (TARGET_VECTORIZE_PREFER_GATHER_SCATTER): Add
+	documentation hook.
+	* doc/tm.texi: Regenerate.
+	* target.def (prefer_gather_scatter): Add target hook under vectorizer.
+	* hooks.cc (hook_bool_mode_int_unsigned_false): New function.
+	* hooks.h (hook_bool_mode_int_unsigned_false): New prototype.
+	* tree-vect-stmts.cc (vect_use_strided_gather_scatters_p): Add
+	parameters group_size and single_element_p, and rework to use
+	targetm.vectorize.prefer_gather_scatter.
+	(get_group_load_store_type): Move some of the condition into
+	vect_use_strided_gather_scatters_p.
+	* config/gcn/gcn.cc (gcn_prefer_gather_scatter): New function.
+	(TARGET_VECTORIZE_PREFER_GATHER_SCATTER): Define hook.
+
+2025-07-30  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn.cc (gcn_option_override): Add note to set default for
+	param_vect_partial_vector_usage to "1".
+	* optc-save-gen.awk: Don't pass through options marked "NoOffload".
+	* params.opt (-param=vect-epilogues-nomask): Add NoOffload.
+	(-param=vect-partial-vector-usage): Likewise.
+	(-param=vect-inner-loop-cost-factor): Likewise.
+
+2025-07-30  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121130
+	* tree-vect-stmts.cc (vectorizable_call): Bail out for
+	.MASK_CALL.
+
+2025-07-30  Pengfei Li  <Pengfei.Li2@arm.com>
+
+	PR tree-optimization/121020
+	* tree-vect-loop-manip.cc (vect_do_peeling): Update the
+	condition of omitting the skip-vector check.
+	* tree-vectorizer.h (LOOP_VINFO_USE_VERSIONING_WITHOUT_PEELING):
+	Add a helper macro.
+
+2025-07-30  Pengfei Li  <Pengfei.Li2@arm.com>
+
+	PR tree-optimization/121190
+	* tree-vect-data-refs.cc (vect_enhance_data_refs_alignment):
+	Increase alignment requirement for speculative loads.
+
+2025-07-30  Alfie Richards  <alfie.richards@arm.com>
+
+	PR target/121300
+	* config/aarch64/aarch64-sve-builtins-sme.def (svamin/svamax): Fix
+	arch gating.
+
+2025-07-30  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (get_group_load_store_type):
+	Process STMT_VINFO_GATHER_SCATTER before reading
+	memory_access_type.
+
+2025-07-30  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (@cond_<optab><mode>): Extend
+	to support partial FP modes.
+	(*cond_<optab><mode>_2_strict): Extend from SVE_FULL_F to SVE_F,
+	use aarch64_predicate_operand.
+	(*cond_<optab><mode>_4_strict): Extend from SVE_FULL_F_B16B16 to
+	SVE_F_B16B16, use aarch64_predicate_operand.
+	(*cond_<optab><mode>_any_strict):  Likewise.
+
+2025-07-30  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (*cond_<optab><mode>_2_relaxed):
+	Extend from SVE_FULL_F to SVE_F.
+	(*cond_<optab><mode>_4_relaxed): Extend from SVE_FULL_F_B16B16
+	to SVE_F_B16B16.
+	(*cond_<optab><mode>_any_relaxed): Likewise.
+
+2025-07-30  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (<optab><mode>4): Extend from
+	SVE_FULL_F_B16B16 to SVE_F_B16B16.  Use aarch64_sve_fp_pred instead
+	of aarch64_ptrue_reg.
+	(@aarch64_pred_<optab><mode>): Extend from SVE_FULL_F_B16B16 to
+	SVE_F_B16B16.  Use aarch64_predicate_operand.
+
+2025-07-30  liuhongt  <hongtao.liu@intel.com>
+
+	* config/i386/i386-modes.def: Remove VECTOR_MODES(FLOAT, 256)
+	and VECTOR_MODE (INT, SI, 64).
+	* config/i386/i386.cc (ix86_hard_regno_nregs): Remove related
+	code for V64SF/V64SImode.
+
+2025-07-30  liuhongt  <hongtao.liu@intel.com>
+
+	PR target/121274
+	* config/i386/sse.md (*vec_concatv2di_0): Add a splitter
+	before it.
+
+2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR rtl-optimization/121302
+	* simplify-rtx.cc (simplify_context::simplify_subreg): Use
+	byte instead of 0 when calling simplify_subreg.
+
+2025-07-29  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-protos.h (aarch64_sve_emit_masked_fp_pred):
+	Declare.
+	* config/aarch64/aarch64-sve.md (and<mode>3):  Change this to...
+	(@and<mode>3): ...this, so that we can use gen_and3.
+	(@cond_<optab><mode>): Extend from SVE_FULL_F_B16B16 to SVE_F_B16B16,
+	use aarch64_predicate_operand.
+	(*cond_<optab><mode>_2_strict): Likewise.
+	(*cond_<optab><mode>_3_strict): Likewise.
+	(*cond_<optab><mode>_any_strict): Likwise.
+	(*cond_<optab><mode>_2_const_strict): Extend from SVE_FULL_F to SVE_F,
+	use aarch64_predicate_operand.
+	(*cond_<optab><mode>_any_const_strict): Likewise.
+	(*cond_sub<mode>_3_const_strict): Likwise.
+	(*cond_sub<mode>_const_strict): Likewise.
+	(*vcond_mask_<mode><vpred>): Use aarch64_predicate_operand, and update
+	the comment here.
+	* config/aarch64/aarch64.cc (aarch64_sve_emit_masked_fp_pred): New
+	function.  Helper to mask the predicate in conditional expanders.
+
+2025-07-29  Dongyan Chen  <chendongyan@isrc.iscas.ac.cn>
+
+	* Makefile.in: Add riscv-mcpu.texi and riscv-mtune.texi to the list
+	of files to be processed by the Texinfo generator.
+	* config/riscv/t-riscv: Add rule for generating riscv-mcpu.texi
+	and riscv-mtune.texi.
+	* doc/invoke.texi: Replace hand‑written extension table with
+	`@include riscv-mcpu.texi` and `@include riscv-mtune.texi` to
+	pull in auto‑generated entries.
+	* config/riscv/gen-riscv-mcpu-texi.cc: New file.
+	* config/riscv/gen-riscv-mtune-texi.cc: New file.
+	* doc/riscv-mcpu.texi: New file.
+	* doc/riscv-mtune.texi: New file.
+
+2025-07-29  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* simplify-rtx.cc (simplify_context::simplify_subreg): Distribute
+	lowpart subregs through AND/IOR/XOR, if doing so eliminates one
+	of the terms.
+	(test_scalar_int_ext_ops): Add some tests of the above for integers.
+	* config/aarch64/aarch64.cc (aarch64_test_sve_folding): Likewise
+	add tests for predicate modes.
+
+2025-07-29  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve-builtins.cc
+	(function_expander::get_reg_target): Check whether the target
+	is a valid register_operand.
+
+2025-07-29  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	PR rtl-optimization/120660
+	* avoid-store-forwarding.cc (process_store_forwarding):
+	Fix instruction generation when haveing multiple stores with
+	base offset.
+
+2025-07-29  Christoph Müllner  <christoph.muellner@vrull.eu>
+
+	* common/config/riscv/riscv-common.cc (riscv_ext_is_subset):
+	Remove use of structured binding to fix compiler warning.
+
+2025-07-29  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	PR rtl-optimization/119795
+	* avoid-store-forwarding.cc
+	(store_forwarding_analyzer::avoid_store_forwarding): Skip
+	transformations for stores that operate on the same address
+	range as deleted ones.
+
+2025-07-29  Pan Li  <pan2.li@intel.com>
+
+	* match.pd: Add mul based unsigned SAT_MUL.
+
+2025-07-29  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120687
+	* tree-ssa-reassoc.cc (reassociate_bb): Do not disturb
+	the sorted operand order in the early pass.
+	* tree-vect-slp.cc (vect_analyze_slp): Dump when a detected
+	reduction chain fails SLP discovery.
+
+2025-07-29  Alfie Richards  <alfie.richards@arm.com>
+
+	PR middle-end/121261
+	* vec.h: Add null ptr check.
+
+2025-07-29  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/121159
+	* calls.cc (can_implement_as_sibling_call_p): Don't reject declared
+	noreturn functions in musttail calls.
+
+2025-07-29  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	* output.h (MAX_ALIGN_MERGABLE): New define.
+	* tree-switch-conversion.cc (switch_conversion::build_one_array):
+	Use MAX_ALIGN_MERGABLE instead of 256.
+	* varasm.cc (mergeable_string_section): Likewise
+	(mergeable_constant_section): Likewise
+
+2025-07-29  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/120523
+	* output.h (mergeable_constant_section): New declaration taking
+	unsigned HOST_WIDE_INT for the size.
+	* tree-switch-conversion.cc (switch_conversion::build_one_array):
+	Increase the alignment of CSWTCH for sizes less than 32bytes.
+	* varasm.cc (mergeable_constant_section): Split out twice.
+	One that takes the size in unsigned HOST_WIDE_INT and the
+	other size in a tree.
+	(default_elf_select_section): Pass DECL_SIZE instead of
+	DECL_MODE to mergeable_constant_section.
+
+2025-07-29  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_load): Un-factor VMAT
+	specific code to their handling blocks.
+
+2025-07-29  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (gather_scatter_info::offset_dt): Remove.
+	* tree-vect-data-refs.cc (vect_describe_gather_scatter_call):
+	Do not set it.
+	(vect_check_gather_scatter): Likewise.
+	* tree-vect-stmts.cc (vect_truncate_gather_scatter_offset):
+	Likewise.
+	(get_group_load_store_type): Use the vector type of the offset
+	SLP child.  Do not re-check vect_is_simple_use validated by
+	SLP build.
+
+2025-07-28  Georg-Johann Lay  <avr@gjlay.de>
+
+	PR target/121277
+	* config/avr/avr.cc (avr_addr_space_convert): When converting
+	from generic AS to __flashx, don't set bit 23.
+	(avr_convert_to_type): Don't -Waddr-space-convert when NULL
+	is converted to __flashx or to __flash.
+
+2025-07-28  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/121236
+	* tree-if-conv.cc (is_cond_scalar_reduction): Instead of phi argument,
+	pass bb and res of the phi.
+	(factor_out_operators): Add iterator for the phi. Remove the phi
+	if this is the first time. Return if we had removed the phi.
+	(predicate_scalar_phi): Add the phi iterator argument.
+	Update call to is_cond_scalar_reduction.
+	Update call to factor_out_operators and set the return value to true
+	when factor_out_operators returns true.
+	(predicate_all_scalar_phis): Don't remove the phi if predicate_scalar_phi
+	already removed it.
+
+2025-07-28  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/121208
+	* config/i386/i386.cc (ix86_tls_get_addr): Issue an error for
+	-mtls-dialect=gnu with no_caller_saved_registers attribute and
+	suggest -mtls-dialect=gnu2.
+
+2025-07-28  Mikael Pettersson  <mikpelinux@gmail.com>
+
+	PR other/121260
+	* diagnostics/changes.cc: Correct nesting of namespaces
+	and #if CHECKING_P blocks.
+	* diagnostics/context.cc: Likewise.
+	* diagnostics/html-sink.cc: Likewise.
+	* diagnostics/output-spec.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+
+2025-07-28  Tobias Burnus  <tburnus@baylibre.com>
+
+	* config/nvptx/nvptx.opt (march-map=): Add sm_100{,f,a},
+	sm_101{,f,a}, sm_103{,a,f}, sm_120{,a,f} and sm_121{,f,a}.
+
+2025-07-28  Tobias Burnus  <tburnus@baylibre.com>
+
+	* config/gcn/gcn.md (atomic_load, atomic_store, atomic_exchange):
+	Fix CDNA3 L2 cache write-back before atomic instructions.
+
+2025-07-28  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (check_load_store_for_partial_vectors):
+	Make *gs_info const.
+	(vect_build_one_gather_load_call): Likewise.
+	(vect_build_one_scatter_store_call): Likewise.
+	(vect_get_gather_scatter_ops): Likewise.
+	(vect_get_strided_load_store_ops): Likewise.
+
+2025-07-28  Tobias Burnus  <tburnus@baylibre.com>
+
+	* config/gcn/gcn.md  (define_attr "vcmp"): Add with values
+	vcmp/vcmpx/no.
+	(*movbi, cstoredi4.., cstore<mode>4): Set it.
+	* config/gcn/gcn-valu.md (vec_cmp<mode>...): Likewise.
+	* config/gcn/gcn.cc (gcn_cmpx_insn_p): Remove.
+	(gcn_md_reorg): Add two new conditions for MI300.
+
+2025-07-28  Tobias Burnus  <tburnus@baylibre.com>
+
+	* config/gcn/gcn-opts.h (enum hsaco_attr_type): Add comment
+	about 'sc0'.
+	* config/gcn/gcn.cc (gcn_md_reorg): Use gen_nops instead of gen_nop.
+	(print_operand_address): Document 'R' and 'V' in the
+	pre-function comment as well.
+	* config/gcn/gcn.md (nops): Add.
+
+2025-07-28  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121256
+	* tree-vect-loop.cc (vectorizable_recurr): Build a correct
+	initialization vector for SLP_TREE_LANES > 1.
+
+2025-07-28  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (_slp_tree::type): Add.
+	(_slp_tree::u): Likewise.
+	(_stmt_vec_info::type): Remove.
+	(STMT_VINFO_TYPE): Likewise.
+	(SLP_TREE_TYPE): New.
+	* tree-vectorizer.cc (vec_info::new_stmt_vec_info): Do not
+	initialize type.
+	* tree-vect-slp.cc (_slp_tree::_slp_tree): Initialize type.
+	(vect_slp_analyze_node_operations): Adjust.
+	(vect_schedule_slp_node): Likewise.
+	* tree-vect-patterns.cc (vect_init_pattern_stmt): Do not
+	copy STMT_VINFO_TYPE.
+	* tree-vect-loop.cc: Set SLP_TREE_TYPE instead of
+	STMT_VINFO_TYPE everywhere.
+	(vect_create_loop_vinfo): Do not set STMT_VINFO_TYPE on
+	loop conditions.
+	* tree-vect-stmts.cc: Set SLP_TREE_TYPE instead of
+	STMT_VINFO_TYPE everywhere.
+	(vect_analyze_stmt): Adjust.
+	(vect_transform_stmt): Likewise.
+	* config/aarch64/aarch64.cc (aarch64_vector_costs::count_ops):
+	Access SLP_TREE_TYPE instead of STMT_VINFO_TYPE.
+	* config/i386/i386.cc (ix86_vector_costs::add_stmt_cost):
+	Remove non-SLP element-wise load/store matching.
+	* config/rs6000/rs6000.cc
+	(rs6000_cost_data::update_target_cost_per_stmt): Pass in
+	the SLP node.  Use that to get at the memory access
+	kind and type.
+	(rs6000_cost_data::add_stmt_cost): Pass down SLP node.
+	* config/riscv/riscv-vector-costs.cc (variable_vectorized_p):
+	Use SLP_TREE_TYPE.
+	(costs::need_additional_vector_vars_p): Likewise.
+	(costs::update_local_live_ranges): Likewise.
+
+2025-07-28  Jennifer Schmitz  <jschmitz@nvidia.com>
+	    Dhruv Chawla  <dhruvc@nvidia.com>
+
+	* config/aarch64/aarch64-cores.def (olympus): Use olympus tuning
+	model.
+	* config/aarch64/aarch64.cc: Include olympus.h.
+	* config/aarch64/tuning_models/olympus.h: New file.
+
+2025-07-28  Lulu Cheng  <chenglulu@loongson.cn>
+
+	* config/loongarch/loongarch.h
+	(CASE_VECTOR_SHORTEN_MODE): Delete.
+
+2025-07-28  Takayuki 'January June' Suwa  <jjsuwa_sys3175@yahoo.co.jp>
+
+	* config/xtensa/xtensa.cc (xtensa_is_insn_L32R_p):
+	Re-rewrite to more accurately capture insns that could be L32R machine
+	instructions wherever possible, and add comments that help understand
+	the intent of the process.
+
+2025-07-27  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/riscv-v.cc (expand_vx_binary_vxrm_vec_vec_dup):
+	Add new case UNSPEC_VAADD.
+	(expand_vx_binary_vxrm_vec_dup_vec): Ditto.
+	* config/riscv/riscv.cc (riscv_rtx_costs): Ditto.
+	* config/riscv/vector-iterators.md: Add new case UNSPEC_VAADD to
+	iterator.
+
+2025-07-27  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR middle-end/120855
+	* cgraphunit.cc (symbol_table::compile): Assert a transparent
+	alias is not an internal identifier.
+	* symtab.cc (symbol_table::change_decl_assembler_name):
+	Likewise.
+	* varasm.cc (assemble_name_resolve): Check for
+	IDENTIFIER_TRANSPARENT_ALIAS instead of just TREE_CHAIN.
+
+2025-07-26  Thomas Schwinge  <tschwinge@baylibre.com>
+
+	* diagnostics/context.cc (context::diagnostic_impl)
+	(context::diagnostic_n_impl): In presence of formal parameter
+	'kind', explicitly state 'diagnostics::' scope for 'kind' enum.
+
+2025-07-26  Thomas Schwinge  <tschwinge@baylibre.com>
+
+	* diagnostics/option-classifier.cc
+	(update_effective_level_from_pragmas): In presence of local
+	variable 'kind', explicitly state 'diagnostics::' scope for 'kind'
+	enum.
+
+2025-07-26  Christoph Müllner  <christoph.muellner@vrull.eu>
+
+	* config/riscv/riscv-ext.def: Add allocated group IDs and
+	group bit positions.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostics/context.cc: Eliminate digraphs::lazy_digraph in
+	favor of lazily_created template.
+	* diagnostics/context.h: Likewise.
+	* diagnostics/digraphs.cc: Likewise, also digraphs::lazy_digraphs.
+	* diagnostics/digraphs.h: Likewise.
+	* diagnostics/html-sink.cc: Likewise.
+	* diagnostics/metadata.h: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* diagnostics/sink.h: Likewise.
+	* diagnostics/text-sink.h: Likewise.
+	* lazily-created.h: New file.
+	* libgdiagnostics.cc: Eliminate digraphs::lazy_digraph in
+	favor of lazily_created template.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h: Use TextOrHtml rather than Sink for template
+	params.
+	* diagnostics/context.h: Likewise.
+	* diagnostics/selftest-context.cc: Use "html_or_text" rather than
+	"sink".
+	* diagnostics/source-printing.cc: Use TextOrHtml rather than Sink
+	for template params.  Use "text_out" and "html_out" rather than
+	"sink".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h (diagnostic_set_caret_max_width): Drop forward
+	decl.
+	(diagnostic_same_line): Make
+	diagnostics::context::m_source_printing private.
+	* diagnostics/context.cc (diagnostic_set_caret_max_width): Convert
+	to...
+	(diagnostics::context::set_caret_max_width): ...this.
+	* diagnostics/context.h
+	(diagnostics::context::get_source_printing_options): New
+	accessors.
+	(diagnostics::context::m_source_printing): Make private.
+	* diagnostics/html-sink.cc: Make
+	diagnostics::context::m_source_printing private.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* diagnostics/selftest-context.cc: Likewise.
+	* diagnostics/selftest-context.h
+	(diagnostics::selftest::test_context::colorize_source): New.
+	(diagnostics::selftest::test_context::show_labels): New.
+	(diagnostics::selftest::test_context::show_line_numbers): New.
+	(diagnostics::selftest::test_context::show_ruler): New.
+	(diagnostics::selftest::test_context::show_event_links): New.
+	(diagnostics::selftest::test_context::set_caret_char): New.
+	* diagnostics/source-printing.cc: Make
+	diagnostics::context::m_source_printing private.
+	* diagnostics/text-sink.h: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* opts.cc: Likewise.
+	* toplev.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h: Eliminate "diagnostic_info" typedef.
+	* diagnostics/context.cc: Update usage of "diagnostic_info" to
+	explicitly refer to "diagnostics::diagnostic_info".
+	* langhooks.cc: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* rtl-error.cc: Likewise.
+	* substring-locations.cc: Likewise.
+	* toplev.cc: Likewise.
+	* tree-diagnostic.cc: Likewise.
+	* tree-diagnostic.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h (enum diagnostics_column_unit): Move to
+	diagnostics/context-options.h.
+	(enum diagnostics_escape_format): Likewise.
+	(enum diagnostics_output_format): Likewise.
+	(enum diagnostic_path_format): Likewise.
+	(enum diagnostics_extra_output_kind): Likewise.
+	(enum diagnostic_text_art_charset): Likewise.
+	* diagnostics/context-options.h: New file, based on the above
+	material.
+	* diagnostics/context.h: Include it.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Add diagnostics/file-cache.o.
+	* diagnostics/changes.cc: Update for file_cache and char_span
+	moving from input.h to diagnostics/file-cache.h and into the
+	"diagnostics::" namespace.
+	* diagnostics/context.cc: Likewise.
+	* diagnostics/diagnostics-selftests.cc: Likewise.
+	* diagnostics/diagnostics-selftests.h: Likewise.
+	* diagnostics/file-cache.cc: New file, based on the file_cache
+	and file_cache_slot material in input.cc.
+	* diagnostics/file-cache.h: Likewise for input.h.
+	* diagnostics/selftest-source-printing.h: Update for file_cache
+	and char_span moving from input.h to diagnostics/file-cache.h and
+	into the "diagnostics::" namespace.
+	* diagnostics/source-printing.cc: Likewise.
+	* final.cc: Likewise.
+	* gcc-rich-location.cc: Likewise.
+	* input.cc (default_charset_callback): Move to
+	diagnostics/file-cache.cc.
+	(file_cache::initialize_input_context): Likewise.
+	(class file_cache_slot): Likewise.
+	(file_cache::tune): Likewise.
+	(file_cache::lookup_file): Likewise.
+	(file_cache::forcibly_evict_file): Likewise.
+	(file_cache::missing_trailing_newline_p): Likewise.
+	(file_cache::add_buffered_content): Likewise.
+	(file_cache::evicted_cache_tab_entry): Likewise.
+	(file_cache::add_file): Likewise.
+	(file_cache::file_cache): Likewise.
+	(file_cache::dump): Likewise.
+	(file_cache::dump): Likewise.
+	(file_cache::lookup_or_add_file): Likewise.
+	(find_end_of_line): Likewise.
+	(file_cache::get_source_line): Likewise.
+	(check_line): Likewise.
+	(test_replacement): Likewise.
+	(test_reading_source_line): Likewise.
+	(test_reading_source_buffer): Likewise.
+	* input.h (class char_span): Move to diagnostics/file-cache.h and
+	into the "diagnostics::" namespace.
+	(class file_cache_slot): Likewise.
+	(class file_cache): Likewise.
+	* libgdiagnostics.cc: Update for file_cache and char_span moving
+	from input.h to diagnostics/file-cache.h and into the
+	"diagnostics::" namespace.
+	* selftest.cc: Likewise.
+	* selftest.h: Likewise.
+	* substring-locations.h: Likewise.
+	* toplev.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostics/selftest-paths.cc: Don't include "demangle.h"
+	and "backtrace.h".
+	* diagnostics/source-printing.cc: Likewise.
+	* diagnostics/state-graphs.cc: Don't include "graphviz.h",
+	"xml.h", "xml-printer.h", or "intl.h".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h: Move include of "unique-argv.h"
+	and various forward decls into...
+	* diagnostics/context.h: ...here.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h (num_digits): Move to...
+	(diagnostics::num_digits): ...here.
+	(get_cwe_url): Move to...
+	(diagnostics::get_cwe_url): ...here.
+	(maybe_line_and_column): Move to...
+	(diagnostics::maybe_line_and_column): ...here.
+	* diagnostic.cc: Move to...
+	* diagnostics/context.cc: ...here.
+	* diagnostics/diagnostics-selftests.cc
+	(run_diagnostics_selftests): Add comment about context_cc_tests.
+	* diagnostics/diagnostics-selftests.h: Add context_cc_tests decl.
+	* diagnostics/sarif-sink.cc: Update for file rename.
+	* diagnostics/source-printing.cc: Update for num_digits moving to
+	diagnostics::num_digits.
+	* errors.cc: Update for file rename.
+	* input.cc: Update for num_digits moving to
+	diagnostics::num_digits.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace diagnostic.o with
+	diagnostics/context.o.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in: Replace diagnostic.def with diagnostics/kinds.def.
+	* config/aarch64/aarch64.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* config/i386/i386-options.cc: Likewise.
+	* config/s390/s390.cc: Likewise.
+	* diagnostic-core.h: Replace typedef diagnostic_t with
+	enum class diagnostics::kind in diagnostics/kinds.h and include
+	it.
+	* diagnostic-global-context.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* diagnostic.cc: Likewise.
+	* diagnostic.h: Likewise.
+	* diagnostics/buffering.cc: Likewise.
+	* diagnostics/buffering.h: Likewise.
+	* diagnostics/context.h: Likewise.
+	* diagnostics/diagnostic-info.h: Likewise.
+	* diagnostics/html-sink.cc: Likewise.
+	* diagnostic.def: Move to...
+	* diagnostics/kinds.def: ...here and update for diagnostic_t
+	becoming enum class diagnostics::kind.
+	* diagnostics/kinds.h: New file, based on material in
+	diagnostic-core.h.
+	* diagnostics/lazy-paths.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* diagnostics/option-classifier.cc: Likewise.
+	* diagnostics/option-classifier.h: Likewise.
+	* diagnostics/output-spec.h: Likewise.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* diagnostics/selftest-context.cc: Likewise.
+	* diagnostics/selftest-context.h: Likewise.
+	* diagnostics/sink.h: Likewise.
+	* diagnostics/source-printing.cc: Likewise.
+	* diagnostics/text-sink.cc: Likewise.
+	* diagnostics/text-sink.h: Likewise.
+	* gcc.cc: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* lto-wrapper.cc: Likewise.
+	* opts-common.cc: Likewise.
+	* opts-diagnostic.h: Likewise.
+	* opts.cc: Likewise.
+	* rtl-error.cc: Likewise.
+	* substring-locations.cc: Likewise.
+	* toplev.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-core.h: Include "diagnostics/option-id.h".
+	(struct diagnostic_option_id): Move there, renaming to
+	diagnostics::option_id.
+	* diagnostic-global-context.cc: Update for renaming of
+	diagnostic_option_id to diagnostics::option_id.
+	* diagnostic.cc: Likewise.
+	* diagnostic.h: Likewise.
+	* diagnostics/context.h: Likewise.
+	* diagnostics/diagnostic-info.h: Likewise.
+	* diagnostics/lazy-paths.cc: Likewise.
+	* diagnostics/option-classifier.cc: Likewise.
+	* diagnostics/option-classifier.h: Likewise.
+	* diagnostics/option-id.h: New file, taken from material in
+	diagnostic-core.h.
+	* diagnostics/selftest-context.cc: Update for renaming of
+	diagnostic_option_id to diagnostics::option_id.
+	* diagnostics/selftest-context.h: Likewise.
+	* ipa-pure-const.cc: Likewise.
+	* lto-wrapper.cc: Likewise.
+	* opts-common.cc: Likewise.
+	* opts-diagnostic.h: Likewise.
+	* opts.cc: Likewise.
+	* substring-locations.cc: Likewise.
+	* substring-locations.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace diagnostic-spec.o with
+	gcc-diagnostic-spec.o.
+	(GTFILES): Replace diagnostic-spec.h with gcc-diagnostic-spec.h.
+	* diagnostic-spec.cc: Move to...
+	* gcc-diagnostic-spec.cc: ...here.
+	* diagnostic-spec.h: Move to...
+	* gcc-diagnostic-spec.h: ...here.
+	* gengtype.cc (open_base_files): Replace diagnostic-spec.h with
+	gcc-diagnostic-spec.h.
+	* warning-control.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace diagnostics/changes.o with
+	diagnostics/edit-context.o.
+	* diagnostic.cc: Update for move of diagnostics/edit-context.h to
+	diagnostics/changes.h.
+	(diagnostics::context::initialize): Update for renaming of
+	m_edit_context_ptr to m_fixits_change_set.
+	(diagnostics::context::finish): Likewise.
+	(diagnostics::context::create_edit_context): Rename to...
+	(diagnostics::context::initialize_fixits_change_set): ...this and
+	update for other renamings.
+	(diagnostics::context::report_diagnostic): Update for renaming of
+	m_edit_context_ptr to m_fixits_change_set.
+	* diagnostic.h (diagnostics::edit_context): Drop forward decl
+	* diagnostics/edit-context.cc: Move to...
+	* diagnostics/changes.cc: ...here.  Update for move of
+	diagnostics/edit-context.h to diagnostics/changes.h.
+	Rename diagnostics::edit_context to
+	diagnostics::changes::change_set.  Rename diagnostics::edited_file
+	to diagnostics::changes::changed_file.  Rename
+	diagnostics::edited_line to diagnostics::changes::changed_line.
+	* diagnostics/edit-context.h: Move to...
+	* diagnostics/changes.h: ...here, renaming as above.
+	* diagnostics/context.h (diagnostics::changes::change_set): New
+	forward decl.
+	(diagnostics::context::create_edit_context): Rename to...
+	(diagnostics::context::initialize_fixits_change_set): ...this.
+	(diagnostics::context::get_edit_context): Rename to...
+	(diagnostics::context::get_fixits_change_set): ...this.
+	(diagnostics::context::m_edit_context_ptr): Rename to...
+	(diagnostics::context::m_fixits_change_set): ..this and update
+	type.
+	* diagnostics/diagnostics-selftests.cc: Update for renaming.
+	* diagnostics/diagnostics-selftests.h: Likewise.
+	* diagnostics/html-sink.cc: Update for above changes.
+	* libgdiagnostics.cc: Likewise.
+	* toplev.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS): Move diagnostics/lazy-paths.o to...
+	(OBJS-libcommon): ...here.  Add
+	diagnostics/diagnostics-selftests.o.
+	* diagnostic.cc: Update for move of
+	selftest::test_diagnostic_context to
+	diagnostics::selftest::test_context.
+	* diagnostics/color.cc: Move selftests from selftest:: to
+	diagnostics::selftest::.
+	* diagnostics/diagnostics-selftests.cc: New file.
+	* diagnostics/diagnostics-selftests.h: New file.
+	* diagnostics/digraphs.cc: Move selftests from selftest:: to
+	diagnostics::selftest::.
+	* diagnostics/edit-context.cc: Likewise.
+	* diagnostics/html-sink.cc: Likewise.
+	* diagnostics/lazy-paths.cc: Likewise.  Eliminate use of "tree"
+	by porting selftests from simple-diagnostic-path.h to
+	diagnostics/selftest-paths.h.
+	* diagnostics/output-spec.cc: Move selftests from selftest:: to
+	diagnostics::selftest::.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* diagnostics/selftest-context.cc: Move
+	selftest::test_diagnostic_context to
+	diagnostics::selftest::test_context.
+	* diagnostics/selftest-context.h: Likewise.
+	* diagnostics/selftest-logical-locations.cc: Move selftests from
+	selftest:: to diagnostics::logical_locations::selftest::.
+	* diagnostics/selftest-logical-locations.h: Move
+	selftest::logical_location_manager to
+	diagnostics::logical_locations::selftest::logical_location_manager.
+	* diagnostics/selftest-paths.cc: Move
+	selftest::test_diagnostic_path to
+	diagnostics::paths::selftest::test_path.  Move
+	selftest::test_diagnostic_event to
+	diagnostics::paths::selftest::test_event.
+	* diagnostics/selftest-paths.h: Likewise, and move
+	selftest::test_diagnostic_thread to
+	diagnostics::paths::selftest::test_thread.
+	* diagnostics/selftest-source-printing.h: Move
+	selftest::diagnostic_show_locus_fixture to
+	diagnostics::selftest::source_printing_fixture.
+	* diagnostics/source-printing.cc: Move selftests from selftest::
+	to diagnostics::selftest:: and update for renames.
+	* diagnostics/state-graphs.cc: Likewise.
+	* selftest-run-tests.cc: Include
+	"diagnostics/diagnostics-selftests.h".
+	(selftest::run_tests): Replace invocation of the various diagnostics
+	selftests with a call to
+	diagnostics::selftest::run_diagnostics_selftests.
+	* selftest.h: Move decls of the various per-file diagnostics
+	invocation functions to diagnostics/diagnostics-selftests.h,
+	renaming due to diagostics prefix being implied by namespace.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* common.opt: Update for move of diagnostic-url.h to
+	diagnostics/url.h.
+	* diagnostic.cc: Likewise.
+	* diagnostics/color.cc: Likewise.
+	* diagnostics/text-sink.cc: Likewise.
+	* diagnostic-url.h: Move to...
+	* diagnostics/url.h: ...here.
+	* libgdiagnostics.cc: Update for move of diagnostic-url.h to
+	diagnostics/url.h.
+	* pretty-print.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace diagnostic-color.o with
+	diagnostics/color.o.
+	(PLUGIN_HEADERS): Remove diagnostic-color.h in favor of glob.
+	* common.opt: Update for move of diagnostic-color.h to
+	diagnostics/color.h.
+	* diagnostic.cc: Likewise.
+	* diagnostic-color.cc: Move to...
+	* diagnostics/color.cc: ...here.
+	* diagnostic-color.h: Move to...
+	* diagnostics/color.h: ...here.
+	* diagnostics/edit-context.cc: Update for move of
+	diagnostic-color.h to diagnostics/color.h.
+	* diagnostics/output-spec.cc: Likewise.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/source-printing.cc: Likewise.
+	* diagnostics/text-sink.cc: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* opts.cc: Likewise.
+	* pretty-print-markup.h: Likewise.
+	* pretty-print.cc: Likewise.
+	* selftest-run-tests.cc: Likewise.
+	* selftest.h: Likewise.
+	* text-art/style.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace selftest-diagnostic.o with
+	diagnostics/selftest-context.o.
+	* diagnostic.cc: Update for move of selftest-diagnostic.h to
+	diagnostics/selftest-context.h.
+	* diagnostics/html-sink.cc: Likewise.
+	* diagnostics/lazy-paths.cc: Likewise.
+	* diagnostics/output-spec.cc: Likewise.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* selftest-diagnostic.cc: Move to...
+	* diagnostics/selftest-context.cc: ...here and update for move of
+	header file.
+	* selftest-diagnostic.h: Move to...
+	* diagnostics/selftest-context.h: ...here.
+	* diagnostics/source-printing.cc: Update for move of
+	selftest-diagnostic.h to diagnostics/selftest-context.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Add diagnostics/buffering.o.
+	* diagnostic.cc (diagnostics::context::set_diagnostic_buffer): Move to
+	diagnostics/buffering.cc.
+	(diagnostics::context::clear_diagnostic_buffer): Likewise.
+	(diagnostics::context::flush_diagnostic_buffer): Likewise.
+	(diagnostics::buffer::buffer): Likewise.
+	(diagnostics::buffer::~buffer): Likewise.
+	(diagnostics::buffer::dump): Likewise.
+	(diagnostics::buffer::empty_p): Likewise.
+	(diagnostics::buffer::move_to): Likewise.
+	(diagnostics::buffer::ensure_per_sink_buffers): Likewise.
+	* diagnostics/buffering.cc: New file, from material in diagnostic.cc.
+	* diagnostics/buffering.h: Fix comment.
+	* diagnostics/text-sink.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Add
+	diagnostics/option-classifier.o.
+	* diagnostic.cc (diagnostics::option_classifier::init): Move to
+	diagnostics/option-classifier.cc.
+	(diagnostics::option_classifier::fini): Likewise.
+	(diagnostics::option_classifier::pch_save): Likewise.
+	(diagnostics::option_classifier::pch_restore): Likewise.
+	(diagnostics::option_classifier::push): Likewise.
+	(diagnostics::option_classifier::pop): Likewise.
+	(diagnostics::option_classifier::classify_diagnostic): Likewise.
+	(diagnostics::option_classifier::update_effective_level_from_pragmas):
+	Likewise.
+	* diagnostics/context.h: Include
+	"diagnostics/option-classifier.h".
+	(class option_classifier): Move to
+	diagnostics/option-classifier.h.
+	* diagnostics/option-classifier.cc: New file, based on material
+	from diagnostic.cc.
+	* diagnostics/option-classifier.h: New file, based on material
+	from diagnostics/context.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* coretypes.h: Update for diagnostic_context becoming
+	diagnostics::context.
+	* diagnostic-global-context.cc: Likewise.
+	* diagnostic.cc: Likewise.  Also for diagnostic_option_classifier
+	becoming diagnostics::option_classifier.
+	* diagnostic.h (diagnostic_text_starter_fn): Rename to...
+	(diagnostics::text_starter_fn): ...this, and move to
+	diagnostics/context.h.
+	(get_printer): Rename to...
+	(diagnostics::get_printer): ...this, and move to
+	diagnostics/context.h.
+	(class diagnostic_option_manager): Rename to...
+	(class diagnostics::option_manager): ...this, and move to
+	diagnostics/context.h.
+	(class diagnostic_option_classifier): Rename to...
+	(class diagnostics::option_classifier): ...this, and move to
+	diagnostics/context.h.
+	(struct diagnostic_source_printing_options): Rename to...
+	(struct diagnostics::source_printing_options): ...this, and move
+	to diagnostics/context.h.
+	(class diagnostic_column_policy): Rename to...
+	(class diagnostics::column_policy): ...this, and move to
+	diagnostics/context.h.
+	(class diagnostic_location_print_policy): Rename to...
+	(class diagnostics::location_print_policy): ...this, and move to
+	diagnostics/context.h.
+	(class html_label_writer): Rename to...
+	(class diagnostics::html_label_writer): ...this, and move to
+	diagnostics/context.h.
+	(class diagnostic_source_print_policy): Rename to...
+	(class diagnostics::source_print_policy): ...this, and move to
+	diagnostics/context.h.
+	(struct diagnostic_counters): Rename to...
+	(struct diagnostics::counters): ...this, and move to
+	diagnostics/context.h.
+	(class diagnostic_context): Rename to...
+	(class diagnostics::context): ...this, and move to
+	diagnostics/context.h.
+	(diagnostic_text_starter): Rename to...
+	(diagnostics::text_starter): ...this, and move to
+	diagnostics/context.h.
+	(diagnostic_start_span): Rename to...
+	(diagnostics::start_span): ...this, and move to
+	diagnostics/context.h.
+	(diagnostic_text_finalizer): Rename to...
+	(diagnostics::text_finalizer): ...this, and move to
+	diagnostics/context.h.
+	Include "diagnostics/context.h".
+	* diagnostics/buffering.h: Update for diagnostic_context becoming
+	diagnostics::context; similarly for diagnostic_counters.
+	* diagnostics/client-data-hooks.h: Likewise.
+	* diagnostics/context.h: New file, taken from material in
+	diagnostic.h.
+	* diagnostics/html-sink.cc: : Update for diagnostic_context
+	becoming diagnostics::context.
+	* diagnostics/html-sink.h: Likewise.
+	* diagnostics/lazy-paths.cc: Likewise for
+	diagnostic_option_manager.
+	* diagnostics/output-file.h: Likewise for diagnostic_context.
+	* diagnostics/output-spec.cc: Likewise.
+	* diagnostics/output-spec.h: Likewise.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* diagnostics/sarif-sink.h: Likewise.
+	* diagnostics/sink.h: Likewise.
+	* diagnostics/source-printing.cc: Likewise.
+	* diagnostics/text-sink.cc: Likewise.
+	* diagnostics/text-sink.h: Likewise.
+	* gcc-rich-location.h: Likewise.
+	* gcc.cc: Likewise.
+	* gdbinit.in: Likewise.
+	* langhooks-def.h: Likewise.
+	* langhooks.cc: Likewise.
+	* langhooks.h: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* optc-gen.awk: Likewise.
+	* opth-gen.awk: Likewise.
+	* opts-common.cc: Likewise.
+	* opts-diagnostic.cc: Likewise.
+	* opts-diagnostic.h: Likewise.
+	* opts-global.cc: Likewise.
+	* opts.cc: Likewise.
+	* opts.h: Likewise.
+	* selftest-diagnostic.cc: Likewise.
+	* selftest-diagnostic.h: Likewise.
+	* toplev.cc: Likewise.
+	* tree-diagnostic-client-data-hooks.cc: Likewise.
+	* tree-diagnostic.cc: Likewise.
+	* tree-diagnostic.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.h (struct diagnostic_info): Move to
+	diagnostics/diagnostic-info.h as diagnostics::diagnostic_info.
+	Add typedef bringing it back into root namespace, for now.
+	* diagnostics/diagnostic-info.h: New file, based on the above.
+	* langhooks-def.h: Update for diagnostic_info moving into
+	namespace diagnostics.
+	* langhooks.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic.cc: Update to add "m_" prefix to fields of
+	diagnostic_info throughout.
+	* diagnostic.h: Likewise.
+	* diagnostics/html-sink.cc: Likewise.
+	* diagnostics/sarif-sink.cc: Likewise.
+	* diagnostics/text-sink.cc: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* substring-locations.cc: Likewise.
+	* tree-diagnostic.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS): Replace diagnostic-macro-unwinding.o with
+	diagnostics/macro-unwinding.o.
+	* diagnostic-macro-unwinding.cc: Move to...
+	* diagnostics/macro-unwinding.cc: ...here, moving content into
+	namespace diagnostics.
+	* diagnostic-macro-unwinding.h: Move to...
+	* diagnostics/macro-unwinding.h: ...here, moving content into
+	namespace diagnostics.
+	* diagnostics/paths-output.cc: Update for move of
+	"diagnostic-macro-unwinding.h" to "diagnostics/macro-unwinding.h".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS): Replace diagnostic-show-locus.o with
+	diagnostics/source-printing.o.
+	* diagnostic.h (class diagnostic_source_effect_info): Replace
+	with...
+	(class diagnotics::source_effect_info): ...this.
+	* diagnostics/paths-output.cc: Likewise.  Update for move of
+	"diagnostic-label-effects.h" to
+	"diagnostics/source-printing-effects.h".
+	* diagnostics/sarif-sink.cc: Update for move of
+	"selftest-diagnostic-show-locus.h" to
+	"diagnostics/selftest-source-printing.h".
+	* selftest-diagnostic-show-locus.h: Move to...
+	* diagnostics/selftest-source-printing.h: ...here.
+	* diagnostic-label-effects.h: Move to...
+	* diagnostics/source-printing-effects.h: ...here, updating
+	for above changes.
+	* diagnostic-show-locus.cc: Move to...
+	* diagnostics/source-printing.cc: ...here.
+	* gcc-rich-location.h: Likewise.
+	* selftest-run-tests.cc: Likewise.
+	* selftest.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS): Replace diagnostic-format-html.o with
+	diagnostics/html-sink.o, diagnostic-format-sarif.o with
+	diagnostics/sarif-sink.o, and diagnostic-format-text.o with
+	diagnostics/text-sink.o.
+	* coretypes.h (class diagnostic_text_output_format): Replace
+	with...
+	(class diagnostics::text_sink): ...this.
+	* diagnostic-global-context.cc: Update for move of diagnostics
+	output formats into namespace "diagnostics", as "sinks".
+	* diagnostic-macro-unwinding.cc: Likewise.
+	* diagnostic-macro-unwinding.h: Likewise.
+	* diagnostic-show-locus.cc: Likewise.
+	* diagnostic.cc: Likewise.
+	* diagnostic.h: Likewise.
+	* diagnostic-buffer.h: Rename to...
+	* diagnostics/buffering.h: ...this, updating for above changes.
+	* diagnostics/client-data-hooks.h: Likewise.
+	* diagnostics/digraphs.cc: Likewise.
+	* diagnostic-format-html.cc: Rename to...
+	* diagnostics/html-sink.cc: ...this, updating for above changes.
+	* diagnostic-format-html.h: Rename to...
+	* diagnostics/html-sink.h: ...this, updating for above changes.
+	* diagnostics/lazy-paths.cc: Likewise.
+	* diagnostics/metadata.h: Likewise.
+	* diagnostics/output-spec.cc: Likewise.
+	* diagnostics/output-spec.h: Likewise.
+	* diagnostics/paths-output.cc: Likewise.
+	* diagnostics/paths.h: Likewise.
+	* diagnostic-format-sarif.cc: Rename to...
+	* diagnostics/sarif-sink.cc: ...this, updating for above changes.
+	* diagnostic-format-sarif.h: Rename to...
+	* diagnostics/sarif-sink.h: ...this, updating for above changes.
+	* diagnostic-format.h: Rename to...
+	* diagnostics/sink.h: ...this, updating for above changes.
+	* diagnostics/state-graphs-to-dot.cc: Likewise.
+	* diagnostic-format-text.cc: Rename to...
+	* diagnostics/text-sink.cc: ...this, updating for above changes.
+	* diagnostic-format-text.h: Rename to...
+	* diagnostics/text-sink.h: ...this, updating for above changes.
+	* gcc.cc: Likewise.
+	* langhooks-def.h: Likewise.
+	* langhooks.cc: Likewise.
+	* langhooks.h: Likewise.
+	* libgdiagnostics.cc: Likewise.
+	* opts-diagnostic.cc: Likewise.
+	* opts.cc: Likewise.
+	* selftest-diagnostic.cc: Likewise.
+	* selftest-run-tests.cc: Likewise.
+	* selftest.h: Likewise.
+	* tree-diagnostic-client-data-hooks.cc: Likewise.
+	* tree-diagnostic.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS): Replace lazy-diagnostic-path.o with
+	diagnostics/lazy-paths.o.
+	(OBJS-libcommon): Replace diagnostic-path.o with
+	diagnostics/paths.o, diagnostic-path-output.o with
+	diagnostics/paths-output.o, and selftest-diagnostic-path.o with
+	diagnostics/selftest-paths.o.
+	(EXTRA_BACKEND_OBJS): Replace lazy-diagnostic-path.o with
+	diagnostics/lazy-paths.o.
+	* diagnostic-format-html.cc: Update #include for
+	"diagnostic-path.h" moving to "diagnostics/paths.h",
+	diagnostic_thread_id_t to diagnostics::paths::thread_id_t,
+	diagnostic_event_id_t to diagnostics::paths::event_id_t,
+	diagnostic_path to diagnostics::paths::path, and
+	diagnostic_thread to diagnostics::paths::thread, and
+	diagnostic_event to diagnostics::paths::event.
+	* diagnostic-format-html.h: Likewise.
+	* diagnostic-format-sarif.cc: Likewise.  Update PROPERTY_PREFIX
+	for threadFlowLocations from "gcc/diagnostic_event/" to
+	"gcc/diagnostics/paths/event/".
+	* diagnostic-format-text.cc: Likewise.
+	* diagnostic-format-text.h: Likewise.
+	* diagnostic.cc: Likewise.
+	* diagnostic.h: Likewise.
+	* diagnostic-event-id.h: Move to...
+	* diagnostics/event-id.h: ...here, updating header guard.
+	(diagnostics:paths:event_id_t): New typedef.
+	(diagnostic_thread_id_t): Replace with...
+	(diagnostics:paths:thread_id_t): New typedef.
+	* lazy-diagnostic-path.cc: Move to...
+	* diagnostics/lazy-paths.cc: ...here.  Update for above changes,
+	lazy_diagnostic_path becomes diagnostics::paths::lazy_path.
+	(lazy_diagnostic_path_cc_tests): Rename to...
+	(diagnostics_lazy_paths_cc_tests): ...this.
+	* lazy-diagnostic-path.h: Move to...
+	* diagnostics/lazy-paths.h: ...here, updating header guard.
+	Update for above changes.
+	* diagnostic-path-output.cc: Move to...
+	* diagnostics/paths-output.cc: ...here.  Update for above changes.
+	(diagnostic_path_output_cc_tests): Rename to...
+	(diagnostics_paths_output_cc_tests): ...this.
+	* diagnostic-path.cc: Move to...
+	* diagnostics/paths.cc: ...here.  Update for above changes.
+	* diagnostic-path.h: Move to...
+	* diagnostics/paths.h: ...here, updating header guard.
+	Update #include for moving "diagnostic-event-id.h" to
+	"diagnostics/event-id.h".
+	(class diagnostic_event): Convert to...
+	(class diagnostics::paths::event): ...this.
+	(class diagnostic_thread): Convert to...
+	(class diagnostics::paths::thread): ...this.
+	(class diagnostic_path): Convert to...
+	(class diagnostics::paths::path): ...this.
+	* diagnostic-show-locus.cc: Update for above changes.
+	* doc/analyzer.texi: Likewise.
+	* selftest-diagnostic-path.cc: Move to...
+	* diagnostics/selftest-paths.cc: ...here.  Update for
+	above changes, and for "selftest-diagnostic-path.h" moving to
+	"diagnostics/selftest-paths.h".
+	* selftest-diagnostic-path.h: Move to...
+	* diagnostics/selftest-paths.h: ...here, updating header guard.
+	Update for above changes.
+	* libgdiagnostics.cc: Update for above changes.
+	* libsarifreplay.cc: Update property prefix for
+	threadFlowLocations from "gcc/diagnostic_event/" to
+	"gcc/diagnostics/paths/event/".
+	* pretty-print-format-impl.h: Update for above changes.
+	* pretty-print.cc: Likewise.
+	* selftest-run-tests.cc (selftest::run_tests): Update for
+	renaming of lazy_diagnostic_path_cc_tests to
+	diagnostics_lazy_paths_cc_tests, and of
+	diagnostic_path_output_cc_tests to
+	diagnostics_paths_output_cc_tests.
+	* selftest.h (lazy_diagnostic_path_cc_tests): Replace decl with...
+	(diagnostics_lazy_paths_cc_tests): ...this.
+	(diagnostic_path_output_cc_tests): Replace decl with...
+	(diagnostics_paths_output_cc_tests): ...this.
+	* simple-diagnostic-path.cc: Clarify that this relates to "tree"
+	and thus shouldn't be in "diagnostics".  Update for above changes.
+	* simple-diagnostic-path.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace diagnostic-output-spec.o
+	with diagnostics/output-spec.o.
+	* diagnostic-output-spec.cc: Move to...
+	* diagnostics/output-spec.cc: ...here.  Update #include for
+	move of diagnostic-output-spec.h to diagnostics/output-spec.h.
+	(namespace::diagnostics_output_spec): Replace with...
+	(namespace diagnostics::output_spec): ...this, removing redundant
+	prefixes.
+	(diagnostics_output_spec::gcc_spec_context): Replace with...
+	(diagnostics::output_spec::dc_spec_context): ...this.
+	(diagnostic_output_spec_cc_tests): Rename to...
+	(diagnostics_output_spec_cc_tests): ...this.
+	* diagnostic-output-spec.h: Move to...
+	* diagnostics/output-spec.h: ...here and update inclusion guard.
+	(namespace::diagnostics_output_spec): Replace with...
+	(namespace diagnostics::output_spec): ...this, removing redundant
+	prefixes.
+	(diagnostics_output_spec::gcc_spec_context): Replace with...
+	(diagnostics::output_spec::dc_spec_context): ...this.
+	* libgdiagnostics.cc: Update #include for move of
+	diagnostic-output-spec.h to diagnostics/output-spec.h.
+	* opts-diagnostic.cc: Likewise.  Update for namespace and name
+	changes.
+	* selftest-run-tests.cc (selftest::run_tests): Update for renaming
+	of diagnostic_output_spec_cc_tests to
+	diagnostics_output_spec_cc_tests.
+	* selftest.h (diagnostic_output_spec_cc_tests): Rename decl to...
+	(diagnostics_output_spec_cc_tests): ...this.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-format-html.cc: Update #include for move of
+	diagnostic-output-file.h to diagnostics/output-file.h.  Update for
+	move of diagnostic_output_file to diagnostics::output_file.
+	* diagnostic-format-html.h: Likewise.
+	* diagnostic-format-sarif.cc: Likewise.
+	* diagnostic-format-sarif.h: Likewise.
+	* diagnostic-output-spec.cc: Likewise.
+	* diagnostic-output-spec.h (along): Likewise.
+	* diagnostic-output-file.h: Move to...
+	* diagnostics/output-file.h: ...here, updating header guard.
+	(class diagnostic_output_file): Move to...
+	(class diagnostics::output_file): ...here.
+	* libgdiagnostics.cc (sarif_sink::sarif_sink): Update for
+	move of diagnostic_output_file to diagnostics::output_file.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-format-html.cc: Update #include for move of
+	diagnostic-client-data-hooks.h to diagnostics/client-data-hooks.h.
+	* diagnostic-format-sarif.cc: Likewise.  Update for move of
+	diagnostic_client_plugin_info to diagnostics::client_plugin_info.
+	* diagnostic-format-text.cc: Likewise.
+	* diagnostic.cc: Likewise.
+	* diagnostic.h: Likewise.
+	* diagnostic-client-data-hooks.h: Move to...
+	* diagnostics/client-data-hooks.h: ...here, updating header guard.
+	(class diagnostic_client_data_hooks): Move to...
+	(class diagnostics::client_data_hooks): ...here.  Remove redundant
+	"diagnostics::" prefixes.
+	(class diagnostic_client_plugin_info): Move to...
+	(class diagnostics::client_plugin_info): ...here.
+	* libgdiagnostics.cc: : Update #include for move of
+	diagnostic-client-data-hooks.h to diagnostics/client-data-hooks.h.
+	Update for move of diagnostic_client_data_hooks to
+	diagnostics::client_data_hooks, and for move of
+	diagnostic_client_version_info to diagnostics::client_version_info.
+	* tree-diagnostic-client-data-hooks.cc: Likewise.
+	* tree-diagnostic.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace edit-context.o with
+	diagnostics/edit-context.o.
+	* diagnostic-format-html.cc: Update #include for move
+	of edit-context.h to diagnostics subdir.
+	* diagnostic.cc: Likewise.  Update for move of edit_context to
+	diagnostics::edit_context.
+	* diagnostic.h: Likewise.
+	* edit-context.cc: Move to...
+	* diagnostics/edit-context.cc: ...here.  Update #include for move
+	of edit-context.h to diagnostics subdir.  Move all code
+	within namespace diagnostics.
+	* edit-context.h: Move to...
+	* diagnostics/edit-context.h: ...here, updating header guard.
+	Move all code within namespace diagnostics.
+	* libgdiagnostics.cc: Update #include for move of edit-context.h
+	to diagnostics subdir.  Update for move of edit_context
+	to diagnostics::edit_context.
+	* toplev.cc: Update #include for move of edit-context.h
+	to diagnostics subdir.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-format-html.cc: Update for move of diagnostic_diagram
+	to diagnostics::diagram.
+	* diagnostic-format-sarif.cc: Update #include for move of
+	"diagnostic-diagram.h" to "diagnostics/diagram.h". Update for move
+	of diagnostic_diagram to diagnostics::diagram.
+	* diagnostic-format-text.cc: Likewise.
+	* diagnostic-format-text.h: Update for move of diagnostic_diagram
+	to diagnostics::diagram.
+	* diagnostic-format.h: Likewise.
+	* diagnostic.cc: Update #include for move of
+	"diagnostic-diagram.h" to "diagnostics/diagram.h". Update for move
+	of diagnostic_diagram to diagnostics::diagram.
+	* diagnostic.h: Update for move of diagnostic_diagram to
+	diagnostics::diagram.
+	* diagnostic-diagram.h: Move to...
+	* diagnostics/diagram.h: ...here.
+	(class diagnostic_diagram): Convert to...
+	(class diagnostics::diagram): ...this.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-core.h: Update for move of diagnostic_metadata to
+	diagnostics::metadata.
+	* diagnostic-format-html.cc: Likewise; also, update #include for
+	move of "diagnostic-metadata.h" to "diagnostics/metadata.h".
+	* diagnostic-format-sarif.cc: Likewise.
+	* diagnostic-format-text.cc: Likewise.
+	* diagnostic-global-context.cc: Likewise.
+	* diagnostic.cc: Likewise.
+	* diagnostic.h: Likewise.
+	* diagnostic-metadata.h: Move to...
+	* diagnostics/metadata.h: ...here, updating header guard.
+	(class diagnostic_metadata): Move to...
+	(class diagnostics::metadata): ...here.  Drop redundant
+	"diagnostics::" qualifiers.
+	* libgdiagnostics.cc: : Update #include for move of
+	"diagnostic-metadata.h" to "diagnostics/metadata.h", and update
+	for move of diagnostic_metadata to diagnostics::metadata.
+	* selftest-diagnostic.cc: Update for move of diagnostic_metadata
+	to diagnostics::metadata.
+	* selftest-diagnostic.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Replace
+	selftest-logical-location.o with
+	diagnostics/selftest-logical-locations.o.
+	* diagnostic-client-data-hooks.h: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, and of logical_location
+	to diagnostics::logical_locations::key.
+	* diagnostic-format-html.cc: Add "using namespace diagnostics;",
+	and drop now-redundant "diagnostics::" prefixes.
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, of logical_location to
+	diagnostics::logical_locations::key, and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+	* diagnostic-format-sarif.cc: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".  Add
+	"using namespace diagnostics;", and drop now-redundant
+	"diagnostics::" prefixes.
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, of logical_location to
+	diagnostics::logical_locations::key, and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+	* diagnostic-format-sarif.h: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location to
+	diagnostics::logical_locations::key.
+	* diagnostic-path-output.cc: Add "using namespace diagnostics;".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, and
+	of logical_location to diagnostics::logical_locations::key,
+	* diagnostic-path.cc: Add "using namespace diagnostics;".
+	Update for conversion of logical_location to
+	diagnostics::logical_locations::key.
+	* diagnostic-path.h: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, and of logical_location
+	to diagnostics::logical_locations::key,
+	* diagnostic.cc: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+	* diagnostic.h: Reorganize forward decls.
+	(class logical_location_manager): Replace this forward decl
+	with...
+	(class diagnostics::logical_locations::manager): ...this.
+	(diagnostic_context::get_logical_location_manager): Update
+	return type.
+	* diagnostics/digraphs.h: : Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location to
+	diagnostics::logical_locations::key.
+	* logical-location.h: Move to...
+	* diagnostics/logical-locations.h: ...here.
+	(enum class logical_location_kind): Convert to...
+	(enum class diagnostics::logical_locations::kind): ... this.
+	(class logical_location_manager::key): Convert to...
+	(class diagnostics::logical_locations::key): ... this.
+	(class logical_location_manager): Convert to...
+	(class diagnostics::logical_locations::manager): ... this.
+	(logical_location): Drop this typedef in favor of
+	diagnostics::logical_locations::key.
+	* selftest-logical-location.cc: Move to...
+	* diagnostics/selftest-logical-locations.cc: ...here.
+	Update #include for move of "selftest-logical-location.h" to
+	"diagnostics/selftest-logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, of logical_location to
+	diagnostics::logical_locations::key, and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+	(selftest_logical_location_cc_tests): Rename to...
+	(diagnostics_selftest_logical_locations_cc_tests): ...this.
+	* selftest-logical-location.h: Move to...
+	* diagnostics/selftest-logical-locations.h: ...here, updating
+	include guard.
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, of logical_location to
+	diagnostics::logical_locations::key, and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+	* diagnostics/state-graphs-to-dot.cc: Add
+	"using namespace diagnostics;" and drop now-redundant
+	"diagnostics::" prefixes.
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager.
+	* diagnostics/state-graphs.h: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager,
+	of logical_location to diagnostics::logical_locations::key.
+	* libgdiagnostics.cc: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, and of logical_location
+	to diagnostics::logical_locations::key.
+	* selftest-diagnostic-path.cc: Update for conversion of
+	logical_location to diagnostics::logical_locations::key.
+	* selftest-diagnostic-path.h: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location to
+	diagnostics::logical_locations::key.
+	* selftest-run-tests.cc (selftest::run_tests): Update for
+	renaming of selftest_logical_location_cc_tests to
+	diagnostics_selftest_logical_locations_cc_tests.
+	* selftest.h: Likewise.
+	* simple-diagnostic-path.h: Update for conversion of
+	logical_location to diagnostics::logical_locations::key.
+	* tree-diagnostic-client-data-hooks.cc: Update for conversion of
+	logical_location_manager to
+	diagnostics::logical_locations::manager, and of logical_location
+	to diagnostics::logical_locations::key.
+	* tree-logical-location.cc: Update for conversions.
+	Update for conversion of logical_location to
+	diagnostics::logical_locations::key, and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+	* tree-logical-location.h: Update #include for move of
+	"logical-location.h" to "diagnostics/logical-locations.h".
+	Update for conversion of logical_location_manager to
+	diagnostics::logical_locations::manager, of logical_location
+	to diagnostics::logical_locations::key, and of
+	enum logical_location_kind to
+	enum diagnostics::logical_locations::kind.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Rename diagnostic-digraphs.o to
+	diagnostics/digraphs.o.  Rename diagnostic-state-graphs.o to
+	diagnostics/state-graphs.o.  Rename diagnostic-state-to-dot.o to
+	diagnostics/state-graphs-to-dot.o.
+	(PLUGIN_HEADERS): Add $(srcdir)/diagnostics/*.h.
+	(install-plugin:): Likewise.
+	* configure: Regenerate.
+	* configure.ac (gccdepdir): Add "diagnostics" to directories.
+	* diagnostic-format-html.cc: Update #includes for move of
+	"diagnostic-digraphs.h" to "diagnostics/digraphs.h" and of
+	move of "diagnostic-state-graphs.h" to
+	"diagnostics/state-graphs.h".
+	* diagnostic-format-sarif.cc: Likewise.
+	* diagnostic-path.cc: Likewise for state-graphs.h
+	* diagnostic-digraphs.cc: Move...
+	* diagnostics/digraphs.cc: ...to here.  Update #include
+	for renaming of digraphs.h.
+	(selftest::diagnostic_digraphs_cc_tests): Rename to...
+	(selftest::diagnostics_digraphs_cc_tests): ...this.
+	* diagnostic-digraphs.h: Move...
+	* diagnostics/digraphs.h: ...to here, updating header guard.
+	* diagnostic-state-to-dot.cc: Move...
+	* diagnostics/state-graphs-to-dot.cc: ...to here.  Update #include
+	of state-graphs.h.
+	* diagnostic-state-graphs.cc: Move...
+	* diagnostics/state-graphs.cc: ...to here.  Update #include
+	of state-graphs.h.
+	(selftest::diagnostic_state_graphs_cc_tests): Rename...
+	(selftest::diagnostics_state_graphs_cc_tests): ...to this.
+	* diagnostic-state-graphs.h: Move...
+	* diagnostics/state-graphs.h: ...to here, updating header guard.
+	* libgdiagnostics.cc: Update #includes for move of
+	"diagnostic-digraphs.h" to "diagnostics/digraphs.h" and of
+	move of "diagnostic-state-graphs.h" to
+	"diagnostics/state-graphs.h".
+	* selftest-run-tests.cc (selftest::run_tests): Update for
+	function renamings that reflect file renamings.
+	* selftest.h (selftest::diagnostic_digraphs_cc_tests): Rename
+	to...
+	(selftest::diagnostics_digraphs_cc_tests): ...this.
+	(selftest::diagnostic_state_graphs_cc_tests): Rename...
+	(selftest::diagnostics_state_graphs_cc_tests): ...to this.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_get_vec_defs): Remove overload
+	with operand vector type.  Remove stmt_vec_info and
+	ncopies argument.
+	* tree-vect-stmts.cc (vect_get_vec_defs): Likewise.
+	(vectorizable_conversion): Adjust by not passing in
+	vector types, stmt_vec_info and ncopies.
+	(vectorizable_bswap): Likewise.
+	(vectorizable_assignment): Likewise.
+	(vectorizable_shift): Likewise.
+	(vectorizable_operation): Likewise.
+	(vectorizable_scan_store): Likewise.
+	(vectorizable_store): Likewise.
+	(vectorizable_condition): Likewise.
+	(vectorizable_comparison_1): Likewise.
+	* tree-vect-loop.cc (vect_transform_reduction): Likewise.
+	(vect_transform_lc_phi): Likewise.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_is_simple_use): Remove stmt-info
+	with vectype output overload and remove stmt-info argument
+	from SLP based API.
+	* tree-vect-loop.cc (vectorizable_lane_reducing): Remove
+	unused def_stmt_info output argument to vect_is_simple_use.
+	Adjust.
+	* tree-vect-stmts.cc (get_group_load_store_type): Get
+	the gather/scatter offset vector type from the SLP child.
+	(vect_check_scalar_mask): Remove stmt_info argument.  Adjust.
+	(vect_check_store_rhs): Likewise.
+	(vectorizable_call): Likewise.
+	(vectorizable_simd_clone_call): Likewise.
+	(vectorizable_conversion): Likewise.
+	(vectorizable_assignment): Likewise.
+	(vectorizable_shift): Likewise.
+	(vectorizable_operation): Likewise.
+	(vectorizable_load): Likewise.
+	(vect_is_simple_cond): Remove stmt_info argument.  Adjust.
+	(vectorizable_condition): Likewise.
+	(vectorizable_comparison_1): Likewise.
+	(vectorizable_store): Likewise.
+	(vect_is_simple_use): Remove overload and non-SLP path.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vect_model_simple_cost): Removed code
+	when no SLP node is passed, remove then unused dt and ndts
+	parameters and fix the number of vector stmts calculation.
+	(vectorizable_call): Adjust.
+	(vectorizable_simd_clone_call): Likewise.
+	(vectorizable_conversion): Likewise.
+	(vectorizable_assignment): Likewise.
+	(vectorizable_shift): Likewise.
+	(vectorizable_operation): Likewise.
+	(vectorizable_condition): Likewise.
+	(vectorizable_comparison_1): Likewise.
+
+2025-07-25  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* config/riscv/riscv-vector-costs.cc (get_live_range):
+	Move compute_local_program_points to cost class.
+	(variable_vectorized_p): Add slp node parameter.
+	(need_additional_vector_vars_p): Move from here...
+	(costs::need_additional_vector_vars_p): ... to here and add slp
+	parameter.
+	(compute_estimated_lmul): Move update_local_live_ranges to cost
+	class.
+	(has_unexpected_spills_p): Move from here...
+	(costs::has_unexpected_spills_p): ... to here.
+	(costs::record_lmul_spills): New function.
+	(costs::add_stmt_cost): Add stmt_info, slp mapping.
+	(costs::finish_cost): Analyze loop.
+	* config/riscv/riscv-vector-costs.h: Move declarations to class.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vect_analyze_loop_2): Get vectype
+	from the SLP node.
+	(vectorize_fold_left_reduction): Likewise.
+	(vect_transform_reduction): Likewise.
+	(vect_transform_cycle_phi): Likewise.
+	(vect_transform_lc_phi): Likewise.
+	* tree-vect-slp.cc (vect_analyze_slp): Likewise.
+	* tree-vect-stmts.cc (vect_truncate_gather_scatter_offset):
+	Pass in vectype and use that.
+	(vect_use_strided_gather_scatters_p): Likewise.
+	(get_group_load_store_type): Adjust.
+	(vect_check_scalar_mask): We always have a SLP node.
+	Use vectype from the SLP node.
+	(vect_check_store_rhs): Likewise.
+	(vect_build_one_gather_load_call): Pass in the vector type
+	and use that.
+	(vect_get_strided_load_store_ops): Likewise.
+	(vectorizable_store): Adjust.
+	(vectorizable_load): Use the vector type from the SLP node
+	and adjust.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* target.h (stmt_vectype): Remove.
+	* tree-vect-stmts.cc (stmt_vectype): Likewise.
+
+2025-07-25  Christoph Müllner  <christoph.muellner@vrull.eu>
+
+	* config/riscv/riscv.cc (enum riscv_privilege_levels): Remove USER_MODE.
+	(riscv_handle_type_attribute): Remove "user" interrupts.
+	(riscv_expand_epilogue): Likewise.
+	(riscv_get_interrupt_type): Likewise.
+	* config/riscv/riscv.md (riscv_uret): Remove URET pattern.
+	* doc/extend.texi: Remove documentation of user interrupts.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (_stmt_vec_info::vec_stmts): Remove.
+	(STMT_VINFO_VEC_STMTS): Likewise.
+	* tree-vectorizer.cc (vec_info::new_stmt_vec_info): Do not
+	initialize it.
+	(vec_info::free_stmt_vec_info): Nor free it.
+	* tree-vect-data-refs.cc (vect_vfa_access_size): Remove
+	check on STMT_VINFO_VEC_STMTS.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_transform_grouped_load): Remove.
+	(vect_record_grouped_load_vectors): Likewise.
+	* tree-vect-data-refs.cc (vect_permute_load_chain): Likewise.
+	(vect_shift_permute_load_chain): Likewise.
+	(vect_transform_grouped_load): Likewise.
+	(vect_record_grouped_load_vectors): Likewise.
+	* tree-vect-stmts.cc (vectorizable_load): Remove comments
+	about load interleaving.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_permute_store_chain): Remove.
+	* tree-vect-data-refs.cc (vect_permute_store_chain): Likewise.
+	* tree-vect-stmts.cc (vectorizable_store): Remove comment
+	about store interleaving.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_get_vec_defs_for_operand): Remove.
+	* tree-vect-stmts.cc (vect_get_vec_defs_for_operand): Likewise.
+	(vect_get_vec_defs): Remove non-SLP path.
+	(check_load_store_for_partial_vectors): We always have an
+	SLP node.
+	(vect_check_store_rhs): Likewise.
+	(vect_get_gather_scatter_ops): Likewise.
+	(vect_create_vectorized_demotion_stmts): Likewise.
+	(vectorizable_store): Adjust.
+	(vectorizable_load): Likewise.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (VMAT_CONTIGUOUS_PERMUTE): Remove.
+	* tree-vect-stmts.cc (check_load_store_for_partial_vectors):
+	Remove checks on VMAT_CONTIGUOUS_PERMUTE.
+	(vectorizable_load): Likewise.
+	(vectorizable_store): Likewise.  Prune dead code.
+
+2025-07-25  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_store): Remove trivially
+	dead code.
+
+2025-07-25  Tobias Burnus  <tburnus@baylibre.com>
+
+	* config/gcn/gcn-opts.h (TARGET_NO_MANUAL_NOPS,
+	TARGET_CDNA3_NOPS): Define.
+	* config/gcn/gcn.md (define_attr "laneselect): Change 'yes' to
+	'read' and 'write'.
+	(define_attr "flatmemaccess"): Add with values store, storex34,
+	load, atomic, atomicwait, cmpswapx2, and no. Replacing ...
+	(define_attr "delayeduse"): Remove.
+	(define_attr "transop"): Add with values yes and no.
+	(various insns): Update 'laneselect', add flatmemaccess and transop,
+	remove delayeduse; fixing an issue for s_load_dwordx4 vs.
+	flat_store_dwordx4 related to delayeduse (now: flatmemaccess).
+	* config/gcn/gcn-valu.md: Update laneselect attribute and add
+	flatmemaccess.
+	* config/gcn/gcn.cc (gcn_cmpx_insn_p): New.
+	(gcn_md_reorg): Update for MI300 to add additional s_nop.
+	Skip s_nop-insertion part for RDNA{2,3}; add "VALU writes EXEC
+	followed by VALU DPP" unconditionally for CDNA2/CDNA3/GCN5.
+
+2025-07-25  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	PR middle-end/121214
+	* genpreds.cc (constraint_max_namelen): Delete.
+	(write_tm_preds_h): Always write insn_constraint_len() and
+	define CONSTRAINT_LEN to it, i.e., remove guard
+	constraint_max_namelen>1.  Remove outer guard
+	constraint_max_namelen>0 and re-indent.
+	(write_insn_preds_c): Remove guard
+	constraint_max_namelen>0 and re-indent.
+
+2025-07-25  Christoph Müllner  <christoph.muellner@vrull.eu>
+
+	* config/riscv/riscv.cc (enum riscv_privilege_levels): Add
+	RNMI_MODE.
+	(riscv_handle_type_attribute): Handle 'rnmi' interrupt attribute.
+	(riscv_expand_epilogue): Generate nmret for RNMI handlers.
+	(riscv_get_interrupt_type): Handle 'rnmi' interrupt attribute.
+	* config/riscv/riscv.md (riscv_rnmi): Add nmret INSN.
+	* doc/extend.texi: Add documentation for 'rnmi' interrupt attribute.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vectorizable_induction): Remove
+	gimple **vec_stmt argument.
+	(vectorizable_phi): Likewise.
+	(vectorizable_recurr): Likewise.
+	(vectorizable_early_exit): Likewise.
+	* tree-vect-loop.cc (vectorizable_phi): Likewise and adjust.
+	(vectorizable_recurr): Likewise.
+	(vectorizable_nonlinear_induction): Likewise.
+	(vectorizable_induction): Likewise.
+	* tree-vect-stmts.cc (vectorizable_bswap): Likewise.
+	(vectorizable_call): Likewise.
+	(vectorizable_simd_clone_call): Likewise.
+	(vectorizable_conversion): Likewise.
+	(vectorizable_assignment): Likewise.
+	(vectorizable_shift): Likewise.
+	(vectorizable_operation): Likewise.
+	(vectorizable_store): Likewise.
+	(vectorizable_load): Likewise.
+	(vectorizable_condition): Likewise.
+	(vectorizable_comparison_1): Likewise.
+	(vectorizable_comparison): Likewise.
+	(vectorizable_early_exit): Likewise.
+	(vect_analyze_stmt): Adjust.
+	(vect_transform_stmt): Likewise.
+	* tree-vect-slp.cc (vect_slp_analyze_operations): Adjust.
+	(vectorize_slp_instance_root_stmt): Likewise.
+
+2025-07-24  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	* doc/cpp.texi (#ifdef): Correct typo.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_early_exit): Remove non-SLP
+	path.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (_stmt_vec_info::simd_clone_info): Remove.
+	(STMT_VINFO_SIMD_CLONE_INFO): Likewise.
+	* tree-vectorizer.cc (vec_info::free_stmt_vec_info): Do not
+	release it.
+	* tree-vect-stmts.cc (vectorizable_simd_clone_call): Remove
+	non-SLP path.
+
+2025-07-24  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* internal-fn.cc (internal_fn_len_index): Adjust indices for new
+	alias_ptr param.
+	(internal_fn_else_index): Ditto.
+	(internal_fn_mask_index): Ditto.
+	(internal_fn_stored_value_index): Ditto.
+	(internal_fn_alias_ptr_index): Ditto.
+	(internal_fn_offset_index): Ditto.
+	(internal_fn_scale_index): Ditto.
+	(internal_gather_scatter_fn_supported_p): Ditto.
+	* internal-fn.h (internal_fn_alias_ptr_index): Ditto.
+	* optabs-query.cc (supports_vec_gather_load_p): Ditto.
+	* tree-vect-data-refs.cc (vect_check_gather_scatter): Add alias
+	pointer.
+	* tree-vect-patterns.cc (vect_recog_gather_scatter_pattern): Add
+	alias pointer.
+	* tree-vect-slp.cc (vect_get_operand_map): Adjust for alias
+	pointer.
+	* tree-vect-stmts.cc (vect_truncate_gather_scatter_offset): Add
+	alias pointer and misalignment handling.
+	(get_load_store_type): Move from here...
+	(get_group_load_store_type): ...To here.
+	(vectorizable_store): Add alias pointer.
+	(vectorizable_load): Ditto.
+	* tree-vectorizer.h (struct gather_scatter_info): Ditto.
+
+2025-07-24  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* config/aarch64/aarch64.cc (aarch64_builtin_support_vector_misalignment):
+	Return true for gather/scatter.
+	* config/arm/arm.cc (arm_builtin_support_vector_misalignment):
+	Ditto.
+	* config/epiphany/epiphany.cc (epiphany_support_vector_misalignment):
+	Ditto.
+	* config/gcn/gcn.cc (gcn_vectorize_support_vector_misalignment):
+	Ditto.
+	* config/loongarch/loongarch.cc (loongarch_builtin_support_vector_misalignment):
+	Ditto.
+	* config/riscv/riscv.cc (riscv_support_vector_misalignment):
+	Add gather/scatter argument.
+	* config/rs6000/rs6000.cc (rs6000_builtin_support_vector_misalignment):
+	Return true for gather/scatter.
+	* config/s390/s390.cc (s390_support_vector_misalignment):
+	Ditto.
+	* doc/tm.texi: Add argument.
+	* target.def: Ditto.
+	* targhooks.cc (default_builtin_support_vector_misalignment):
+	Ditto.
+	* targhooks.h (default_builtin_support_vector_misalignment):
+	Ditto.
+	* tree-vect-data-refs.cc (vect_supportable_dr_alignment):
+	Ditto.
+
+2025-07-24  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* tree-vect-slp.cc (GATHER_SCATTER_OFFSET): New define.
+	(vect_get_and_check_slp_defs): Use.
+	* tree-vectorizer.h (GATHER_SCATTER_LEGACY_P): New define.
+	(GATHER_SCATTER_IFN_P): Ditto.
+	(GATHER_SCATTER_EMULATED_P): Ditto.
+	* tree-vect-stmts.cc (vectorizable_store): Use.
+	(vectorizable_load): Use.
+
+2025-07-24  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* internal-fn.cc (expand_scatter_store_optab_fn): Use new
+	function.
+	(expand_gather_load_optab_fn): Ditto.
+	(internal_fn_offset_index): Ditto.
+	(internal_fn_scale_index): Ditto.
+	* internal-fn.h (internal_fn_offset_index): New function.
+	(internal_fn_scale_index): Ditto.
+	* tree-vect-data-refs.cc (vect_describe_gather_scatter_call):
+	Use new function.
+
+2025-07-24  Alfie Richards  <alfie.richards@arm.com>
+
+	* tree-vect-data-refs.cc (vect_create_data_ref_ptr): Remove unnecessary
+	casts to aggr_ptr_type.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_comparison_1): Remove
+	non-SLP path.
+	(vectorizable_comparison): Likewise.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_condition): Remove
+	non-SLP paths.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_scan_store): Remove
+	non-SLP path and unused parameters.
+	(vectorizable_store): Adjust.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_shift): Remove non-SLP paths.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_assignment): Remove
+	non-SLP paths.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_call): Remove non-SLP path.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (vectorizable_bswap): Remove non-SLP path.
+
+2025-07-24  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vectorizable_recurr): Remove non-SLP path.
+
+2025-07-24  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (*cond_<optab><mode>_2_relaxed):
+	Extend from SVE_FULL_F_B16B16 to SVE_F_B16B16.
+	(*cond_<optab><mode>_3_relaxed): Likewise.
+	(*cond_<optab><mode>_any_relaxed): Likwise.
+	(*cond_<optab><mode>_any_const_relaxed): Extend from SVE_FULL_F
+	to SVE_F.
+	(*cond_add<mode>_2_const_relaxed): Likewise.
+	(*cond_add<mode>_any_const_relaxed): Likewise.
+	(*cond_sub<mode>_3_const_relaxed): Likewise.
+	(*cond_sub<mode>_const_relaxed): Likewise.
+
+2025-07-24  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md: (@aarch64_sve_<optab><mode>):
+	Extend from SVE_FULL_F to SVE_F, use aarch64_predicate_operand.
+	(@aarch64_frecpe<mode>): Extend from SVE_FULL_F to SVE_F.
+	(@aarch64_frecps<mode>): Likewise.
+	(div<mode>3): Likewise, use aarch64_sve_fp_pred.
+	* config/aarch64/iterators.md: Add warnings above SVE_FP_UNARY
+	and SVE_FP_BINARY.
+
+2025-07-24  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (<optab><mode>3): Extend from
+	SVE_FULL_F to SVE_F, use aarch64_sve_fp_pred.
+	(*post_ra_<sve_fp_op><mode>3): Extend from SVE_FULL_F to SVE_F.
+	(@aarch64_pred_<optab><mode>): Extend from SVE_FULL_F to SVE_F,
+	use aarch64_predicate_operand (ADD/SUB/MUL/MAX/MIN).
+	(split for using unpredicated insns): Move SVE_RELAXED_GP into
+	the pattern, rather than testing for it in the condition.
+	* config/aarch64/aarch64-sve2.md (@aarch64_pred_<optab><mode>):
+	Extend from VNx8BF_ONLY to SVE_BF.
+
+2025-07-24  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/autovec-opt.md (*uavg_floor_vx_<mode>): Rename
+	from...
+	(*<sat_op_v_vdup>_vx_<mode>): Rename to...
+	(*<sat_op_vdup_v>_vx_<mode>): Rename to...
+	* config/riscv/riscv-protos.h (enum insn_flags): Add vxrm
+	RNE, ROD type.
+	(enum insn_type): Add RNE_P, ROD_P type.
+	(expand_vx_binary_vxrm_vec_vec_dup): Add new func decl.
+	(expand_vx_binary_vxrm_vec_dup_vec): Ditto.
+	* config/riscv/riscv-v.cc (get_insn_type_by_vxrm_val): Add
+	helper to get insn type by vxrm value.
+	(expand_vx_binary_vxrm_vec_vec_dup): Add new func impl
+	to expand vec + vec_dup pattern.
+	(expand_vx_binary_vxrm_vec_dup_vec): Ditto but for
+	vec_dup + vec pattern.
+	* config/riscv/vector-iterators.md: Add helper iterator
+	for sat vx combine.
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (*cond_<optab><mode>_2_relaxed):
+	Extend from SVE_FULL_F to SVE_F.
+	(*cond_<optab><mode>_any_relaxed): Likewise.
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (<optab><mode>2): Replace use of
+	aarch64_ptrue_reg with aarch64_sve_fp_pred.
+	(@aarch64_pred_<optab><mode>): Extend from SVE_FULL_F to SVE_F,
+	and use aarch64_predicate_operand.
+	* config/aarch64/iterators.md: Split FABS/FNEG out of
+	SVE_COND_FP_UNARY (into new SVE_COND_FP_UNARY_BITWISE).
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md
+	(*cond_<optab>_nontrunc<SVE_PARTIAL_F:mode><SVE_HSDI:mode>_relaxed):
+	New FCVT/SEL combiner pattern.
+	(*cond_<optab>_trunc<VNx2DF_ONLY:mode><VNx2SI_ONLY:mode>_relaxed):
+	New FCVTZ{S,U}/SEL combiner pattern.
+	(*cond_<optab>_nonextend<SVE_HSDI:mode><SVE_PARTIAL_F:mode>_relaxed):
+	New {S,U}CVTF/SEL combiner pattern.
+	(*cond_<optab>_trunc<SVE_SDF:mode><SVE_PARTIAL_HSF:mode>):
+	New FCVT/SEL combiner pattern.
+	(*cond_<optab>_nontrunc<SVE_PARTIAL_HSF:mode><SVE_SDF:mode>_relaxed):
+	New FCVTZ{S,U}/SEL combiner pattern.
+	* config/aarch64/iterators.md: New mode iterator for VNx2SI.
+
+2025-07-23  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/121073
+	* config/riscv/autovec-opt.md: Use new helpers.
+	* config/riscv/autovec.md: Ditto.
+	* config/riscv/predicates.md (strided_broadcast_mask_operand):
+	New predicate.
+	(strided_broadcast_operand): Ditto.
+	(any_broadcast_operand): Ditto.
+	* config/riscv/riscv-protos.h (expand_broadcast): Declare.
+	(expand_set_first): Ditto.
+	(expand_set_first_tu): Ditto.
+	(strided_broadcast_p): Ditto.
+	* config/riscv/riscv-string.cc (expand_vec_setmem): Use new
+	helpers.
+	* config/riscv/riscv-v.cc (expand_broadcast): New functionk.
+	(expand_set_first): Ditto.
+	(expand_set_first_tu): Ditto.
+	(expand_const_vec_duplicate): Use new helpers.
+	(expand_const_vector_duplicate_repeating): Ditto.
+	(expand_const_vector_duplicate_default): Ditto.
+	(sew64_scalar_helper): Ditto.
+	(expand_vector_init_merge_repeating_sequence): Ditto.
+	(expand_reduction): Ditto.
+	(strided_broadcast_p): New function.
+	(whole_reg_to_reg_move_p): Use new helpers.
+	* config/riscv/riscv-vector-builtins-bases.cc: Use either
+	broadcast or strided broadcast.
+	* config/riscv/riscv-vector-builtins.cc (function_expander::use_ternop_insn):
+	Ditto.
+	(function_expander::use_widen_ternop_insn): Ditto.
+	(function_expander::use_scalar_broadcast_insn): Ditto.
+	* config/riscv/riscv-vector-builtins.h: Declare scalar
+	broadcast.
+	* config/riscv/vector.md (*pred_broadcast<mode>): Split into
+	regular and strided broadcast.
+	(*pred_broadcast<mode>_zvfh): Split.
+	(pred_broadcast<mode>_zvfh): Ditto.
+	(*pred_broadcast<mode>_zvfhmin): Ditto.
+	(@pred_strided_broadcast<mode>): Ditto.
+	(*pred_strided_broadcast<mode>): Ditto.
+	(*pred_strided_broadcast<mode>_zvfhmin): Ditto.
+
+2025-07-23  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR target/120119
+	* config/aarch64/cortex-a57-fma-steering.cc (func_fma_steering::analyze):
+	Skip if renaming fails.
+
+2025-07-23  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121220
+	* tree-ssa-sink.cc (statement_sink_location): For stores
+	handle sinking to paths ending in a store.  Skip loads
+	that do not use the store.
+
+2025-07-23  Martin Jambor  <mjambor@suse.cz>
+
+	PR tree-optimization/119085
+	* tree-sra.cc (sort_and_splice_var_accesses): Prevent total
+	scalarization if two incompatible aggregates access the same place.
+
+2025-07-23  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* config.gcc <sparc*-*-solaris2*> (with_cpu): Default to ultrasparc3.
+
+2025-07-23  Richard Biener  <rguenther@suse.de>
+
+	* config/aarch64/aarch64.cc (aarch64_vector_costs::add_stmt_cost):
+	Check vectype is non-NULL before accessing it.
+
+2025-07-23  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/121216
+	* expr.cc (constant_byte_string): Check the string type
+	size fits an uhwi before converting to uhwi.
+
+2025-07-22  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/109267
+	* tree-cfgcleanup.cc (execute_cleanup_cfg_post_optimizing): If the first
+	non debug statement in the first (and only) basic block is a call
+	to __builtin_unreachable change it to a call to __builtin_trap or an
+	infinite loop.
+
+2025-07-22  Takayuki 'January June' Suwa  <jjsuwa_sys3175@yahoo.co.jp>
+
+	* config/xtensa/xtensa.cc (xtensa_is_insn_L32R_p):
+	Rewrite to capture insns that could be L32R machine instructions
+	wherever possible.
+	(xtensa_rtx_costs): Fix to consider that moves larger than a
+	natural word can take multiple L32R machine instructions.
+	(constantpool_address_p): Cosmetics.
+	* config/xtensa/xtensa.md (movdi_internal, movdf_internal):
+	Add missing insn attributes.
+
+2025-07-22  Takayuki 'January June' Suwa  <jjsuwa_sys3175@yahoo.co.jp>
+
+	* config/xtensa/xtensa.md
+	(movsi_internal, movhi_internal, movsf_internal):
+	Change the value of the "type" attribute from "move" to "load"
+	when the source operand constraint is "Y".
+
+2025-07-22  Karl Meakin  <karl.meakin@arm.com>
+
+	* tree-vect-data-refs.cc (vect_check_gather_scatter): Swap
+	`base` and `off` in more scenarios. Also assert at the end of
+	the function that `base` and `off` are loop-invariant and not
+	loop-invariant respectively.
+
+2025-07-22  Jeff Law  <jlaw@ventanamicro.com>
+
+	* config/riscv/generic-vector-ooo.md: Restrict insn reservations to
+	generic_ooo and generic tuning models.
+
+2025-07-22  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121202
+	* tree-vect-slp.cc (vect_schedule_slp_node): Do not take
+	an out-of-region stmt as "last".
+
+2025-07-22  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* genpreds.cc (write_insn_constraint_len): Replace rawmemchr by
+	a loop.
+
+2025-07-21  Jeff Law  <jlaw@ventanamicro.com>
+
+	* config/riscv/mips-p8700.md: Add support for "ghost" insn types.
+	* config/riscv/xiangshan.md: Add support for "sf_vc" and "sf_vc_se"
+	insn types.
+
+2025-07-21  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/110949
+	PR tree-optimization/95906
+	* match.pd (cmp - 1): New pattern.
+
+2025-07-21  Andreas Schwab  <schwab@suse.de>
+
+	PR target/121121
+	* config/riscv/riscv.cc (riscv_allocate_and_probe_stack_space):
+	Use temp2 instead of temp1 for the CFA note.
+
+2025-07-21  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/vector.md: Allow VLS DImode for sat_op vx pattern.
+
+2025-07-21  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/autovec-opt.md (*uavg_floor_vx_<mode>): Add
+	pattern for vaaddu.vx combine.
+	* config/riscv/riscv.cc (get_vector_binary_rtx_cost): Add UNSPEC
+	handling for UNSPEC_VAADDU.
+	(riscv_rtx_costs): Ditto.
+
+2025-07-21  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-simd.md (aarch64_simd_vec_set_zero<mode>):
+	Enable only when optimizing for size.
+
+2025-07-21  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64.cc (aarch64_rtx_costs): Add extra_cost values
+	only when speed is true for CONST_VECTOR, VEC_DUPLICATE, VEC_SELECT
+	cases.
+	* config/aarch64/aarch64-cost-tables.h (qdf24xx_extra_costs,
+	thunderx_extra_costs, thunderx2t99_extra_costs,
+	thunderx3t110_extra_costs, tsv110_extra_costs, a64fx_extra_costs,
+	ampere1_extra_costs, ampere1a_extra_costs, ampere1b_extra_costs):
+	Reduce cost of movi, dup, extract fields by COSTS_N_INSNS (1).
+	* config/arm/aarch-cost-tables.h (generic_extra_costs,
+	cortexa53_extra_costs, cortexa57_extra_costs, cortexa76_extra_costs,
+	exynosm1_extra_costs, xgene1_extra_costs): Likewise.
+
+2025-07-21  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121194
+	* tree-vect-loop.cc (vectorizable_lc_phi): Verify
+	vector types are compatible.
+
+2025-07-21  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn-valu.md (gather_load<mode><vndi>): New.
+	(scatter_store<mode><vndi>): New.
+	(mask_gather_load<mode><vndi>): New.
+	(mask_scatter_store<mode><vndi>): New.
+	* config/gcn/gcn.cc (gcn_expand_scaled_offsets): Support DImode.
+
+2025-07-21  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn.cc (GEN_VNM_NOEXEC): Use USE_QHF.
+	(GEN_VNM): Likewise, and call for new ashl and mul variants.
+
+2025-07-21  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn-valu.md (add<mode>3_dup): New.
+	(add<mode>3_dup_exec): New.
+	(<su>mul<mode>3_highpart_dup<exec>): New.
+	(mul<mode>3_dup): Move the vec_duplicate to operand 1.
+	(mul<mode>3_dup_exec): New.
+	(vec_series<mode>): Adjust call to gen_mul<mode>3_dup.
+	* config/gcn/gcn.cc (gcn_expand_vector_init): Likewise.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* genoutput.cc (main): Emit function
+	verify_reg_names_in_constraints() for run-time validation.
+	(mdep_constraint_len): Deal with hard register constraints.
+	* output.h (verify_reg_names_in_constraints): New function
+	declaration.
+	* toplev.cc (backend_init): If checking is enabled, call into
+	verify_reg_names_in_constraints().
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* cfgexpand.cc (n_occurrences): Move this ...
+	(check_operand_nalternatives): and this ...
+	(expand_asm_stmt): and the call to gimplify.cc.
+	* config/s390/s390.cc (s390_md_asm_adjust): Pass null pointer to
+	parse_{input,output}_constraint().
+	* gimple-walk.cc (walk_gimple_asm): Pass null pointer to
+	parse_{input,output}_constraint().
+	(walk_stmt_load_store_addr_ops): Ditto.
+	* gimplify-me.cc (gimple_regimplify_operands): Ditto.
+	* gimplify.cc (num_occurrences): Moved from cfgexpand.cc.
+	(num_alternatives): Ditto.
+	(gimplify_asm_expr): Deal with hard register constraints.
+	* stmt.cc (eliminable_regno_p): New helper.
+	(hardreg_ok_p): Perform a similar check as done in
+	make_decl_rtl().
+	(parse_output_constraint): Add parameter for gimplify_reg_info
+	and validate hard register constrained operands.
+	(parse_input_constraint): Ditto.
+	* stmt.h (class gimplify_reg_info): Forward declaration.
+	(parse_output_constraint): Add parameter.
+	(parse_input_constraint): Ditto.
+	* tree-ssa-operands.cc
+	(operands_scanner::get_asm_stmt_operands): Pass null pointer
+	to parse_{input,output}_constraint().
+	* tree-ssa-structalias.cc (find_func_aliases): Pass null pointer
+	to parse_{input,output}_constraint().
+	* varasm.cc (assemble_asm): Pass null pointer to
+	parse_{input,output}_constraint().
+	* gimplify_reg_info.h: New file.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* config/cris/cris.cc (cris_md_asm_adjust): Deal with hard
+	register constraint.
+	* config/i386/i386.cc (map_egpr_constraints): Ditto.
+	* config/s390/s390.cc (f_constraint_p): Ditto.
+	* doc/extend.texi: Document hard register constraints.
+	* doc/md.texi: Ditto.
+	* function.cc (match_asm_constraints_2): Have a unique pseudo
+	for each operand with a hard register constraint.
+	(pass_match_asm_constraints::execute): Calling into new helper
+	match_asm_constraints_2().
+	* genoutput.cc (mdep_constraint_len): Return the length of a
+	hard register constraint.
+	* genpreds.cc (write_insn_constraint_len): Support hard register
+	constraints for insn_constraint_len().
+	* ira.cc (valid_replacement_for_asm_input_p_1): New helper.
+	(valid_replacement_for_asm_input_p): New helper.
+	(decrease_live_ranges_number): Similar to
+	match_asm_constraints_2() ensure that each operand has a unique
+	pseudo if constrained by a hard register.
+	* lra-constraints.cc (process_alt_operands): Install hard
+	register filter according to constraint.
+	* recog.cc (asm_operand_ok): Accept register type for hard
+	register constrained asm operands.
+	(constrain_operands): Validate hard register constraints.
+	* stmt.cc (decode_hard_reg_constraint): Parse a hard register
+	constraint into the corresponding register number or bail out.
+	(parse_output_constraint): Parse hard register constraint and
+	set *ALLOWS_REG.
+	(parse_input_constraint): Ditto.
+	* stmt.h (decode_hard_reg_constraint): Declaration of new
+	function.
+
+2025-07-21  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_analyze_data_refs): Remove min_vf
+	output.
+	* tree-vect-data-refs.cc (vect_analyze_data_refs): Likewise.
+	* tree-vect-loop.cc (vect_analyze_loop_2): Remove early
+	out based on bogus min_vf.
+	* tree-vect-slp.cc (vect_slp_analyze_bb_1): Adjust.
+
+2025-07-20  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/autovec.md: Add (const_int 1) as the op2 of
+	ashiftrt.
+
+2025-07-19  Dimitar Dimitrov  <dimitar@dinux.eu>
+
+	PR target/121124
+	* config/pru/pru-pragma.cc (pru_pragma_ctable_entry): Handle the
+	ctable base address as signed 32-bit value, and sign-extend to
+	HOST_WIDE_INT.
+	* config/pru/pru-protos.h (struct pru_ctable_entry): Store the
+	ctable base address as signed.
+	(pru_get_ctable_exact_base_index): Pass base address as signed.
+	(pru_get_ctable_base_index): Ditto.
+	(pru_get_ctable_base_offset): Ditto.
+	* config/pru/pru.cc (pru_get_ctable_exact_base_index): Ditto.
+	(pru_get_ctable_base_index): Ditto.
+	(pru_get_ctable_base_offset): Ditto.
+	(pru_print_operand_address): Ditto.
+
+2025-07-19  Paul-Antoine Arras  <parras@baylibre.com>
+
+	PR target/119100
+	* config/riscv/autovec-opt.md (*vfwnmacc_vf_<mode>): New pattern.
+	(*vfwnmsac_vf_<mode>): New pattern.
+	* config/riscv/riscv.cc (get_vector_binary_rtx_cost): Add support for a
+	vec_duplicate in a neg.
+
+2025-07-19  Artemiy Volkov  <artemiyv@acm.org>
+
+	* config/riscv/riscv.cc (riscv_macro_fusion_pair_p): Protect
+	from a NULL PREV_SET or CURR_SET.
+
+2025-07-19  Georg-Johann Lay  <avr@gjlay.de>
+
+	* config/avr/avr-passes.cc (avr_optimize_casesi): Fuse
+	get_insns() with end_sequence().
+
+2025-07-18  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/autovec.md (avg<mode>3_ceil): Add new pattern
+	of avg3_ceil for RVV DImode
+
+2025-07-18  Martin Jambor  <mjambor@suse.cz>
+
+	PR tree-optimization/117423
+	* tree-sra.cc (analyze_access_subtree): Fix computation of grp_covered
+	flag.
+
+2025-07-18  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121126
+	* tree-vect-stmts.cc (vect_analyze_stmt): Analyze the
+	live lane extract for LC PHIs that are vect_internal_def.
+
+2025-07-18  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vectorizable_live_operation_1):
+	Remove stmt_info and ncopies parameters.  Remove !slp_node
+	paths.
+	(vectorizable_live_operation): Remove !slp_node paths.
+
+2025-07-18  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120924
+	* params.opt (uninit-max-chain-len): Up from 8 to 12.
+
+2025-07-18  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121048
+	* tree-vect-loop.cc (vect_determine_vectype_for_stmt_1):
+	Remove rejecting vector(1) vector types.
+	(vect_set_stmts_vectype): Likewise.
+	* tree-vect-slp.cc (vect_make_slp_decision): Only
+	count instances with non-vector(1) root towards whether
+	we have any interesting instances to vectorize.
+
+2025-07-18  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/121131
+	* gimple-fold.cc (fold_nonarray_ctor_reference): Use
+	TREE_INT_CST_LOW (TYPE_SIZE ()) instead of
+	GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE ()) for BLKmode BITINT_TYPEs.
+	Don't compute encoding_size at all for little endian targets.
+
+2025-07-17  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/121095
+	* gcse.cc (execute_hardreg_pre): Skip if the hardreg which is never live.
+
+2025-07-17  Filip Kastl  <fkastl@suse.cz>
+
+	* tree-ssa-structalias.cc (lookup_vi_for_tree): Fix GNU style.
+	(process_constraint): Fix GNU style.
+	(get_constraint_for_component_ref): Fix GNU style.
+	(get_constraint_for_1): Fix GNU style.
+	(get_function_part_constraint): Fix GNU style.
+	(handle_lhs_call): Fix GNU style.
+	(find_func_aliases_for_builtin_call): Fix GNU style.
+	(find_func_aliases): Fix GNU style.
+	(find_func_clobbers): Fix GNU style.
+	(struct shared_bitmap_hasher): Fix GNU style.
+	(shared_bitmap_hasher::hash): Fix GNU style.
+	(pt_solution_includes_global): Fix GNU style.
+	(init_base_vars): Fix GNU style.
+	(visit_loadstore): Fix GNU style.
+	(compute_dependence_clique): Fix GNU style.
+	(struct pt_solution): Fix GNU style.
+	(ipa_pta_execute): Fix GNU style.
+
+2025-07-17  Filip Kastl  <fkastl@suse.cz>
+
+	* pta-andersen.cc (struct constraint_graph): Fix GNU style.
+	(constraint_equal): Fix GNU style.
+	(set_union_with_increment): Fix GNU style.
+	(insert_into_complex): Fix GNU style.
+	(merge_node_constraints): Fix GNU style.
+	(unify_nodes): Fix GNU style.
+	(do_ds_constraint): Fix GNU style.
+	(scc_info::scc_info): Fix GNU style.
+	(find_indirect_cycles): Fix GNU style.
+	(equiv_class_lookup_or_add): Fix GNU style.
+	(label_visit): Fix GNU style.
+	(dump_pred_graph): Fix GNU style.
+	(perform_var_substitution): Fix GNU style.
+	(eliminate_indirect_cycles): Fix GNU style.
+	(solve_graph): Fix GNU style.
+	(solve_constraints): Fix GNU style.
+	* tree-ssa-structalias.cc (first_vi_for_offset): Fix GNU style.
+	(debug_constraint): Fix GNU style.
+	* tree-ssa-structalias.h (struct constraint_expr): Fix GNU
+	style.
+	(struct variable_info): Fix GNU style.
+
+2025-07-17  H.J. Lu  <hjl.tools@gmail.com>
+
+	* config/i386/i386-expand.cc (ix86_expand_ternlog): Don't change
+	mode for XOR.
+
+2025-07-17  Filip Kastl  <fkastl@suse.cz>
+
+	* Makefile.in: Add pta-andersen.o.
+	* tree-ssa-structalias.cc (create_variable_info_for): Just move
+	around.
+	(unify_nodes): Move to pta-andersen.cc.
+	(struct constraint): Move to tree-ssa-structalias.h.
+	(EXECUTE_IF_IN_NONNULL_BITMAP): Move to pta-andersen.cc.
+	(struct variable_info): Move to tree-ssa-structalias.h.
+	(struct constraint_stats): Move to tree-ssa-structalias.h.
+	(first_vi_for_offset): External linkage, move to namespace
+	pointer_analysis.
+	(first_or_preceding_vi_for_offset): External linkage, move to namespace
+	pointer_analysis.
+	(dump_constraint): External linkage, move to namespace
+	pointer_analysis.
+	(debug_constraint): External linkage, move to namespace
+	pointer_analysis.
+	(dump_constraints): External linkage, move to namespace
+	pointer_analysis.
+	(debug_constraints): External linkage, move to namespace
+	pointer_analysis.
+	(lookup_vi_for_tree): Move around inside tree-ssa-structalias.cc.
+	(type_can_have_subvars): Move around inside tree-ssa-structalias.cc.
+	(make_param_constraints): Move around inside tree-ssa-structalias.cc.
+	(dump_solution_for_var): External linkage, move to namespace
+	pointer_analysis.  find (...) -> var_rep[...].
+	(get_varinfo): Move to tree-ssa-structalias.h.
+	(debug_solution_for_var): External linkage, move to namespace
+	pointer_analysis.
+	(vi_next): Move to tree-ssa-structalias.h.
+	(dump_sa_stats): External linkage, move to namespace pointer_analysis.
+	(new_var_info): Just move around.
+	(dump_sa_points_to_info): External linkage, move to namespace
+	pointer_analysis.
+	(debug_sa_points_to_info): External linkage, move to namespace
+	pointer_analysis.
+	(get_call_vi): Just move around.
+	(dump_varinfo): External linkage, move to namespace pointer_analysis.
+	(lookup_call_use_vi): Just move around.
+	(lookup_call_clobber_vi): Just move around.
+	(get_call_use_vi): Just move around.
+	(get_call_clobber_vi): Just move around.
+	(enum constraint_expr_type): Move to tree-ssa-structalias.h.
+	(struct constraint_expr): Move to tree-ssa-structalias.h.
+	(UNKNOWN_OFFSET): Move to tree-ssa-structalias.h.
+	(get_constraint_for_1): Just move around.
+	(get_constraint_for): Just move around.
+	(get_constraint_for_rhs): Just move around.
+	(do_deref): Just move around.
+	(constraint_pool): Just move around.
+	(struct constraint_graph): Move to pta-andersen.h.
+	(FIRST_REF_NODE): Move to pta-andersen.cc.
+	(LAST_REF_NODE): Move to pta-andersen.cc.
+	(find): Move to pta-andersen.cc.
+	(unite): Move to pta-andersen.cc.
+	(new_constraint): Just move around.
+	(debug_constraint_graph): External linkage, move to namespace
+	pointer_analysis.
+	(debug_varinfo): External linkage, move to namespace pointer_analysis.
+	(debug_varmap): External linkage, move to namespace pointer_analysis.
+	(dump_constraint_graph): External linkage, move to namespace
+	pointer_analysis.
+	(constraint_expr_equal): Move to pta-andersen.cc.
+	(constraint_expr_less): Move to pta-andersen.cc.
+	(constraint_less): Move to pta-andersen.cc.
+	(constraint_equal): Move to pta-andersen.cc.
+	(constraint_vec_find): Move to pta-andersen.cc.
+	(constraint_set_union): Move to pta-andersen.cc.
+	(solution_set_expand): Move to pta-andersen.cc.
+	(set_union_with_increment): Move to pta-andersen.cc.
+	(insert_into_complex): Move to pta-andersen.cc.
+	(merge_node_constraints): Move to pta-andersen.cc.
+	(clear_edges_for_node): Move to pta-andersen.cc.
+	(merge_graph_nodes): Move to pta-andersen.cc.
+	(add_implicit_graph_edge): Move to pta-andersen.cc.
+	(add_pred_graph_edge): Move to pta-andersen.cc.
+	(add_graph_edge): Move to pta-andersen.cc.
+	(init_graph): Move to pta-andersen.cc.  Initialize
+	predbitmap_obstack here.
+	(build_pred_graph): Move to pta-andersen.cc.
+	(build_succ_graph): Move to pta-andersen.cc.
+	(class scc_info): Move to pta-andersen.cc.
+	(scc_visit): Move to pta-andersen.cc.
+	(solve_add_graph_edge): Move to pta-andersen.cc.
+	(do_sd_constraint): Move to pta-andersen.cc.
+	(do_ds_constraint): Move to pta-andersen.cc.
+	(do_complex_constraint): Move to pta-andersen.cc.
+	(scc_info::scc_info): Move to pta-andersen.cc.
+	(scc_info::~scc_info): Move to pta-andersen.cc.
+	(find_indirect_cycles): Move to pta-andersen.cc.
+	(topo_visit): Move to pta-andersen.cc.
+	(compute_topo_order): Move to pta-andersen.cc.
+	(struct equiv_class_hasher): Move to pta-andersen.cc.
+	(equiv_class_hasher::hash): Move to pta-andersen.cc.
+	(equiv_class_hasher::equal): Move to pta-andersen.cc.
+	(equiv_class_lookup_or_add): Move to pta-andersen.cc.
+	(condense_visit): Move to pta-andersen.cc.
+	(label_visit): Move to pta-andersen.cc.
+	(dump_pred_graph): External linkage, move to namespace
+	pointer_analysis.
+	(dump_varmap): External linkage, move to namespace pointer_analysis.
+	(perform_var_substitution): Move to pta-andersen.cc.
+	(free_var_substitution_info): Move to pta-andersen.cc.
+	(find_equivalent_node): Move to pta-andersen.cc.
+	(unite_pointer_equivalences): Move to pta-andersen.cc.
+	(move_complex_constraints): Move to pta-andersen.cc.
+	(rewrite_constraints): Move to pta-andersen.cc.
+	(eliminate_indirect_cycles): Move to pta-andersen.cc.
+	(solve_graph): Move to pta-andersen.cc.
+	(set_uids_in_ptset): find (...) -> var_rep[...].
+	(find_what_var_points_to): find (...) -> var_rep[...].
+	(init_alias_vars): Don't initialize predbitmap_obstack here.
+	(remove_preds_and_fake_succs): Move to pta-andersen.cc.
+	(solve_constraints): Move to pta-andersen.cc.  Call
+	delete_graph() at the end.
+	(delete_points_to_sets): Don't delete graph here.  Delete var_rep here.
+	(visit_loadstore): find (...) -> var_rep[...].
+	(compute_dependence_clique): find (...) -> var_rep[...].
+	(ipa_pta_execute): find (...) -> var_rep[...].
+	* pta-andersen.cc: New file.
+	* pta-andersen.h: New file.
+	* tree-ssa-structalias.h: New file.
+
+2025-07-17  Richard Sandiford  <richard.sandiford@arm.com>
+	    Yury Khrustalev  <yury.khrustalev@arm.com>
+
+	* doc/sourcebuild.texi (aarch64_sme_hw): Document.
+
+2025-07-17  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* config/s390/s390.md (signbit_tdc): Rename expander.
+	(signbit<mode>2): New expander.
+	(signbit<mode>2_z10): New expander.
+
+2025-07-17  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* config/s390/s390.cc (s390_register_move_cost): Add costing for
+	vlvg/vlgv.
+
+2025-07-17  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* config/s390/vector.md (bhfgq): Add scalar modes.
+	(*movdi<mode>_zero_extend_A): New insn.
+	(*movsi<mode>_zero_extend_A): New insn.
+	(*movdi<mode>_zero_extend_B): New insn.
+	(*movsi<mode>_zero_extend_B): New insn.
+
+2025-07-17  Xi Ruoyao  <xry111@xry111.site>
+
+	PR target/121064
+	* config/loongarch/lsx.md (lsx_vshuf_<lsxfmt_f>): Add '@' to
+	generate a mode-aware helper.  Use <VIMODE> as the mode of the
+	operand 1 (selector).
+	* config/loongarch/lasx.md (lasx_xvshuf_<lasxfmt_f>): Likewise.
+	* config/loongarch/loongarch.cc
+	(loongarch_try_expand_lsx_vshuf_const): Create a new pseudo for
+	the selector.  Use the mode-aware helper to simplify the code.
+	(loongarch_expand_vec_perm_const): Likewise.
+
+2025-07-17  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-slp.cc (vect_build_slp_tree_1): Reject
+	single-lane vector types.
+
+2025-07-17  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121035
+	* tree-ssa-pre.cc (find_or_generate_expression): Handle
+	values without expression.
+
+2025-07-16  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-state-to-dot.cc (state_diagram::m_show_tags): Drop
+	unused field.
+
+2025-07-16  Kwok Cheung Yeung  <kcyeung@baylibre.com>
+
+	* gimplify.cc (gimplify_omp_affinity): Use OMP_ITERATOR_DECL_P.
+	(compute_omp_iterator_count): New.
+	(build_omp_iterator_loop): New.
+	(gimplify_omp_depend): Use OMP_ITERATOR_DECL_P,
+	compute_omp_iterator_count and build_omp_iterator_loop.
+	* tree-inline.cc (copy_tree_body_r): Use OMP_ITERATOR_DECL_P.
+	* tree-pretty-print.cc (dump_omp_clause): Likewise.
+	* tree.h (OMP_ITERATOR_DECL_P): New macro.
+
+2025-07-16  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/121062
+	* config/i386/i386.cc (ix86_convert_const_vector_to_integer):
+	Handle E_V1SImode and E_V1DImode.
+	* config/i386/mmx.md (V_16_32_64): Add V1SI, V2BF and V1DI.
+	(mmxinsnmode): Add V1DI and V1SI.
+	Add V_16_32_64 splitter for constant vector loads from constant
+	vector pool.
+	(V_16_32_64:*mov<mode>_imm): Moved after V_16_32_64 splitter.
+	Replace lowpart_subreg with adjust_address.
+
+2025-07-16  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120881
+	PR testsuite/121078
+	* config/i386/i386-options.cc (ix86_option_override_internal):
+	Warn -pg without -mfentry only on glibc targets.
+
+2025-07-16  Uros Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/i386-expand.cc (ix86_expand_move):
+	Use MEM_P predicate instead of open coding it.
+	(ix86_erase_embedded_rounding):
+	Use NONJUMP_INSN_P predicate instead of open coding it.
+	* config/i386/i386-features.cc (convertible_comparison_p):
+	Use REG_P predicate instead of open coding it.
+	* config/i386/i386.cc (ix86_rtx_costs):
+	Use SUBREG_P predicate instead of open coding it.
+
+2025-07-16  Uros Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/i386.cc (symbolic_reference_mentioned_p):
+	Use LABEL_REF_P predicate instead of open coding it.
+	(ix86_legitimate_constant_p): Ditto.
+	(legitimate_pic_address_disp_p): Ditto.
+	(ix86_legitimate_address_p): Ditto.
+	(legitimize_pic_address): Ditto.
+	(ix86_print_operand): Ditto.
+	(ix86_print_operand_address_as): Ditto.
+	(ix86_rip_relative_addr_p): Ditto.
+	* config/i386/i386.h (SYMBOLIC_CONST): Ditto.
+	* config/i386/i386.md (*anddi_1 to *andsi_1_zext splitter): Ditto.
+	* config/i386/predicates.md (symbolic_operand): Ditto.
+	(local_symbolic_operand): Ditto.
+	(vsib_address_operand): Ditto.
+
+2025-07-16  Uros Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/i386-expand.cc (ix86_expand_move):
+	Use SYMBOL_REF_P predicate instead of open coding it.
+	(ix86_split_long_move): Ditto.
+	(construct_plt_address): Ditto.
+	(ix86_expand_call): Ditto.
+	(ix86_notrack_prefixed_insn_p): Ditto.
+	* config/i386/i386-features.cc
+	(rest_of_insert_endbr_and_patchable_area): Ditto.
+	* config/i386/i386.cc (symbolic_reference_mentioned_p): Ditto.
+	(ix86_force_load_from_GOT_p): Ditto.
+	(ix86_legitimate_constant_p): Ditto.
+	(legitimate_pic_operand_p): Ditto.
+	(legitimate_pic_address_disp_p): Ditto.
+	(ix86_legitimate_address_p): Ditto.
+	(legitimize_pic_address): Ditto.
+	(ix86_legitimize_address): Ditto.
+	(ix86_delegitimize_tls_address): Ditto.
+	(ix86_print_operand): Ditto.
+	(ix86_print_operand_address_as): Ditto.
+	(ix86_rip_relative_addr_p): Ditto.
+	(symbolic_base_address_p): Ditto.
+	* config/i386/i386.h (SYMBOLIC_CONST): Ditto.
+	* config/i386/i386.md (*anddi_1 to *andsi_1_zext splitter): Ditto.
+	* config/i386/predicates.md (symbolic_operand): Ditto.
+	(local_symbolic_operand): Ditto.
+	(local_func_symbolic_operand): Ditto.
+
+2025-07-16  Uros Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/i386-expand.cc (ix86_expand_vector_logical_operator):
+	Use CONST_VECTOR_P instead of open coding it.
+	(ix86_expand_int_sse_cmp): Ditto.
+	(ix86_extract_perm_from_pool_constant): Ditto.
+	(ix86_split_to_parts): Ditto.
+	(const_vector_equal_evenodd_p): Ditto.
+	* config/i386/i386.cc (ix86_print_operand): Ditto.
+	* config/i386/predicates.md (zero_extended_scalar_load_operand): Ditto.
+	(float_vector_all_ones_operand): Ditto.
+	* config/i386/sse.md (avx512vl_vextractf128<mode>): Ditto.
+
+2025-07-16  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121049
+	* internal-fn.h (widening_evenodd_fn_p): Declare.
+	* internal-fn.cc (widening_evenodd_fn_p): New function.
+	* tree-vect-stmts.cc (vectorizable_conversion): When using
+	an even/odd widening function disable loop masking.
+
+2025-07-16  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/119920
+	PR tree-optimization/112324
+	PR tree-optimization/110015
+	* tree-if-conv.cc (find_different_opnum): New function.
+	(factor_out_operators): New function.
+	(predicate_scalar_phi): Call factor_out_operators when
+	there is only 2 elements of a phi.
+
+2025-07-16  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	* tree-if-conv.cc (fold_build_cond_expr): Return early if lhs and rhs
+	are the same.
+
+2025-07-16  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	* tree-if-conv.cc (combine_blocks): Remove predicated
+	dynamic array.
+
+2025-07-16  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121116
+	* tree-vect-loop.cc (vectorizable_induction): Use the
+	step vector element type for further processing.
+
+2025-07-16  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn-valu.md (add<mode>3_vcc_dup<exec_vcc>): Change
+	operand 2 to allow gcn_alu_operand.  Swap the operands in the VCC
+	update RTL.
+	(add<mode>3_vcc_zext_dup): Likewise.
+	(add<mode>3_vcc_zext_dup_exec): Likewise.
+	(add<mode>3_vcc_zext_dup2): Likewise.
+	(add<mode>3_vcc_zext_dup2_exec): Likewise.
+
+2025-07-16  Spencer Abson  <spencer.abson@arm.com>
+
+	PR target/117850
+	* config/aarch64/aarch64-builtins.cc (LO_HI_PAIRINGS): New, group the
+	lo/hi pairs from aarch64-builtin-pairs.def.
+	(aarch64_get_highpart_builtin): New function.
+	(aarch64_v128_highpart_ref): New function, helper to look for vector
+	highparts.
+	(aarch64_build_vector_cst): New function, helper to build duplicated
+	VECTOR_CSTs.
+	(aarch64_fold_lo_call_to_hi): New function.
+	(aarch64_general_gimple_fold_builtin): Add cases for the lo builtins
+	in aarch64-builtin-pairs.def.
+	* config/aarch64/aarch64-builtin-pairs.def: New file, declare the
+	parirs of lowpart-operating and highpart-operating builtins.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* tree.cc (get_clone_versions): New function.
+	(get_clone_attr_versions): New function.
+	(get_version): New function.
+	* tree.h (get_clone_versions): New function.
+	(get_clone_attr_versions): New function.
+	(get_target_version): New function.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* attribs.cc (make_attribute): Change arguments.
+	* attribs.h (make_attribute): Change arguments.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* pretty-print.cc (format_phase_2): Add support for string_slice.
+	* vec.cc (string_slice::tokenize): New static method.
+	(string_slice::strcmp): New static method.
+	(string_slice::strip): New method.
+	(test_string_slice_initializers): New test.
+	(test_string_slice_tokenize): Ditto.
+	(test_string_slice_strcmp): Ditto.
+	(test_string_slice_equality): Ditto.
+	(test_string_slice_inequality): Ditto.
+	(test_string_slice_invalid): Ditto.
+	(test_string_slice_strip): Ditto.
+	(vec_cc_tests): Add new tests.
+	* vec.h (class string_slice): New class.
+
+2025-07-16  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR middle-end/121065
+	* cfgexpand.cc (expand_debug_expr): Allow fixed-point modes for
+	RDIV_EXPR.
+	* optabs-tree.cc (optab_for_tree_code): Ditto.
+
+2025-07-16  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/120297
+	* config/riscv/riscv-vsetvl.def: Do not forget ratio demand of
+	previous vsetvl.
+
+2025-07-16  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-sve2.md (*aarch64_sve2_bsl2n_eon<mode>):
+	New pattern.
+	(*aarch64_sve2_eon_bsl2n_unpred<mode>): Likewise.
+
+2025-07-16  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-sve2.md (*aarch64_sve2_unpred_nor<mode>):
+	New define_insn.
+	(*aarch64_sve2_nand_unpred<mode>): Likewise.
+
+2025-07-16  Jeremy Rifkin  <jeremy@rifkin.dev>
+
+	PR c/82134
+	* gimplify.cc (gimplify_modify_expr): Add suppress_warning
+	* tree-cfg.cc (do_warn_unused_result): Check warning_suppressed_p
+
+2025-07-16  Haochen Jiang  <haochen.jiang@intel.com>
+
+	* common/config/i386/i386-common.cc
+	(OPTION_MASK_ISA2_AMX_AVX512_SET): Do not set AVX10.2.
+	(OPTION_MASK_ISA2_AVX10_2_UNSET): Remove AMX-AVX512 unset.
+	(OPTION_MASK_ISA2_AVX512F_UNSET): Unset AMX-AVX512.
+	(ix86_handle_option): Imply AVX512F for AMX-AVX512.
+
+2025-07-16  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/autovec.md (avg<mode>3_floor): Add new
+	pattern of avg3_floor for rvv DImode.
+
+2025-07-15  David Malcolm  <dmalcolm@redhat.com>
+
+	* spellcheck.cc: Define INCLUDE_ALGORITHM.
+	(CASE_COST, BASE_COST): Convert to...
+	(case_cost, base_cost): ...these, in an anonymous namespace.
+	(get_edit_distance): Update for above.  Use std::min rather than
+	MIN.
+	(get_edit_distance_cutoff): Likewise.  Use std::max rather than
+	MAX.
+	(selftest::test_edit_distances): Update for BASE_COST renaming.
+	(selftest::get_old_cutoff): Likewise.  Use std::max.
+	(selftest::assert_not_suggested_for): Use nullptr.
+	(selftest::test_find_closest_string): Likewise.
+	* spellcheck.h: Replace TYPE with StringLikeType in templates,
+	and use CamelCase.
+
+2025-07-15  David Malcolm  <dmalcolm@redhat.com>
+
+	PR sarif-replay/120792
+	* auto-obstack.h: New file, based on material taken from
+	pretty-print.cc.
+	* diagnostic-digraphs.h
+	(diagnostics::digraphs::digraph::set_description): New.
+	(diagnostics::digraphs::node::set_label): New.
+	* doc/libgdiagnostics/topics/compatibility.rst: Add
+	LIBGDIAGNOSTICS_ABI_4.
+	* doc/libgdiagnostics/topics/diagnostics.rst
+	(diagnostic_finish_via_msg_buf): Document new entrypoint.
+	* doc/libgdiagnostics/topics/execution-paths.rst
+	(diagnostic_execution_path_add_event_via_msg_buf): Document new
+	entrypoint.
+	* doc/libgdiagnostics/topics/index.rst: Add message-buffers.rst.
+	* doc/libgdiagnostics/topics/message-buffers.rst: New file.
+	* doc/libgdiagnostics/topics/message-formatting.rst: Add note
+	about message buffers.
+	* doc/libgdiagnostics/topics/physical-locations.rst
+	(diagnostic_add_location_with_label_via_msg_buf): Add.
+	* doc/libgdiagnostics/tutorial/07-execution-paths.rst: Link to
+	next section.
+	* doc/libgdiagnostics/tutorial/08-message-buffers.rst: New file.
+	* doc/libgdiagnostics/tutorial/index.rst: Add
+	08-message-buffers.rst.
+	* libgdiagnostics++.h (libgdiagnostics::message_buffer): New
+	class.
+	(libgdiagnostics::execution_path::add_event_via_msg_buf): New.
+	(libgdiagnostics::diagnostic::add_location_with_label): New.
+	(libgdiagnostics::diagnostic::finish_via_msg_buf): New.
+	(libgdiagnostics::graph::set_description): New overload.
+	(libgdiagnostics::graph::add_edge): New overload.
+	(libgdiagnostics::node::set_label): New overload.
+	* libgdiagnostics-private.h
+	(private_diagnostic_execution_path_add_event_2): Drop decl.
+	(private_diagnostic_execution_path_add_event_3): New decl.
+	* libgdiagnostics.cc:  Include "pretty-print-format-impl.h",
+	"pretty-print-markup.h", and "auto-obstack.h".
+	(class copying_token_printer): New.
+	(struct diagnostic_message_buffer): New.
+	(class pp_element_message_buffer): New.
+	(libgdiagnostics_path_event::libgdiagnostics_path_event): Replace
+	params "gmsgid" and "args" with "msg_buf".
+	(libgdiagnostics_path_event::print_desc): Reimplement using
+	pp_element_message_buffer to replay m_msg_buf into "pp".
+	(libgdiagnostics_path_event::m_desc_uncolored): Drop field.
+	(libgdiagnostics_path_event::m_desc_colored): Drop field.
+	(libgdiagnostics_path_event::msg_buf): New field.
+	(diagnostic_execution_path::add_event_va): Reimplement.
+	(diagnostic_execution_path::add_event_via_msg_buf): New.
+	(diagnostic::add_location_with_label): New overload, using
+	msg_buf.
+	(diagnostic_manager::emit): Reimplement with...
+	(diagnostic_manager::emit_va): ...this.
+	(diagnostic_manager::emit_msg_buf): New.
+	(FAIL_IF_NULL): Rename "p" to "ptr_arg".
+	(diagnostic_finish_va): Update to use diagnostic_manager::emit_va.
+	(diagnostic_graph::add_node_with_id): Rename "id" to "node_id".
+	(diagnostic_graph_add_node): Likewise.
+	(diagnostic_graph_add_edge): Rename "id" to "edge_id".
+	(diagnostic_graph_get_node_by_id): Rename "id" to "node_id".
+	(diagnostic_graph_get_edge_by_id): Rename "id" to "edge_id".
+	(private_diagnostic_execution_path_add_event_2): Delete.
+	(diagnostic_message_buffer_new): New public entrypoint.
+	(diagnostic_message_buffer_release): Likewise.
+	(diagnostic_message_buffer_append_str): Likewise.
+	(diagnostic_message_buffer_append_text): Likewise.
+	(diagnostic_message_buffer_append_byte): Likewise.
+	(diagnostic_message_buffer_append_printf): Likewise.
+	(diagnostic_message_buffer_append_event_id): Likewise.
+	(diagnostic_message_buffer_begin_url): Likewise.
+	(diagnostic_message_buffer_end_url): Likewise.
+	(diagnostic_message_buffer_begin_quote): Likewise.
+	(diagnostic_message_buffer_end_quote): Likewise.
+	(diagnostic_message_buffer_begin_color): Likewise.
+	(diagnostic_message_buffer_end_color): Likewise.
+	(diagnostic_message_buffer_dump): Likewise.
+	(diagnostic_finish_via_msg_buf): Likewise.
+	(diagnostic_add_location_with_label_via_msg_buf): Likewise.
+	(diagnostic_execution_path_add_event_via_msg_buf): Likewise.
+	(diagnostic_graph_set_description_via_msg_buf): Likewise.
+	(diagnostic_graph_add_edge_via_msg_buf): Likewise.
+	(diagnostic_node_set_label_via_msg_buf): Likewise.
+	(private_diagnostic_execution_path_add_event_3): New private
+	entrypoint.
+	* libgdiagnostics.h (LIBGDIAGNOSTICS_PARAM_FORMAT_STRING): New macro.
+	(LIBGDIAGNOSTICS_PARAM_PRINTF_FORMAT_STRING): New macro.
+	(diagnostic_message_buffer): New typedef.
+	(LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer): New define.
+	(diagnostic_message_buffer_new): New decl.
+	(diagnostic_message_buffer_release): New decl.
+	(diagnostic_message_buffer_append_str): New decl.
+	(diagnostic_message_buffer_append_text): New decl.
+	(diagnostic_message_buffer_append_byte): New decl.
+	(diagnostic_message_buffer_append_printf): New decl.
+	(diagnostic_message_buffer_append_event_id): New decl.
+	(diagnostic_message_buffer_begin_url): New decl.
+	(diagnostic_message_buffer_end_url): New decl.
+	(diagnostic_message_buffer_begin_quote): New decl.
+	(diagnostic_message_buffer_end_quote): New decl.
+	(diagnostic_message_buffer_begin_color): New decl.
+	(diagnostic_message_buffer_end_color): New decl.
+	(diagnostic_message_buffer_dump): New decl.
+	(diagnostic_finish_via_msg_buf): New decl.
+	(diagnostic_add_location_with_label_via_msg_buf): New decl.
+	(diagnostic_execution_path_add_event_via_msg_buf): New decl.
+	(diagnostic_graph_set_description_via_msg_buf): New decl.
+	(diagnostic_graph_add_edge_via_msg_buf): New decl.
+	(diagnostic_node_set_label_via_msg_buf): New decl.
+	* libgdiagnostics.map (LIBGDIAGNOSTICS_ABI_3): Drop
+	private_diagnostic_execution_path_add_event_2.
+	(LIBGDIAGNOSTICS_ABI_4): New.
+	* libsarifreplay.cc (class annotation): Use
+	libgdiagnostics::message_buffer rather than label_text.
+	(add_any_annotations): Likewise.
+	(sarif_replayer::handle_result_obj): Likewise.
+	(make_plain_text_within_result_message): Likewise.
+	(handle_thread_flow_location_object): Likewise.
+	(handle_location_object): Likewise.
+	(sarif_replayer::handle_graph_object): Likewise.
+	(sarif_replayer::handle_node_object): Likewise.
+	(sarif_replayer::handle_edge_object): Likewise.
+	* pretty-print-format-impl.h (pp_token_list::push_back_byte): New
+	decl.
+	* pretty-print-markup.h (pp_markup::context::begin_url): New decl.
+	(pp_markup::context::end_url): New decl.
+	(pp_markup::context::add_event_id): New decl.
+	* pretty-print.cc: Include "auto-obstack.h".
+	(pp_token_list::push_back_byte): New.
+	(struct auto_obstack): Move to auto-obstack.h.
+	(default_token_printer): Make non-static.
+	(pp_markup::context::begin_url): New.
+	(pp_markup::context::end_url): New.
+	(pp_markup::context::add_event_id): New.
+
+2025-07-15  Umesh Kalappa  <ukalappa.mips@gmail.com>
+
+	* config/riscv/riscv-cores.def (RISCV_CORE): Updated the supported march.
+	* config/riscv/riscv-ext-mips.def (DEFINE_RISCV_EXT):
+	New file added for mips conditional mov extension.
+	* config/riscv/riscv-ext.def: Likewise.
+	* config/riscv/t-riscv: Generates riscv-ext.opt
+	* config/riscv/riscv-ext.opt: Generated file.
+	* config/riscv/riscv.cc (riscv_expand_conditional_move): Updated for mips cmov
+	and outlined some code that handle arch cond move.
+	* config/riscv/riscv.md (mov<mode>cc): updated expand for MIPS CCMOV.
+	* config/riscv/mips-insn.md: New file for mips-p8700 ccmov insn.
+	* doc/riscv-ext.texi: Updated for mips cmov.
+
+2025-07-15  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	* avoid-store-forwarding.cc (generate_bit_insert_sequence):
+	Remove adjustment of bitfield insertion's starting position
+	when BYTES_BIG_ENDIAN != BITS_BIG_ENDIAN.
+	(process_store_forwarding): Update offset check in base reg
+	initialization to take into account the target's endianness.
+
+2025-07-15  Soumya AR  <soumyaa@nvidia.com>
+
+	* config/aarch64/aarch64-tuning-flags.def (AARCH64_EXTRA_TUNING_OPTION):
+	Add AVOID_LDAPUR tuning flag.
+	* config/aarch64/aarch64.cc (aarch64_adjust_generic_arch_tuning):
+	Set AVOID_LDAPUR for architectures before armv8.8-a.
+	(aarch64_override_options_internal): Apply generic tuning adjustments
+	to generic_armv8_a_tunings and generic_armv9_a_tunings.
+	* config/aarch64/aarch64.h (TARGET_ENABLE_LDAPUR): New macro to
+	control LDAPUR usage based on RCPC2 and tuning flags.
+	* config/aarch64/aarch64.md: Add enable_ldapur attribute.
+	* config/aarch64/atomics.md (aarch64_atomic_load<mode>_rcpc): Modify
+	to emit LDAPUR for cores with RCPC2.
+	(*aarch64_atomic_load<ALLX:mode>_rcpc_zext): Likewise.
+	(*aarch64_atomic_load<ALLX:mode>_rcpc_sext): Update constraint to Ust.
+	* config/aarch64/tuning_models/cortexx925.h: Add AVOID_LDAPUR flag.
+	* config/aarch64/tuning_models/neoversev2.h: Likewise.
+	* config/aarch64/tuning_models/neoversev3.h: Likewise.
+	* config/aarch64/tuning_models/neoversev3ae.h: Likewise.
+
+2025-07-15  Richard Biener  <rguenther@suse.de>
+	    Richard Sandiford   <richard.sandiford@arm.com>
+
+	PR tree-optimization/121059
+	* tree-vect-stmts.cc (vectorizable_operation): Query
+	scalar_cond_masked_set with the correct number of masks.
+
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* common.opt (Wunused-but-set-parameter=, Wunused-but-set-variable=):
+	New options.
+	(Wunused-but-set-parameter, Wunused-but-set-variable): Turn into
+	aliases.
+	* common.opt.urls: Regenerate.
+	* diagnostic-spec.cc (nowarn_spec_t::nowarn_spec_t): Use
+	OPT_Wunused_but_set_variable_ instead of OPT_Wunused_but_set_variable
+	and OPT_Wunused_but_set_parameter_ instead of
+	OPT_Wunused_but_set_parameter.
+	* gimple-ssa-store-merging.cc (find_bswap_or_nop_1): Remove unused
+	but set variable tmp.
+	* ipa-strub.cc (pass_ipa_strub::execute): Cast named_args to
+	(void) if ATTR_FNSPEC_DECONST_WATERMARK is not defined.
+	* doc/invoke.texi (Wunused-but-set-parameter=,
+	Wunused-but-set-variable=): Document new options.
+	(Wunused-but-set-parameter, Wunused-but-set-variable): Adjust
+	documentation now that they are just aliases.
+
+2025-07-15  Alfie Richards  <alfie.richards@arm.com>
+
+	* config/aarch64/aarch64-sme.md (@aarch64_sme_<faminmax_uns_op><mode>):
+	Change gating and comment.
+
+2025-07-15  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	Revert:
+	2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-simd.md (*eor3qdi4): New
+	define_insn_and_split.
+
+2025-07-15  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (*fcm<cmp_op><mode>_and_combine):
+	Extend from SVE_FULL_F to SVE_F.
+	(*fcmuo<mode>_and_combine): Likewise.
+	(*fcm<cmp_op><mode>_bic_combine): Likewise.
+	(*fcm<cmp_op><mode>_nor_combine): Likewise.
+	(*fcmuo<mode>_bic_combine): Likewise.
+	(*fcmuo<mode>_nor_combine): Likewise.  Move the comment here to
+	above fcmuo<mode>_bic_combine, since it applies to both patterns.
+
+2025-07-15  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* tree.cc: Include memmodel.h.
+
+2025-07-14  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn-valu.md (vec_cmpu<mode>di_exec): Call gen_vec_cmp*,
+	not gen_vec_cmpu*.
+
+2025-07-14  Richard Biener  <rguenther@suse.de>
+
+	Revert:
+	2025-07-14  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121059
+	* tree-vect-stmts.cc (vectorizable_operation): Record a
+	loop mask for mask AND operations.
+
+2025-07-14  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* config/s390/vector.md (reduc_plus_scal_<mode>): Implement.
+	(reduc_plus_scal_v2df): Implement.
+	(reduc_plus_scal_v4sf): Implement.
+	(REDUC_FMINMAX): New int iterator.
+	(reduc_fminmax_name): New int attribute.
+	(reduc_minmax): New code iterator.
+	(reduc_minmax_name): New code attribute.
+	(reduc_<reduc_fminmax_name>_scal_v2df): Implement.
+	(reduc_<reduc_fminmax_name>_scal_v4sf): Implement.
+	(reduc_<reduc_minmax_name>_scal_v2df): Implement.
+	(reduc_<reduc_minmax_name>_scal_v4sf): Implement.
+	(REDUCBIN): New code iterator.
+	(reduc_bin_insn): New code attribute.
+	(reduc_<reduc_bin_insn>_scal_v2di): Implement.
+	(reduc_<reduc_bin_insn>_scal_v4si): Implement.
+	(reduc_<reduc_bin_insn>_scal_v8hi): Implement.
+	(reduc_<reduc_bin_insn>_scal_v16qi): Implement.
+
+2025-07-14  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* config/s390/s390.cc (s390_option_override_internal): Remove override.
+
+2025-07-14  Andrew Stubbs  <ams@baylibre.com>
+
+	* config/gcn/gcn-valu.md (add<mode>3<exec_clobber>): Rename ...
+	(add<mode>3<exec>): ... to this, remove the clobber, and change the
+	instruction from v_add_co_u32 to v_add_u32.
+	(add<mode>3_dup<exec_clobber>): Rename ...
+	(add<mode>3_dup<exec>): ... to this, and likewise.
+	(sub<mode>3<exec_clobber>): Rename ...
+	(sub<mode>3<exec>): ... to this, and likewise
+	* config/gcn/gcn.md (addsi3): Remove the DI clobber, and change the
+	instruction from v_add_co_u32 to v_add_u32.
+	(addsi3_scc): Likewise.
+	(subsi3): Likewise, but for v_sub_co_u32.
+	(muldi3): Likewise.
+
+2025-07-14  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121059
+	* tree-vect-stmts.cc (vectorizable_operation): Record a
+	loop mask for mask AND operations.
+
+2025-07-14  Pan Li  <pan2.li@intel.com>
+
+	* match.pd: Make sure widen mul has twice bitsize
+	of the inputs in SAT_MUL pattern.
+
+2025-07-14  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/121015
+	* config/i386/i386-features.cc (ix86_broadcast_inner): Check all
+	0s/1s vectors with standard_sse_constant_p.
+
+2025-07-14  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120881
+	* config.in: Regenerated.
+	* configure: Likewise.
+	* configure.ac: Add --enable-x86-64-mfentry.
+	* config/i386/i386-options.cc (ix86_option_override_internal):
+	Enable __fentry__ in 64-bit mode if ENABLE_X86_64_MFENTRY is set
+	to 1.  Warn -pg without -mfentry with shrink wrapping enabled.
+	* doc/install.texi: Document --enable-x86-64-mfentry.
+
+2025-07-14  François-Xavier Coudert  <fxcoudert@gcc.gnu.org>
+
+	PR target/120645
+	* config/darwin-driver.cc: Account for latest macOS numbering
+	scheme.
+
+2025-07-14  Paul-Antoine Arras  <parras@baylibre.com>
+
+	PR target/119100
+	* config/riscv/autovec-opt.md (*vfwmacc_vf_<mode>): New pattern to
+	handle both vfwmacc and vfwmsac.
+	(*extend_vf_<mode>): New pattern that serves as an intermediate combine
+	step.
+	* config/riscv/vector-iterators.md (vsubel): New mode attribute. This is
+	just the lower-case version of VSUBEL.
+	* config/riscv/vector.md (@pred_widen_mul_<optab><mode>_scalar): Reorder
+	and swap operands to match the RTL emitted by expand, i.e. first
+	float_extend then vec_duplicate.
+
+2025-07-14  Alfie Richards  <alfie.richards@arm.com>
+
+	* config/aarch64/aarch64-sme.md (@aarch64_sme_<faminmax_uns_op><mode>):
+	New patterns.
+	* config/aarch64/aarch64-sve-builtins-sme.def (svamin): New intrinsics.
+	(svamax): New intrinsics.
+	* config/aarch64/aarch64-sve-builtins-sve2.cc (class faminmaximpl): New
+	class.
+	(svamin): New function.
+	(svamax): New function.
+
+2025-07-14  Haochen Jiang  <haochen.jiang@intel.com>
+
+	* config/i386/i386.h (PTA_PANTHERLAKE): Revmoe KL and WIDEKL.
+	(PTA_CLEARWATERFOREST): Ditto.
+	* doc/invoke.texi: Revise documentation.
+
+2025-07-13  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/120866
+	* tree.cc: Add include to tm_p.h.
+
+2025-07-13  Benjamin Wu  <bwu25@cs.washington.edu>
+
+	* gimple.h (GTMA_DOES_GO_IRREVOCABLE): Fix typo.
+
+2025-07-12  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc (function_instance::~function_instance):
+	Move down in source.
+	(string_table::get_cgraph_node): New member function with
+	logic broken out from ...
+	(function_instance::get_cgraph_node): ... here.
+	(match_with_target): Fix formating.
+	(function_instance::match): Fix formating; do not use iterators
+	after modifying map; remove incorrect set of warned flag.
+	(autofdo_source_profile::offline_external_functions): Keep
+	seen set up to date.
+	(function_instance::read_function_instance): Fix formating.
+
+2025-07-12  Uros Bizjak  <ubizjak@gmail.com>
+
+	* config/i386/mmx.md (mov<V_32:mode>):
+	Use nonimm_or_0_operand predicate for operand 1.
+	(*mov<V_32:mode>_internal): Ditto.
+	(movv2qi): Ditto.
+	(*movv2qi_internal): Ditto.  Use ix86_hardreg_mov_ok
+	in insn condition.
+
+2025-07-12  Xi Ruoyao  <xry111@xry111.site>
+
+	PR rtl-optimization/120983
+	* lra-constraints.cc (process_alt_operands): Allow reloading
+	user hard registers unless the insn is an asm.
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-format-html.cc: Include "diagnostic-format-sarif.h",
+	Replace include of "diagnostic-state.h" with includes of
+	"diagnostic-digraphs.h" and "diagnostic-state-graphs.h".
+	(html_generation_options::html_generation_options): Update for
+	field renaming.
+	(html_builder::m_body_element): New field.
+	(html_builder::html_builder): Initialize m_body_element.
+	(html_builder::maybe_make_state_diagram): Port from XML
+	implementation to state graph implementation.
+	(html_builder::make_element_for_diagnostic): Add any
+	per-diagnostic graphs.
+	(html_builder::add_graph): New.
+	(html_builder::emit_global_graph): New.
+	(html_output_format::report_global_digraph): New.
+	* diagnostic-format-html.h
+	(html_generation_options::m_show_state_diagram_xml): Replace
+	with...
+	(html_generation_options::m_show_state_diagrams_sarif): ...this.
+	(html_generation_options::m_show_state_diagram_dot_src): Rename
+	to...
+	(html_generation_options::m_show_state_diagrams_dot_src): ...this.
+	* diagnostic-format-sarif.cc: Include "diagnostic-digraphs.h" and
+	"diagnostic-state-graphs.h".
+	(sarif_builder::m_run_graphs): New field.
+	(sarif_result::on_nested_diagnostic): Update call to
+	make_location_object to pass arg by pointer.
+	(sarif_builder::sarif_builder): Initialize m_run_graphs.
+	(sarif_builder::report_global_digraph): New.
+	(sarif_builder::make_result_object): Add any graphs to
+	the result object.
+	(sarif_builder::make_locations_arr): Update call to
+	make_location_object to pass arg by pointer.
+	(sarif_builder::make_location_object): Pass param "loc_mgr" by
+	pointer rather than by reference so that it can be null, and
+	handle this case.
+	(copy_any_property_bag): New.
+	(make_sarif_graph): New.
+	(make_sarif_node): New.
+	(make_sarif_edge): New.
+	(sarif_property_bag::set_graph): New.
+	(populate_thread_flow_location_object): Port from XML
+	implementation to state graph implementation.
+	(make_run_object): Store any graphs.
+	(sarif_output_format::report_global_digraph): New.
+	(sarif_generation_options::sarif_generation_options): Rename
+	m_xml_state to m_state_graph.
+	(selftest::test_make_location_object): Update for change to
+	make_location_object.
+	* diagnostic-format-sarif.h:
+	(sarif_generation_options::m_xml_state): Replace with...
+	(sarif_generation_options::m_state_graph): ...this.
+	(class sarif_location_manager): Add forward decl.
+	(diagnostics::digraphs::digraph): New forward decl.
+	(diagnostics::digraphs::node): New forward decl.
+	(diagnostics::digraphs::edge): New forward decl.
+	(sarif_property_bag::set_graph): New decl.
+	(class sarif_graph): New.
+	(class sarif_node): New.
+	(class sarif_edge): New.
+	(make_sarif_graph): New decl.
+	(make_sarif_node): New decl.
+	(make_sarif_edge): New decl.
+	* diagnostic-format-text.h
+	(diagnostic_text_output_format::report_global_digraph): New.
+	* diagnostic-format.h
+	(diagnostic_output_format::report_global_digraph): New vfunc.
+	* diagnostic-digraphs.cc: New file.
+	* diagnostic-digraphs.h: New file.
+	* diagnostic-metadata.h (diagnostics::digraphs::lazy_digraphs):
+	New forward decl.
+	(diagnostic_metadata::diagnostic_metadata): Initialize
+	m_lazy_digraphs.
+	(diagnostic_metadata::set_lazy_digraphs): New.
+	(diagnostic_metadata::get_lazy_digraphs): New.
+	(diagnostic_metadata::m_lazy_digraphs): New field.
+	* diagnostic-output-spec.cc (sarif_scheme_handler::make_sink):
+	Update for XML to state graph changes.
+	(sarif_scheme_handler::make_sarif_gen_opts): Likewise.
+	(html_scheme_handler::make_sink): Rename "show-state-diagram-xml"
+	to "show-state-diagrams-sarif" and use pluralization consistently.
+	* diagnostic-path.cc: Replace include of "xml.h" with
+	"diagnostic-state-graphs.h".
+	(diagnostic_event::maybe_make_xml_state): Replace with...
+	(diagnostic_event::maybe_make_diagnostic_state_graph): ...this.
+	* diagnostic-path.h (diagnostics::digraphs::digraph): New forward
+	decl.
+	(diagnostic_event::maybe_make_xml_state): Replace with...
+	(diagnostic_event::maybe_make_diagnostic_state_graph): ...this.
+	* diagnostic-state-graphs.cc: New file.
+	* diagnostic-state-graphs.h: New file.
+	* diagnostic-state-to-dot.cc: Port implementation from XML to
+	state graphs.
+	* diagnostic-state.h: Deleted file.
+	* diagnostic.cc (diagnostic_context::report_global_digraph): New.
+	* diagnostic.h (diagnostics::digraphs::lazy_digraph): New forward
+	decl.
+	(diagnostic_context::report_global_digraph): New decl.
+	* doc/analyzer.texi (Debugging the Analyzer): Update to reflect
+	change from XML to state graphs.
+	* doc/invoke.texi ("sarif" diagnostics sink): Replace "xml-state"
+	with "state-graphs".
+	("experimental-html" diagnostics sink): Replace
+	"show-state-diagrams-xml" with "show-state-diagrams-sarif"
+	* doc/libgdiagnostics/topics/compatibility.rst
+	(LIBGDIAGNOSTICS_ABI_3): New.
+	* doc/libgdiagnostics/topics/graphs.rst: New file.
+	* doc/libgdiagnostics/topics/index.rst: Add graphs.rst.
+	* graphviz.h (node_id::operator=): New.
+	* json.h (json::value::dyn_cast_string): New.
+	(json::object::get_num_keys): New accessor.
+	(json::object::get_key): New accessor.
+	(json::string::dyn_cast_string): New.
+	* libgdiagnostics++.h (class libgdiagnostics::graph): New.
+	(class libgdiagnostics::node): New.
+	(class libgdiagnostics::edge): New.
+	(class libgdiagnostics::diagnostic::take_graph): New.
+	(class libgdiagnostics::manager::take_global_graph): New.
+	(class libgdiagnostics::graph::set_description): New.
+	(class libgdiagnostics::graph::get_node_by_id): New.
+	(class libgdiagnostics::graph::get_edge_by_id): New.
+	(class libgdiagnostics::graph::add_edge): New.
+	(class libgdiagnostics::node::set_label): New.
+	(class libgdiagnostics::node::set_location): New.
+	(class libgdiagnostics::node::set_logical_location): New.
+	* libgdiagnostics-private.h: New file.
+	* libgdiagnostics.cc: Define INCLUDE_STRING.  Include
+	"diagnostic-digraphs.h", "diagnostic-state-graphs.h", and
+	"libgdiagnostics-private.h".
+	(struct diagnostic_graph): New.
+	(struct diagnostic_node): New.
+	(struct diagnostic_edge): New.
+	(libgdiagnostics_path_event::libgdiagnostics_path_event): Add
+	state_graph param.
+	(libgdiagnostics_path_event::maybe_make_diagnostic_state_graph):
+	New.
+	(libgdiagnostics_path_event::m_state_graph): New field.
+	(diagnostic_execution_path::add_event_va): Add state_graph param.
+	(class prebuilt_digraphs): New.
+	(diagnostic::diagnostic): Use m_graphs in m_metadata.
+	(diagnostic::take_graph): New.
+	(diagnostic::get_graphs): New accessor.
+	(diagnostic::m_graphs): New field.
+	(diagnostic_manager::take_global_graph): New.
+	(diagnostic_execution_path_add_event): Update for new param to
+	add_event_va.
+	(diagnostic_execution_path_add_event_va): Likewise.
+	(diagnostic_graph::add_node_with_id): New public entrypoint.
+	(diagnostic_graph::add_edge_with_label): New public entrypoint.
+	(diagnostic_manager_new_graph): New public entrypoint.
+	(diagnostic_manager_take_global_graph): New public entrypoint.
+	(diagnostic_take_graph): New public entrypoint.
+	(diagnostic_graph_release): New public entrypoint.
+	(diagnostic_graph_set_description): New public entrypoint.
+	(diagnostic_graph_add_node): New public entrypoint.
+	(diagnostic_graph_add_edge): New public entrypoint.
+	(diagnostic_graph_get_node_by_id): New public entrypoint.
+	(diagnostic_graph_get_edge_by_id): New public entrypoint.
+	(diagnostic_node_set_location): New public entrypoint.
+	(diagnostic_node_set_label): New public entrypoint.
+	(diagnostic_node_set_logical_location): New public entrypoint.
+	(private_diagnostic_execution_path_add_event_2): New private
+	entrypoint.
+	(private_diagnostic_graph_set_property_bag): New private
+	entrypoint.
+	(private_diagnostic_node_set_property_bag): New private
+	entrypoint.
+	(private_diagnostic_edge_set_property_bag): New private
+	entrypoint.
+	* libgdiagnostics.h (diagnostic_graph): New typedef.
+	(diagnostic_node): New typedef.
+	(diagnostic_edge): New typedef.
+	(diagnostic_manager_new_graph): New decl.
+	(diagnostic_manager_take_global_graph): New decl.
+	(diagnostic_take_graph): New decl.
+	(diagnostic_graph_release): New decl.
+	(diagnostic_graph_set_description): New decl.
+	(diagnostic_graph_add_node): New decl.
+	(diagnostic_graph_add_edge): New decl.
+	(diagnostic_graph_get_node_by_id): New decl.
+	(diagnostic_graph_get_edge_by_id): New decl.
+	(diagnostic_node_set_label): New decl.
+	(diagnostic_node_set_location): New decl.
+	(diagnostic_node_set_logical_location): New decl.
+	* libgdiagnostics.map (LIBGDIAGNOSTICS_ABI_3): New.
+	* libsarifreplay.cc: Include "libgdiagnostics-private.h".
+	(id_map): New "using".
+	(sarif_replayer::report_invalid_sarif): Update for change to
+	report_problem params.
+	(sarif_replayer::report_unhandled_sarif): Likewise.
+	(sarif_replayer::report_note): New.
+	(sarif_replayer::report_problem): Pass param "ref" by
+	pointer rather than reference and handle it being null.
+	(sarif_replayer::maybe_get_property_bag): New.
+	(sarif_replayer::maybe_get_property_bag_value): New.
+	(sarif_replayer::handle_run_obj): Handle run-level "graphs" as per
+	§3.14.20.
+	(sarif_replayer::handle_result_obj): Handle result-level "graphs"
+	as per §3.27.19.
+	(handle_thread_flow_location_object): Optionally handle graphs
+	stored in property "gcc/diagnostic_event/state_graph" as state
+	graphs.
+	(sarif_replayer::handle_graph_object): New.
+	(sarif_replayer::handle_node_object): New.
+	(sarif_replayer::handle_edge_object): New.
+	(sarif_replayer::get_graph_node_by_id_property): New.
+	* selftest-run-tests.cc (selftest::run_tests): Call
+	selftest::diagnostic_graph_cc_tests and
+	selftest::diagnostic_state_graph_cc_tests.
+	* selftest.h (selftest::diagnostic_graph_cc_tests): New decl.
+	(selftest::diagnostic_state_graph_cc_tests): New decl.
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* Makefile.in (OBJS-libcommon): Add diagnostic-digraphs.o and
+	diagnostic-state-graphs.o.
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* json.cc (json::object::clone): New.
+	(json::object::clone_as_object): New.
+	(json::array::clone): New.
+	(json::float_number::clone): New.
+	(json::integer_number::clone): New.
+	(json::string::clone): New.
+	(json::literal::clone): New.
+	(selftest::test_cloning): New test.
+	(selftest::json_cc_tests): Call it.
+	* json.h (json::value::clone): New vfunc.
+	(json::object::clone): New decl.
+	(json::object::clone_as_object): New decl.
+	(json::array::clone): New decl.
+	(json::float_number::clone): New decl.
+	(json::integer_number::clone): New decl.
+	(json::string::clone): New decl.
+	(json::literal::clone): New decl.
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* json.cc (string::string): When constructing from pointer and
+	length, ensure the new buffer is null-terminated.
+	(selftest::test_strcmp): New.
+	(selftest::json_cc_tests): Likewise.
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* doc/libgdiagnostics/topics/compatibility.rst
+	(_LIBGDIAGNOSTICS_ABI_2): Add missing anchor.
+	* doc/libgdiagnostics/topics/diagnostic-manager.rst
+	(diagnostic_manager_add_sink_from_spec): Add links to GCC's
+	documentation of "-fdiagnostics-add-output=".  Fix parameter
+	markup.
+	(diagnostic_manager_set_analysis_target): Fix parameter markup.
+	Add link to SARIF spec.
+	* doc/libgdiagnostics/topics/logical-locations.rst: Markup fix.
+	* doc/libgdiagnostics/tutorial/02-physical-locations.rst: Clarify
+	wording of what "the source file" means, and that a range can't
+	have multiple files.
+
+2025-07-11  Vladimir N. Makarov  <vmakarov@redhat.com>
+
+	* lra-constraints.cc (process_address_1): When changing base reg
+	on a reg of the base class, fall back to reload of whole inner address.
+	(process_address): Constrain the iteration number.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/119064
+	* doc/invoke.texi (Wc++26-compat): Document.
+
+2025-07-11  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR target/121027
+	* config/aarch64/aarch64.cc (aarch64_evpc_sve_tbl): Punt on 2-input
+	operations that can be handled by vec_perm.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-simd.md (*eor3qdi4): New
+	define_insn_and_split.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-simd.md (*bcaxqdi4): New
+	define_insn_and_split.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-simd.md (eor3q<mode>4): Use VDQ_I mode
+	iterator.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64-simd.md (bcaxq<mode>4): Use VDQ_I mode
+	iterator.
+
+2025-07-11  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121034
+	* tree-vect-loop.cc (vectorizable_reduction): Cleanup
+	reduction chain following code.
+
+2025-07-11  Jan Hubicka  <hubicka@ucw.cz>
+
+	* opts.cc (finish_options): Enable debug_nonbind_markers_p for
+	auto-profile.
+	* tree-cfg.cc (struct locus_discrim_map): Remove.
+	(struct locus_discrim_hasher): Remove.
+	(locus_discrim_hasher::hash): Remove.
+	(locus_discrim_hasher::equal): Remove.
+	(first_non_label_nondebug_stmt): Remove.
+	(build_gimple_cfg): Do not allocate discriminator tables.
+	(next_discriminator_for_locus): Remove.
+	(same_line_p): Remove.
+	(struct discrim_entry): New structure.
+	(assign_discriminator): Rewrite.
+	(assign_discriminators): Rewrite.
+
+2025-07-11  Jan Hubicka  <hubicka@ucw.cz>
+
+	PR ipa/114790
+	* cgraph.cc (cgraph_update_edges_for_call_stmt_node): Resolve devirtualization
+	if call statement was optimized out or turned to direct call.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+	    Martin Jambor  <mjambor@suse.cz>
+
+	PR ipa/121023
+	* ipa-fnsummary.cc (compute_fn_summary): Disallow signature changes
+	on cfun->has_musttail functions.
+
+2025-07-11  Hu, Lin1  <lin1.hu@intel.com>
+
+	PR target/91384
+	* config/i386/i386.md: Add new peeophole2 for optimize *negsi_1
+	followed by *cmpsi_ccno_1 with APX_F.
+
+2025-07-11  Richard Biener  <rguenther@suse.de>
+
+	* config/i386/i386.cc (ix86_vector_costs::add_stmt_cost): Use
+	the LHS of a scalar stmt to determine mode and whether it is FP.
+
+2025-07-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.cc (aarch64_vector_costs::add_stmt_cost):
+	Guard VF-based costing with !m_costing_for_scalar.
+
+2025-07-10  Qing Zhao  <qing.zhao@oracle.com>
+
+	* internal-fn.cc (expand_ACCESS_WITH_SIZE): Update comments.
+	* internal-fn.def (ACCESS_WITH_SIZE): Update comments.
+	* tree-object-size.cc (access_with_size_object_size): Update comments.
+	Adjust the arguments per the new design.
+
+2025-07-10  Qing Zhao  <qing.zhao@oracle.com>
+
+	PR middle-end/121000
+	* internal-fn.cc (expand_ACCESS_WITH_SIZE): Update comments.
+	* internal-fn.def (ACCESS_WITH_SIZE): Update comments.
+	* tree-object-size.cc (access_with_size_object_size): Update comments.
+	Get the element_size from the 6th argument directly.
+
+2025-07-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve2.md (aarch64_gather_ld1q): Replace with...
+	(@aarch64_gather_ld1q<mode>): ...this, parameterizing based on mode.
+	* config/aarch64/aarch64-sve-builtins-sve2.cc
+	(svld1q_gather_impl::expand): Update accordingly.
+	(svst1q_scatter_impl::expand): Use aarch64_sve_reinterpret
+	instead of force_lowpart_subreg.
+
+2025-07-10  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc: Include output.h.
+	(function_instance::set_call_location): Also sanity check
+	that location is known.
+	(raw_symbol_name): Two new static functions.
+	(dump_inline_stack): Use it.
+	(string_table::get_index_by_decl): Likewise.
+	(function_instance::get_cgraph_node): Likewise.
+	(function_instance::get_function_instance_by_decl): Fix typo
+	in warning; use raw names; fix lineno decoding.
+	(match_with_target): Add containing funciton parameter;
+	correctly output function and call location in warning.
+	(function_instance::lookup_count): Fix warning locations.
+	(function_instance::match): Fix warning locations; avoid
+	crash with mismatched callee; do not warn about broken callsites
+	twice.
+	(autofdo_source_profile::offline_external_functions): Use
+	raw_assembler_name.
+	(walk_block): Use raw_assembler_name.
+
+2025-07-10  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/121014
+	* cfgexpand.cc (expand_debug_expr): Assert FLOAT_MODE_P.
+	* optabs-tree.cc (optab_for_tree_code): Assert FLOAT_TYPE_P.
+	* tree-vect-loop.cc (vect_get_loop_len): Use EXACT_DIV_EXPR.
+
+2025-07-10  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/118734
+	* config/riscv/constraints.md (Wdm): Use tunable for Wdm
+	constraint.
+	* config/riscv/riscv-protos.h (emit_avltype_insn): Declare.
+	(can_be_broadcasted_p): Rename to...
+	(can_be_broadcast_p): ...this.
+	* config/riscv/predicates.md: Use renamed function.
+	(strided_load_broadcast_p): Declare.
+	* config/riscv/riscv-selftests.cc (run_broadcast_selftests):
+	Only run broadcast selftest if strided broadcasts are OK.
+	* config/riscv/riscv-v.cc (emit_avltype_insn): New function.
+	(sew64_scalar_helper): Only emit a pred_broadcast if the new
+	tunable says so.
+	(can_be_broadcasted_p): Rename to...
+	(can_be_broadcast_p): ...this and use new tunable.
+	* config/riscv/riscv.cc (struct riscv_tune_param): Add strided
+	broad tunable.
+	(strided_load_broadcast_p): Implement.
+	* config/riscv/vector.md: Use strided_load_broadcast_p () and
+	work around 64-bit broadcast on rv32 targets.
+
+2025-07-10  Co-authored-by: Jeff Law  <jlaw@ventanamicro.com>
+
+	* config/riscv/riscv.cc (riscv_fusion_pairs): Add new cases.
+	(riscv_set_is_add): New function.
+	(riscv_set_is_addi, riscv_set_is_adduw, riscv_set_is_shNadd): Likewise.
+	(riscv_set_is_shNadduw): Likewise.
+	(riscv_macro_fusion_pair_p): Add new fusion cases.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-slp.cc (vect_analyze_slp): Fail for non-canonical
+	gconds.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-slp.cc (vect_build_slp_instance): Do not use
+	SLP_TREE_VECTYPE to determine the conversion back to the
+	reduction IV.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vectorizable_reduction): Avoid
+	vect_is_simple_use and record a vector type if we come
+	up with one.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-stmts.cc (get_load_store_type): Do not use
+	vect_is_simple_use to fill gather/scatter offset operand
+	vectype and dt.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vect_model_reduction_cost): Get SLP
+	node instead of stmt_info and use that when recording costs.
+
+2025-07-10  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	PR target/120999
+	* config/aarch64/aarch64-sve2.md (*aarch64_sve2_nor<mode>):
+	Adjust movprfx alternative.
+
+2025-07-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* doc/sourcebuild.texi (aarch64_sve2_hw, aarch64_sve2p1_hw): Document.
+	* config/aarch64/aarch64.cc (aarch64_evpc_hvla): Extend to
+	BYTES_BIG_ENDIAN.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vect_analyze_stmt): Remove stmt-info
+	and need_to_vectorize arguments.
+	* tree-vect-slp.cc (vect_slp_analyze_node_operations_1):
+	Adjust.
+	* tree-vect-stmts.cc (can_vectorize_live_stmts): Remove
+	stmt_info argument and remove non-SLP path.
+	(vect_analyze_stmt): Remove stmt_info and need_to_vectorize
+	argument and prune paths no longer reachable.
+	(vect_transform_stmt): Adjust.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	* config/i386/x86-tune.def: Change "Tunning the" to "tuning" in
+	comment and use semicolon instead of dot in comment.
+	* loop-unroll.cc (decide_unroll_stupid): Comment spelling fix,
+	tunning -> tuning.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	* tree-vect-loop.cc (scale_profile_for_vect_loop): Comment
+	spelling fix: bellow -> below.
+	* ipa-polymorphic-call.cc (record_known_type): Likewise.
+	* config/i386/x86-tune.def: Likewise.
+	* config/riscv/vector.md (*vsetvldi_no_side_effects_si_extend):
+	Likewise.
+	* tree-scalar-evolution.cc (iv_can_overflow_p): Likewise.
+	* ipa-devirt.cc (add_type_duplicate): Likewise.
+	* tree-ssa-loop-niter.cc (maybe_lower_iteration_bound): Likewise.
+	* gimple-ssa-sccopy.cc: Likewise.
+	* cgraphunit.cc: Likewise.
+	* graphite.h (struct poly_dr): Likewise.
+	* ipa-reference.cc (ignore_edge_p): Likewise.
+	* tree-ssa-alias.cc (ao_compare::compare_ao_refs): Likewise.
+	* profile-count.h (profile_probability::probably_reliable_p):
+	Likewise.
+	* ipa-inline-transform.cc (inline_call): Likewise.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vect_dissolve_slp_only_groups): Remove.
+	(vect_analyze_loop_2): Do not call it.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vect_active_double_reduction_p): Remove.
+	(vect_analyze_loop_operations): Remove.
+	(vect_analyze_loop_2): Do not call it.
+
+2025-07-10  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vect_determine_vf_for_stmt_1): Rename
+	to ...
+	(vect_determine_vectype_for_stmt_1): ... this and only set
+	STMT_VINFO_VECTYPE.  Fail for single-element vector types.
+	(vect_determine_vf_for_stmt): Rename to ...
+	(vect_determine_vectype_for_stmt): ... this and only set
+	STMT_VINFO_VECTYPE. Fail for single-element vector types.
+	(vect_determine_vectorization_factor): Rename to ...
+	(vect_set_stmts_vectype): ... this and only set STMT_VINFO_VECTYPE.
+	(vect_update_vf_for_slp): Remove.
+	(vect_analyze_loop_operations): Remove walk over stmts.
+	(vect_analyze_loop_2): Call vect_set_stmts_vectype instead of
+	vect_determine_vectorization_factor.  Set vectorization factor
+	from LOOP_VINFO_SLP_UNROLLING_FACTOR.  Fail if vect_detect_hybrid_slp
+	detects hybrid stmts or when vect_make_slp_decision finds
+	nothing to SLP.
+	* tree-vect-slp.cc (vect_detect_hybrid_slp): Move check
+	whether we have any hybrid stmts here from vect_update_vf_for_slp
+	* tree-vect-stmts.cc (vect_analyze_stmt): Remove loop over
+	stmts.
+	* tree-vectorizer.h (vect_detect_hybrid_slp): Update.
+
+2025-07-09  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.cc (aarch64_simd_valid_imm): Account
+	for FLOAT_WORDS_BIG_ENDIAN when building a floating-point value.
+
+2025-07-09  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc (afdo_adjust_guessed_profile): Add forgotten
+	if (dump_file) guard.
+
+2025-07-09  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64.cc (aarch64_sve_index_series_p): New
+	function, split out from...
+	(aarch64_simd_valid_imm): ...here.  Account for the different
+	SVE and Advanced SIMD element orders on big-endian targets.
+	Check each vector in a structure mode.
+
+2025-07-09  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* read-rtl-function.cc (function_reader::read_rtx_operand_r): Use
+	hard_regno_nregs to work out REG_NREGS for hard registers.
+
+2025-07-09  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/riscv-v.cc (expand_vx_binary_vec_vec_dup): Add
+	new case SS_MINUS.
+	* config/riscv/riscv.cc (riscv_rtx_costs): Ditto.
+	* config/riscv/vector-iterators.md: Add new op ss_minus.
+
+2025-07-09  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* ext-dce.cc (ext_dce_process_uses): Apply is_constant directly
+	to the subreg_lsb.
+
+2025-07-09  Jan Dubiec  <jdx@o2.pl>
+
+	PR target/109286
+	* config.gcc: Include elfos.h before h8300/h8300.h.
+	* config/h8300/h8300.h (INIT_SECTION_ASM_OP): Override
+	default version from elfos.h.
+	(FINI_SECTION_ASM_OP): Ditto.
+	(ASM_DECLARE_FUNCTION_NAME): Ditto.
+	(ASM_GENERATE_INTERNAL_LABEL): Macro removed because it was
+	being overridden in elfos.h anyway.
+	(ASM_OUTPUT_SKIP): Ditto.
+
+2025-07-09  Icen Zeyada  <Icen.Zeyada2@arm.com>
+
+	PR tree-optimization/119196
+	* match.pd: Allow scalar optimizations with bitwise AND/OR/XOR to apply to vectors.
+
+2025-07-09  Icen Zeyada  <Icen.Zeyada2@arm.com>
+
+	PR tree-optimization/119196
+	* match.pd: Merge multiple vec_cond_expr in a single one for
+	bit_and, bit_ior and bit_xor.
+
+2025-07-09  Jeff Law  <jlaw@ventanamicro.com>
+
+	PR target/120642
+	* config/riscv/riscv-avlprop.cc (pass_avlprop::execute): Do not do
+	constant AVL propagation for xtheadvector.
+
+2025-07-09  Richard Biener  <rguenther@suse.de>
+
+	* tree-vect-loop.cc (vectorizable_reduction): Get the
+	output vector type from slp_for_stmt_info.
+	* tree-vect-stmts.cc (vect_analyze_stmt): Bail out earlier
+	for PURE_SLP_STMT when doing loop stmt analysis.
+
+2025-07-09  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc (struct scale): New structure.
+	(add_scale): Also record weights.
+	(afdo_adjust_guessed_profile): Compute robust average
+	of scales and cap by max count in function.
+
+2025-07-09  Jan Hubicka  <hubicka@ucw.cz>
+
+	* tree-inline.cc (initialize_cfun): Use num and den for scaling.
+
+2025-07-09  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc (get_original_name): Fix loop walking the
+	suffixes.
+
+2025-07-09  Christophe Lyon  <christophe.lyon@linaro.org>
+
+	* config/arm/arm_neon.h: Remove useless push/pop pragmas.
+
+2025-07-09  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/120922
+	* tree-vect-loop-manip.cc (vect_gen_vector_loop_niters): Support range
+	for partial vectors.
+
+2025-07-09  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/120922
+	* tree-vect-loop-manip.cc (vect_gen_vector_loop_niters): Don't set range
+	for partial vectors.
+
+2025-07-08  Takayuki 'January June' Suwa  <jjsuwa_sys3175@yahoo.co.jp>
+
+	* config/xtensa/xtensa.cc (xtensa_b4const_or_zero):
+	Remove.
+	(xtensa_b4const): Add a case where the value is 0, and rename
+	to xtensa_b4const_or_zero.
+	(xtensa_rtx_costs): Fix to also consider the result of
+	xtensa_b4constu().
+
+2025-07-08  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* config/s390/s390.md (stack_protect_get_tpsi): New insn.
+	(stack_protect_get_tpdi): New insn.
+	(stack_protect_set): Use new insn.
+	(stack_protect_test): Use new insn.
+
+2025-07-08  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/120461
+	* config/riscv/riscv-v.cc (emit_vlmax_insn_lra): Do not emit
+	vsetivli for XTHeadVector.
+
+2025-07-08  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/113829
+	* config/riscv/riscv-vector-builtins.cc (registered_function::overloaded_hash):
+	Skip non-type arguments.
+
+2025-07-08  Andreas Schwab  <schwab@suse.de>
+
+	PR target/120995
+	* config/riscv/sync.md (zacas_atomic_cas_value_strong<mode>):
+	Allow op3 to be zero.
+
+2025-07-08  Richard Biener  <rguenther@suse.de>
+
+	* config/i386/x86-tune.def (X86_TUNE_AVX512_MASKED_EPILOGUES):
+	New tunable, default on for m_ZNVER4 and m_ZNVER5.
+	* config/i386/i386.cc (ix86_vector_costs::finish_cost): With
+	X86_TUNE_AVX512_MASKED_EPILOGUES and when the main loop
+	had a vectorization factor > 2 use a masked epilogue when
+	possible and when not obviously problematic.
+
+2025-07-08  Richard Biener  <rguenther@suse.de>
+
+	* tree-vectorizer.h (vector_costs::suggested_epilogue_mode):
+	Add masked output parameter and return m_masked_epilogue.
+	(vector_costs::m_masked_epilogue): New tristate flag.
+	(vector_costs::vector_costs): Initialize m_masked_epilogue.
+	* tree-vect-loop.cc (vect_analyze_loop_1): Pass in masked
+	flag to optionally initialize can_use_partial_vectors_p.
+	(vect_analyze_loop): For epilogues also get whether to use
+	a masked epilogue for this loop from the target and use
+	that for the first epilogue mode we try.
+
+2025-07-08  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120358
+	* tree-ssa-structalias.cc (get_constraint_for_1): Adjust
+	pruning of sub-variables according to the imprecise
+	known start offset.
+
+2025-07-08  Alexandre Oliva  <oliva@adacore.com>
+
+	* config/vxworks-dummy.h (TARGET_VXWORKS_VAROFF): New.
+	(TARGET_VXWORKS_GOTTPIC): New.
+	* config/vxworks.h (TARGET_VXWORKS_VAROFF): Override.
+	(TARGET_VXWORKS_GOTTPIC): Likewise.
+	* config/i386/i386.cc (output_set_got): Disable VxWorks6 GOT
+	sequence on VxWorks7.
+	(legitimize_pic_address): Accept relative addressing of
+	labels on VxWorks7.
+	(ix86_delegitimize_address_1): Likewise.
+	(ix86_output_addr_diff_elt): Likewise.
+	* config/i386/i386.md (tablejump): Likewise.
+	(set_got, set_got_labelled): Set no-red-zone flag on VxWorks7.
+	* config/i386/predicates.md (gotoff_operand): Test
+	TARGET_VXWORKS_VAROFF.
+
+2025-07-08  Alexandre Oliva  <oliva@adacore.com>
+
+	* config.gcc (vxworks-dummy.h): Add to aarch64-*-* as well.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	* doc/extend.texi: Extend counted_by attribute to pointer fields in
+	structures. Add one more requirement to pointers with counted_by
+	attribute.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-01  Qing Zhao  <qing.zhao@oracle.com>
+
+	* tree-object-size.cc (access_with_size_object_size): Update comments
+	for pointers with .ACCESS_WITH_SIZE.
+	(collect_object_sizes_for): Propagate size info through GIMPLE_ASSIGN
+	for pointers with .ACCESS_WITH_SIZE.
+
+2025-07-07  Martin Jambor  <mjambor@suse.cz>
+
+	* value-range.h (class irange): Mark member function verify_range
+	with override.
+	(class prange): Mark member function verify_range with final override.
+	(class frange): Mark member function verify_range with override.
+
+2025-07-07  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120888
+	* config/xtensa/xtensa.cc (xtensa_promote_function_mode): New.
+	(TARGET_PROMOTE_FUNCTION_MODE): Use.
+	(TARGET_PROMOTE_PROTOTYPES): Removed.
+
+2025-07-07  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* config/s390/s390.md: Update UNSPECs
+	* config/s390/vector.md (fmax<mode>3): New expander.
+	(fmin<mode>3): New expander.
+	* config/s390/vx-builtins.md (*fmin<mode>): New insn.
+	(vfmin<mode>): Redefined to use new insn.
+	(*fmax<mode>): New insn.
+	(vfmax<mode>): Redefined to use new insn.
+
+2025-07-07  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120917
+	* doc/invoke.texi: Add -Wno-abbreviated-auto-in-template-arg.
+
+2025-07-07  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* config/aarch64/aarch64.md (popcountti2): Add TARGET_SVE path.
+
+2025-07-07  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120817
+	* tree-ssa-dse.cc (initialize_ao_ref_for_dse): Use
+	ao_ref_init_from_ptr_and_range with unknown size for
+	.MASK_STORE and .MASK_LEN_STORE.
+
+2025-07-07  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/riscv-protos.h (riscv_expand_usmul): Add new func
+	decl.
+	* config/riscv/riscv.cc (riscv_expand_xmode_usmul): Add new func
+	to expand Xmode SAT_MUL.
+	(riscv_expand_non_xmode_usmul): Ditto but for non-Xmode.
+	(riscv_expand_usmul): Add new func to implment SAT_MUL.
+	* config/riscv/riscv.md (usmul<mode>3): Add new pattern to match
+	standard name usmul.
+
+2025-07-07  Pan Li  <pan2.li@intel.com>
+
+	* match.pd: Add new match pattern for unsigned SAT_MUL.
+	* tree-ssa-math-opts.cc (gimple_unsigned_integer_sat_mul):
+	new decl for pattern match func.
+	(match_unsigned_saturation_mul): Add new func to match unsigned
+	SAT_MUL.
+	(math_opts_dom_walker::after_dom_children): Try to match
+	unsigned SAT_MUL on NOP.
+
+2025-07-07  Pan Li  <pan2.li@intel.com>
+
+	* internal-fn.cc (commutative_binary_fn_p): Add new case
+	for SAT_MUL.
+	* internal-fn.def (SAT_MUL): Add new IFN_SAT_MUL.
+	* optabs.def (OPTAB_NL): Remove fixed point limitation.
+
+2025-07-07  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* config/s390/s390.md: Removed unused unspecs.
+	* config/s390/vector.md (avg<mode>3_ceil): New expander.
+	(uavg<mode>3_ceil): New expander.
+	(smul<mode>3_highpart): New expander.
+	(umul<mode>3_highpart): New expander.
+	* config/s390/vx-builtins.md (vec_umulh<mode>): Remove unspec.
+	(vec_smulh<mode>): Remove unspec.
+
+2025-07-07  Spencer Abson  <spencer.abson@arm.com>
+
+	* config/aarch64/aarch64-sve.md (vec_cmp<mode><vpred>): Extend
+	to handle partial FP modes.
+	(@aarch64_pred_fcm<cmp_op><mode>): Likewise.
+	(@aarch64_pred_fcmuo<mode>): Likewise.
+	(*one_cmpl<mode>3): Rename to...
+	(@aarch64_pred_one_cmpl<mode>_z): ... this.
+	* config/aarch64/aarch64.cc (aarch64_emit_sve_fp_cond): Allow the
+	target and governing predicates to have different modes.
+	(aarch64_emit_sve_or_fp_conds): Likewise.
+	(aarch64_emit_sve_invert_fp_cond): Likewise.
+	(aarch64_expand_sve_vec_cmp_float): Likewise.
+
+2025-07-07  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR tree-optimization/118891
+	* tree-vect-stmts.cc (supportable_widening_operation): Swap the
+	hi and lo internal functions on big-endian targets.
+
+2025-07-07  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* ext-dce.cc (ext_dce_process_uses): Apply is_constant directly
+	to the subreg_lsb.
+
+2025-07-07  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* config/aarch64/aarch64-sve.md (@aarch64_sve_set_neonq_<mode>):
+	Use %Z instead of lowpart_subreg.  Tweak formatting.
+
+2025-07-07  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR target/118891
+	* config/aarch64/aarch64.cc (aarch64_expand_vector_init): Fix the
+	ZIP1 operand order for big-endian targets.
+
+2025-07-07  Jan Hubicka  <hubicka@ucw.cz>
+
+	* tree-ssa-live.cc (dump_scope_block): Print discriminators
+	of inlined functions.
+
+2025-07-07  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120670
+	PR target/120683
+	* config/i386/i386-expand.cc (expand_set_or_cpymem_via_loop):
+	Don't generate the loop if the loop count is 1.
+	(expand_cpymem_epilogue): Use move_by_pieces.
+	(setmem_epilogue_gen_val): New.
+	(expand_setmem_epilogue): Use store_by_pieces.
+	(expand_small_cpymem_or_setmem): Choose cpymem mode from MOVE_MAX.
+	For memset with vector and the size is smaller than the vector
+	size, first try the narrower vector, otherwise, use the scalar
+	value.
+	(promote_duplicated_reg): Duplicate the scalar value for vector.
+	(ix86_expand_set_or_cpymem): Always expand vector-version of
+	memset for vector_loop.  Use misaligned prologue if alignment
+	isn't needed and destination isn't aligned.  Always initialize
+	vec_promoted_val from the promoted scalar value for vector_loop.
+
+2025-07-07  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/120709
+	* builtins.cc (expand_builtin_crc_table_based): Error out
+	instead of asserting the 3rd argument is an integer constant.
+	* internal-fn.cc (expand_crc_optab_fn): Likewise.
+	* doc/extend.texi (crc): Document requirement of the poly argument
+	being a constant.
+
+2025-07-06  Georg-Johann Lay  <avr@gjlay.de>
+
+	* config/avr/avr-mcus.def: -mmcu= takes lower case MCU names.
+	* doc/avr-mmcu.texi: Rebuild.
+
+2025-07-06  Georg-Johann Lay  <avr@gjlay.de>
+
+	* config/avr/avr-mcus.def (avr32da28S, avr32da32S, avr32da48S)
+	(avr64da28S, avr64da32S, avr64da48S avr64da64S)
+	(avr128da28S, avr128da32S, avr128da48S, avr128da64S): Add devices.
+	* doc/avr-mmcu.texi: Rebuild.
+
+2025-07-06  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/120951
+	* tree-call-cdce.cc (use_internal_fn): For non-call exceptions
+	with EQ_EXPR can throw for floating point types, then create
+	the EQ_EXPR seperately.
+
+2025-07-06  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/120921
+	* tree-cfg.cc (verify_gimple_assign_single): Reject constant and address expression LHS.
+	For non-empty vector constructors, make sure the LHS is an is_gimple_reg.
+
+2025-07-06  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc
+	(autofdo_source_profile::read): Scale cutoff.
+	(read_autofdo_file): Initialize cutoff
+	* coverage.cc (read_counts_file): Initialize cutoff to 1.
+	* gcov-io.h (struct gcov_summary): Add cutoff field.
+	* ipa-inline.cc (inline_small_functions): mac_count can be non-zero
+	also with auto_profile.
+	* lto-cgraph.cc (output_profile_summary): Write cutoff
+	and sum_max.
+	(input_profile_summary): Read cutoff and sum max.
+	(merge_profile_summaries): Initialize and scale global cutoffs
+	and sum max.
+	* profile-count.cc: Include profile.h
+	(profile_count::force_nonzero): move here from ...; use cutoff.
+	* profile-count.h: (profile_count::force_nonzero): ... here.
+
+2025-07-06  Jan Hubicka  <hubicka@ucw.cz>
+
+	* profile-count.cc (profile_count::operator*): fix overflow check.
+
+2025-07-05  Alexandre Oliva  <oliva@adacore.com>
+
+	* config/rs6000/vxworks.h (SUBTARGET_DRIVER_SELF_SPECS):
+	Redefine to select word size matching TARGET_VXWORKS64.
+	(TARGET_VXWORKS64): Redefine in terms of TARGET_64BIT.
+
+2025-07-04  Vineet Gupta  <vineetg@rivosinc.com>
+
+	PR target/118241
+	* config/riscv/riscv.md (prefetch): Add alternative "r".
+
+2025-07-04  Vineet Gupta  <vineetg@rivosinc.com>
+
+	* config/riscv/predicates.md (prefetch_operand): mack 5 bits.
+
+2025-07-04  Raphael Moreira Zinsly  <rzinsly@ventanamicro.com>
+
+	* config/sh/predicates.md
+	(treg_set_expr_not_const01): call sh_recog_treg_set_expr_not_01
+	* config/sh/sh-protos.h
+	(sh_recog_treg_set_expr_not_01): New function
+	* config/sh/sh.cc (sh_recog_treg_set_expr_not_01): Likewise
+
+2025-07-04  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR c/118948
+	* fold-const.cc (tree_expr_nonnegative_warnv_p): Use
+	error_operand_p instead of checking for error_mark_node directly.
+
+2025-07-04  Pan Li  <pan2.li@intel.com>
+
+	* config/riscv/riscv-v.cc (expand_vx_binary_vec_dup_vec): Add
+	new case SS_PLUS.
+	(expand_vx_binary_vec_vec_dup): Ditto.
+	* config/riscv/riscv.cc (riscv_rtx_costs): Ditto.
+	* config/riscv/vector-iterators.md: Add new op ss_plus.
+
+2025-07-04  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120944
+	* tree-ssa-sccvn.cc (vn_reference_lookup_3): Gate optimizations
+	invalid when volatile is involved.
+
+2025-07-04  Jan Hubicka  <hubicka@ucw.cz>
+
+	* common.opt: Add period.
+	* common.opt.urls: Regenerate.
+
+2025-07-04  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120927
+	* tree-vect-data-refs.cc (vect_compute_data_ref_alignment):
+	Do not force a DRs base alignment when analyzing an
+	epilog loop.  Check whether the step preserves alignment
+	for all VFs possibly involved sofar.
+
+2025-07-04  Xi Ruoyao  <xry111@xry111.site>
+
+	* config/loongarch/loongarch.md (crc_combine): Avoid nested
+	subreg.
+
+2025-07-04  Shreya Munnangi  <smunnangi1@ventanamicro.com>
+
+	* config/riscv/riscv.cc (riscv_macro_fusion_pair_p): Add basic
+	instrumentation to all cases where fusion is detected.  Fix
+	minor formatting goofs found in the process.
+
+2025-07-04  panciyan  <panciyan@eswincomputing.com>
+
+	* match.pd: Add signed scalar SAT_ADD IMM form2 matching.
+
+2025-07-03  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* config/s390/s390.cc (expand_perm_with_merge): Add size change cases.
+	(expand_perm_with_pack): New function.
+	(vectorize_vec_perm_const_1): Wire up new function.
+
+2025-07-03  Co-authored-by: Daniel Barboza  <dbarboza@ventanamicro.com>
+	    Co-authored-by: Shreya Munnangi  <smunnangi1@ventanamicro.com>
+
+	PR target/118886
+	* config/riscv/riscv.cc (riscv_macro_fusion_pair_p): Check
+	for fusion being disabled earlier.  If PREV is already fused,
+	then it can't be fused again.  Be more selective about fusing
+	when the destination registers do not match.  Don't fuse into
+	loads that aren't scalar integer modes.  Revamp store pair
+	commit support.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64.md (aarch64_cbz<optab><mode>1): Move
+	above rules for CBB<cond>/CBH<cond>/CB<cond>.
+	(*aarch64_tbz<optab><mode>1): Likewise.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64-protos.h (aarch64_cb_rhs): New function.
+	* config/aarch64/aarch64.cc (aarch64_cb_rhs): Likewise.
+	* config/aarch64/aarch64.md (cbranch<mode>4): Rename to ...
+	(cbranch<GPI:mode>4): ...here, and emit CMPBR if possible.
+	(cbranch<SHORT:mode>4): New expand rule.
+	(aarch64_cb<INT_CMP:code><GPI:mode>): New insn rule.
+	(aarch64_cb<INT_CMP:code><SHORT:mode>): Likewise.
+	* config/aarch64/constraints.md (Uc0): New constraint.
+	(Uc1): Likewise.
+	(Uc2): Likewise.
+	* config/aarch64/iterators.md (cmpbr_suffix): New mode attr.
+	(INT_CMP): New code iterator.
+	(cmpbr_imm_constraint): New code attr.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64-option-extensions.def (cmpbr): New
+	option.
+	* config/aarch64/aarch64.h (TARGET_CMPBR): New macro.
+	* doc/invoke.texi (cmpbr): New option.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64.md (far_branch): Replace 0/1 with
+	no/yes.
+	(aarch64_bcond): Handle rename.
+	(aarch64_cbz<optab><mode>1): Likewise.
+	(*aarch64_tbz<optab><mode>1): Likewise.
+	(@aarch64_tbz<optab><ALLI:mode><GPI:mode>): Likewise.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64.md (BRANCH_LEN_P_1MiB): New constant.
+	(BRANCH_LEN_N_1MiB): Likewise.
+	(BRANCH_LEN_P_32KiB): Likewise.
+	(BRANCH_LEN_N_32KiB): Likewise.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64.md (condjump): Rename to ...
+	(aarch64_bcond): ...here.
+	(*compare_condjump<GPI:mode>): Rename to ...
+	(*aarch64_bcond_wide_imm<GPI:mode>): ...here.
+	(aarch64_cb<optab><mode>): Rename to ...
+	(aarch64_cbz<optab><mode>1): ...here.
+	(*cb<optab><mode>1): Rename to ...
+	(*aarch64_tbz<optab><mode>1): ...here.
+	(@aarch64_tb<optab><ALLI:mode><GPI:mode>): Rename to ...
+	(@aarch64_tbz<optab><ALLI:mode><GPI:mode>): ...here.
+	(restore_stack_nonlocal): Handle rename.
+	(stack_protect_combined_test): Likewise.
+	* config/aarch64/aarch64-simd.md (cbranch<mode>4): Likewise.
+	* config/aarch64/aarch64-sme.md (aarch64_restore_za): Likewise.
+	* config/aarch64/aarch64.cc (aarch64_gen_test_and_branch): Likewise.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64.md (cbranch<mode>4): Reformat.
+	(cbranchcc4): Likewise.
+	(condjump): Likewise.
+	(*compare_condjump<GPI:mode>): Likewise.
+	(aarch64_cb<optab><mode>1): Likewise.
+	(*cb<optab><mode>1): Likewise.
+	(tbranch_<code><mode>3): Likewise.
+	(@aarch64_tb<optab><ALLI:mode><GPI:mode>): Likewise.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* config/aarch64/aarch64.md (condjump): Move.
+	(*compare_condjump<GPI:mode>): Likewise.
+	(aarch64_cb<optab><mode>1): Likewise.
+	(*cb<optab><mode>1): Likewise.
+	(tbranch_<code><mode>3): Likewise.
+	(@aarch64_tb<optab><ALLI:mode><GPI:mode>): Likewise.
+
+2025-07-03  Siddhesh Poyarekar  <siddhesh@gotplt.org>
+
+	PR tree-optimization/120780
+	* tree-object-size.cc (inner_at_offset,
+	get_wholesize_for_memref): New functions.
+	(addr_object_size): Call get_wholesize_for_memref.
+
+2025-07-03  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120936
+	* config/i386/i386.cc (x86_print_call_or_nop): Add a label
+	argument and use it to print label.
+	(x86_function_profiler): Emit label only when __mcount_loc
+	section is used.
+
+2025-07-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc (get_combined_location): Handle negative
+	offsets; output better diagnostics.
+	(get_relative_location_for_locus): Reutrn -1 for unknown location.
+	(function_instance::get_cgraph_node): New member function.
+	(match_with_target): New function.
+	(dump_stmt): New function.
+	(function_instance::lookup_count): New function.
+	(mark_expr_locations): New function.
+	(function_instance::match): New function.
+	(autofdo_source_profile::offline_external_functions): Do
+	not repeat renaming; manage two worklists and do matching.
+	(autofdo_source_profile::offline_unrealized_inlines): Simplify.
+	(afdo_set_bb_count): do not look for lost discriminators.
+	(auto_profile): Do not ICE when profile reading failed.
+	* common.opt (Wauto-profile): New warning flag
+	* doc/invoke.texi (-Wauto-profile): Document.
+
+2025-07-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-fnsummary.cc (analyze_function_body): For loop
+	heuristics use header count instead of preheader count.
+
+2025-07-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-cp.cc (update_profiling_info): Watch for division by zero.
+
+2025-07-03  Alex Coplan  <alex.coplan@arm.com>
+
+	* config/aarch64/aarch64-sve.md
+	(vec_load_lanes<mode><vsingle>): Expand else operand in
+	subvector mode, as per optab documentation.
+	(vec_mask_load_lanes<mode><vsingle>): Add missing mode for
+	operand 3.
+	* config/aarch64/predicates.md (aarch64_maskload_else_operand):
+	Remove const_int.
+
+2025-07-03  Alex Coplan  <alex.coplan@arm.com>
+
+	* doc/md.texi (Standard Names): Clarify mode of else operand for
+	vec_mask_load_lanesmn optab.
+
+2025-07-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-cp.cc (cs_interesting_for_ipcp_p): Handle
+	correctly GLOBAL0 afdo counts.
+	(ipcp_cloning_candidate_p): Do not rule out nodes
+	!node->optimize_for_size_p ().
+	(good_cloning_opportunity_p): Handle afdo counts
+	as non-zero.
+
+2025-07-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* ipa-cp.cc (hint_time_bonus): Return sreal and avoid
+	conversions to integer.
+	(good_cloning_opportunity_p): Avoid sreal to integer
+	conversions
+	(perform_estimation_of_a_value): Update.
+
+2025-07-03  Jan Hubicka  <hubicka@ucw.cz>
+
+	* auto-profile.cc (afdo_hot_bb_threshod): New global
+	variable.
+	(maybe_hot_afdo_count_p): New function.
+	(autofdo_source_profile::read): Do not set up dump file;
+	set afdo_hot_bb_threshod.
+	(afdo_annotate_cfg): Handle partial training.
+	(afdo_callsite_hot_enough_for_early_inline):
+	Use maybe_hot_afdo_count_p.
+	(auto_profile_offline::execute): Read autofdo file.
+	* auto-profile.h (maybe_hot_afdo_count_p): Declare.
+	(afdo_hot_bb_threshold): Declare.
+	* coverage.cc (read_counts_file): Also set gcov_profile_info.
+	(coverage_init): Do not read autofdo file.
+	* opts.cc (enable_fdo_optimizations): Add autofdo parameter;
+	do not set flag_branch_probabilities and flag_profile_values
+	with it.
+	(common_handle_option): Update.
+	* passes.cc (finish_optimization_passes): Do not end branch
+	prob here.
+	(pass_manager::dump_profile_report): Also mark change after
+	autofdo pass.
+	* profile.cc: Include auto-profile.h
+	(gcov_profile_info): New global variable.
+	(struct afdo_fdo_record): New struture.
+	(compute_branch_probabilities): Record afdo profile.
+	(end_branch_prob): Dump afdo/fdo profile comparsion.
+	* profile.h (gcov_profile_info): Declarre.
+	* tree-profile.cc (tree_profiling): Call end_branch_prob
+	(pass_ipa_tree_profile::gate): Also enable with autoFDO
+
+2025-07-03  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/118669
+	* tree-vect-stmts.cc (vectorizable_load): Emit loads
+	with proper (element) alignment.
+	(vectorizable_store): Likewise.
+
+2025-07-03  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120908
+	* config/i386/i386.cc (legitimize_tls_address): Pass RDI to
+	gen_tls_local_dynamic_64.
+	* config/i386/i386.md (*tls_global_dynamic_64_largepic): Add
+	RDI clobber and use it to generate LEA.
+	(*tls_local_dynamic_64_<mode>): Likewise.
+	(*tls_local_dynamic_base_64_largepic): Likewise.
+	(@tls_local_dynamic_64_<mode>): Add a clobber.
+
 2025-07-02  H.J. Lu  <hjl.tools@gmail.com>
 
 	PR target/120908
diff --git a/gcc/DATESTAMP b/gcc/DATESTAMP
index 6952979..963d4c7 100644
--- a/gcc/DATESTAMP
+++ b/gcc/DATESTAMP
@@ -1 +1 @@
-20250703
+20250803
diff --git a/gcc/Makefile.in b/gcc/Makefile.in
index d5ceccc..d7d5cbe 100644
--- a/gcc/Makefile.in
+++ b/gcc/Makefile.in
@@ -1069,7 +1069,7 @@ TREE_SSA_H = tree-ssa.h tree-ssa-operands.h \
 PRETTY_PRINT_H = pretty-print.h $(INPUT_H) $(OBSTACK_H) wide-int-print.h
 TREE_PRETTY_PRINT_H = tree-pretty-print.h $(PRETTY_PRINT_H)
 GIMPLE_PRETTY_PRINT_H = gimple-pretty-print.h $(TREE_PRETTY_PRINT_H)
-DIAGNOSTIC_CORE_H = diagnostic-core.h $(INPUT_H) bversion.h diagnostic.def
+DIAGNOSTIC_CORE_H = diagnostic-core.h $(INPUT_H) bversion.h diagnostics/kinds.def
 DIAGNOSTIC_H = diagnostic.h $(DIAGNOSTIC_CORE_H) $(PRETTY_PRINT_H)
 C_PRETTY_PRINT_H = c-family/c-pretty-print.h $(PRETTY_PRINT_H) \
 	$(C_COMMON_H) $(TREE_H)
@@ -1595,7 +1595,6 @@ OBJS = \
 	jump.o \
 	langhooks.o \
 	late-combine.o \
-	lazy-diagnostic-path.o \
 	lcm.o \
 	lists.o \
 	loop-doloop.o \
@@ -1796,6 +1795,7 @@ OBJS = \
 	tree-ssa-sink.o \
 	tree-ssa-strlen.o \
 	tree-ssa-structalias.o \
+	pta-andersen.o \
 	tree-ssa-tail-merge.o \
 	tree-ssa-ter.o \
 	tree-ssa-threadbackward.o \
@@ -1851,28 +1851,39 @@ OBJS = \
 
 # Objects in libcommon.a, potentially used by all host binaries and with
 # no target dependencies.
-OBJS-libcommon = diagnostic-spec.o diagnostic.o diagnostic-color.o \
-	diagnostic-format-html.o \
-	diagnostic-format-sarif.o \
-	diagnostic-format-text.o \
+OBJS-libcommon = \
 	diagnostic-global-context.o \
-	diagnostic-macro-unwinding.o \
-	diagnostic-output-spec.o \
-	diagnostic-path.o \
-	diagnostic-path-output.o \
-	diagnostic-show-locus.o \
-	diagnostic-state-to-dot.o \
-	edit-context.o \
+	diagnostics/buffering.o \
+	diagnostics/changes.o \
+	diagnostics/color.o \
+	diagnostics/context.o \
+	diagnostics/digraphs.o \
+	diagnostics/file-cache.o \
+	diagnostics/output-spec.o \
+	diagnostics/html-sink.o \
+	diagnostics/sarif-sink.o \
+	diagnostics/text-sink.o \
+	diagnostics/lazy-paths.o \
+	diagnostics/macro-unwinding.o \
+	diagnostics/option-classifier.o \
+	diagnostics/paths.o \
+	diagnostics/paths-output.o \
+	diagnostics/source-printing.o \
+	diagnostics/state-graphs.o \
+	diagnostics/state-graphs-to-dot.o \
+	diagnostics/selftest-context.o \
+	diagnostics/selftest-logical-locations.o \
+	diagnostics/selftest-paths.o \
+	diagnostics/diagnostics-selftests.o \
+	gcc-diagnostic-spec.o \
 	graphviz.o pex.o \
 	pretty-print.o intl.o \
 	json.o json-parsing.o \
 	xml.o \
 	sbitmap.o \
 	vec.o input.o hash-table.o ggc-none.o memory-block.o \
-	selftest.o selftest-diagnostic.o sort.o \
-	selftest-diagnostic-path.o \
+	selftest.o sort.o \
 	selftest-json.o \
-	selftest-logical-location.o \
 	text-art/box-drawing.o \
 	text-art/canvas.o \
 	text-art/ruler.o \
@@ -1919,7 +1930,7 @@ endif
 
 # libbackend.a objs that might not be in some cases linked into the compiler,
 # yet they are supposed to be part of the plugin ABI.  See PR116143.
-EXTRA_BACKEND_OBJS = simple-diagnostic-path.o lazy-diagnostic-path.o
+EXTRA_BACKEND_OBJS = simple-diagnostic-path.o diagnostics/lazy-paths.o
 
 BACKEND = $(EXTRA_BACKEND_OBJS) libbackend.a main.o libcommon-target.a \
 	libcommon.a $(CPPLIB) $(LIBDECNUMBER)
@@ -3036,7 +3047,7 @@ GTFILES = $(CPPLIB_H) $(srcdir)/input.h $(srcdir)/coretypes.h \
   $(srcdir)/ipa-modref-tree.h \
   $(srcdir)/ipa-locality-cloning.cc \
   $(srcdir)/signop.h \
-  $(srcdir)/diagnostic-spec.h $(srcdir)/diagnostic-spec.cc \
+  $(srcdir)/gcc-diagnostic-spec.h $(srcdir)/gcc-diagnostic-spec.cc \
   $(srcdir)/dwarf2out.h \
   $(srcdir)/dwarf2asm.cc \
   $(srcdir)/dwarf2cfi.cc \
@@ -3709,7 +3720,7 @@ TEXI_GCC_FILES = gcc.texi gcc-common.texi gcc-vers.texi frontends.texi	\
 	 contribute.texi compat.texi funding.texi gnu.texi gpl_v3.texi	\
 	 fdl.texi contrib.texi cppenv.texi cppopts.texi avr-mmcu.texi	\
 	 implement-c.texi implement-cxx.texi gcov-tool.texi gcov-dump.texi \
-	 lto-dump.texi riscv-ext.texi
+	 lto-dump.texi riscv-ext.texi riscv-mcpu.texi riscv-mtune.texi
 
 # we explicitly use $(srcdir)/doc/tm.texi here to avoid confusion with
 # the generated tm.texi; the latter might have a more recent timestamp,
@@ -4069,7 +4080,7 @@ PLUGIN_HEADERS = $(TREE_H) $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) \
   fold-const.h fold-const-call.h tree-cfg.h tree-into-ssa.h tree-ssanames.h \
   print-tree.h varasm.h context.h tree-phinodes.h stor-layout.h \
   ssa-iterators.h $(RESOURCE_H) tree-cfgcleanup.h attribs.h calls.h \
-  cfgexpand.h diagnostic-color.h gcc-symtab.h gimple-builder.h gimple-low.h \
+  cfgexpand.h gcc-symtab.h gimple-builder.h gimple-low.h \
   gimple-walk.h gimplify-me.h pass_manager.h print-rtl.h stmt.h \
   tree-dfa.h tree-hasher.h tree-nested.h tree-object-size.h tree-outof-ssa.h \
   tree-parloops.h tree-ssa-address.h tree-ssa-coalesce.h tree-ssa-dom.h \
@@ -4079,7 +4090,7 @@ PLUGIN_HEADERS = $(TREE_H) $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) \
   hash-set.h dominance.h cfg.h cfgrtl.h cfganal.h cfgbuild.h cfgcleanup.h \
   lcm.h cfgloopmanip.h file-prefix-map.h builtins.def $(INSN_ATTR_H) \
   pass-instances.def params.list $(srcdir)/../include/gomp-constants.h \
-  $(EXPR_H) $(srcdir)/analyzer/*.h
+  $(EXPR_H) $(srcdir)/analyzer/*.h $(srcdir)/diagnostics/*.h
 
 # generate the 'build fragment' b-header-vars
 s-header-vars: Makefile
@@ -4117,6 +4128,7 @@ install-plugin: installdirs lang.install-plugin s-header-vars install-gengtype
 	  fi; \
 	  case $$path in \
 	  "$(srcdir)"/analyzer/* \
+	  | "$(srcdir)"/diagnostics/* \
 	  | "$(srcdir)"/config/* | "$(srcdir)"/common/config/* \
 	  | "$(srcdir)"/c-family/* | "$(srcdir)"/*.def ) \
 	    base=`echo "$$path" | sed -e "s|$$srcdirstrip/||"`;; \
diff --git a/gcc/ada/ChangeLog b/gcc/ada/ChangeLog
index 8aaa006..85464e3 100644
--- a/gcc/ada/ChangeLog
+++ b/gcc/ada/ChangeLog
@@ -1,3 +1,1020 @@
+2025-07-31  Eric Botcazou  <ebotcazou@gcc.gnu.org>
+
+	Revert:
+	2025-07-31  Nicolas Boulenguez  <nicolas@debian.org>
+
+	PR ada/114065
+	* Makefile.rtl (GNATRTL_NONTASKING_OBJS): Add g-c_time$(objext) and
+	s-c_time$(objext).
+	(Aarch64/Android): Do not use s-osinte__android.adb.
+	(SPARC/Solaris): Do not use s-osprim__solaris.adb.
+	(x86/Solaris): Likewise.
+	(LynxOS178): Do not use s-parame__posix2008.ads.
+	(RTEMS): Likewise.
+	(x32/Linux): Likewise, as well as s-linux__x32.ads.  Replace
+	s-osprim__x32.adb with s-osprim__posix.adb.
+	(LIBGNAT_OBJS): Remove cal.o.
+	* cal.c: Delete.
+	* doc/gnat_rm/the_gnat_library.rst (GNAT.C_Time): New entry.
+	(GNAT.Calendar): Do not mention the obsolete conversion functions.
+	* impunit.adb (Non_Imp_File_Names_95): Add g-c_time.
+	* libgnarl/a-exetim__posix.adb: Add with clause for System.C_Time
+	(Clock): Use type and functions from System.C_Time.
+	* libgnarl/s-linux.ads: Remove with clause for System.Parameters.
+	Remove declarations of C time types.
+	* libgnarl/s-linux__alpha.ads: Likewise.
+	* libgnarl/s-linux__android-aarch64.ads: Likewise.
+	* libgnarl/s-linux__android-arm.ads: Likewise.
+	* libgnarl/s-linux__hppa.ads: Likewise.
+	* libgnarl/s-linux__loongarch.ads: Likewise.
+	* libgnarl/s-linux__mips.ads: Likewise.
+	* libgnarl/s-linux__riscv.ads: Likewise.
+	* libgnarl/s-linux__sparc.ads: Likewise.
+	* libgnarl/s-osinte__aix.ads: Likewise.
+	* libgnarl/s-osinte__android.ads: Likewise.
+	* libgnarl/s-osinte__cheribsd.ads: Likewise.
+	* libgnarl/s-osinte__darwin.ads: Likewise.
+	* libgnarl/s-osinte__dragonfly.ads: Likewise.
+	* libgnarl/s-osinte__freebsd.ads: Likewise.
+	* libgnarl/s-osinte__gnu.ads: Likewise.
+	* libgnarl/s-osinte__hpux.ads: Likewise.
+	* libgnarl/s-osinte__kfreebsd-gnu.ads: Likewise.
+	* libgnarl/s-osinte__linux.ads: Likewise.
+	* libgnarl/s-osinte__lynxos178e.ads: Likewise.
+	* libgnarl/s-osinte__qnx.ads: Likewise.
+	* libgnarl/s-osinte__rtems.ads: Likewise.
+	* libgnarl/s-osinte__solaris.ads: Likewise.
+	* libgnarl/s-osinte__vxworks.ads: Likewise.
+	* libgnarl/s-qnx.ads: Likewise.
+	* libgnarl/s-linux__x32.ads: Delete.
+	* libgnarl/s-osinte__darwin.adb (To_Duration): Remove.
+	(To_Timespec): Likewise.
+	* libgnarl/s-osinte__aix.adb: Likewise.
+	* libgnarl/s-osinte__dragonfly.adb: Likewise.
+	* libgnarl/s-osinte__freebsd.adb: Likewise.
+	* libgnarl/s-osinte__gnu.adb: Likewise.
+	* libgnarl/s-osinte__lynxos178.adb: Likewise.
+	* libgnarl/s-osinte__posix.adb: Likewise.
+	* libgnarl/s-osinte__qnx.adb: Likewise.
+	* libgnarl/s-osinte__rtems.adb: Likewise.
+	* libgnarl/s-osinte__solaris.adb: Likewise.
+	* libgnarl/s-osinte__vxworks.adb: Likewise.
+	* libgnarl/s-osinte__x32.adb: Likewise.
+	* libgnarl/s-taprop__solaris.adb: Add with clause for System.C_Time.
+	(Monotonic_Clock): Use type and functions from System.C_Time.
+	(RT_Resolution): Likewise.
+	(Timed_Sleep): Likewise.
+	(Timed_Delay): Likewise.
+	* libgnarl/s-taprop__vxworks.adb: Likewise.
+	* libgnarl/s-tpopmo.adb: Likewise.
+	* libgnarl/s-osinte__android.adb: Delete.
+	* libgnat/g-c_time.ads: New file.
+	* libgnat/g-calend.adb: Delegate to System.C_Time.
+	* libgnat/g-calend.ads: Likewise.
+	* libgnat/g-socket.adb: Likewise.
+	* libgnat/g-socthi.adb: Likewise.
+	* libgnat/g-socthi__vxworks.adb: Likewise.
+	* libgnat/g-sothco.ads: Likewise.
+	* libgnat/g-spogwa.adb: Likewise.
+	* libgnat/s-c_time.adb: New file.
+	* libgnat/s-c_time.ads: Likewise.
+	* libgnat/s-optide.adb: Import nanosleep here.
+	* libgnat/s-os_lib.ads (time_t): Remove.
+	(To_Ada): Adjust.
+	(To_C): Likewise.
+	* libgnat/s-os_lib.adb: Likewise.
+	* libgnat/s-osprim__darwin.adb: Delegate to System.C_Time.
+	* libgnat/s-osprim__posix.adb: Likewise.
+	* libgnat/s-osprim__posix2008.adb: Likewise.
+	* libgnat/s-osprim__rtems.adb: Likewise.
+	* libgnat/s-osprim__unix.adb: Likewise.
+	* libgnat/s-osprim__solaris.adb: Delete.
+	* libgnat/s-osprim__x32.adb: Likewise.
+	* libgnat/s-parame.ads (time_t_bits): Remove.
+	* libgnat/s-parame__hpux.ads: Likewise.
+	* libgnat/s-parame__vxworks.ads: Likewise.
+	* libgnat/s-parame__posix2008.ads: Delete.
+	* s-oscons-tmplt.c (SIZEOF_tv_nsec): New constant.
+
+2025-07-31  Eric Botcazou  <ebotcazou@gcc.gnu.org>
+
+	PR ada/120440
+	* gcc-interface/Makefile.in (GNATLINK_OBJS): Add s-excmac.o.
+	(GNATMAKE_OBJS): Likewise.
+
+2025-07-31  Nicolas Boulenguez  <nicolas@debian.org>
+
+	PR ada/114065
+	* Makefile.rtl (GNATRTL_NONTASKING_OBJS): Add g-c_time$(objext) and
+	s-c_time$(objext).
+	(Aarch64/Android): Do not use s-osinte__android.adb.
+	(SPARC/Solaris): Do not use s-osprim__solaris.adb.
+	(x86/Solaris): Likewise.
+	(LynxOS178): Do not use s-parame__posix2008.ads.
+	(RTEMS): Likewise.
+	(x32/Linux): Likewise, as well as s-linux__x32.ads.  Replace
+	s-osprim__x32.adb with s-osprim__posix.adb.
+	(LIBGNAT_OBJS): Remove cal.o.
+	* cal.c: Delete.
+	* doc/gnat_rm/the_gnat_library.rst (GNAT.C_Time): New entry.
+	(GNAT.Calendar): Do not mention the obsolete conversion functions.
+	* impunit.adb (Non_Imp_File_Names_95): Add g-c_time.
+	* libgnarl/a-exetim__posix.adb: Add with clause for System.C_Time
+	(Clock): Use type and functions from System.C_Time.
+	* libgnarl/s-linux.ads: Remove with clause for System.Parameters.
+	Remove declarations of C time types.
+	* libgnarl/s-linux__alpha.ads: Likewise.
+	* libgnarl/s-linux__android-aarch64.ads: Likewise.
+	* libgnarl/s-linux__android-arm.ads: Likewise.
+	* libgnarl/s-linux__hppa.ads: Likewise.
+	* libgnarl/s-linux__loongarch.ads: Likewise.
+	* libgnarl/s-linux__mips.ads: Likewise.
+	* libgnarl/s-linux__riscv.ads: Likewise.
+	* libgnarl/s-linux__sparc.ads: Likewise.
+	* libgnarl/s-osinte__aix.ads: Likewise.
+	* libgnarl/s-osinte__android.ads: Likewise.
+	* libgnarl/s-osinte__cheribsd.ads: Likewise.
+	* libgnarl/s-osinte__darwin.ads: Likewise.
+	* libgnarl/s-osinte__dragonfly.ads: Likewise.
+	* libgnarl/s-osinte__freebsd.ads: Likewise.
+	* libgnarl/s-osinte__gnu.ads: Likewise.
+	* libgnarl/s-osinte__hpux.ads: Likewise.
+	* libgnarl/s-osinte__kfreebsd-gnu.ads: Likewise.
+	* libgnarl/s-osinte__linux.ads: Likewise.
+	* libgnarl/s-osinte__lynxos178e.ads: Likewise.
+	* libgnarl/s-osinte__qnx.ads: Likewise.
+	* libgnarl/s-osinte__rtems.ads: Likewise.
+	* libgnarl/s-osinte__solaris.ads: Likewise.
+	* libgnarl/s-osinte__vxworks.ads: Likewise.
+	* libgnarl/s-qnx.ads: Likewise.
+	* libgnarl/s-linux__x32.ads: Delete.
+	* libgnarl/s-osinte__darwin.adb (To_Duration): Remove.
+	(To_Timespec): Likewise.
+	* libgnarl/s-osinte__aix.adb: Likewise.
+	* libgnarl/s-osinte__dragonfly.adb: Likewise.
+	* libgnarl/s-osinte__freebsd.adb: Likewise.
+	* libgnarl/s-osinte__gnu.adb: Likewise.
+	* libgnarl/s-osinte__lynxos178.adb: Likewise.
+	* libgnarl/s-osinte__posix.adb: Likewise.
+	* libgnarl/s-osinte__qnx.adb: Likewise.
+	* libgnarl/s-osinte__rtems.adb: Likewise.
+	* libgnarl/s-osinte__solaris.adb: Likewise.
+	* libgnarl/s-osinte__vxworks.adb: Likewise.
+	* libgnarl/s-osinte__x32.adb: Likewise.
+	* libgnarl/s-taprop__solaris.adb: Add with clause for System.C_Time.
+	(Monotonic_Clock): Use type and functions from System.C_Time.
+	(RT_Resolution): Likewise.
+	(Timed_Sleep): Likewise.
+	(Timed_Delay): Likewise.
+	* libgnarl/s-taprop__vxworks.adb: Likewise.
+	* libgnarl/s-tpopmo.adb: Likewise.
+	* libgnarl/s-osinte__android.adb: Delete.
+	* libgnat/g-c_time.ads: New file.
+	* libgnat/g-calend.adb: Delegate to System.C_Time.
+	* libgnat/g-calend.ads: Likewise.
+	* libgnat/g-socket.adb: Likewise.
+	* libgnat/g-socthi.adb: Likewise.
+	* libgnat/g-socthi__vxworks.adb: Likewise.
+	* libgnat/g-sothco.ads: Likewise.
+	* libgnat/g-spogwa.adb: Likewise.
+	* libgnat/s-c_time.adb: New file.
+	* libgnat/s-c_time.ads: Likewise.
+	* libgnat/s-optide.adb: Import nanosleep here.
+	* libgnat/s-os_lib.ads (time_t): Remove.
+	(To_Ada): Adjust.
+	(To_C): Likewise.
+	* libgnat/s-os_lib.adb: Likewise.
+	* libgnat/s-osprim__darwin.adb: Delegate to System.C_Time.
+	* libgnat/s-osprim__posix.adb: Likewise.
+	* libgnat/s-osprim__posix2008.adb: Likewise.
+	* libgnat/s-osprim__rtems.adb: Likewise.
+	* libgnat/s-osprim__unix.adb: Likewise.
+	* libgnat/s-osprim__solaris.adb: Delete.
+	* libgnat/s-osprim__x32.adb: Likewise.
+	* libgnat/s-parame.ads (time_t_bits): Remove.
+	* libgnat/s-parame__hpux.ads: Likewise.
+	* libgnat/s-parame__vxworks.ads: Likewise.
+	* libgnat/s-parame__posix2008.ads: Delete.
+	* s-oscons-tmplt.c (SIZEOF_tv_nsec): New constant.
+
+2025-07-28  Marc Poulhiès  <poulhies@adacore.com>
+
+	* gcc-interface/trans.cc (gnat_to_gnu): Fix typo in comment.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc-interface/misc.cc: Make
+	diagnostics::context::m_source_printing private.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc-interface/trans.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc-interface/misc.cc: Update for diagnostic_context becoming
+	diagnostics::context.
+
+2025-07-25  Marc Poulhiès  <poulhies@adacore.com>
+
+	* gcc-interface/utils.cc (update_pointer_to): Renamed ptr/old_ptr, ref/old_ref.
+
+2025-07-25  Alexandre Oliva  <oliva@adacore.com>
+
+	* s-oscons-tmplt.c (CLOCK_RT_Ada) [__vxworks]: Define to
+	CLOCK_REALTIME on VxWorks6.
+	* gsocket.h [__vxworks]: Include strings.h if available.
+	* sysdep.c [__vxworks]: Likewise.
+
+2025-07-25  Steve Baird  <baird@adacore.com>
+
+	* exp_ch6.adb (Apply_Access_Discrims_Accessibility_Check): Do
+	nothing and simply return if either Ada_Version <= Ada_95 or if
+	the function being returned from lacks the extra formal parameter
+	needed to perform the check (typically because the result is
+	tagged).
+
+2025-07-25  Bob Duff  <duff@adacore.com>
+
+	* sem_ch12.adb (Check_Formal_Package_Instance):
+	Do nothing in case of E_Subprogram_Body.
+
+2025-07-25  Ronan Desplanques  <desplanques@adacore.com>
+
+	* sem_ch3.adb (Process_Discriminants): Update comments
+
+2025-07-25  Ronan Desplanques  <desplanques@adacore.com>
+
+	* sem_ch3.adb (Build_Derived_Record_Type): Set flag appropriately.
+
+2025-07-25  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* libgnat/a-calend.adb (To_Struct_Timespec_64): Deal with negative
+	Duration values and truncate the nanoseconds too.
+	* libgnat/g-calend.adb (timeval_to_duration): Unsuppress overflow
+	checks.
+	(duration_to_timeval): Likewise.  Deal with negative Duration values
+	as well as integral Duration values.
+	* libgnat/g-socket.adb (To_Timeval): Simplify the implementation.
+
+2025-07-24  Steve Baird  <baird@adacore.com>
+
+	* exp_aggr.adb (Convert_To_Assignments): Add calls to Ensure_Defined
+	before generating assignments to components that could be
+	associated with a not-yet-defined itype.
+
+2025-07-24  Steve Baird  <baird@adacore.com>
+
+	* accessibility.adb (Function_Call_Or_Allocator_Level): Handle the
+	case where a function that has an Extra_Accessibility_Of_Result
+	parameter returns as its result a call to another such function.
+	In that case, the extra parameter should be passed along.
+	(Check_Return_Construct_Accessibility): Replace a warning about an
+	inevitable failure of a dynamic check with a legality-rule-violation
+	error message; adjust the text of the message accordingly.
+	* exp_ch6.ads (Apply_Access_Discrims_Accessibility_Check): New
+	procedure, following example of the existing
+	Apply_CW_Accessibility procedure.
+	* exp_ch6.adb (Apply_Access_Discrims_Accessibility_Check): body
+	for new procedure.
+	(Expand_Simple_Function_Return): Add call to new
+	Apply_Access_Discrims_Accessibility_Check procedure.
+	* exp_ch3.adb (Make_Allocator_For_Return): Add call to new
+	Apply_Access_Discrims_Accessibility_Check procedure.
+
+2025-07-24  Tonu Naks  <naks@adacore.com>
+
+	* doc/gnat_rm/standard_and_implementation_defined_restrictions.rst:
+	clarify parameter description.
+	* gnat_rm.texi: Regenerate.
+
+2025-07-22  Gary Dismukes  <dismukes@adacore.com>
+
+	* sem_ch8.adb (End_Use_Type): Add a test for there not being an earlier
+	use_type_clause for the same type as an additional criterion for turning
+	off In_Use and Current_Use_Clause.
+
+2025-07-22  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_attr.adb (Eval_Attribute): Only fold array attributes when prefix
+	is static or at least safe to evaluate
+
+2025-07-22  Ronan Desplanques  <desplanques@adacore.com>
+
+	* einfo.ads (Is_Controlled_Active): Fix pasto in comment.
+	* sem_util.ads (Propagate_Controlled_Flags): Update comment for
+	Destructor aspect.
+
+2025-07-22  Ronan Desplanques  <desplanques@adacore.com>
+
+	* doc/gnat_rm/gnat_language_extensions.rst: Document new extension.
+	* snames.ads-tmpl: Add name for new aspect.
+	* gen_il-fields.ads (Has_Destructor, Is_Destructor): Add new fields.
+	* gen_il-gen-gen_entities.adb (E_Procedure, Type_Kind): Add new fields.
+	* einfo.ads (Has_Destructor, Is_Destructor): Document new fields.
+	* aspects.ads: Add new aspect.
+	* sem_ch13.adb (Analyze_Aspect_Specifications,
+	Check_Aspect_At_Freeze_Point, Check_Aspect_At_End_Of_Declarations):
+	Add semantic analysis for new aspect.
+	(Resolve_Finalization_Procedure): New function.
+	(Resolve_Finalizable_Argument): Use new function above.
+	* sem_util.adb (Propagate_Controlled_Flags): Extend for new field.
+	* freeze.adb (Freeze_Entity): Add legality check for new aspect.
+	* exp_ch3.adb (Expand_Freeze_Record_Type, Predefined_Primitive_Bodies):
+	Use new field.
+	* exp_ch7.adb (Build_Finalize_Statements): Add expansion for
+	destructors.
+	(Make_Final_Call, Build_Record_Deep_Procs): Adapt to new Has_Destructor
+	field.
+	(Build_Adjust_Statements): Tweak to handle cases of empty lists.
+	* gnat_rm.texi: Regenerate.
+
+2025-07-22  Denis Mazzucato  <mazzucato@adacore.com>
+
+	* sem_ch6.adb (Might_Need_BIP_Task_Actuals): Before retrieving the original corresponding
+	operation we retrieve first the root of the aliased chain.
+
+2025-07-22  Ronan Desplanques  <desplanques@adacore.com>
+
+	* gen_il-fields.ads (Is_Implicit_Full_View): New field.
+	* gen_il-gen-gen_entities.adb (Type_Kind): Use new field.
+	* einfo.ads (Is_Implicit_Full_View): Document new field.
+	* exp_ch7.adb (Make_Adjust_Call, Make_Init_Call, Make_Final_Call): Use
+	new field.
+	* exp_util.adb (Finalize_Address): Likewise.
+	* sem_ch3.adb (Copy_And_Build): Set new field.
+
+2025-07-22  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* exp_util.ads (May_Generate_Large_Temp): Delete.
+	* exp_util.adb (May_Generate_Large_Temp): Likewise.
+	(Safe_Unchecked_Type_Conversion): Do not take stack checking into
+	account to compute the result.
+
+2025-07-22  Javier Miranda  <miranda@adacore.com>
+
+	* sinfo.ads (Is_Expanded_Dispatching_Call): New flag.
+	(Tag_Propagated): New flag.
+	* exp_ch6.adb (Expand_Call_Helper): Propagate the tag when
+	the dispatching call is placed in conditionl expressions or
+	case-expressions.
+	* sem_ch5.adb (Analyze_Assignment): For assignment of tag-
+	indeterminate expression, do not propagate the tag if
+	previously done.
+	* sem_disp.adb (Is_Tag_Indeterminate): Add missing support
+	for conditional expression and case expression.
+	* exp_disp.ads (Is_Expanded_Dispatching_Call): Removed. Function
+	replaced by a new flag in the nodes.
+	* exp_disp.adb (Expand_Dispatching_Call): Set a flag in the
+	call node to remember that the call has been expanded.
+	(Is_Expanded_Dispatching_Call): Function removed.
+	* gen_il-fields.ads (Tag_Propagated): New flag.
+	(Is_Expanded_Dispatching_Call): New flag.
+	* gen_il-gen-gen_nodes.adb (Tag_Propagated): New flag.
+	(Is_Expanded_Dispatching_Call): New flag.
+
+2025-07-22  Gary Dismukes  <dismukes@adacore.com>
+
+	* libgnat/a-cbhama.ads (Empty): Add missing default to Capacity formal.
+	* libgnat/a-cbhama.adb (Empty): Add missing default to Capacity formal.
+	* exp_aggr.adb (Build_Size_Expr): Test for presence of Capacity
+	discriminant as additional criterion for generating the call to
+	the Length function. Update comments.
+
+2025-07-22  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* exp_util.adb (Safe_Unchecked_Type_Conversion): Always return True
+	if the expression is the prefix of an N_Selected_Component.
+
+2025-07-22  Denis Mazzucato  <mazzucato@adacore.com>
+
+	* sem_ch6.adb (Might_Need_BIP_Task_Actuals): Check whether No_Task_Parts is enabled in any
+	of the derived types, or interfaces, from the user-defined primitive return type.
+	* sem_ch13.adb (Analyze_Aspect_Specifications): Add No_Task_Parts and No_Controlled_Parts to
+	the representation chain to be visible in the full view of private types.
+	* aspects.ads (Nonoverridable_Aspect_Id): As per GNAT RM, No_Task_Parts is nonoverridable.
+	* sem_util.adb (Check_Inherited_Nonoverridable_Aspects): Likewise.
+	* sem_util.ads: Fix typo and style.
+	* sem_disp.adb: Missing comment.
+
+2025-07-22  Javier Miranda  <miranda@adacore.com>
+
+	* einfo.ads (Extra_Formals): Complete documentation.
+	(Has_First_Controlling_Parameter_Aspect): Place it in alphabetical order.
+	(Has_Frozen_Extra_Formals): New attribute.
+	* gen_il-fields.ads (Has_Frozen_Extra_Formals): New entity field.
+	* gen_il-gen-gen_entities.adb (Has_Frozen_Extra_Formals): Adding new
+	entity flag to subprograms, subprogram types, and and entries.
+	* gen_il-internals.adb (Image): Adding Has_Frozen_Extra_Formals.
+	* exp_ch3.adb (Build_Array_Init_Proc): Freeze its extra formals.
+	(Build_Init_Procedure): Freeze its extra formals.
+	(Expand_Freeze_Record_Type): For tagged types with foreign convention
+	create the extra formals of primitives with convention Ada.
+	* exp_ch6.ads (Create_Extra_Actuals): New subprogram.
+	* exp_ch6.adb (Check_BIP_Actuals): Adding assertions.
+	(Create_Extra_Actuals): New subprogram that factorizes code from
+	Expand_Call_Helper.
+	(Expand_Call_Helper): Adding support to defer the addition of extra
+	actuals. Move the code that adds the extra actuals to a new subprogram.
+	(Is_Unchecked_Union_Equality): Renamed as Is_Unchecked_Union_Predefined_
+	Equality_Call.
+	* exp_ch7.adb (Create_Finalizer): Freeze its extra formals.
+	(Wrap_Transient_Expression): Link the temporary with its relocated
+	expression to facilitate locating the expression in the expanded code.
+	* exp_ch9.ads (Expand_N_Entry_Declaration): Adding one formal.
+	* exp_ch9.adb (Expand_N_Entry_Declaration): Defer the expansion of
+	the entry if the extra formals are not available; analyze the built
+	declarations for the record type that holds all the parameters if
+	the expansion of the entry declaration was deferred.
+	* exp_disp.adb (Expand_Dispatching_Call): Handle deferred extra formals.
+	(Set_CPP_Constructors): Freeze its extra formals.
+	* freeze.adb (Freeze_Entity): Create the extra actuals of acccess to
+	subprograms whose designated type is a subprogram type.
+	(Freeze_Subprogram): Adjust assertion to support deferred extra formals,
+	and freeze extra formals of non-dispatching subprograms with foreign
+	convention. Added assertion to check matching of formals in thunks.
+	* sem_aux.adb (Get_Called_Entity): Adding documentation.
+	* sem_ch3.adb (Analyze_Full_Type_Declaration): Create the extra formals
+	of deferred subprograms, subprogram types and entries; create also the
+	extra actuals of deferred calls.
+	* sem_ch6.ads (Freeze_Extra_Formals): New subprogram.
+	(Deferred_Extra_Formals_Support): New package.
+	* sem_ch6.adb (Analyze_Subprogram_Body_Helper): Create the extra formals
+	of subprograms without separate spec.
+	(Add_Extra_Formal): Add documentation.
+	(Has_Extra_Formals): Removed.
+	(Parent_Subprogram): Adding documentation.
+	(Create_Extra_Formals): Defer adding extra formals if the underlying_type
+	of some formal type or return type is not available.
+	(Extra_Formals_Match_OK): Add missing check on the extra formals of
+	unchecked unions.
+	(Freeze_Extra_Formals): New subprogram.
+	(Deferred_Extra_Formals_Support): New package.
+	* sem_ch9.adb (Analyze_Entry_Declaration): Freeze its extra formals.
+	* sem_ch13.adb (New_Put_Image_Subprogram): ditto.
+	* sem_util.ads (Is_Unchecked_Union_Equality): New subprogram.
+	* sem_util.adb (Is_Unchecked_Union_Equality): ditto.
+
+2025-07-22  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_util.adb (Get_Actual_Subtype): Do the same for GCC and GNATprove
+	backends.
+
+2025-07-22  Martin Clochard  <clochard@adacore.com>
+
+	* exp_spark.adb (Expand_SPARK): Add expansion of continue statements.
+	(Expand_SPARK_N_Continue_Statement): Expand continue statements resolved
+	as procedure calls into said procedure calls.
+
+2025-07-22  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_res.adb (Resolve_Call): Look at the base type of actual parameter
+	when checking call to Set_Handler.
+
+2025-07-22  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* sem_util.adb (Get_Actual_Subtype): Only create a new subtype when
+	the expander is active.  Remove a useless test of type inequality,
+	as well as a useless call to Set_Has_Delayed_Freeze on the subtype.
+
+2025-07-22  Gary Dismukes  <dismukes@adacore.com>
+
+	* exp_aggr.adb (Build_Size_Expr): Change test of "not Present (...)"
+	to "No (...)".
+
+2025-07-22  Gary Dismukes  <dismukes@adacore.com>
+
+	* exp_aggr.adb (Build_Size_Expr): Determine the length of a container
+	aggregate association in the case where it's an iteration over an
+	object of a container type coming from an instantiation of a predefined
+	container generic. Minor updates to existing comments.
+
+2025-07-22  Ghjuvan Lacambre  <lacambre@adacore.com>
+
+	* exp_util.adb (Finalize_Address): Prevent infinite loop
+
+2025-07-22  Steve Baird  <baird@adacore.com>
+
+	* sem_aux.ads: Declare new function Unique_Component_Name.
+	* sem_aux.adb: Implement new function Unique_Component_Name.
+
+2025-07-22  Viljar Indus  <indus@adacore.com>
+
+	* sem_ch13.adb (Check_Aspect_At_End_Of_Declarations):
+	Ensure the Expression_Copy always has a parent before
+	calling any analyze.
+
+2025-07-22  Steve Baird  <baird@adacore.com>
+
+	* exp_aggr.adb (Build_Record_Aggr_Code.Gen_Assign): In the case of
+	an aggregate component where the component type is mutably tagged
+	and the component value is provided by a qualified aggregate (and
+	qualified with a specific type), avoid incorrectly rejecting the
+	inner aggregate for violating the rule that the type of an
+	aggregate shall not be class-wide.
+	* exp_attr.adb: For a predefined streaming operation (i.e., Read,
+	Write, Input, or Output) of a class-wide type, the external name
+	of the tag of the value is normally written out by Output and read
+	in by Input. In the case of a mutably tagged type, this is instead
+	done in Write and Read.
+	* exp_ch4.adb (Expand_Composite_Equality): In the case of an
+	equality comparison for a type having a mutably tagged component,
+	we want the component comparison to compare two values of the
+	mutably tagged type, not two values of the corresponding
+	array-of-bytes-ish representation type. Even if there are no
+	user-defined equality functions anywhere in sight, comparing the
+	array values still doesn't work because undefined bits may end up
+	participating in the comparison (resulting in an incorrect result
+	of False).
+	* exp_put_image.adb: In the case of a class-wide type, the
+	predefined Image attribute includes the name of the specific type
+	(and a "'" character, to follow qualified expression syntax) to
+	indicate the tag of the value. With the introduction of mutably
+	tagged types, this case can now arise in the case of a component
+	(of either an enclosing array or an enclosing record), not just
+	for a top-level object. So we factor the code to do this into a
+	new procedure, Put_Specific_Type_Name_Qualifier, so that it can be
+	called from more than one place. This reorganization also involves
+	replacing the procedure Put_String_Exp with a new procedure,
+	Put_String_Exp_To_Buffer, declared in a less nested scope. For
+	mutably tagged components (at the source level) the component type
+	(at the GNAT tree level) is an array of bytes (actually a two
+	field record containing an array of bytes, but that's a detail).
+	Appropriate conversions need to be generated so that we don't end
+	up generating an image for an array of bytes; this is done at the
+	same places where Put_Specific_Type_Name_Qualifier is called
+	(for components) by calling Make_Mutably_Tagged_Conversion.
+	* exp_strm.adb (Make_Field_Attribute): Add
+	Make_Mutably_Tagged_Conversion call where we construct a
+	Selected_Component node and the corresponding component type is
+	the internal representation type for a mutably tagged type.
+	(Stream_Base_Type): Return the mutably
+	tagged type if given the corresponding internal representation type.
+	* sem_ch3.adb (Array_Type_Declaration): In the case where the
+	source-level component type of an array type is mutably tagged,
+	set the Component_Type field of the base type of the declared
+	array type (as opposed to that of the first subtype of the array
+	type) to the corresponding internal representation type.
+	* sem_ch4.adb (Analyze_Selected_Component): In the case of a
+	selected component name which references a component whose type is
+	the internal representation type of a mutably tagged type,
+	generate a conversion to the mutably tagged type.
+
+2025-07-21  Eric Botcazou  <ebotcazou@gcc.gnu.org>
+
+	PR ada/121184
+	* styleg.adb (Check_Comment): Use consistent warning message.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc-interface/trans.cc (gnat_to_gnu): Pass null pointer to
+	parse_{input,output}_constraint().
+
+2025-07-18  Steve Baird  <baird@adacore.com>
+
+	* sem_ch12.adb (Validate_Derived_Type_Instance): Cope with the case
+	where the ancestor type for a formal derived type is declared in
+	an earlier formal package but Get_Instance_Of does not return the
+	corresponding type from the corresponding actual package.
+
+2025-07-18  Bob Duff  <duff@adacore.com>
+
+	* tbuild.adb (Unchecked_Convert_To): Back out
+	change.
+
+2025-07-18  Marc Poulhiès  <poulhies@adacore.com>
+	    Eric Botcazou  <botcazou@adacore.com>
+
+	* exp_ch6.adb (Convert): Do not call Expand_Inlined_Call for
+	unsupported cases.
+	* inline.adb (Expand_Inlined_Call): Add assert to catch unsupported
+	case.
+
+2025-07-18  Gary Dismukes  <dismukes@adacore.com>
+
+	* einfo.ads: Document new field Overridden_Inherited_Operation and
+	list it as a field for the entity kinds that it applies to.
+	* gen_il-fields.ads (type Opt_Field_Enum): Add new literal
+	Overridden_Inherited_Operation to the type.
+	* gen_il-gen-gen_entities.adb: Add Overridden_Inherited_Operation as
+	a field of entities of kinds E_Enumeration_Literal and Subprogram_Kind.
+	* sem_ch4.adb (Is_Callable_Private_Overriding): Change name (was
+	Is_Private_Overriding). Replace Is_Hidden test on Overridden_Operation
+	with test of Is_Hidden on the new field Overridden_Inherited_Operation.
+	* sem_ch6.adb (New_Overloaded_Entity): Set the new field
+	Overridden_Inherited_Operation on an operation derived from
+	an interface to refer to the inherited operation of a private
+	extension that's overridden by the derived operation. Also set
+	that field in the more common cases of an explicit subprogram
+	that overrides, to refer to the inherited subprogram that is
+	overridden. (Contrary to its name, the Overridden_Operation
+	field of the overriding subprogram, which is also set in these
+	places, refers to the *parent* subprogram from which the inherited
+	subprogram is derived.) Also, remove a redundant Present (Alias (S))
+	test in an if_statement and the dead "else" part of that statement.
+
+2025-07-18  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_util.adb (Build_Elaboration_Entity): Set ghost mode to none
+	before creating the elaboration entity; restore the ghost mode
+	afterwards.
+
+2025-07-18  Javier Miranda  <miranda@adacore.com>
+
+	* exp_aggr.adb (Gen_Assign): Code cleanup.
+	(Initialize_Component): Do not adjust the tag when the type of
+	the aggregate components is a mutably tagged type.
+
+2025-07-14  Eric Botcazou  <ebotcazou@adacore.com>
+
+	PR ada/121056
+	* sem_ch4.adb (Try_Object_Operation.Try_Primitive_Operation): Add
+	test on Is_Record_Type before accessing Underlying_Record_View.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	* par-load.adb: Comment spelling fix: bellow -> below.
+	* libgnarl/s-taskin.ads: Likewise.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/Make-lang.in (ACATSDIR): Change to acats-4.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/utils.cc (make_packable_type): Clear the TYPE_PACKED
+	flag in the case where the alignment is bumped.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/trans.cc (Subprogram_Body_to_gnu): Do not generate
+	a block-copy out for a null initialization procedure when the _Init
+	parameter is not passed in.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/decl.cc (gnat_to_gnu_subprog_type): Only apply the
+	transformation to integer types.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/decl.cc (gnat_to_gnu_subprog_type): Add guards.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/decl.cc (gnat_to_gnu_subprog_type): In the case of a
+	subprogram using the Copy-In/Copy-Out mechanism, deal specially with
+	the case of 2 parameters of differing sizes.
+	* gcc-interface/trans.cc (Subprogram_Body_to_gnu): In the case of a
+	subprogram using the Copy-In/Copy-Out mechanism, make sure the types
+	are consistent on the two sides for all the parameters.
+
+2025-07-04  Steve Baird  <baird@adacore.com>
+
+	* sem_res.adb (Resolve_Type_Conversion): Replace code for
+	detecting a similar case with a more comprehensive test.
+
+2025-07-04  Bob Duff  <duff@adacore.com>
+
+	* doc/gnat_rm/implementation_defined_pragmas.rst
+	(Short_Circuit_And_Or): Add more documentation.
+	* sem_ch8.adb (Analyze_Subprogram_Renaming):
+	Disallow renamings.
+	* gnat_rm.texi: Regenerate.
+
+2025-07-04  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch7.adb (Make_Final_Call): Tweak search of Finalize primitive.
+	* exp_util.adb (Finalize_Address): Likewise.
+
+2025-07-04  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* freeze.adb (Check_Compile_Time_Size): Try harder to see whether
+	the bounds of array types are known at compile time.
+
+2025-07-04  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_aux.ads (First_Discriminant): Remove space before period.
+
+2025-07-04  Steve Baird  <baird@adacore.com>
+
+	* sem_ch13.adb (Analyze_Record_Representation_Clause): In deciding
+	whether to generate a warning about a missing component clause, in
+	addition to calling Is_Unchecked_Union also call a new local
+	function, Unchecked_Union_Pragma_Pending, which checks for the
+	case of a not-yet-analyzed Unchecked_Union pragma occurring later
+	in the declaration list.
+
+2025-07-04  Steve Baird  <baird@adacore.com>
+
+	* mutably_tagged.adb (Make_CW_Size_Compile_Check): Include the
+	value of the Size'Class limit in the message generated via a
+	Compile_Time_Error pragma.
+
+2025-07-04  Ronan Desplanques  <desplanques@adacore.com>
+
+	* sem_ch13.adb (Check_Aspect_At_Freeze_Point): Remove obsolete bits.
+
+2025-07-04  Ronan Desplanques  <desplanques@adacore.com>
+
+	* sem_ch13.adb (Analyze_Aspect_Specifications): Fix error emission.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/Makefile.in (gnatlib-sjlj): Delete.
+	(gnatlib-zcx): Do not modify Frontend_Exceptions constant.
+	* libgnat/system-linux-loongarch.ads (Frontend_Exceptions): Delete.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/decl.cc (type_contains_only_integral_data): Do not
+	return false only because the type contains pointer data.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/decl.cc (gnat_to_gnu_entity): Use default messages
+	for errors reported for Object_Size clauses.
+	(validate_size): Give an error for stand-alone objects of composite
+	types if the specified size is not a multiple of the alignment.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/trans.cc (addressable_p): Add COMPG third parameter.
+	<COMPONENT_REF>: Do not return true out of alignment considerations
+	for non-strict-alignment targets if COMPG is set.
+	(Call_to_gnu): Pass true as COMPG in the call to the addressable_p
+	predicate if the called subprogram is an initialization procedure.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/trans.cc (gnat_to_gnu) <N_Allocator>: Allocate the
+	bounds alongside the data if the Is_Constr_Array_Subt_With_Bounds
+	flag is set on the designated type.
+	<N_Free_Statement>: Take into account the allocated bounds if the
+	Is_Constr_Array_Subt_With_Bounds flag is set on the designated type.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gcc-interface/decl.cc (gnat_to_gnu_component_type): Validate the
+	Component_Size like the size of a type only if the component type
+	is actually packed.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_elab.adb (Check_Overriding_Primitive): Find early call region
+	of the subprogram body declaration, not of the subprogram body stub.
+
+2025-07-03  Bob Duff  <duff@adacore.com>
+
+	* gen_il-gen-gen_nodes.adb (N_Unchecked_Type_Conversion):
+	Remove useless Nmake_Assert.
+	* tbuild.adb (Unchecked_Convert_To):
+	Narrow the bitfield-related conditions.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_util.adb (Insert_Actions): Fix check.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch6.adb (Expand_Ctrl_Function_Call): Precisify comment.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* exp_ch6.adb (Expand_Ctrl_Function_Call): Do not bail out for the
+	declarations of return objects.
+
+2025-07-03  Daniel King  <dmking@adacore.com>
+
+	* Makefile.rtl (LIBGNAT_TARGET_PAIRS): New unit s-tsgsba__cheri.adb for morello-freebsd.
+	* libgnarl/s-tassta.adb (Get_Stack_Base): New function.
+	* libgnarl/s-tsgsba__cheri.adb: New file for CHERI targets.
+	* libgnarl/s-tsgsba.adb: New default file for non-CHERI targets.
+	* libgnat/s-stausa.adb (Fill_Stack, Compute_Result): Port to CHERI.
+	* libgnat/s-stausa.ads (Initialize_Analyzer, Stack_Analyzer): Port to CHERI.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_ch3.adb (Check_Return_Subtype_Indication): Use Original_Node.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_ch3.adb (Check_Return_Subtype_Indication): Use type from
+	explicit subtype indication, when possible.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_ch3.adb (Check_Return_Subtype_Indication): Adjust error message
+	to match the RM wording.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_ch3.adb (Check_Return_Subtype_Indication): Use the nominal
+	subtype of a return object; literally implement the RM rule about
+	elementary types; check for static subtype compatibility both when
+	the subtype is given as a subtype mark and a subtype indication.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* repinfo.adb (First_Comp_Or_Discr.Is_Placed_Before): Return True
+	only if the components are in the same component list.
+
+2025-07-03  Denis Mazzucato  <mazzucato@adacore.com>
+
+	* sem_disp.adb (Check_Dispatching_call): Fix uninitialized Subp_Entity.
+	* sem_util.adb (Update_Controlling_Argument): No need to replace controlling argument
+	in case of functions.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* errid.ads: Adjust header to renaming and fix copyright line.
+	* errid.adb: Adjust header to renaming and add blank line.
+	* erroutc-pretty_emitter.ads: Adjust header to renaming.
+	* erroutc-pretty_emitter.adb: Likewise.
+	* erroutc-sarif_emitter.ads: Likewise.
+	* erroutc-sarif_emitter.adb: Likewise.
+	* errsw.ads: Adjust header to renaming and add blank line.
+	* errsw.adb: Likewise.
+	* json_utils.ads: Likewise.
+	* json_utils.adb: Adjust header to renaming.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* errid.ads (Diagnostic_Entries): Now a constant.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* errid.ads (Diagnostic_Entries): Remove nested aggregate.
+	* errsw.adb (Switches): Likewise.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch7.adb (Make_Deep_Record_Body): Fix case of absent Initialize
+	primitive.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_ch3.adb (Count_Default_Sized_Task_Stacks): Refine subtypes of
+	parameters; same for callsites.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* exp_imgv.adb (Expand_Value_Attribute): Do not call Set_Etype on N
+	after rewriting it by means of OK_Convert_To.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_aggr.adb (Generate_Finalization_Actions): Stop assuming that
+	initialize primitive exists.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch7.adb (Build_Record_Deep_Procs): Fix typo in comment.
+
+2025-07-03  Gary Dismukes  <dismukes@adacore.com>
+
+	* sem_ch12.adb (Install_Spec): Remove "not Is_Generic_Instance (Par)"
+	in test for setting Instance_Parent_Unit. Revise comment to no longer
+	say "noninstance", plus remove "???".
+	(Remove_Parent): Restructure if_statement to allow for both "elsif"
+	parts to be executed (by changing them to be separate if_statements
+	within an "else" part).
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch3.adb (Predefined_Primitive_Bodies): Fix comment.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* exp_tss.adb (TSS): Refactor IF condition to make code smaller.
+	* lib.adb (Increment_Serial_Number, Synchronize_Serial_Number):
+	Use type of renamed object when creating renaming.
+	* lib.ads (Unit_Record): Refine subtype of dependency number.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* libgnat/s-valuef.adb: Document the prerequisites more precisely.
+	* libgnat/a-tifiio.adb (OK_Get_32): Adjust to the prerequisites.
+	(OK_Get_64): Likewise.
+	* libgnat/a-tifiio__128.adb (OK_Get_32): Likewise.
+	(OK_Get_64): Likewise.
+	(OK_Get_128): Likewise.
+	* libgnat/a-wtfiio.adb (OK_Get_32): Likewise.
+	(OK_Get_64): Likewise.
+	* libgnat/a-wtfiio__128.adb (OK_Get_32): Likewise.
+	(OK_Get_64): Likewise.
+	(OK_Get_128): Likewise.
+	* libgnat/a-ztfiio.adb (OK_Get_32): Likewise.
+	(OK_Get_64): Likewise.
+	* libgnat/a-ztfiio__128.adb (OK_Get_32): Likewise.
+	(OK_Get_64): Likewise.
+	(OK_Get_128): Likewise.
+	* exp_imgv.adb (Expand_Value_Attribute): Adjust the conditions under
+	which the RE_Value_Fixed{32,64,128} routines are called for ordinary
+	fixed-point types.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch3.adb (Make_Predefined_Primitive_Specs): Fix comment.
+
+2025-07-03  Ronan Desplanques  <desplanques@adacore.com>
+
+	* exp_ch7.adb (Insert_Actions_In_Scope_Around): Fix condition.
+
+2025-07-03  Bob Duff  <duff@adacore.com>
+
+	* checks.adb: Remove unnecessary "return;" statements.
+	* eval_fat.adb: Likewise.
+	* exp_aggr.adb: Likewise.
+	* exp_attr.adb: Likewise.
+	* exp_ch3.adb: Likewise.
+	* exp_ch4.adb: Likewise.
+	* exp_ch5.adb: Likewise.
+	* exp_ch6.adb: Likewise.
+	* exp_unst.adb: Likewise.
+	* krunch.adb: Likewise.
+	* layout.adb: Likewise.
+	* libgnat/s-excdeb.adb: Likewise.
+	* libgnat/s-trasym__dwarf.adb: Likewise.
+	* par-endh.adb: Likewise.
+	* par-tchk.adb: Likewise.
+	* sem.adb: Likewise.
+	* sem_attr.adb: Likewise.
+	* sem_ch6.adb: Likewise.
+	* sem_elim.adb: Likewise.
+	* sem_eval.adb: Likewise.
+	* sfn_scan.adb: Likewise.
+
+2025-07-03  Bob Duff  <duff@adacore.com>
+
+	* doc/gnat_rm/implementation_defined_characteristics.rst:
+	Change Ignore to Disable.
+	* sem_ch13.ads (Analyze_Aspect_Specifications):
+	Minor: Remove incorrect comment; there is no need to check
+	Has_Aspects (N) at the call site.
+	* gnat_rm.texi: Regenerate.
+	* gnat_ugn.texi: Regenerate.
+
+2025-07-03  Bob Duff  <duff@adacore.com>
+
+	* types.ads (Empty_Or_Error): Remove.
+	* atree.adb: Remove reference to Empty_Or_Error.
+	* par-endh.adb: Likewise.
+	* sem_ch12.adb: Likewise.
+	* sem_ch3.adb: Likewise.
+	* sem_util.adb: Likewise.
+	* treepr.adb: Likewise.
+
+2025-07-03  Viljar Indus  <indus@adacore.com>
+
+	* sem_ch10.adb(Analyze_With_Clause): Call Semantics instead
+	of Analyze to bring Current_Sem_Unit up to date.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* lib-xref-spark_specific.adb
+	(Enclosing_Subprogram_Or_Library_Package): Traverse subunits and body
+	stubs.
+
+2025-07-03  Tonu Naks  <naks@adacore.com>
+
+	* libgnat/i-cstrin.ads (Value): add documentation
+
+2025-07-03  Aleksandra Pasek  <pasek@adacore.com>
+
+	* libgnat/a-strsup.adb (Super_Delete): Fix index check.
+	* libgnat/a-stwisu.adb (Super_Delete): Likewise.
+	* libgnat/a-stzsup.adb (Super_Delete): Likewise.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* exp_ch4.adb (Handle_Changed_Representation): Alphabetize local
+	variables.  Set the No_Finalize_Actions flag on the assignment.
+
+2025-07-03  Joffrey Huguet  <huguet@adacore.com>
+
+	* aspects.ads: Define an identifier for Potentially_Invalid.
+	* doc/gnat_rm/implementation_defined_aspects.rst: Add section for Potentially_Invalid.
+	* sem_attr.adb (Analyze_Attribute_Old_Result): Attribute Old is allowed to occur in a
+	Potentially_Invalid aspect.
+	* sem_ch13.adb (Analyze_Aspect_Specifications): Handle Potentially_Invalid.
+	* sem_util.adb (Has_Potentially_Invalid): Returns True iff an entity is subject to the
+	Potentially_Invalid aspect.
+	* sem_util.ads (Has_Potentially_Invalid): Idem.
+	* snames.ads-tmpl: New name for Potentially_Invalid.
+	* gnat_rm.texi: Regenerate.
+
+2025-07-03  Piotr Trojanek  <trojanek@adacore.com>
+
+	* sem_ch10.adb (Analyze_Compilation_Unit): Ignored ghost unit need no
+	elaboration checks.
+
+2025-07-03  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* libgnat/s-valued.adb (Integer_to_Decimal): Use truncation for the
+	scaled divide operation performed for bases other than 10.
+
 2025-07-01  Eric Botcazou  <ebotcazou@adacore.com>
 
 	PR ada/120705
diff --git a/gcc/ada/accessibility.adb b/gcc/ada/accessibility.adb
index 0b8d3f7..c780054 100644
--- a/gcc/ada/accessibility.adb
+++ b/gcc/ada/accessibility.adb
@@ -327,8 +327,23 @@ package body Accessibility is
                if In_Return_Value (N)
                  or else In_Return_Context
                then
-                  return Make_Level_Literal
-                           (Subprogram_Access_Level (Current_Subprogram));
+                  if Present (Extra_Accessibility_Of_Result
+                                (Current_Subprogram))
+                  then
+                     --  If a function is passed an extra "level of the
+                     --  master of the call" parameter and that function
+                     --  returns a call to another such function (or
+                     --  possibly to the same function, in the case of a
+                     --  recursive call), then that parameter should be
+                     --  "passed along".
+
+                     return New_Occurrence_Of
+                              (Extra_Accessibility_Of_Result
+                                (Current_Subprogram), Loc);
+                  else
+                     return Make_Level_Literal
+                              (Subprogram_Access_Level (Current_Subprogram));
+                  end if;
                end if;
             end if;
 
@@ -1683,16 +1698,14 @@ package body Accessibility is
                 Condition => Check_Cond,
                 Reason    => PE_Accessibility_Check_Failed));
 
-            --  If constant folding has happened on the condition for the
-            --  generated error, then warn about it being unconditional when
-            --  we know an error will be raised.
+            --  ??? Is this how we want to detect RM 6.5(5.9) violations?
 
             if Nkind (Check_Cond) = N_Identifier
               and then Entity (Check_Cond) = Standard_True
             then
                Error_Msg_N
-                 ("access discriminant in return object could be a dangling"
-                  & " reference??", Return_Stmt);
+                 ("level of type of access discriminant value of return object"
+                    & " is statically too deep", Return_Stmt);
             end if;
          end if;
 
diff --git a/gcc/ada/aspects.ads b/gcc/ada/aspects.ads
index d8861bf..737f151 100644
--- a/gcc/ada/aspects.ads
+++ b/gcc/ada/aspects.ads
@@ -92,6 +92,7 @@ package Aspects is
       Aspect_Default_Value,
       Aspect_Depends,                       -- GNAT
       Aspect_Designated_Storage_Model,      -- GNAT
+      Aspect_Destructor,                    -- GNAT
       Aspect_Dimension,                     -- GNAT
       Aspect_Dimension_System,              -- GNAT
       Aspect_Dispatching_Domain,
@@ -259,6 +260,7 @@ package Aspects is
                                  | Aspect_Iterator_Element
                                  | Aspect_Max_Entry_Queue_Length
                                  | Aspect_No_Controlled_Parts
+                                 | Aspect_No_Task_Parts
                                  | Aspect_Real_Literal
                                  | Aspect_String_Literal
                                  | Aspect_Variable_Indexing;
@@ -293,6 +295,7 @@ package Aspects is
       Aspect_CUDA_Global                => True,
       Aspect_Depends                    => True,
       Aspect_Designated_Storage_Model   => True,
+      Aspect_Destructor                 => True,
       Aspect_Dimension                  => True,
       Aspect_Dimension_System           => True,
       Aspect_Disable_Controlled         => True,
@@ -447,6 +450,7 @@ package Aspects is
       Aspect_Default_Value              => Expression,
       Aspect_Depends                    => Expression,
       Aspect_Designated_Storage_Model   => Name,
+      Aspect_Destructor                 => Name,
       Aspect_Dimension                  => Expression,
       Aspect_Dimension_System           => Expression,
       Aspect_Dispatching_Domain         => Expression,
@@ -551,6 +555,7 @@ package Aspects is
       Aspect_Default_Value                => True,
       Aspect_Depends                      => False,
       Aspect_Designated_Storage_Model     => True,
+      Aspect_Destructor                   => False,
       Aspect_Dimension                    => False,
       Aspect_Dimension_System             => False,
       Aspect_Dispatching_Domain           => False,
@@ -726,6 +731,7 @@ package Aspects is
       Aspect_Default_Value                => Name_Default_Value,
       Aspect_Depends                      => Name_Depends,
       Aspect_Designated_Storage_Model     => Name_Designated_Storage_Model,
+      Aspect_Destructor                   => Name_Destructor,
       Aspect_Dimension                    => Name_Dimension,
       Aspect_Dimension_System             => Name_Dimension_System,
       Aspect_Disable_Controlled           => Name_Disable_Controlled,
@@ -994,6 +1000,7 @@ package Aspects is
       Aspect_Default_Value                => Always_Delay,
       Aspect_Default_Component_Value      => Always_Delay,
       Aspect_Designated_Storage_Model     => Always_Delay,
+      Aspect_Destructor                   => Always_Delay,
       Aspect_Discard_Names                => Always_Delay,
       Aspect_Dispatching_Domain           => Always_Delay,
       Aspect_Dynamic_Predicate            => Always_Delay,
diff --git a/gcc/ada/doc/gnat_rm/gnat_language_extensions.rst b/gcc/ada/doc/gnat_rm/gnat_language_extensions.rst
index f313179..ff111dd 100644
--- a/gcc/ada/doc/gnat_rm/gnat_language_extensions.rst
+++ b/gcc/ada/doc/gnat_rm/gnat_language_extensions.rst
@@ -1793,3 +1793,71 @@ statement in the sequence of statements of the specified loop_statement.
 
 Note that ``continue`` is a keyword but it is not a reserved word. This is a
 configuration that does not exist in standard Ada.
+
+Destructors
+-----------
+
+The ``Destructor`` aspect can be applied to any record type, tagged or not.
+It must denote a primitive of the type that is a procedure with one parameter
+of the type and of mode ``in out``:
+
+.. code-block:: ada
+
+   type T is record
+      ...
+   end record with Destructor => Foo;
+
+   procedure Foo (X : in out T);
+
+This is equivalent to the following code that uses ``Finalizable``:
+
+.. code-block:: ada
+
+   type T is record
+      ...
+   end record with Finalizable => (Finalize => Foo);
+
+   procedure Foo (X : in out T);
+
+Unlike ``Finalizable``, however, ``Destructor`` can be specified on a derived
+type. And when it is, the effect of the aspect combines with the destructors of
+the parent type. Take, for example:
+
+.. code-block:: ada
+
+   type T1 is record
+      ...
+   end record with Destructor => Foo;
+
+   procedure Foo (X : in out T1);
+
+   type T2 is new T1 with Destructor => Bar;
+
+   procedure Bar (X : in out T2);
+
+Here, when an object of type ``T2`` is finalized, a call to ``Bar``
+will be performed and it will be followed by a call to ``Foo``.
+
+The ``Destructor`` aspect comes with a legality rule: if a primitive procedure
+of a type is denoted by a ``Destructor`` aspect specification, it is illegal to
+override this procedure in a derived type. For example, the following is illegal:
+
+.. code-block:: ada
+
+   type T1 is record
+      ...
+   end record with Destructor => Foo;
+
+   procedure Foo (X : in out T1);
+
+   type T2 is new T1;
+
+   overriding
+   procedure Foo (X : in out T2); -- Error here
+
+It is possible to specify ``Destructor`` on the completion of a private type,
+but there is one more restriction in that case: the denoted primitive must
+be private to the enclosing package. This is necessary due to the previously
+mentioned legality rule, to prevent breaking the privacy of the type when
+imposing that rule on outside types that derive from the private view of the
+type.
diff --git a/gcc/ada/doc/gnat_rm/implementation_defined_pragmas.rst b/gcc/ada/doc/gnat_rm/implementation_defined_pragmas.rst
index 02013f1..3986298 100644
--- a/gcc/ada/doc/gnat_rm/implementation_defined_pragmas.rst
+++ b/gcc/ada/doc/gnat_rm/implementation_defined_pragmas.rst
@@ -5914,13 +5914,33 @@ Syntax:
   pragma Short_Circuit_And_Or;
 
 
-This configuration pragma causes any occurrence of the AND operator applied to
-operands of type Standard.Boolean to be short-circuited (i.e. the AND operator
-is treated as if it were AND THEN). Or is similarly treated as OR ELSE. This
-may be useful in the context of certification protocols requiring the use of
-short-circuited logical operators. If this configuration pragma occurs locally
-within the file being compiled, it applies only to the file being compiled.
+This configuration pragma causes the predefined AND and OR operators of
+type Standard.Boolean to have short-circuit semantics. That is, they
+behave like AND THEN and OR ELSE; the right-hand side is not evaluated
+if the left-hand side determines the result. This may be useful in the
+context of certification protocols requiring the use of short-circuited
+logical operators.
+
 There is no requirement that all units in a partition use this option.
+However, mixing of short-circuit and non-short-circuit semantics can be
+confusing. Therefore, the recommended use is to put the pragma in a
+configuration file that applies to the whole program. Alternatively, if
+you have a legacy library that should not use this pragma, you can put
+it in a separate library project that does not use the pragma.
+In any case, fine-grained mixing of the different semantics is not
+recommended. If pragma ``Short_Circuit_And_Or`` is specified, then it
+is illegal to rename the predefined Boolean AND and OR, or to pass
+them to generic formal functions; this corresponds to the fact that
+AND THEN and OR ELSE cannot be renamed nor passed as generic formal
+functions.
+
+Note that this pragma has no effect on other logical operators --
+predefined operators of modular types, array-of-boolean types and types
+derived from Standard.Boolean, nor user-defined operators.
+
+See also the pragma ``Unevaluated_Use_Of_Old`` and the restriction
+``No_Direct_Boolean_Operators``, which may be useful in conjunction
+with ``Short_Circuit_And_Or``.
 
 Pragma Short_Descriptors
 ========================
diff --git a/gcc/ada/doc/gnat_rm/standard_and_implementation_defined_restrictions.rst b/gcc/ada/doc/gnat_rm/standard_and_implementation_defined_restrictions.rst
index 0e9162a..1f0aa03 100644
--- a/gcc/ada/doc/gnat_rm/standard_and_implementation_defined_restrictions.rst
+++ b/gcc/ada/doc/gnat_rm/standard_and_implementation_defined_restrictions.rst
@@ -333,9 +333,9 @@ must define with the following profile:
                  "__gnat_last_chance_handler");
 
 
-The parameter is a C null-terminated string representing a message to be
-associated with the exception (typically the source location of the raise
-statement generated by the compiler). The Line parameter when nonzero
+The ``Source_Location`` parameter is a C null-terminated string representing a
+message to be associated with the exception (typically the source location of
+the raise statement generated by the compiler). The Line parameter when nonzero
 represents the line number in the source program where the raise occurs.
 
 No_Exception_Propagation
diff --git a/gcc/ada/einfo.ads b/gcc/ada/einfo.ads
index ba79fe4..916d9c6 100644
--- a/gcc/ada/einfo.ads
+++ b/gcc/ada/einfo.ads
@@ -1279,9 +1279,10 @@ package Einfo is
 --       that represents an activation record pointer is an extra formal.
 
 --    Extra_Formals
---       Applies to subprograms, subprogram types, entries, and entry
---       families. Returns first extra formal of the subprogram or entry.
---       Returns Empty if there are no extra formals.
+--       Applies to subprograms, subprogram types, entries, and entry families.
+--       Returns the first extra formal of the subprogram or entry. An entity
+--       has no extra formals when this attribute is Empty, and its attribute
+--       Extra_Formals_Known is True.
 
 --    Finalization_Collection [root type only]
 --       Defined in access-to-controlled or access-to-class-wide types. The
@@ -1594,6 +1595,11 @@ package Einfo is
 --       set, signalling that Freeze.Inherit_Delayed_Rep_Aspects must be called
 --       at the freeze point of the derived type.
 
+--    Has_Destructor
+--       Defined in all type and subtype entities. Set only for record type
+--       entities for which at least one ancestor has the Destructor aspect
+--       specified.
+
 --    Has_DIC (synthesized)
 --       Defined in all type entities. Set for a private type and its full view
 --       when the type is subject to pragma Default_Initial_Condition (DIC), or
@@ -1640,11 +1646,6 @@ package Einfo is
 --       that this does not imply a representation with holes, since the rep
 --       clause may merely confirm the default 0..N representation.
 
---    Has_First_Controlling_Parameter_Aspect
---       Defined in tagged types, concurrent types and concurrent record types.
---       Set to indicate that the type has a First_Controlling_Parameter of
---       True (whether by an aspect_specification, a pragma, or inheritance).
-
 --    Has_Exit
 --       Defined in loop entities. Set if the loop contains an exit statement.
 
@@ -1654,6 +1655,12 @@ package Einfo is
 --       flag prevents double expansion of a contract when a construct is
 --       rewritten into something else and subsequently reanalyzed/expanded.
 
+--    Has_First_Controlling_Parameter_Aspect
+--       Defined in tagged types, concurrent types, and concurrent record
+--       types. Set to indicate that the type has a First_Controlling_Parameter
+--       of True (whether by an aspect_specification, a pragma, or
+--       inheritance).
+
 --    Has_Foreign_Convention (synthesized)
 --       Applies to all entities. Determines if the Convention for the entity
 --       is a foreign convention, i.e. non-native: other than Convention_Ada,
@@ -1668,6 +1675,12 @@ package Einfo is
 --       the instance will conflict with the linear elaboration of front-end
 --       inlining.
 
+--    Extra_Formals_Known
+--       Defined in subprograms, subprogram types, entries, and entry families.
+--       Set when the extra formals have been determined. An entity has no
+--       extra formals when this attribute is True, and its attribute
+--       Extra_Formals is Empty.
+
 --    Has_Fully_Qualified_Name
 --       Defined in all entities. Set if the name in the Chars field has been
 --       replaced by the fully qualified name, as used for debug output. See
@@ -2515,11 +2528,12 @@ package Einfo is
 
 --    Is_Controlled_Active [base type only]
 --       Defined in all type entities. Indicates that the type is controlled,
---       i.e. has been declared with the Finalizable aspect or has inherited
---       the Finalizable aspect from an ancestor. Can only be set for record
---       types, tagged or untagged. System.Finalization_Root.Root_Controlled
---       is an example of the former case while Ada.Finalization.Controlled
---       and Ada.Finalization.Limited_Controlled are examples of the latter.
+--       i.e. has been declared with the Finalizable or the Destructor aspect
+--       or has inherited the aspect from an ancestor. Can only be set for
+--       record types, tagged or untagged.
+--       System.Finalization_Root.Root_Controlled is an example of the former
+--       case while Ada.Finalization.Controlled and
+--       Ada.Finalization.Limited_Controlled are examples of the latter.
 
 --    Is_Controlled (synth) [base type only]
 --       Defined in all type entities. Set if Is_Controlled_Active is set for
@@ -2545,6 +2559,10 @@ package Einfo is
 --       Defined in all entities. True if the entity is type System.Address,
 --       or (recursively) a subtype or derived type of System.Address.
 
+--    Is_Destructor
+--       Defined in procedure entities. True if the procedure is denoted by the
+--       Destructor aspect on some type.
+
 --    Is_DIC_Procedure
 --       Defined in functions and procedures. Set for a generated procedure
 --       which verifies the assumption of pragma Default_Initial_Condition at
@@ -2779,6 +2797,10 @@ package Einfo is
 --       identifiers in standard library packages, and to implement the
 --       restriction No_Implementation_Identifiers.
 
+--    Is_Implicit_Full_View
+--       Defined in types. Set on types that the compiler generates to act as
+--       full views of types that are derivations of private types.
+
 --    Is_Imported
 --       Defined in all entities. Set if the entity is imported. For now we
 --       only allow the import of exceptions, functions, procedures, packages,
@@ -3940,9 +3962,21 @@ package Einfo is
 --       Defined in constants and variables. Set if there is an address clause
 --       that causes the entity to overlay a constant object.
 
+--    Overridden_Inherited_Operation
+--       Defined in subprograms and enumeration literals. When set on a
+--       subprogram S, indicates an inherited subprogram that S overrides.
+--       In the case of a privately declared explicit subprogram E that
+--       overrides a private inherited subprogram, and the inherited
+--       subprogram itself overrides another inherited subprogram declared
+--       for a private extension, the field on E will reference the subprogram
+--       inherited by the private extension. This field is used for properly
+--       handling visibility for such privately declared subprograms. This
+--       field is always Empty for enumeration literal entities.
+
 --    Overridden_Operation
 --       Defined in subprograms. For overriding operations, points to the
---       user-defined parent subprogram that is being overridden.
+--       user-defined parent subprogram from which the inherited subprogram
+--       that is being overridden is derived.
 
 --    Package_Instantiation
 --       Defined in packages and generic packages. When defined, this field
@@ -5381,11 +5415,12 @@ package Einfo is
    --    Scope_Depth_Value
    --    Protection_Object                    (protected kind)
    --    Contract_Wrapper
-   --    Extra_Formals
    --    Contract
    --    SPARK_Pragma                         (protected kind)
    --    Default_Expressions_Processed
    --    Entry_Accepted
+   --    Extra_Formals
+   --    Extra_Formals_Known
    --    Has_Yield_Aspect
    --    Has_Expanded_Contract
    --    Ignore_SPARK_Mode_Pragmas
@@ -5413,6 +5448,7 @@ package Einfo is
    --    Enumeration_Pos
    --    Enumeration_Rep
    --    Alias
+   --    Overridden_Inherited_Operation
    --    Enumeration_Rep_Expr
    --    Interface_Name $$$
    --    Renamed_Object $$$
@@ -5502,9 +5538,11 @@ package Einfo is
    --    Subps_Index                          (non-generic case only)
    --    Interface_Alias
    --    LSP_Subprogram                       (non-generic case only)
+   --    Overridden_Inherited_Operation
    --    Overridden_Operation
    --    Wrapped_Entity                       (non-generic case only)
    --    Extra_Formals
+   --    Extra_Formals_Known                  (non-generic case only)
    --    Anonymous_Collections                (non-generic case only)
    --    Corresponding_Equality               (implicit /= only)
    --    Thunk_Entity                         (thunk case only)
@@ -5705,9 +5743,12 @@ package Einfo is
    --    Extra_Accessibility_Of_Result
    --    Last_Entity
    --    Subps_Index
+   --    Overridden_Inherited_Operation
    --    Overridden_Operation
    --    Linker_Section_Pragma
    --    Contract
+   --    Extra_Formals
+   --    Extra_Formals_Known
    --    Import_Pragma
    --    LSP_Subprogram
    --    SPARK_Pragma
@@ -5858,9 +5899,11 @@ package Einfo is
    --    Subps_Index                          (non-generic case only)
    --    Interface_Alias
    --    LSP_Subprogram                       (non-generic case only)
+   --    Overridden_Inherited_Operation
    --    Overridden_Operation                 (never for init proc)
    --    Wrapped_Entity                       (non-generic case only)
    --    Extra_Formals
+   --    Extra_Formals_Known                  (non-generic case only)
    --    Anonymous_Collections                (non-generic case only)
    --    Static_Initialization                (init_proc only)
    --    Thunk_Entity                         (thunk case only)
@@ -5899,6 +5942,7 @@ package Einfo is
    --    Is_Class_Wide_Wrapper
    --    Is_Constructor
    --    Is_CUDA_Kernel
+   --    Is_Destructor                        (non-generic case only)
    --    Is_DIC_Procedure                     (non-generic case only)
    --    Is_Elaboration_Checks_OK_Id
    --    Is_Elaboration_Warnings_OK_Id
@@ -6088,6 +6132,7 @@ package Einfo is
    --    Last_Entity
    --    Scope_Depth_Value
    --    Extra_Formals
+   --    Extra_Formals_Known
    --    Anonymous_Collections
    --    Contract
    --    SPARK_Pragma
@@ -6101,6 +6146,7 @@ package Einfo is
    --    Extra_Accessibility_Of_Result
    --    Directly_Designated_Type
    --    Extra_Formals
+   --    Extra_Formals_Known
    --    Access_Subprogram_Wrapper
    --    First_Formal                         (synth)
    --    First_Formal_With_Extras             (synth)
diff --git a/gcc/ada/exp_aggr.adb b/gcc/ada/exp_aggr.adb
index 9ff69ec..cd98369 100644
--- a/gcc/ada/exp_aggr.adb
+++ b/gcc/ada/exp_aggr.adb
@@ -1422,8 +1422,11 @@ package body Exp_Aggr is
                elsif Is_Mutably_Tagged_Type (Comp_Typ)
                  and then Nkind (Expr) = N_Qualified_Expression
                then
-                  Analyze_And_Resolve (Expr_Q, Etype (Expr));
+                  --  Avoid class-wide expected type for aggregate
+                  --  (which would be rejected as illegal)
+                  --  if the aggregate is explicitly qualified.
 
+                  Analyze_And_Resolve (Expr_Q, Etype (Expr));
                else
                   Analyze_And_Resolve (Expr_Q, Comp_Typ);
                end if;
@@ -1457,54 +1460,12 @@ package body Exp_Aggr is
          end if;
 
          if Present (Expr) then
-
-            --  For mutably tagged abstract class-wide types, we rely on the
-            --  type of the initializing expression to initialize the tag of
-            --  each array component.
-
-            --  Generate:
-            --     expr_type!(Indexed_Comp) := expr;
-            --     expr_type!(Indexed_Comp)._tag := expr_type'Tag;
-
-            if Is_Mutably_Tagged_Type (Comp_Typ)
-              and then Is_Abstract_Type (Root_Type (Comp_Typ))
-            then
-               declare
-                  Expr_Type : Entity_Id;
-
-               begin
-                  if Nkind (Expr) in N_Has_Etype
-                    and then Present (Etype (Expr))
-                  then
-                     Expr_Type := Etype (Expr);
-
-                  elsif Nkind (Expr) = N_Qualified_Expression then
-                     Analyze (Subtype_Mark (Expr));
-                     Expr_Type := Etype (Subtype_Mark (Expr));
-
-                  --  Unsupported case
-
-                  else
-                     pragma Assert (False);
-                     raise Program_Error;
-                  end if;
-
-                  Initialize_Component
-                    (N          => N,
-                     Comp       => Unchecked_Convert_To (Expr_Type,
-                                     Indexed_Comp),
-                     Comp_Typ   => Expr_Type,
-                     Init_Expr  => Expr,
-                     Stmts      => Stmts);
-               end;
-            else
-               Initialize_Component
-                 (N          => N,
-                  Comp       => Indexed_Comp,
-                  Comp_Typ   => Comp_Typ,
-                  Init_Expr  => Expr,
-                  Stmts      => Stmts);
-            end if;
+            Initialize_Component
+              (N          => N,
+               Comp       => Indexed_Comp,
+               Comp_Typ   => Comp_Typ,
+               Init_Expr  => Expr,
+               Stmts      => Stmts);
 
          --  Ada 2005 (AI-287): In case of default initialized component, call
          --  the initialization subprogram associated with the component type.
@@ -1519,10 +1480,10 @@ package body Exp_Aggr is
 
          else
             --  For mutably tagged class-wide types, default initialization is
-            --  performed by the init procedure of their root type.
+            --  performed by the init procedure of their specific type.
 
             if Is_Mutably_Tagged_Type (Comp_Typ) then
-               Comp_Typ := Root_Type (Comp_Typ);
+               Comp_Typ := Find_Specific_Type (Comp_Typ);
             end if;
 
             if Present (Base_Init_Proc (Comp_Typ)) then
@@ -4388,6 +4349,7 @@ package body Exp_Aggr is
         and then Is_Limited_Type (Typ)
       then
          Target_Expr := New_Copy_Tree (Name (Parent_Node));
+         Ensure_Defined (Typ, Parent_Node);
          Insert_Actions (Parent_Node,
            Build_Record_Aggr_Code (N, Typ, Target_Expr));
          Rewrite (Parent_Node, Make_Null_Statement (Loc));
@@ -4413,6 +4375,7 @@ package body Exp_Aggr is
 
             if Nkind (N) in N_Aggregate | N_Extension_Aggregate then
                Target_Expr := New_Copy_Tree (Lhs);
+               Ensure_Defined (Typ, Parent_Node);
                Insert_Actions (Parent_Node,
                  Build_Record_Aggr_Code (N, Typ, Target_Expr));
                Rewrite (Parent_Node, Make_Null_Statement (Loc));
@@ -6771,6 +6734,7 @@ package body Exp_Aggr is
       function Build_Size_Expr (Comp : Node_Id) return Node_Id is
          Lo, Hi       : Node_Id;
          It           : Node_Id;
+         It_Subt      : Entity_Id;
          Siz_Exp      : Node_Id := Empty;
          Choice       : Node_Id;
          Temp_Siz_Exp : Node_Id;
@@ -6845,20 +6809,22 @@ package body Exp_Aggr is
          elsif Nkind (Comp) = N_Iterated_Component_Association then
             if Present (Iterator_Specification (Comp)) then
 
-               --  If the static size of the iterable object is known,
+               --  If the size of the iterable object can be determined,
                --  attempt to return it.
 
                It := Name (Iterator_Specification (Comp));
                Preanalyze (It);
 
-               --  Handle the simplest cases for now where It denotes an array
-               --  object.
+               It_Subt := Etype (It);
+
+               --  Handle the simplest cases for now, where It denotes an array
+               --  object or a container object.
 
                if Nkind (It) in N_Identifier
-                 and then Ekind (Etype (It)) = E_Array_Subtype
+                 and then Ekind (It_Subt) = E_Array_Subtype
                then
                   declare
-                     Idx_N : Node_Id := First_Index (Etype (It));
+                     Idx_N : Node_Id := First_Index (It_Subt);
                      Siz_Exp : Node_Id := Empty;
                   begin
                      while Present (Idx_N) loop
@@ -6892,6 +6858,96 @@ package body Exp_Aggr is
 
                      return Siz_Exp;
                   end;
+
+               --  Case of iterating over a container object. Note that this
+               --  must be a simple object, and not something like a function
+               --  call (which might have side effects, and we wouldn't want
+               --  it to be evaluated more than once). We take advantage of
+               --  RM22 4.3.5(40/5), which allows implementation-defined
+               --  behavior for the parameter passed to the Empty function,
+               --  and here use the container Length function when available.
+               --  Class-wide objects are also excluded, since those would
+               --  lead to dispatching, which could call a user-defined
+               --  overriding of Length that might have arbitrary effects.
+
+               elsif Is_Entity_Name (It)
+                 and then Is_Object (Entity (It))
+                 and then Ekind (It_Subt) in Record_Kind
+                 and then not Is_Class_Wide_Type (It_Subt)
+               then
+                  declare
+                     Aggr_Base    : constant Entity_Id := Base_Type (Typ);
+                     It_Base      : constant Entity_Id := Base_Type (It_Subt);
+                     Empty_Formal : constant Entity_Id :=
+                                      First_Formal (Entity (Empty_Subp));
+                     Length_Subp  : Entity_Id;
+                     Param_List   : List_Id;
+
+                  begin
+                     --  We only determine a nondefault capacity in the case
+                     --  of containers of predefined container types, which
+                     --  generally have a Length function. User-defined
+                     --  containers don't necessarily have such a function,
+                     --  or it may be named differently, or it may have
+                     --  the wrong semantics. The base subtypes are tested,
+                     --  since their Sloc will refer to the original container
+                     --  generics in the predefined library, even though the
+                     --  types are declared in a package instantiation in some
+                     --  other unit. Also, this is only done when Empty_Subp
+                     --  has a formal parameter (generally named Capacity),
+                     --  and not in the case of a parameterless Empty function.
+                     --  Finally, we test for the container aggregate's type
+                     --  having a first discriminant with the name Capacity,
+                     --  since determining capacity via Length is only sensible
+                     --  for container types with that discriminant (bounded
+                     --  containers).
+
+                     if Present (Empty_Formal)
+                       and then In_Predefined_Unit (It_Base)
+                       and then In_Predefined_Unit (Aggr_Base)
+                       and then Has_Discriminants (Aggr_Base)
+                       and then
+                         Get_Name_String
+                           (Chars (First_Discriminant (Aggr_Base)))
+                           = "capacity"
+                     then
+                        --  Look for the container type's Length function in
+                        --  the package where it's defined.
+
+                        Push_Scope (Scope (It_Base));
+
+                        Length_Subp := Current_Entity_In_Scope (Name_Length);
+
+                        Pop_Scope;
+
+                        --  If we found a Length function that has a single
+                        --  parameter of the iterator object's container type,
+                        --  then expand a call to that, passing the object,
+                        --  and return that call, which will be used as the
+                        --  "size" of the current element association of the
+                        --  bounded container aggregate.
+
+                        if Present (Length_Subp)
+                          and then Ekind (Length_Subp) = E_Function
+                          and then
+                            Present (First_Entity (Length_Subp))
+                          and then
+                            No (Next_Entity (First_Entity (Length_Subp)))
+                          and then
+                            Base_Type
+                              (Etype (First_Entity (Length_Subp))) = It_Base
+                        then
+                           Param_List :=
+                             New_List (New_Occurrence_Of (Entity (It), Loc));
+
+                           return
+                             Make_Function_Call (Loc,
+                               Name =>
+                                 New_Occurrence_Of (Length_Subp, Loc),
+                               Parameter_Associations => Param_List);
+                        end if;
+                     end if;
+                  end;
                end if;
 
                return Empty;
@@ -8864,7 +8920,15 @@ package body Exp_Aggr is
       else
          Set_No_Ctrl_Actions (Init_Stmt);
 
-         if Tagged_Type_Expansion and then Is_Tagged_Type (Comp_Typ) then
+         if Tagged_Type_Expansion
+           and then Is_Tagged_Type (Comp_Typ)
+
+         --  Cannot adjust the tag when the expected type of the component is
+         --  a mutably tagged (and therefore class-wide) type; each component
+         --  of the aggregate has the tag of its initializing expression.
+
+           and then not Is_Mutably_Tagged_Type (Comp_Typ)
+         then
             declare
                Typ : Entity_Id := Underlying_Type (Comp_Typ);
 
diff --git a/gcc/ada/exp_attr.adb b/gcc/ada/exp_attr.adb
index 4f9f16c..810248d 100644
--- a/gcc/ada/exp_attr.adb
+++ b/gcc/ada/exp_attr.adb
@@ -1915,6 +1915,15 @@ package body Exp_Attr is
       --  call to the appropriate TSS procedure. Pname is the entity for the
       --  procedure to call.
 
+      procedure Read_Controlling_Tag
+        (P_Type : Entity_Id; Cntrl : out Node_Id);
+      --  Read the external tag from the stream and use it to construct the
+      --  controlling operand for a dispatching call.
+
+      procedure Write_Controlling_Tag (P_Type : Entity_Id);
+      --  Write the external tag of the given attribute prefix type to
+      --  the stream. Also perform the accompanying accessibility check.
+
       -------------------------------------
       -- Build_And_Insert_Type_Attr_Subp --
       -------------------------------------
@@ -2175,6 +2184,153 @@ package body Exp_Attr is
          Analyze (N);
       end Rewrite_Attribute_Proc_Call;
 
+      --------------------------
+      -- Read_Controlling_Tag --
+      --------------------------
+
+      procedure Read_Controlling_Tag
+        (P_Type : Entity_Id; Cntrl : out Node_Id)
+      is
+         Strm    : constant Node_Id := First (Exprs);
+         Expr    : Node_Id; -- call to Descendant_Tag
+         Get_Tag : Node_Id; -- expression to read the 'Tag
+
+      begin
+         --  Read the internal tag (RM 13.13.2(34)) and use it to
+         --  initialize a dummy tag value. We used to unconditionally
+         --  generate:
+         --
+         --     Descendant_Tag (String'Input (Strm), P_Type);
+         --
+         --  which turns into a call to String_Input_Blk_IO. However,
+         --  if the input is malformed, that could try to read an
+         --  enormous String, causing chaos. So instead we call
+         --  String_Input_Tag, which does the same thing as
+         --  String_Input_Blk_IO, except that if the String is
+         --  absurdly long, it raises an exception.
+         --
+         --  However, if the No_Stream_Optimizations restriction
+         --  is active, we disable this unnecessary attempt at
+         --  robustness; we really need to read the string
+         --  character-by-character.
+         --
+         --  This value is used only to provide a controlling
+         --  argument for the eventual _Input call. Descendant_Tag is
+         --  called rather than Internal_Tag to ensure that we have a
+         --  tag for a type that is descended from the prefix type and
+         --  declared at the same accessibility level (the exception
+         --  Tag_Error will be raised otherwise). The level check is
+         --  required for Ada 2005 because tagged types can be
+         --  extended in nested scopes (AI-344).
+
+         --  Note: we used to generate an explicit declaration of a
+         --  constant Ada.Tags.Tag object, and use an occurrence of
+         --  this constant in Cntrl, but this caused a secondary stack
+         --  leak.
+
+         if Restriction_Active (No_Stream_Optimizations) then
+            Get_Tag :=
+              Make_Attribute_Reference (Loc,
+                Prefix         =>
+                  New_Occurrence_Of (Standard_String, Loc),
+                Attribute_Name => Name_Input,
+                Expressions    => New_List (
+                  Relocate_Node (Duplicate_Subexpr (Strm))));
+         else
+            Get_Tag :=
+              Make_Function_Call (Loc,
+                Name                   =>
+                  New_Occurrence_Of
+                    (RTE (RE_String_Input_Tag), Loc),
+                Parameter_Associations => New_List (
+                  Relocate_Node (Duplicate_Subexpr (Strm))));
+         end if;
+
+         Expr :=
+           Make_Function_Call (Loc,
+             Name                   =>
+               New_Occurrence_Of (RTE (RE_Descendant_Tag), Loc),
+             Parameter_Associations => New_List (
+               Get_Tag,
+               Make_Attribute_Reference (Loc,
+                 Prefix         => New_Occurrence_Of (P_Type, Loc),
+                 Attribute_Name => Name_Tag)));
+
+         Set_Etype (Expr, RTE (RE_Tag));
+
+         --  Construct a controlling operand for a dispatching call.
+
+         Cntrl := Unchecked_Convert_To (P_Type, Expr);
+         Set_Etype (Cntrl, P_Type);
+         Set_Parent (Cntrl, N);
+      end Read_Controlling_Tag;
+
+      ----------------------------
+      --  Write_Controlling_Tag --
+      ----------------------------
+
+      procedure Write_Controlling_Tag (P_Type : Entity_Id) is
+         Strm : constant Node_Id := First (Exprs);
+         Item : constant Node_Id := Next (Strm);
+      begin
+         --  Ada 2005 (AI-344): Check that the accessibility level
+         --  of the type of the output object is not deeper than
+         --  that of the attribute's prefix type.
+
+         --  if Get_Access_Level (Item'Tag)
+         --       /= Get_Access_Level (P_Type'Tag)
+         --  then
+         --     raise Tag_Error;
+         --  end if;
+
+         --  String'Output (Strm, External_Tag (Item'Tag));
+
+         --  We cannot figure out a practical way to implement this
+         --  accessibility check on virtual machines, so we omit it.
+
+         if Ada_Version >= Ada_2005
+           and then Tagged_Type_Expansion
+         then
+            Insert_Action (N,
+              Make_Implicit_If_Statement (N,
+                Condition =>
+                  Make_Op_Ne (Loc,
+                    Left_Opnd  =>
+                      Build_Get_Access_Level (Loc,
+                        Make_Attribute_Reference (Loc,
+                          Prefix         =>
+                            Relocate_Node (
+                              Duplicate_Subexpr (Item,
+                                Name_Req => True)),
+                          Attribute_Name => Name_Tag)),
+
+                    Right_Opnd =>
+                      Make_Integer_Literal (Loc,
+                        Type_Access_Level (P_Type))),
+
+                Then_Statements =>
+                  New_List (Make_Raise_Statement (Loc,
+                              New_Occurrence_Of (
+                                RTE (RE_Tag_Error), Loc)))));
+         end if;
+
+         Insert_Action (N,
+           Make_Attribute_Reference (Loc,
+             Prefix => New_Occurrence_Of (Standard_String, Loc),
+             Attribute_Name => Name_Output,
+             Expressions => New_List (
+               Relocate_Node (Duplicate_Subexpr (Strm)),
+               Make_Function_Call (Loc,
+                 Name =>
+                   New_Occurrence_Of (RTE (RE_External_Tag), Loc),
+                 Parameter_Associations => New_List (
+                  Make_Attribute_Reference (Loc,
+                    Prefix =>
+                      Relocate_Node
+                        (Duplicate_Subexpr (Item, Name_Req => True)),
+                    Attribute_Name => Name_Tag))))));
+      end Write_Controlling_Tag;
+
       Typ  : constant Entity_Id    := Etype (N);
       Btyp : constant Entity_Id    := Base_Type (Typ);
       Ptyp : constant Entity_Id    := Etype (Pref);
@@ -4487,94 +4643,57 @@ package body Exp_Attr is
 
             elsif Is_Class_Wide_Type (P_Type) then
 
-               --  No need to do anything else compiling under restriction
-               --  No_Dispatching_Calls. During the semantic analysis we
-               --  already notified such violation.
+               if Is_Mutably_Tagged_Type (P_Type) then
 
-               if Restriction_Active (No_Dispatching_Calls) then
-                  return;
-               end if;
+                  --  In mutably tagged case, rewrite
+                  --    T'Class'Input (Strm)
+                  --  as (roughly)
+                  --    declare
+                  --       Result : T'Class;
+                  --       T'Class'Read (Strm, Result);
+                  --    begin
+                  --      Result;
+                  --    end;
 
-               declare
-                  Rtyp : constant Entity_Id := Root_Type (P_Type);
+                  declare
+                     Result_Temp : constant Entity_Id :=
+                       Make_Temporary (Loc, 'I');
 
-                  Expr    : Node_Id; -- call to Descendant_Tag
-                  Get_Tag : Node_Id; -- expression to read the 'Tag
+                     --  Gets default initialization
+                     Result_Temp_Decl : constant Node_Id :=
+                       Make_Object_Declaration (Loc,
+                         Defining_Identifier => Result_Temp,
+                         Object_Definition =>
+                           New_Occurrence_Of (P_Type, Loc));
 
-               begin
-                  --  Read the internal tag (RM 13.13.2(34)) and use it to
-                  --  initialize a dummy tag value. We used to unconditionally
-                  --  generate:
-                  --
-                  --     Descendant_Tag (String'Input (Strm), P_Type);
-                  --
-                  --  which turns into a call to String_Input_Blk_IO. However,
-                  --  if the input is malformed, that could try to read an
-                  --  enormous String, causing chaos. So instead we call
-                  --  String_Input_Tag, which does the same thing as
-                  --  String_Input_Blk_IO, except that if the String is
-                  --  absurdly long, it raises an exception.
-                  --
-                  --  However, if the No_Stream_Optimizations restriction
-                  --  is active, we disable this unnecessary attempt at
-                  --  robustness; we really need to read the string
-                  --  character-by-character.
-                  --
-                  --  This value is used only to provide a controlling
-                  --  argument for the eventual _Input call. Descendant_Tag is
-                  --  called rather than Internal_Tag to ensure that we have a
-                  --  tag for a type that is descended from the prefix type and
-                  --  declared at the same accessibility level (the exception
-                  --  Tag_Error will be raised otherwise). The level check is
-                  --  required for Ada 2005 because tagged types can be
-                  --  extended in nested scopes (AI-344).
-
-                  --  Note: we used to generate an explicit declaration of a
-                  --  constant Ada.Tags.Tag object, and use an occurrence of
-                  --  this constant in Cntrl, but this caused a secondary stack
-                  --  leak.
-
-                  if Restriction_Active (No_Stream_Optimizations) then
-                     Get_Tag :=
-                       Make_Attribute_Reference (Loc,
-                         Prefix         =>
-                           New_Occurrence_Of (Standard_String, Loc),
-                         Attribute_Name => Name_Input,
-                         Expressions    => New_List (
-                           Relocate_Node (Duplicate_Subexpr (Strm))));
-                  else
-                     Get_Tag :=
-                       Make_Function_Call (Loc,
-                         Name                   =>
-                           New_Occurrence_Of
-                             (RTE (RE_String_Input_Tag), Loc),
-                         Parameter_Associations => New_List (
-                           Relocate_Node (Duplicate_Subexpr (Strm))));
-                  end if;
+                     function Result_Temp_Name return Node_Id is
+                       (New_Occurrence_Of (Result_Temp, Loc));
 
-                  Expr :=
-                    Make_Function_Call (Loc,
-                      Name                   =>
-                        New_Occurrence_Of (RTE (RE_Descendant_Tag), Loc),
-                      Parameter_Associations => New_List (
-                        Get_Tag,
-                        Make_Attribute_Reference (Loc,
-                          Prefix         => New_Occurrence_Of (P_Type, Loc),
-                          Attribute_Name => Name_Tag)));
+                     Actions : constant List_Id := New_List (
+                       Result_Temp_Decl,
+                       Make_Attribute_Reference (Loc,
+                         Prefix => New_Occurrence_Of (P_Type, Loc),
+                         Attribute_Name => Name_Read,
+                         Expressions => New_List (
+                           Relocate_Node (Strm), Result_Temp_Name)));
+                  begin
+                     Rewrite (N, Make_Expression_With_Actions (Loc,
+                                   Actions, Result_Temp_Name));
+                     Analyze_And_Resolve (N, P_Type);
+                     return;
+                  end;
+               end if;
 
-                  Set_Etype (Expr, RTE (RE_Tag));
+               --  No need to do anything else compiling under restriction
+               --  No_Dispatching_Calls. During the semantic analysis we
+               --  already notified such violation.
 
-                  --  Now we need to get the entity for the call, and construct
-                  --  a function call node, where we preset a reference to Dnn
-                  --  as the controlling argument (doing an unchecked convert
-                  --  to the class-wide tagged type to make it look like a real
-                  --  tagged object).
+               if Restriction_Active (No_Dispatching_Calls) then
+                  return;
+               end if;
 
-                  Fname := Find_Prim_Op (Rtyp, TSS_Stream_Input);
-                  Cntrl := Unchecked_Convert_To (P_Type, Expr);
-                  Set_Etype (Cntrl, P_Type);
-                  Set_Parent (Cntrl, N);
-               end;
+               Read_Controlling_Tag (P_Type, Cntrl);
+               Fname := Find_Prim_Op (Root_Type (P_Type), TSS_Stream_Input);
 
             --  For tagged types, use the primitive Input function
 
@@ -5957,6 +6076,14 @@ package body Exp_Attr is
                      Attr_Ref => N);
                end;
 
+            --  In the mutably tagged case, T'Class'Output calls T'Class'Write;
+            --  T'Write will take care of writing out the external tag.
+
+            elsif Is_Mutably_Tagged_Type (P_Type) then
+               Set_Attribute_Name (N, Name_Write);
+               Analyze (N);
+               return;
+
             --  Class-wide case, first output external tag, then dispatch
             --  to the appropriate primitive Output function (RM 13.13.2(31)).
 
@@ -5970,68 +6097,7 @@ package body Exp_Attr is
                   return;
                end if;
 
-               Tag_Write : declare
-                  Strm : constant Node_Id := First (Exprs);
-                  Item : constant Node_Id := Next (Strm);
-
-               begin
-                  --  Ada 2005 (AI-344): Check that the accessibility level
-                  --  of the type of the output object is not deeper than
-                  --  that of the attribute's prefix type.
-
-                  --  if Get_Access_Level (Item'Tag)
-                  --       /= Get_Access_Level (P_Type'Tag)
-                  --  then
-                  --     raise Tag_Error;
-                  --  end if;
-
-                  --  String'Output (Strm, External_Tag (Item'Tag));
-
-                  --  We cannot figure out a practical way to implement this
-                  --  accessibility check on virtual machines, so we omit it.
-
-                  if Ada_Version >= Ada_2005
-                    and then Tagged_Type_Expansion
-                  then
-                     Insert_Action (N,
-                       Make_Implicit_If_Statement (N,
-                         Condition =>
-                           Make_Op_Ne (Loc,
-                             Left_Opnd  =>
-                               Build_Get_Access_Level (Loc,
-                                 Make_Attribute_Reference (Loc,
-                                   Prefix         =>
-                                     Relocate_Node (
-                                       Duplicate_Subexpr (Item,
-                                         Name_Req => True)),
-                                   Attribute_Name => Name_Tag)),
-
-                             Right_Opnd =>
-                               Make_Integer_Literal (Loc,
-                                 Type_Access_Level (P_Type))),
-
-                         Then_Statements =>
-                           New_List (Make_Raise_Statement (Loc,
-                                       New_Occurrence_Of (
-                                         RTE (RE_Tag_Error), Loc)))));
-                  end if;
-
-                  Insert_Action (N,
-                    Make_Attribute_Reference (Loc,
-                      Prefix => New_Occurrence_Of (Standard_String, Loc),
-                      Attribute_Name => Name_Output,
-                      Expressions => New_List (
-                        Relocate_Node (Duplicate_Subexpr (Strm)),
-                        Make_Function_Call (Loc,
-                          Name =>
-                            New_Occurrence_Of (RTE (RE_External_Tag), Loc),
-                          Parameter_Associations => New_List (
-                           Make_Attribute_Reference (Loc,
-                             Prefix =>
-                               Relocate_Node
-                                 (Duplicate_Subexpr (Item, Name_Req => True)),
-                             Attribute_Name => Name_Tag))))));
-               end Tag_Write;
+               Write_Controlling_Tag (P_Type);
 
                Pname := Find_Prim_Op (U_Type, TSS_Stream_Output);
 
@@ -6793,6 +6859,7 @@ package body Exp_Attr is
          P_Type  : constant Entity_Id := Entity (Pref);
          B_Type  : constant Entity_Id := Base_Type (P_Type);
          U_Type  : constant Entity_Id := Underlying_Type (P_Type);
+         Cntrl   : Node_Id := Empty; -- nonempty only if P_Type mutably tagged
          Pname   : Entity_Id;
          Decl    : Node_Id;
          Prag    : Node_Id;
@@ -6941,6 +7008,11 @@ package body Exp_Attr is
             --  this will dispatch in the class-wide case which is what we want
 
             elsif Is_Tagged_Type (U_Type) then
+
+               if Is_Mutably_Tagged_Type (U_Type) then
+                  Read_Controlling_Tag (P_Type, Cntrl);
+               end if;
+
                Pname := Find_Prim_Op (U_Type, TSS_Stream_Read);
 
             --  All other record type cases, including protected records. The
@@ -7001,6 +7073,46 @@ package body Exp_Attr is
 
          Rewrite_Attribute_Proc_Call (Pname);
 
+         if Present (Cntrl) then
+            pragma Assert (Is_Mutably_Tagged_Type (U_Type));
+            pragma Assert (Nkind (N) = N_Procedure_Call_Statement);
+
+            --  Assign the Tag value that was read from the stream
+            --  to the tag of the out-mode actual parameter so that
+            --  we dispatch correctly. This isn't quite right.
+            --  We should assign a complete object (not just
+            --  the tag), but that would require a dispatching call to
+            --  perform default initialization of the source object and
+            --  dispatching default init calls are currently not supported.
+
+            declare
+               function Select_Tag (Prefix : Node_Id) return Node_Id is
+                 (Make_Selected_Component (Loc,
+                    Prefix => Prefix,
+                    Selector_Name =>
+                      New_Occurrence_Of (First_Tag_Component
+                                           (Etype (Prefix)), Loc)));
+
+               Controlling_Actual : constant Node_Id :=
+                 Next (First (Parameter_Associations (N)));
+
+               pragma Assert (Is_Controlling_Actual (Controlling_Actual));
+
+               Assign_Tag : Node_Id;
+            begin
+               Remove_Side_Effects (Controlling_Actual, Name_Req => True);
+
+               Assign_Tag :=
+                 Make_Assignment_Statement (Loc,
+                   Name =>
+                     Select_Tag (New_Copy_Tree (Controlling_Actual)),
+                   Expression => Select_Tag (Cntrl));
+
+               Insert_Before (Before => N, Node => Assign_Tag);
+               Analyze (Assign_Tag);
+            end;
+         end if;
+
          if not Is_Tagged_Type (P_Type) then
             Cached_Attribute_Ops.Read_Map.Set (U_Type, Pname);
          end if;
@@ -8611,6 +8723,14 @@ package body Exp_Attr is
             --  this will dispatch in the class-wide case which is what we want
 
             elsif Is_Tagged_Type (U_Type) then
+
+               --  If T'Class is mutably tagged, then the external tag
+               --  is written out by T'Class'Write, not by T'Class'Output.
+
+               if Is_Mutably_Tagged_Type (U_Type) then
+                  Write_Controlling_Tag (P_Type);
+               end if;
+
                Pname := Find_Prim_Op (U_Type, TSS_Stream_Write);
 
             --  All other record type cases, including protected records.
diff --git a/gcc/ada/exp_ch3.adb b/gcc/ada/exp_ch3.adb
index 5a47a5a..6cf7c9c 100644
--- a/gcc/ada/exp_ch3.adb
+++ b/gcc/ada/exp_ch3.adb
@@ -942,10 +942,11 @@ package body Exp_Ch3 is
                Make_Handled_Sequence_Of_Statements (Loc,
                  Statements => Body_Stmts)));
 
-         Mutate_Ekind       (Proc_Id, E_Procedure);
-         Set_Is_Public      (Proc_Id, Is_Public (A_Type));
-         Set_Is_Internal    (Proc_Id);
-         Set_Has_Completion (Proc_Id);
+         Mutate_Ekind         (Proc_Id, E_Procedure);
+         Set_Is_Public        (Proc_Id, Is_Public (A_Type));
+         Set_Is_Internal      (Proc_Id);
+         Set_Has_Completion   (Proc_Id);
+         Freeze_Extra_Formals (Proc_Id);
 
          if not Debug_Generated_Code then
             Set_Debug_Info_Off (Proc_Id);
@@ -3204,6 +3205,7 @@ package body Exp_Ch3 is
          end if;
 
          Set_Parameter_Specifications (Proc_Spec_Node, Parameters);
+         Freeze_Extra_Formals (Proc_Id);
          Set_Specification (Body_Node, Proc_Spec_Node);
          Set_Declarations (Body_Node, Decls);
 
@@ -6496,7 +6498,7 @@ package body Exp_Ch3 is
          end;
       end if;
 
-      if Has_Controlled_Component (Typ) then
+      if Has_Controlled_Component (Typ) or else Has_Destructor (Typ) then
          Build_Controlling_Procs (Typ);
       end if;
 
@@ -6570,17 +6572,16 @@ package body Exp_Ch3 is
       --  procedure, because a self-referential type might call one of these
       --  primitives in the body of the init_proc itself.
       --
-      --  This is not needed:
-      --    1) If expansion is disabled, because extra formals are only added
-      --       when we are generating code.
+      --  This is not needed when expansion is disabled, because extra formals
+      --  are only added when we are generating code.
       --
-      --    2) For types with foreign convention since primitives with foreign
-      --       convention don't have extra formals and AI95-117 requires that
-      --       all primitives of a tagged type inherit the convention.
+      --  Notice that for tagged types with foreign convention this is also
+      --  required because (although primitives with foreign convention don't
+      --  have extra formals), a tagged type with foreign convention may have
+      --  primitives with convention Ada.
 
       if Expander_Active
         and then Is_Tagged_Type (Typ)
-        and then not Has_Foreign_Convention (Typ)
       then
          declare
             Elmt : Elmt_Id;
@@ -7500,6 +7501,12 @@ package body Exp_Ch3 is
                Apply_CW_Accessibility_Check (Expr, Func_Id);
             end if;
 
+            if Has_Anonymous_Access_Discriminant (Etype (Expr)) then
+               --  Check that access discrims do not designate entities
+               --  that the function result could outlive.
+               Apply_Access_Discrims_Accessibility_Check (Expr, Func_Id);
+            end if;
+
             Alloc_Expr := New_Copy_Tree (Expr);
 
             if Etype (Alloc_Expr) /= Alloc_Typ then
@@ -9058,6 +9065,10 @@ package body Exp_Ch3 is
                if Is_Class_Wide_Type (Etype (Func_Id)) then
                   Apply_CW_Accessibility_Check (Expr_Q, Func_Id);
                end if;
+
+               --  ??? Usually calls to Apply_CW_Accessibility_Check and to
+               --  Apply_Access_Discrims_Accessibility_Check come in pairs.
+               --  Do we need a (conditional) call here to A_A_D_A_C ?
             end;
          end if;
 
@@ -12845,25 +12856,27 @@ package body Exp_Ch3 is
             Append_To (Res, Decl);
          end if;
 
-         Fin_Call := Empty;
-         Decl     := Predef_Deep_Spec (Loc, Tag_Typ, TSS_Deep_Finalize, True);
+         if not Has_Destructor (Tag_Typ) then
+            Fin_Call := Empty;
+            Decl := Predef_Deep_Spec (Loc, Tag_Typ, TSS_Deep_Finalize, True);
 
-         if Is_Controlled (Tag_Typ) then
-            Fin_Call :=
-              Make_Final_Call
-                (Obj_Ref => Make_Identifier (Loc, Name_V),
-                 Typ     => Tag_Typ);
-         end if;
+            if Is_Controlled (Tag_Typ) then
+               Fin_Call :=
+                 Make_Final_Call
+                   (Obj_Ref => Make_Identifier (Loc, Name_V), Typ => Tag_Typ);
+            end if;
 
-         if No (Fin_Call) then
-            Fin_Call := Make_Null_Statement (Loc);
-         end if;
+            if No (Fin_Call) then
+               Fin_Call := Make_Null_Statement (Loc);
+            end if;
 
-         Set_Handled_Statement_Sequence (Decl,
-           Make_Handled_Sequence_Of_Statements (Loc,
-             Statements => New_List (Fin_Call)));
+            Set_Handled_Statement_Sequence
+              (Decl,
+               Make_Handled_Sequence_Of_Statements
+                 (Loc, Statements => New_List (Fin_Call)));
 
-         Append_To (Res, Decl);
+            Append_To (Res, Decl);
+         end if;
       end if;
 
       return Res;
diff --git a/gcc/ada/exp_ch4.adb b/gcc/ada/exp_ch4.adb
index 76386fc..43c94f3 100644
--- a/gcc/ada/exp_ch4.adb
+++ b/gcc/ada/exp_ch4.adb
@@ -2354,6 +2354,7 @@ package body Exp_Ch4 is
       Rhs        : Node_Id) return Node_Id
    is
       Loc       : constant Source_Ptr := Sloc (Nod);
+      CW_Comp   : Boolean := False;
       Full_Type : Entity_Id;
       Eq_Op     : Entity_Id;
 
@@ -2383,10 +2384,17 @@ package body Exp_Ch4 is
          Full_Type := Underlying_Type (Full_Type);
       end if;
 
+      if Is_Class_Wide_Equivalent_Type (Full_Type) then
+         CW_Comp := True;
+         Full_Type :=
+           Get_Corresponding_Mutably_Tagged_Type_If_Present (Full_Type);
+         pragma Assert (Is_Tagged_Type (Full_Type));
+      end if;
+
       --  Case of tagged record types
 
       if Is_Tagged_Type (Full_Type) then
-         Eq_Op := Find_Primitive_Eq (Comp_Type);
+         Eq_Op := Find_Primitive_Eq (if CW_Comp then Full_Type else Comp_Type);
          pragma Assert (Present (Eq_Op));
 
          return
diff --git a/gcc/ada/exp_ch6.adb b/gcc/ada/exp_ch6.adb
index c24c8c6..eb7422c 100644
--- a/gcc/ada/exp_ch6.adb
+++ b/gcc/ada/exp_ch6.adb
@@ -734,6 +734,258 @@ package body Exp_Ch6 is
       Add_Extra_Actual_To_Call (Function_Call, Chain_Formal, Chain_Actual);
    end Add_Task_Actuals_To_Build_In_Place_Call;
 
+   ----------------------------------------------
+   -- Apply_Access_Discrims_Accesibility_Check --
+   ----------------------------------------------
+
+   procedure Apply_Access_Discrims_Accessibility_Check
+     (Exp : Node_Id; Func : Entity_Id)
+   is
+      Loc : constant Source_Ptr := Sloc (Exp);
+
+      --  Some of the code here in this procedure may need to be factored
+      --  out at some point because it seems like some of the same
+      --  functionality would be needed for accessibility checking of a
+      --  return statement when the function result type is an anonymous
+      --  access type (as opposed to a type that has an anonymous access
+      --  discriminant).
+      --
+      --  Another case that is not addressed today is the case where
+      --  we need to check an access discriminant subcomponent of the
+      --  function result other than a discriminant of the function result.
+      --  This can only happen if the function result type has an unconstrained
+      --  subcomponent subtype that has an access discriminant (which implies
+      --  that the function result type must be limited).
+      --
+      --  A further corner case of that corner case arises if the limited
+      --  function result type is class-wide and it is not known statically
+      --  that this access-discriminant-bearing subcomponent exists. The
+      --  easiest way to address this properly would probably involve adding
+      --  a new compiler-generated dispatching procedure; a dispatching call
+      --  could then be used to perform the check in a context where we know
+      --  statically the specific type of the function result. Finding a
+      --  less important unimplemented case would be challenging.
+
+      function Constraint_Bearing_Subtype_If_Any
+        (Exp : Node_Id) return Node_Id;
+      --  If we can locate a constrained subtype whose constraint applies
+      --  to Exp, then return that. Otherwise, return Etype (Exp).
+
+      function Discr_Expression
+        (Typ : Entity_Id; Discr_Index : Positive) return Node_Id;
+      --  Typ is a constrained discriminated subtype.
+      --  Return the constraint expression for the indexed discriminant.
+
+      function Has_Level_Tied_To_Explicitly_Aliased_Param
+        (Constraint_Exp : Node_Id) return Boolean;
+      --  Constraint_Exp is the value given for an access discriminant
+      --  in a discriminant constraint for Exp. Return True iff the
+      --  accessibility of the type of that discriminant of Exp is the level
+      --  of an explicitly aliased parameter of Func. If true, this indicates
+      --  that no check should be performed for this discriminant.
+
+      ---------------------------------------
+      -- Constraint_Bearing_Subtype_If_Any --
+      ---------------------------------------
+
+      function Constraint_Bearing_Subtype_If_Any
+        (Exp : Node_Id) return Entity_Id
+      is
+         Result : Entity_Id := Etype (Exp);
+      begin
+         if Is_Constrained (Result) then
+            return Result;
+         end if;
+
+         --  Look through expansion-generated levels of indirection
+         --  to find a constrained subtype. Yuck. This comes up in
+         --  some cases when the unexpanded source returns an aggregate.
+
+         if Nkind (Exp) = N_Explicit_Dereference
+           and then Nkind (Prefix (Exp)) = N_Identifier
+           and then Ekind (Entity (Prefix (Exp))) = E_Constant
+         then
+            declare
+               Acc_Const       : Entity_Id := Entity (Prefix (Exp));
+               Acc_Const_Value : Node_Id := Empty;
+            begin
+               --  look through constants initialized to constants
+               loop
+                  exit when Nkind (Parent (Acc_Const)) /= N_Object_Declaration;
+
+                  Acc_Const_Value := Expression (Parent (Acc_Const));
+
+                  if Nkind (Acc_Const_Value) = N_Identifier
+                    and then Ekind (Entity (Acc_Const_Value)) = E_Constant
+                  then
+                     Acc_Const := Entity (Acc_Const_Value);
+                  else
+                     exit;
+                  end if;
+               end loop;
+
+               if Nkind (Acc_Const_Value) = N_Allocator
+                 and then Nkind (Expression (Acc_Const_Value))
+                             = N_Qualified_Expression
+               then
+                  Result :=
+                    Etype (Expression (Acc_Const_Value));
+               end if;
+            end;
+         end if;
+
+         if Is_Constrained (Result) then
+            return Result;
+         end if;
+
+         --  no constrained subtype found
+         return Etype (Exp);
+      end Constraint_Bearing_Subtype_If_Any;
+
+      ----------------------
+      -- Discr_Expression --
+      ----------------------
+
+      function Discr_Expression
+        (Typ : Entity_Id; Discr_Index : Positive) return Node_Id
+      is
+         Constraint_Elmt : Elmt_Id :=
+           First_Elmt (Discriminant_Constraint (Typ));
+      begin
+         for Skip in 1 .. Discr_Index - 1 loop
+            Next_Elmt (Constraint_Elmt);
+         end loop;
+         return Node (Constraint_Elmt);
+      end Discr_Expression;
+
+      -------------------------------------------------
+      --  Has_Level_Tied_To_Explicitly_Aliased_Param --
+      -------------------------------------------------
+
+      function Has_Level_Tied_To_Explicitly_Aliased_Param
+        (Constraint_Exp : Node_Id) return Boolean
+      is
+         Discr_Exp   : Node_Id := Constraint_Exp;
+         Attr_Prefix : Node_Id;
+      begin
+         --  look through constants
+         while Nkind (Discr_Exp) = N_Identifier
+           and then Ekind (Entity (Discr_Exp)) = E_Constant
+           and then Nkind (Parent (Entity (Discr_Exp))) = N_Object_Declaration
+         loop
+            Discr_Exp := Expression (Parent (Entity (Discr_Exp)));
+         end loop;
+
+         if Nkind (Discr_Exp) = N_Attribute_Reference
+           and then Get_Attribute_Id
+                      (Attribute_Name (Discr_Exp)) = Attribute_Access
+         then
+            Attr_Prefix := Ultimate_Prefix (Prefix (Discr_Exp));
+            if Is_Entity_Name (Attr_Prefix)
+              and then Is_Explicitly_Aliased (Entity (Attr_Prefix))
+              and then Scope (Entity (Attr_Prefix)) = Func
+            then
+               return True;
+            end if;
+         end if;
+
+         return False;
+      end Has_Level_Tied_To_Explicitly_Aliased_Param;
+
+      Discr       : Entity_Id := First_Discriminant (Etype (Exp));
+      Discr_Index : Positive  := 1;
+      Discr_Exp   : Node_Id;
+
+      Constrained_Subtype : constant Entity_Id :=
+        Constraint_Bearing_Subtype_If_Any (Exp);
+   begin
+      --  ??? Do not generate a check if version is Ada 95 (or earlier).
+      --  It is unclear whether this is really correct, or is just a stopgap
+      --  measure. Investigation is needed to decide how post-Ada-95 binding
+      --  interpretation changes in RM 3.10.2 should interact with Ada 95's
+      --  return-by-reference model for functions with limited result types
+      --  (which was abandoned in Ada 2005).
+
+      if Ada_Version <= Ada_95 then
+         return;
+      end if;
+
+      --  If we are returning a function call then that function will
+      --  perform the needed check.
+
+      if Nkind (Unqualify (Exp)) = N_Function_Call then
+         return;
+      end if;
+
+     --  ??? Cope with the consequences of the Disable_Tagged_Cases flag
+     --  in accessibility.adb (which can cause the extra formal parameter
+     --  needed for the check(s) generated here to be missing in the case
+     --  of a tagged result type); this is a workaround and can
+     --  prevent generation of a required check.
+
+      if No (Extra_Accessibility_Of_Result (Func)) then
+         return;
+      end if;
+
+      Remove_Side_Effects (Exp);
+
+      while Present (Discr) loop
+         if Is_Anonymous_Access_Type (Etype (Discr)) then
+            if Is_Constrained (Constrained_Subtype) then
+               Discr_Exp :=
+                 New_Copy_Tree
+                   (Discr_Expression (Constrained_Subtype, Discr_Index));
+            else
+               Discr_Exp :=
+                 Make_Selected_Component (Loc,
+                   Prefix => New_Copy_Tree (Exp),
+                   Selector_Name => New_Occurrence_Of (Discr, Loc));
+            end if;
+
+            if not Has_Level_Tied_To_Explicitly_Aliased_Param (Discr_Exp) then
+               declare
+                  --  We could do this min operation earlier, as is done
+                  --  for other implicit level parameters. Motivation for
+                  --  doing this min operation (earlier or not) is as for
+                  --  Generate_Minimum_Accessibility (see sem_ch6.adb):
+                  --  if a level value is too big, then the caller and the
+                  --  callee disagree about what it means.
+
+                  Level_Of_Master_Of_Call : constant Node_Id :=
+                    Make_Attribute_Reference (Loc,
+                      Prefix => New_Occurrence_Of (Standard_Natural, Loc),
+                      Attribute_Name => Name_Min,
+                      Expressions => New_List (
+                        Make_Integer_Literal (Loc, Scope_Depth (Func)),
+                        New_Occurrence_Of
+                          (Extra_Accessibility_Of_Result (Func), Loc)));
+
+                  Discrim_Level : Node_Id;
+               begin
+                  Analyze (Level_Of_Master_Of_Call);
+                  Analyze (Discr_Exp);
+
+                  Discrim_Level :=
+                    Accessibility_Level (Discr_Exp, Level => Dynamic_Level);
+                  Analyze (Discrim_Level);
+
+                  Insert_Action (Exp,
+                    Make_Raise_Program_Error (Loc,
+                      Condition =>
+                        Make_Op_Gt (Loc,
+                          Left_Opnd  => Discrim_Level,
+                          Right_Opnd => Level_Of_Master_Of_Call),
+                      Reason    => PE_Accessibility_Check_Failed),
+                    Suppress => Access_Check);
+               end;
+            end if;
+         end if;
+
+         Next_Discriminant (Discr);
+         Discr_Index := Discr_Index + 1;
+      end loop;
+   end Apply_Access_Discrims_Accessibility_Check;
+
    ----------------------------------
    -- Apply_CW_Accessibility_Check --
    ----------------------------------
@@ -1155,13 +1407,18 @@ package body Exp_Ch6 is
      (Subp_Call : Node_Id;
       Subp_Id   : Entity_Id) return Boolean
    is
-      Formal : Entity_Id;
+      use Deferred_Extra_Formals_Support;
+
       Actual : Node_Id;
+      Formal : Entity_Id;
 
    begin
       pragma Assert (Nkind (Subp_Call) in N_Entry_Call_Statement
                                         | N_Function_Call
                                         | N_Procedure_Call_Statement);
+      pragma Assert (Extra_Formals_Known (Subp_Id)
+        or else not Expander_Active
+        or else Is_Unsupported_Extra_Actuals_Call (Subp_Call, Subp_Id));
 
       --  In CodePeer_Mode, the tree for `'Elab_Spec` procedures will be
       --  malformed because GNAT does not perform the usual expansion that
@@ -2866,15 +3123,17 @@ package body Exp_Ch6 is
    -----------------
 
    procedure Expand_Call (N : Node_Id) is
-      function Is_Unchecked_Union_Equality (N : Node_Id) return Boolean;
+      function Is_Unchecked_Union_Predefined_Equality_Call
+        (N : Node_Id) return Boolean;
       --  Return True if N is a call to the predefined equality operator of an
       --  unchecked union type, or a renaming thereof.
 
-      ---------------------------------
-      -- Is_Unchecked_Union_Equality --
-      ---------------------------------
+      -------------------------------------------------
+      -- Is_Unchecked_Union_Predefined_Equality_Call --
+      -------------------------------------------------
 
-      function Is_Unchecked_Union_Equality (N : Node_Id) return Boolean is
+      function Is_Unchecked_Union_Predefined_Equality_Call
+        (N : Node_Id) return Boolean is
       begin
          if Is_Entity_Name (Name (N))
            and then Ekind (Entity (Name (N))) = E_Function
@@ -2899,7 +3158,7 @@ package body Exp_Ch6 is
          else
             return False;
          end if;
-      end Is_Unchecked_Union_Equality;
+      end Is_Unchecked_Union_Predefined_Equality_Call;
 
       --  If this is an indirect call through an Access_To_Subprogram
       --  with contract specifications, it is rewritten as a call to
@@ -2996,7 +3255,7 @@ package body Exp_Ch6 is
       --  Case of a call to the predefined equality operator of an unchecked
       --  union type, which requires specific processing.
 
-      elsif Is_Unchecked_Union_Equality (N) then
+      elsif Is_Unchecked_Union_Predefined_Equality_Call (N) then
          declare
             Eq : constant Entity_Id := Entity (Name (N));
 
@@ -3020,29 +3279,12 @@ package body Exp_Ch6 is
       end if;
    end Expand_Call;
 
-   ------------------------
-   -- Expand_Call_Helper --
-   ------------------------
-
-   --  This procedure handles expansion of function calls and procedure call
-   --  statements (i.e. it serves as the body for Expand_N_Function_Call and
-   --  Expand_N_Procedure_Call_Statement). Processing for calls includes:
-
-   --    Replace call to Raise_Exception by Raise_Exception_Always if possible
-   --    Provide values of actuals for all formals in Extra_Formals list
-   --    Replace "call" to enumeration literal function by literal itself
-   --    Rewrite call to predefined operator as operator
-   --    Replace actuals to in-out parameters that are numeric conversions,
-   --     with explicit assignment to temporaries before and after the call.
-
-   --   Note that the list of actuals has been filled with default expressions
-   --   during semantic analysis of the call. Only the extra actuals required
-   --   for the 'Constrained attribute and for accessibility checks are added
-   --   at this point.
+   --------------------------
+   -- Create_Extra_Actuals --
+   --------------------------
 
-   procedure Expand_Call_Helper (N : Node_Id; Post_Call : out List_Id) is
-      Loc           : constant Source_Ptr := Sloc (N);
-      Call_Node     : Node_Id := N;
+   procedure Create_Extra_Actuals (Call_Node : Node_Id) is
+      Loc           : constant Source_Ptr := Sloc (Call_Node);
       Extra_Actuals : List_Id := No_List;
       Prev          : Node_Id := Empty;
 
@@ -3072,88 +3314,6 @@ package body Exp_Ch6 is
       --  expression for the value of the actual, EF is the entity for the
       --  extra formal.
 
-      procedure Add_View_Conversion_Invariants
-        (Formal : Entity_Id;
-         Actual : Node_Id);
-      --  Adds invariant checks for every intermediate type between the range
-      --  of a view converted argument to its ancestor (from parent to child).
-
-      function Can_Fold_Predicate_Call (P : Entity_Id) return Boolean;
-      --  Try to constant-fold a predicate check, which often enough is a
-      --  simple arithmetic expression that can be computed statically if
-      --  its argument is static. This cleans up the output of CCG, even
-      --  though useless predicate checks will be generally removed by
-      --  back-end optimizations.
-
-      procedure Check_Subprogram_Variant;
-      --  Emit a call to the internally generated procedure with checks for
-      --  aspect Subprogram_Variant, if present and enabled.
-
-      function Inherited_From_Formal (S : Entity_Id) return Entity_Id;
-      --  Within an instance, a type derived from an untagged formal derived
-      --  type inherits from the original parent, not from the actual. The
-      --  current derivation mechanism has the derived type inherit from the
-      --  actual, which is only correct outside of the instance. If the
-      --  subprogram is inherited, we test for this particular case through a
-      --  convoluted tree traversal before setting the proper subprogram to be
-      --  called.
-
-      function In_Unfrozen_Instance (E : Entity_Id) return Boolean;
-      --  Return true if E comes from an instance that is not yet frozen
-
-      function Is_Class_Wide_Interface_Type (E : Entity_Id) return Boolean;
-      --  Return True when E is a class-wide interface type or an access to
-      --  a class-wide interface type.
-
-      function Is_Direct_Deep_Call (Subp : Entity_Id) return Boolean;
-      --  Determine if Subp denotes a non-dispatching call to a Deep routine
-
-      function New_Value (From : Node_Id) return Node_Id;
-      --  From is the original Expression. New_Value is equivalent to a call
-      --  to Duplicate_Subexpr with an explicit dereference when From is an
-      --  access parameter.
-
-      --------------------------
-      -- Add_Actual_Parameter --
-      --------------------------
-
-      procedure Add_Actual_Parameter (Insert_Param : Node_Id) is
-         Actual_Expr : constant Node_Id :=
-                         Explicit_Actual_Parameter (Insert_Param);
-
-      begin
-         --  Case of insertion is first named actual
-
-         if No (Prev) or else
-            Nkind (Parent (Prev)) /= N_Parameter_Association
-         then
-            Set_Next_Named_Actual
-              (Insert_Param, First_Named_Actual (Call_Node));
-            Set_First_Named_Actual (Call_Node, Actual_Expr);
-
-            if No (Prev) then
-               if No (Parameter_Associations (Call_Node)) then
-                  Set_Parameter_Associations (Call_Node, New_List);
-               end if;
-
-               Append (Insert_Param, Parameter_Associations (Call_Node));
-
-            else
-               Insert_After (Prev, Insert_Param);
-            end if;
-
-         --  Case of insertion is not first named actual
-
-         else
-            Set_Next_Named_Actual
-              (Insert_Param, Next_Named_Actual (Parent (Prev)));
-            Set_Next_Named_Actual (Parent (Prev), Actual_Expr);
-            Append (Insert_Param, Parameter_Associations (Call_Node));
-         end if;
-
-         Prev := Actual_Expr;
-      end Add_Actual_Parameter;
-
       --------------------------------------
       -- Add_Cond_Expression_Extra_Actual --
       --------------------------------------
@@ -3368,14 +3528,14 @@ package body Exp_Ch6 is
             if Etype (Formal) = Standard_Natural then
                Actual := Make_Integer_Literal (Loc, Uint_0);
                Analyze_And_Resolve (Actual, Standard_Natural);
-               Add_Extra_Actual_To_Call (N, Formal, Actual);
+               Add_Extra_Actual_To_Call (Call_Node, Formal, Actual);
 
             --  BIPtaskmaster
 
             elsif Etype (Formal) = Standard_Integer then
                Actual := Make_Integer_Literal (Loc, Uint_0);
                Analyze_And_Resolve (Actual, Standard_Integer);
-               Add_Extra_Actual_To_Call (N, Formal, Actual);
+               Add_Extra_Actual_To_Call (Call_Node, Formal, Actual);
 
             --  BIPstoragepool, BIPcollection, BIPactivationchain,
             --  and BIPaccess.
@@ -3383,7 +3543,7 @@ package body Exp_Ch6 is
             elsif Is_Access_Type (Etype (Formal)) then
                Actual := Make_Null (Loc);
                Analyze_And_Resolve (Actual, Etype (Formal));
-               Add_Extra_Actual_To_Call (N, Formal, Actual);
+               Add_Extra_Actual_To_Call (Call_Node, Formal, Actual);
 
             else
                pragma Assert (False);
@@ -3402,6 +3562,47 @@ package body Exp_Ch6 is
          pragma Assert (Check_BIP_Actuals (Call_Node, Function_Id));
       end Add_Dummy_Build_In_Place_Actuals;
 
+      --------------------------
+      -- Add_Actual_Parameter --
+      --------------------------
+
+      procedure Add_Actual_Parameter (Insert_Param : Node_Id) is
+         Actual_Expr : constant Node_Id :=
+                         Explicit_Actual_Parameter (Insert_Param);
+
+      begin
+         --  Case of insertion is first named actual
+
+         if No (Prev)
+           or else Nkind (Parent (Prev)) /= N_Parameter_Association
+         then
+            Set_Next_Named_Actual
+              (Insert_Param, First_Named_Actual (Call_Node));
+            Set_First_Named_Actual (Call_Node, Actual_Expr);
+
+            if No (Prev) then
+               if No (Parameter_Associations (Call_Node)) then
+                  Set_Parameter_Associations (Call_Node, New_List);
+               end if;
+
+               Append (Insert_Param, Parameter_Associations (Call_Node));
+
+            else
+               Insert_After (Prev, Insert_Param);
+            end if;
+
+         --  Case of insertion is not first named actual
+
+         else
+            Set_Next_Named_Actual
+              (Insert_Param, Next_Named_Actual (Parent (Prev)));
+            Set_Next_Named_Actual (Parent (Prev), Actual_Expr);
+            Append (Insert_Param, Parameter_Associations (Call_Node));
+         end if;
+
+         Prev := Actual_Expr;
+      end Add_Actual_Parameter;
+
       ----------------------
       -- Add_Extra_Actual --
       ----------------------
@@ -3427,6 +3628,421 @@ package body Exp_Ch6 is
          end if;
       end Add_Extra_Actual;
 
+      --  Local variables
+
+      use Deferred_Extra_Formals_Support;
+
+      Actual        : Node_Id;
+      Formal        : Entity_Id;
+      Param_Count   : Positive;
+      Subp          : constant Entity_Id := Get_Called_Entity (Call_Node);
+
+   --  Start of processing for Create_Extra_Actuals
+
+   begin
+      --  Special case: Thunks must not compute the extra actuals; they must
+      --  just propagate their extra actuals to the target primitive.
+
+      if Is_Thunk (Current_Scope)
+        and then Thunk_Entity (Current_Scope) = Subp
+      then
+         declare
+            Target_Formal : Entity_Id;
+            Thunk_Formal  : Entity_Id;
+
+         begin
+            pragma Assert (Extra_Formals_Known (Subp)
+              and then Extra_Formals_Match_OK (Current_Scope, Subp));
+
+            Target_Formal := Extra_Formals (Subp);
+            Thunk_Formal  := Extra_Formals (Current_Scope);
+            while Present (Target_Formal) loop
+               Add_Extra_Actual
+                  (Expr => New_Occurrence_Of (Thunk_Formal, Loc),
+                   EF   => Thunk_Formal);
+
+               Target_Formal := Extra_Formal (Target_Formal);
+               Thunk_Formal  := Extra_Formal (Thunk_Formal);
+            end loop;
+
+            while Is_Non_Empty_List (Extra_Actuals) loop
+               Add_Actual_Parameter (Remove_Head (Extra_Actuals));
+            end loop;
+
+            return;
+         end;
+      end if;
+
+      pragma Assert (Extra_Formals_Known (Subp)
+        or else Is_Unsupported_Extra_Formals_Entity (Subp));
+
+      --  First step, compute extra actuals, corresponding to any Extra_Formals
+      --  present. Note that we do not access Extra_Formals directly; instead
+      --  we generate and collect the corresponding actuals in Extra_Actuals.
+
+      Formal := First_Formal (Subp);
+      Actual := First_Actual (Call_Node);
+      Param_Count := 1;
+      while Present (Formal) loop
+         --  Prepare to examine current entry
+
+         Prev := Actual;
+
+         --  Create possible extra actual for constrained case. Usually, the
+         --  extra actual is of the form actual'constrained, but since this
+         --  attribute is only available for unconstrained records, TRUE is
+         --  expanded if the type of the formal happens to be constrained (for
+         --  instance when this procedure is inherited from an unconstrained
+         --  record to a constrained one) or if the actual has no discriminant
+         --  (its type is constrained). An exception to this is the case of a
+         --  private type without discriminants. In this case we pass FALSE
+         --  because the object has underlying discriminants with defaults.
+
+         if Present (Extra_Constrained (Formal)) then
+            if Is_Mutably_Tagged_Type (Etype (Actual))
+              or else (Is_Private_Type (Etype (Prev))
+                        and then not Has_Discriminants
+                                       (Base_Type (Etype (Prev))))
+            then
+               Add_Extra_Actual
+                 (Expr => New_Occurrence_Of (Standard_False, Loc),
+                  EF   => Extra_Constrained (Formal));
+
+            elsif Is_Constrained (Etype (Formal))
+              or else not Has_Discriminants (Etype (Prev))
+            then
+               Add_Extra_Actual
+                 (Expr => New_Occurrence_Of (Standard_True, Loc),
+                  EF   => Extra_Constrained (Formal));
+
+            --  Do not produce extra actuals for Unchecked_Union parameters.
+            --  Jump directly to the end of the loop.
+
+            elsif Is_Unchecked_Union (Base_Type (Etype (Actual))) then
+               goto Skip_Extra_Actual_Generation;
+
+            else
+               --  If the actual is a type conversion, then the constrained
+               --  test applies to the actual, not the target type.
+
+               declare
+                  Act_Prev : Node_Id;
+
+               begin
+                  --  Test for unchecked conversions as well, which can occur
+                  --  as out parameter actuals on calls to stream procedures.
+
+                  Act_Prev := Prev;
+                  while Nkind (Act_Prev) in N_Type_Conversion
+                                          | N_Unchecked_Type_Conversion
+                  loop
+                     Act_Prev := Expression (Act_Prev);
+                  end loop;
+
+                  --  If the expression is a conversion of a dereference, this
+                  --  is internally generated code that manipulates addresses,
+                  --  e.g. when building interface tables. No check should
+                  --  occur in this case, and the discriminated object is not
+                  --  directly at hand.
+
+                  if not Comes_From_Source (Actual)
+                    and then Nkind (Actual) = N_Unchecked_Type_Conversion
+                    and then Nkind (Act_Prev) = N_Explicit_Dereference
+                  then
+                     Add_Extra_Actual
+                       (Expr => New_Occurrence_Of (Standard_False, Loc),
+                        EF   => Extra_Constrained (Formal));
+
+                  else
+                     Add_Extra_Actual
+                       (Expr =>
+                          Make_Attribute_Reference (Sloc (Prev),
+                            Prefix         =>
+                              Duplicate_Subexpr_No_Checks
+                                (Act_Prev, Name_Req => True),
+                            Attribute_Name => Name_Constrained),
+                        EF   => Extra_Constrained (Formal));
+                  end if;
+               end;
+            end if;
+         end if;
+
+         --  Create possible extra actual for accessibility level
+
+         if Present (Extra_Accessibility (Formal)) then
+
+            --  Ada 2005 (AI-251): Thunks must propagate the extra actuals of
+            --  accessibility levels.
+
+            if Is_Thunk (Current_Scope) then
+               declare
+                  Parm_Ent : Entity_Id;
+
+               begin
+                  if Is_Controlling_Actual (Actual) then
+
+                     --  Find the corresponding actual of the thunk
+
+                     Parm_Ent := First_Entity (Current_Scope);
+                     for J in 2 .. Param_Count loop
+                        Next_Entity (Parm_Ent);
+                     end loop;
+
+                  --  Handle unchecked conversion of access types generated
+                  --  in thunks (cf. Expand_Interface_Thunk).
+
+                  elsif Is_Access_Type (Etype (Actual))
+                    and then Nkind (Actual) = N_Unchecked_Type_Conversion
+                  then
+                     Parm_Ent := Entity (Expression (Actual));
+
+                  else pragma Assert (Is_Entity_Name (Actual));
+                     Parm_Ent := Entity (Actual);
+                  end if;
+
+                  Add_Extra_Actual
+                    (Expr => Accessibility_Level
+                               (Expr            => Parm_Ent,
+                                Level           => Dynamic_Level,
+                                Allow_Alt_Model => False),
+                     EF   => Extra_Accessibility (Formal));
+               end;
+
+            --  Conditional expressions
+
+            elsif Nkind (Prev) = N_Expression_With_Actions
+              and then Nkind (Original_Node (Prev)) in
+                         N_If_Expression | N_Case_Expression
+            then
+               Add_Cond_Expression_Extra_Actual (Formal);
+
+            --  Internal constant generated to remove side effects (normally
+            --  from the expansion of dispatching calls).
+
+            --  First verify the actual is internal
+
+            elsif not Comes_From_Source (Prev)
+              and then not Is_Rewrite_Substitution (Prev)
+
+              --  Next check that the actual is a constant
+
+              and then Nkind (Prev) = N_Identifier
+              and then Ekind (Entity (Prev)) = E_Constant
+              and then Nkind (Parent (Entity (Prev))) = N_Object_Declaration
+            then
+               --  Generate the accessibility level based on the expression in
+               --  the constant's declaration.
+
+               declare
+                  Ent : Entity_Id := Entity (Prev);
+
+               begin
+                  --  Handle deferred constants
+
+                  if Present (Full_View (Ent)) then
+                     Ent := Full_View (Ent);
+                  end if;
+
+                  Add_Extra_Actual
+                    (Expr => Accessibility_Level
+                               (Expr            => Expression (Parent (Ent)),
+                                Level           => Dynamic_Level,
+                                Allow_Alt_Model => False),
+                     EF   => Extra_Accessibility (Formal));
+               end;
+
+            --  Normal case
+
+            else
+               Add_Extra_Actual
+                 (Expr => Accessibility_Level
+                            (Expr            => Prev,
+                             Level           => Dynamic_Level,
+                             Allow_Alt_Model => False),
+                  EF   => Extra_Accessibility (Formal));
+            end if;
+         end if;
+
+         --  This label is required when skipping extra actual generation for
+         --  Unchecked_Union parameters.
+
+         <<Skip_Extra_Actual_Generation>>
+
+         Param_Count := Param_Count + 1;
+         Next_Actual (Actual);
+         Next_Formal (Formal);
+      end loop;
+
+      --  If we are calling an Ada 2012 function which needs to have the
+      --  "accessibility level determined by the point of call" (AI05-0234)
+      --  passed in to it, then pass it in.
+
+      if Ekind (Subp) in E_Function | E_Operator | E_Subprogram_Type
+        and then
+          Present (Extra_Accessibility_Of_Result (Ultimate_Alias (Subp)))
+      then
+         declare
+            Extra_Form : Node_Id := Empty;
+            Level      : Node_Id := Empty;
+
+         begin
+            --  Detect cases where the function call has been internally
+            --  generated by examining the original node and return library
+            --  level - taking care to avoid ignoring function calls expanded
+            --  in prefix notation.
+
+            if Nkind (Original_Node (Call_Node)) not in N_Function_Call
+                                                      | N_Selected_Component
+                                                      | N_Indexed_Component
+            then
+               Level := Make_Integer_Literal
+                          (Loc, Scope_Depth (Standard_Standard));
+
+            --  Otherwise get the level normally based on the call node
+
+            else
+               Level := Accessibility_Level
+                          (Expr            => Call_Node,
+                           Level           => Dynamic_Level,
+                           Allow_Alt_Model => False);
+            end if;
+
+            --  It may be possible that we are re-expanding an already
+            --  expanded call when are are dealing with dispatching ???
+
+            if No (Parameter_Associations (Call_Node))
+              or else Nkind (Last (Parameter_Associations (Call_Node)))
+                        /= N_Parameter_Association
+              or else not Is_Accessibility_Actual
+                              (Last (Parameter_Associations (Call_Node)))
+            then
+               Extra_Form := Extra_Accessibility_Of_Result
+                               (Ultimate_Alias (Subp));
+
+               Add_Extra_Actual
+                 (Expr => Level,
+                  EF   => Extra_Form);
+            end if;
+         end;
+      end if;
+
+      --  Second step: In the previous loop we gathered the extra actuals (the
+      --  ones that correspond to Extra_Formals), so now they can be appended.
+
+      if Is_Non_Empty_List (Extra_Actuals) then
+         declare
+            Num_Extra_Actuals : constant Nat := List_Length (Extra_Actuals);
+
+         begin
+            while Is_Non_Empty_List (Extra_Actuals) loop
+               Add_Actual_Parameter (Remove_Head (Extra_Actuals));
+            end loop;
+
+            --  Add dummy extra BIP actuals if we are calling a function that
+            --  inherited the BIP extra actuals but does not require them.
+
+            if Nkind (Call_Node) = N_Function_Call
+              and then Is_Function_Call_With_BIP_Formals (Call_Node)
+              and then not Is_Build_In_Place_Function_Call (Call_Node)
+            then
+               Add_Dummy_Build_In_Place_Actuals (Subp,
+                 Num_Added_Extra_Actuals => Num_Extra_Actuals);
+            end if;
+         end;
+
+      --  Add dummy extra BIP actuals if we are calling a function that
+      --  inherited the BIP extra actuals but does not require them.
+
+      elsif Nkind (Call_Node) = N_Function_Call
+        and then Is_Function_Call_With_BIP_Formals (Call_Node)
+        and then not Is_Build_In_Place_Function_Call (Call_Node)
+      then
+         Add_Dummy_Build_In_Place_Actuals (Subp);
+      end if;
+
+      --  For non build-in-place calls formals and actuals must match;
+      --  for build-in-place function calls, the pending bip actuals are
+      --  added by the following subprograms as part of the bottom-up
+      --  expansion of the call (and this check will be performed there):
+      --    Make_Build_In_Place_Call_In_Allocator
+      --    Make_Build_In_Place_Call_In_Anonymous_Context
+      --    Make_Build_In_Place_Call_In_Assignment
+      --    Make_Build_In_Place_Call_In_Object_Declaration
+      --    Make_Build_In_Place_Iface_Call_In_Allocator
+      --    Make_Build_In_Place_Iface_Call_In_Anonymous_Context
+      --    Make_Build_In_Place_Iface_Call_In_Object_Declaration
+
+      pragma Assert (Is_Build_In_Place_Function_Call (Call_Node)
+        or else (Check_Number_Of_Actuals (Call_Node, Subp)
+                   and then Check_BIP_Actuals (Call_Node, Subp)));
+   end Create_Extra_Actuals;
+
+   ------------------------
+   -- Expand_Call_Helper --
+   ------------------------
+
+   --  This procedure handles expansion of function calls and procedure call
+   --  statements (i.e. it serves as the body for Expand_N_Function_Call and
+   --  Expand_N_Procedure_Call_Statement). Processing for calls includes:
+
+   --    Replace call to Raise_Exception by Raise_Exception_Always if possible
+   --    Provide values of actuals for all formals in Extra_Formals list
+   --    Replace "call" to enumeration literal function by literal itself
+   --    Rewrite call to predefined operator as operator
+   --    Replace actuals to in-out parameters that are numeric conversions,
+   --     with explicit assignment to temporaries before and after the call.
+
+   --   Note that the list of actuals has been filled with default expressions
+   --   during semantic analysis of the call. Only the extra actuals required
+   --   for the 'Constrained attribute and for accessibility checks are added
+   --   at this point.
+
+   procedure Expand_Call_Helper (N : Node_Id; Post_Call : out List_Id) is
+      Loc       : constant Source_Ptr := Sloc (N);
+      Call_Node : Node_Id := N;
+      Prev      : Node_Id := Empty;
+
+      procedure Add_View_Conversion_Invariants
+        (Formal : Entity_Id;
+         Actual : Node_Id);
+      --  Adds invariant checks for every intermediate type between the range
+      --  of a view converted argument to its ancestor (from parent to child).
+
+      function Can_Fold_Predicate_Call (P : Entity_Id) return Boolean;
+      --  Try to constant-fold a predicate check, which often enough is a
+      --  simple arithmetic expression that can be computed statically if
+      --  its argument is static. This cleans up the output of CCG, even
+      --  though useless predicate checks will be generally removed by
+      --  back-end optimizations.
+
+      procedure Check_Subprogram_Variant;
+      --  Emit a call to the internally generated procedure with checks for
+      --  aspect Subprogram_Variant, if present and enabled.
+
+      function Inherited_From_Formal (S : Entity_Id) return Entity_Id;
+      --  Within an instance, a type derived from an untagged formal derived
+      --  type inherits from the original parent, not from the actual. The
+      --  current derivation mechanism has the derived type inherit from the
+      --  actual, which is only correct outside of the instance. If the
+      --  subprogram is inherited, we test for this particular case through a
+      --  convoluted tree traversal before setting the proper subprogram to be
+      --  called.
+
+      function In_Unfrozen_Instance (E : Entity_Id) return Boolean;
+      --  Return true if E comes from an instance that is not yet frozen
+
+      function Is_Class_Wide_Interface_Type (E : Entity_Id) return Boolean;
+      --  Return True when E is a class-wide interface type or an access to
+      --  a class-wide interface type.
+
+      function Is_Direct_Deep_Call (Subp : Entity_Id) return Boolean;
+      --  Determine if Subp denotes a non-dispatching call to a Deep routine
+
+      function New_Value (From : Node_Id) return Node_Id;
+      --  From is the original Expression. New_Value is equivalent to a call
+      --  to Duplicate_Subexpr with an explicit dereference when From is an
+      --  access parameter.
+
       ------------------------------------
       -- Add_View_Conversion_Invariants --
       ------------------------------------
@@ -3943,6 +4559,9 @@ package body Exp_Ch6 is
       Subp          : Entity_Id;
 
       CW_Interface_Formals_Present : Boolean := False;
+      Defer_Extra_Actuals          : Boolean := False;
+
+      use Deferred_Extra_Formals_Support;
 
    --  Start of processing for Expand_Call_Helper
 
@@ -4029,12 +4648,6 @@ package body Exp_Ch6 is
          end if;
       end if;
 
-      --  Ensure that the called subprogram has all its formals
-
-      if not Is_Frozen (Subp) then
-         Create_Extra_Formals (Subp);
-      end if;
-
       --  Ada 2005 (AI-345): We have a procedure call as a triggering
       --  alternative in an asynchronous select or as an entry call in
       --  a conditional or timed select. Check whether the procedure call
@@ -4080,6 +4693,50 @@ package body Exp_Ch6 is
          end;
       end if;
 
+      --  Ensure that the called subprogram has all its formals; extra formals
+      --  of init procs were added when they were built.
+
+      if not Extra_Formals_Known (Subp) then
+         Create_Extra_Formals (Subp);
+
+         --  If the previous call to Create_Extra_Formals could not add the
+         --  extra formals, then we must defer adding the extra actuals of
+         --  this call until we know the underlying type of all the formals
+         --  and return type of the called subprogram or entry. Deferral of
+         --  extra actuals occurs in two cases:
+         --  1) In the body of internally built dynamic call helpers of
+         --     class-wide preconditions.
+         --  2) In the body of expanded expression functions.
+
+         if not Extra_Formals_Known (Subp) then
+            declare
+               Scop_Id : Entity_Id := Current_Scope;
+
+            begin
+               --  Locate the enclosing subprogram or entry since it is
+               --  required to register this deferred call.
+
+               Scop_Id := Current_Scope;
+               while Present (Scop_Id)
+                 and then Scop_Id /= Standard_Standard
+                 and then not Is_Subprogram_Or_Entry (Scop_Id)
+               loop
+                  Scop_Id := Scope (Scop_Id);
+               end loop;
+
+               pragma Assert (Is_Subprogram_Or_Entry (Scop_Id));
+               pragma Assert (Is_Deferred_Extra_Formals_Entity (Subp));
+               Register_Deferred_Extra_Formals_Call (Call_Node, Scop_Id);
+
+               Defer_Extra_Actuals := True;
+            end;
+         end if;
+      end if;
+
+      pragma Assert (Extra_Formals_Known (Subp)
+        or else Is_Deferred_Extra_Formals_Entity (Subp)
+        or else Is_Unsupported_Extra_Formals_Entity (Subp));
+
       --  If this is a call to a predicate function, try to constant fold it
 
       if Nkind (Call_Node) = N_Function_Call
@@ -4091,56 +4748,39 @@ package body Exp_Ch6 is
       end if;
 
       --  First step, compute extra actuals, corresponding to any Extra_Formals
-      --  present. Note that we do not access Extra_Formals directly, instead
+      --  present. Note that we do not access Extra_Formals directly; instead
       --  we simply note the presence of the extra formals as we process the
       --  regular formals collecting corresponding actuals in Extra_Actuals.
 
-      --  We also generate any required range checks for actuals for in formals
-      --  as we go through the loop, since this is a convenient place to do it.
-      --  (Though it seems that this would be better done in Expand_Actuals???)
+      --  We also generate any required range checks for actuals for in-mode
+      --  formals as we go through the loop, since this is a convenient place
+      --  to do it. (Though it seems that this would be better done in
+      --  Expand_Actuals???)
 
       --  Special case: Thunks must not compute the extra actuals; they must
-      --  just propagate to the target primitive their extra actuals.
+      --  just propagate their extra actuals to the target primitive (this
+      --  propagation is performed by Create_Extra_Actuals).
 
       if Is_Thunk (Current_Scope)
         and then Thunk_Entity (Current_Scope) = Subp
+        and then Extra_Formals_Known (Subp)
         and then Present (Extra_Formals (Subp))
       then
-         pragma Assert (Extra_Formals_Match_OK (Current_Scope, Subp));
+         Create_Extra_Actuals (N);
 
-         declare
-            Target_Formal : Entity_Id;
-            Thunk_Formal  : Entity_Id;
-
-         begin
-            Target_Formal := Extra_Formals (Subp);
-            Thunk_Formal  := Extra_Formals (Current_Scope);
-            while Present (Target_Formal) loop
-               Add_Extra_Actual
-                 (Expr => New_Occurrence_Of (Thunk_Formal, Loc),
-                  EF   => Thunk_Formal);
-
-               Target_Formal := Extra_Formal (Target_Formal);
-               Thunk_Formal  := Extra_Formal (Thunk_Formal);
-            end loop;
-
-            while Is_Non_Empty_List (Extra_Actuals) loop
-               Add_Actual_Parameter (Remove_Head (Extra_Actuals));
-            end loop;
+         --  Mark the call as an expanded build-in-place call; required
+         --  to avoid adding the extra formals twice.
 
-            --  Mark the call as processed build-in-place call; required
-            --  to avoid adding the extra formals twice.
+         if Nkind (Call_Node) = N_Function_Call then
+            Set_Is_Expanded_Build_In_Place_Call (Call_Node);
+         end if;
 
-            if Nkind (Call_Node) = N_Function_Call then
-               Set_Is_Expanded_Build_In_Place_Call (Call_Node);
-            end if;
+         Expand_Actuals (Call_Node, Subp, Post_Call);
 
-            Expand_Actuals (Call_Node, Subp, Post_Call);
-            pragma Assert (Is_Empty_List (Post_Call));
-            pragma Assert (Check_Number_Of_Actuals (Call_Node, Subp));
-            pragma Assert (Check_BIP_Actuals (Call_Node, Subp));
-            return;
-         end;
+         pragma Assert (Is_Empty_List (Post_Call));
+         pragma Assert (Check_Number_Of_Actuals (Call_Node, Subp));
+         pragma Assert (Check_BIP_Actuals (Call_Node, Subp));
+         return;
       end if;
 
       Formal := First_Formal (Subp);
@@ -4158,180 +4798,6 @@ package body Exp_Ch6 is
            CW_Interface_Formals_Present
              or else Is_Class_Wide_Interface_Type (Etype (Formal));
 
-         --  Create possible extra actual for constrained case. Usually, the
-         --  extra actual is of the form actual'constrained, but since this
-         --  attribute is only available for unconstrained records, TRUE is
-         --  expanded if the type of the formal happens to be constrained (for
-         --  instance when this procedure is inherited from an unconstrained
-         --  record to a constrained one) or if the actual has no discriminant
-         --  (its type is constrained). An exception to this is the case of a
-         --  private type without discriminants. In this case we pass FALSE
-         --  because the object has underlying discriminants with defaults.
-
-         if Present (Extra_Constrained (Formal)) then
-            if Is_Mutably_Tagged_Type (Etype (Actual))
-              or else (Is_Private_Type (Etype (Prev))
-                        and then not Has_Discriminants
-                                       (Base_Type (Etype (Prev))))
-            then
-               Add_Extra_Actual
-                 (Expr => New_Occurrence_Of (Standard_False, Loc),
-                  EF   => Extra_Constrained (Formal));
-
-            elsif Is_Constrained (Etype (Formal))
-              or else not Has_Discriminants (Etype (Prev))
-            then
-               Add_Extra_Actual
-                 (Expr => New_Occurrence_Of (Standard_True, Loc),
-                  EF   => Extra_Constrained (Formal));
-
-            --  Do not produce extra actuals for Unchecked_Union parameters.
-            --  Jump directly to the end of the loop.
-
-            elsif Is_Unchecked_Union (Base_Type (Etype (Actual))) then
-               goto Skip_Extra_Actual_Generation;
-
-            else
-               --  If the actual is a type conversion, then the constrained
-               --  test applies to the actual, not the target type.
-
-               declare
-                  Act_Prev : Node_Id;
-
-               begin
-                  --  Test for unchecked conversions as well, which can occur
-                  --  as out parameter actuals on calls to stream procedures.
-
-                  Act_Prev := Prev;
-                  while Nkind (Act_Prev) in N_Type_Conversion
-                                          | N_Unchecked_Type_Conversion
-                  loop
-                     Act_Prev := Expression (Act_Prev);
-                  end loop;
-
-                  --  If the expression is a conversion of a dereference, this
-                  --  is internally generated code that manipulates addresses,
-                  --  e.g. when building interface tables. No check should
-                  --  occur in this case, and the discriminated object is not
-                  --  directly at hand.
-
-                  if not Comes_From_Source (Actual)
-                    and then Nkind (Actual) = N_Unchecked_Type_Conversion
-                    and then Nkind (Act_Prev) = N_Explicit_Dereference
-                  then
-                     Add_Extra_Actual
-                       (Expr => New_Occurrence_Of (Standard_False, Loc),
-                        EF   => Extra_Constrained (Formal));
-
-                  else
-                     Add_Extra_Actual
-                       (Expr =>
-                          Make_Attribute_Reference (Sloc (Prev),
-                            Prefix         =>
-                              Duplicate_Subexpr_No_Checks
-                                (Act_Prev, Name_Req => True),
-                            Attribute_Name => Name_Constrained),
-                        EF   => Extra_Constrained (Formal));
-                  end if;
-               end;
-            end if;
-         end if;
-
-         --  Create possible extra actual for accessibility level
-
-         if Present (Extra_Accessibility (Formal)) then
-            --  Ada 2005 (AI-251): Thunks must propagate the extra actuals of
-            --  accessibility levels.
-
-            if Is_Thunk (Current_Scope) then
-               declare
-                  Parm_Ent : Entity_Id;
-
-               begin
-                  if Is_Controlling_Actual (Actual) then
-
-                     --  Find the corresponding actual of the thunk
-
-                     Parm_Ent := First_Entity (Current_Scope);
-                     for J in 2 .. Param_Count loop
-                        Next_Entity (Parm_Ent);
-                     end loop;
-
-                  --  Handle unchecked conversion of access types generated
-                  --  in thunks (cf. Expand_Interface_Thunk).
-
-                  elsif Is_Access_Type (Etype (Actual))
-                    and then Nkind (Actual) = N_Unchecked_Type_Conversion
-                  then
-                     Parm_Ent := Entity (Expression (Actual));
-
-                  else pragma Assert (Is_Entity_Name (Actual));
-                     Parm_Ent := Entity (Actual);
-                  end if;
-
-                  Add_Extra_Actual
-                    (Expr => Accessibility_Level
-                               (Expr            => Parm_Ent,
-                                Level           => Dynamic_Level,
-                                Allow_Alt_Model => False),
-                     EF   => Extra_Accessibility (Formal));
-               end;
-
-            --  Conditional expressions
-
-            elsif Nkind (Prev) = N_Expression_With_Actions
-              and then Nkind (Original_Node (Prev)) in
-                         N_If_Expression | N_Case_Expression
-            then
-               Add_Cond_Expression_Extra_Actual (Formal);
-
-            --  Internal constant generated to remove side effects (normally
-            --  from the expansion of dispatching calls).
-
-            --  First verify the actual is internal
-
-            elsif not Comes_From_Source (Prev)
-              and then not Is_Rewrite_Substitution (Prev)
-
-              --  Next check that the actual is a constant
-
-              and then Nkind (Prev) = N_Identifier
-              and then Ekind (Entity (Prev)) = E_Constant
-              and then Nkind (Parent (Entity (Prev))) = N_Object_Declaration
-            then
-               --  Generate the accessibility level based on the expression in
-               --  the constant's declaration.
-
-               declare
-                  Ent : Entity_Id := Entity (Prev);
-
-               begin
-                  --  Handle deferred constants
-
-                  if Present (Full_View (Ent)) then
-                     Ent := Full_View (Ent);
-                  end if;
-
-                  Add_Extra_Actual
-                    (Expr => Accessibility_Level
-                               (Expr            => Expression (Parent (Ent)),
-                                Level           => Dynamic_Level,
-                                Allow_Alt_Model => False),
-                     EF   => Extra_Accessibility (Formal));
-               end;
-
-            --  Normal case
-
-            else
-               Add_Extra_Actual
-                 (Expr => Accessibility_Level
-                            (Expr            => Prev,
-                             Level           => Dynamic_Level,
-                             Allow_Alt_Model => False),
-                  EF   => Extra_Accessibility (Formal));
-            end if;
-         end if;
-
          --  Perform the check of 4.6(49) that prevents a null value from being
          --  passed as an actual to an access parameter. Note that the check
          --  is elided in the common cases of passing an access attribute or
@@ -4525,66 +4991,11 @@ package body Exp_Ch6 is
          --  This label is required when skipping extra actual generation for
          --  Unchecked_Union parameters.
 
-         <<Skip_Extra_Actual_Generation>>
-
          Param_Count := Param_Count + 1;
          Next_Actual (Actual);
          Next_Formal (Formal);
       end loop;
 
-      --  If we are calling an Ada 2012 function which needs to have the
-      --  "accessibility level determined by the point of call" (AI05-0234)
-      --  passed in to it, then pass it in.
-
-      if Ekind (Subp) in E_Function | E_Operator | E_Subprogram_Type
-        and then
-          Present (Extra_Accessibility_Of_Result (Ultimate_Alias (Subp)))
-      then
-         declare
-            Extra_Form : Node_Id := Empty;
-            Level      : Node_Id := Empty;
-
-         begin
-            --  Detect cases where the function call has been internally
-            --  generated by examining the original node and return library
-            --  level - taking care to avoid ignoring function calls expanded
-            --  in prefix notation.
-
-            if Nkind (Original_Node (Call_Node)) not in N_Function_Call
-                                                      | N_Selected_Component
-                                                      | N_Indexed_Component
-            then
-               Level := Make_Integer_Literal
-                          (Loc, Scope_Depth (Standard_Standard));
-
-            --  Otherwise get the level normally based on the call node
-
-            else
-               Level := Accessibility_Level
-                          (Expr            => Call_Node,
-                           Level           => Dynamic_Level,
-                           Allow_Alt_Model => False);
-            end if;
-
-            --  It may be possible that we are re-expanding an already
-            --  expanded call when are are dealing with dispatching ???
-
-            if No (Parameter_Associations (Call_Node))
-              or else Nkind (Last (Parameter_Associations (Call_Node)))
-                        /= N_Parameter_Association
-              or else not Is_Accessibility_Actual
-                              (Last (Parameter_Associations (Call_Node)))
-            then
-               Extra_Form := Extra_Accessibility_Of_Result
-                               (Ultimate_Alias (Subp));
-
-               Add_Extra_Actual
-                 (Expr => Level,
-                  EF   => Extra_Form);
-            end if;
-         end;
-      end if;
-
       --  If we are expanding the RHS of an assignment we need to check if tag
       --  propagation is needed. You might expect this processing to be in
       --  Analyze_Assignment but has to be done earlier (bottom-up) because the
@@ -4597,27 +5008,34 @@ package body Exp_Ch6 is
       then
          declare
             Ass : Node_Id := Empty;
+            Par : Node_Id := Parent (Call_Node);
 
          begin
-            if Nkind (Parent (Call_Node)) = N_Assignment_Statement then
-               Ass := Parent (Call_Node);
+            --  Search for the LHS of an enclosing assignment statement to a
+            --  classwide type object (if present) and propagate the tag to
+            --  this function call.
+
+            while Nkind (Par) in N_Case_Expression
+                               | N_Case_Expression_Alternative
+                               | N_Explicit_Dereference
+                               | N_If_Expression
+                               | N_Qualified_Expression
+                               | N_Unchecked_Type_Conversion
+            loop
+               if Nkind (Par) = N_Case_Expression_Alternative then
+                  Par := Parent (Par);
+               end if;
 
-            elsif Nkind (Parent (Call_Node)) = N_Qualified_Expression
-              and then Nkind (Parent (Parent (Call_Node))) =
-                                                  N_Assignment_Statement
-            then
-               Ass := Parent (Parent (Call_Node));
+               exit when not Is_Tag_Indeterminate (Par);
 
-            elsif Nkind (Parent (Call_Node)) = N_Explicit_Dereference
-              and then Nkind (Parent (Parent (Call_Node))) =
-                                                  N_Assignment_Statement
-            then
-               Ass := Parent (Parent (Call_Node));
-            end if;
+               Par := Parent (Par);
+            end loop;
 
-            if Present (Ass)
-              and then Is_Class_Wide_Type (Etype (Name (Ass)))
+            if Nkind (Par) = N_Assignment_Statement
+              and then Is_Class_Wide_Type (Etype (Name (Par)))
             then
+               Ass := Par;
+
                --  Move the error messages below to sem???
 
                if Is_Access_Type (Etype (Call_Node)) then
@@ -4630,6 +5048,12 @@ package body Exp_Ch6 is
                          Call_Node, Root_Type (Etype (Name (Ass))));
                   else
                      Propagate_Tag (Name (Ass), Call_Node);
+
+                     --  Remember that the tag has been propagated to avoid
+                     --  propagating it again, as part of the (bottom-up)
+                     --  analysis of the enclosing assignment.
+
+                     Set_Tag_Propagated (Name (Ass));
                   end if;
 
                elsif Etype (Call_Node) /= Root_Type (Etype (Name (Ass))) then
@@ -4640,6 +5064,12 @@ package body Exp_Ch6 is
 
                else
                   Propagate_Tag (Name (Ass), Call_Node);
+
+                  --  Remember that the tag has been propagated to avoid
+                  --  propagating it again, as part of the (bottom-up)
+                  --  analysis of the enclosing assignment.
+
+                  Set_Tag_Propagated (Name (Ass));
                end if;
 
                --  The call will be rewritten as a dispatching call, and
@@ -4778,38 +5208,12 @@ package body Exp_Ch6 is
       then
          null;
 
-      --  During that loop we gathered the extra actuals (the ones that
-      --  correspond to Extra_Formals), so now they can be appended.
-
-      elsif Is_Non_Empty_List (Extra_Actuals) then
-         declare
-            Num_Extra_Actuals : constant Nat := List_Length (Extra_Actuals);
-
-         begin
-            while Is_Non_Empty_List (Extra_Actuals) loop
-               Add_Actual_Parameter (Remove_Head (Extra_Actuals));
-            end loop;
-
-            --  Add dummy extra BIP actuals if we are calling a function that
-            --  inherited the BIP extra actuals but does not require them.
-
-            if Nkind (Call_Node) = N_Function_Call
-              and then Is_Function_Call_With_BIP_Formals (Call_Node)
-              and then not Is_Build_In_Place_Function_Call (Call_Node)
-            then
-               Add_Dummy_Build_In_Place_Actuals (Subp,
-                 Num_Added_Extra_Actuals => Num_Extra_Actuals);
-            end if;
-         end;
-
-      --  Add dummy extra BIP actuals if we are calling a function that
-      --  inherited the BIP extra actuals but does not require them.
+      elsif not Defer_Extra_Actuals then
+         Create_Extra_Formals (Subp);
 
-      elsif Nkind (Call_Node) = N_Function_Call
-        and then Is_Function_Call_With_BIP_Formals (Call_Node)
-        and then not Is_Build_In_Place_Function_Call (Call_Node)
-      then
-         Add_Dummy_Build_In_Place_Actuals (Subp);
+         if Extra_Formals_Known (Subp) then
+            Create_Extra_Actuals (N);
+         end if;
       end if;
 
       --  At this point we have all the actuals, so this is the point at which
@@ -5227,6 +5631,10 @@ package body Exp_Ch6 is
          --  also Build_Renamed_Body) cannot be expanded here because this may
          --  give rise to order-of-elaboration issues for the types of the
          --  parameters of the subprogram, if any.
+         --
+         --  Expand_Inlined_Call procedure does not support the frontend
+         --  inlining of calls that return unconstrained types used as actuals
+         --  or in return statements.
 
          elsif Present (Unit_Declaration_Node (Subp))
            and then Nkind (Unit_Declaration_Node (Subp)) =
@@ -5235,6 +5643,8 @@ package body Exp_Ch6 is
            and then
              Nkind (Body_To_Inline (Unit_Declaration_Node (Subp))) not in
                                                                        N_Entity
+           and then Nkind (Parent (N)) /= N_Function_Call
+           and then Nkind (Parent (N)) /= N_Simple_Return_Statement
          then
             Expand_Inlined_Call (Call_Node, Subp, Orig_Subp);
 
@@ -7159,6 +7569,16 @@ package body Exp_Ch6 is
       then
          Apply_CW_Accessibility_Check (Exp, Scope_Id);
 
+      --  Check that result's access discrims (if any) do not designate
+      --  entities that the function result could outlive. See preceding
+      --  comment about extended return statements and thunks.
+
+      elsif Has_Anonymous_Access_Discriminant (Exp_Typ)
+        and then not Comes_From_Extended_Return_Statement (N)
+        and then not Is_Thunk (Scope_Id)
+      then
+         Apply_Access_Discrims_Accessibility_Check (Exp, Scope_Id);
+
       --  Ada 2012 (AI05-0073): If the result subtype of the function is
       --  defined by an access_definition designating a specific tagged
       --  type T, a check is made that the result value is null or the tag
@@ -8557,6 +8977,8 @@ package body Exp_Ch6 is
       Rewrite (Allocator, New_Occurrence_Of (Return_Obj_Access, Loc));
 
       Analyze_And_Resolve (Allocator, Acc_Type);
+
+      pragma Assert (Returns_By_Ref (Function_Id));
       pragma Assert (Check_Number_Of_Actuals (Func_Call, Function_Id));
       pragma Assert (Check_BIP_Actuals (Func_Call, Function_Id));
    end Make_Build_In_Place_Call_In_Allocator;
@@ -8662,6 +9084,7 @@ package body Exp_Ch6 is
 
          Set_Is_Expanded_Build_In_Place_Call (Func_Call);
 
+         pragma Assert (Returns_By_Ref (Function_Id));
          pragma Assert (Check_Number_Of_Actuals (Func_Call, Function_Id));
          pragma Assert (Check_BIP_Actuals (Func_Call, Function_Id));
       end if;
@@ -8763,6 +9186,8 @@ package body Exp_Ch6 is
       Insert_After_And_Analyze (Ptr_Typ_Decl, Obj_Decl);
 
       Rewrite (Assign, Make_Null_Statement (Loc));
+
+      pragma Assert (Returns_By_Ref (Func_Id));
       pragma Assert (Check_Number_Of_Actuals (Func_Call, Func_Id));
       pragma Assert (Check_BIP_Actuals (Func_Call, Func_Id));
    end Make_Build_In_Place_Call_In_Assignment;
@@ -9187,6 +9612,7 @@ package body Exp_Ch6 is
          end if;
       end if;
 
+      pragma Assert (Returns_By_Ref (Function_Id));
       pragma Assert (Check_Number_Of_Actuals (Func_Call, Function_Id));
       pragma Assert (Check_BIP_Actuals (Func_Call, Function_Id));
    end Make_Build_In_Place_Call_In_Object_Declaration;
@@ -9824,35 +10250,16 @@ package body Exp_Ch6 is
             =>
                declare
                   Call_Node : Node_Id renames Nod;
-                  Subp      : Entity_Id;
+                  Subp      : constant Entity_Id := Get_Called_Entity (Nod);
 
                begin
-                  --  Call using access to subprogram with explicit dereference
-
-                  if Nkind (Name (Call_Node)) = N_Explicit_Dereference then
-                     Subp := Etype (Name (Call_Node));
-
-                  --  Prefix notation calls
-
-                  elsif Nkind (Name (Call_Node)) = N_Selected_Component then
-                     Subp := Entity (Selector_Name (Name (Call_Node)));
-
-                  --  Call to member of entry family, where Name is an indexed
-                  --  component, with the prefix being a selected component
-                  --  giving the task and entry family name, and the index
-                  --  being the entry index.
-
-                  elsif Nkind (Name (Call_Node)) = N_Indexed_Component then
-                     Subp :=
-                       Entity (Selector_Name (Prefix (Name (Call_Node))));
+                  pragma Assert (Check_BIP_Actuals (Call_Node, Subp));
 
-                  --  Normal case
+                  --  Build-in-place function calls return their result by
+                  --  reference.
 
-                  else
-                     Subp := Entity (Name (Call_Node));
-                  end if;
-
-                  pragma Assert (Check_BIP_Actuals (Call_Node, Subp));
+                  pragma Assert (not Is_Build_In_Place_Function (Subp)
+                    or else Returns_By_Ref (Subp));
                end;
 
             --  Skip generic bodies
diff --git a/gcc/ada/exp_ch6.ads b/gcc/ada/exp_ch6.ads
index 118d994..5919627 100644
--- a/gcc/ada/exp_ch6.ads
+++ b/gcc/ada/exp_ch6.ads
@@ -101,7 +101,20 @@ package Exp_Ch6 is
    --  Adds Extra_Actual as a named parameter association for the formal
    --  Extra_Formal in Subprogram_Call.
 
+   procedure Create_Extra_Actuals (Call_Node : Node_Id);
+   --  Create the extra actuals of the given call and add them to its
+   --  actual parameters list.
+
+   procedure Apply_Access_Discrims_Accessibility_Check
+     (Exp : Node_Id; Func : Entity_Id);
+   --  Exp is an expression being returned from a function Func.
+   --  If the result type of the function has access discriminants, insert
+   --  checks that the accessibility level of each entity designated by an
+   --  access discriminant of the result is not deeper than the level of the
+   --  master of the call.
+
    procedure Apply_CW_Accessibility_Check (Exp : Node_Id; Func : Entity_Id);
+   --  Exp is an expression being returned from a function Func.
    --  Ada 2005 (AI95-344): If the result type is class-wide, insert a check
    --  that the level of the return expression's underlying type is not deeper
    --  than the level of the master enclosing the function. Always generate the
diff --git a/gcc/ada/exp_ch7.adb b/gcc/ada/exp_ch7.adb
index e4daf4b..dd864b7 100644
--- a/gcc/ada/exp_ch7.adb
+++ b/gcc/ada/exp_ch7.adb
@@ -59,6 +59,7 @@ with Sinfo.Nodes;    use Sinfo.Nodes;
 with Sinfo.Utils;    use Sinfo.Utils;
 with Sem;            use Sem;
 with Sem_Aux;        use Sem_Aux;
+with Sem_Ch6;        use Sem_Ch6;
 with Sem_Ch7;        use Sem_Ch7;
 with Sem_Ch8;        use Sem_Ch8;
 with Sem_Res;        use Sem_Res;
@@ -2331,6 +2332,8 @@ package body Exp_Ch7 is
 
                Ensure_Freeze_Node (Fin_Id);
                Insert_After (Fin_Spec, Freeze_Node (Fin_Id));
+               Mutate_Ekind (Fin_Id, E_Procedure);
+               Freeze_Extra_Formals (Fin_Id);
                Set_Is_Frozen (Fin_Id);
 
                Append_To (Stmts, Fin_Body);
@@ -3586,18 +3589,22 @@ package body Exp_Ch7 is
 
    procedure Build_Record_Deep_Procs (Typ : Entity_Id) is
    begin
-      Set_TSS (Typ,
-        Make_Deep_Proc
-          (Prim  => Initialize_Case,
-           Typ   => Typ,
-           Stmts => Make_Deep_Record_Body (Initialize_Case, Typ)));
-
-      if not Is_Inherently_Limited_Type (Typ) then
-         Set_TSS (Typ,
-           Make_Deep_Proc
-             (Prim  => Adjust_Case,
-              Typ   => Typ,
-              Stmts => Make_Deep_Record_Body (Adjust_Case, Typ)));
+      if Has_Controlled_Component (Typ) then
+         Set_TSS
+           (Typ,
+            Make_Deep_Proc
+              (Prim  => Initialize_Case,
+               Typ   => Typ,
+               Stmts => Make_Deep_Record_Body (Initialize_Case, Typ)));
+
+         if not Is_Inherently_Limited_Type (Typ) then
+            Set_TSS
+              (Typ,
+               Make_Deep_Proc
+                 (Prim  => Adjust_Case,
+                  Typ   => Typ,
+                  Stmts => Make_Deep_Record_Body (Adjust_Case, Typ)));
+         end if;
       end if;
 
       --  Do not generate Deep_Finalize and Finalize_Address if finalization is
@@ -5598,7 +5605,10 @@ package body Exp_Ch7 is
 
       --  Deal with untagged derivation of private views
 
-      if Present (Utyp) and then Is_Untagged_Derivation (Typ) then
+      if Present (Utyp)
+        and then Is_Untagged_Derivation (Typ)
+        and then Is_Implicit_Full_View (Utyp)
+      then
          Utyp := Underlying_Type (Root_Type (Base_Type (Typ)));
          Ref  := Unchecked_Convert_To (Utyp, Ref);
          Set_Assignment_OK (Ref);
@@ -6635,6 +6645,16 @@ package body Exp_Ch7 is
       --       Raised : Boolean := False;
       --
       --    begin
+      --       begin
+      --          <Destructor_Proc> (V);  --  If applicable
+      --       exception
+      --          when others =>
+      --             if not Raised then
+      --                Raised := True;
+      --                Save_Occurrence (E, Get_Current_Excep.all.all);
+      --             end if;
+      --       end;
+      --
       --       if F then
       --          begin
       --             Finalize (V);  --  If applicable
@@ -6690,6 +6710,8 @@ package body Exp_Ch7 is
       --
       --       begin
       --          Deep_Finalize (V._parent, False);  --  If applicable
+      --  or
+      --          Deep_Finalize (Parent_Type (V), False); -- Untagged case
       --       exception
       --          when Id : others =>
       --             if not Raised then
@@ -7094,7 +7116,7 @@ package body Exp_Ch7 is
          --  or the type is not controlled.
 
          if Is_Empty_List (Bod_Stmts) then
-            Append_To (Bod_Stmts, Make_Null_Statement (Loc));
+            Append_New_To (Bod_Stmts, Make_Null_Statement (Loc));
 
             return Bod_Stmts;
 
@@ -7581,9 +7603,13 @@ package body Exp_Ch7 is
 
          --    Deep_Finalize (Obj._parent, False);
 
-         if Is_Tagged_Type (Typ) and then Is_Derived_Type (Typ) then
+         if Is_Derived_Type (Typ) then
             declare
-               Par_Typ  : constant Entity_Id := Parent_Field_Type (Typ);
+               Tagd     : constant Boolean := Is_Tagged_Type (Typ);
+               Par_Typ  : constant Entity_Id :=
+                 (if Tagd
+                  then Parent_Field_Type (Typ)
+                  else Etype (Base_Type (Typ)));
                Call     : Node_Id;
                Fin_Stmt : Node_Id;
 
@@ -7592,10 +7618,16 @@ package body Exp_Ch7 is
                   Call :=
                     Make_Final_Call
                       (Obj_Ref   =>
-                         Make_Selected_Component (Loc,
-                           Prefix        => Make_Identifier (Loc, Name_V),
-                           Selector_Name =>
-                             Make_Identifier (Loc, Name_uParent)),
+                         (if Tagd
+                          then
+                            Make_Selected_Component
+                              (Loc,
+                               Prefix        => Make_Identifier (Loc, Name_V),
+                               Selector_Name =>
+                                 Make_Identifier (Loc, Name_uParent))
+                          else
+                            Convert_To
+                              (Par_Typ, Make_Identifier (Loc, Name_V))),
                        Typ       => Par_Typ,
                        Skip_Self => True);
 
@@ -7611,6 +7643,21 @@ package body Exp_Ch7 is
                   --               Get_Current_Excep.all.all);
                   --          end if;
                   --    end;
+                  --
+                  --  in the tagged case. In the untagged case, which arises
+                  --  with the Destructor aspect, generate:
+                  --
+                  --    begin
+                  --       Deep_Finalize (Parent_Type (V), False);
+
+                  --    exception
+                  --       when Id : others =>
+                  --          if not Raised then
+                  --             Raised := True;
+                  --             Save_Occurrence (E,
+                  --               Get_Current_Excep.all.all);
+                  --          end if;
+                  --    end;
 
                   if Present (Call) then
                      Fin_Stmt := Call;
@@ -7656,7 +7703,7 @@ package body Exp_Ch7 is
                         --  than before, the extension components. That might
                         --  be more intuitive (as discussed in preceding
                         --  comment), but it is not required.
-                        Prepend_To (Bod_Stmts, Fin_Stmt);
+                        Prepend_New_To (Bod_Stmts, Fin_Stmt);
                      end if;
                   end if;
                end if;
@@ -7707,12 +7754,58 @@ package body Exp_Ch7 is
                                  (Finalizer_Data))));
                   end if;
 
-                  Prepend_To (Bod_Stmts,
+                  Prepend_New_To (Bod_Stmts,
                     Make_If_Statement (Loc,
                       Condition       => Make_Identifier (Loc, Name_F),
                       Then_Statements => New_List (Fin_Stmt)));
                end if;
             end;
+
+            declare
+               ASN : constant Opt_N_Aspect_Specification_Id :=
+                 Get_Rep_Item (Typ, Name_Destructor, False);
+
+               Stmt : Node_Id;
+               Proc : Entity_Id;
+            begin
+               if Present (ASN) then
+                  --  Generate:
+                  --    begin
+                  --       <Destructor_Proc> (V);
+
+                  --    exception
+                  --       when others =>
+                  --          if not Raised then
+                  --             Raised := True;
+                  --             Save_Occurrence (E,
+                  --               Get_Current_Excep.all.all);
+                  --          end if;
+                  --    end;
+
+                  Proc := Entity (Expression (ASN));
+                  Stmt :=
+                    Make_Procedure_Call_Statement
+                      (Loc,
+                       Name                   => New_Occurrence_Of (Proc, Loc),
+                       Parameter_Associations =>
+                         New_List (Make_Identifier (Loc, Name_V)));
+                  if Exceptions_OK then
+                     Stmt :=
+                       Make_Block_Statement
+                         (Loc,
+                          Handled_Statement_Sequence =>
+                            Make_Handled_Sequence_Of_Statements
+                              (Loc,
+                               Statements         => New_List (Stmt),
+                               Exception_Handlers =>
+                                 New_List
+                                   (Build_Exception_Handler
+                                      (Finalizer_Data))));
+                  end if;
+
+                  Prepend_New_To (Bod_Stmts, Stmt);
+               end if;
+            end;
          end if;
 
          --  At this point either all finalization statements have been
@@ -7906,7 +7999,7 @@ package body Exp_Ch7 is
       if Is_Untagged_Derivation (Typ) then
          if Is_Protected_Type (Typ) then
             Utyp := Corresponding_Record_Type (Root_Type (Base_Type (Typ)));
-         else
+         elsif Is_Implicit_Full_View (Utyp) then
             Utyp := Underlying_Type (Root_Type (Base_Type (Typ)));
 
             if Is_Protected_Type (Utyp) then
@@ -7966,7 +8059,7 @@ package body Exp_Ch7 is
          return Empty;
 
       elsif Skip_Self then
-         if Has_Controlled_Component (Utyp) then
+         if Has_Controlled_Component (Utyp) or else Has_Destructor (Utyp) then
             if Is_Tagged_Type (Utyp) then
                Fin_Id := Find_Optional_Prim_Op (Utyp, TSS_Deep_Finalize);
             else
@@ -7979,6 +8072,7 @@ package body Exp_Ch7 is
       elsif Is_Class_Wide_Type (Typ)
         or else Is_Interface (Typ)
         or else Has_Controlled_Component (Utyp)
+        or else Has_Destructor (Utyp)
       then
          if Is_Tagged_Type (Utyp) then
             Fin_Id := Find_Optional_Prim_Op (Utyp, TSS_Deep_Finalize);
@@ -8476,7 +8570,10 @@ package body Exp_Ch7 is
 
       --  Deal with untagged derivation of private views
 
-      if Is_Untagged_Derivation (Typ) and then not Is_Conc then
+      if Is_Untagged_Derivation (Typ)
+        and then not Is_Conc
+        and then Is_Implicit_Full_View (Utyp)
+      then
          Utyp := Underlying_Type (Root_Type (Base_Type (Typ)));
          Ref  := Unchecked_Convert_To (Utyp, Ref);
 
@@ -9444,9 +9541,16 @@ package body Exp_Ch7 is
    procedure Wrap_Transient_Expression (N : Node_Id) is
       Loc  : constant Source_Ptr := Sloc (N);
       Expr : Node_Id             := Relocate_Node (N);
-      Temp : constant Entity_Id  := Make_Temporary (Loc, 'E', N);
       Typ  : constant Entity_Id  := Etype (N);
 
+      Temp : constant Entity_Id  := Make_Temporary (Loc, 'E',
+                                      Related_Node => Expr);
+      --  We link the temporary with its relocated expression to facilitate
+      --  locating the expression in the expanded code; this simplifies the
+      --  implementation of the function that searchs in the expanded code
+      --  for a function call that has been wrapped in a transient block
+      --  (see Get_Relocated_Function_Call).
+
    begin
       --  Generate:
 
diff --git a/gcc/ada/exp_ch9.adb b/gcc/ada/exp_ch9.adb
index 9cfc6b5..c979cf6 100644
--- a/gcc/ada/exp_ch9.adb
+++ b/gcc/ada/exp_ch9.adb
@@ -4691,11 +4691,11 @@ package body Exp_Ch9 is
 
       --  The availability of the activation chain entity does not ensure
       --  that we have tasks to activate because it may have been declared
-      --  by the frontend to pass a required extra formal to a build-in-place
+      --  by the front end to pass a required extra formal to a build-in-place
       --  subprogram call. If we are within the scope of a protected type and
       --  pragma Detect_Blocking is active we can assume that no tasks will be
       --  activated; if tasks are created in a protected object and this pragma
-      --  is active then the frontend emits a warning and Program_Error is
+      --  is active then the front end emits a warning and Program_Error is
       --  raised at runtime.
 
       elsif Detect_Blocking and then Within_Protected_Type (Current_Scope) then
@@ -8094,12 +8094,18 @@ package body Exp_Ch9 is
    --  access type. Finally the Entry_Component of each formal is set to
    --  reference the corresponding record component.
 
-   procedure Expand_N_Entry_Declaration (N : Node_Id) is
+   procedure Expand_N_Entry_Declaration
+     (N            : Node_Id;
+      Was_Deferred : Boolean := False)
+   is
+      use Deferred_Extra_Formals_Support;
+
       Loc        : constant Source_Ptr := Sloc (N);
       Entry_Ent  : constant Entity_Id  := Defining_Identifier (N);
       Components : List_Id;
       Formal     : Node_Id;
       Ftype      : Entity_Id;
+      First_Decl : Node_Id;
       Last_Decl  : Node_Id;
       Component  : Entity_Id;
       Ctype      : Entity_Id;
@@ -8108,7 +8114,21 @@ package body Exp_Ch9 is
       Acc_Ent    : Entity_Id;
 
    begin
+      --  No action if the addition of the extra formals was deferred,
+      --  since it means that the underlying type of some formal is not
+      --  available, and hence we cannot build the record type that will
+      --  hold all the parameter values.
+
+      if Present (First_Formal (Entry_Ent))
+        and then not Extra_Formals_Known (Entry_Ent)
+        and then not Is_Unsupported_Extra_Formals_Entity (Entry_Ent)
+      then
+         pragma Assert (Is_Deferred_Extra_Formals_Entity (Entry_Ent));
+         return;
+      end if;
+
       Formal := First_Formal (Entry_Ent);
+      First_Decl := N;
       Last_Decl := N;
 
       --  Most processing is done only if parameters are present
@@ -8184,6 +8204,24 @@ package body Exp_Ch9 is
                  Subtype_Indication => New_Occurrence_Of (Rec_Ent, Loc)));
 
          Insert_After (Last_Decl, Decl);
+         Last_Decl := Decl;
+
+         --  Analyze all the inserted declarations. This is required when
+         --  the entry has formals and the addition of its extra formals
+         --  was deferred; otherwise their analysis will be performed as
+         --  as part of the regular flow of the front end at the end of
+         --  analysis of the enclosing task/protected type declaration.
+
+         if Was_Deferred then
+            Push_Scope (Scope (Entry_Ent));
+
+            while First_Decl /= Last_Decl loop
+               Next (First_Decl);
+               Analyze (First_Decl);
+            end loop;
+
+            End_Scope;
+         end if;
       end if;
    end Expand_N_Entry_Declaration;
 
diff --git a/gcc/ada/exp_ch9.ads b/gcc/ada/exp_ch9.ads
index cae6cb3..6811141 100644
--- a/gcc/ada/exp_ch9.ads
+++ b/gcc/ada/exp_ch9.ads
@@ -227,9 +227,16 @@ package Exp_Ch9 is
    procedure Expand_N_Delay_Until_Statement      (N : Node_Id);
    procedure Expand_N_Entry_Body                 (N : Node_Id);
    procedure Expand_N_Entry_Call_Statement       (N : Node_Id);
-   procedure Expand_N_Entry_Declaration          (N : Node_Id);
    procedure Expand_N_Protected_Body             (N : Node_Id);
 
+   procedure Expand_N_Entry_Declaration
+     (N            : Node_Id;
+      Was_Deferred : Boolean := False);
+   --  Expands an entry declaration, building a record type to hold all the
+   --  parameter values. Was_Deferred is True when this expansion was deferred
+   --  because the underlying type of some formal was not available to build
+   --  the record.
+
    procedure Expand_N_Protected_Type_Declaration (N : Node_Id);
    --  Expands protected type declarations. This results, among other things,
    --  in the declaration of a record type for the representation of protected
diff --git a/gcc/ada/exp_disp.adb b/gcc/ada/exp_disp.adb
index 080a2e1..619ac40 100644
--- a/gcc/ada/exp_disp.adb
+++ b/gcc/ada/exp_disp.adb
@@ -801,7 +801,7 @@ package body Exp_Disp is
 
         --  No action needed if the dispatching call has been already expanded
 
-        or else Is_Expanded_Dispatching_Call (Name (Call_Node))
+        or else Is_Expanded_Dispatching_Call (Call_Node)
       then
          return;
       end if;
@@ -926,6 +926,8 @@ package body Exp_Disp is
          New_Formal  : Entity_Id;
          Last_Formal : Entity_Id := Empty;
 
+         use Deferred_Extra_Formals_Support;
+
       begin
          if Present (Old_Formal) then
             New_Formal := New_Copy (Old_Formal);
@@ -962,51 +964,21 @@ package body Exp_Disp is
          end if;
 
          --  Now that the explicit formals have been duplicated, any extra
-         --  formals needed by the subprogram must be duplicated; we know
-         --  that extra formals are available because they were added when
-         --  the tagged type was frozen (see Expand_Freeze_Record_Type).
+         --  formals needed by the subprogram must be added; we know that
+         --  extra formals are available because they were added when the
+         --  tagged type was frozen (see Expand_Freeze_Record_Type).
 
          pragma Assert (Is_Frozen (Typ));
 
-         --  Warning: The addition of the extra formals cannot be performed
-         --  here invoking Create_Extra_Formals since we must ensure that all
-         --  the extra formals of the pointer type and the target subprogram
-         --  match (and for functions that return a tagged type the profile of
-         --  the built subprogram type always returns a class-wide type, which
-         --  may affect the addition of some extra formals).
-
-         if Present (Last_Formal)
-           and then Present (Extra_Formal (Last_Formal))
-         then
-            Old_Formal := Extra_Formal (Last_Formal);
-            New_Formal := New_Copy (Old_Formal);
-            Set_Scope (New_Formal, Subp_Typ);
-
-            Set_Extra_Formal (Last_Formal, New_Formal);
-            Set_Extra_Formals (Subp_Typ, New_Formal);
-
-            if Ekind (Subp) = E_Function
-              and then Present (Extra_Accessibility_Of_Result (Subp))
-              and then Extra_Accessibility_Of_Result (Subp) = Old_Formal
-            then
-               Set_Extra_Accessibility_Of_Result (Subp_Typ, New_Formal);
-            end if;
-
-            Old_Formal := Extra_Formal (Old_Formal);
-            while Present (Old_Formal) loop
-               Set_Extra_Formal (New_Formal, New_Copy (Old_Formal));
-               New_Formal := Extra_Formal (New_Formal);
-               Set_Scope (New_Formal, Subp_Typ);
+         if Extra_Formals_Known (Subp) then
+            Create_Extra_Formals (Subp_Typ);
 
-               if Ekind (Subp) = E_Function
-                 and then Present (Extra_Accessibility_Of_Result (Subp))
-                 and then Extra_Accessibility_Of_Result (Subp) = Old_Formal
-               then
-                  Set_Extra_Accessibility_Of_Result (Subp_Typ, New_Formal);
-               end if;
+         --  Extra formals were previously deferred
 
-               Old_Formal := Extra_Formal (Old_Formal);
-            end loop;
+         else
+            pragma Assert (Is_Deferred_Extra_Formals_Entity (Subp));
+            Register_Deferred_Extra_Formals_Entity (Subp_Typ);
+            Register_Deferred_Extra_Formals_Call (Call_Node, Current_Scope);
          end if;
       end;
 
@@ -1237,6 +1209,8 @@ package body Exp_Disp is
       --  the generation of spurious warnings under ZFP run-time.
 
       Analyze_And_Resolve (Call_Node, Call_Typ, Suppress => All_Checks);
+
+      Set_Is_Expanded_Dispatching_Call (Call_Node);
    end Expand_Dispatching_Call;
 
    ---------------------------------
@@ -2378,17 +2352,6 @@ package body Exp_Disp is
         and then not Restriction_Active (No_Dispatching_Calls);
    end Has_DT;
 
-   ----------------------------------
-   -- Is_Expanded_Dispatching_Call --
-   ----------------------------------
-
-   function Is_Expanded_Dispatching_Call (N : Node_Id) return Boolean is
-   begin
-      return Nkind (N) in N_Subprogram_Call
-        and then Nkind (Name (N)) = N_Explicit_Dereference
-        and then Is_Dispatch_Table_Entity (Etype (Name (N)));
-   end Is_Expanded_Dispatching_Call;
-
    -------------------------------------
    -- Is_Predefined_Dispatching_Alias --
    -------------------------------------
@@ -8345,13 +8308,15 @@ package body Exp_Disp is
                         Defining_Unit_Name       => IP,
                         Parameter_Specifications => Parms)));
 
-               Set_Init_Proc (Typ, IP);
-               Set_Is_Imported    (IP);
-               Set_Is_Constructor (IP);
-               Set_Interface_Name (IP, Interface_Name (E));
-               Set_Convention     (IP, Convention_CPP);
-               Set_Is_Public      (IP);
-               Set_Has_Completion (IP);
+               Set_Init_Proc   (Typ, IP);
+               Set_Is_Imported      (IP);
+               Set_Is_Constructor   (IP);
+               Set_Interface_Name   (IP, Interface_Name (E));
+               Set_Convention       (IP, Convention_CPP);
+               Set_Is_Public        (IP);
+               Set_Has_Completion   (IP);
+               Mutate_Ekind         (IP, E_Procedure);
+               Freeze_Extra_Formals (IP);
 
             --  Case 2: Constructor of a tagged type
 
@@ -8484,6 +8449,8 @@ package body Exp_Disp is
 
                   Discard_Node (IP_Body);
                   Set_Init_Proc (Typ, IP);
+                  Mutate_Ekind (IP, E_Procedure);
+                  Freeze_Extra_Formals (IP);
                end;
             end if;
 
@@ -8549,6 +8516,8 @@ package body Exp_Disp is
 
             Discard_Node (IP_Body);
             Set_Init_Proc (Typ, IP);
+            Mutate_Ekind (IP, E_Procedure);
+            Freeze_Extra_Formals (IP);
          end;
       end if;
 
diff --git a/gcc/ada/exp_disp.ads b/gcc/ada/exp_disp.ads
index 3cba8ca..76f5923 100644
--- a/gcc/ada/exp_disp.ads
+++ b/gcc/ada/exp_disp.ads
@@ -236,9 +236,6 @@ package Exp_Disp is
    function Has_CPP_Constructors (Typ : Entity_Id) return Boolean;
    --  Returns true if the type has CPP constructors
 
-   function Is_Expanded_Dispatching_Call (N : Node_Id) return Boolean;
-   --  Returns true if N is the expanded code of a dispatching call
-
    function Make_DT (Typ : Entity_Id) return List_Id;
    --  Expand the declarations for the Dispatch Table of Typ
 
diff --git a/gcc/ada/exp_put_image.adb b/gcc/ada/exp_put_image.adb
index 40b2a65..ce3390b 100644
--- a/gcc/ada/exp_put_image.adb
+++ b/gcc/ada/exp_put_image.adb
@@ -77,8 +77,28 @@ package body Exp_Put_Image is
    --  reference). The Loc parameter is used as the Sloc of the created entity.
 
    function Put_Image_Base_Type (E : Entity_Id) return Entity_Id;
-   --  Returns the base type, except for an array type whose whose first
-   --  subtype is constrained, in which case it returns the first subtype.
+   --  For an array type whose whose first subtype is constrained, return
+   --  the first subtype. For the internal representation type corresponding
+   --  to a mutably tagged type, return the mutably tagged type. Otherwise,
+   --  return the base type. Similar to Exp_Strm.Stream_Base_Type.
+
+   procedure Put_Specific_Type_Name_Qualifier
+              (Loc               : Source_Ptr;
+               Stms              : List_Id;
+               Tagged_Obj        : Node_Id;
+               Buffer_Name       : Node_Id;
+               Is_Interface_Type : Boolean);
+   --  Append to the given statement list calls to add into the
+   --  buffer the name of the given object's tag and then a "'".
+
+   function Put_String_Exp_To_Buffer
+     (Loc         : Source_Ptr;
+      String_Exp  : Node_Id;
+      Buffer_Name : Node_Id;
+      Wide_Wide   : Boolean := False) return Node_Id;
+   --  Generate a call to evaluate a String (or Wide_Wide_String, depending
+   --  on the Wide_Wide Boolean parameter) expression and output it into
+   --  the buffer.
 
    -------------------------------------
    -- Build_Array_Put_Image_Procedure --
@@ -189,7 +209,7 @@ package body Exp_Put_Image is
       Ndim : constant Pos        := Number_Dimensions (Typ);
       Ctyp : constant Entity_Id  := Component_Type (Typ);
 
-      Stm         : Node_Id;
+      Stms        : List_Id := New_List;
       Exl         : constant List_Id := New_List;
       PI_Entity   : Entity_Id;
 
@@ -220,15 +240,36 @@ package body Exp_Put_Image is
          Append_To (Exl, Make_Identifier (Loc, New_External_Name ('L', Dim)));
       end loop;
 
-      Stm :=
-        Make_Attribute_Reference (Loc,
-          Prefix => New_Occurrence_Of (Put_Image_Base_Type (Ctyp), Loc),
-          Attribute_Name => Name_Put_Image,
-          Expressions => New_List (
-            Make_Identifier (Loc, Name_S),
-            Make_Indexed_Component (Loc,
-              Prefix      => Make_Identifier (Loc, Name_V),
-              Expressions => Exl)));
+      declare
+         Ctype_For_Call : constant Entity_Id := Put_Image_Base_Type (Ctyp);
+         Indexed_Comp   : constant Node_Id :=
+           Make_Indexed_Component (Loc,
+             Prefix      => Make_Identifier (Loc, Name_V),
+             Expressions => Exl);
+      begin
+         if Is_Mutably_Tagged_Type (Ctype_For_Call) then
+            pragma Assert (not Is_Mutably_Tagged_Type (Component_Type (Typ)));
+
+            Make_Mutably_Tagged_Conversion (Indexed_Comp,
+                                            Typ => Ctype_For_Call);
+
+            pragma Assert (Is_Mutably_Tagged_Type (Etype (Indexed_Comp)));
+
+            Put_Specific_Type_Name_Qualifier (Loc,
+               Stms              => Stms,
+               Tagged_Obj        => Indexed_Comp,
+               Buffer_Name       => Make_Identifier (Loc, Name_S),
+               Is_Interface_Type => False);
+         end if;
+
+         Append_To (Stms,
+           Make_Attribute_Reference (Loc,
+             Prefix => New_Occurrence_Of (Ctype_For_Call, Loc),
+             Attribute_Name => Name_Put_Image,
+             Expressions => New_List (
+               Make_Identifier (Loc, Name_S),
+               Indexed_Comp)));
+      end;
 
       --  The corresponding attribute for the component type of the array might
       --  be user-defined, and frozen after the array type. In that case,
@@ -245,46 +286,42 @@ package body Exp_Put_Image is
       --  Loop through the dimensions, innermost first, generating a loop for
       --  each dimension.
 
-      declare
-         Stms : List_Id := New_List (Stm);
-      begin
-         for Dim in reverse 1 .. Ndim loop
-            declare
-               New_Stms : constant List_Id := New_List;
-               Between_Proc : RE_Id;
-            begin
-               --  For a one-dimensional array of elementary type, use
-               --  RE_Simple_Array_Between. The same applies to the last
-               --  dimension of a multidimensional array.
+      for Dim in reverse 1 .. Ndim loop
+         declare
+            New_Stms : constant List_Id := New_List;
+            Between_Proc : RE_Id;
+         begin
+            --  For a one-dimensional array of elementary type, use
+            --  RE_Simple_Array_Between. The same applies to the last
+            --  dimension of a multidimensional array.
 
-               if Is_Elementary_Type (Ctyp) and then Dim = Ndim then
-                  Between_Proc := RE_Simple_Array_Between;
-               else
-                  Between_Proc := RE_Array_Between;
-               end if;
+            if Is_Elementary_Type (Ctyp) and then Dim = Ndim then
+               Between_Proc := RE_Simple_Array_Between;
+            else
+               Between_Proc := RE_Array_Between;
+            end if;
 
-               Append_To (New_Stms,
-                 Make_Procedure_Call_Statement (Loc,
-                   Name => New_Occurrence_Of (RTE (RE_Array_Before), Loc),
-                   Parameter_Associations => New_List
-                     (Make_Identifier (Loc, Name_S))));
+            Append_To (New_Stms,
+              Make_Procedure_Call_Statement (Loc,
+                Name => New_Occurrence_Of (RTE (RE_Array_Before), Loc),
+                Parameter_Associations => New_List
+                  (Make_Identifier (Loc, Name_S))));
 
-               Append_To
-                 (New_Stms,
-                  Wrap_In_Loop (Stms, Dim, Indices (Dim), Between_Proc));
+            Append_To
+              (New_Stms,
+               Wrap_In_Loop (Stms, Dim, Indices (Dim), Between_Proc));
 
-               Append_To (New_Stms,
-                 Make_Procedure_Call_Statement (Loc,
-                   Name => New_Occurrence_Of (RTE (RE_Array_After), Loc),
-                   Parameter_Associations => New_List
-                     (Make_Identifier (Loc, Name_S))));
+            Append_To (New_Stms,
+              Make_Procedure_Call_Statement (Loc,
+                Name => New_Occurrence_Of (RTE (RE_Array_After), Loc),
+                Parameter_Associations => New_List
+                  (Make_Identifier (Loc, Name_S))));
 
-               Stms := New_Stms;
-            end;
-         end loop;
+            Stms := New_Stms;
+         end;
+      end loop;
 
-         Build_Put_Image_Proc (Loc, Typ, Decl, Pnam, Stms);
-      end;
+      Build_Put_Image_Proc (Loc, Typ, Decl, Pnam, Stms);
    end Build_Array_Put_Image_Procedure;
 
    -------------------------------------
@@ -379,7 +416,8 @@ package body Exp_Put_Image is
          begin
             --  We have built a dispatching call to handle calls to
             --  descendants (since they are not available through rtsfind).
-            --  Further details available in the body of Put_String_Exp.
+            --  Further details available in the body of
+            --  Put_String_Exp_To_Buffer.
 
             return Put_Call;
          end;
@@ -691,19 +729,33 @@ package body Exp_Put_Image is
       ---------------------------
 
       procedure Append_Component_Attr (Clist : List_Id; C : Entity_Id) is
-         Component_Typ : constant Entity_Id :=
-           Put_Image_Base_Type
-             (Get_Corresponding_Mutably_Tagged_Type_If_Present (Etype (C)));
+         Component_Typ : constant Entity_Id := Put_Image_Base_Type (Etype (C));
+         Selected_Comp : constant Node_Id :=
+           Make_Selected_Component (Loc,
+             Prefix        => Make_Identifier (Loc, Name_V),
+             Selector_Name => New_Occurrence_Of (C, Loc));
       begin
+         if Is_Mutably_Tagged_Type (Component_Typ) then
+            pragma Assert (not Is_Mutably_Tagged_Type (Etype (C)));
+
+            Make_Mutably_Tagged_Conversion (Selected_Comp,
+                                            Typ => Component_Typ);
+
+            pragma Assert (Is_Mutably_Tagged_Type (Etype (Selected_Comp)));
+
+            Put_Specific_Type_Name_Qualifier (Loc,
+               Stms              => Clist,
+               Tagged_Obj        => Selected_Comp,
+               Buffer_Name       => Make_Identifier (Loc, Name_S),
+               Is_Interface_Type => False);
+         end if;
+
          Append_To (Clist,
            Make_Attribute_Reference (Loc,
              Prefix         => New_Occurrence_Of (Component_Typ, Loc),
              Attribute_Name => Name_Put_Image,
-             Expressions    => New_List (
-               Make_Identifier (Loc, Name_S),
-               Make_Selected_Component (Loc,
-                 Prefix        => Make_Identifier (Loc, Name_V),
-                 Selector_Name => New_Occurrence_Of (C, Loc)))));
+             Expressions    => New_List (Make_Identifier (Loc, Name_S),
+                                         Selected_Comp)));
       end Append_Component_Attr;
 
       -------------------------------
@@ -1303,105 +1355,20 @@ package body Exp_Put_Image is
                 New_Occurrence_Of (Sink_Entity, Loc))));
       Actions : List_Id;
 
-      function Put_String_Exp (String_Exp : Node_Id;
-                               Wide_Wide  : Boolean := False) return Node_Id;
-      --  Generate a call to evaluate a String (or Wide_Wide_String, depending
-      --  on the Wide_Wide Boolean parameter) expression and output it into
-      --  the buffer.
-
-      --------------------
-      -- Put_String_Exp --
-      --------------------
-
-      function Put_String_Exp (String_Exp : Node_Id;
-                               Wide_Wide  : Boolean := False) return Node_Id is
-         Put_Id : constant RE_Id :=
-           (if Wide_Wide then RE_Wide_Wide_Put else RE_Put_UTF_8);
-
-         --  We could build a nondispatching call here, but to make
-         --  that work we'd have to change Rtsfind spec to make available
-         --  corresponding callees out of Ada.Strings.Text_Buffers.Unbounded
-         --  (as opposed to from Ada.Strings.Text_Buffers). Seems simpler to
-         --  introduce a type conversion and leave it to the optimizer to
-         --  eliminate the dispatching. This does not *introduce* any problems
-         --  if a no-dispatching-allowed restriction is in effect, since we
-         --  are already in the middle of generating a call to T'Class'Image.
-
-         Sink_Exp : constant Node_Id :=
-           Make_Type_Conversion (Loc,
-             Subtype_Mark =>
-               New_Occurrence_Of
-                 (Class_Wide_Type (RTE (RE_Root_Buffer_Type)), Loc),
-             Expression   => New_Occurrence_Of (Sink_Entity, Loc));
-      begin
-         return
-           Make_Procedure_Call_Statement (Loc,
-             Name => New_Occurrence_Of (RTE (Put_Id), Loc),
-             Parameter_Associations => New_List (Sink_Exp, String_Exp));
-      end Put_String_Exp;
-
-      --  Local variables
-
-      Tag_Node : Node_Id;
-
    --  Start of processing for Build_Image_Call
 
    begin
       if Is_Class_Wide_Type (U_Type) then
+         Actions := New_List (Sink_Decl);
 
-         --  For interface types we must generate code to displace the pointer
-         --  to the object to reference the base of the underlying object.
-
-         --  Generate:
-         --    To_Tag_Ptr (Image_Prefix'Address).all
-
-         --  Note that Image_Prefix'Address is recursively expanded into a
-         --  call to Ada.Tags.Base_Address (Image_Prefix'Address).
-
-         if Is_Interface (U_Type) then
-            Tag_Node :=
-              Make_Explicit_Dereference (Loc,
-                Unchecked_Convert_To (RTE (RE_Tag_Ptr),
-                  Make_Attribute_Reference (Loc,
-                    Prefix => Duplicate_Subexpr (Image_Prefix),
-                    Attribute_Name => Name_Address)));
+         Put_Specific_Type_Name_Qualifier (Loc,
+           Stms              => Actions,
+           Tagged_Obj        => Image_Prefix,
+           Buffer_Name       => New_Occurrence_Of (Sink_Entity, Loc),
+           Is_Interface_Type => Is_Interface (U_Type));
 
-         --  Common case
-
-         else
-            Tag_Node :=
-              Make_Attribute_Reference (Loc,
-                Prefix         => Duplicate_Subexpr (Image_Prefix),
-                Attribute_Name => Name_Tag);
-         end if;
-
-         --  Generate qualified-expression syntax; qualification name comes
-         --  from calling Ada.Tags.Wide_Wide_Expanded_Name.
-
-         declare
-            --  The copy of Image_Prefix will be evaluated before the
-            --  original, which is ok if no side effects are involved.
-
-            pragma Assert (Side_Effect_Free (Image_Prefix));
-
-            Specific_Type_Name : constant Node_Id :=
-              Put_String_Exp
-                (Make_Function_Call (Loc,
-                   Name => New_Occurrence_Of
-                             (RTE (RE_Wide_Wide_Expanded_Name), Loc),
-                   Parameter_Associations => New_List (Tag_Node)),
-                 Wide_Wide => True);
-
-            Qualification : constant Node_Id :=
-              Put_String_Exp (Make_String_Literal (Loc, "'"));
-         begin
-            Actions := New_List
-                         (Sink_Decl,
-                          Specific_Type_Name,
-                          Qualification,
-                          Put_Im,
-                          Result_Decl);
-         end;
+         Append_To (Actions, Put_Im);
+         Append_To (Actions, Result_Decl);
       else
          Actions := New_List (Sink_Decl, Put_Im, Result_Decl);
       end if;
@@ -1485,9 +1452,89 @@ package body Exp_Put_Image is
          return E;
       elsif Is_Private_Type (Base_Type (E)) and not Is_Private_Type (E) then
          return Implementation_Base_Type (E);
+      elsif Is_Mutably_Tagged_CW_Equivalent_Type (E) then
+         return Get_Corresponding_Mutably_Tagged_Type_If_Present (E);
       else
          return Base_Type (E);
       end if;
    end Put_Image_Base_Type;
 
+   --------------------------------------
+   -- Put_Specific_Type_Name_Qualifier --
+   --------------------------------------
+
+   procedure Put_Specific_Type_Name_Qualifier
+     (Loc               : Source_Ptr;
+      Stms              : List_Id;
+      Tagged_Obj        : Node_Id;
+      Buffer_Name       : Node_Id;
+      Is_Interface_Type : Boolean)
+   is
+      Tag_Node : Node_Id;
+   begin
+      if Is_Interface_Type then
+         Tag_Node :=
+           Make_Explicit_Dereference (Loc,
+             Unchecked_Convert_To (RTE (RE_Tag_Ptr),
+               Make_Attribute_Reference (Loc,
+                 Prefix => New_Copy_Tree (Tagged_Obj),
+                 Attribute_Name => Name_Address)));
+      else
+         Tag_Node :=
+           Make_Attribute_Reference (Loc,
+             Prefix         => New_Copy_Tree (Tagged_Obj),
+             Attribute_Name => Name_Tag);
+      end if;
+
+      Append_To (Stms,
+                 Put_String_Exp_To_Buffer (Loc,
+                   String_Exp =>
+                     Make_Function_Call (Loc,
+                       Name => New_Occurrence_Of
+                                 (RTE (RE_Wide_Wide_Expanded_Name), Loc),
+                       Parameter_Associations => New_List (Tag_Node)),
+                   Buffer_Name => Buffer_Name,
+                   Wide_Wide => True));
+
+      Append_To (Stms,
+                 Put_String_Exp_To_Buffer (Loc,
+                   String_Exp  => Make_String_Literal (Loc, "'"),
+                   Buffer_Name => New_Copy_Tree (Buffer_Name)));
+   end Put_Specific_Type_Name_Qualifier;
+
+   ------------------------------
+   -- Put_String_Exp_To_Buffer --
+   ------------------------------
+
+   function Put_String_Exp_To_Buffer
+     (Loc         : Source_Ptr;
+      String_Exp  : Node_Id;
+      Buffer_Name : Node_Id;
+      Wide_Wide   : Boolean := False) return Node_Id
+   is
+      Put_Id : constant RE_Id :=
+        (if Wide_Wide then RE_Wide_Wide_Put else RE_Put_UTF_8);
+
+      --  We could build a nondispatching call here, but to make
+      --  that work we'd have to change Rtsfind spec to make available
+      --  corresponding callees out of Ada.Strings.Text_Buffers.Unbounded
+      --  (as opposed to from Ada.Strings.Text_Buffers). Seems simpler to
+      --  introduce a type conversion and leave it to the optimizer to
+      --  eliminate the dispatching. This does not *introduce* any problems
+      --  if a no-dispatching-allowed restriction is in effect, since we
+      --  are already in the middle of generating a call to T'Class'Image.
+
+      Sink_Exp : constant Node_Id :=
+        Make_Type_Conversion (Loc,
+          Subtype_Mark =>
+            New_Occurrence_Of
+              (Class_Wide_Type (RTE (RE_Root_Buffer_Type)), Loc),
+          Expression   => Buffer_Name);
+   begin
+      return
+        Make_Procedure_Call_Statement (Loc,
+          Name => New_Occurrence_Of (RTE (Put_Id), Loc),
+          Parameter_Associations => New_List (Sink_Exp, String_Exp));
+   end Put_String_Exp_To_Buffer;
+
 end Exp_Put_Image;
diff --git a/gcc/ada/exp_spark.adb b/gcc/ada/exp_spark.adb
index 6e1c86a..a75a507 100644
--- a/gcc/ada/exp_spark.adb
+++ b/gcc/ada/exp_spark.adb
@@ -73,6 +73,10 @@ package body Exp_SPARK is
    procedure Expand_SPARK_N_Attribute_Reference (N : Node_Id);
    --  Perform attribute-reference-specific expansion
 
+   procedure Expand_SPARK_N_Continue_Statement (N : Node_Id);
+   --  Expand continue statements which are resolved as procedure calls, into
+   --  said procedure calls. Real continue statements are left as-is.
+
    procedure Expand_SPARK_N_Delta_Aggregate (N : Node_Id);
    --  Perform delta-aggregate-specific expansion
 
@@ -191,6 +195,9 @@ package body Exp_SPARK is
 
          --  In SPARK mode, no other constructs require expansion
 
+         when N_Continue_Statement =>
+            Expand_SPARK_N_Continue_Statement (N);
+
          when others =>
             null;
       end case;
@@ -435,6 +442,23 @@ package body Exp_SPARK is
       end if;
    end Expand_SPARK_Delta_Or_Update;
 
+   ---------------------------------------
+   -- Expand_SPARK_N_Continue_Statement --
+   ---------------------------------------
+
+   procedure Expand_SPARK_N_Continue_Statement (N : Node_Id) is
+      X : constant Node_Id := Call_Or_Target_Loop (N);
+   begin
+      if No (X) then
+         return;
+      end if;
+
+      if Nkind (X) = N_Procedure_Call_Statement then
+         Replace (N, X);
+         Analyze (N);
+      end if;
+   end Expand_SPARK_N_Continue_Statement;
+
    ------------------------------
    -- Expand_SPARK_N_Aggregate --
    ------------------------------
diff --git a/gcc/ada/exp_strm.adb b/gcc/ada/exp_strm.adb
index 250efd2..5e1c913 100644
--- a/gcc/ada/exp_strm.adb
+++ b/gcc/ada/exp_strm.adb
@@ -29,6 +29,7 @@ with Einfo.Entities; use Einfo.Entities;
 with Einfo.Utils;    use Einfo.Utils;
 with Elists;         use Elists;
 with Exp_Util;       use Exp_Util;
+with Mutably_Tagged; use Mutably_Tagged;
 with Namet;          use Namet;
 with Nlists;         use Nlists;
 with Nmake;          use Nmake;
@@ -101,13 +102,10 @@ package body Exp_Strm is
    --  Loc parameter is used as the Sloc of the created entity.
 
    function Stream_Base_Type (E : Entity_Id) return Entity_Id;
-   --  Stream attributes work on the basis of the base type except for the
-   --  array case. For the array case, we do not go to the base type, but
-   --  to the first subtype if it is constrained. This avoids problems with
-   --  incorrect conversions in the packed array case. Stream_Base_Type is
-   --  exactly this function (returns the base type, unless we have an array
-   --  type whose first subtype is constrained, in which case it returns the
-   --  first subtype).
+   --  For an array type whose whose first subtype is constrained, return
+   --  the first subtype. For the internal representation type corresponding
+   --  to a mutably tagged type, return the mutably tagged type. Otherwise,
+   --  return the base type. Similar to Exp_Put_Image.Put_Image_Base_Type.
 
    --------------------------------
    -- Build_Array_Input_Function --
@@ -1502,6 +1500,7 @@ package body Exp_Strm is
 
       function Make_Field_Attribute (C : Entity_Id) return Node_Id is
          Field_Typ : constant Entity_Id := Stream_Base_Type (Etype (C));
+         Selected  : Node_Id;
 
          TSS_Names : constant array (Name_Input .. Name_Write) of
                        TSS_Name_Type :=
@@ -1524,15 +1523,23 @@ package body Exp_Strm is
             return Make_Null_Statement (Loc);
          end if;
 
+         Selected := Make_Selected_Component (Loc,
+                       Prefix        => Make_Identifier (Loc, Name_V),
+                       Selector_Name => New_Occurrence_Of (C, Loc));
+
+         if Is_Mutably_Tagged_CW_Equivalent_Type (Etype (C)) then
+            Make_Mutably_Tagged_Conversion
+              (Selected,
+               Typ => Get_Corresponding_Mutably_Tagged_Type_If_Present
+                        (Etype (C)));
+         end if;
+
          return
            Make_Attribute_Reference (Loc,
              Prefix         => New_Occurrence_Of (Field_Typ, Loc),
              Attribute_Name => Nam,
-             Expressions    => New_List (
-               Make_Identifier (Loc, Name_S),
-               Make_Selected_Component (Loc,
-                 Prefix        => Make_Identifier (Loc, Name_V),
-                 Selector_Name => New_Occurrence_Of (C, Loc))));
+             Expressions    => New_List (Make_Identifier (Loc, Name_S),
+                                         Selected));
       end Make_Field_Attribute;
 
       ---------------------------
@@ -1808,6 +1815,10 @@ package body Exp_Strm is
 
    function Stream_Base_Type (E : Entity_Id) return Entity_Id is
    begin
+      if Is_Class_Wide_Equivalent_Type (E) then
+         return Corresponding_Mutably_Tagged_Type (E);
+      end if;
+
       if Is_Array_Type (E)
         and then Is_First_Subtype (E)
       then
diff --git a/gcc/ada/exp_util.adb b/gcc/ada/exp_util.adb
index 4f98779..5a6fca0 100644
--- a/gcc/ada/exp_util.adb
+++ b/gcc/ada/exp_util.adb
@@ -6189,7 +6189,7 @@ package body Exp_Util is
          if Is_Protected_Type (Btyp) then
             Utyp := Corresponding_Record_Type (Root_Type (Btyp));
 
-         else
+         elsif Is_Implicit_Full_View (Utyp) then
             Utyp := Underlying_Type (Root_Type (Btyp));
 
             if Is_Protected_Type (Utyp) then
@@ -11730,34 +11730,6 @@ package body Exp_Util is
       end if;
    end Matching_Standard_Type;
 
-   -----------------------------
-   -- May_Generate_Large_Temp --
-   -----------------------------
-
-   --  At the current time, the only types that we return False for (i.e. where
-   --  we decide we know they cannot generate large temps) are ones where we
-   --  know the size is 256 bits or less at compile time, and we are still not
-   --  doing a thorough job on arrays and records.
-
-   function May_Generate_Large_Temp (Typ : Entity_Id) return Boolean is
-   begin
-      if not Size_Known_At_Compile_Time (Typ) then
-         return False;
-      end if;
-
-      if Known_Esize (Typ) and then Esize (Typ) <= 256 then
-         return False;
-      end if;
-
-      if Is_Array_Type (Typ)
-        and then Present (Packed_Array_Impl_Type (Typ))
-      then
-         return May_Generate_Large_Temp (Packed_Array_Impl_Type (Typ));
-      end if;
-
-      return True;
-   end May_Generate_Large_Temp;
-
    ---------------------------------------
    -- Move_To_Initialization_Statements --
    ---------------------------------------
@@ -13760,11 +13732,12 @@ package body Exp_Util is
    --  The above requirements should be documented in Sinfo ???
 
    function Safe_Unchecked_Type_Conversion (Exp : Node_Id) return Boolean is
+      Pexp : constant Node_Id := Parent (Exp);
+
       Otyp   : Entity_Id;
       Ityp   : Entity_Id;
       Oalign : Uint;
       Ialign : Uint;
-      Pexp   : constant Node_Id := Parent (Exp);
 
    begin
       --  If the expression is the RHS of an assignment or object declaration
@@ -13782,18 +13755,12 @@ package body Exp_Util is
          return True;
 
       --  If the expression is the prefix of an N_Selected_Component we should
-      --  also be OK because GCC knows to look inside the conversion except if
-      --  the type is discriminated. We assume that we are OK anyway if the
-      --  type is not set yet or if it is controlled since we can't afford to
-      --  introduce a temporary in this case.
+      --  also be OK because GCC knows to look inside the conversion.
 
       elsif Nkind (Pexp) = N_Selected_Component
         and then Prefix (Pexp) = Exp
       then
-         return No (Etype (Pexp))
-           or else not Is_Type (Etype (Pexp))
-           or else not Has_Discriminants (Etype (Pexp))
-           or else Is_Constrained (Etype (Pexp));
+         return True;
       end if;
 
       --  Set the output type, this comes from Etype if it is set, otherwise we
@@ -13866,14 +13833,7 @@ package body Exp_Util is
       --  known size, but we can't consider them that way here, because we are
       --  talking about the actual size of the object.
 
-      --  We also make sure that in addition to the size being known, we do not
-      --  have a case which might generate an embarrassingly large temp in
-      --  stack checking mode.
-
       elsif Size_Known_At_Compile_Time (Otyp)
-        and then
-          (not Stack_Checking_Enabled
-            or else not May_Generate_Large_Temp (Otyp))
         and then not (Is_Record_Type (Otyp) and then not Is_Constrained (Otyp))
       then
          return True;
diff --git a/gcc/ada/exp_util.ads b/gcc/ada/exp_util.ads
index b8b7525..4226fcc 100644
--- a/gcc/ada/exp_util.ads
+++ b/gcc/ada/exp_util.ads
@@ -1064,16 +1064,6 @@ package Exp_Util is
    --  typically return Standard_Short_Integer. For fixed-point types, this
    --  will return integer types of the corresponding size.
 
-   function May_Generate_Large_Temp (Typ : Entity_Id) return Boolean;
-   --  Determines if the given type, Typ, may require a large temporary of the
-   --  kind that causes back-end trouble if stack checking is enabled. The
-   --  result is True only the size of the type is known at compile time and
-   --  large, where large is defined heuristically by the body of this routine.
-   --  The purpose of this routine is to help avoid generating troublesome
-   --  temporaries that interfere with stack checking mechanism. Note that the
-   --  caller has to check whether stack checking is actually enabled in order
-   --  to guide the expansion (typically of a function call).
-
    procedure Move_To_Initialization_Statements (Decl, Stop : Node_Id);
    --  Decl is an N_Object_Declaration node and Stop is a node past Decl in
    --  the same list. Move all the nodes on the list between Decl and Stop
diff --git a/gcc/ada/freeze.adb b/gcc/ada/freeze.adb
index be2115a..dbd7cf4 100644
--- a/gcc/ada/freeze.adb
+++ b/gcc/ada/freeze.adb
@@ -765,6 +765,9 @@ package body Freeze is
       --  in fact constrained by non-static discriminant values. Could be made
       --  more precise ???
 
+      function Value_Known (Exp : Node_Id) return Boolean;
+      --  Return True if the value of expression Exp is known at compile time
+
       --------------------
       -- Set_Small_Size --
       --------------------
@@ -880,13 +883,13 @@ package body Freeze is
                      High := Type_High_Bound (Etype (Index));
                   end if;
 
-                  if not Compile_Time_Known_Value (Low)
-                    or else not Compile_Time_Known_Value (High)
-                    or else Etype (Index) = Any_Type
-                  then
+                  if Etype (Index) = Any_Type then
                      return False;
 
-                  else
+                  elsif Compile_Time_Known_Value (Low)
+                    and then Compile_Time_Known_Value (High)
+                  then
+
                      Dim := Expr_Value (High) - Expr_Value (Low) + 1;
 
                      if Dim > Uint_0 then
@@ -894,6 +897,12 @@ package body Freeze is
                      else
                         Size := Uint_0;
                      end if;
+
+                  elsif Value_Known (Low) and then Value_Known (High) then
+                     Size := Uint_0;
+
+                  else
+                     return False;
                   end if;
 
                   Next_Index (Index);
@@ -1160,6 +1169,70 @@ package body Freeze is
          return True;
       end Static_Discriminated_Components;
 
+      -----------------
+      -- Value_Known --
+      -----------------
+
+      function Value_Known (Exp : Node_Id) return Boolean is
+      begin
+         --  This is the immediate case
+
+         if Compile_Time_Known_Value (Exp) then
+            return True;
+         end if;
+
+         --  The value may be known only to the back end, the typical example
+         --  being the alignment or the various sizes of composite types; in
+         --  the latter case, we may mutually recurse with Size_Known.
+
+         case Nkind (Exp) is
+            when N_Attribute_Reference =>
+               declare
+                  P : constant Node_Id := Prefix (Exp);
+
+               begin
+                  if not Is_Entity_Name (P)
+                    or else not Is_Type (Entity (P))
+                  then
+                     return False;
+                  end if;
+
+                  case Get_Attribute_Id (Attribute_Name (Exp)) is
+                     when Attribute_Alignment =>
+                        return True;
+
+                     when Attribute_Component_Size =>
+                        return Size_Known (Component_Type (Entity (P)));
+
+                     when Attribute_Object_Size
+                        | Attribute_Size
+                        | Attribute_Value_Size
+                     =>
+                        return Size_Known (Entity (P));
+
+                     when others =>
+                        return False;
+                  end case;
+               end;
+
+            when N_Binary_Op =>
+               return Value_Known (Left_Opnd (Exp))
+                 and then Value_Known (Right_Opnd (Exp));
+
+            when N_Qualified_Expression
+               | N_Type_Conversion
+               | N_Unchecked_Type_Conversion
+            =>
+               return Value_Known (Expression (Exp));
+
+            when N_Unary_Op =>
+               return Value_Known (Right_Opnd (Exp));
+
+            when others =>
+               return False;
+         end case;
+      end Value_Known;
+
    --  Start of processing for Check_Compile_Time_Size
 
    begin
@@ -7158,6 +7231,35 @@ package body Freeze is
             end if;
 
             Inherit_Aspects_At_Freeze_Point (E);
+
+            --  Destructor legality check
+
+            if Present (Primitive_Operations (E)) then
+               declare
+                  Subp             : Entity_Id;
+                  Parent_Operation : Entity_Id;
+
+                  Elmt : Elmt_Id := First_Elmt (Primitive_Operations (E));
+
+               begin
+                  while Present (Elmt) loop
+                     Subp := Node (Elmt);
+
+                     if Present (Overridden_Operation (Subp)) then
+                        Parent_Operation := Overridden_Operation (Subp);
+
+                        if Ekind (Parent_Operation) = E_Procedure
+                          and then Is_Destructor (Parent_Operation)
+                        then
+                           Error_Msg_N ("cannot override destructor", Subp);
+                        end if;
+                     end if;
+
+                     Next_Elmt (Elmt);
+                  end loop;
+               end;
+            end if;
+
          end if;
 
          --  Case of array type
@@ -8057,6 +8159,7 @@ package body Freeze is
          if Ekind (E) = E_Anonymous_Access_Subprogram_Type
            and then Ekind (Designated_Type (E)) = E_Subprogram_Type
          then
+            Create_Extra_Formals (Designated_Type (E));
             Layout_Type (Etype (Designated_Type (E)));
          end if;
 
@@ -10320,6 +10423,8 @@ package body Freeze is
 
       --  Local variables
 
+      use Deferred_Extra_Formals_Support;
+
       F      : Entity_Id;
       Retype : Entity_Id;
 
@@ -10420,8 +10525,11 @@ package body Freeze is
             Create_Extra_Formals (E);
 
             pragma Assert
-              ((Ekind (E) = E_Subprogram_Type
-                  and then Extra_Formals_OK (E))
+              ((Extra_Formals_Known (E)
+                 or else Is_Deferred_Extra_Formals_Entity (E))
+               or else
+                 (Ekind (E) = E_Subprogram_Type
+                   and then Extra_Formals_OK (E))
                or else
                  (Is_Subprogram (E)
                    and then Extra_Formals_OK (E)
@@ -10450,6 +10558,10 @@ package body Freeze is
       else
          Set_Mechanisms (E);
 
+         if not Extra_Formals_Known (E) then
+            Freeze_Extra_Formals (E);
+         end if;
+
          --  For foreign conventions, warn about return of unconstrained array
 
          if Ekind (E) = E_Function then
@@ -10505,6 +10617,11 @@ package body Freeze is
          end if;
       end if;
 
+      --  Check formals matching in thunks
+
+      pragma Assert (not Is_Thunk (E)
+        or else Extra_Formals_Match_OK (Thunk_Entity (E), E));
+
       --  Pragma Inline_Always is disallowed for dispatching subprograms
       --  because the address of such subprograms is saved in the dispatch
       --  table to support dispatching calls, and dispatching calls cannot
diff --git a/gcc/ada/gcc-interface/Make-lang.in b/gcc/ada/gcc-interface/Make-lang.in
index 1c93816..bbbd697 100644
--- a/gcc/ada/gcc-interface/Make-lang.in
+++ b/gcc/ada/gcc-interface/Make-lang.in
@@ -1096,7 +1096,7 @@ check-ada-subtargets: check-acats-subtargets check-gnat-subtargets
 # No ada-specific selftests
 selftest-ada:
 
-ACATSDIR = $(TESTSUITEDIR)/ada/acats-2
+ACATSDIR = $(TESTSUITEDIR)/ada/acats-4
 ACATSCMD = run_acats.sh
 
 check_acats_numbers0:=1 2 3 4 5 6 7 8 9
diff --git a/gcc/ada/gcc-interface/Makefile.in b/gcc/ada/gcc-interface/Makefile.in
index 3557b46..d456ac1 100644
--- a/gcc/ada/gcc-interface/Makefile.in
+++ b/gcc/ada/gcc-interface/Makefile.in
@@ -314,16 +314,16 @@ Makefile: ../config.status $(srcdir)/ada/gcc-interface/Makefile.in $(srcdir)/ada
 GNATLINK_OBJS = gnatlink.o \
  a-except.o ali.o alloc.o butil.o casing.o csets.o debug.o fmap.o fname.o \
  gnatvsn.o hostparm.o indepsw.o interfac.o i-c.o i-cstrin.o namet.o opt.o \
- osint.o output.o rident.o s-exctab.o s-secsta.o s-stalib.o s-stoele.o \
- sdefault.o snames.o stylesw.o switch.o system.o table.o targparm.o \
- types.o validsw.o widechar.o
+ osint.o output.o rident.o s-excmac.o s-exctab.o s-secsta.o s-stalib.o \
+ s-stoele.o sdefault.o snames.o stylesw.o switch.o system.o table.o \
+ targparm.o types.o validsw.o widechar.o
 
 GNATMAKE_OBJS = a-except.o ali.o ali-util.o aspects.o s-casuti.o alloc.o \
  atree.o binderr.o butil.o casing.o csets.o debug.o elists.o einfo.o errout.o \
  erroutc.o errutil.o err_vars.o fmap.o fname.o fname-uf.o fname-sf.o \
  gnatmake.o gnatvsn.o hostparm.o interfac.o i-c.o i-cstrin.o krunch.o lib.o \
  make.o makeusg.o make_util.o namet.o nlists.o opt.o osint.o osint-m.o \
- output.o restrict.o rident.o s-exctab.o s-cautns.o \
+ output.o restrict.o rident.o s-cautns.o s-excmac.o s-exctab.o \
  s-secsta.o s-stalib.o s-stoele.o scans.o scng.o sdefault.o sfn_scan.o \
  s-purexc.o s-htable.o scil_ll.o sem_aux.o sinfo.o sinput.o sinput-c.o \
  snames.o stand.o stringt.o styleg.o stylesw.o system.o validsw.o \
@@ -840,35 +840,6 @@ gnatlib-shared:
 	     PICFLAG_FOR_TARGET="$(PICFLAG_FOR_TARGET)" \
 	     $(GNATLIB_SHARED)
 
-# When building a SJLJ runtime for VxWorks, we need to ensure that the extra
-# linker options needed for ZCX are not passed to prevent the inclusion of
-# useless objects and potential troubles from the presence of extra symbols
-# and references in some configurations.  The inhibition is performed by
-# commenting the pragma instead of deleting the line, as the latter might
-# result in getting multiple blank lines, hence possible style check errors.
-gnatlib-sjlj:
-	$(MAKE) $(FLAGS_TO_PASS) \
-	     EH_MECHANISM="" \
-	     MULTISUBDIR="$(MULTISUBDIR)" \
-	     THREAD_KIND="$(THREAD_KIND)" \
-	     LN_S="$(LN_S)" \
-	     ../stamp-gnatlib1-$(RTSDIR)
-	sed \
-	  -e 's/Frontend_Exceptions.*/Frontend_Exceptions       : constant Boolean := True;/' \
-	  -e 's/ZCX_By_Default.*/ZCX_By_Default            : constant Boolean := False;/' \
-	  $(RTSDIR)/system.ads > $(RTSDIR)/s.ads
-	$(MV) $(RTSDIR)/s.ads $(RTSDIR)/system.ads
-	$(MAKE) $(FLAGS_TO_PASS) \
-	     EH_MECHANISM="" \
-	     GNATLIBFLAGS="$(GNATLIBFLAGS)" \
-	     GNATLIBCFLAGS="$(GNATLIBCFLAGS)" \
-	     GNATLIBCFLAGS_FOR_C="$(GNATLIBCFLAGS_FOR_C)" \
-	     FORCE_DEBUG_ADAFLAGS="$(FORCE_DEBUG_ADAFLAGS)" \
-	     MULTISUBDIR="$(MULTISUBDIR)" \
-	     THREAD_KIND="$(THREAD_KIND)" \
-	     LN_S="$(LN_S)" \
-	     gnatlib
-
 gnatlib-zcx:
 	$(MAKE) $(FLAGS_TO_PASS) \
 	     EH_MECHANISM="-gcc" \
@@ -877,7 +848,6 @@ gnatlib-zcx:
 	     LN_S="$(LN_S)" \
 	     ../stamp-gnatlib1-$(RTSDIR)
 	sed \
-	  -e 's/Frontend_Exceptions.*/Frontend_Exceptions       : constant Boolean := False;/' \
 	  -e 's/ZCX_By_Default.*/ZCX_By_Default            : constant Boolean := True;/' \
 	  $(RTSDIR)/system.ads > $(RTSDIR)/s.ads
 	$(MV) $(RTSDIR)/s.ads $(RTSDIR)/system.ads
diff --git a/gcc/ada/gcc-interface/decl.cc b/gcc/ada/gcc-interface/decl.cc
index 903ec84..86cbf5b 100644
--- a/gcc/ada/gcc-interface/decl.cc
+++ b/gcc/ada/gcc-interface/decl.cc
@@ -6421,6 +6421,33 @@ gnat_to_gnu_subprog_type (Entity_Id gnat_subprog, bool definition,
 	 since structures are incomplete for the back-end.  */
       else if (Convention (gnat_subprog) != Convention_Stubbed)
 	{
+	  /* If we have two entries that may be returned in integer registers,
+	     the larger has power-of-2 size and the smaller is integer, then
+	     extend the smaller to this power-of-2 size to get a return type
+	     with power-of-2 size and no holes, again to speed up accesses.  */
+	  if (list_length (gnu_cico_field_list) == 2
+	      && gnu_cico_only_integral_type)
+	    {
+	      tree typ1 = TREE_TYPE (gnu_cico_field_list);
+	      tree typ2 = TREE_TYPE (DECL_CHAIN (gnu_cico_field_list));
+	      if (TREE_CODE (typ1) == INTEGER_TYPE
+		  && integer_pow2p (TYPE_SIZE (typ2))
+		  && compare_tree_int (TYPE_SIZE (typ2),
+				       MAX_FIXED_MODE_SIZE) <= 0
+		  && tree_int_cst_lt (TYPE_SIZE (typ1), TYPE_SIZE (typ2)))
+		TREE_TYPE (gnu_cico_field_list)
+		  = gnat_type_for_size (TREE_INT_CST_LOW (TYPE_SIZE (typ2)),
+					TYPE_UNSIGNED (typ1));
+	      else if (TREE_CODE (typ2) == INTEGER_TYPE
+		       && integer_pow2p (TYPE_SIZE (typ1))
+		       && compare_tree_int (TYPE_SIZE (typ1),
+					    MAX_FIXED_MODE_SIZE) <= 0
+		       && tree_int_cst_lt (TYPE_SIZE (typ2), TYPE_SIZE (typ1)))
+		TREE_TYPE (DECL_CHAIN (gnu_cico_field_list))
+		  = gnat_type_for_size (TREE_INT_CST_LOW (TYPE_SIZE (typ1)),
+					TYPE_UNSIGNED (typ2));
+	    }
+
 	  finish_record_type (gnu_cico_return_type,
 			      nreverse (gnu_cico_field_list),
 			      0, false);
diff --git a/gcc/ada/gcc-interface/misc.cc b/gcc/ada/gcc-interface/misc.cc
index 128040e..7711f8b 100644
--- a/gcc/ada/gcc-interface/misc.cc
+++ b/gcc/ada/gcc-interface/misc.cc
@@ -271,7 +271,7 @@ gnat_post_options (const char **pfilename ATTRIBUTE_UNUSED)
 
   /* No caret by default for Ada.  */
   if (!OPTION_SET_P (flag_diagnostics_show_caret))
-    global_dc->m_source_printing.enabled = false;
+    global_dc->get_source_printing_options ().enabled = false;
 
   /* Copy global settings to local versions.  */
   gnat_encodings = global_options.x_gnat_encodings;
@@ -292,7 +292,7 @@ gnat_post_options (const char **pfilename ATTRIBUTE_UNUSED)
 /* Here is the function to handle the compiler error processing in GCC.  */
 
 static void
-internal_error_function (diagnostic_context *context, const char *msgid,
+internal_error_function (diagnostics::context *context, const char *msgid,
 			 va_list *ap)
 {
   char *buffer, *p, *loc;
diff --git a/gcc/ada/gcc-interface/trans.cc b/gcc/ada/gcc-interface/trans.cc
index e02804b..fd1d39c 100644
--- a/gcc/ada/gcc-interface/trans.cc
+++ b/gcc/ada/gcc-interface/trans.cc
@@ -1510,7 +1510,7 @@ Pragma_to_gnu (Node_Id gnat_node)
 	const location_t location = input_location;
 	struct cl_option_handlers handlers;
 	unsigned int option_index;
-	diagnostic_t kind;
+	enum diagnostics::kind kind;
 	bool imply;
 
 	gnat_temp = First (Pragma_Argument_Associations (gnat_node));
@@ -1521,12 +1521,12 @@ Pragma_to_gnu (Node_Id gnat_node)
 	    switch (id)
 	      {
 	      case Pragma_Warning_As_Error:
-		kind = DK_ERROR;
+		kind = diagnostics::kind::error;
 		imply = false;
 		break;
 
 	      case Pragma_Warnings:
-		kind = DK_WARNING;
+		kind = diagnostics::kind::warning;
 		imply = true;
 		break;
 
@@ -1543,11 +1543,11 @@ Pragma_to_gnu (Node_Id gnat_node)
 	    switch (Chars (Expression (gnat_temp)))
 	      {
 		case Name_Off:
-		  kind = DK_IGNORED;
+		  kind = diagnostics::kind::ignored;
 		  break;
 
 		case Name_On:
-		  kind = DK_WARNING;
+		  kind = diagnostics::kind::warning;
 		  break;
 
 		default:
@@ -1569,7 +1569,7 @@ Pragma_to_gnu (Node_Id gnat_node)
 		gnat_expr = Empty;
 
 		/* For pragma Warnings (Off), we save the current state...  */
-		if (kind == DK_IGNORED)
+		if (kind == diagnostics::kind::ignored)
 		  diagnostic_push_diagnostics (global_dc, location);
 
 		/* ...so that, for pragma Warnings (On), we do not enable all
@@ -4049,7 +4049,7 @@ Subprogram_Body_to_gnu (Node_Id gnat_node)
 	    tree gnu_decl;
 
 	    /* Skip any entries that have been already filled in; they must
-	       correspond to In Out parameters.  */
+	       correspond to In Out parameters or previous Out parameters.  */
 	    while (gnu_cico_entry && TREE_VALUE (gnu_cico_entry))
 	      gnu_cico_entry = TREE_CHAIN (gnu_cico_entry);
 
@@ -4059,11 +4059,22 @@ Subprogram_Body_to_gnu (Node_Id gnat_node)
 	    if (DECL_BY_REF_P (gnu_decl))
 	      gnu_decl = build_unary_op (INDIRECT_REF, NULL_TREE, gnu_decl);
 
-	    /* Do any needed references for padded types.  */
-	    TREE_VALUE (gnu_cico_entry)
-	      = convert (TREE_TYPE (TREE_PURPOSE (gnu_cico_entry)), gnu_decl);
+	    TREE_VALUE (gnu_cico_entry) = gnu_decl;
 	  }
+
+      /* Finally, ensure type consistency between TREE_PURPOSE and TREE_VALUE
+	 so that the assignment of the latter to the former can be done.  */
+      tree gnu_cico_entry = gnu_cico_list;
+      while (gnu_cico_entry)
+	{
+	  if (!VOID_TYPE_P (TREE_VALUE (gnu_cico_entry)))
+	    TREE_VALUE (gnu_cico_entry)
+	      = convert (TREE_TYPE (TREE_PURPOSE (gnu_cico_entry)),
+			 TREE_VALUE (gnu_cico_entry));
+	  gnu_cico_entry = TREE_CHAIN (gnu_cico_entry);
+	}
     }
+
   else
     vec_safe_push (gnu_return_label_stack, NULL_TREE);
 
@@ -4161,9 +4172,13 @@ Subprogram_Body_to_gnu (Node_Id gnat_node)
 	    }
 	}
 
-      /* Otherwise, if this is a procedure or a function which does not return
-	 by invisible reference, we can do a direct block-copy out.  */
-      else
+      /* Otherwise, if this is a procedure or a function that does not return
+	 by invisible reference, we can do a direct block-copy out, but we do
+	 not need to do it for a null initialization procedure when the _Init
+	 parameter is not passed in since we would copy uninitialized bits.  */
+      else if (!(Is_Null_Init_Proc (gnat_subprog)
+		 && list_length (gnu_cico_list) == 1
+		 && TREE_CODE (TREE_VALUE (gnu_cico_list)) == VAR_DECL))
 	{
 	  tree gnu_retval;
 
@@ -8461,7 +8476,8 @@ gnat_to_gnu (Node_Id gnat_node)
 	      oconstraints[i] = constraint;
 
 	      if (parse_output_constraint (&constraint, i, ninputs, noutputs,
-					   &allows_mem, &allows_reg, &fake))
+					   &allows_mem, &allows_reg, &fake,
+					   nullptr))
 		{
 		  /* If the operand is going to end up in memory,
 		     mark it addressable.  Note that we don't test
@@ -8489,9 +8505,9 @@ gnat_to_gnu (Node_Id gnat_node)
 	      constraint
 		= TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (tail)));
 
-	      if (parse_input_constraint (&constraint, i, ninputs, noutputs,
-					  0, oconstraints,
-					  &allows_mem, &allows_reg))
+	      if (parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
+					  oconstraints, &allows_mem,
+					  &allows_reg, nullptr))
 		{
 		  /* If the operand is going to end up in memory,
 		     mark it addressable.  */
@@ -8737,7 +8753,7 @@ gnat_to_gnu (Node_Id gnat_node)
 
   /* Set the location information on the result if it's not a simple name
      or something that contains a simple name, for example a tag, because
-     we don"t want all the references to get the location of the first use.
+     we don't want all the references to get the location of the first use.
      Note that we may have no result if we tried to build a CALL_EXPR node
      to a procedure with no side-effects and optimization is enabled.  */
   else if (kind != N_Identifier
diff --git a/gcc/ada/gcc-interface/utils.cc b/gcc/ada/gcc-interface/utils.cc
index 23737c3..f501915 100644
--- a/gcc/ada/gcc-interface/utils.cc
+++ b/gcc/ada/gcc-interface/utils.cc
@@ -1225,7 +1225,6 @@ make_packable_type (tree type, bool in_record, unsigned int max_align)
      Note that we rely on the pointer equality created here for
      TYPE_NAME to look through conversions in various places.  */
   TYPE_NAME (new_type) = TYPE_NAME (type);
-  TYPE_PACKED (new_type) = 1;
   TYPE_JUSTIFIED_MODULAR_P (new_type) = TYPE_JUSTIFIED_MODULAR_P (type);
   TYPE_CONTAINS_TEMPLATE_P (new_type) = TYPE_CONTAINS_TEMPLATE_P (type);
   TYPE_REVERSE_STORAGE_ORDER (new_type) = TYPE_REVERSE_STORAGE_ORDER (type);
@@ -1240,6 +1239,8 @@ make_packable_type (tree type, bool in_record, unsigned int max_align)
       new_size = ceil_pow2 (size);
       new_align = MIN (new_size, BIGGEST_ALIGNMENT);
       SET_TYPE_ALIGN (new_type, new_align);
+      /* build_aligned_type needs to be able to adjust back the alignment.  */
+      TYPE_PACKED (new_type) = 0;
     }
   else
     {
@@ -1261,6 +1262,7 @@ make_packable_type (tree type, bool in_record, unsigned int max_align)
       if (max_align > 0 && new_align > max_align)
 	new_align = max_align;
       SET_TYPE_ALIGN (new_type, MIN (align, new_align));
+      TYPE_PACKED (new_type) = 1;
     }
 
   TYPE_USER_ALIGN (new_type) = 1;
@@ -4508,8 +4510,8 @@ build_unc_object_type_from_ptr (tree thin_fat_ptr_type, tree object_type,
 void
 update_pointer_to (tree old_type, tree new_type)
 {
-  tree ptr = TYPE_POINTER_TO (old_type);
-  tree ref = TYPE_REFERENCE_TO (old_type);
+  const tree old_ptr = TYPE_POINTER_TO (old_type);
+  const tree old_ref = TYPE_REFERENCE_TO (old_type);
   tree t;
 
   /* If this is the main variant, process all the other variants first.  */
@@ -4518,7 +4520,7 @@ update_pointer_to (tree old_type, tree new_type)
       update_pointer_to (t, new_type);
 
   /* If no pointers and no references, we are done.  */
-  if (!ptr && !ref)
+  if (!old_ptr && !old_ref)
     return;
 
   /* Merge the old type qualifiers in the new type.
@@ -4552,12 +4554,13 @@ update_pointer_to (tree old_type, tree new_type)
   if (TREE_CODE (new_type) != UNCONSTRAINED_ARRAY_TYPE)
     {
       tree new_ptr, new_ref;
+      tree ptr, ref;
 
       /* If pointer or reference already points to new type, nothing to do.
 	 This can happen as update_pointer_to can be invoked multiple times
 	 on the same couple of types because of the type variants.  */
-      if ((ptr && TREE_TYPE (ptr) == new_type)
-	  || (ref && TREE_TYPE (ref) == new_type))
+      if ((old_ptr && TREE_TYPE (old_ptr) == new_type)
+	  || (old_ref && TREE_TYPE (old_ref) == new_type))
 	return;
 
       /* Chain PTR and its variants at the end.  */
@@ -4566,13 +4569,13 @@ update_pointer_to (tree old_type, tree new_type)
 	{
 	  while (TYPE_NEXT_PTR_TO (new_ptr))
 	    new_ptr = TYPE_NEXT_PTR_TO (new_ptr);
-	  TYPE_NEXT_PTR_TO (new_ptr) = ptr;
+	  TYPE_NEXT_PTR_TO (new_ptr) = old_ptr;
 	}
       else
-	TYPE_POINTER_TO (new_type) = ptr;
+	TYPE_POINTER_TO (new_type) = old_ptr;
 
       /* Now adjust them.  */
-      for (; ptr; ptr = TYPE_NEXT_PTR_TO (ptr))
+      for (ptr = old_ptr; ptr; ptr = TYPE_NEXT_PTR_TO (ptr))
 	for (t = TYPE_MAIN_VARIANT (ptr); t; t = TYPE_NEXT_VARIANT (t))
 	  {
 	    TREE_TYPE (t) = new_type;
@@ -4587,13 +4590,13 @@ update_pointer_to (tree old_type, tree new_type)
 	{
 	  while (TYPE_NEXT_REF_TO (new_ref))
 	    new_ref = TYPE_NEXT_REF_TO (new_ref);
-	  TYPE_NEXT_REF_TO (new_ref) = ref;
+	  TYPE_NEXT_REF_TO (new_ref) = old_ref;
 	}
       else
-	TYPE_REFERENCE_TO (new_type) = ref;
+	TYPE_REFERENCE_TO (new_type) = old_ref;
 
       /* Now adjust them.  */
-      for (; ref; ref = TYPE_NEXT_REF_TO (ref))
+      for (ref = old_ref; ref; ref = TYPE_NEXT_REF_TO (ref))
 	for (t = TYPE_MAIN_VARIANT (ref); t; t = TYPE_NEXT_VARIANT (t))
 	  {
 	    TREE_TYPE (t) = new_type;
@@ -4612,20 +4615,20 @@ update_pointer_to (tree old_type, tree new_type)
     {
       tree new_ptr = TYPE_POINTER_TO (new_type);
 
-      gcc_assert (TYPE_IS_FAT_POINTER_P (ptr));
+      gcc_assert (TYPE_IS_FAT_POINTER_P (old_ptr));
 
       /* If PTR already points to NEW_TYPE, nothing to do.  This can happen
 	 since update_pointer_to can be invoked multiple times on the same
 	 couple of types because of the type variants.  */
-      if (TYPE_UNCONSTRAINED_ARRAY (ptr) == new_type)
+      if (TYPE_UNCONSTRAINED_ARRAY (old_ptr) == new_type)
 	return;
 
       update_pointer_to
-	(TREE_TYPE (TREE_TYPE (TYPE_FIELDS (ptr))),
+	(TREE_TYPE (TREE_TYPE (TYPE_FIELDS (old_ptr))),
 	 TREE_TYPE (TREE_TYPE (TYPE_FIELDS (new_ptr))));
 
       update_pointer_to
-	(TREE_TYPE (TREE_TYPE (DECL_CHAIN (TYPE_FIELDS (ptr)))),
+	(TREE_TYPE (TREE_TYPE (DECL_CHAIN (TYPE_FIELDS (old_ptr)))),
 	 TREE_TYPE (TREE_TYPE (DECL_CHAIN (TYPE_FIELDS (new_ptr)))));
 
       update_pointer_to (TYPE_OBJECT_RECORD_TYPE (old_type),
diff --git a/gcc/ada/gen_il-fields.ads b/gcc/ada/gen_il-fields.ads
index 2d16e12..a1e284f 100644
--- a/gcc/ada/gen_il-fields.ads
+++ b/gcc/ada/gen_il-fields.ads
@@ -255,6 +255,7 @@ package Gen_IL.Fields is
       Is_Entry_Barrier_Function,
       Is_Expanded_Build_In_Place_Call,
       Is_Expanded_Constructor_Call,
+      Is_Expanded_Dispatching_Call,
       Is_Expanded_Prefixed_Call,
       Is_Folded_In_Parser,
       Is_Generic_Contract_Pragma,
@@ -400,6 +401,7 @@ package Gen_IL.Fields is
       Suppress_Loop_Warnings,
       Synchronized_Present,
       Tagged_Present,
+      Tag_Propagated,
       Target,
       Call_Or_Target_Loop,
       Target_Type,
@@ -539,6 +541,7 @@ package Gen_IL.Fields is
       Extra_Constrained,
       Extra_Formal,
       Extra_Formals,
+      Extra_Formals_Known,
       Finalization_Collection,
       Finalization_Master_Node,
       Finalize_Storage_Only,
@@ -572,6 +575,7 @@ package Gen_IL.Fields is
       Has_Delayed_Aspects,
       Has_Delayed_Freeze,
       Has_Delayed_Rep_Aspects,
+      Has_Destructor,
       Has_Discriminants,
       Has_Dispatch_Table,
       Has_Dynamic_Predicate_Aspect,
@@ -699,6 +703,7 @@ package Gen_IL.Fields is
       Is_CPP_Class,
       Is_CUDA_Kernel,
       Is_Descendant_Of_Address,
+      Is_Destructor,
       Is_DIC_Procedure,
       Is_Discrim_SO_Function,
       Is_Discriminant_Check_Function,
@@ -730,6 +735,7 @@ package Gen_IL.Fields is
       Is_Ignored_Ghost_Entity,
       Is_Immediately_Visible,
       Is_Implementation_Defined,
+      Is_Implicit_Full_View,
       Is_Imported,
       Is_Independent,
       Is_Initial_Condition_Procedure,
@@ -850,6 +856,7 @@ package Gen_IL.Fields is
       Original_Protected_Subprogram,
       Original_Record_Component,
       Overlays_Constant,
+      Overridden_Inherited_Operation,
       Overridden_Operation,
       Package_Instantiation,
       Packed_Array_Impl_Type,
diff --git a/gcc/ada/gen_il-gen-gen_entities.adb b/gcc/ada/gen_il-gen-gen_entities.adb
index 8cbed8a..0fedfbc 100644
--- a/gcc/ada/gen_il-gen-gen_entities.adb
+++ b/gcc/ada/gen_il-gen-gen_entities.adb
@@ -467,6 +467,7 @@ begin -- Gen_IL.Gen.Gen_Entities
         Sm (Full_View, Node_Id),
         Sm (Has_Completion_In_Body, Flag),
         Sm (Has_Constrained_Partial_View, Flag, Base_Type_Only),
+        Sm (Has_Destructor, Flag, Base_Type_Only),
         Sm (Has_Discriminants, Flag),
         Sm (Has_Dispatch_Table, Flag,
             Pre => "Is_Tagged_Type (N)"),
@@ -502,6 +503,7 @@ begin -- Gen_IL.Gen.Gen_Entities
         Sm (Is_Fixed_Lower_Bound_Array_Subtype, Flag),
         Sm (Is_Fixed_Lower_Bound_Index_Subtype, Flag),
         Sm (Is_Generic_Actual_Type, Flag),
+        Sm (Is_Implicit_Full_View, Flag),
         Sm (Is_Mutably_Tagged_Type, Flag),
         Sm (Is_Non_Static_Subtype, Flag),
         Sm (Is_Private_Composite, Flag),
@@ -935,11 +937,13 @@ begin -- Gen_IL.Gen.Gen_Entities
        (Sm (Access_Subprogram_Wrapper, Node_Id),
         Sm (Extra_Accessibility_Of_Result, Node_Id),
         Sm (Extra_Formals, Node_Id),
+        Sm (Extra_Formals_Known, Flag),
         Sm (Needs_No_Actuals, Flag)));
 
    Ab (Overloadable_Kind, Entity_Kind,
        (Sm (Renamed_Or_Alias, Node_Id),
         Sm (Extra_Formals, Node_Id),
+        Sm (Extra_Formals_Known, Flag),
         Sm (Is_Abstract_Subprogram, Flag),
         Sm (Is_Primitive, Flag),
         Sm (Needs_No_Actuals, Flag),
@@ -953,6 +957,7 @@ begin -- Gen_IL.Gen.Gen_Entities
         Sm (Enumeration_Rep_Expr, Node_Id),
         Sm (Esize, Uint),
         Sm (Alignment, Unat),
+        Sm (Overridden_Inherited_Operation, Node_Id),
         Sm (Interface_Name, Node_Id)));
 
    Ab (Subprogram_Kind, Overloadable_Kind,
@@ -981,6 +986,7 @@ begin -- Gen_IL.Gen.Gen_Entities
         Sm (Is_Machine_Code_Subprogram, Flag),
         Sm (Last_Entity, Node_Id),
         Sm (Linker_Section_Pragma, Node_Id),
+        Sm (Overridden_Inherited_Operation, Node_Id),
         Sm (Overridden_Operation, Node_Id),
         Sm (Protected_Body_Subprogram, Node_Id),
         Sm (No_Raise, Flag),
@@ -1050,6 +1056,7 @@ begin -- Gen_IL.Gen.Gen_Entities
         Sm (Is_Asynchronous, Flag),
         Sm (Is_Called, Flag),
         Sm (Is_CUDA_Kernel, Flag),
+        Sm (Is_Destructor, Flag),
         Sm (Is_DIC_Procedure, Flag),
         Sm (Is_Generic_Actual_Subprogram, Flag),
         Sm (Is_Initial_Condition_Procedure, Flag),
@@ -1125,6 +1132,7 @@ begin -- Gen_IL.Gen.Gen_Entities
         Sm (Entry_Accepted, Flag),
         Sm (Entry_Parameters_Type, Node_Id),
         Sm (Extra_Formals, Node_Id),
+        Sm (Extra_Formals_Known, Flag),
         Sm (First_Entity, Node_Id),
         Sm (Has_Out_Or_In_Out_Parameter, Flag),
         Sm (Ignore_SPARK_Mode_Pragmas, Flag),
@@ -1326,6 +1334,7 @@ begin -- Gen_IL.Gen.Gen_Entities
        (Sm (Anonymous_Collections, Elist_Id),
         Sm (Contract, Node_Id),
         Sm (Extra_Formals, Node_Id),
+        Sm (Extra_Formals_Known, Flag),
         Sm (First_Entity, Node_Id),
         Sm (Ignore_SPARK_Mode_Pragmas, Flag),
         Sm (Interface_Name, Node_Id),
diff --git a/gcc/ada/gen_il-gen-gen_nodes.adb b/gcc/ada/gen_il-gen-gen_nodes.adb
index f4e7917..412565f 100644
--- a/gcc/ada/gen_il-gen-gen_nodes.adb
+++ b/gcc/ada/gen_il-gen-gen_nodes.adb
@@ -149,6 +149,7 @@ begin -- Gen_IL.Gen.Gen_Nodes
         Sm (Is_Controlling_Actual, Flag),
         Sm (Is_Overloaded, Flag),
         Sm (Is_Static_Expression, Flag),
+        Sm (Is_Expanded_Dispatching_Call, Flag),
         Sm (Must_Not_Freeze, Flag),
         Sm (Raises_Constraint_Error, Flag)));
 
@@ -181,7 +182,8 @@ begin -- Gen_IL.Gen.Gen_Nodes
         Sm (Is_Elaboration_Warnings_OK_Node, Flag),
         Sm (Is_SPARK_Mode_On_Node, Flag),
         Sm (Original_Discriminant, Node_Id),
-        Sm (Redundant_Use, Flag)));
+        Sm (Redundant_Use, Flag),
+        Sm (Tag_Propagated, Flag)));
 
    Cc (N_Operator_Symbol, N_Direct_Name,
        (Sy (Strval, String_Id)));
@@ -346,7 +348,8 @@ begin -- Gen_IL.Gen.Gen_Nodes
        (Sy (Prefix, Node_Id),
         Sm (Actual_Designated_Subtype, Node_Id),
         Sm (Atomic_Sync_Required, Flag),
-        Sm (Has_Dereference_Action, Flag)));
+        Sm (Has_Dereference_Action, Flag),
+        Sm (Tag_Propagated, Flag)));
 
    Cc (N_Expression_With_Actions, N_Subexpr,
        (Sy (Actions, List_Id, Default_No_List),
@@ -463,6 +466,7 @@ begin -- Gen_IL.Gen.Gen_Nodes
         Sm (Do_Length_Check, Flag),
         Sm (Do_Overflow_Check, Flag),
         Sm (Float_Truncate, Flag),
+        Sm (Tag_Propagated, Flag),
         Sm (Rounded_Result, Flag)));
 
    Cc (N_Unchecked_Type_Conversion, N_Subexpr,
@@ -905,6 +909,7 @@ begin -- Gen_IL.Gen.Gen_Nodes
         Sm (Cleanup_Actions, List_Id),
         Sm (Exception_Junk, Flag),
         Sm (Is_Abort_Block, Flag),
+        Sm (Is_Expanded_Dispatching_Call, Flag),
         Sm (Is_Initialization_Block, Flag),
         Sm (Is_Task_Master, Flag)));
 
diff --git a/gcc/ada/gen_il-internals.adb b/gcc/ada/gen_il-internals.adb
index 8d0dfc7..3fa8b94 100644
--- a/gcc/ada/gen_il-internals.adb
+++ b/gcc/ada/gen_il-internals.adb
@@ -277,6 +277,8 @@ package body Gen_IL.Internals is
             return "DT_Offset_To_Top_Func";
          when DT_Position =>
             return "DT_Position";
+         when Extra_Formals_Known =>
+            return "Extra_Formals_Known";
          when Forwards_OK =>
             return "Forwards_OK";
          when Has_First_Controlling_Parameter_Aspect =>
diff --git a/gcc/ada/gnat_rm.texi b/gcc/ada/gnat_rm.texi
index 79fb225..5d7bedc 100644
--- a/gcc/ada/gnat_rm.texi
+++ b/gcc/ada/gnat_rm.texi
@@ -19,7 +19,7 @@
 
 @copying
 @quotation
-GNAT Reference Manual , Jun 27, 2025
+GNAT Reference Manual , Jul 24, 2025
 
 AdaCore
 
@@ -931,6 +931,7 @@ Experimental Language Extensions
 * External_Initialization Aspect:: 
 * Finally construct:: 
 * Continue statement:: 
+* Destructors:: 
 
 Storage Model
 
@@ -7548,13 +7549,33 @@ Syntax:
 pragma Short_Circuit_And_Or;
 @end example
 
-This configuration pragma causes any occurrence of the AND operator applied to
-operands of type Standard.Boolean to be short-circuited (i.e. the AND operator
-is treated as if it were AND THEN). Or is similarly treated as OR ELSE. This
-may be useful in the context of certification protocols requiring the use of
-short-circuited logical operators. If this configuration pragma occurs locally
-within the file being compiled, it applies only to the file being compiled.
+This configuration pragma causes the predefined AND and OR operators of
+type Standard.Boolean to have short-circuit semantics. That is, they
+behave like AND THEN and OR ELSE; the right-hand side is not evaluated
+if the left-hand side determines the result. This may be useful in the
+context of certification protocols requiring the use of short-circuited
+logical operators.
+
 There is no requirement that all units in a partition use this option.
+However, mixing of short-circuit and non-short-circuit semantics can be
+confusing. Therefore, the recommended use is to put the pragma in a
+configuration file that applies to the whole program. Alternatively, if
+you have a legacy library that should not use this pragma, you can put
+it in a separate library project that does not use the pragma.
+In any case, fine-grained mixing of the different semantics is not
+recommended. If pragma @code{Short_Circuit_And_Or} is specified, then it
+is illegal to rename the predefined Boolean AND and OR, or to pass
+them to generic formal functions; this corresponds to the fact that
+AND THEN and OR ELSE cannot be renamed nor passed as generic formal
+functions.
+
+Note that this pragma has no effect on other logical operators –
+predefined operators of modular types, array-of-boolean types and types
+derived from Standard.Boolean, nor user-defined operators.
+
+See also the pragma @code{Unevaluated_Use_Of_Old} and the restriction
+@code{No_Direct_Boolean_Operators}, which may be useful in conjunction
+with @code{Short_Circuit_And_Or}.
 
 @node Pragma Short_Descriptors,Pragma Side_Effects,Pragma Short_Circuit_And_Or,Implementation Defined Pragmas
 @anchor{gnat_rm/implementation_defined_pragmas pragma-short-descriptors}@anchor{ed}
@@ -13051,9 +13072,9 @@ pragma Export (C, Last_Chance_Handler,
                "__gnat_last_chance_handler");
 @end example
 
-The parameter is a C null-terminated string representing a message to be
-associated with the exception (typically the source location of the raise
-statement generated by the compiler). The Line parameter when nonzero
+The @code{Source_Location} parameter is a C null-terminated string representing a
+message to be associated with the exception (typically the source location of
+the raise statement generated by the compiler). The Line parameter when nonzero
 represents the line number in the source program where the raise occurs.
 
 @node No_Exception_Propagation,No_Exception_Registration,No_Exception_Handlers,Partition-Wide Restrictions
@@ -31216,6 +31237,7 @@ Features activated via @code{-gnatX0} or
 * External_Initialization Aspect:: 
 * Finally construct:: 
 * Continue statement:: 
+* Destructors:: 
 
 @end menu
 
@@ -32568,7 +32590,7 @@ Abort/ATC (asynchronous transfer of control) cannot interrupt a finally block, n
 execution, that is the finally block must be executed in full even if the containing task is
 aborted, or if the control is transferred out of the block.
 
-@node Continue statement,,Finally construct,Experimental Language Extensions
+@node Continue statement,Destructors,Finally construct,Experimental Language Extensions
 @anchor{gnat_rm/gnat_language_extensions continue-statement}@anchor{472}
 @subsection Continue statement
 
@@ -32586,8 +32608,78 @@ statement in the sequence of statements of the specified loop_statement.
 Note that @code{continue} is a keyword but it is not a reserved word. This is a
 configuration that does not exist in standard Ada.
 
+@node Destructors,,Continue statement,Experimental Language Extensions
+@anchor{gnat_rm/gnat_language_extensions destructors}@anchor{473}
+@subsection Destructors
+
+
+The @code{Destructor} aspect can be applied to any record type, tagged or not.
+It must denote a primitive of the type that is a procedure with one parameter
+of the type and of mode @code{in out}:
+
+@example
+type T is record
+   ...
+end record with Destructor => Foo;
+
+procedure Foo (X : in out T);
+@end example
+
+This is equivalent to the following code that uses @code{Finalizable}:
+
+@example
+type T is record
+   ...
+end record with Finalizable => (Finalize => Foo);
+
+procedure Foo (X : in out T);
+@end example
+
+Unlike @code{Finalizable}, however, @code{Destructor} can be specified on a derived
+type. And when it is, the effect of the aspect combines with the destructors of
+the parent type. Take, for example:
+
+@example
+type T1 is record
+   ...
+end record with Destructor => Foo;
+
+procedure Foo (X : in out T1);
+
+type T2 is new T1 with Destructor => Bar;
+
+procedure Bar (X : in out T2);
+@end example
+
+Here, when an object of type @code{T2} is finalized, a call to @code{Bar}
+will be performed and it will be followed by a call to @code{Foo}.
+
+The @code{Destructor} aspect comes with a legality rule: if a primitive procedure
+of a type is denoted by a @code{Destructor} aspect specification, it is illegal to
+override this procedure in a derived type. For example, the following is illegal:
+
+@example
+type T1 is record
+   ...
+end record with Destructor => Foo;
+
+procedure Foo (X : in out T1);
+
+type T2 is new T1;
+
+overriding
+procedure Foo (X : in out T2); -- Error here
+@end example
+
+It is possible to specify @code{Destructor} on the completion of a private type,
+but there is one more restriction in that case: the denoted primitive must
+be private to the enclosing package. This is necessary due to the previously
+mentioned legality rule, to prevent breaking the privacy of the type when
+imposing that rule on outside types that derive from the private view of the
+type.
+
 @node Security Hardening Features,Obsolescent Features,GNAT language extensions,Top
-@anchor{gnat_rm/security_hardening_features doc}@anchor{473}@anchor{gnat_rm/security_hardening_features id1}@anchor{474}@anchor{gnat_rm/security_hardening_features security-hardening-features}@anchor{15}
+@anchor{gnat_rm/security_hardening_features doc}@anchor{474}@anchor{gnat_rm/security_hardening_features id1}@anchor{475}@anchor{gnat_rm/security_hardening_features security-hardening-features}@anchor{15}
 @chapter Security Hardening Features
 
 
@@ -32609,7 +32701,7 @@ change.
 @end menu
 
 @node Register Scrubbing,Stack Scrubbing,,Security Hardening Features
-@anchor{gnat_rm/security_hardening_features register-scrubbing}@anchor{475}
+@anchor{gnat_rm/security_hardening_features register-scrubbing}@anchor{476}
 @section Register Scrubbing
 
 
@@ -32645,7 +32737,7 @@ programming languages, see @cite{Using the GNU Compiler Collection (GCC)}.
 @c Stack Scrubbing:
 
 @node Stack Scrubbing,Hardened Conditionals,Register Scrubbing,Security Hardening Features
-@anchor{gnat_rm/security_hardening_features stack-scrubbing}@anchor{476}
+@anchor{gnat_rm/security_hardening_features stack-scrubbing}@anchor{477}
 @section Stack Scrubbing
 
 
@@ -32789,7 +32881,7 @@ Bar_Callable_Ptr.
 @c Hardened Conditionals:
 
 @node Hardened Conditionals,Hardened Booleans,Stack Scrubbing,Security Hardening Features
-@anchor{gnat_rm/security_hardening_features hardened-conditionals}@anchor{477}
+@anchor{gnat_rm/security_hardening_features hardened-conditionals}@anchor{478}
 @section Hardened Conditionals
 
 
@@ -32879,7 +32971,7 @@ be used with other programming languages supported by GCC.
 @c Hardened Booleans:
 
 @node Hardened Booleans,Control Flow Redundancy,Hardened Conditionals,Security Hardening Features
-@anchor{gnat_rm/security_hardening_features hardened-booleans}@anchor{478}
+@anchor{gnat_rm/security_hardening_features hardened-booleans}@anchor{479}
 @section Hardened Booleans
 
 
@@ -32940,7 +33032,7 @@ and more details on that attribute, see @cite{Using the GNU Compiler Collection
 @c Control Flow Redundancy:
 
 @node Control Flow Redundancy,,Hardened Booleans,Security Hardening Features
-@anchor{gnat_rm/security_hardening_features control-flow-redundancy}@anchor{479}
+@anchor{gnat_rm/security_hardening_features control-flow-redundancy}@anchor{47a}
 @section Control Flow Redundancy
 
 
@@ -33108,7 +33200,7 @@ see @cite{Using the GNU Compiler Collection (GCC)}.  These options
 can be used with other programming languages supported by GCC.
 
 @node Obsolescent Features,Compatibility and Porting Guide,Security Hardening Features,Top
-@anchor{gnat_rm/obsolescent_features doc}@anchor{47a}@anchor{gnat_rm/obsolescent_features id1}@anchor{47b}@anchor{gnat_rm/obsolescent_features obsolescent-features}@anchor{16}
+@anchor{gnat_rm/obsolescent_features doc}@anchor{47b}@anchor{gnat_rm/obsolescent_features id1}@anchor{47c}@anchor{gnat_rm/obsolescent_features obsolescent-features}@anchor{16}
 @chapter Obsolescent Features
 
 
@@ -33127,7 +33219,7 @@ compatibility purposes.
 @end menu
 
 @node pragma No_Run_Time,pragma Ravenscar,,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features id2}@anchor{47c}@anchor{gnat_rm/obsolescent_features pragma-no-run-time}@anchor{47d}
+@anchor{gnat_rm/obsolescent_features id2}@anchor{47d}@anchor{gnat_rm/obsolescent_features pragma-no-run-time}@anchor{47e}
 @section pragma No_Run_Time
 
 
@@ -33140,7 +33232,7 @@ preferred usage is to use an appropriately configured run-time that
 includes just those features that are to be made accessible.
 
 @node pragma Ravenscar,pragma Restricted_Run_Time,pragma No_Run_Time,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features id3}@anchor{47e}@anchor{gnat_rm/obsolescent_features pragma-ravenscar}@anchor{47f}
+@anchor{gnat_rm/obsolescent_features id3}@anchor{47f}@anchor{gnat_rm/obsolescent_features pragma-ravenscar}@anchor{480}
 @section pragma Ravenscar
 
 
@@ -33149,7 +33241,7 @@ The pragma @code{Ravenscar} has exactly the same effect as pragma
 is part of the new Ada 2005 standard.
 
 @node pragma Restricted_Run_Time,pragma Task_Info,pragma Ravenscar,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features id4}@anchor{480}@anchor{gnat_rm/obsolescent_features pragma-restricted-run-time}@anchor{481}
+@anchor{gnat_rm/obsolescent_features id4}@anchor{481}@anchor{gnat_rm/obsolescent_features pragma-restricted-run-time}@anchor{482}
 @section pragma Restricted_Run_Time
 
 
@@ -33159,7 +33251,7 @@ preferred since the Ada 2005 pragma @code{Profile} is intended for
 this kind of implementation dependent addition.
 
 @node pragma Task_Info,package System Task_Info s-tasinf ads,pragma Restricted_Run_Time,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features id5}@anchor{482}@anchor{gnat_rm/obsolescent_features pragma-task-info}@anchor{483}
+@anchor{gnat_rm/obsolescent_features id5}@anchor{483}@anchor{gnat_rm/obsolescent_features pragma-task-info}@anchor{484}
 @section pragma Task_Info
 
 
@@ -33185,7 +33277,7 @@ in the spec of package System.Task_Info in the runtime
 library.
 
 @node package System Task_Info s-tasinf ads,,pragma Task_Info,Obsolescent Features
-@anchor{gnat_rm/obsolescent_features package-system-task-info}@anchor{484}@anchor{gnat_rm/obsolescent_features package-system-task-info-s-tasinf-ads}@anchor{485}
+@anchor{gnat_rm/obsolescent_features package-system-task-info}@anchor{485}@anchor{gnat_rm/obsolescent_features package-system-task-info-s-tasinf-ads}@anchor{486}
 @section package System.Task_Info (@code{s-tasinf.ads})
 
 
@@ -33195,7 +33287,7 @@ to support the @code{Task_Info} pragma. The predefined Ada package
 standard replacement for GNAT’s @code{Task_Info} functionality.
 
 @node Compatibility and Porting Guide,GNU Free Documentation License,Obsolescent Features,Top
-@anchor{gnat_rm/compatibility_and_porting_guide doc}@anchor{486}@anchor{gnat_rm/compatibility_and_porting_guide compatibility-and-porting-guide}@anchor{17}@anchor{gnat_rm/compatibility_and_porting_guide id1}@anchor{487}
+@anchor{gnat_rm/compatibility_and_porting_guide doc}@anchor{487}@anchor{gnat_rm/compatibility_and_porting_guide compatibility-and-porting-guide}@anchor{17}@anchor{gnat_rm/compatibility_and_porting_guide id1}@anchor{488}
 @chapter Compatibility and Porting Guide
 
 
@@ -33217,7 +33309,7 @@ applications developed in other Ada environments.
 @end menu
 
 @node Writing Portable Fixed-Point Declarations,Compatibility with Ada 83,,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide id2}@anchor{488}@anchor{gnat_rm/compatibility_and_porting_guide writing-portable-fixed-point-declarations}@anchor{489}
+@anchor{gnat_rm/compatibility_and_porting_guide id2}@anchor{489}@anchor{gnat_rm/compatibility_and_porting_guide writing-portable-fixed-point-declarations}@anchor{48a}
 @section Writing Portable Fixed-Point Declarations
 
 
@@ -33339,7 +33431,7 @@ If you follow this scheme you will be guaranteed that your fixed-point
 types will be portable.
 
 @node Compatibility with Ada 83,Compatibility between Ada 95 and Ada 2005,Writing Portable Fixed-Point Declarations,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-ada-83}@anchor{48a}@anchor{gnat_rm/compatibility_and_porting_guide id3}@anchor{48b}
+@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-ada-83}@anchor{48b}@anchor{gnat_rm/compatibility_and_porting_guide id3}@anchor{48c}
 @section Compatibility with Ada 83
 
 
@@ -33367,7 +33459,7 @@ following subsections treat the most likely issues to be encountered.
 @end menu
 
 @node Legal Ada 83 programs that are illegal in Ada 95,More deterministic semantics,,Compatibility with Ada 83
-@anchor{gnat_rm/compatibility_and_porting_guide id4}@anchor{48c}@anchor{gnat_rm/compatibility_and_porting_guide legal-ada-83-programs-that-are-illegal-in-ada-95}@anchor{48d}
+@anchor{gnat_rm/compatibility_and_porting_guide id4}@anchor{48d}@anchor{gnat_rm/compatibility_and_porting_guide legal-ada-83-programs-that-are-illegal-in-ada-95}@anchor{48e}
 @subsection Legal Ada 83 programs that are illegal in Ada 95
 
 
@@ -33467,7 +33559,7 @@ the fix is usually simply to add the @code{(<>)} to the generic declaration.
 @end itemize
 
 @node More deterministic semantics,Changed semantics,Legal Ada 83 programs that are illegal in Ada 95,Compatibility with Ada 83
-@anchor{gnat_rm/compatibility_and_porting_guide id5}@anchor{48e}@anchor{gnat_rm/compatibility_and_porting_guide more-deterministic-semantics}@anchor{48f}
+@anchor{gnat_rm/compatibility_and_porting_guide id5}@anchor{48f}@anchor{gnat_rm/compatibility_and_porting_guide more-deterministic-semantics}@anchor{490}
 @subsection More deterministic semantics
 
 
@@ -33495,7 +33587,7 @@ which open select branches are executed.
 @end itemize
 
 @node Changed semantics,Other language compatibility issues,More deterministic semantics,Compatibility with Ada 83
-@anchor{gnat_rm/compatibility_and_porting_guide changed-semantics}@anchor{490}@anchor{gnat_rm/compatibility_and_porting_guide id6}@anchor{491}
+@anchor{gnat_rm/compatibility_and_porting_guide changed-semantics}@anchor{491}@anchor{gnat_rm/compatibility_and_porting_guide id6}@anchor{492}
 @subsection Changed semantics
 
 
@@ -33537,7 +33629,7 @@ covers only the restricted range.
 @end itemize
 
 @node Other language compatibility issues,,Changed semantics,Compatibility with Ada 83
-@anchor{gnat_rm/compatibility_and_porting_guide id7}@anchor{492}@anchor{gnat_rm/compatibility_and_porting_guide other-language-compatibility-issues}@anchor{493}
+@anchor{gnat_rm/compatibility_and_porting_guide id7}@anchor{493}@anchor{gnat_rm/compatibility_and_porting_guide other-language-compatibility-issues}@anchor{494}
 @subsection Other language compatibility issues
 
 
@@ -33570,7 +33662,7 @@ include @code{pragma Interface} and the floating point type attributes
 @end itemize
 
 @node Compatibility between Ada 95 and Ada 2005,Implementation-dependent characteristics,Compatibility with Ada 83,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide compatibility-between-ada-95-and-ada-2005}@anchor{494}@anchor{gnat_rm/compatibility_and_porting_guide id8}@anchor{495}
+@anchor{gnat_rm/compatibility_and_porting_guide compatibility-between-ada-95-and-ada-2005}@anchor{495}@anchor{gnat_rm/compatibility_and_porting_guide id8}@anchor{496}
 @section Compatibility between Ada 95 and Ada 2005
 
 
@@ -33642,7 +33734,7 @@ can declare a function returning a value from an anonymous access type.
 @end itemize
 
 @node Implementation-dependent characteristics,Compatibility with Other Ada Systems,Compatibility between Ada 95 and Ada 2005,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide id9}@anchor{496}@anchor{gnat_rm/compatibility_and_porting_guide implementation-dependent-characteristics}@anchor{497}
+@anchor{gnat_rm/compatibility_and_porting_guide id9}@anchor{497}@anchor{gnat_rm/compatibility_and_porting_guide implementation-dependent-characteristics}@anchor{498}
 @section Implementation-dependent characteristics
 
 
@@ -33665,7 +33757,7 @@ transition from certain Ada 83 compilers.
 @end menu
 
 @node Implementation-defined pragmas,Implementation-defined attributes,,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide id10}@anchor{498}@anchor{gnat_rm/compatibility_and_porting_guide implementation-defined-pragmas}@anchor{499}
+@anchor{gnat_rm/compatibility_and_porting_guide id10}@anchor{499}@anchor{gnat_rm/compatibility_and_porting_guide implementation-defined-pragmas}@anchor{49a}
 @subsection Implementation-defined pragmas
 
 
@@ -33687,7 +33779,7 @@ avoiding compiler rejection of units that contain such pragmas; they are not
 relevant in a GNAT context and hence are not otherwise implemented.
 
 @node Implementation-defined attributes,Libraries,Implementation-defined pragmas,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide id11}@anchor{49a}@anchor{gnat_rm/compatibility_and_porting_guide implementation-defined-attributes}@anchor{49b}
+@anchor{gnat_rm/compatibility_and_porting_guide id11}@anchor{49b}@anchor{gnat_rm/compatibility_and_porting_guide implementation-defined-attributes}@anchor{49c}
 @subsection Implementation-defined attributes
 
 
@@ -33701,7 +33793,7 @@ Ada 83, GNAT supplies the attributes @code{Bit}, @code{Machine_Size} and
 @code{Type_Class}.
 
 @node Libraries,Elaboration order,Implementation-defined attributes,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide id12}@anchor{49c}@anchor{gnat_rm/compatibility_and_porting_guide libraries}@anchor{49d}
+@anchor{gnat_rm/compatibility_and_porting_guide id12}@anchor{49d}@anchor{gnat_rm/compatibility_and_porting_guide libraries}@anchor{49e}
 @subsection Libraries
 
 
@@ -33730,7 +33822,7 @@ be preferable to retrofit the application using modular types.
 @end itemize
 
 @node Elaboration order,Target-specific aspects,Libraries,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide elaboration-order}@anchor{49e}@anchor{gnat_rm/compatibility_and_porting_guide id13}@anchor{49f}
+@anchor{gnat_rm/compatibility_and_porting_guide elaboration-order}@anchor{49f}@anchor{gnat_rm/compatibility_and_porting_guide id13}@anchor{4a0}
 @subsection Elaboration order
 
 
@@ -33766,7 +33858,7 @@ pragmas either globally (as an effect of the `-gnatE' switch) or locally
 @end itemize
 
 @node Target-specific aspects,,Elaboration order,Implementation-dependent characteristics
-@anchor{gnat_rm/compatibility_and_porting_guide id14}@anchor{4a0}@anchor{gnat_rm/compatibility_and_porting_guide target-specific-aspects}@anchor{4a1}
+@anchor{gnat_rm/compatibility_and_porting_guide id14}@anchor{4a1}@anchor{gnat_rm/compatibility_and_porting_guide target-specific-aspects}@anchor{4a2}
 @subsection Target-specific aspects
 
 
@@ -33779,10 +33871,10 @@ on the robustness of the original design.  Moreover, Ada 95 (and thus
 Ada 2005, Ada 2012, and Ada 2022) are sometimes
 incompatible with typical Ada 83 compiler practices regarding implicit
 packing, the meaning of the Size attribute, and the size of access values.
-GNAT’s approach to these issues is described in @ref{4a2,,Representation Clauses}.
+GNAT’s approach to these issues is described in @ref{4a3,,Representation Clauses}.
 
 @node Compatibility with Other Ada Systems,Representation Clauses,Implementation-dependent characteristics,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-other-ada-systems}@anchor{4a3}@anchor{gnat_rm/compatibility_and_porting_guide id15}@anchor{4a4}
+@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-other-ada-systems}@anchor{4a4}@anchor{gnat_rm/compatibility_and_porting_guide id15}@anchor{4a5}
 @section Compatibility with Other Ada Systems
 
 
@@ -33825,7 +33917,7 @@ far beyond this minimal set, as described in the next section.
 @end itemize
 
 @node Representation Clauses,Compatibility with HP Ada 83,Compatibility with Other Ada Systems,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide id16}@anchor{4a5}@anchor{gnat_rm/compatibility_and_porting_guide representation-clauses}@anchor{4a2}
+@anchor{gnat_rm/compatibility_and_porting_guide id16}@anchor{4a6}@anchor{gnat_rm/compatibility_and_porting_guide representation-clauses}@anchor{4a3}
 @section Representation Clauses
 
 
@@ -33918,7 +34010,7 @@ with thin pointers.
 @end itemize
 
 @node Compatibility with HP Ada 83,,Representation Clauses,Compatibility and Porting Guide
-@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-hp-ada-83}@anchor{4a6}@anchor{gnat_rm/compatibility_and_porting_guide id17}@anchor{4a7}
+@anchor{gnat_rm/compatibility_and_porting_guide compatibility-with-hp-ada-83}@anchor{4a7}@anchor{gnat_rm/compatibility_and_porting_guide id17}@anchor{4a8}
 @section Compatibility with HP Ada 83
 
 
@@ -33948,7 +34040,7 @@ extension of package System.
 @end itemize
 
 @node GNU Free Documentation License,Index,Compatibility and Porting Guide,Top
-@anchor{share/gnu_free_documentation_license doc}@anchor{4a8}@anchor{share/gnu_free_documentation_license gnu-fdl}@anchor{1}@anchor{share/gnu_free_documentation_license gnu-free-documentation-license}@anchor{4a9}
+@anchor{share/gnu_free_documentation_license doc}@anchor{4a9}@anchor{share/gnu_free_documentation_license gnu-fdl}@anchor{1}@anchor{share/gnu_free_documentation_license gnu-free-documentation-license}@anchor{4aa}
 @chapter GNU Free Documentation License
 
 
diff --git a/gcc/ada/gsocket.h b/gcc/ada/gsocket.h
index 79836e8..5a553d1 100644
--- a/gcc/ada/gsocket.h
+++ b/gcc/ada/gsocket.h
@@ -64,6 +64,12 @@
 #include <ioLib.h>
 #include <hostLib.h>
 
+#if __has_include ("strings.h")
+/* On VxWorks6, FD_ZERO uses bzero, but since it's not a standard header, don't
+   require it.  */
+#include "strings.h"
+#endif
+
 #define SHUT_RD		0
 #define SHUT_WR		1
 #define SHUT_RDWR	2
diff --git a/gcc/ada/inline.adb b/gcc/ada/inline.adb
index e8eeebd..5e2f033 100644
--- a/gcc/ada/inline.adb
+++ b/gcc/ada/inline.adb
@@ -3797,6 +3797,17 @@ package body Inline is
                           and then Is_Unc;
       end if;
 
+      --  Inlining function calls returning an object of unconstrained type as
+      --  function actuals or in a return statement is not supported: a
+      --  temporary variable will be declared of unconstrained type without
+      --  initializing expression.
+
+      pragma Assert
+        (not Uses_Back_End
+           or else Nkind (Parent (N)) not in
+             N_Function_Call | N_Simple_Return_Statement
+           or else not Is_Unc);
+
       --  Check for an illegal attempt to inline a recursive procedure. If the
       --  subprogram has parameters this is detected when trying to supply a
       --  binding for parameters that already have one. For parameterless
diff --git a/gcc/ada/libgnarl/s-taskin.ads b/gcc/ada/libgnarl/s-taskin.ads
index d68e199..dbf2e7b 100644
--- a/gcc/ada/libgnarl/s-taskin.ads
+++ b/gcc/ada/libgnarl/s-taskin.ads
@@ -390,7 +390,7 @@ package System.Tasking is
    System_Domain : Dispatching_Domain_Access;
    --  All processors belong to default system dispatching domain at start up.
    --  We use a pointer which creates the actual variable for the reasons
-   --  explained bellow in Dispatching_Domain_Tasks.
+   --  explained below in Dispatching_Domain_Tasks.
 
    Dispatching_Domains_Frozen : Boolean := False;
    --  True when the main procedure has been called. Hence, no new dispatching
diff --git a/gcc/ada/libgnat/a-calend.adb b/gcc/ada/libgnat/a-calend.adb
index 4f89a40..0be5673 100644
--- a/gcc/ada/libgnat/a-calend.adb
+++ b/gcc/ada/libgnat/a-calend.adb
@@ -1068,19 +1068,28 @@ is
          tv_nsec : out Long_Integer)
       is
          pragma Unsuppress (Overflow_Check);
-         Secs      : Duration;
-         Nano_Secs : Duration;
 
       begin
-         --  Seconds extraction, avoid potential rounding errors
-
-         Secs   := D - 0.5;
-         tv_sec := Long_Long_Integer (Secs);
-
-         --  Nanoseconds extraction
+         if D = 0.0 then
+            tv_sec  := 0;
+            tv_nsec := 0;
+
+         elsif D < 0.0 then
+            tv_sec := Long_Long_Integer (D + 0.5);
+            if D = Duration (tv_sec) then
+               tv_nsec := 0;
+            else
+               tv_nsec := Long_Integer ((D - Duration (tv_sec)) * Nano + 0.5);
+            end if;
 
-         Nano_Secs := D - Duration (tv_sec);
-         tv_nsec := Long_Integer (Nano_Secs * Nano);
+         else
+            tv_sec := Long_Long_Integer (D - 0.5);
+            if D = Duration (tv_sec) then
+               tv_nsec := 0;
+            else
+               tv_nsec := Long_Integer ((D - Duration (tv_sec)) * Nano - 0.5);
+            end if;
+         end if;
       end To_Struct_Timespec_64;
 
       ------------------
diff --git a/gcc/ada/libgnat/a-cbhama.adb b/gcc/ada/libgnat/a-cbhama.adb
index ee6584d..b2d7964 100644
--- a/gcc/ada/libgnat/a-cbhama.adb
+++ b/gcc/ada/libgnat/a-cbhama.adb
@@ -368,7 +368,7 @@ is
    -- Empty --
    -----------
 
-   function Empty (Capacity : Count_Type) return Map is
+   function Empty (Capacity : Count_Type := 10) return Map is
    begin
       return Result : Map (Capacity, 0) do
          null;
diff --git a/gcc/ada/libgnat/a-cbhama.ads b/gcc/ada/libgnat/a-cbhama.ads
index 6ffc815..c741b40 100644
--- a/gcc/ada/libgnat/a-cbhama.ads
+++ b/gcc/ada/libgnat/a-cbhama.ads
@@ -71,7 +71,7 @@ is
    --  Map objects declared without an initialization expression are
    --  initialized to the value Empty_Map.
 
-   function Empty (Capacity : Count_Type) return Map;
+   function Empty (Capacity : Count_Type := 10) return Map;
 
    No_Element : constant Cursor;
    --  Cursor objects declared without an initialization expression are
diff --git a/gcc/ada/libgnat/g-calend.adb b/gcc/ada/libgnat/g-calend.adb
index e410c3e..a2bc77c 100644
--- a/gcc/ada/libgnat/g-calend.adb
+++ b/gcc/ada/libgnat/g-calend.adb
@@ -344,6 +344,8 @@ package body GNAT.Calendar is
       sec   : aliased C.Extensions.long_long;
       usec  : aliased C.long;
 
+      pragma Unsuppress (Overflow_Check);
+
    begin
       timeval_to_duration (T, sec'Access, usec'Access);
       pragma Annotate (CodePeer, Modified, sec);
@@ -369,13 +371,28 @@ package body GNAT.Calendar is
       sec    : C.Extensions.long_long;
       usec   : C.long;
 
+      pragma Unsuppress (Overflow_Check);
+
    begin
       if D = 0.0 then
          sec  := 0;
          usec := 0;
+
+      elsif D < 0.0 then
+         sec := C.Extensions.long_long (D + 0.5);
+         if D = Duration (sec) then
+            usec := 0;
+         else
+            usec := C.long ((D - Duration (sec)) * Micro + 0.5);
+         end if;
+
       else
-         sec  := C.Extensions.long_long (D - 0.5);
-         usec := C.long ((D - Duration (sec)) * Micro - 0.5);
+         sec := C.Extensions.long_long (D - 0.5);
+         if D = Duration (sec) then
+            usec := 0;
+         else
+            usec := C.long ((D - Duration (sec)) * Micro - 0.5);
+         end if;
       end if;
 
       duration_to_timeval (sec, usec, Result'Access);
diff --git a/gcc/ada/libgnat/g-socket.adb b/gcc/ada/libgnat/g-socket.adb
index 5042dac..0fed791 100644
--- a/gcc/ada/libgnat/g-socket.adb
+++ b/gcc/ada/libgnat/g-socket.adb
@@ -3059,12 +3059,11 @@ package body GNAT.Sockets is
       --  Normal case where we do round down
 
       else
-         S  := time_t (Val - 0.5);
-         uS := suseconds_t (1_000_000 * (Val - Selector_Duration (S)) - 0.5);
-
-         if uS = -1 then
-            --  It happen on integer duration
+         S := time_t (Val - 0.5);
+         if Val = Timeval_Duration (S) then
             uS := 0;
+         else
+            uS := suseconds_t ((Val - Timeval_Duration (S)) * 1_000_000 - 0.5);
          end if;
       end if;
 
diff --git a/gcc/ada/libgnat/system-linux-loongarch.ads b/gcc/ada/libgnat/system-linux-loongarch.ads
index 77a2139..683b7a4 100644
--- a/gcc/ada/libgnat/system-linux-loongarch.ads
+++ b/gcc/ada/libgnat/system-linux-loongarch.ads
@@ -139,7 +139,6 @@ private
    Always_Compatible_Rep     : constant Boolean := False;
    Suppress_Standard_Library : constant Boolean := False;
    Use_Ada_Main_Program_Name : constant Boolean := False;
-   Frontend_Exceptions       : constant Boolean := False;
    ZCX_By_Default            : constant Boolean := True;
 
 end System;
diff --git a/gcc/ada/mutably_tagged.adb b/gcc/ada/mutably_tagged.adb
index 153d168..b04ba92 100644
--- a/gcc/ada/mutably_tagged.adb
+++ b/gcc/ada/mutably_tagged.adb
@@ -40,6 +40,7 @@ with Sinfo.Nodes;    use Sinfo.Nodes;
 with Sinfo.Utils;    use Sinfo.Utils;
 with Stringt;        use Stringt;
 with Tbuild;         use Tbuild;
+with Uintp;          use Uintp;
 
 package body Mutably_Tagged is
 
@@ -205,21 +206,41 @@ package body Mutably_Tagged is
       Mut_Tag_Typ : Entity_Id) return Node_Id
    is
       Loc : constant Source_Ptr := Sloc (New_Typ);
+
+      CW_Size : constant Uint := RM_Size (Mut_Tag_Typ);
+
+      function To_Mixed_Case (S : String) return String;
+      --  convert string to mixed case
+
+      -------------------
+      -- To_Mixed_Case --
+      -------------------
+
+      function To_Mixed_Case (S : String) return String is
+         Buf : Bounded_String;
+      begin
+         Append (Buf, S);
+         Set_Casing (Buf, Mixed_Case);
+         return +Buf;
+      end To_Mixed_Case;
+
+   --  Start of processing for Make_CW_Size_Compile_Check
+
    begin
-      --  Generate a string literal for New_Typ's name which is needed for
-      --  printing within the Compile_Time_Error.
+      --  Build a Compile_Time_Error pragma in order to defer the
+      --  (compile-time) size check until after the back end has
+      --  determined sizes.
+      --
+      --  It would be nice if we could somehow include the value of
+      --  New_Type'Size in the error message, but it is not clear how to
+      --  accomplish that with the current FE/BE interfaces.
+
+      --  Get New_Typ's name (in mixed case) into the name buffer;
+      --  this is used immediately afterwards in the Make_Pragma call.
 
       Get_Decoded_Name_String (Chars (New_Typ));
       Set_Casing (Mixed_Case);
 
-      --  Build a pragma Compile_Time_Error to force the backend to
-      --  preform appropriate sizing checks.
-
-      --  Generate:
-      --    pragma Compile_Time_Error
-      --             (New_Typ'Size < Mut_Tag_Typ'Size,
-      --              "class size for by-reference type ""New_Typ"" too small")
-
       return
         Make_Pragma (Loc,
           Chars                        => Name_Compile_Time_Error,
@@ -233,19 +254,18 @@ package body Mutably_Tagged is
                       Prefix         =>
                         New_Occurrence_Of (New_Typ, Loc)),
                   Right_Opnd =>
-                    Make_Integer_Literal (Loc,
-                      RM_Size (Mut_Tag_Typ))))),
+                    Make_Integer_Literal (Loc, CW_Size)))),
              Make_Pragma_Argument_Association (Loc,
                Expression =>
-
-                 --  Is it possible to print the size of New_Typ via
-                 --  Validate_Compile_Time_Warning_Or_Error after the back-end
-                 --  has run to generate the error message manually ???
-
                  Make_String_Literal (Loc,
-                   "class size for by-reference type """
-                   & To_String (String_From_Name_Buffer)
-                   & """ too small"))));
+                   To_String (String_From_Name_Buffer)
+                     & "'Size exceeds "
+                     & To_Mixed_Case (
+                         To_String (Fully_Qualified_Name_String
+                                     (Find_Specific_Type (Mut_Tag_Typ),
+                                      Append_NUL => False)))
+                     & "'Size'Class limit of "
+                     & UI_Image (CW_Size)))));
    end Make_CW_Size_Compile_Check;
 
    ------------------------------------
diff --git a/gcc/ada/par-load.adb b/gcc/ada/par-load.adb
index 96fa7e8..4a97f14 100644
--- a/gcc/ada/par-load.adb
+++ b/gcc/ada/par-load.adb
@@ -83,7 +83,7 @@ procedure Load is
    --  withed units and the second round handles Ada 2005 limited-withed units.
    --  This is required to allow the low-level circuitry that detects circular
    --  dependencies of units the correct notification of errors (see comment
-   --  bellow). This variable is used to indicate that the second round is
+   --  below). This variable is used to indicate that the second round is
    --  required.
 
    function Same_File_Name_Except_For_Case
diff --git a/gcc/ada/s-oscons-tmplt.c b/gcc/ada/s-oscons-tmplt.c
index 97537ef..7a5e987 100644
--- a/gcc/ada/s-oscons-tmplt.c
+++ b/gcc/ada/s-oscons-tmplt.c
@@ -1976,7 +1976,7 @@ CND(CLOCK_THREAD_CPUTIME_ID, "Thread CPU clock")
 #if defined(__linux__) || defined(__FreeBSD__) \
  || (defined(_AIX) && defined(_AIXVERSION_530)) \
  || defined(__DragonFly__) || defined(__QNX__) \
- || defined (__vxworks)
+ || (defined (__vxworks) && /* VxWorks7 */ defined (_VSB_CONFIG_FILE))
 /** On these platforms use system provided monotonic clock instead of
  ** the default CLOCK_REALTIME. We then need to set up cond var attributes
  ** appropriately (see thread.c).
@@ -1985,6 +1985,10 @@ CND(CLOCK_THREAD_CPUTIME_ID, "Thread CPU clock")
  ** pthread_cond_timedwait (and does not have pthread_condattr_setclock),
  ** hence the conditionalization on AIX version above). _AIXVERSION_530
  ** is defined in AIX 5.3 and more recent versions.
+ **
+ ** VxWorks6 lacks pthread_condattr_setclock, so define CLOCK_RT_Ada to
+ ** CLOCK_REALTIME to get the dummy definition of __gnat_pthread_condattr_setup
+ ** in libgnarl/thread.c.
  **/
 # define CLOCK_RT_Ada "CLOCK_MONOTONIC"
 
diff --git a/gcc/ada/sem_attr.adb b/gcc/ada/sem_attr.adb
index 9602944..f38380c 100644
--- a/gcc/ada/sem_attr.adb
+++ b/gcc/ada/sem_attr.adb
@@ -9359,6 +9359,20 @@ package body Sem_Attr is
       when Attribute_First =>
          Set_Bounds;
 
+         --  In GNATprove mode we only fold array attributes when prefix is
+         --  static (because that's required by the Ada rules) or at least can
+         --  be evaluated without checks (because GNATprove would miss them).
+
+         if GNATprove_Mode
+            and then
+              not (Static
+                   or else (Is_Entity_Name (P) and then Is_Type (Entity (P)))
+                   or else Statically_Names_Object (P)
+                   or else Ekind (P_Type) = E_String_Literal_Subtype)
+         then
+            return;
+         end if;
+
          if Compile_Time_Known_Value (Lo_Bound) then
             if Is_Real_Type (P_Type) then
                Fold_Ureal (N, Expr_Value_R (Lo_Bound), Static);
@@ -9572,6 +9586,20 @@ package body Sem_Attr is
       when Attribute_Last =>
          Set_Bounds;
 
+         --  In GNATprove mode we only fold array attributes when prefix is
+         --  static (because that's required by the Ada rules) or at least can
+         --  be evaluated without checks (because GNATprove would miss them).
+
+         if GNATprove_Mode
+            and then
+              not (Static
+                   or else (Is_Entity_Name (P) and then Is_Type (Entity (P)))
+                   or else Statically_Names_Object (P)
+                   or else Ekind (P_Type) = E_String_Literal_Subtype)
+         then
+            return;
+         end if;
+
          if Compile_Time_Known_Value (Hi_Bound) then
             if Is_Real_Type (P_Type) then
                Fold_Ureal (N, Expr_Value_R (Hi_Bound), Static);
@@ -9655,6 +9683,20 @@ package body Sem_Attr is
 
          Set_Bounds;
 
+         --  In GNATprove mode we only fold array attributes when prefix is
+         --  static (because that's required by the Ada rules) or at least can
+         --  be evaluated without checks (because GNATprove would miss them).
+
+         if GNATprove_Mode
+            and then
+              not (Static
+                   or else (Is_Entity_Name (P) and then Is_Type (Entity (P)))
+                   or else Statically_Names_Object (P)
+                   or else Ekind (P_Type) = E_String_Literal_Subtype)
+         then
+            return;
+         end if;
+
          --  For two compile time values, we can compute length
 
          if Compile_Time_Known_Value (Lo_Bound)
diff --git a/gcc/ada/sem_aux.adb b/gcc/ada/sem_aux.adb
index bb1624d..0aa74e3 100644
--- a/gcc/ada/sem_aux.adb
+++ b/gcc/ada/sem_aux.adb
@@ -25,7 +25,6 @@
 
 with Atree;          use Atree;
 with Einfo;          use Einfo;
-with Einfo.Entities; use Einfo.Entities;
 with Einfo.Utils;    use Einfo.Utils;
 with Nlists;         use Nlists;
 with Sinfo;          use Sinfo;
@@ -454,16 +453,28 @@ package body Sem_Aux is
       Id  : Entity_Id;
 
    begin
+      --  Call using access to subprogram with explicit dereference
+
       if Nkind (Nam) = N_Explicit_Dereference then
          Id := Etype (Nam);
          pragma Assert (Ekind (Id) = E_Subprogram_Type);
 
+      --  Case of call to simple entry, where the Name is a selected component
+      --  whose prefix is the task or protected record, and whose selector name
+      --  is the entry name.
+
       elsif Nkind (Nam) = N_Selected_Component then
          Id := Entity (Selector_Name (Nam));
 
+      --  Case of call to member of entry family, where Name is an indexed
+      --  component, with the prefix being a selected component giving the
+      --  task and entry family name, and the index being the entry index.
+
       elsif Nkind (Nam) = N_Indexed_Component then
          Id := Entity (Selector_Name (Prefix (Nam)));
 
+      --  Normal case
+
       else
          Id := Entity (Nam);
       end if;
@@ -1546,6 +1557,81 @@ package body Sem_Aux is
       return E;
    end Ultimate_Alias;
 
+   ---------------------------
+   -- Unique_Component_Name --
+   ---------------------------
+
+   function Unique_Component_Name
+     (Component : Record_Field_Kind_Id) return Name_Id
+   is
+      Homographic_Component_Count : Pos := 1;
+      Hcc                         : Pos renames Homographic_Component_Count;
+      Enclosing_Type              : Entity_Id :=
+        Underlying_Type (Base_Type (Scope (Component)));
+   begin
+      if Ekind (Enclosing_Type) = E_Record_Type
+        and then Is_Tagged_Type (Enclosing_Type)
+        and then Has_Private_Ancestor (Enclosing_Type)
+      then
+         --  traverse ancestors to determine Hcc value
+         loop
+            declare
+               Type_Decl : constant Node_Id :=
+                 Parent (Underlying_Type (Base_Type (Enclosing_Type)));
+               Type_Def : constant Node_Id := Type_Definition (Type_Decl);
+            begin
+               exit when Nkind (Type_Def) /= N_Derived_Type_Definition;
+               Enclosing_Type :=
+                 Underlying_Type (Base_Type (Etype (Enclosing_Type)));
+
+               declare
+                  Ancestor_Comp : Opt_Record_Field_Kind_Id :=
+                    First_Component_Or_Discriminant (Enclosing_Type);
+               begin
+                  while Present (Ancestor_Comp) loop
+                     if Chars (Ancestor_Comp) = Chars (Component) then
+                        Hcc := Hcc + 1;
+                        exit; -- exit not required, but might as well
+                     end if;
+                     Next_Component_Or_Discriminant (Ancestor_Comp);
+                  end loop;
+               end;
+            end;
+         end loop;
+      end if;
+
+      if Hcc = 1 then
+         --  the usual case
+         return Chars (Component);
+      else
+         declare
+            Buff : Bounded_String;
+         begin
+            Append (Buff, Chars (Component));
+
+            Append (Buff, "__");
+            --  A double underscore in an identifier is legal in C, not in Ada.
+            --  Returning a result that is not a legal Ada identifier
+            --  ensures that we won't have problems with collisions.
+            --  If we have a component named Foo and we just append a
+            --  number (without any underscores), that new name might match
+            --  the name of another component (which would be bad).
+            --  The result of this function is intended for use as an
+            --  identifier in generated C code, so it needs to be a
+            --  legal C identifer.
+
+            Append (Buff, Hcc);
+            --  Should we instead append Hcc - 1 here? This is a human
+            --  readability question. If parent type and extension each
+            --  have a Foo component, do we want the name returned for the
+            --  second Foo to be "foo__2" or "foo__1" ? Does it matter?
+            --  Either way, the name returned for the first Foo will be "foo".
+
+            return Name_Find (Buff);
+         end;
+      end if;
+   end Unique_Component_Name;
+
    --------------------------
    -- Unit_Declaration_Node --
    --------------------------
diff --git a/gcc/ada/sem_aux.ads b/gcc/ada/sem_aux.ads
index 5a63002..1a298a9 100644
--- a/gcc/ada/sem_aux.ads
+++ b/gcc/ada/sem_aux.ads
@@ -31,6 +31,7 @@
 --  require more than minimal semantic knowledge.
 
 with Alloc;
+with Einfo.Entities; use Einfo.Entities;
 with Namet; use Namet;
 with Table;
 with Types; use Types;
@@ -105,7 +106,7 @@ package Sem_Aux is
    --  this is equivalent to First_Entity. The exception arises for tagged
    --  types, where the tag itself is prepended to the front of the entity
    --  chain, so the First_Discriminant function steps past the tag if it is
-   --  present.  When called on a private type with unknown discriminants, the
+   --  present. When called on a private type with unknown discriminants, the
    --  function always returns Empty.
 
    --  WARNING: There is a matching C declaration of this subprogram in fe.h
@@ -405,6 +406,19 @@ package Sem_Aux is
    --  Return the last entity in the chain of aliased entities of Prim. If Prim
    --  has no alias return Prim.
 
+   function Unique_Component_Name
+     (Component : Record_Field_Kind_Id) return Name_Id;
+   --  Usually, a record type cannot have two components with the same name.
+   --  But in the case of a component declared in an extension of a tagged
+   --  private (or private extension) parent type, it is possible that some
+   --  ancestor type also has a (non-visible) component with the same name.
+   --  In the common case, this function simply returns the Chars attribute
+   --  of its argument.
+   --  But in the multiple-components-with-the-same-name case, it appends
+   --  a uniquifying suffix. The result in this case will not be a
+   --  syntactically valid Ada identifier, but it will be a syntactically
+   --  valid C identifier.
+
    function Unit_Declaration_Node (Unit_Id : Entity_Id) return Node_Id;
    --  Unit_Id is the simple name of a program unit, this function returns the
    --  corresponding xxx_Declaration node for the entity. Also applies to the
diff --git a/gcc/ada/sem_ch12.adb b/gcc/ada/sem_ch12.adb
index e80aea5..b5c9e88 100644
--- a/gcc/ada/sem_ch12.adb
+++ b/gcc/ada/sem_ch12.adb
@@ -7574,6 +7574,12 @@ package body Sem_Ch12 is
                     or else not Same_Instantiated_Function (E1, E2));
             end if;
 
+         --  No check is needed if this is the body of a subprogram that is
+         --  implicitly created in the case of class-wide predefined functions.
+
+         elsif Ekind (E1) = E_Subprogram_Body then
+            null;
+
          else
             raise Program_Error;
          end if;
@@ -14371,8 +14377,21 @@ package body Sem_Ch12 is
          elsif
            Scope (Scope (Base_Type (Etype (A_Gen_T)))) = Scope (A_Gen_T)
          then
-            Ancestor :=
-              Get_Instance_Of (Base_Type (Etype (A_Gen_T)));
+            declare
+               Formal_Ancestor : constant Entity_Id :=
+                 Base_Type (Etype (A_Gen_T));
+            begin
+               Ancestor := Get_Instance_Of (Formal_Ancestor);
+
+               --  Handle (rare) case where Get_Instance_Of found nothing in
+               --  the map.
+
+               if Ancestor = Formal_Ancestor then
+                  Ancestor :=
+                    Get_Instance_Of
+                      (Base_Type (Etype (Get_Instance_Of (A_Gen_T))));
+               end if;
+            end;
 
          --  The type may be a local derivation, or a type extension of a
          --  previous formal, or of a formal of a parent package.
diff --git a/gcc/ada/sem_ch13.adb b/gcc/ada/sem_ch13.adb
index 1e88ef4..b7ada50 100644
--- a/gcc/ada/sem_ch13.adb
+++ b/gcc/ada/sem_ch13.adb
@@ -337,6 +337,13 @@ package body Sem_Ch13 is
    --  Resolve each one of the arguments specified in the specification of
    --  aspect Finalizable.
 
+   function Resolve_Finalization_Procedure
+     (N   : Node_Id;
+      Typ : Entity_Id) return Boolean;
+   --  Resolve a procedure argument specified in the specification of one of
+   --  the finalization aspects, i.e. Finalizable and Destructor. Returns True
+   --  if successful, False otherwise.
+
    procedure Resolve_Iterable_Operation
      (N      : Node_Id;
       Cursor : Entity_Id;
@@ -4634,6 +4641,7 @@ package body Sem_Ch13 is
 
                when Aspect_Designated_Storage_Model =>
                   if not All_Extensions_Allowed then
+                     Error_Msg_Name_1 := Nam;
                      Error_Msg_GNAT_Extension ("aspect %", Loc);
                      goto Continue;
 
@@ -4646,6 +4654,20 @@ package body Sem_Ch13 is
                      goto Continue;
                   end if;
 
+               when Aspect_Destructor =>
+                  if not All_Extensions_Allowed then
+                     Error_Msg_Name_1 := Nam;
+                     Error_Msg_GNAT_Extension ("aspect %", Loc);
+                     goto Continue;
+
+                  elsif not Is_Type (E) then
+                     Error_Msg_N ("can only be specified for a type", Aspect);
+                     goto Continue;
+                  end if;
+
+                  Set_Has_Destructor (E);
+                  Set_Is_Controlled_Active (E);
+
                when Aspect_Storage_Model_Type =>
                   if not All_Extensions_Allowed then
                      Error_Msg_Name_1 := Nam;
@@ -5063,6 +5085,14 @@ package body Sem_Ch13 is
                         Check_Expr_Is_OK_Static_Expression (Expr, Any_Boolean);
                      end if;
 
+                     --  Record the No_Task_Parts aspects as a rep item so it
+                     --  can be consistently looked up on the full view of the
+                     --  type.
+
+                     if Is_Private_Type (E) then
+                        Record_Rep_Item (E, Aspect);
+                     end if;
+
                      goto Continue;
 
                   --  Ada 2022 (AI12-0075): static expression functions
@@ -8861,6 +8891,43 @@ package body Sem_Ch13 is
             Num_Repped_Components   : Nat := 0;
             Num_Unrepped_Components : Nat := 0;
 
+            function Unchecked_Union_Pragma_Pending return Boolean;
+            --  Return True in the corner case of an Unchecked_Union pragma
+            --  occuring after the record representation clause (which
+            --  means that Is_Unchecked_Union will return False for Rectype,
+            --  even though it would return True if called later after the
+            --  pragma is analyzed).
+
+            ------------------------------------
+            -- Unchecked_Union_Pragma_Pending --
+            ------------------------------------
+
+            function Unchecked_Union_Pragma_Pending return Boolean is
+               Decl_List_Element : Node_Id := N;
+               Pragma_Arg : Node_Id;
+            begin
+               while Present (Decl_List_Element) loop
+                  if Nkind (Decl_List_Element) = N_Pragma
+                    and then Get_Pragma_Id (Decl_List_Element) =
+                             Pragma_Unchecked_Union
+                    and then not Is_Empty_List (Pragma_Argument_Associations
+                                                  (Decl_List_Element))
+                  then
+                     Pragma_Arg := Get_Pragma_Arg
+                                     (First (Pragma_Argument_Associations
+                                               (Decl_List_Element)));
+                     if Nkind (Pragma_Arg) = N_Identifier
+                       and then Chars (Pragma_Arg) = Chars (Rectype)
+                     then
+                        return True;
+                     end if;
+                  end if;
+
+                  Next (Decl_List_Element);
+               end loop;
+               return False;
+            end Unchecked_Union_Pragma_Pending;
+
          begin
             --  First count number of repped and unrepped components
 
@@ -8899,8 +8966,10 @@ package body Sem_Ch13 is
                     --  Ignore discriminant in unchecked union, since it is
                     --  not there, and cannot have a component clause.
 
-                    and then (not Is_Unchecked_Union (Rectype)
-                               or else Ekind (Comp) /= E_Discriminant)
+                    and then (Ekind (Comp) /= E_Discriminant
+                              or else not (Is_Unchecked_Union (Rectype)
+                                           or else
+                                             Unchecked_Union_Pragma_Pending))
                   then
                      Error_Msg_Sloc := Sloc (Comp);
                      Error_Msg_NE
@@ -11201,6 +11270,13 @@ package body Sem_Ch13 is
    --  Start of processing for Check_Aspect_At_End_Of_Declarations
 
    begin
+      --  Indicate that the expression comes from an aspect specification,
+      --  which is used in subsequent analysis even if expansion is off.
+
+      if Present (End_Decl_Expr) then
+         Set_Parent (End_Decl_Expr, ASN);
+      end if;
+
       --  In an instance we do not perform the consistency check between freeze
       --  point and end of declarations, because it was done already in the
       --  analysis of the generic. Furthermore, the delayed analysis of an
@@ -11230,6 +11306,7 @@ package body Sem_Ch13 is
       --  the one available at at the freeze point.
 
       elsif A_Id in Aspect_Constructor
+                  | Aspect_Destructor
                   | Aspect_Input
                   | Aspect_Output
                   | Aspect_Read
@@ -11292,13 +11369,6 @@ package body Sem_Ch13 is
             end if;
          end if;
 
-         --  Indicate that the expression comes from an aspect specification,
-         --  which is used in subsequent analysis even if expansion is off.
-
-         if Present (End_Decl_Expr) then
-            Set_Parent (End_Decl_Expr, ASN);
-         end if;
-
          --  In a generic context the original aspect expressions have not
          --  been preanalyzed, so do it now. There are no conformance checks
          --  to perform in this case. As before, we have to make components
@@ -11406,24 +11476,16 @@ package body Sem_Ch13 is
    ----------------------------------
 
    procedure Check_Aspect_At_Freeze_Point (ASN : Node_Id) is
-      Ident : constant Node_Id := Identifier (ASN);
-      --  Identifier (use Entity field to save expression)
-
       Expr : constant Node_Id := Expression (ASN);
-      --  For cases where using Entity (Identifier) doesn't work
 
-      A_Id : constant Aspect_Id := Get_Aspect_Id (Chars (Ident));
+      A_Id : constant Aspect_Id := Get_Aspect_Id (Chars (Identifier (ASN)));
 
       T : Entity_Id := Empty;
       --  Type required for preanalyze call
 
    begin
-      --  On entry to this procedure, Entity (Ident) contains a copy of the
-      --  original expression from the aspect, saved for this purpose.
-
-      --  On exit from this procedure Entity (Ident) is unchanged, still
-      --  containing that copy, but Expression (Ident) is a preanalyzed copy
-      --  of the expression, preanalyzed just after the freeze point.
+      --  On exit from this procedure, Expression (ASN) is a copy of the
+      --  original expression, preanalyzed just after the freeze point.
 
       --  Make a copy of the expression to be preanalyzed
 
@@ -11702,6 +11764,67 @@ package body Sem_Ch13 is
             Analyze (Expression (ASN));
             return;
 
+         when Aspect_Destructor =>
+            if not Is_Record_Type (Entity (ASN)) then
+               Error_Msg_N
+                 ("aspect Destructor can only be specified for a "
+                  & "record type",
+                  ASN);
+               return;
+            end if;
+
+            Set_Has_Destructor (Entity (ASN));
+            Set_Is_Controlled_Active (Entity (ASN));
+
+            Analyze (Expression (ASN));
+
+            if not Resolve_Finalization_Procedure
+                     (Expression (ASN), Entity (ASN))
+            then
+               Error_Msg_N
+                 ("destructor must be local procedure whose only formal "
+                  & "parameter has mode `IN OUT` and is of the type the "
+                  & "destructor is for",
+                  Expression (ASN));
+            end if;
+
+            Set_Is_Destructor (Entity (Expression (ASN)));
+
+            declare
+               Proc  : constant Entity_Id := Entity (Expression (ASN));
+               Overr : constant Opt_N_Entity_Id :=
+                 Overridden_Inherited_Operation (Proc);
+               Orig  : constant Entity_Id :=
+                 (if Present (Overr) then Overr else Proc);
+
+               Decl : constant Node_Id :=
+                 Parent
+                   (if Nkind (Parent (Orig)) = N_Procedure_Specification
+                    then Parent (Orig)
+                    else Orig);
+
+               Encl : constant Node_Id := Parent (Decl);
+
+               Is_Private : constant Boolean :=
+                 Nkind (Encl) = N_Package_Specification
+                 and then Is_List_Member (Decl)
+                 and then List_Containing (Decl) = Private_Declarations (Encl);
+
+            begin
+
+               if Has_Private_Declaration (Entity (ASN))
+                 and then not Aspect_On_Partial_View (ASN)
+                 and then not Is_Private
+               then
+                  Error_Msg_N
+                    ("aspect Destructor on full view cannot denote public "
+                     & "primitive",
+                     ASN);
+               end if;
+            end;
+
+            return;
+
          when Aspect_Storage_Model_Type =>
 
             --  The aggregate argument of Storage_Model_Type is optional, and
@@ -15855,6 +15978,8 @@ package body Sem_Ch13 is
          --  We may freeze Subp_Id immediately since Ent has just been frozen.
          --  This will help to shield us from potential late freezing issues.
 
+         Mutate_Ekind (Subp_Id, E_Procedure);
+         Freeze_Extra_Formals (Subp_Id);
          Set_Is_Frozen (Subp_Id);
 
       else
@@ -17302,6 +17427,35 @@ package body Sem_Ch13 is
       Typ : Entity_Id;
       Nam : Name_Id)
    is
+   begin
+      if Nam = Name_Relaxed_Finalization then
+         Resolve (N, Any_Boolean);
+
+         if Is_OK_Static_Expression (N) then
+            Set_Has_Relaxed_Finalization (Typ, Is_True (Static_Boolean (N)));
+
+         else
+            Flag_Non_Static_Expr
+              ("expression of aspect Finalizable must be static!", N);
+         end if;
+
+         return;
+      end if;
+
+      if Resolve_Finalization_Procedure (N, Typ) then
+         return;
+      end if;
+
+      Error_Msg_N
+        ("finalizable primitive must be local procedure whose only formal " &
+         "parameter has mode `IN OUT` and is of the finalizable type", N);
+   end Resolve_Finalizable_Argument;
+
+   function Resolve_Finalization_Procedure
+     (N   : Node_Id;
+      Typ : Entity_Id)
+      return Boolean
+   is
       function Is_Finalizable_Primitive (E : Entity_Id) return Boolean;
       --  Check whether E is a finalizable primitive for Typ
 
@@ -17319,29 +17473,15 @@ package body Sem_Ch13 is
            and then No (Next_Formal (First_Formal (E)));
       end Is_Finalizable_Primitive;
 
-   --  Start of processing for Resolve_Finalizable_Argument
+   --  Start of processing for Resolve_Finalization_Procedure
 
    begin
-      if Nam = Name_Relaxed_Finalization then
-         Resolve (N, Any_Boolean);
-
-         if Is_OK_Static_Expression (N) then
-            Set_Has_Relaxed_Finalization (Typ, Is_True (Static_Boolean (N)));
-
-         else
-            Flag_Non_Static_Expr
-              ("expression of aspect Finalizable must be static!", N);
-         end if;
-
-         return;
-      end if;
-
       if not Is_Entity_Name (N) then
          null;
 
       elsif not Is_Overloaded (N) then
          if Is_Finalizable_Primitive (Entity (N)) then
-            return;
+            return True;
          end if;
 
       else
@@ -17357,7 +17497,7 @@ package body Sem_Ch13 is
             while Present (It.Typ) loop
                if Is_Finalizable_Primitive (It.Nam) then
                   Set_Entity (N, It.Nam);
-                  return;
+                  return True;
                end if;
 
                Get_Next_Interp (I, It);
@@ -17365,10 +17505,8 @@ package body Sem_Ch13 is
          end;
       end if;
 
-      Error_Msg_N
-        ("finalizable primitive must be local procedure whose only formal " &
-         "parameter has mode `IN OUT` and is of the finalizable type", N);
-   end Resolve_Finalizable_Argument;
+      return False;
+   end Resolve_Finalization_Procedure;
 
    --------------------------------
    -- Resolve_Iterable_Operation --
diff --git a/gcc/ada/sem_ch3.adb b/gcc/ada/sem_ch3.adb
index 5354d82..3726169 100644
--- a/gcc/ada/sem_ch3.adb
+++ b/gcc/ada/sem_ch3.adb
@@ -3020,6 +3020,8 @@ package body Sem_Ch3 is
    -----------------------------------
 
    procedure Analyze_Full_Type_Declaration (N : Node_Id) is
+      use Deferred_Extra_Formals_Support;
+
       Def    : constant Node_Id   := Type_Definition (N);
       Def_Id : constant Entity_Id := Defining_Identifier (N);
       T      : Entity_Id;
@@ -3558,6 +3560,16 @@ package body Sem_Ch3 is
          end if;
       end if;
 
+      --  If we have some subprogram, subprogram type, or entry, with deferred
+      --  addition of its extra formals (because the underlying type of this
+      --  type was not previously available), then try creating now its extra
+      --  formals. Create also the extra actuals of deferred calls to entities
+      --  with deferred extra formals.
+
+      if Has_Deferred_Extra_Formals (T) then
+         Add_Deferred_Extra_Params (T);
+      end if;
+
       if Ekind (T) = E_Record_Type
         and then Is_Large_Unconstrained_Definite (T)
         and then not Is_Limited_Type (T)
@@ -6819,7 +6831,7 @@ package body Sem_Ch3 is
       --  that the element type is constrained.
 
       if Is_Mutably_Tagged_Type (Element_Type) then
-         Set_Component_Type (T,
+         Set_Component_Type (Base_Type (T),
            Class_Wide_Equivalent_Type (Element_Type));
 
       elsif not Is_Definite_Subtype (Element_Type) then
@@ -8440,15 +8452,17 @@ package body Sem_Ch3 is
          Set_Has_Private_Declaration (Full_Der);
          Set_Has_Private_Declaration (Derived_Type);
 
-         Set_Scope                (Full_Der, Scope (Derived_Type));
-         Set_Is_First_Subtype     (Full_Der, Is_First_Subtype (Derived_Type));
-         Set_Has_Size_Clause      (Full_Der, False);
-         Set_Has_Alignment_Clause (Full_Der, False);
-         Set_Has_Delayed_Freeze   (Full_Der);
-         Set_Is_Frozen            (Full_Der, False);
-         Set_Freeze_Node          (Full_Der, Empty);
-         Set_Depends_On_Private   (Full_Der, Has_Private_Component (Full_Der));
-         Set_Is_Public            (Full_Der, Is_Public (Derived_Type));
+         Set_Scope                 (Full_Der, Scope (Derived_Type));
+         Set_Is_First_Subtype      (Full_Der, Is_First_Subtype (Derived_Type));
+         Set_Has_Size_Clause       (Full_Der, False);
+         Set_Has_Alignment_Clause  (Full_Der, False);
+         Set_Has_Delayed_Freeze    (Full_Der);
+         Set_Is_Frozen             (Full_Der, False);
+         Set_Freeze_Node           (Full_Der, Empty);
+         Set_Depends_On_Private
+           (Full_Der, Has_Private_Component (Full_Der));
+         Set_Is_Public             (Full_Der, Is_Public (Derived_Type));
+         Set_Is_Implicit_Full_View (Full_Der);
 
          --  The convention on the base type may be set in the private part
          --  and not propagated to the subtype until later, so we obtain the
@@ -9646,6 +9660,8 @@ package body Sem_Ch3 is
            (New_Decl, Parent_Base, New_Base,
             Is_Completion => False, Derive_Subps => False);
 
+         Set_Is_Implicit_Full_View (New_Base);
+
          --  ??? This needs re-examination to determine whether the
          --  following call can simply be replaced by a call to Analyze.
 
@@ -21279,8 +21295,8 @@ package body Sem_Ch3 is
       --  On entry, the current scope is the composite type.
 
       --  The discriminants are initially entered into the scope of the type
-      --  via Enter_Name with the default Ekind of E_Void to prevent premature
-      --  use, as explained at the end of this procedure.
+      --  via Enter_Name with Is_Not_Self_Hidden set to False to prevent
+      --  premature use, as explained at the end of this procedure.
 
       Discr := First (Discriminant_Specifications (N));
       while Present (Discr) loop
@@ -21553,12 +21569,12 @@ package body Sem_Ch3 is
       --  expressions of a discriminant part if the specification of the
       --  discriminant is itself given in the discriminant part. (RM 3.7.1)
 
-      --  To detect this, the discriminant names are entered initially with an
-      --  Ekind of E_Void (which is the default Ekind given by Enter_Name). Any
-      --  attempt to use a void entity (for example in an expression that is
-      --  type-checked) produces the error message: premature usage. Now after
-      --  completing the semantic analysis of the discriminant part, we can set
-      --  the Ekind of all the discriminants appropriately.
+      --  To detect this, the discriminant names are entered initially with
+      --  Is_Not_Self_Hidden set to False. Any attempt to use a self-hidden
+      --  entity (for example in an expression that is type-checked) produces
+      --  the error message: premature usage. Now after completing the semantic
+      --  analysis of the discriminant part, we can set Is_Not_Self_Hidden on
+      --  all the discriminants appropriately.
 
       Discr := First (Discriminant_Specifications (N));
       Discr_Number := Uint_1;
diff --git a/gcc/ada/sem_ch4.adb b/gcc/ada/sem_ch4.adb
index dc81467..018c8a0 100644
--- a/gcc/ada/sem_ch4.adb
+++ b/gcc/ada/sem_ch4.adb
@@ -5603,6 +5603,10 @@ package body Sem_Ch4 is
                   if No (Act_Decl) then
                      Set_Etype (N, Etype (Comp));
 
+                     if Is_Mutably_Tagged_CW_Equivalent_Type (Etype (N)) then
+                        Make_Mutably_Tagged_Conversion (N);
+                     end if;
+
                   else
                      --  If discriminants were present in the component
                      --  declaration, they have been replaced by the
@@ -10406,11 +10410,14 @@ package body Sem_Ch4 is
          --  may be candidates, so that Try_Primitive_Operations can examine
          --  them if no real primitive is found.
 
-         function Is_Private_Overriding (Op : Entity_Id) return Boolean;
+         function Is_Callable_Private_Overriding
+           (Op : Entity_Id) return Boolean;
          --  An operation that overrides an inherited operation in the private
          --  part of its package may be hidden, but if the inherited operation
-         --  is visible a direct call to it will dispatch to the private one,
-         --  which is therefore a valid candidate.
+         --  that it overrides is visible, then a direct call to it will
+         --  dispatch to the private one, which is therefore a valid candidate.
+         --  Returns True if the operation can be called from outside the
+         --  enclosing package.
 
          function Names_Match
            (Obj_Type : Entity_Id;
@@ -10581,11 +10588,13 @@ package body Sem_Ch4 is
             return Op_List;
          end Extended_Primitive_Ops;
 
-         ---------------------------
-         -- Is_Private_Overriding --
-         ---------------------------
+         ------------------------------------
+         -- Is_Callable_Private_Overriding --
+         ------------------------------------
 
-         function Is_Private_Overriding (Op : Entity_Id) return Boolean is
+         function Is_Callable_Private_Overriding
+           (Op : Entity_Id) return Boolean
+         is
             Visible_Op : Entity_Id;
 
          begin
@@ -10607,7 +10616,10 @@ package body Sem_Ch4 is
                   --  have found what we're looking for.
 
                   if not Is_Hidden (Visible_Op)
-                    or else not Is_Hidden (Overridden_Operation (Op))
+                    or else
+                      (Present (Overridden_Inherited_Operation (Op))
+                        and then not Is_Hidden
+                                       (Overridden_Inherited_Operation (Op)))
                   then
                      return True;
                   end if;
@@ -10617,7 +10629,7 @@ package body Sem_Ch4 is
             end loop;
 
             return False;
-         end Is_Private_Overriding;
+         end Is_Callable_Private_Overriding;
 
          -----------------
          -- Names_Match --
@@ -10692,6 +10704,7 @@ package body Sem_Ch4 is
 
               or else
                 (Has_Unknown_Discriminants (Typ)
+                  and then Is_Record_Type (Base_Type (Obj_Type))
                   and then Typ = Underlying_Record_View (Base_Type (Obj_Type)))
 
                --  Prefix can be dereferenced
@@ -10759,13 +10772,15 @@ package body Sem_Ch4 is
 
                  --  Do not consider hidden primitives unless the type is in an
                  --  open scope or we are within an instance, where visibility
-                 --  is known to be correct, or else if this is an overriding
-                 --  operation in the private part for an inherited operation.
+                 --  is known to be correct, or else if this is an operation
+                 --  declared in the private part that overrides a visible
+                 --  inherited operation.
 
                  or else (Is_Hidden (Prim_Op)
                            and then not Is_Immediately_Visible (Obj_Type)
                            and then not In_Instance
-                           and then not Is_Private_Overriding (Prim_Op))
+                           and then
+                             not Is_Callable_Private_Overriding (Prim_Op))
                then
                   goto Continue;
                end if;
diff --git a/gcc/ada/sem_ch5.adb b/gcc/ada/sem_ch5.adb
index e1d6be4..0661e64 100644
--- a/gcc/ada/sem_ch5.adb
+++ b/gcc/ada/sem_ch5.adb
@@ -807,7 +807,14 @@ package body Sem_Ch5 is
 
       if Is_Tag_Indeterminate (Rhs) then
          if Is_Class_Wide_Type (T1) then
-            Propagate_Tag (Lhs, Rhs);
+
+            --  No need to propagate the tag when the RHS has function calls
+            --  that already propagated it (see Expand_Call_Helper), or if
+            --  some error was reported analyzing RHS.
+
+            if not (Error_Posted (Rhs) or else Tag_Propagated (Lhs)) then
+               Propagate_Tag (Lhs, Rhs);
+            end if;
 
          elsif Nkind (Rhs) = N_Function_Call
            and then Is_Entity_Name (Name (Rhs))
diff --git a/gcc/ada/sem_ch6.adb b/gcc/ada/sem_ch6.adb
index 48dcf8e..709f625 100644
--- a/gcc/ada/sem_ch6.adb
+++ b/gcc/ada/sem_ch6.adb
@@ -3864,9 +3864,14 @@ package body Sem_Ch6 is
          Spec_Id := Build_Internal_Protected_Declaration (N);
       end if;
 
-      --  If a separate spec is present, then deal with freezing issues
+      --  Separate spec is not present
 
-      if Present (Spec_Id) then
+      if No (Spec_Id) then
+         Create_Extra_Formals (Body_Id);
+
+      --  Separate spec is present; deal with freezing issues
+
+      else
          Spec_Decl := Unit_Declaration_Node (Spec_Id);
          Verify_Overriding_Indicator;
 
@@ -3882,6 +3887,8 @@ package body Sem_Ch6 is
            and then not Has_BIP_Formals (Spec_Id)
          then
             Create_Extra_Formals (Spec_Id);
+            pragma Assert (not Expander_Active
+              or else Extra_Formals_Known (Spec_Id));
             Compute_Returns_By_Ref (Spec_Id);
          end if;
 
@@ -8564,14 +8571,13 @@ package body Sem_Ch6 is
       --  without coordinating with CodePeer, which makes use of these to
       --  provide better messages.
 
+      --  A and B denote extra formals for unchecked unions equality. See
+      --  exp_ch3.Build_Variant_Record_Equality.
       --  O denotes the Constrained bit.
       --  L denotes the accessibility level.
       --  BIP_xxx denotes an extra formal for a build-in-place function. See
       --  the full list in exp_ch6.BIP_Formal_Kind.
 
-      function Has_Extra_Formals (E : Entity_Id) return Boolean;
-      --  Determines if E has its extra formals
-
       function Might_Need_BIP_Task_Actuals (E : Entity_Id) return Boolean;
       --  Determines if E is a function or an access to a function returning a
       --  limited tagged type object. On dispatching primitives this predicate
@@ -8610,14 +8616,6 @@ package body Sem_Ch6 is
          EF : Entity_Id;
 
       begin
-         --  A little optimization. Never generate an extra formal for the
-         --  _init operand of an initialization procedure, since it could
-         --  never be used.
-
-         if Chars (Formal) = Name_uInit then
-            return Empty;
-         end if;
-
          EF := Make_Defining_Identifier (Sloc (Assoc_Entity),
                  Chars => New_External_Name (Chars (Assoc_Entity),
                                              Suffix => Suffix));
@@ -8643,25 +8641,22 @@ package body Sem_Ch6 is
          return EF;
       end Add_Extra_Formal;
 
-      -----------------------
-      -- Has_Extra_Formals --
-      -----------------------
-
-      function Has_Extra_Formals (E : Entity_Id) return Boolean is
-      begin
-         return Present (Extra_Formals (E))
-           or else
-             (Ekind (E) = E_Function
-                and then Present (Extra_Accessibility_Of_Result (E)));
-      end Has_Extra_Formals;
-
       ---------------------------------
       -- Might_Need_BIP_Task_Actuals --
       ---------------------------------
 
       function Might_Need_BIP_Task_Actuals (E : Entity_Id) return Boolean is
          Subp_Id  : Entity_Id;
-         Func_Typ : Entity_Id;
+         Original : Entity_Id;
+         Root     : Entity_Id;
+
+         function Has_No_Task_Parts_Enabled (E : Entity_Id) return Boolean
+         is (Has_Enabled_Aspect (E, Aspect_No_Task_Parts));
+
+         function Collect_Ancestors_With_No_Task_Parts is new
+           Collect_Types_In_Hierarchy (Predicate => Has_No_Task_Parts_Enabled);
+
+      --  Start of processing for Might_Need_BIP_Task_Actuals
 
       begin
          if Global_No_Tasking or else No_Run_Time_Mode then
@@ -8689,21 +8684,29 @@ package body Sem_Ch6 is
          then
             Subp_Id := Protected_Body_Subprogram (E);
 
-         else
+         --  For access-to-subprogram types we look at the return type of the
+         --  subprogram type itself, as it cannot be overridden or inherited.
+
+         elsif Ekind (E) = E_Subprogram_Type then
             Subp_Id := E;
-         end if;
 
-         --  We check the root type of the return type since the same
-         --  decision must be taken for all descendants overriding a
-         --  dispatching operation.
+         --  Otherwise, we need to return the same value we would return for
+         --  the original corresponding operation of the root of the aliased
+         --  chain.
+
+         else
+            Subp_Id := Original_Corresponding_Operation (Ultimate_Alias (E));
+         end if;
 
-         Func_Typ := Root_Type (Underlying_Type (Etype (Subp_Id)));
+         Original := Underlying_Type (Etype (Subp_Id));
+         Root := Underlying_Type (Root_Type (Original));
 
          return Ekind (Subp_Id) in E_Function | E_Subprogram_Type
-           and then not Has_Foreign_Convention (Func_Typ)
-           and then Is_Tagged_Type (Func_Typ)
-           and then Is_Limited_Type (Func_Typ)
-           and then not Has_Aspect (Func_Typ, Aspect_No_Task_Parts);
+           and then Is_Inherently_Limited_Type (Original)
+           and then not Has_Foreign_Convention (Root)
+           and then Is_Tagged_Type (Root)
+           and then Is_Empty_Elmt_List
+             (Collect_Ancestors_With_No_Task_Parts (Original));
       end Might_Need_BIP_Task_Actuals;
 
       -------------------------------------
@@ -8792,10 +8795,12 @@ package body Sem_Ch6 is
                --  we have no direct way to climb to the corresponding parent
                --  subprogram but this internal entity has the extra formals
                --  (if any) required for the purpose of checking the extra
-               --  formals of Subp_Id.
+               --  formals of Subp_Id because its extra formals are shared
+               --  with its parent subprogram (see Sem_Ch3.Derive_Subprogram).
 
                else
                   pragma Assert (not Comes_From_Source (Ovr_E));
+                  Freeze_Extra_Formals (Ovr_E);
                end if;
 
             --  Use as our reference entity the ultimate renaming of the
@@ -8818,10 +8823,14 @@ package body Sem_Ch6 is
 
       --  Local variables
 
-      Formal_Type      : Entity_Id;
-      May_Have_Alias   : Boolean;
+      use Deferred_Extra_Formals_Support;
+
+      Can_Be_Deferred  : constant Boolean :=
+                           not Is_Unsupported_Extra_Formals_Entity (E);
       Alias_Formal     : Entity_Id := Empty;
       Alias_Subp       : Entity_Id := Empty;
+      Formal_Type      : Entity_Id;
+      May_Have_Alias   : Boolean;
       Parent_Formal    : Entity_Id := Empty;
       Parent_Subp      : Entity_Id := Empty;
       Ref_E            : Entity_Id;
@@ -8832,10 +8841,18 @@ package body Sem_Ch6 is
       pragma Assert (Is_Subprogram_Or_Entry (E)
         or else Ekind (E) in E_Subprogram_Type);
 
+      --  No action needed if extra formals were already handled. This
+      --  situation may arise because of a previous call to create the
+      --  extra formals, and also for subprogram types created as part
+      --  of dispatching calls (see Expand_Dispatching_Call).
+
+      if Extra_Formals_Known (E) then
+         return;
+
       --  We never generate extra formals if expansion is not active because we
       --  don't need them unless we are generating code.
 
-      if not Expander_Active then
+      elsif not Expander_Active then
          return;
 
       --  Enumeration literals have no extra formal; this case occurs when
@@ -8844,25 +8861,38 @@ package body Sem_Ch6 is
       elsif Ekind (E) = E_Function
         and then Ekind (Ultimate_Alias (E)) = E_Enumeration_Literal
       then
+         Freeze_Extra_Formals (E);
          return;
 
-      --  Extra formals of Initialization procedures are added by the function
-      --  Exp_Ch3.Init_Formals
+      --  Extra formals of init procs are added by Exp_Ch3.Init_Formals and
+      --  Set_CPP_Constructors when they are built, but we must handle here
+      --  aliased init procs.
 
       elsif Is_Init_Proc (E) then
+         pragma Assert (Present (Alias (E)));
+         pragma Assert (Extra_Formals_Known (Ultimate_Alias (E)));
+         Freeze_Extra_Formals (E);
          return;
 
       --  No need to generate extra formals in thunks whose target has no extra
       --  formals, but we can have two of them chained (interface and stack).
 
-      elsif Is_Thunk (E) and then No (Extra_Formals (Thunk_Target (E))) then
+      elsif Is_Thunk (E)
+        and then Extra_Formals_Known (Thunk_Target (E))
+        and then No (Extra_Formals (Thunk_Target (E)))
+      then
+         Freeze_Extra_Formals (E);
          return;
 
-      --  If Extra_Formals were already created, don't do it again. This
-      --  situation may arise for subprogram types created as part of
-      --  dispatching calls (see Expand_Dispatching_Call).
+      --  Handle alias of unchecked union equality with frozen extra formals
 
-      elsif Has_Extra_Formals (E) then
+      elsif Is_Overloadable (E)
+        and then Present (Alias (E))
+        and then Extra_Formals_Known (Ultimate_Alias (E))
+        and then Is_Unchecked_Union_Equality (Ultimate_Alias (E))
+      then
+         Set_Extra_Formals (E, Extra_Formals (Ultimate_Alias (E)));
+         Freeze_Extra_Formals (E);
          return;
 
       --  Extra formals of renamings of generic actual subprograms and
@@ -8880,6 +8910,8 @@ package body Sem_Ch6 is
            = Is_Generic_Instance (Ultimate_Alias (E)));
 
          Create_Extra_Formals (Ultimate_Alias (E));
+         pragma Assert (not Expander_Active
+           or else Extra_Formals_Known (Ultimate_Alias (E)));
 
          --  Share the extra formals
 
@@ -8891,17 +8923,72 @@ package body Sem_Ch6 is
          end if;
 
          pragma Assert (Extra_Formals_OK (E));
+         Freeze_Extra_Formals (E);
          return;
       end if;
 
-      --  Locate the last formal; required by Add_Extra_Formal.
+      --  Check if the addition of the extra formals must be deferred
 
       Formal := First_Formal (E);
       while Present (Formal) loop
-         Last_Extra := Formal;
+         if No (Underlying_Type (Etype (Formal)))
+           and then Can_Be_Deferred
+         then
+            Register_Deferred_Extra_Formals_Entity (E);
+            return;
+         end if;
+
          Next_Formal (Formal);
       end loop;
 
+      if Ekind (E) in E_Function
+                    | E_Subprogram_Type
+        and then No (Underlying_Type (Etype (E)))
+        and then Can_Be_Deferred
+      then
+         Register_Deferred_Extra_Formals_Entity (E);
+         return;
+      end if;
+
+      --  Here we start adding the extra formals
+
+      --  We we know that either the underlying type of all the formals and
+      --  returned results of E are known, or this is an special case where
+      --  some underlying type is still not available.
+
+      --  In the former case, we can already mark functions that return their
+      --  result by reference; in the latter case, we can mark them only if the
+      --  underlying return type is available (and it will be marked later).
+
+      if not Is_Unsupported_Extra_Formals_Entity (E)
+        or else (Ekind (E) in E_Function | E_Subprogram_Type
+                   and then Present (Underlying_Type (Etype (E))))
+      then
+         Compute_Returns_By_Ref (E);
+      end if;
+
+      --  Locate the last formal (required by Add_Extra_Formal)
+
+      if Present (First_Formal (E))
+        and then Is_Unchecked_Union (Etype (First_Formal (E)))
+        and then Present (Extra_Formals (E))
+        and then Has_Suffix (Extra_Formals (E), 'A')
+      then
+         --  An unchecked union equality has two extra formals per discriminant
+
+         First_Extra := Extra_Formals (E);
+         Last_Extra  := First_Extra;
+         while Present (Last_Extra) loop
+            pragma Assert (Has_Suffix (Last_Extra, 'A'));
+            Last_Extra := Extra_Formal (Last_Extra);
+
+            pragma Assert (Has_Suffix (Last_Extra, 'B'));
+            Last_Extra := Extra_Formal (Last_Extra);
+         end loop;
+      else
+         Last_Extra := Last_Formal (E);
+      end if;
+
       --  We rely on three entities to ensure consistency of extra formals of
       --  entity E:
       --
@@ -8961,6 +9048,7 @@ package body Sem_Ch6 is
         or else (Present (Alias_Subp)
                    and then Has_Foreign_Convention (Alias_Subp))
       then
+         Freeze_Extra_Formals (E);
          return;
       end if;
 
@@ -9039,14 +9127,44 @@ package body Sem_Ch6 is
          --  Here we establish our priority for deciding on the extra
          --  formals: 1) Parent primitive 2) Aliased primitive 3) Identity
 
-         if Present (Parent_Formal) then
-            Formal_Type := Etype (Parent_Formal);
+         --  Common case: the underlying type of all the formals is known
+         --  to be available.
+
+         if Can_Be_Deferred then
+            if Present (Parent_Formal) then
+               Formal_Type := Underlying_Type (Etype (Parent_Formal));
+            elsif Present (Alias_Formal) then
+               Formal_Type := Underlying_Type (Etype (Alias_Formal));
+            else
+               Formal_Type := Underlying_Type (Etype (Formal));
+            end if;
+
+            pragma Assert (Present (Formal_Type));
 
-         elsif Present (Alias_Formal) then
-            Formal_Type := Etype (Alias_Formal);
+         --  Special case: The underlying type of some formal is not available.
+         --  We use the underlying type when present. More work needed here???
 
          else
-            Formal_Type := Etype (Formal);
+            if Present (Parent_Formal) then
+               Formal_Type := Etype (Parent_Formal);
+
+               if Present (Underlying_Type (Formal_Type)) then
+                  Formal_Type := Underlying_Type (Formal_Type);
+               end if;
+
+            elsif Present (Alias_Formal) then
+               Formal_Type := Etype (Alias_Formal);
+
+               if Present (Underlying_Type (Formal_Type)) then
+                  Formal_Type := Underlying_Type (Formal_Type);
+               end if;
+            else
+               Formal_Type := Etype (Formal);
+
+               if Present (Underlying_Type (Formal_Type)) then
+                  Formal_Type := Underlying_Type (Formal_Type);
+               end if;
+            end if;
          end if;
 
          --  Create extra formal for supporting the attribute 'Constrained.
@@ -9093,12 +9211,13 @@ package body Sem_Ch6 is
               and then (Is_Definite_Subtype (Formal_Type)
                          or else Is_Mutably_Tagged_Type (Formal_Type))
               and then (Ada_Version < Ada_2012
-                         or else No (Underlying_Type (Formal_Type))
+                         or else
+                           (not Can_Be_Deferred
+                             and then No (Underlying_Type (Formal_Type)))
                          or else not
                            (Is_Limited_Type (Formal_Type)
                              and then
-                               Is_Tagged_Type
-                                 (Underlying_Type (Formal_Type))))
+                               Is_Tagged_Type (Formal_Type)))
             then
                Set_Extra_Constrained
                  (Formal, Add_Extra_Formal (Formal, Standard_Boolean, E, "O"));
@@ -9337,6 +9456,8 @@ package body Sem_Ch6 is
          Set_Extra_Formals (Alias (E), Extra_Formals (E));
       end if;
 
+      Freeze_Extra_Formals (E);
+
       pragma Assert (No (Alias_Subp)
         or else Extra_Formals_Match_OK (E, Alias_Subp));
 
@@ -9651,6 +9772,19 @@ package body Sem_Ch6 is
                   return False;
                end if;
 
+            --  Extra formals (A and B) of Unchecked_Unions (see Build_Variant_
+            --  Record_Equality)
+
+            elsif Has_Suffix (Formal_1, 'A') then
+               if not Has_Suffix (Formal_2, 'A') then
+                  return False;
+               end if;
+
+            elsif Has_Suffix (Formal_1, 'B') then
+               if not Has_Suffix (Formal_2, 'B') then
+                  return False;
+               end if;
+
             elsif BIP_Suffix_Kind (Formal_1) /= BIP_Suffix_Kind (Formal_2) then
                return False;
             end if;
@@ -10003,6 +10137,16 @@ package body Sem_Ch6 is
       return Empty;
    end Find_Corresponding_Spec;
 
+   --------------------------
+   -- Freeze_Extra_Formals --
+   --------------------------
+
+   procedure Freeze_Extra_Formals (E : Entity_Id) is
+   begin
+      pragma Assert (not Extra_Formals_Known (E));
+      Set_Extra_Formals_Known (E);
+   end Freeze_Extra_Formals;
+
    ----------------------
    -- Fully_Conformant --
    ----------------------
@@ -10622,6 +10766,10 @@ package body Sem_Ch6 is
       Formal : Entity_Id := First_Formal_With_Extras (E);
 
    begin
+      --  It makes no sense to perform this check if the extra formals
+      --  have not been added.
+      pragma Assert (Extra_Formals_Known (E));
+
       while Present (Formal) loop
          if Is_Build_In_Place_Entity (Formal) then
             return True;
@@ -12133,36 +12281,51 @@ package body Sem_Ch6 is
         and then Present (Find_Dispatching_Type (Alias (S)))
         and then Is_Interface (Find_Dispatching_Type (Alias (S)))
       then
-         --  For private types, when the full-view is processed we propagate to
-         --  the full view the non-overridden entities whose attribute "alias"
-         --  references an interface primitive. These entities were added by
-         --  Derive_Subprograms to ensure that interface primitives are
-         --  covered.
-
-         --  Inside_Freeze_Actions is non zero when S corresponds with an
-         --  internal entity that links an interface primitive with its
-         --  covering primitive through attribute Interface_Alias (see
-         --  Add_Internal_Interface_Entities).
-
-         if Inside_Freezing_Actions = 0
-           and then Is_Package_Or_Generic_Package (Current_Scope)
-           and then In_Private_Part (Current_Scope)
-           and then Parent_Kind (E) = N_Private_Extension_Declaration
-           and then Nkind (Parent (S)) = N_Full_Type_Declaration
-           and then Full_View (Defining_Identifier (Parent (E)))
-                      = Defining_Identifier (Parent (S))
-           and then Alias (E) = Alias (S)
-         then
-            Check_Operation_From_Private_View (S, E);
-            Set_Is_Dispatching_Operation (S);
+         declare
+            Private_Operation_Exported_By_Visible_Part : constant Boolean :=
+              Is_Package_Or_Generic_Package (Current_Scope)
+              and then In_Private_Part (Current_Scope)
+              and then Parent_Kind (E) = N_Private_Extension_Declaration
+              and then Nkind (Parent (S)) = N_Full_Type_Declaration
+              and then Full_View (Defining_Identifier (Parent (E)))
+                         = Defining_Identifier (Parent (S));
+
+         begin
+            --  For private types, when the full view is processed we propagate
+            --  to the full view the nonoverridden entities whose attribute
+            --  "alias" references an interface primitive. These entities were
+            --  added by Derive_Subprograms to ensure that interface primitives
+            --  are covered.
+
+            --  Inside_Freeze_Actions is nonzero when S corresponds to an
+            --  internal entity that links an interface primitive with its
+            --  covering primitive through attribute Interface_Alias (see
+            --  Add_Internal_Interface_Entities).
+
+            if Inside_Freezing_Actions = 0
+              and then Private_Operation_Exported_By_Visible_Part
+              and then Alias (E) = Alias (S)
+            then
+               Check_Operation_From_Private_View (S, E);
+               Set_Is_Dispatching_Operation (S);
 
-         --  Common case
+            --  Common case
 
-         else
-            Enter_Overloaded_Entity (S);
-            Check_Dispatching_Operation (S, Empty);
-            Check_For_Primitive_Subprogram (Is_Primitive_Subp);
-         end if;
+            else
+               Enter_Overloaded_Entity (S);
+               Check_Dispatching_Operation (S, Empty);
+               Check_For_Primitive_Subprogram (Is_Primitive_Subp);
+            end if;
+
+            if Private_Operation_Exported_By_Visible_Part
+              and then Type_Conformant (E, S)
+            then
+               --  Record the actual inherited subprogram that's being
+               --  overridden.
+
+               Set_Overridden_Inherited_Operation (S, E);
+            end if;
+         end;
 
          return;
       end if;
@@ -12601,6 +12764,26 @@ package body Sem_Ch6 is
                           and then not Is_Dispatch_Table_Wrapper (S)))
                   then
                      Set_Overridden_Operation    (S, Alias (E));
+
+                     --  Record the actual inherited subprogram that's being
+                     --  overridden. In the case where a subprogram declared
+                     --  in a private part overrides an inherited subprogram
+                     --  that itself is also declared in the private part,
+                     --  and that subprogram in turns overrides a subprogram
+                     --  declared in a package visible part (inherited via
+                     --  a private extension), we record the visible subprogram
+                     --  as the overridden one, so that we can determine
+                     --  visibility properly for prefixed calls to the
+                     --  subprogram made from outside the package. (See
+                     --  Try_Primitive_Operation in Sem_Ch4.)
+
+                     if Present (Overridden_Inherited_Operation (E)) then
+                        Set_Overridden_Inherited_Operation
+                          (S, Overridden_Inherited_Operation (E));
+                     else
+                        Set_Overridden_Inherited_Operation (S, E);
+                     end if;
+
                      Inherit_Subprogram_Contract (S, Alias (E));
 
                      Set_Is_Ada_2022_Only (S, Is_Ada_2022_Only (Alias (E)));
@@ -12760,6 +12943,530 @@ package body Sem_Ch6 is
          end if;
    end New_Overloaded_Entity;
 
+   ------------------------------------
+   -- Deferred_Extra_Formals_Support --
+   ------------------------------------
+
+   package body Deferred_Extra_Formals_Support is
+      Calls_List       : Elist_Id := No_Elist;
+      Calls_Scope_List : Elist_Id := No_Elist;
+      --  Calls to subprograms or entries with some unknown underlying type
+      --  in their parameters or result type, and the scope where each call
+      --  is performed.
+
+      Entities_List    : Elist_Id := No_Elist;
+      --  Subprograms, entries, and subprogram types with some unknown
+      --  underlying type in their formals or result type.
+
+      Types_List       : Elist_Id := No_Elist;
+      --  Types with no underlying type
+
+      function Underlying_Types_Available (E : Entity_Id) return Boolean;
+      --  Determines if the underlying type of all the formals and result
+      --  type of the given subprogram, subprogram type, or entry are
+      --  available.
+
+      -------------------------------
+      -- Add_Deferred_Extra_Params --
+      -------------------------------
+
+      procedure Add_Deferred_Extra_Params (Typ : Entity_Id) is
+
+         procedure Check_Registered_Calls;
+         --  Check all the registered calls; for each registered call that
+         --  has the underlying type of all the parameters and result types
+         --  of the called entity available, call Create_Extra_Actuals, and
+         --  unregister the call.
+
+         procedure Check_Registered_Entities;
+         --  Check all the registered entities (subprograms, entries and
+         --  subprogram types); for each registered entity E that has all
+         --  its underlying types available, call Create_Extra_Formals,
+         --  and unregister E.
+
+         ----------------------------
+         -- Check_Registered_Calls --
+         ----------------------------
+
+         procedure Check_Registered_Calls is
+
+            function Get_Relocated_Function_Call (N : Node_Id) return Node_Id;
+            --  Given a node N that references a function call that has been
+            --  relocated to remove possible side effects of the call (see
+            --  Remove_Side_Effects) or to wrap the call in a transient scope
+            --  (see Wrap_Transient_Expression), search and return the function
+            --  call. Notice that this function does not use the Original_Node
+            --  field of N; it searchs for the actual call associated with N
+            --  in the expanded code (since we need to add to such call its
+            --  missing extra actuals).
+
+            ---------------------------------
+            -- Get_Relocated_Function_Call --
+            ---------------------------------
+
+            function Get_Relocated_Function_Call (N : Node_Id) return Node_Id
+            is
+               Current_Node : Node_Id;
+               Decl         : Node_Id;
+               Id           : Entity_Id;
+
+            begin
+               Current_Node := N;
+
+               while Nkind (Current_Node) /= N_Function_Call loop
+                  case Nkind (Current_Node) is
+                     when N_Identifier =>
+                        Id   := Entity (Current_Node);
+                        Decl := Parent (Id);
+
+                        if Nkind (Decl) = N_Object_Renaming_Declaration then
+                           Current_Node := Name (Decl);
+
+                        else
+                           pragma Assert (Nkind (Decl) = N_Object_Declaration);
+
+                           if Present (Expression (Decl)) then
+                              Current_Node := Expression (Decl);
+
+                           elsif Present (BIP_Initialization_Call (Id)) then
+                              Decl := BIP_Initialization_Call (Id);
+                              pragma Assert (Present (Expression (Decl)));
+                              Current_Node := Expression (Decl);
+
+                           elsif Present (Related_Expression (Id)) then
+                              Current_Node := Related_Expression (Id);
+
+                           else
+                              pragma Assert (False);
+                              raise Program_Error;
+                           end if;
+                        end if;
+
+                     when N_Explicit_Dereference | N_Reference =>
+                        Current_Node := Prefix (Current_Node);
+
+                     when others =>
+                        pragma Assert (False);
+                        raise Program_Error;
+                  end case;
+               end loop;
+
+               return Current_Node;
+            end Get_Relocated_Function_Call;
+
+            --  Local variables
+
+            Call_Node        : Node_Id;
+            Call_Id          : Entity_Id;
+            Elmt_Call        : Elmt_Id;
+            Elmt_Scope       : Elmt_Id;
+            Remove_Call      : Boolean;
+            Scop_Id          : Entity_Id;
+
+         --  Start of processing for Check_Registered_Calls
+
+         begin
+            --  Perform a single traversal of both lists simultaneously,
+            --  since they have the same number of elements with a 1-to-1
+            --  relationship.
+
+            Elmt_Scope := First_Elmt (Calls_Scope_List);
+            Elmt_Call  := First_Elmt (Calls_List);
+
+            while Present (Elmt_Scope) loop
+               Scop_Id     := Node (Elmt_Scope);
+               Remove_Call := False;
+
+               --  Check the enclosing scope of the call: if the underlying
+               --  type of some formal or return type of the enclosing scope
+               --  of this call is not available then we must skip processing
+               --  this call.
+
+               if Underlying_Types_Available (Scop_Id) then
+                  Call_Node := Node (Elmt_Call);
+
+                  if Nkind (Call_Node) in N_Entry_Call_Statement
+                                        | N_Function_Call
+                                        | N_Procedure_Call_Statement
+                  then
+                     Call_Id := Get_Called_Entity (Call_Node);
+
+                  --  Handle expanded function calls that could have side
+                  --  effects.
+
+                  else
+                     pragma Assert
+                       (Nkind (Original_Node (Call_Node)) = N_Function_Call);
+
+                     Call_Node := Get_Relocated_Function_Call (Call_Node);
+                     Call_Id := Get_Called_Entity (Call_Node);
+                  end if;
+
+                  --  If the underlying types of all the formal and return
+                  --  types of this called entity are available then create
+                  --  its extra actuals and remove it from the list of
+                  --  registered calls.
+
+                  if Underlying_Types_Available (Call_Id) then
+
+                     --  Given that the call is placed in the body of an
+                     --  internally built subprogram, ensure that the extra
+                     --  formals of the enclosing scope are available before
+                     --  adding the extra actuals of this call.
+
+                     Create_Extra_Formals (Scop_Id);
+                     Create_Extra_Formals (Call_Id);
+
+                     pragma Assert (Extra_Formals_Known (Scop_Id));
+                     pragma Assert (Extra_Formals_Known (Call_Id));
+
+                     --  Mark functions that return a result by reference
+
+                     Compute_Returns_By_Ref (Scop_Id);
+                     Compute_Returns_By_Ref (Call_Id);
+
+                     Push_Scope (Scop_Id);
+                     Create_Extra_Actuals (Call_Node);
+                     Pop_Scope;
+
+                     Remove_Call := True;
+                  end if;
+               end if;
+
+               --  In order to safely remove these elements from their
+               --  containing lists, remember these elements before moving
+               --  to the next list elements.
+
+               if Remove_Call then
+                  declare
+                     Removed_Call  : constant Elmt_Id := Elmt_Call;
+                     Removed_Scope : constant Elmt_Id := Elmt_Scope;
+
+                  begin
+                     Next_Elmt (Elmt_Scope);
+                     Next_Elmt (Elmt_Call);
+
+                     Remove_Elmt (Calls_List, Removed_Call);
+                     Remove_Elmt (Calls_Scope_List, Removed_Scope);
+                  end;
+               else
+                  Next_Elmt (Elmt_Scope);
+                  Next_Elmt (Elmt_Call);
+               end if;
+
+            end loop;
+         end Check_Registered_Calls;
+
+         -------------------------------
+         -- Check_Registered_Entities --
+         -------------------------------
+
+         procedure Check_Registered_Entities is
+            Elmt       : Elmt_Id;
+            Found_Elmt : Elmt_Id;
+            Id         : Entity_Id;
+
+         begin
+            Elmt := First_Elmt (Entities_List);
+
+            while Present (Elmt) loop
+               Id := Node (Elmt);
+
+               --  If the underlying type of some formal or return type of this
+               --  entity is not available then skip this element.
+
+               if not Underlying_Types_Available (Id) then
+                  Next_Elmt (Elmt);
+
+               --  Otherwise, create its extra formals and remove it from the
+               --  list of entities that require adding the extra formals.
+
+               else
+                  --  In order to safely remove this element from the list,
+                  --  temporarily remember this element, and move to the next
+                  --  element.
+
+                  Found_Elmt := Elmt;
+                  Next_Elmt (Elmt);
+
+                  --  Create the extra formals, and mark functions that return
+                  --  by reference (not be done before if the underying return
+                  --  type was previously unknown).
+
+                  Create_Extra_Formals (Id);
+                  Compute_Returns_By_Ref (Id);
+
+                  Remove_Elmt (Entities_List, Found_Elmt);
+
+                  --  For deferred entries and entry families, the expansion of
+                  --  their entry declaration was deferred, and must be done
+                  --  now (after adding their extra formals).
+
+                  if Ekind (Id) in E_Entry | E_Entry_Family then
+                     Expand_N_Entry_Declaration (Parent (Id),
+                       Was_Deferred => True);
+                  end if;
+               end if;
+            end loop;
+         end Check_Registered_Entities;
+
+      --  Start of processing for Add_Deferred_Extra_Params
+
+      begin
+         pragma Assert (Present (Underlying_Type (Typ)));
+
+         if Present (Entities_List) then
+            Check_Registered_Entities;
+         end if;
+
+         if Present (Calls_List) then
+            Check_Registered_Calls;
+         end if;
+
+         Remove (Types_List, Typ);
+      end Add_Deferred_Extra_Params;
+
+      --------------------------------
+      -- Has_Deferred_Extra_Formals --
+      --------------------------------
+
+      function Has_Deferred_Extra_Formals (Typ : Entity_Id) return Boolean is
+      begin
+         return Contains (Types_List, Typ);
+      end Has_Deferred_Extra_Formals;
+
+      --------------------------------------
+      -- Is_Deferred_Extra_Formals_Entity --
+      --------------------------------------
+
+      function Is_Deferred_Extra_Formals_Entity
+        (Id : Entity_Id) return Boolean is
+      begin
+         return Contains (Entities_List, Id);
+      end Is_Deferred_Extra_Formals_Entity;
+
+      ---------------------------------------
+      -- Is_Unsupported_Extra_Actuals_Call --
+      ---------------------------------------
+
+      --  Similarly to Is_Unsupported_Extra_Formals_Entity, we cannot
+      --  determine if the extra formals are needed when the underlying
+      --  type of some formal or result type is not available, and we are
+      --  compiling the body of a subprogram or package. However, for calls
+      --  we must also handle internal calls generated by the compiler as
+      --  part of compiling a package spec. For example, internal calls
+      --  performed in thunks of secondary dispatch table entries.
+      --
+      --  Example
+      --  -------
+      --  package P is
+      --     type T is tagged null record;
+      --  end;
+      --
+      --  limited with P;
+      --  package Q is
+      --     type Iface is interface;
+      --     procedure Prim (Self : Iface; Current : P.T) is abstract;
+      --  end;
+      --
+      --  limited with P;
+      --  with Q;
+      --  package R is
+      --     type Root is tagged null record;
+      --     type DT is new Root and Q.Iface with null record;
+      --
+      --     procedure Prim (Self : DT; Current : P.T);
+      --  end;
+      --
+      --  The initialization of the secondary dispatch table of tagged type
+      --  DT has an internally generated thunk that displaces the pointer to
+      --  the object and calls the primitive Prim (and the underlying type
+      --  of type T is not available).
+
+      function Is_Unsupported_Extra_Actuals_Call
+        (Call_Node : Node_Id; Id : Entity_Id) return Boolean
+      is
+         Comp_Unit : constant Entity_Id :=
+                       Cunit_Entity (Get_Source_Unit (Call_Node));
+      begin
+         return not Underlying_Types_Available (Id)
+           and then Is_Compilation_Unit (Comp_Unit)
+           and then Ekind (Comp_Unit) in E_Package
+                                       | E_Package_Body
+                                       | E_Subprogram_Body;
+      end Is_Unsupported_Extra_Actuals_Call;
+
+      -----------------------------------------
+      -- Is_Unsupported_Extra_Formals_Entity --
+      -----------------------------------------
+
+      --  We cannot determine if the extra formals are needed when the
+      --  underlying type of some formal or result type is not available,
+      --  and we are compiling the body of a subprogram or package. The
+      --  scenery for this case is a package spec that has a limited_with_
+      --  clause on unit Q, and its body has no regular with-clause on Q
+      --  (AI05-0151-1/08).
+
+      function Is_Unsupported_Extra_Formals_Entity
+        (Id : Entity_Id) return Boolean
+      is
+         Comp_Unit : constant Entity_Id :=
+                       Cunit_Entity (Get_Source_Unit (Id));
+      begin
+         return not Underlying_Types_Available (Id)
+           and then Is_Compilation_Unit (Comp_Unit)
+           and then Ekind (Comp_Unit) in E_Package_Body
+                                       | E_Subprogram_Body;
+      end Is_Unsupported_Extra_Formals_Entity;
+
+      --------------------------------------------
+      -- Register_Deferred_Extra_Formals_Entity --
+      --------------------------------------------
+
+      procedure Register_Deferred_Extra_Formals_Entity (Id : Entity_Id) is
+
+         procedure Register_Type (Typ : Entity_Id);
+         --  Register the given type in Types_List; for types visible though
+         --  limited_with_clauses, register their non-limited view.
+
+         -------------------
+         -- Register_Type --
+         -------------------
+
+         procedure Register_Type (Typ : Entity_Id) is
+         begin
+            --  Handle entities visible through limited_with_clauses
+
+            if Has_Non_Limited_View (Typ) then
+               Append_Unique_Elmt (Non_Limited_View (Typ), Types_List);
+            else
+               Append_Unique_Elmt (Typ, Types_List);
+            end if;
+         end Register_Type;
+
+         --  Local variables
+
+         Formal : Entity_Id;
+
+      --  Start of processing for Register_Deferred_Extra_Formals_Entity
+
+      begin
+         pragma Assert (Is_Subprogram_Or_Entry (Id)
+           or else Ekind (Id) in E_Subprogram_Type);
+
+         if not Is_Deferred_Extra_Formals_Entity (Id) then
+            if No (Types_List) then
+               Types_List := New_Elmt_List;
+            end if;
+
+            if No (Entities_List) then
+               Entities_List := New_Elmt_List;
+            end if;
+
+            --  Register all the types of the subprogram profile that are not
+            --  fully known.
+
+            Formal := First_Formal (Id);
+            while Present (Formal) loop
+
+               if No (Underlying_Type (Etype (Formal))) then
+                  Register_Type (Etype (Formal));
+               end if;
+
+               Next_Formal (Formal);
+            end loop;
+
+            if Ekind (Id) in E_Function | E_Subprogram_Type
+              and then No (Underlying_Type (Etype (Id)))
+            then
+               Register_Type (Etype (Id));
+            end if;
+
+            --  Register this subprogram
+
+            Append_Elmt (Id, Entities_List);
+         end if;
+      end Register_Deferred_Extra_Formals_Entity;
+
+      ------------------------------------------
+      -- Register_Deferred_Extra_Formals_Call --
+      ------------------------------------------
+
+      procedure Register_Deferred_Extra_Formals_Call
+        (Call_Node : Node_Id;
+         Scope_Id  : Entity_Id) is
+      begin
+         pragma Assert (Nkind (Call_Node) in N_Subprogram_Call
+                                           | N_Entry_Call_Statement);
+         if No (Calls_List) then
+            Calls_List := New_Elmt_List;
+            Calls_Scope_List := New_Elmt_List;
+         end if;
+
+         --  Avoid registering any call twice; this may occur in dispatching
+         --  calls with deferred extra actuals because Expand_Call_Helper
+         --  registers the call and invokes Expand_Dispatching_Call (which
+         --  tries again to register the expanded call).
+
+         if not Contains (Calls_List, Call_Node) then
+            Append_Elmt (Call_Node, Calls_List);
+            Append_Elmt (Scope_Id, Calls_Scope_List);
+         end if;
+      end Register_Deferred_Extra_Formals_Call;
+
+      --------------------------------
+      -- Underlying_Types_Available --
+      --------------------------------
+
+      function Underlying_Types_Available (E : Entity_Id) return Boolean is
+         Formal     : Entity_Id;
+         Formal_Typ : Entity_Id;
+         Func_Typ   : Entity_Id;
+
+      begin
+         --  If the extra formals are available, then the nonlimited view
+         --  of all the types referenced in the profile are available.
+
+         if Extra_Formals_Known (E) then
+            return True;
+         end if;
+
+         --  Check the return type
+
+         if Ekind (E) in E_Function | E_Subprogram_Type then
+            Func_Typ := Etype (E);
+
+            if Has_Non_Limited_View (Func_Typ) then
+               Func_Typ := Non_Limited_View (Func_Typ);
+            end if;
+
+            if No (Underlying_Type (Func_Typ)) then
+               return False;
+            end if;
+         end if;
+
+         --  Check the type of the formals
+
+         Formal := First_Formal (E);
+         while Present (Formal) loop
+            Formal_Typ := Etype (Formal);
+
+            if Has_Non_Limited_View (Formal_Typ) then
+               Formal_Typ := Non_Limited_View (Formal_Typ);
+            end if;
+
+            if No (Underlying_Type (Formal_Typ)) then
+               return False;
+            end if;
+
+            Next_Formal (Formal);
+         end loop;
+
+         return True;
+      end Underlying_Types_Available;
+
+   end Deferred_Extra_Formals_Support;
+
    ---------------------
    -- Process_Formals --
    ---------------------
diff --git a/gcc/ada/sem_ch6.ads b/gcc/ada/sem_ch6.ads
index 7ebbcaa..4ef5b65 100644
--- a/gcc/ada/sem_ch6.ads
+++ b/gcc/ada/sem_ch6.ads
@@ -190,6 +190,14 @@ package Sem_Ch6 is
    --  Use the subprogram specification in the body to retrieve the previous
    --  subprogram declaration, if any.
 
+   procedure Freeze_Extra_Formals (E : Entity_Id);
+   --  Given a subprogram, subprogram type, or entry, flag E to indicate that
+   --  its extra formals (if any) are known (by setting Extra_Formals_Known).
+   --  This subprogram serves three purposes: (1) Document the places where
+   --  the extra formals are known, (2) Ensure that extra formals are added
+   --  only once, and (3) Provide a convenient place for setting a debugger
+   --  breakpoint to locate when extra formals are known.
+
    function Fully_Conformant (New_Id, Old_Id : Entity_Id) return Boolean;
    --  Determine whether two callable entities (subprograms, entries,
    --  literals) are fully conformant (RM 6.3.1(17))
@@ -299,4 +307,156 @@ package Sem_Ch6 is
    procedure Valid_Operator_Definition (Designator : Entity_Id);
    --  Verify that an operator definition has the proper number of formals
 
+   ------------------------------------
+   -- Deferred_Extra_Formals_Support --
+   ------------------------------------
+
+   --  This package provides support for deferring the addition of extra
+   --  formals to subprograms, entries, and subprogram types; it also provides
+   --  support for deferring the addition of extra actuals to direct calls to
+   --  subprograms and entries, and indirect calls through subprogram types.
+   --  The addition of the extra formals and actuals is deferred until the
+   --  underlying type of all the parameters and result types of registered
+   --  subprograms, entries, and subprogram types is known.
+
+   --  Functional Description
+   --  ----------------------
+   --
+   --  When Create_Extra_Formals identifies that the underlying type of
+   --  some parameter or result type of an entity E is not available, E is
+   --  registered by this package, and the addition of its extra formals is
+   --  deferred. As part of this registration, the types of all the params
+   --  and result types of E with no underlying type are also registered.
+   --
+   --  When Expand_Call_Helper identifies that the underlying type of some
+   --  parameter or result type of a called entity is not available, the call
+   --  is registered by Register_Deferred_Extra_Formals_Call, and the addition
+   --  of its extra actuals is deferred.
+   --
+   --  When the full type declaration of some registered type T is analyzed,
+   --  the subprogram Add_Deferred_Extra_Params is invoked; this subprogram
+   --  does the following actions:
+   --    1) Check all the registered entities (subprograms, entries, and
+   --       subprogram types); for each registered entity that has all its
+   --       underlying types available, call Create_Extra_Formals, and
+   --       unregister the entity.
+   --    2) Check all the registered calls; for each registered call that
+   --       has available the underlying type of all the parameters and result
+   --       types of the called entity, call Create_Extra_Actuals, and
+   --       unregister the call.
+   --    3) Unregister T.
+   --
+   --  Example 1
+   --  ---------
+   --  A package spec has a private type declaration T, and declarations of
+   --  expression functions and/or primitives with class-wide conditions
+   --  invoking primitives of type T before the full view of T is defined.
+   --
+   --  As part of processing the early freezing of the called subprograms
+   --  (and as part of processing the calls) the functions are registered as
+   --  subprograms with deferred extra formals, and the calls are registered
+   --  as calls with deferred extra actuals.
+   --
+   --  When the full type declaration of T is analyzed, extra formals are
+   --  added to all the registered subprograms, and extra actuals are added
+   --  to all the registered calls with deferred extra actuals.
+   --
+   --  Example 2
+   --  ---------
+   --  The specification of package P has a limited_with_clause on package Q,
+   --  and the type of the formals of subprograms defined in P are types
+   --  defined in Q.
+   --
+   --  When compiling the spec of P, similarly to the previous example,
+   --  subprograms with incomplete formals are registered as subprograms
+   --  with deferred extra formals; if the spec of P has calls to these
+   --  subprograms, then these calls are registered as calls with deferred
+   --  extra actuals. That is, when the analysis of package P completes,
+   --  deferred extra formals and actuals have not been added.
+   --
+   --  When another compilation unit is analyzed (including the body of
+   --  package P), and a regular with-clause on Q is processed, when the
+   --  full type declaration of deferred entities is analyzed, deferred
+   --  extra formals and deferred extra actuals are added.
+   --
+   --  This machinery relies on the GNAT Compilation Model; that is, when
+   --  we analyze the spec of P (for which we generally don't generate code),
+   --  it is safe to complete the compilation and still have entities with
+   --  deferred extra formals, and calls with deferred extra actuals.
+   --
+   --  The body package P generally has a regular with-clause on package Q.
+   --  Hence, when we compile the body of package P, the implicit dependence
+   --  on its package spec causes the analysis of the spec of P (thus
+   --  registering deferred entities), followed by the analysis of context
+   --  clauses in the body of P. When the regular with-clause on package Q
+   --  is analyzed, we add the extra formals and extra actuals to deferred
+   --  entities. Thus, the generated code will have all the needed formals.
+   --
+   --  The (still) unsupported case is when the body of package P does not
+   --  have a regular with-clause on package Q (AI05-0151-1/08). This case
+   --  is left documented in the front-end sources by means of calls to
+   --  the following subprograms: Is_Unsupported_Extra_Formals_Entity, and
+   --  Is_Unsupported_Extra_Actuals_Call.
+
+   package Deferred_Extra_Formals_Support is
+
+      procedure Add_Deferred_Extra_Params (Typ : Entity_Id);
+      --  Check all the registered subprograms, entries, and subprogram types
+      --  with deferred addition of their extra formals; if the underlying
+      --  types of all their formals is available then add their extra formals.
+      --  Check also all the registered calls with deferred addition of their
+      --  extra actuals; add their extra actuals if the underlying types of all
+      --  their parameters and result types are available. Finally unregister
+      --  Typ from the list of types used for the deferral of extra formals/
+      --  actuals.
+
+      procedure Register_Deferred_Extra_Formals_Entity (Id : Entity_Id);
+      --  Register the given subprogram, entry, or subprogram type to defer the
+      --  addition of its extra formals.
+
+      procedure Register_Deferred_Extra_Formals_Call
+        (Call_Node : Node_Id;
+         Scope_Id  : Entity_Id);
+      --  Register the given call, performed from the given scope, to defer the
+      --  addition of its extra actuals.
+
+      function Has_Deferred_Extra_Formals (Typ : Entity_Id) return Boolean;
+      --  Return True if there some registered subprogram, subprogram type, or
+      --  entry with deferred extra formals that has some formal type or
+      --  result type of type Typ (i.e. which depends on the given type to
+      --  add its extra formals).
+
+      function Is_Deferred_Extra_Formals_Entity
+        (Id : Entity_Id) return Boolean;
+      --  Return True if Id is a subprogram, subprogram type, or entry that has
+      --  been registered to defer the addition of its extra formals.
+
+      function Is_Unsupported_Extra_Formals_Entity
+        (Id : Entity_Id) return Boolean;
+      --  Id is a subprogram, subprogram type, or entry. Return True if Id is
+      --  unsupported for deferring the addition of its extra formals; that is,
+      --  it is defined in a compilation unit that is a package body or a
+      --  subprogram body, and the underlying type of some of its parameters
+      --  or result type is not available.
+      --
+      --  The context for this case is an unsupported case of AI05-0151-1/08
+      --  that allows incomplete tagged types as parameter and result types.
+      --  More concretely, a type T is visible in a package spec through a
+      --  limited_with_clause, and the body of the package has no regular
+      --  with_clause. In such a case, the machinery for deferring the
+      --  addition of extra formals does not work because the underlying
+      --  type of the type is not seen during the compilation of the
+      --  package body.
+      --
+      --  The purpose of this function is to facilitate locating in the sources
+      --  the places where the front end performs the current (incomplete)
+      --  management of such case (to facilitate further work) ???
+
+      function Is_Unsupported_Extra_Actuals_Call
+        (Call_Node : Node_Id; Id : Entity_Id) return Boolean;
+      --  Same as previous function but applicable to a call to the given
+      --  entity Id.
+
+   end Deferred_Extra_Formals_Support;
+
 end Sem_Ch6;
diff --git a/gcc/ada/sem_ch8.adb b/gcc/ada/sem_ch8.adb
index db892d0..e6ef658 100644
--- a/gcc/ada/sem_ch8.adb
+++ b/gcc/ada/sem_ch8.adb
@@ -4270,6 +4270,40 @@ package body Sem_Ch8 is
          Local_Restrict.Check_Actual_Subprogram_For_Instance
            (Actual_Subp_Name => Nam, Formal_Subp => Formal_Spec);
       end if;
+
+      --  If pragma Short_Circuit_And_Or is specified, then we give an error
+      --  for renaming an operator that is made short circuit.
+      --  For example, this is illegal:
+      --
+      --      function My_And (X, Y: Boolean) return Boolean renames "and";
+      --
+      --  if "and" denotes the usual predefined Boolean operator. Otherwise,
+      --  the semantics are confusing (sometimes short circuit, and sometimes
+      --  not, for calls to My_And). If we ever relax this rule, we will need
+      --  to clean up that run-time semantics.
+
+      if Short_Circuit_And_Or
+        and then Chars (Old_S) in Name_Op_And | Name_Op_Or
+        and then In_Extended_Main_Source_Unit (N)
+        and then Etype (Old_S) = Standard_Boolean
+        and then Is_Intrinsic_Subprogram (Old_S)
+      then
+         if Comes_From_Source (N) then
+            Error_Msg_N
+              ("pragma Short_Circuit_And_Or disallows renaming of " &
+               "operator", N);
+
+         --  Same error in case of an instantiation with My_And => "and"
+
+         elsif Present (Corresponding_Formal_Spec (N)) then
+            Error_Msg_N
+              ("pragma Short_Circuit_And_Or disallows passing of " &
+               "operator as a generic actual", N);
+
+         else
+            raise Program_Error;
+         end if;
+      end if;
    end Analyze_Subprogram_Renaming;
 
    -------------------------
@@ -5410,7 +5444,15 @@ package body Sem_Ch8 is
          elsif In_Open_Scopes (Scope (Base_Type (T))) then
             null;
 
-         elsif not Redundant_Use (Id) then
+         --  Turn off the use_type_clause on the type unless the clause is
+         --  redundant, or there's a previous use_type_clause. (The case where
+         --  a use_type_clause without "all" is followed by one with "all" in
+         --  a more nested scope is not considered redundant, necessitating
+         --  the test for a previous clause. One might expect the latter test
+         --  to suffice, but it turns out there are cases where Redundant_Use
+         --  is set, but Prev_Use_Clause is not set. ???)
+
+         elsif not Redundant_Use (Id) and then No (Prev_Use_Clause (N)) then
             Set_In_Use (T, False);
             Set_In_Use (Base_Type (T), False);
             Set_Current_Use_Clause (T, Empty);
diff --git a/gcc/ada/sem_ch9.adb b/gcc/ada/sem_ch9.adb
index e32612e..bf387d3 100644
--- a/gcc/ada/sem_ch9.adb
+++ b/gcc/ada/sem_ch9.adb
@@ -1700,6 +1700,12 @@ package body Sem_Ch9 is
          Process_Formals (Formals, N);
          Create_Extra_Formals (Def_Id);
          End_Scope;
+
+      --  If the entry has no formals, extra formals are definitely not
+      --  required.
+
+      else
+         Freeze_Extra_Formals (Def_Id);
       end if;
 
       if Ekind (Def_Id) = E_Entry then
diff --git a/gcc/ada/sem_disp.adb b/gcc/ada/sem_disp.adb
index 9d03eff..5a8bd58 100644
--- a/gcc/ada/sem_disp.adb
+++ b/gcc/ada/sem_disp.adb
@@ -2416,6 +2416,8 @@ package body Sem_Disp is
       Formal    : Entity_Id;
       Ctrl_Type : Entity_Id;
 
+   --  Start of processing for Find_Dispatching_Type
+
    begin
       if Ekind (Subp) in E_Function | E_Procedure
         and then Present (DTC_Entity (Subp))
@@ -3083,6 +3085,52 @@ package body Sem_Disp is
       then
          return Is_Tag_Indeterminate (Prefix (Orig_Node));
 
+      --  An if-expression is tag-indeterminate if all of the dependent
+      --  expressions are tag-indeterminate (RM 4.5.7 (17/3)).
+
+      elsif Nkind (Orig_Node) = N_If_Expression then
+         declare
+            Cond : constant Node_Id := First (Expressions (Orig_Node));
+            Expr : Node_Id := Next (Cond);
+
+         begin
+            if not Is_Tag_Indeterminate (Original_Node (Expr)) then
+               return False;
+            end if;
+
+            Next (Expr);
+
+            if Present (Expr)
+              and then not Is_Tag_Indeterminate (Original_Node (Expr))
+            then
+               return False;
+            end if;
+
+            return True;
+         end;
+
+      --  A case-expression is tag-indeterminate if all of the dependent
+      --  expressions are tag-indeterminate (RM 4.5.7 (17/3)).
+
+      elsif Nkind (Orig_Node) = N_Case_Expression then
+         declare
+            Alt  : Node_Id := First (Alternatives (Orig_Node));
+            Expr : Node_Id;
+
+         begin
+            while Present (Alt) loop
+               Expr := Expression (Alt);
+
+               if not Is_Tag_Indeterminate (Original_Node (Expr)) then
+                  return False;
+               end if;
+
+               Next (Alt);
+            end loop;
+
+            return True;
+         end;
+
       else
          return False;
       end if;
@@ -3243,6 +3291,7 @@ package body Sem_Disp is
       elsif Nkind (Actual) = N_Explicit_Dereference
         and then Nkind (Original_Node (Prefix (Actual))) = N_Function_Call
       then
+         pragma Assert (Is_Expanded_Dispatching_Call (Actual));
          return;
 
       --  When expansion is suppressed, an unexpanded call to 'Input can occur,
diff --git a/gcc/ada/sem_res.adb b/gcc/ada/sem_res.adb
index 96e8da6..29b7766 100644
--- a/gcc/ada/sem_res.adb
+++ b/gcc/ada/sem_res.adb
@@ -7270,7 +7270,9 @@ package body Sem_Res is
 
       if Restriction_Check_Required (No_Relative_Delay)
         and then Is_RTE (Nam, RE_Set_Handler)
-        and then Is_RTE (Etype (Next_Actual (First_Actual (N))), RE_Time_Span)
+        and then
+          Is_RTE
+            (Base_Type (Etype (Next_Actual (First_Actual (N)))), RE_Time_Span)
       then
          Check_Restriction (No_Relative_Delay, N);
       end if;
@@ -12463,16 +12465,6 @@ package body Sem_Res is
          Orig_N := Original_Node (Expression (Orig_N));
          Orig_T := Target_Typ;
 
-         --  If the node is part of a larger expression, the Target_Type
-         --  may not be the original type of the node if the context is a
-         --  condition. Recover original type to see if conversion is needed.
-
-         if Is_Boolean_Type (Orig_T)
-          and then Nkind (Parent (N)) in N_Op
-         then
-            Orig_T := Etype (Parent (N));
-         end if;
-
          --  If we have an entity name, then give the warning if the entity
          --  is the right type, or if it is a loop parameter covered by the
          --  original type (that's needed because loop parameters have an
@@ -12548,6 +12540,16 @@ package body Sem_Res is
             then
                null;
 
+            --  Do not warn if original source-level conversion was
+            --  between two different types.
+
+            elsif Nkind (Original_Node (N)) = N_Type_Conversion
+              and then
+                Base_Type (Etype (Subtype_Mark (Original_Node (N))))
+                  /= Base_Type (Etype (Expression (Original_Node (N))))
+            then
+               null;
+
             --  Here we give the redundant conversion warning. If it is an
             --  entity, give the name of the entity in the message. If not,
             --  just mention the expression.
diff --git a/gcc/ada/sem_util.adb b/gcc/ada/sem_util.adb
index 74de26a..b2b4fed 100644
--- a/gcc/ada/sem_util.adb
+++ b/gcc/ada/sem_util.adb
@@ -35,6 +35,7 @@ with Exp_Ch11;       use Exp_Ch11;
 with Exp_Util;       use Exp_Util;
 with Fname;          use Fname;
 with Freeze;         use Freeze;
+with Ghost;          use Ghost;
 with Itypes;         use Itypes;
 with Lib;            use Lib;
 with Lib.Xref;       use Lib.Xref;
@@ -1923,6 +1924,10 @@ package body Sem_Util is
    -- Build_Elaboration_Entity --
    ------------------------------
 
+   --  WARNING: This routine manages Ghost regions. Return statements must be
+   --  replaced by gotos which jump to the end of the routine and restore the
+   --  Ghost mode.
+
    procedure Build_Elaboration_Entity (N : Node_Id; Spec_Id : Entity_Id) is
       Loc      : constant Source_Ptr := Sloc (N);
       Decl     : Node_Id;
@@ -1956,6 +1961,12 @@ package body Sem_Util is
          end if;
       end Set_Package_Name;
 
+      --  Local variables
+
+      Saved_GM  : constant Ghost_Mode_Type := Ghost_Mode;
+      Saved_IGR : constant Node_Id         := Ignored_Ghost_Region;
+      --  Save the Ghost-related attributes to restore on exit
+
    --  Start of processing for Build_Elaboration_Entity
 
    begin
@@ -2003,6 +2014,11 @@ package body Sem_Util is
          return;
       end if;
 
+      --  Elaboration entity is never a ghost object, regardless of the context
+      --  in which this routine is called.
+
+      Install_Ghost_Region (None, N);
+
       --  Here we need the elaboration entity
 
       --  Construct name of elaboration entity as xxx_E, where xxx is the unit
@@ -2043,6 +2059,8 @@ package body Sem_Util is
 
       Set_Has_Qualified_Name       (Elab_Ent);
       Set_Has_Fully_Qualified_Name (Elab_Ent);
+
+      Restore_Ghost_Region (Saved_GM, Saved_IGR);
    end Build_Elaboration_Entity;
 
    --------------------------------
@@ -3688,6 +3706,7 @@ package body Sem_Util is
            Aspect_Aggregate,
            Aspect_Max_Entry_Queue_Length
            --  , Aspect_No_Controlled_Parts
+           --  , Aspect_No_Task_Parts
           );
 
       --  Note that none of these 8 aspects can be specified (for a type)
@@ -10043,16 +10062,19 @@ package body Sem_Util is
            and then not Has_Unknown_Discriminants (Utyp)
            and then not (Ekind (Utyp) = E_String_Literal_Subtype)
       then
-         --  Nothing to do if in spec expression (why not???)
+         --  If the type has no discriminants, there is no subtype to build,
+         --  even if the underlying type is discriminated.
 
-         if In_Spec_Expression then
+         if Is_Private_Type (Typ) and then not Has_Discriminants (Typ) then
             return Typ;
 
-         elsif Is_Private_Type (Typ) and then not Has_Discriminants (Typ) then
-
-            --  If the type has no discriminants, there is no subtype to
-            --  build, even if the underlying type is discriminated.
+         --  If we are performing preanalysis on a conjured-up copy of a name
+         --  (see calls to Preanalyze_Range in sem_ch5.adb) then we don't want
+         --  to freeze Atyp, now or ever. In this case, the tree we eventually
+         --  pass to the back end should contain no references to Atyp (and a
+         --  freeze node would contain such a reference).
 
+         elsif not (Expander_Active or GNATprove_Mode) then
             return Typ;
 
          --  Else build the actual subtype
@@ -10068,42 +10090,21 @@ package body Sem_Util is
 
             Atyp := Defining_Identifier (Decl);
 
-            --  If Build_Actual_Subtype generated a new declaration then use it
-
-            if Atyp /= Typ then
-
-               --  The actual subtype is an Itype, so analyze the declaration,
-               --  but do not attach it to the tree, to get the type defined.
-
-               Set_Parent (Decl, N);
-               Set_Is_Itype (Atyp);
-               Analyze (Decl, Suppress => All_Checks);
-               Set_Associated_Node_For_Itype (Atyp, N);
-               if Expander_Active then
-                  Set_Has_Delayed_Freeze (Atyp, False);
-
-                  --  We need to freeze the actual subtype immediately. This is
-                  --  needed because otherwise this Itype will not get frozen
-                  --  at all; it is always safe to freeze on creation because
-                  --  any associated types must be frozen at this point.
+            --  The actual subtype is an Itype, so analyze the declaration
+            --  after attaching it to the tree, to get the type defined.
 
-                  --  On the other hand, if we are performing preanalysis on
-                  --  a conjured-up copy of a name (see calls to
-                  --  Preanalyze_Range in sem_ch5.adb) then we don't want
-                  --  to freeze Atyp, now or ever. In this case, the tree
-                  --  we eventually pass to the back end should contain no
-                  --  references to Atyp (and a freeze node would contain
-                  --  such a reference). That's why Expander_Active is tested.
+            Set_Parent (Decl, N);
+            Set_Is_Itype (Atyp);
+            Analyze (Decl, Suppress => All_Checks);
+            Set_Associated_Node_For_Itype (Atyp, N);
 
-                  Freeze_Itype (Atyp, N);
-               end if;
-               return Atyp;
-
-            --  Otherwise we did not build a declaration, so return original
+            --  We need to freeze the actual subtype immediately. This is
+            --  needed because otherwise this Itype will not get frozen
+            --  at all; it is always safe to freeze on creation because
+            --  any associated types must be frozen at this point.
 
-            else
-               return Typ;
-            end if;
+            Freeze_Itype (Atyp, N);
+            return Atyp;
          end if;
 
       --  For all remaining cases, the actual subtype is the same as
@@ -15017,6 +15018,7 @@ package body Sem_Util is
                | Aspect_Iterator_Element
                | Aspect_Max_Entry_Queue_Length
                | Aspect_No_Controlled_Parts
+               | Aspect_No_Task_Parts
             =>
                return;
          end case;
@@ -16276,8 +16278,9 @@ package body Sem_Util is
                  Names_Match (Assign_Indexed_1, Assign_Indexed_2);
             end;
 
-         --  Checking for this aspect is performed elsewhere during freezing
-         when Aspect_No_Controlled_Parts =>
+         --  Checking for these aspects is performed elsewhere during freezing
+         when Aspect_No_Controlled_Parts
+            | Aspect_No_Task_Parts =>
             return True;
 
          --  scalar-valued aspects; compare (static) values.
@@ -21373,6 +21376,18 @@ package body Sem_Util is
       return False;
    end Is_Unchecked_Conversion_Instance;
 
+   ---------------------------------
+   -- Is_Unchecked_Union_Equality --
+   ---------------------------------
+
+   function Is_Unchecked_Union_Equality (Id : Entity_Id) return Boolean is
+   begin
+      return Ekind (Id) = E_Function
+        and then Present (First_Formal (Id))
+        and then Is_Unchecked_Union (Etype (First_Formal (Id)))
+        and then Id = TSS (Etype (First_Formal (Id)), TSS_Composite_Equality);
+   end Is_Unchecked_Union_Equality;
+
    -------------------------------
    -- Is_Universal_Numeric_Type --
    -------------------------------
@@ -26955,6 +26970,10 @@ package body Sem_Util is
       if Has_Relaxed_Finalization (From_Typ) then
          Set_Has_Relaxed_Finalization (Typ);
       end if;
+
+      if Deriv and then Has_Destructor (From_Typ) then
+         Set_Has_Destructor (Typ);
+      end if;
    end Propagate_Controlled_Flags;
 
    ------------------------------
diff --git a/gcc/ada/sem_util.ads b/gcc/ada/sem_util.ads
index efeafda..4554f24 100644
--- a/gcc/ada/sem_util.ads
+++ b/gcc/ada/sem_util.ads
@@ -1578,7 +1578,7 @@ package Sem_Util is
    --  underlying type).
 
    function Has_Suffix (E : Entity_Id; Suffix : Character) return Boolean;
-   --  Returns true if the last character of E is Suffix. Used in Assertions.
+   --  Returns true if the last character of E is Suffix.
 
    function Has_Tagged_Component (Typ : Entity_Id) return Boolean;
    --  Returns True if Typ is a composite type (array or record) that is either
@@ -2131,7 +2131,7 @@ package Sem_Util is
    --  object as per RM C.6(8).
 
    function Is_Inherited_Operation (E : Entity_Id) return Boolean;
-   --  E is a subprogram. Return True is E is an implicit operation inherited
+   --  E is a subprogram. Return True if E is an implicit operation inherited
    --  by a derived type declaration.
 
    function Is_Inlinable_Expression_Function (Subp : Entity_Id) return Boolean;
@@ -2196,7 +2196,7 @@ package Sem_Util is
    --  the encapsulated expression is nontrivial.
 
    function Is_Null_Extension
-    (T : Entity_Id; Ignore_Privacy : Boolean := False) return Boolean;
+     (T : Entity_Id; Ignore_Privacy : Boolean := False) return Boolean;
    --  Given a tagged type, returns True if argument is a type extension
    --  that introduces no new components (discriminant or nondiscriminant).
    --  Ignore_Privacy should be True for use in implementing dynamic semantics.
@@ -2449,6 +2449,10 @@ package Sem_Util is
    --  Determine whether an arbitrary entity denotes an instance of function
    --  Ada.Unchecked_Conversion.
 
+   function Is_Unchecked_Union_Equality (Id : Entity_Id) return Boolean;
+   --  Determine whether an arbitrary entity denotes the predefined equality
+   --  function of an Unchecked_Union type (see Build_Variant_Record_Equality).
+
    function Is_Unconstrained_Or_Tagged_Item (Item : Entity_Id) return Boolean;
    --  Subsidiary to Collect_Subprogram_Inputs_Outputs and the analysis of
    --  pragma Depends. Determine whether the type of dependency item Item is
@@ -2973,11 +2977,11 @@ package Sem_Util is
       Comp     : Boolean := False;
       Deriv    : Boolean := False);
    --  Set Disable_Controlled, Finalize_Storage_Only, Has_Controlled_Component,
-   --  Has_Relaxed_Finalization, and Is_Controlled_Active on Typ when the flags
-   --  are set on From_Typ. If Comp is True, From_Typ is assumed to be the type
-   --  of a component of Typ while, if Deriv is True, From_Typ is assumed to be
-   --  the parent type of Typ. This procedure can only set flags for Typ, and
-   --  never clear them.
+   --  Has_Destructor, Has_Relaxed_Finalization, and Is_Controlled_Active on
+   --  Typ when the flags are set on From_Typ. If Comp is True, From_Typ is
+   --  assumed to be the type of a component of Typ while, if Deriv is True,
+   --  From_Typ is assumed to be the parent type of Typ. This procedure can
+   --  only set flags for Typ, and never clear them.
 
    procedure Propagate_DIC_Attributes
      (Typ      : Entity_Id;
diff --git a/gcc/ada/sinfo.ads b/gcc/ada/sinfo.ads
index c63a97d..3d11d5c 100644
--- a/gcc/ada/sinfo.ads
+++ b/gcc/ada/sinfo.ads
@@ -402,16 +402,17 @@ package Sinfo is
    --       Has_Secondary_Private_View  set in generic units
 
    --    "plus fields for expression"
-   --       Paren_Count               number of parentheses levels
-   --       Etype                     type of the expression
-   --       Is_Overloaded             >1 type interpretation exists
-   --       Is_Static_Expression      set for static expression
-   --       Raises_Constraint_Error   evaluation raises CE
-   --       Must_Not_Freeze           set if must not freeze
-   --       Do_Range_Check            set if a range check needed
-   --       Has_Dynamic_Length_Check  set if length check inserted
-   --       Assignment_OK             set if modification is OK
-   --       Is_Controlling_Actual     set for controlling argument
+   --       Paren_Count                  number of parentheses levels
+   --       Etype                        type of the expression
+   --       Is_Overloaded                >1 type interpretation exists
+   --       Is_Static_Expression         set for static expression
+   --       Raises_Constraint_Error      evaluation raises CE
+   --       Must_Not_Freeze              set if must not freeze
+   --       Do_Range_Check               set if a range check needed
+   --       Has_Dynamic_Length_Check     set if length check inserted
+   --       Assignment_OK                set if modification is OK
+   --       Is_Controlling_Actual        set for controlling argument
+   --       Is_Expanded_Dispatching_Call set for expanded dispatching calls
 
    --  Note: see under (EXPRESSION) for further details on the use of
    --  the Paren_Count field to record the number of parentheses levels.
@@ -1664,6 +1665,10 @@ package Sinfo is
    --    actuals to support a build-in-place style of call have been added to
    --    the call.
 
+   --  Is_Expanded_Dispatching_Call
+   --    This flag is set in N_Block_Statement, and expression nodes to
+   --    indicate that it is an expanded dispatching call.
+
    --  Is_Expanded_Prefixed_Call
    --    This flag is set in N_Function_Call and N_Procedure_Call_Statement
    --    nodes to indicate that it is an expanded prefixed call.
@@ -2321,6 +2326,13 @@ package Sinfo is
    --    statement applies to. Finally, if Analyze_Continue_Statement detects
    --    an error, this field is set to Empty.
 
+   --  Tag_Propagated
+   --    This flag is set in N_Identifier, N_Explicit_Dereference, and N_Type_
+   --    Conversion nodes that are the LHS of an assignment statement. Used to
+   --    remember that the RHS of the assignment has tag indeterminate function
+   --    calls and the tag has been propagated to the calls (as part of the
+   --    bottom-up analysis of the RHS of the assignment statement).
+
    --  Target_Type
    --    Used in an N_Validate_Unchecked_Conversion node to point to the target
    --    type entity for the unchecked conversion instantiation which gigi must
@@ -2507,6 +2519,7 @@ package Sinfo is
       --  Has_Private_View (set in generic units)
       --  Has_Secondary_Private_View (set in generic units)
       --  Redundant_Use
+      --  Tag_Propagated
       --  Atomic_Sync_Required
       --  plus fields for expression
 
@@ -3820,6 +3833,7 @@ package Sinfo is
       --  Prefix
       --  Actual_Designated_Subtype
       --  Has_Dereference_Action
+      --  Tag_Propagated
       --  Atomic_Sync_Required
       --  plus fields for expression
 
@@ -4755,6 +4769,7 @@ package Sinfo is
       --  Conversion_OK
       --  Do_Overflow_Check
       --  Rounded_Result
+      --  Tag_Propagated
       --  plus fields for expression
 
       --  Note: if a range check is required, then the Do_Range_Check flag
@@ -5196,6 +5211,7 @@ package Sinfo is
       --  Has_Created_Identifier
       --  Is_Abort_Block
       --  Is_Asynchronous_Call_Block
+      --  Is_Expanded_Dispatching_Call
       --  Is_Initialization_Block
       --  Is_Task_Allocation_Block
       --  Is_Task_Master
diff --git a/gcc/ada/snames.ads-tmpl b/gcc/ada/snames.ads-tmpl
index f26515e..272e10b 100644
--- a/gcc/ada/snames.ads-tmpl
+++ b/gcc/ada/snames.ads-tmpl
@@ -151,6 +151,7 @@ package Snames is
    Name_Default_Value                  : constant Name_Id := N + $;
    Name_Default_Component_Value        : constant Name_Id := N + $;
    Name_Designated_Storage_Model       : constant Name_Id := N + $;
+   Name_Destructor                     : constant Name_Id := N + $;
    Name_Dimension                      : constant Name_Id := N + $;
    Name_Dimension_System               : constant Name_Id := N + $;
    Name_Disable_Controlled             : constant Name_Id := N + $;
diff --git a/gcc/ada/styleg.adb b/gcc/ada/styleg.adb
index 7b7f252..20945fb 100644
--- a/gcc/ada/styleg.adb
+++ b/gcc/ada/styleg.adb
@@ -499,7 +499,8 @@ package body Styleg is
                   if Is_Box_Comment
                     or else Style_Check_Comments_Spacing = 1
                   then
-                     Error_Space_Required (Scan_Ptr + 2);
+                     Error_Msg -- CODEFIX
+                       ("(style) space required?c?", Scan_Ptr + 2);
                   else
                      Error_Msg -- CODEFIX
                        ("(style) two spaces required?c?", Scan_Ptr + 2);
@@ -526,7 +527,8 @@ package body Styleg is
          --  box comment.
 
          elsif not Is_Box_Comment then
-            Error_Space_Required (Scan_Ptr + 3);
+            Error_Msg -- CODEFIX
+              ("(style) space required?c?", Scan_Ptr + 3);
          end if;
       end if;
    end Check_Comment;
diff --git a/gcc/ada/sysdep.c b/gcc/ada/sysdep.c
index 3dc76f9..afbb362 100644
--- a/gcc/ada/sysdep.c
+++ b/gcc/ada/sysdep.c
@@ -40,6 +40,12 @@
    - either they are defined as ENOENT (vx7r2);
    - or the corresponding system includes are not provided (Helix Cert).  */
 
+#if __has_include ("strings.h")
+/* On VxWorks6, FD_ZERO uses bzero, and index is also declared in strings.h,
+   but since it's not a standard header, don't require it.  */
+#include "strings.h"
+#endif
+
 #if __has_include ("dosFsLib.h")
 /* On helix-cert, this include is only provided for RTPs.  */
 #include "dosFsLib.h"
diff --git a/gcc/ada/tbuild.adb b/gcc/ada/tbuild.adb
index b89c408..52fdbfc 100644
--- a/gcc/ada/tbuild.adb
+++ b/gcc/ada/tbuild.adb
@@ -926,11 +926,11 @@ package body Tbuild is
       --  conversion of an unchecked conversion. Extra unchecked conversions
       --  make the .dg output less readable. We can't do this in cases
       --  involving bitfields, because the sizes might not match. The
-      --  "not Is_Scalar_Type" checks avoid such cases.
+      --  Is_Composite_Type checks avoid such cases.
 
       elsif Nkind (Expr) = N_Unchecked_Type_Conversion
-        and then not Is_Scalar_Type (Etype (Expr))
-        and then not Is_Scalar_Type (Typ)
+        and then Is_Composite_Type (Etype (Expr))
+        and then Is_Composite_Type (Typ)
       then
          Set_Subtype_Mark (Expr, New_Occurrence_Of (Typ, Loc));
          Result := Relocate_Node (Expr);
diff --git a/gcc/analyzer/ChangeLog b/gcc/analyzer/ChangeLog
index d31cbbc..8e5d38e 100644
--- a/gcc/analyzer/ChangeLog
+++ b/gcc/analyzer/ChangeLog
@@ -1,3 +1,148 @@
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* program-point.cc: Make diagnostics::context::m_source_printing
+	private.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* pending-diagnostic.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* program-point.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* program-point.cc: : Update for diagnostic_context becoming
+	diagnostics::context, and for diagnostic_source_print_policy
+	becoming diagnostics::source_print_policy.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* ana-state-to-diagnostic-state.cc: Update for move of
+	diagnostics output formats into namespace "diagnostics" as
+	"sinks".
+	* bounds-checking.cc: Likewise.
+	* call-details.cc: Likewise.
+	* checker-event.cc: Likewise.
+	* checker-event.h: Likewise.
+	* diagnostic-manager.cc: Likewise.
+	* diagnostic-manager.h: Likewise.
+	* infinite-loop.cc: Likewise.
+	* infinite-recursion.cc: Likewise.
+	* pending-diagnostic.h: Likewise.
+	* region-model.cc: Likewise.
+	* sm-taint.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* access-diagram.cc: Update for changes to diagnostic paths:
+	"diagnostic-path.h" moving to "diagnostics/paths.h",
+	"diagnostic-event-id.h" moving to "diagnostics/event-id.h",
+	diagnostic_event_id_t to diagnostics::paths::event_id_t,
+	diagnostic_path to diagnostics::paths::path, and
+	diagnostic_event to diagnostics::paths::event.
+	* access-diagram.h: Likewise.
+	* analyzer.cc: Likewise.
+	* bounds-checking.cc: Likewise.
+	* call-info.cc: Likewise.
+	* checker-event.cc: Likewise.
+	* checker-event.h: Likewise.
+	* checker-path.cc: Likewise.
+	* checker-path.h: Likewise.
+	* common.h: Likewise.
+	* diagnostic-manager.cc: Likewise.
+	* pending-diagnostic.cc: Likewise.
+	* pending-diagnostic.h: Likewise.
+	* program-point.cc: Likewise.
+	* program-state.cc: Likewise.
+	* region-model.cc: Likewise.
+	* sm-fd.cc: Likewise.
+	* sm-file.cc: Likewise.
+	* sm-malloc.cc: Likewise.
+	* sm-pattern-test.cc: Likewise.
+	* sm-sensitive.cc: Likewise.
+	* sm-signal.cc: Likewise.
+	* sm-taint.cc: Likewise.
+	* varargs.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* bounds-checking.cc: Update #include for move of
+	"diagnostic-diagram.h" to "diagnostics/diagram.h". Update for
+	move of diagnostic_diagram to diagnostics::diagram.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* diagnostic-manager.cc: Update for move of diagnostic_metadata to
+	diagnostics::metadata.
+	* kf.cc: Likewise.
+	* pending-diagnostic.h: Likewise; also, update #include for move
+	of "diagnostic-metadata.h" to "diagnostics/metadata.h".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* checker-event.h (checker_event::get_logical_location): Update
+	for conversion of logical_location to
+	diagnostics::logical_locations::key.
+	(checker_event::m_logical_loc): Likewise.
+	* diagnostic-manager.cc
+	(diagnostic_manager::get_logical_location_manager): Likewise.
+	* diagnostic-manager.h
+	(diagnostic_manager::get_logical_location_manager): Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* ana-state-to-diagnostic-state.cc: Update #include for move of
+	"diagnostic-state-graphs.h" to "diagnostics/state-graphs.h".
+	* ana-state-to-diagnostic-state.h: Likewise.
+	* checker-event.cc: Likewise.
+	* checker-event.h: Update #include for move of
+	"diagnostic-digraphs.h" to "diagnostics/digraphs.h".
+	* program-state.cc: : Update #include for move of
+	"diagnostic-state-graphs.h" to "diagnostics/state-graphs.h".
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* region-model-asm.cc (region_model::on_asm_stmt): Pass null
+	pointer to parse_{input,output}_constraint().
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* ana-state-to-diagnostic-state.cc: Reimplement, replacing
+	XML-based implementation with one based on state graphs.
+	* ana-state-to-diagnostic-state.h: Likewise.
+	* checker-event.cc: Replace include of "xml.h" with include of
+	"diagnostic-state-graphs.h".
+	(checker_event::maybe_make_xml_state): Replace with...
+	(checker_event::maybe_make_diagnostic_state_graph): ...this.
+	* checker-event.h: Add include of "diagnostic-digraphs.h".
+	(checker_event::maybe_make_xml_state): Replace decl with...
+	(checker_event::maybe_make_diagnostic_state_graph): ...this.
+	* engine.cc (exploded_node::on_stmt_pre): Replace
+	"_analyzer_dump_xml" with "__analyzer_dump_sarif".
+	* program-state.cc: Replace include of "diagnostic-state.h" with
+	"diagnostic-state-graphs.h".
+	(program_state::dump_dot): Port from XML to state graphs.
+	* program-state.h: Drop reduntant forward decl of xml::document.
+	(program_state::make_xml): Replace decl with...
+	(program_state::make_diagnostic_state_graph): ...this.
+	(program_state::dump_xml_to_pp): Drop decl.
+	(program_state::dump_xml_to_file): Drop decl.
+	(program_state::dump_xml): Drop decl.
+	(program_state::dump_dump_sarif): New decl.
+	* sm-malloc.cc (get_dynalloc_state_for_state): New.
+	(malloc_state_machine::add_state_to_xml): Replace with...
+	(malloc_state_machine::add_state_to_state_graph): ...this.
+	* sm.cc (state_machine::add_state_to_xml): Replace with...
+	(state_machine::add_state_to_state_graph): ...this.
+	(state_machine::add_global_state_to_xml): Replace with...
+	(state_machine::add_global_state_to_state_graph): ...this.
+	* sm.h (class xml_state): Drop forward decl.
+	(class analyzer_state_graph): New forward decl.
+	(state_machine::add_state_to_xml): Replace decl with...
+	(state_machine::add_state_to_state_graph): ...this.
+	(state_machine::add_global_state_to_xml): Replace decl with...
+	(state_machine::add_global_state_to_state_graph): ...this.
+
 2025-06-30  David Malcolm  <dmalcolm@redhat.com>
 
 	* access-diagram.cc: Use nullptr rather than NULL where
diff --git a/gcc/analyzer/access-diagram.cc b/gcc/analyzer/access-diagram.cc
index 90b396c..166be08 100644
--- a/gcc/analyzer/access-diagram.cc
+++ b/gcc/analyzer/access-diagram.cc
@@ -1300,7 +1300,7 @@ class valid_region_spatial_item : public spatial_item
 {
 public:
   valid_region_spatial_item (const access_operation &op,
-			     diagnostic_event_id_t region_creation_event_id,
+			     diagnostics::paths::event_id_t region_creation_event_id,
 			     const theme &theme)
   : m_op (op),
     m_region_creation_event_id (region_creation_event_id),
@@ -1523,7 +1523,7 @@ public:
 
 private:
   const access_operation &m_op;
-  diagnostic_event_id_t m_region_creation_event_id;
+  diagnostics::paths::event_id_t m_region_creation_event_id;
   mutable const boundaries *m_boundaries;
   const svalue *m_existing_sval;
   std::unique_ptr<spatial_item> m_existing_sval_spatial_item;
@@ -2012,7 +2012,7 @@ class access_diagram_impl : public vbox_widget
 {
 public:
   access_diagram_impl (const access_operation &op,
-		       diagnostic_event_id_t region_creation_event_id,
+		       diagnostics::paths::event_id_t region_creation_event_id,
 		       style_manager &sm,
 		       const theme &theme,
 		       logger *logger)
@@ -2564,7 +2564,7 @@ private:
   }
 
   const access_operation &m_op;
-  diagnostic_event_id_t m_region_creation_event_id;
+  diagnostics::paths::event_id_t m_region_creation_event_id;
   style_manager &m_sm;
   const theme &m_theme;
   logger *m_logger;
@@ -2662,7 +2662,7 @@ direction_widget::paint_to_canvas (canvas &canvas)
    an access_diagram_impl.  */
 
 access_diagram::access_diagram (const access_operation &op,
-				diagnostic_event_id_t region_creation_event_id,
+				diagnostics::paths::event_id_t region_creation_event_id,
 				style_manager &sm,
 				const theme &theme,
 				logger *logger)
diff --git a/gcc/analyzer/access-diagram.h b/gcc/analyzer/access-diagram.h
index e31dd28..d558f53 100644
--- a/gcc/analyzer/access-diagram.h
+++ b/gcc/analyzer/access-diagram.h
@@ -152,7 +152,7 @@ class access_diagram : public text_art::wrapper_widget
 {
 public:
   access_diagram (const access_operation &op,
-		  diagnostic_event_id_t region_creation_event_id,
+		  diagnostics::paths::event_id_t region_creation_event_id,
 		  text_art::style_manager &sm,
 		  const text_art::theme &theme,
 		  logger *logger);
diff --git a/gcc/analyzer/ana-state-to-diagnostic-state.cc b/gcc/analyzer/ana-state-to-diagnostic-state.cc
index b85a2f1..996538c 100644
--- a/gcc/analyzer/ana-state-to-diagnostic-state.cc
+++ b/gcc/analyzer/ana-state-to-diagnostic-state.cc
@@ -1,4 +1,4 @@
-/* Converting ana::program_state to XML state documents.
+/* Creating diagnostic state graphs from ana::program_state.
    Copyright (C) 2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -23,8 +23,8 @@ along with GCC; see the file COPYING3.  If not see
 #define INCLUDE_SET
 #include "analyzer/common.h"
 
-#include "xml.h"
-#include "xml-printer.h"
+#include "diagnostics/state-graphs.h"
+#include "diagnostics/sarif-sink.h"
 
 #include "analyzer/region-model.h"
 #include "analyzer/program-state.h"
@@ -39,57 +39,47 @@ along with GCC; see the file COPYING3.  If not see
 
 namespace ana {
 
+using namespace ::diagnostics::state_graphs;
+
 static void
-set_wi_attr (xml::element &e,
+set_wi_attr (state_node_ref state_node,
 	     const char *attr_name,
 	     const wide_int_ref &w,
 	     signop sgn)
 {
   pretty_printer pp;
   pp_wide_int (&pp, w, sgn);
-  e.set_attr (attr_name, pp_formatted_text (&pp));
+  state_node.set_attr (attr_name, pp_formatted_text (&pp));
 }
 
 static void
-set_type_attr (xml::element &e, const_tree type)
+set_type_attr (state_node_ref state_node, const_tree type)
 {
   gcc_assert (type);
   pretty_printer pp;
   pp_format_decoder (&pp) = default_tree_printer;
   pp_printf (&pp, "%T", type);
-  e.set_attr ("type", pp_formatted_text (&pp));
+  state_node.set_type (pp_formatted_text (&pp));
 }
 
 static void
-set_bits_attr (xml::element &e,
+set_bits_attr (state_node_ref state_node,
 	       bit_range bits)
 {
   pretty_printer pp;
   bits.dump_to_pp (&pp);
-  e.set_attr ("bits", pp_formatted_text (&pp));
+  state_node.set_attr ("bits", pp_formatted_text (&pp));
 }
 
-static void
-set_region_id_attr (xml::element &e,
-		    const region &reg)
-{
-  e.set_attr ("region_id", std::to_string (reg.get_id ()));
-}
+// class analyzer_state_graph : public diagnostics::digraphs::digraph
 
-// class xml_state : public xml::document
-
-xml_state::xml_state (const program_state &state,
-		      const extrinsic_state &ext_state)
-: xml::document (),
-  m_state (state),
+analyzer_state_graph::analyzer_state_graph (const program_state &state,
+					    const extrinsic_state &ext_state)
+: m_state (state),
   m_ext_state (ext_state),
   m_mgr (*ext_state.get_engine ()->get_model_manager ()),
-  m_root (nullptr)
+  m_next_id (0)
 {
-  auto root = std::make_unique<xml::element> ("state-diagram", false);
-  m_root = root.get ();
-  add_child (std::move (root));
-
   /* Find pointers to heap-allocated regions, and record their types,
      so that we have a user-friendly way of showing the memory
      (by field, rather than by byte offset).  */
@@ -117,14 +107,14 @@ xml_state::xml_state (const program_state &state,
   for (int i = state.m_region_model->get_stack_depth () - 1; i >= 0; --i)
     {
       const frame_region *reg = state.m_region_model->get_frame_at_index (i);
-      get_or_create_element (*reg);
+      get_or_create_state_node (*reg);
     }
 
   /* Create bound memory.  */
   for (auto iter : *state.m_region_model->get_store ())
     {
       const bool create_all = false; // "true" for verbose, for debugging
-      create_elements_for_binding_cluster (*iter.second, create_all);
+      create_state_nodes_for_binding_cluster (*iter.second, create_all);
     }
 
   /* TODO: Constraints.  */
@@ -137,52 +127,165 @@ xml_state::xml_state (const program_state &state,
       {
 	auto &sm = ext_state.get_sm (i);
 	for (const auto &iter : *smap)
-	  sm.add_state_to_xml (*this, *iter.first, iter.second.m_state);
+	  sm.add_state_to_state_graph (*this, *iter.first, iter.second.m_state);
 	if (auto s = smap->get_global_state ())
-	  sm.add_global_state_to_xml (*this, s);
+	  sm.add_global_state_to_state_graph (*this, s);
       }
   }
+
+  /* Process pending edges.  */
+  while (m_pending_edges.size () > 0)
+    {
+      pending_edge item = m_pending_edges.back ();
+      m_pending_edges.pop_back ();
+
+      /* Ensure we have a node for the dst region.  This
+	 could lead to additional pending edges.  */
+      auto dst_node = get_or_create_state_node (item.m_dst_reg);
+      add_edge (nullptr, item.m_src_node.m_node, dst_node.m_node);
+    }
 }
 
-xml::element &
-xml_state::get_or_create_element (const region &reg)
+state_node_ref
+analyzer_state_graph::get_or_create_state_node (const region &reg)
 {
-  auto existing = m_region_to_element_map.find (&reg);
-  if (existing != m_region_to_element_map.end ())
+  auto existing = m_region_to_state_node_map.find (&reg);
+  if (existing != m_region_to_state_node_map.end ())
     return *existing->second;
 
-  auto &e = create_and_add_element (reg);
-  m_region_to_element_map[&reg] = &e;
-  return e;
+  auto ref = create_and_add_state_node (reg);
+  m_region_to_state_node_map[&reg] = &ref.m_node;
+  return ref;
 }
 
-xml::element&
-xml_state::create_and_add_element (const region &reg)
+state_node_ref
+analyzer_state_graph::create_and_add_state_node (const region &reg)
 {
-  auto e = create_element (reg);
-  xml::element &result = *e;
+  auto node = create_state_node (reg);
+
+  state_node_ref result = *node;
   if (auto parent_reg = reg.get_parent_region ())
+    if (parent_reg->get_kind () != RK_ROOT)
+      {
+	auto parent_state_node = get_or_create_state_node (*parent_reg);
+	parent_state_node.m_node.add_child (std::move (node));
+	return result;
+      }
+  add_node (std::move (node));
+  return result;
+}
+
+std::string
+analyzer_state_graph::make_node_id (const char *prefix)
+{
+  return std::string (prefix) + "-" + std::to_string (m_next_id++);
+}
+
+std::string
+analyzer_state_graph::make_node_id (const region &reg)
+{
+  const char *prefix = nullptr;
+  switch (reg.get_kind ())
     {
-      auto parent_element = &get_or_create_element (*parent_reg);
-      parent_element->add_child (std::move (e));
+    case RK_ROOT:
+    default:
+      gcc_unreachable ();
+      break;
+
+    case RK_GLOBALS:
+      return "globals";
+    case RK_CODE:
+      return "code";
+    case RK_STACK:
+      return "stack";
+    case RK_HEAP:
+      return "heap";
+
+    case RK_FRAME:
+      prefix = "frame-region";
+      break;
+    case RK_FUNCTION:
+      prefix = "function-region";
+      break;
+    case RK_LABEL:
+      prefix = "label-region";
+      break;
+    case RK_THREAD_LOCAL:
+      prefix = "thread-local-region";
+      break;
+    case RK_SYMBOLIC:
+      prefix = "symbolic-region";
+      break;
+    case RK_DECL:
+      prefix = "decl-region";
+      break;
+    case RK_FIELD:
+      prefix = "field-region";
+      break;
+    case RK_ELEMENT:
+      prefix = "element-region";
+      break;
+    case RK_OFFSET:
+      prefix = "offset-region";
+      break;
+    case RK_SIZED:
+      prefix = "sized-region";
+      break;
+    case RK_CAST:
+      prefix = "cast-region";
+      break;
+    case RK_HEAP_ALLOCATED:
+      prefix = "heap-allocated-region";
+      break;
+    case RK_ALLOCA:
+      prefix = "alloca-region";
+      break;
+    case RK_STRING:
+      prefix = "string-region";
+      break;
+    case RK_BIT_RANGE:
+      prefix = "bit-range-region";
+      break;
+    case RK_VAR_ARG:
+      prefix = "var-arg-region";
+      break;
+    case RK_ERRNO:
+      prefix = "errno-region";
+      break;
+    case RK_PRIVATE:
+      prefix = "private-region";
+      break;
+    case RK_UNKNOWN:
+      prefix = "unknown-region";
+      break;
     }
-    else
-      m_root->add_child (std::move (e));
-  return result;
+  return std::string (prefix) + "-" + std::to_string (reg.get_id ());
+}
+
+std::unique_ptr<diagnostics::digraphs::node>
+analyzer_state_graph::
+make_state_node (diagnostics::state_graphs::node_kind kind,
+		 std::string id)
+{
+  auto node = std::make_unique<diagnostics::digraphs::node> (*this, std::move (id));
+  state_node_ref node_ref (*node);
+  node_ref.set_node_kind (kind);
+  return node;
 }
 
-std::unique_ptr<xml::element>
-xml_state::make_memory_space_element (const char *label)
+std::unique_ptr<diagnostics::digraphs::node>
+analyzer_state_graph::
+make_memspace_state_node (const region &reg,
+			  diagnostics::state_graphs::node_kind kind)
 {
-  auto e = std::make_unique<xml::element> ("memory-space", false);
-  e->set_attr ("label", label);
-  return e;
+  return make_state_node (kind, make_node_id (reg));
 }
 
-std::unique_ptr<xml::element>
-xml_state::create_element (const region &reg)
+std::unique_ptr<diagnostics::digraphs::node>
+analyzer_state_graph::create_state_node (const region &reg)
 {
-  std::unique_ptr<xml::element> e;
+  std::unique_ptr<diagnostics::digraphs::node> node;
+
   switch (reg.get_kind ())
     {
     default:
@@ -190,110 +293,102 @@ xml_state::create_element (const region &reg)
 
     case RK_FRAME:
       {
-	e = std::make_unique<xml::element> ("stack-frame", false);
 	const frame_region &frame_reg
 	  = static_cast<const frame_region &> (reg);
+
+	node = make_state_node (diagnostics::state_graphs::node_kind::stack_frame,
+				make_node_id (reg));
+	node->set_logical_loc
+	  (m_logical_loc_mgr.key_from_tree (frame_reg.get_fndecl ()));
 	{
 	  pretty_printer pp;
 	  pp_format_decoder (&pp) = default_tree_printer;
 	  pp_printf (&pp, "%E", frame_reg.get_fndecl ());
-	  e->set_attr ("function", pp_formatted_text (&pp));
+	  node->set_attr (STATE_NODE_PREFIX, "function",
+			  pp_formatted_text (&pp));
 	}
       }
       break;
+
     case RK_GLOBALS:
-      e = make_memory_space_element ("Globals");
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::globals);
       break;
     case RK_CODE:
-      e = make_memory_space_element ("Code");
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::code);
       break;
     case RK_FUNCTION:
-      e = std::make_unique<xml::element> ("function", false);
-      // TODO
-      break;
-    case RK_LABEL:
-      e = std::make_unique<xml::element> ("label", false);
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::function);
       // TODO
       break;
+
     case RK_STACK:
-      e = std::make_unique<xml::element> ("stack", false);
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::stack);
       break;
     case RK_HEAP:
-      e = make_memory_space_element ("Heap");
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::heap_);
       break;
     case RK_THREAD_LOCAL:
-      e = make_memory_space_element ("Thread-local");
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::thread_local_);
       break;
     case RK_ROOT:
-      e = std::make_unique<xml::element> ("memory-regions", false);
+      gcc_unreachable ();
       break;
     case RK_SYMBOLIC:
-      e = std::make_unique<xml::element> ("symbolic-region", false);
-      // TODO
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::other);
       break;
+
     case RK_DECL:
       {
-	e = std::make_unique<xml::element> ("variable", false);
+	node = make_state_node (diagnostics::state_graphs::node_kind::variable,
+				make_node_id (reg));
 	const decl_region &decl_reg
 	  = static_cast<const decl_region &> (reg);
+	state_node_ref node_ref (*node);
 	{
 	  pretty_printer pp;
 	  pp_format_decoder (&pp) = default_tree_printer;
 	  pp_printf (&pp, "%E", decl_reg.get_decl ());
-	  e->set_attr ("name", pp_formatted_text (&pp));
+	  node_ref.set_name (pp_formatted_text (&pp));
 	}
-	set_type_attr (*e, TREE_TYPE (decl_reg.get_decl ()));
+	set_type_attr (*node, TREE_TYPE (decl_reg.get_decl ()));
       }
       break;
+
     case RK_FIELD:
-      e = std::make_unique<xml::element> ("field", false);
-      break;
     case RK_ELEMENT:
-      e = std::make_unique<xml::element> ("element", false);
+      /* These should be handled in populate_state_node_for_typed_region.  */
+      gcc_unreachable ();
       break;
+
+    case RK_LABEL:
     case RK_OFFSET:
-      e = std::make_unique<xml::element> ("offset-region", false);
-      // TODO
-      break;
     case RK_SIZED:
-      e = std::make_unique<xml::element> ("sized-region", false);
-      // TODO
-      break;
     case RK_CAST:
-      e = std::make_unique<xml::element> ("cast-region", false);
-      // TODO
-      break;
-    case RK_HEAP_ALLOCATED:
-      e = std::make_unique<xml::element> ("heap-buffer", false);
-      set_attr_for_dynamic_extents (reg, *e);
-      break;
-    case RK_ALLOCA:
-      e = std::make_unique<xml::element> ("alloca-buffer", false);
-      set_attr_for_dynamic_extents (reg, *e);
-      break;
     case RK_STRING:
-      e = std::make_unique<xml::element> ("string-region", false);
-      // TODO
-      break;
     case RK_BIT_RANGE:
-      e = std::make_unique<xml::element> ("RK_BIT_RANGE", false); // TODO
-      break;
     case RK_VAR_ARG:
-      e = std::make_unique<xml::element> ("RK_VAR_ARG", false); // TODO
-      break;
     case RK_ERRNO:
-      e = std::make_unique<xml::element> ("errno", false);
-      break;
     case RK_PRIVATE:
-      e = std::make_unique<xml::element> ("RK_PRIVATE", false); // TODO
-      break;
     case RK_UNKNOWN:
-      e = std::make_unique<xml::element> ("RK_UNKNOWN", false); // TODO
+      node = make_state_node (diagnostics::state_graphs::node_kind::other,
+				make_node_id (reg));
       break;
-    }
-  gcc_assert (e);
 
-  set_region_id_attr (*e, reg);
+    case RK_HEAP_ALLOCATED:
+    case RK_ALLOCA:
+      node = make_memspace_state_node (reg,
+				       diagnostics::state_graphs::node_kind::dynalloc_buffer);
+      set_attr_for_dynamic_extents (reg, *node);
+      break;
+    }
+  gcc_assert (node);
 
   if (reg.get_base_region () == &reg)
     if (!reg.get_type ())
@@ -303,16 +398,17 @@ xml_state::create_element (const region &reg)
 	if (search != m_types_for_untyped_regions.end ())
 	  {
 	    tree type_to_use = search->second;
-	    set_type_attr (*e, type_to_use);
+	    set_type_attr (*node, type_to_use);
 	  }
       }
 
-  return e;
+  return node;
 }
 
 void
-xml_state::create_elements_for_binding_cluster (const binding_cluster &cluster,
-						bool create_all)
+analyzer_state_graph::
+create_state_nodes_for_binding_cluster (const binding_cluster &cluster,
+					bool create_all)
 {
   /* TODO:
      - symbolic bindings
@@ -326,12 +422,12 @@ xml_state::create_elements_for_binding_cluster (const binding_cluster &cluster,
       if (auto conc_key = key->dyn_cast_concrete_binding ())
 	conc_bindings[conc_key->get_bit_range ()] = svalue;
       if (const region *reg = svalue->maybe_get_region ())
-	get_or_create_element (*reg);
+	get_or_create_state_node (*reg);
     }
 
-  auto &e = get_or_create_element (*cluster.get_base_region ());
+  auto ref = get_or_create_state_node (*cluster.get_base_region ());
 
-  e.add_child (create_element_for_conc_bindings (conc_bindings));
+  ref.m_node.add_child (create_state_node_for_conc_bindings (conc_bindings));
 
   const region *typed_reg = cluster.get_base_region ();
   if (!typed_reg->get_type ())
@@ -346,44 +442,45 @@ xml_state::create_elements_for_binding_cluster (const binding_cluster &cluster,
     }
 
   if (typed_reg->get_type ())
-    populate_element_for_typed_region (e,
-				       *typed_reg,
-				       conc_bindings,
-				       create_all);
+    populate_state_node_for_typed_region (ref,
+					  *typed_reg,
+					  conc_bindings,
+					  create_all);
   else
     {
       // TODO
     }
 }
 
-std::unique_ptr<xml::element>
-xml_state::create_element_for_conc_bindings (const concrete_bindings_t &conc_bindings)
+std::unique_ptr<diagnostics::digraphs::node>
+analyzer_state_graph::create_state_node_for_conc_bindings (const concrete_bindings_t &conc_bindings)
 {
-  auto e = std::make_unique<xml::element> ("concrete-bindings", false);
+  auto node = make_state_node (diagnostics::state_graphs::node_kind::other,
+			       make_node_id ("concrete-bindings"));
   for (auto iter : conc_bindings)
     {
       const bit_range bits = iter.first;
       const svalue *sval = iter.second;
-      auto binding_element
-	= std::make_unique<xml::element> ("binding", false);
-      set_bits_attr (*binding_element, bits);
+      auto binding_state_node
+	= make_state_node (diagnostics::state_graphs::node_kind::other,
+			   make_node_id ("binding"));
+      set_bits_attr (*binding_state_node, bits);
       {
 	pretty_printer pp;
 	pp_format_decoder (&pp) = default_tree_printer;
 	sval->dump_to_pp (&pp, true);
-	binding_element->set_attr ("value", pp_formatted_text (&pp));
-	if (auto svalue_element = create_element_for_svalue (sval))
-	  binding_element->add_child (std::move (svalue_element));
+	binding_state_node->set_attr (STATE_NODE_PREFIX, "value",
+				      pp_formatted_text (&pp));
       }
-      e->add_child (std::move (binding_element));
+      node->add_child (std::move (binding_state_node));
     }
-  return e;
+  return node;
 }
 
 // Try to get the bit_range of REG within its base region
 bool
-xml_state::get_bit_range_within_base_region (const region &reg,
-					     bit_range &out)
+analyzer_state_graph::get_bit_range_within_base_region (const region &reg,
+							bit_range &out)
 {
   region_offset start_offset = reg.get_offset (&m_mgr);
   if (!start_offset.concrete_p ())
@@ -398,30 +495,28 @@ xml_state::get_bit_range_within_base_region (const region &reg,
 }
 
 void
-xml_state::populate_element_for_typed_region (xml::element &e,
-					      const region &reg,
-					      const concrete_bindings_t &conc_bindings,
-					      bool create_all)
+analyzer_state_graph::
+populate_state_node_for_typed_region (state_node_ref node,
+				      const region &reg,
+				      const concrete_bindings_t &conc_bindings,
+				      bool create_all)
 {
   const_tree reg_type = reg.get_type ();
   gcc_assert (reg_type);
-  set_type_attr (e, reg_type);
+  set_type_attr (node, reg_type);
 
   bit_range bits (0, 0);
   if (get_bit_range_within_base_region (reg, bits))
     {
-      set_bits_attr (e, bits);
+      set_bits_attr (node, bits);
 
       auto search = conc_bindings.find (bits);
       if (search != conc_bindings.end ())
 	{
 	  const svalue *bound_sval = search->second;
-	  if (auto svalue_element = create_element_for_svalue (bound_sval))
-	    {
-	      xml::printer xp (e);
-	      xp.push_tag ("value-of-region");
-	      xp.append (std::move (svalue_element));
-	    }
+	  node.set_json_attr ("value", bound_sval->to_json ());
+	  if (const region *dst_reg = bound_sval->maybe_get_region ())
+	      m_pending_edges.push_back ({node, *dst_reg});
 	}
     }
 
@@ -454,22 +549,23 @@ xml_state::populate_element_for_typed_region (xml::element &e,
 	      = m_mgr.get_element_region (&reg,
 					  const_cast<tree> (element_type),
 					  sval_index);
-	    if (show_child_element_for_child_region_p (*child_reg,
+	    if (show_child_state_node_for_child_region_p (*child_reg,
 						       conc_bindings,
 						       create_all))
 	      {
-		// Here "element" is in the xml sense
-		auto child_element
-		  = std::make_unique<xml::element> ("element", false);
-		set_wi_attr (*child_element, "index", idx, UNSIGNED);
-		set_region_id_attr (*child_element, *child_reg);
+		auto child_state_node
+		  = make_state_node
+		      (diagnostics::state_graphs::node_kind::element,
+		       make_node_id (*child_reg));
+		set_wi_attr (*child_state_node, "index", idx, UNSIGNED);
+
 		// Recurse:
 		gcc_assert (element_type);
-		populate_element_for_typed_region (*child_element,
-						   *child_reg,
-						   conc_bindings,
-						   create_all);
-		e.add_child (std::move (child_element));
+		populate_state_node_for_typed_region (*child_state_node,
+						      *child_reg,
+						      conc_bindings,
+						      create_all);
+		node.m_node.add_child (std::move (child_state_node));
 	      }
 	  }
       }
@@ -485,15 +581,17 @@ xml_state::populate_element_for_typed_region (xml::element &e,
 		const bit_range bits (0, item.m_bit_range.m_size_in_bits);
 		const region *child_reg
 		  = m_mgr.get_bit_range (&reg, NULL_TREE, bits);
-		if (show_child_element_for_child_region_p (*child_reg,
-							   conc_bindings,
-							   create_all))
+		if (show_child_state_node_for_child_region_p (*child_reg,
+							      conc_bindings,
+							      create_all))
 		  {
-		    auto child_element
-		      = std::make_unique<xml::element> ("padding", false);
-		    set_wi_attr (*child_element, "num_bits",
+		    auto child_state_node
+		      = make_state_node
+			  (diagnostics::state_graphs::node_kind::padding,
+			   make_node_id (*child_reg));
+		    set_wi_attr (*child_state_node, "num_bits",
 				 item.m_bit_range.m_size_in_bits, SIGNED);
-		    e.add_child (std::move (child_element));
+		    node.m_node.add_child (std::move (child_state_node));
 		  }
 	      }
 	    else
@@ -501,26 +599,28 @@ xml_state::populate_element_for_typed_region (xml::element &e,
 		const region *child_reg
 		  = m_mgr.get_field_region (&reg,
 					    const_cast<tree> (item.m_field));
-		if (show_child_element_for_child_region_p (*child_reg,
+		if (show_child_state_node_for_child_region_p (*child_reg,
 							   conc_bindings,
 							   create_all))
 		  {
-		    auto child_element
-		      = std::make_unique<xml::element> ("field", false);
+		    auto child_state_node
+		      = make_state_node
+			  (diagnostics::state_graphs::node_kind::field,
+			   make_node_id (*child_reg));
 		    {
 		      pretty_printer pp;
 		      pp_format_decoder (&pp) = default_tree_printer;
 		      pp_printf (&pp, "%D", item.m_field);
-		      child_element->set_attr ("name",
-					       pp_formatted_text (&pp));
+		      child_state_node->set_attr (STATE_NODE_PREFIX, "name",
+						  pp_formatted_text (&pp));
 		    }
-		    set_region_id_attr (*child_element, *child_reg);
+
 		    // Recurse:
-		    populate_element_for_typed_region (*child_element,
+		    populate_state_node_for_typed_region (*child_state_node,
 						       *child_reg,
 						       conc_bindings,
 						       create_all);
-		    e.add_child (std::move (child_element));
+		    node.m_node.add_child (std::move (child_state_node));
 		  }
 	      }
 	  }
@@ -530,7 +630,8 @@ xml_state::populate_element_for_typed_region (xml::element &e,
 }
 
 void
-xml_state::set_attr_for_dynamic_extents (const region &reg, xml::element &e)
+analyzer_state_graph::set_attr_for_dynamic_extents (const region &reg,
+						    state_node_ref node_ref)
 {
   const svalue *sval = m_state.m_region_model->get_dynamic_extents (&reg);
   if (sval)
@@ -541,13 +642,13 @@ xml_state::set_attr_for_dynamic_extents (const region &reg, xml::element &e)
 	pp_wide_int (&pp, wi::to_wide (cst), UNSIGNED);
       else
 	sval->dump_to_pp (&pp, true);
-      e.set_attr ("dynamic-extents", pp_formatted_text (&pp));
+      node_ref.set_attr ("dynamic-extents", pp_formatted_text (&pp));
     }
 }
 
 bool
-xml_state::
-show_child_element_for_child_region_p (const region &reg,
+analyzer_state_graph::
+show_child_state_node_for_child_region_p (const region &reg,
 				       const concrete_bindings_t &conc_bindings,
 				       bool create_all)
 {
@@ -569,216 +670,18 @@ show_child_element_for_child_region_p (const region &reg,
   return false;
 }
 
-std::unique_ptr<xml::element>
-xml_state::create_element_for_svalue (const svalue *sval)
+std::unique_ptr<diagnostics::digraphs::digraph>
+program_state::
+make_diagnostic_state_graph (const extrinsic_state &ext_state) const
 {
-  if (!sval)
-    return nullptr;
-
-  std::unique_ptr<xml::element> result;
-  switch (sval->get_kind ())
-    {
-    default:
-      gcc_unreachable ();
-    case SK_REGION:
-      {
-	const region_svalue *region_sval = (const region_svalue *)sval;
-	result
-	  = std::make_unique<xml::element> ("pointer-to-region", false);
-	set_region_id_attr (*result, *region_sval->get_pointee ());
-      }
-      break;
-    case SK_CONSTANT:
-      {
-	const constant_svalue *constant_sval = (const constant_svalue *)sval;
-	result = std::make_unique<xml::element> ("constant", false);
-	pretty_printer pp;
-	pp_format_decoder (&pp) = default_tree_printer;
-	pp_printf (&pp, "%E", constant_sval->get_constant ());
-	result->set_attr ("value", pp_formatted_text (&pp));
-      }
-      break;
-    case SK_UNKNOWN:
-      result = std::make_unique<xml::element> ("unknown", false);
-      break;
-    case SK_POISONED:
-      {
-	const poisoned_svalue *poisoned_sval = (const poisoned_svalue *)sval;
-	switch (poisoned_sval->get_poison_kind ())
-	  {
-	  default:
-	    gcc_unreachable ();
-	  case poison_kind::uninit:
-	    result = std::make_unique<xml::element> ("uninitialized", false);
-	    break;
-	  case poison_kind::freed:
-	    result = std::make_unique<xml::element> ("freed", false);
-	    break;
-	  case poison_kind::deleted:
-	    result = std::make_unique<xml::element> ("deleted", false);
-	    break;
-	  case poison_kind::popped_stack:
-	    result = std::make_unique<xml::element> ("popped-stack", false);
-	    break;
-	  }
-      }
-      break;
-    case SK_SETJMP:
-      {
-	//const setjmp_svalue *setjmp_sval = (const setjmp_svalue *)sval;
-	result = std::make_unique<xml::element> ("setjmp-buffer", false);
-	// TODO
-      }
-      break;
-    case SK_INITIAL:
-      {
-	const initial_svalue *initial_sval = (const initial_svalue *)sval;
-	result = std::make_unique<xml::element> ("initial-value-of", false);
-	set_region_id_attr (*result, *initial_sval->get_region ());
-      }
-      break;
-    case SK_UNARYOP:
-      {
-	const unaryop_svalue *unaryop_sval = (const unaryop_svalue *)sval;
-	result = std::make_unique<xml::element> ("unary-op", false);
-	result->set_attr ("op", get_tree_code_name (unaryop_sval->get_op ()));
-	result->add_child
-	  (create_element_for_svalue (unaryop_sval->get_arg ()));
-      }
-      break;
-    case SK_BINOP:
-      {
-	const binop_svalue *binop_sval = (const binop_svalue *)sval;
-	result = std::make_unique<xml::element> ("binary-op", false);
-	result->set_attr ("op", get_tree_code_name (binop_sval->get_op ()));
-	result->add_child (create_element_for_svalue (binop_sval->get_arg0 ()));
-	result->add_child (create_element_for_svalue (binop_sval->get_arg1 ()));
-      }
-      break;
-    case SK_SUB:
-      {
-	//const sub_svalue *sub_sval = (const sub_svalue *)sval;
-	result = std::make_unique<xml::element> ("subregion-value", false);
-	// TODO
-      }
-      break;
-    case SK_REPEATED:
-      {
-	const repeated_svalue *repeated_sval = (const repeated_svalue *)sval;
-	result = std::make_unique<xml::element> ("repeated-value", false);
-	result->add_child
-	  (create_element_for_svalue (repeated_sval->get_outer_size ()));
-	result->add_child
-	  (create_element_for_svalue (repeated_sval->get_inner_svalue ()));
-      }
-      break;
-    case SK_BITS_WITHIN:
-      {
-	const bits_within_svalue *bits_within_sval
-	  = (const bits_within_svalue *)sval;
-	result = std::make_unique<xml::element> ("bits-within", false);
-	set_bits_attr (*result, bits_within_sval->get_bits ());
-	result->add_child
-	  (create_element_for_svalue (bits_within_sval->get_inner_svalue ()));
-      }
-      break;
-    case SK_UNMERGEABLE:
-      {
-	const unmergeable_svalue *unmergeable_sval
-	  = (const unmergeable_svalue *)sval;
-	result = std::make_unique<xml::element> ("unmergeable", false);
-	result->add_child
-	  (create_element_for_svalue (unmergeable_sval->get_arg ()));
-      }
-      break;
-    case SK_PLACEHOLDER:
-      {
-	const placeholder_svalue *placeholder_sval
-	  = (const placeholder_svalue *)sval;
-	result = std::make_unique<xml::element> ("placeholder", false);
-	result->set_attr ("name", placeholder_sval->get_name ());
-      }
-      break;
-    case SK_WIDENING:
-      {
-	//const widening_svalue *widening_sval = (const widening_svalue *)sval;
-	result = std::make_unique<xml::element> ("iterating-value", false);
-	// TODO
-      }
-      break;
-    case SK_COMPOUND:
-      {
-	//const compound_svalue *compound_sval = (const compound_svalue *)sval;
-	result = std::make_unique<xml::element> ("compound-value", false);
-	// TODO
-      }
-      break;
-    case SK_CONJURED:
-      {
-	//const conjured_svalue *conjured_sval = (const conjured_svalue *)sval;
-	result = std::make_unique<xml::element> ("conjured-value", false);
-	// TODO
-      }
-      break;
-    case SK_ASM_OUTPUT:
-      {
-	/* const asm_output_svalue *asm_output_sval
-	   = (const asm_output_svalue *)sval; */
-	result = std::make_unique<xml::element> ("asm-output", false);
-	// TODO
-      }
-      break;
-    case SK_CONST_FN_RESULT:
-      {
-	/* const const_fn_result_svalue *const_fn_result_sval
-	   = (const const_fn_result_svalue *)sval; */
-	result = std::make_unique<xml::element> ("const-fn-result", false);
-	// TODO
-      }
-    }
-
-  if (result)
-    {
-      if (sval->get_type ())
-	set_type_attr (*result, sval->get_type ());
-
-      pretty_printer pp;
-      pp_format_decoder (&pp) = default_tree_printer;
-      sval->dump_to_pp (&pp, true);
-      result->set_attr ("dump-text", pp_formatted_text (&pp));
-    }
-
-  return result;
-}
-
-std::unique_ptr<xml::document>
-program_state::make_xml (const extrinsic_state &ext_state) const
-{
-  return std::make_unique<xml_state> (*this, ext_state);
-}
-
-void
-program_state::dump_xml_to_pp (const extrinsic_state &ext_state,
-			       pretty_printer *pp) const
-{
-  auto doc = make_xml (ext_state);
-  doc->write_as_xml (pp, 0, true);
-}
-
-void
-program_state::dump_xml_to_file (const extrinsic_state &ext_state,
-				 FILE *outf) const
-{
-  pretty_printer pp;
-  pp.set_output_stream (outf);
-  dump_xml_to_pp (ext_state, &pp);
-  pp_flush (&pp);
+  return std::make_unique<analyzer_state_graph> (*this, ext_state);
 }
 
 void
-program_state::dump_xml (const extrinsic_state &ext_state) const
+program_state::dump_sarif (const extrinsic_state &ext_state) const
 {
-  dump_xml_to_file (ext_state, stderr);
+  auto g = make_diagnostic_state_graph (ext_state);
+  g->dump ();
 }
 
 } // namespace ana
diff --git a/gcc/analyzer/ana-state-to-diagnostic-state.h b/gcc/analyzer/ana-state-to-diagnostic-state.h
index bd6aa46..3a5ccc1 100644
--- a/gcc/analyzer/ana-state-to-diagnostic-state.h
+++ b/gcc/analyzer/ana-state-to-diagnostic-state.h
@@ -1,4 +1,4 @@
-/* XML documents for dumping state in an easier-to-read form.
+/* Creating diagnostic state graphs from ana::program_state.
    Copyright (C) 2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -18,41 +18,52 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_ANALYZER_ANA_STATE_TO_XML_STATE_H
-#define GCC_ANALYZER_ANA_STATE_TO_XML_STATE_H
+#ifndef GCC_ANALYZER_ANA_STATE_TO_DIAGNOSTIC_STATE_H
+#define GCC_ANALYZER_ANA_STATE_TO_DIAGNOSTIC_STATE_H
 
-#include "xml.h"
+#include "diagnostics/state-graphs.h"
+#include "tree-logical-location.h"
 
 namespace ana {
 
-class xml_state : public xml::document
+class analyzer_state_graph : public diagnostics::digraphs::digraph
 {
 public:
-  xml_state (const program_state &state,
-	     const extrinsic_state &ext_state);
-
-  xml::element &
-  get_or_create_element (const region &reg);
+  analyzer_state_graph (const program_state &state,
+			const extrinsic_state &ext_state);
+  diagnostics::state_graphs::state_node_ref
+  get_or_create_state_node (const region &reg);
 
 private:
-  xml::element&
-  create_and_add_element (const region &reg);
-
-  static std::unique_ptr<xml::element>
-  make_memory_space_element (const char *label);
-
-  std::unique_ptr<xml::element>
-  create_element (const region &reg);
+  struct pending_edge
+  {
+    diagnostics::state_graphs::state_node_ref m_src_node;
+    const region &m_dst_reg;
+  };
+  
+  diagnostics::state_graphs::state_node_ref
+  create_and_add_state_node (const region &reg);
+
+  std::unique_ptr<diagnostics::digraphs::node>
+  make_state_node (diagnostics::state_graphs::node_kind kind,
+		   std::string id);
+
+  std::unique_ptr<diagnostics::digraphs::node>
+  make_memspace_state_node (const region &reg,
+			    enum diagnostics::state_graphs::node_kind kind);
+
+  std::unique_ptr<diagnostics::digraphs::node>
+  create_state_node (const region &reg);
 
   /* Spatially sorted concrete bindings.  */
   typedef std::map<bit_range, const svalue *> concrete_bindings_t;
 
   void
-  create_elements_for_binding_cluster (const binding_cluster &cluster,
-				       bool create_all);
+  create_state_nodes_for_binding_cluster (const binding_cluster &cluster,
+					  bool create_all);
 
-  std::unique_ptr<xml::element>
-  create_element_for_conc_bindings (const concrete_bindings_t &conc_bindings);
+  std::unique_ptr<diagnostics::digraphs::node>
+  create_state_node_for_conc_bindings (const concrete_bindings_t &conc_bindings);
 
   // Try to get the bit_range of REG within its base region
   bool
@@ -60,30 +71,36 @@ private:
 				    bit_range &out);
 
   void
-  populate_element_for_typed_region (xml::element &e,
-				     const region &reg,
-				     const concrete_bindings_t &conc_bindings,
-				     bool create_all);
+  populate_state_node_for_typed_region (diagnostics::state_graphs::state_node_ref,
+					const region &reg,
+					const concrete_bindings_t &conc_bindings,
+					bool create_all);
 
   void
-  set_attr_for_dynamic_extents (const region &reg, xml::element &e);
+  set_attr_for_dynamic_extents (const region &reg,
+				diagnostics::state_graphs::state_node_ref);
 
   bool
-  show_child_element_for_child_region_p (const region &reg,
-					 const concrete_bindings_t &conc_bindings,
-					 bool create_all);
+  show_child_state_node_for_child_region_p (const region &reg,
+					    const concrete_bindings_t &conc_bindings,
+					    bool create_all);
+
+  std::unique_ptr<diagnostics::digraphs::node>
+  create_state_node_for_svalue (const svalue *sval);
 
-  std::unique_ptr<xml::element>
-  create_element_for_svalue (const svalue *sval);
+  std::string make_node_id (const region &reg);
+  std::string make_node_id (const char *prefix);
 
+  tree_logical_location_manager m_logical_loc_mgr;
   const program_state &m_state;
   const extrinsic_state &m_ext_state;
   region_model_manager &m_mgr;
-  xml::element *m_root;
-  std::map<const region *, xml::element *> m_region_to_element_map;
+  std::map<const region *, diagnostics::digraphs::node *> m_region_to_state_node_map;
   std::map<const region *, tree> m_types_for_untyped_regions;
+  unsigned m_next_id;
+  std::vector<pending_edge> m_pending_edges;
 };
 
 } // namespace ana
 
-#endif /* GCC_ANALYZER_ANA_STATE_TO_XML_STATE_H */
+#endif /* GCC_ANALYZER_ANA_STATE_TO_DIAGNOSTIC_STATE_H */
diff --git a/gcc/analyzer/analyzer.cc b/gcc/analyzer/analyzer.cc
index 938ad6d..9f6f7b4 100644
--- a/gcc/analyzer/analyzer.cc
+++ b/gcc/analyzer/analyzer.cc
@@ -21,7 +21,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "analyzer/common.h"
 
 #include "tree-pretty-print.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "tree-dfa.h"
 #include "intl.h"
 
@@ -232,7 +232,7 @@ tree_to_json (tree node)
    for unknown).  */
 
 std::unique_ptr<json::value>
-diagnostic_event_id_to_json (const diagnostic_event_id_t &event_id)
+diagnostic_event_id_to_json (const diagnostics::paths::event_id_t &event_id)
 {
   if (event_id.known_p ())
     {
diff --git a/gcc/analyzer/bounds-checking.cc b/gcc/analyzer/bounds-checking.cc
index d2e2b34..921ad16 100644
--- a/gcc/analyzer/bounds-checking.cc
+++ b/gcc/analyzer/bounds-checking.cc
@@ -20,8 +20,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "analyzer/common.h"
 
 #include "intl.h"
-#include "diagnostic-diagram.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/diagram.h"
+#include "diagnostics/sarif-sink.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/region-model.h"
@@ -51,7 +51,7 @@ public:
     }
     void prepare_for_emission (checker_path *path,
 			       pending_diagnostic *pd,
-			       diagnostic_event_id_t emission_id) override
+			       diagnostics::paths::event_id_t emission_id) override
     {
       region_creation_event_capacity::prepare_for_emission (path,
 							    pd,
@@ -101,10 +101,11 @@ public:
 							       *this));
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/out_of_bounds/"
     props.set_string (PROPERTY_PREFIX "dir",
 		      get_dir () == access_direction::read ? "read" : "write");
@@ -181,7 +182,7 @@ protected:
 	       a problem.  Give up if that's happened.  */
 	    return;
 	  }
-	diagnostic_diagram diagram
+	diagnostics::diagram diagram
 	  (canvas,
 	   /* Alt text.  */
 	   _("Diagram visualizing the predicted out-of-bounds access"));
@@ -203,7 +204,7 @@ protected:
   const region *m_reg;
   tree m_diag_arg;
   const svalue *m_sval_hint;
-  diagnostic_event_id_t m_region_creation_event_id;
+  diagnostics::paths::event_id_t m_region_creation_event_id;
 };
 
 /* Abstract base class for all out-of-bounds warnings where the
@@ -228,11 +229,11 @@ public:
 	    && m_out_of_bounds_bits == other.m_out_of_bounds_bits);
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const override
   {
     out_of_bounds::maybe_add_sarif_properties (result_obj);
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/concrete_out_of_bounds/"
     props.set (PROPERTY_PREFIX "out_of_bounds_bits",
 	       m_out_of_bounds_bits.to_json ());
@@ -294,11 +295,11 @@ public:
 							       *this));
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
     concrete_out_of_bounds::maybe_add_sarif_properties (result_obj);
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/concrete_past_the_end/"
     props.set (PROPERTY_PREFIX "bit_bound",
 	       tree_to_json (m_bit_bound));
@@ -966,11 +967,12 @@ public:
 	    && pending_diagnostic::same_tree_p (m_capacity, other.m_capacity));
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
     out_of_bounds::maybe_add_sarif_properties (result_obj);
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/symbolic_past_the_end/"
     props.set (PROPERTY_PREFIX "offset", tree_to_json (m_offset));
     props.set (PROPERTY_PREFIX "num_bytes", tree_to_json (m_num_bytes));
diff --git a/gcc/analyzer/call-details.cc b/gcc/analyzer/call-details.cc
index cb95843..ede1229 100644
--- a/gcc/analyzer/call-details.cc
+++ b/gcc/analyzer/call-details.cc
@@ -25,7 +25,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-pretty-print.h"
 #include "stringpool.h"
 #include "attribs.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/sarif-sink.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/region-model.h"
@@ -528,10 +528,10 @@ public:
     return true;
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/overlapping_buffers/"
     props.set (PROPERTY_PREFIX "bytes_range_a",
 	       m_byte_range_a.to_json ());
diff --git a/gcc/analyzer/call-info.cc b/gcc/analyzer/call-info.cc
index 9a698ef..f431143 100644
--- a/gcc/analyzer/call-info.cc
+++ b/gcc/analyzer/call-info.cc
@@ -24,7 +24,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "cfg.h"
 #include "digraph.h"
 #include "sbitmap.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/supergraph.h"
diff --git a/gcc/analyzer/checker-event.cc b/gcc/analyzer/checker-event.cc
index af336df..4eac945 100644
--- a/gcc/analyzer/checker-event.cc
+++ b/gcc/analyzer/checker-event.cc
@@ -1,4 +1,4 @@
-/* Subclasses of diagnostic_event for analyzer diagnostics.
+/* Subclasses of diagnostics::paths::event for analyzer diagnostics.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -27,8 +27,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-iterator.h"
 #include "inlining-iterator.h"
 #include "tree-logical-location.h"
-#include "diagnostic-format-sarif.h"
-#include "xml.h"
+#include "diagnostics/sarif-sink.h"
+#include "diagnostics/state-graphs.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/sm.h"
@@ -101,7 +101,7 @@ event_kind_to_string (enum event_kind ek)
     }
 }
 
-/* class checker_event : public diagnostic_event.  */
+/* class checker_event : public diagnostics::paths::event.  */
 
 /* checker_event's ctor.  */
 
@@ -131,24 +131,24 @@ checker_event::checker_event (enum event_kind kind,
     }
 }
 
-/* No-op implementation of diagnostic_event::get_meaning vfunc for
+/* No-op implementation of diagnostics::paths::event::get_meaning vfunc for
    checker_event: checker events have no meaning by default.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 checker_event::get_meaning () const
 {
-  return meaning ();
+  return diagnostics::paths::event::meaning ();
 }
 
-/* Implementation of diagnostic_event::maybe_add_sarif_properties
+/* Implementation of diagnostics::paths::event::maybe_add_sarif_properties
    for checker_event.  */
 
 void
 checker_event::
-maybe_add_sarif_properties (sarif_builder &builder,
-			    sarif_object &thread_flow_loc_obj) const
+maybe_add_sarif_properties (diagnostics::sarif_builder &builder,
+			    diagnostics::sarif_object &thread_flow_loc_obj) const
 {
-  sarif_property_bag &props = thread_flow_loc_obj.get_or_create_properties ();
+  auto &props = thread_flow_loc_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/checker_event/"
   props.set (PROPERTY_PREFIX "emission_id",
 	     diagnostic_event_id_to_json  (m_emission_id));
@@ -215,7 +215,7 @@ checker_event::debug () const
 void
 checker_event::prepare_for_emission (checker_path *path,
 				     pending_diagnostic *pd,
-				     diagnostic_event_id_t emission_id)
+				     diagnostics::paths::event_id_t emission_id)
 {
   m_path = path;
   m_pending_diagnostic = pd;
@@ -225,8 +225,8 @@ checker_event::prepare_for_emission (checker_path *path,
   print_desc (*pp.get ());
 }
 
-std::unique_ptr<xml::document>
-checker_event::maybe_make_xml_state (bool debug) const
+std::unique_ptr<diagnostics::digraphs::digraph>
+checker_event::maybe_make_diagnostic_state_graph (bool debug) const
 {
   const program_state *state = get_program_state ();
   if (!state)
@@ -235,14 +235,16 @@ checker_event::maybe_make_xml_state (bool debug) const
   gcc_assert (m_path);
   const extrinsic_state &ext_state = m_path->get_ext_state ();
 
-  auto result = state->make_xml (ext_state);
+  auto result = state->make_diagnostic_state_graph (ext_state);
 
   if (debug)
     {
       pretty_printer pp;
       text_art::theme *theme = global_dc->get_diagram_theme ();
       text_art::dump_to_pp (*state, theme, &pp);
-      result->add_comment (pp_formatted_text (&pp));
+      result->set_attr (STATE_GRAPH_PREFIX,
+			"analyzer/program_state/",
+			pp_formatted_text (&pp));
     }
 
   return result;
@@ -250,7 +252,7 @@ checker_event::maybe_make_xml_state (bool debug) const
 
 /* class debug_event : public checker_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    debug_event.
    Use the saved string as the event's description.  */
 
@@ -262,7 +264,7 @@ debug_event::print_desc (pretty_printer &pp) const
 
 /* class precanned_custom_event : public custom_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    precanned_custom_event.
    Use the saved string as the event's description.  */
 
@@ -285,7 +287,7 @@ statement_event::statement_event (const gimple *stmt, tree fndecl, int depth,
 {
 }
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    statement_event.
    Use the statement's dump form as the event's description.  */
 
@@ -381,7 +383,7 @@ function_entry_event::function_entry_event (const program_point &dst_point,
 {
 }
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    function_entry_event.
 
    Use a string such as "entry to 'foo'" as the event's description.  */
@@ -392,10 +394,10 @@ function_entry_event::print_desc (pretty_printer &pp) const
   pp_printf (&pp, "entry to %qE", m_effective_fndecl);
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    function entry.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 function_entry_event::get_meaning () const
 {
   return meaning (verb::enter, noun::function);
@@ -427,7 +429,7 @@ state_change_event::state_change_event (const supernode *node,
 {
 }
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    state_change_event.
 
    Attempt to generate a nicer human-readable description.
@@ -469,7 +471,7 @@ state_change_event::print_desc (pretty_printer &pp) const
 		pp_string (&pp, "NULL origin");
 
 	      /* Get any "meaning" of event.  */
-	      diagnostic_event::meaning meaning = get_meaning ();
+	      diagnostics::paths::event::meaning meaning = get_meaning ();
 	      pp_string (&pp, ", meaning: ");
 	      meaning.dump_to_pp (&pp);
 	      pp_string (&pp, ")");
@@ -506,11 +508,11 @@ state_change_event::print_desc (pretty_printer &pp) const
     }
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    state change events: delegate to the pending_diagnostic to
    get any meaning.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 state_change_event::get_meaning () const
 {
   if (m_pending_diagnostic)
@@ -528,16 +530,17 @@ state_change_event::get_meaning () const
 
 /* class superedge_event : public checker_event.  */
 
-/* Implementation of diagnostic_event::maybe_add_sarif_properties
+/* Implementation of diagnostics::paths::event::maybe_add_sarif_properties
    for superedge_event.  */
 
 void
-superedge_event::maybe_add_sarif_properties (sarif_builder &builder,
-					     sarif_object &thread_flow_loc_obj)
+superedge_event::
+maybe_add_sarif_properties (diagnostics::sarif_builder &builder,
+			    diagnostics::sarif_object &thread_flow_loc_obj)
   const
 {
   checker_event::maybe_add_sarif_properties (builder, thread_flow_loc_obj);
-  sarif_property_bag &props = thread_flow_loc_obj.get_or_create_properties ();
+  auto &props = thread_flow_loc_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/superedge_event/"
   if (m_sedge)
     props.set (PROPERTY_PREFIX "superedge", m_sedge->to_json ());
@@ -624,10 +627,10 @@ cfg_edge_event::cfg_edge_event (enum event_kind kind,
   gcc_assert (eedge.m_sedge->m_kind == SUPEREDGE_CFG_EDGE);
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    CFG edge events.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 cfg_edge_event::get_meaning () const
 {
   const cfg_superedge& cfg_sedge = get_cfg_superedge ();
@@ -641,7 +644,7 @@ cfg_edge_event::get_meaning () const
 
 /* class start_cfg_edge_event : public cfg_edge_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    start_cfg_edge_event.
 
    If -fanalyzer-verbose-edges, then generate low-level descriptions, such
@@ -842,7 +845,7 @@ call_event::call_event (const exploded_edge &eedge,
    m_dest_snode = eedge.m_dest->get_supernode ();
 }
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    call_event.
 
    If this call event passes critical state for an sm-based warning,
@@ -874,10 +877,10 @@ call_event::print_desc (pretty_printer &pp) const
 	     get_caller_fndecl ());
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    function call events.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 call_event::get_meaning () const
 {
   return meaning (verb::call, noun::function);
@@ -926,7 +929,7 @@ return_event::return_event (const exploded_edge &eedge,
   m_dest_snode = eedge.m_dest->get_supernode ();
 }
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    return_event.
 
    If this return event returns critical state for an sm-based warning,
@@ -958,10 +961,10 @@ return_event::print_desc (pretty_printer &pp) const
 	     m_src_snode->m_fun->decl);
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    function return events.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 return_event::get_meaning () const
 {
   return meaning (verb::return_, noun::function);
@@ -985,10 +988,10 @@ start_consolidated_cfg_edges_event::print_desc (pretty_printer &pp) const
 	     m_edge_sense ? "true" : "false");
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    start_consolidated_cfg_edges_event.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 start_consolidated_cfg_edges_event::get_meaning () const
 {
   return meaning (verb::branch,
@@ -1006,10 +1009,10 @@ inlined_call_event::print_desc (pretty_printer &pp) const
 	     m_apparent_caller_fndecl);
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    reconstructed inlined function calls.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 inlined_call_event::get_meaning () const
 {
   return meaning (verb::call, noun::function);
@@ -1017,7 +1020,7 @@ inlined_call_event::get_meaning () const
 
 /* class setjmp_event : public checker_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    setjmp_event.  */
 
 void
@@ -1036,7 +1039,7 @@ setjmp_event::print_desc (pretty_printer &pp) const
 void
 setjmp_event::prepare_for_emission (checker_path *path,
 				    pending_diagnostic *pd,
-				    diagnostic_event_id_t emission_id)
+				    diagnostics::paths::event_id_t emission_id)
 {
   checker_event::prepare_for_emission (path, pd, emission_id);
   path->record_setjmp_event (m_enode, emission_id);
@@ -1075,7 +1078,7 @@ rewind_event::rewind_event (const exploded_edge *eedge,
 
 /* class rewind_from_longjmp_event : public rewind_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    rewind_from_longjmp_event.  */
 
 void
@@ -1099,7 +1102,7 @@ rewind_from_longjmp_event::print_desc (pretty_printer &pp) const
 
 /* class rewind_to_setjmp_event : public rewind_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    rewind_to_setjmp_event.  */
 
 void
@@ -1148,7 +1151,7 @@ rewind_to_setjmp_event::print_desc (pretty_printer &pp) const
 void
 rewind_to_setjmp_event::prepare_for_emission (checker_path *path,
 					      pending_diagnostic *pd,
-					      diagnostic_event_id_t emission_id)
+					      diagnostics::paths::event_id_t emission_id)
 {
   checker_event::prepare_for_emission (path, pd, emission_id);
   path->get_setjmp_event (m_rewind_info->get_enode_origin (),
@@ -1201,7 +1204,7 @@ unwind_event::print_desc (pretty_printer &pp) const
 
 /* class warning_event : public checker_event.  */
 
-/* Implementation of diagnostic_event::print_desc vfunc for
+/* Implementation of diagnostics::paths::event::print_desc vfunc for
    warning_event.
 
    If the pending diagnostic implements describe_final_event, use it,
@@ -1246,10 +1249,10 @@ warning_event::print_desc (pretty_printer &pp) const
     pp_string (&pp, "here");
 }
 
-/* Implementation of diagnostic_event::get_meaning vfunc for
+/* Implementation of diagnostics::paths::event::get_meaning vfunc for
    warning_event.  */
 
-diagnostic_event::meaning
+diagnostics::paths::event::meaning
 warning_event::get_meaning () const
 {
   return meaning (verb::danger, noun::unknown);
diff --git a/gcc/analyzer/checker-event.h b/gcc/analyzer/checker-event.h
index 7c44f1e..909e388 100644
--- a/gcc/analyzer/checker-event.h
+++ b/gcc/analyzer/checker-event.h
@@ -1,4 +1,4 @@
-/* Subclasses of diagnostic_event for analyzer diagnostics.
+/* Subclasses of diagnostics::paths::event for analyzer diagnostics.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -24,6 +24,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-logical-location.h"
 #include "analyzer/program-state.h"
 #include "analyzer/event-loc-info.h"
+#include "diagnostics/digraphs.h"
 
 namespace ana {
 
@@ -61,7 +62,7 @@ extern const char *event_kind_to_string (enum event_kind ek);
    The class hierarchy looks like this (using indentation to show
    inheritance, and with event_kinds shown for the concrete subclasses):
 
-   diagnostic_event
+   diagnostics::paths::event
      checker_event
        debug_event (event_kind::debug)
        custom_event (event_kind::custom)
@@ -90,30 +91,32 @@ extern const char *event_kind_to_string (enum event_kind ek);
        unwind_event (event_kind::unwind)
        warning_event (event_kind::warning).  */
 
-/* Abstract subclass of diagnostic_event; the base class for use in
-   checker_path (the analyzer's diagnostic_path subclass).  */
+/* Abstract subclass of diagnostics::paths::event; the base class for use in
+   checker_path (the analyzer's diagnostics::paths::path subclass).  */
 
-class checker_event : public diagnostic_event
+class checker_event : public diagnostics::paths::event
 {
 public:
-  /* Implementation of diagnostic_event.  */
+  /* Implementation of diagnostics::paths::event.  */
 
   location_t get_location () const final override { return m_loc; }
   int get_stack_depth () const final override { return m_effective_depth; }
-  logical_location get_logical_location () const final override
+  diagnostics::logical_locations::key
+  get_logical_location () const final override
   {
     return m_logical_loc;
   }
   meaning get_meaning () const override;
   bool connect_to_next_event_p () const override { return false; }
-  diagnostic_thread_id_t get_thread_id () const final override
+  diagnostics::paths::thread_id_t get_thread_id () const final override
   {
     return 0;
   }
 
   void
-  maybe_add_sarif_properties (sarif_builder &,
-			      sarif_object &thread_flow_loc_obj) const override;
+  maybe_add_sarif_properties (diagnostics::sarif_builder &,
+			      diagnostics::sarif_object &thread_flow_loc_obj)
+    const override;
 
   /* Additional functionality.  */
   enum event_kind get_kind () const { return m_kind; }
@@ -123,19 +126,19 @@ public:
 
   virtual void prepare_for_emission (checker_path *,
 				     pending_diagnostic *pd,
-				     diagnostic_event_id_t emission_id);
+				     diagnostics::paths::event_id_t emission_id);
   virtual bool is_call_p () const { return false; }
   virtual bool is_function_entry_p () const  { return false; }
   virtual bool is_return_p () const  { return false; }
 
+  std::unique_ptr<diagnostics::digraphs::digraph>
+  maybe_make_diagnostic_state_graph (bool debug) const final override;
+
   virtual const program_state *
   get_program_state () const { return nullptr; }
 
-  std::unique_ptr<xml::document>
-  maybe_make_xml_state (bool debug) const final override;
-
   /* For use with %@.  */
-  const diagnostic_event_id_t *get_id_ptr () const
+  const diagnostics::paths::event_id_t *get_id_ptr () const
   {
     return &m_emission_id;
   }
@@ -159,8 +162,8 @@ protected:
   int m_original_depth;
   int m_effective_depth;
   pending_diagnostic *m_pending_diagnostic;
-  diagnostic_event_id_t m_emission_id; // only set once all pruning has occurred
-  logical_location m_logical_loc;
+  diagnostics::paths::event_id_t m_emission_id; // only set once all pruning has occurred
+  diagnostics::logical_locations::key m_logical_loc;
 };
 
 /* A concrete event subclass for a purely textual event, for use in
@@ -421,8 +424,9 @@ public:
 class superedge_event : public checker_event
 {
 public:
-  void maybe_add_sarif_properties (sarif_builder &,
-				   sarif_object &thread_flow_loc_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_builder &,
+			      diagnostics::sarif_object &thread_flow_loc_obj)
     const override;
 
   /* Mark this edge event as being either an interprocedural call or
@@ -664,7 +668,7 @@ public:
 
   void prepare_for_emission (checker_path *path,
 			     pending_diagnostic *pd,
-			     diagnostic_event_id_t emission_id) final override;
+			     diagnostics::paths::event_id_t emission_id) final override;
 
 private:
   const exploded_node *m_enode;
@@ -730,10 +734,10 @@ public:
 
   void prepare_for_emission (checker_path *path,
 			     pending_diagnostic *pd,
-			     diagnostic_event_id_t emission_id) final override;
+			     diagnostics::paths::event_id_t emission_id) final override;
 
 private:
-  diagnostic_event_id_t m_original_setjmp_event_id;
+  diagnostics::paths::event_id_t m_original_setjmp_event_id;
 };
 
 /* An abstract subclass for throwing/rethrowing an exception.  */
diff --git a/gcc/analyzer/checker-path.cc b/gcc/analyzer/checker-path.cc
index 45593e0..646d72c 100644
--- a/gcc/analyzer/checker-path.cc
+++ b/gcc/analyzer/checker-path.cc
@@ -1,4 +1,4 @@
-/* Subclass of diagnostic_path for analyzer diagnostics.
+/* Subclass of diagnostics::paths::path for analyzer diagnostics.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
diff --git a/gcc/analyzer/checker-path.h b/gcc/analyzer/checker-path.h
index 3c174bf..490d21b 100644
--- a/gcc/analyzer/checker-path.h
+++ b/gcc/analyzer/checker-path.h
@@ -1,4 +1,4 @@
-/* Subclass of diagnostic_path for analyzer diagnostics.
+/* Subclass of diagnostics::paths::path for analyzer diagnostics.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -26,28 +26,29 @@ along with GCC; see the file COPYING3.  If not see
 
 namespace ana {
 
-/* Subclass of diagnostic_path for analyzer diagnostics.  */
+/* Subclass of diagnostic path for analyzer diagnostics.  */
 
-class checker_path : public diagnostic_path
+class checker_path : public diagnostics::paths::path
 {
 public:
-  checker_path (const logical_location_manager &logical_loc_mgr,
+  checker_path (const diagnostics::logical_locations::manager &logical_loc_mgr,
 		const extrinsic_state &ext_state,
 		logger *logger)
-  : diagnostic_path (logical_loc_mgr),
+  : diagnostics::paths::path (logical_loc_mgr),
     m_ext_state (ext_state),
     m_thread ("main"),
     m_logger (logger)
   {}
 
-  /* Implementation of diagnostic_path vfuncs.  */
+  /* Implementation of diagnostics::paths::path vfuncs.  */
 
   unsigned num_events () const final override
   {
     return m_events.length ();
   }
 
-  const diagnostic_event & get_event (int idx) const final override
+  const diagnostics::paths::event &
+  get_event (int idx) const final override
   {
     return *m_events[idx];
   }
@@ -55,8 +56,8 @@ public:
   {
     return 1;
   }
-  const diagnostic_thread &
-  get_thread (diagnostic_thread_id_t) const final override
+  const diagnostics::paths::thread &
+  get_thread (diagnostics::paths::thread_id_t) const final override
   {
     return m_thread;
   }
@@ -115,21 +116,22 @@ public:
     checker_event *e;
     int i;
     FOR_EACH_VEC_ELT (m_events, i, e)
-      e->prepare_for_emission (this, pd, diagnostic_event_id_t (i));
+      e->prepare_for_emission (this, pd, diagnostics::paths::event_id_t (i));
   }
 
   void fixup_locations (pending_diagnostic *pd);
 
   void record_setjmp_event (const exploded_node *enode,
-			    diagnostic_event_id_t setjmp_emission_id)
+			    diagnostics::paths::event_id_t setjmp_emission_id)
   {
     m_setjmp_event_ids.put (enode, setjmp_emission_id);
   }
 
   bool get_setjmp_event (const exploded_node *enode,
-			 diagnostic_event_id_t *out_emission_id)
+			 diagnostics::paths::event_id_t *out_emission_id)
   {
-    if (diagnostic_event_id_t *emission_id = m_setjmp_event_ids.get (enode))
+    if (diagnostics::paths::event_id_t *emission_id
+	  = m_setjmp_event_ids.get (enode))
       {
 	*out_emission_id = *emission_id;
 	return true;
@@ -154,7 +156,7 @@ private:
   /* During prepare_for_emission (and after), the setjmp_event for each
      exploded_node *, so that rewind events can refer to them in their
      descriptions.  */
-  hash_map <const exploded_node *, diagnostic_event_id_t> m_setjmp_event_ids;
+  hash_map <const exploded_node *, diagnostics::paths::event_id_t> m_setjmp_event_ids;
 
   logger *m_logger;
 };
diff --git a/gcc/analyzer/common.h b/gcc/analyzer/common.h
index 148bfdd..ac3048c 100644
--- a/gcc/analyzer/common.h
+++ b/gcc/analyzer/common.h
@@ -34,7 +34,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "options.h"
 #include "bitmap.h"
 #include "diagnostic-core.h"
-#include "diagnostic-path.h"
+#include "diagnostics/paths.h"
 #include "rich-location.h"
 #include "function.h"
 #include "json.h"
@@ -443,7 +443,7 @@ extern std::unique_ptr<json::value>
 tree_to_json (tree node);
 
 extern std::unique_ptr<json::value>
-diagnostic_event_id_to_json (const diagnostic_event_id_t &);
+diagnostic_event_id_to_json (const diagnostics::paths::event_id_t &);
 
 extern std::unique_ptr<json::value>
 bit_offset_to_json (const bit_offset_t &offset);
diff --git a/gcc/analyzer/diagnostic-manager.cc b/gcc/analyzer/diagnostic-manager.cc
index c083b8c..88f72d1 100644
--- a/gcc/analyzer/diagnostic-manager.cc
+++ b/gcc/analyzer/diagnostic-manager.cc
@@ -29,7 +29,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "cgraph.h"
 #include "digraph.h"
 #include "gcc-rich-location.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/sarif-sink.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/sm.h"
@@ -1017,9 +1017,10 @@ saved_diagnostic::emit_any_notes () const
    This extra data is intended for use when debugging the analyzer.  */
 
 void
-saved_diagnostic::maybe_add_sarif_properties (sarif_object &result_obj) const
+saved_diagnostic::
+maybe_add_sarif_properties (diagnostics::sarif_object &result_obj) const
 {
-  sarif_property_bag &props = result_obj.get_or_create_properties ();
+  auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/saved_diagnostic/"
   if (m_sm)
     props.set_string (PROPERTY_PREFIX "sm", m_sm->get_name ());
@@ -1044,7 +1045,7 @@ saved_diagnostic::maybe_add_sarif_properties (sarif_object &result_obj) const
       auto duplicates_arr = std::make_unique<json::array> ();
       for (auto iter : m_duplicates)
 	{
-	  auto sd_obj = std::make_unique<sarif_object> ();
+	  auto sd_obj = std::make_unique<diagnostics::sarif_object> ();
 	  iter->maybe_add_sarif_properties (*sd_obj);
 	  duplicates_arr->append (std::move (sd_obj));
 	}
@@ -1541,12 +1542,12 @@ diagnostic_manager::emit_saved_diagnostics (const exploded_graph &eg)
   best_candidates.emit_best (this, eg);
 }
 
-/* Custom subclass of diagnostic_metadata which, for SARIF output,
+/* Custom subclass of diagnostics::metadata which, for SARIF output,
    populates the property bag of the diagnostic's "result" object
    with information from the saved_diagnostic and the
    pending_diagnostic.  */
 
-class pending_diagnostic_metadata : public diagnostic_metadata
+class pending_diagnostic_metadata : public diagnostics::metadata
 {
 public:
   pending_diagnostic_metadata (const saved_diagnostic &sd)
@@ -1555,7 +1556,8 @@ public:
   }
 
   void
-  maybe_add_sarif_properties (sarif_object &result_obj) const override
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
+    const override
   {
     m_sd.maybe_add_sarif_properties (result_obj);
   }
@@ -1583,7 +1585,7 @@ diagnostic_manager::emit_saved_diagnostic (const exploded_graph &eg,
   /* Precompute all enodes from which the diagnostic is reachable.  */
   path_builder pb (eg, *epath, sd.get_feasibility_problem (), sd);
 
-  /* This is the diagnostic_path subclass that will be built for
+  /* This is the diagnostics::paths::path subclass that will be built for
      the diagnostic.  */
   checker_path emission_path (get_logical_location_manager (),
 			      eg.get_ext_state (),
@@ -1663,7 +1665,7 @@ diagnostic_manager::emit_saved_diagnostic (const exploded_graph &eg,
     }
 }
 
-const logical_location_manager &
+const diagnostics::logical_locations::manager &
 diagnostic_manager::get_logical_location_manager () const
 {
   gcc_assert (global_dc);
diff --git a/gcc/analyzer/diagnostic-manager.h b/gcc/analyzer/diagnostic-manager.h
index aa0bd79..d9cf109 100644
--- a/gcc/analyzer/diagnostic-manager.h
+++ b/gcc/analyzer/diagnostic-manager.h
@@ -67,7 +67,8 @@ public:
 
   void emit_any_notes () const;
 
-  void maybe_add_sarif_properties (sarif_object &result_obj) const;
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj) const;
 
   //private:
   const state_machine *m_sm;
@@ -191,7 +192,7 @@ public:
   }
 
 private:
-  const logical_location_manager &
+  const diagnostics::logical_locations::manager &
   get_logical_location_manager () const;
 
   void build_emission_path (const path_builder &pb,
diff --git a/gcc/analyzer/engine.cc b/gcc/analyzer/engine.cc
index 67024e9..745ef7e 100644
--- a/gcc/analyzer/engine.cc
+++ b/gcc/analyzer/engine.cc
@@ -1580,9 +1580,9 @@ exploded_node::on_stmt_pre (exploded_graph &eg,
 	  state->dump (eg.get_ext_state (), true);
 	  return;
 	}
-      else if (is_special_named_call_p (call, "__analyzer_dump_xml", 0))
+      else if (is_special_named_call_p (call, "__analyzer_dump_sarif", 0))
 	{
-	  state->dump_xml (eg.get_ext_state ());
+	  state->dump_sarif (eg.get_ext_state ());
 	  return;
 	}
       else if (is_special_named_call_p (call, "__analyzer_dump_dot", 0))
diff --git a/gcc/analyzer/infinite-loop.cc b/gcc/analyzer/infinite-loop.cc
index ec0b079..a53807c 100644
--- a/gcc/analyzer/infinite-loop.cc
+++ b/gcc/analyzer/infinite-loop.cc
@@ -25,7 +25,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-pretty-print.h"
 #include "cgraph.h"
 #include "digraph.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/sarif-sink.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/call-string.h"
@@ -307,10 +307,11 @@ public:
       }
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/infinite_loop_diagnostic/"
     props.set (PROPERTY_PREFIX "inf_loop", m_inf_loop->to_json ());
 #undef PROPERTY_PREFIX
diff --git a/gcc/analyzer/infinite-recursion.cc b/gcc/analyzer/infinite-recursion.cc
index b80b94a..960b487 100644
--- a/gcc/analyzer/infinite-recursion.cc
+++ b/gcc/analyzer/infinite-recursion.cc
@@ -25,6 +25,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-pretty-print.h"
 #include "cgraph.h"
 #include "digraph.h"
+#include "diagnostics/sarif-sink.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/call-string.h"
@@ -40,7 +41,6 @@ along with GCC; see the file COPYING3.  If not see
 #include "analyzer/exploded-graph.h"
 #include "analyzer/checker-path.h"
 #include "analyzer/feasible-graph.h"
-#include "diagnostic-format-sarif.h"
 
 /* A subclass of pending_diagnostic for complaining about suspected
    infinite recursion.  */
@@ -223,10 +223,11 @@ public:
     return false;
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/infinite_recursion_diagnostic/"
     props.set_integer (PROPERTY_PREFIX "prev_entry_enode",
 		       m_prev_entry_enode->m_index);
diff --git a/gcc/analyzer/kf.cc b/gcc/analyzer/kf.cc
index fe25520..2a7c357 100644
--- a/gcc/analyzer/kf.cc
+++ b/gcc/analyzer/kf.cc
@@ -783,7 +783,7 @@ public:
 
     /* SEI CERT C Coding Standard: "POS34-C. Do not call putenv() with a
        pointer to an automatic variable as the argument".  */
-    diagnostic_metadata::precanned_rule
+    diagnostics::metadata::precanned_rule
       rule ("POS34-C", "https://wiki.sei.cmu.edu/confluence/x/6NYxBQ");
     ctxt.add_rule (rule);
 
diff --git a/gcc/analyzer/pending-diagnostic.cc b/gcc/analyzer/pending-diagnostic.cc
index 5e95edd..cc2d795 100644
--- a/gcc/analyzer/pending-diagnostic.cc
+++ b/gcc/analyzer/pending-diagnostic.cc
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "cpplib.h"
 #include "digraph.h"
 #include "ordered-hash-map.h"
@@ -92,7 +92,7 @@ diagnostic_emission_context::warn (const char *gmsgid, ...)
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
-  const bool result = emit_diagnostic_valist_meta (DK_WARNING,
+  const bool result = emit_diagnostic_valist_meta (diagnostics::kind::warning,
 						   &m_rich_loc, &m_metadata,
 						   pd.get_controlling_option (),
 						   gmsgid, &ap);
@@ -110,7 +110,7 @@ diagnostic_emission_context::inform (const char *gmsgid, ...)
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
-  emit_diagnostic_valist_meta (DK_NOTE,
+  emit_diagnostic_valist_meta (diagnostics::kind::note,
 			       &m_rich_loc, &m_metadata,
 			       pd.get_controlling_option (),
 			       gmsgid, &ap);
diff --git a/gcc/analyzer/pending-diagnostic.h b/gcc/analyzer/pending-diagnostic.h
index 469513c..b5d90a2 100644
--- a/gcc/analyzer/pending-diagnostic.h
+++ b/gcc/analyzer/pending-diagnostic.h
@@ -21,8 +21,7 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_ANALYZER_PENDING_DIAGNOSTIC_H
 #define GCC_ANALYZER_PENDING_DIAGNOSTIC_H
 
-#include "diagnostic-metadata.h"
-#include "diagnostic-path.h"
+#include "diagnostics/metadata.h"
 #include "analyzer/sm.h"
 
 namespace ana {
@@ -44,7 +43,7 @@ struct interesting_t
 };
 
 /* Various bundles of information used for generating more precise
-   messages for events within a diagnostic_path, for passing to the
+   messages for events within a diagnostic path, for passing to the
    various "describe_*" vfuncs of pending_diagnostic.  See those
    for more information.  */
 
@@ -58,7 +57,7 @@ struct state_change
 		tree origin,
 		state_machine::state_t old_state,
 		state_machine::state_t new_state,
-		diagnostic_event_id_t event_id,
+		diagnostics::paths::event_id_t event_id,
 		const state_change_event &event)
   : m_expr (expr), m_origin (origin),
     m_old_state (old_state), m_new_state (new_state),
@@ -71,7 +70,7 @@ struct state_change
   tree m_origin;
   state_machine::state_t m_old_state;
   state_machine::state_t m_new_state;
-  diagnostic_event_id_t m_event_id;
+  diagnostics::paths::event_id_t m_event_id;
   const state_change_event &m_event;
 };
 
@@ -131,14 +130,14 @@ struct final_event
     pending_diagnostic::emit vfunc.
 
     The rich_location will have already been populated with a
-    diagnostic_path.  */
+    diagnostics::paths::path.  */
 
 class diagnostic_emission_context
 {
 public:
   diagnostic_emission_context (const saved_diagnostic &sd,
 			       rich_location &rich_loc,
-			       diagnostic_metadata &metadata,
+			       diagnostics::metadata &metadata,
 			       logger *logger)
   : m_sd (sd),
     m_rich_loc (rich_loc),
@@ -156,7 +155,7 @@ public:
   logger *get_logger () const { return m_logger; }
 
   void add_cwe (int cwe) { m_metadata.add_cwe (cwe); }
-  void add_rule (const diagnostic_metadata::rule &r)
+  void add_rule (const diagnostics::metadata::rule &r)
   {
     m_metadata.add_rule (r);
   }
@@ -164,7 +163,7 @@ public:
 private:
   const saved_diagnostic &m_sd;
   rich_location &m_rich_loc;
-  diagnostic_metadata &m_metadata;
+  diagnostics::metadata &m_metadata;
   logger *m_logger;
 };
 
@@ -182,7 +181,7 @@ private:
 
    As well as emitting a diagnostic, the class has various "precision of
    wording" virtual functions, for generating descriptions for events
-   within a diagnostic_path.  These are optional, but implementing these
+   within a diagnostic path.  These are optional, but implementing these
    allows for more precise wordings than the more generic
    implementation.  */
 
@@ -238,7 +237,7 @@ class pending_diagnostic
   virtual location_t fixup_location (location_t loc, bool primary) const;
 
   /* Precision-of-wording vfunc for describing a critical state change
-     within the diagnostic_path.
+     within the diagnostic path.
 
      For example, a double-free diagnostic might use the descriptions:
      - "first 'free' happens here"
@@ -260,13 +259,13 @@ class pending_diagnostic
     return false;
   }
 
-  /* Vfunc for implementing diagnostic_event::get_meaning for
+  /* Vfunc for implementing event::get_meaning for
      state_change_event.  */
-  virtual diagnostic_event::meaning
+  virtual diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &) const
   {
     /* Default no-op implementation.  */
-    return diagnostic_event::meaning ();
+    return diagnostics::paths::event::meaning ();
   }
 
   /* Precision-of-wording vfunc for describing an interprocedural call
@@ -285,7 +284,7 @@ class pending_diagnostic
   }
 
   /* Precision-of-wording vfunc for describing an interprocedural return
-     within the diagnostic_path that carries critial state for the
+     within the diagnostic path that carries critial state for the
      diagnostic, from callee back to caller.
 
      For example, a deref-of-unchecked-malloc diagnostic might use:
@@ -301,7 +300,7 @@ class pending_diagnostic
   }
 
   /* Precision-of-wording vfunc for describing the final event within a
-     diagnostic_path.
+     diagnostic path.
 
      For example a double-free diagnostic might use:
       - "second 'free' here; first 'free' was at (3)"
@@ -402,7 +401,8 @@ class pending_diagnostic
      the opportunity to add diagnostic-specific properties to the SARIF
      "result" object for the diagnostic.
      This is intended for use when debugging a diagnostic.  */
-  virtual void maybe_add_sarif_properties (sarif_object &/*result_obj*/) const
+  virtual void
+  maybe_add_sarif_properties (diagnostics::sarif_object &/*result_obj*/) const
   {
     /* Default no-op implementation.  */
   }
diff --git a/gcc/analyzer/program-point.cc b/gcc/analyzer/program-point.cc
index 1f82559..9baa007 100644
--- a/gcc/analyzer/program-point.cc
+++ b/gcc/analyzer/program-point.cc
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "gcc-rich-location.h"
 #include "gimple-pretty-print.h"
 #include "sbitmap.h"
@@ -234,17 +234,18 @@ function_point::before_supernode (const supernode *supernode,
   return function_point (supernode, from_edge, 0, PK_BEFORE_SUPERNODE);
 }
 
-/* A subclass of diagnostic_context for use by
+/* A subclass of diagnostics::context for use by
    program_point::print_source_line.  */
 
-class debug_diagnostic_context : public diagnostic_context
+class debug_diagnostic_context : public diagnostics::context
 {
 public:
   debug_diagnostic_context ()
   {
     diagnostic_initialize (this, 0);
-    m_source_printing.show_line_numbers_p = true;
-    m_source_printing.enabled = true;
+    auto &source_printing_opts = get_source_printing_options ();
+    source_printing_opts.show_line_numbers_p = true;
+    source_printing_opts.enabled = true;
   }
   ~debug_diagnostic_context ()
   {
@@ -263,9 +264,9 @@ function_point::print_source_line (pretty_printer *pp) const
   // TODO: monospace font
   debug_diagnostic_context tmp_dc;
   gcc_rich_location richloc (stmt->location);
-  diagnostic_source_print_policy source_policy (tmp_dc);
+  diagnostics::source_print_policy source_policy (tmp_dc);
   gcc_assert (pp);
-  source_policy.print (*pp, richloc, DK_ERROR, nullptr);
+  source_policy.print (*pp, richloc, diagnostics::kind::error, nullptr);
   pp_string (pp, pp_formatted_text (tmp_dc.get_reference_printer ()));
 }
 
diff --git a/gcc/analyzer/program-state.cc b/gcc/analyzer/program-state.cc
index c0befac..e16a50f 100644
--- a/gcc/analyzer/program-state.cc
+++ b/gcc/analyzer/program-state.cc
@@ -27,8 +27,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "gimple-iterator.h"
 #include "cgraph.h"
 #include "digraph.h"
-#include "diagnostic-event-id.h"
-#include "diagnostic-state.h"
+#include "diagnostics/event-id.h"
+#include "diagnostics/state-graphs.h"
 #include "graphviz.h"
 
 #include "text-art/tree-widget.h"
@@ -1230,8 +1230,14 @@ program_state::make_dump_widget (const text_art::dump_widget_info &dwi) const
 void
 program_state::dump_dot (const extrinsic_state &ext_state) const
 {
-  auto doc = make_xml (ext_state);
-  auto graph = make_dot_graph_from_xml_state (*doc);
+  auto state_graph = make_diagnostic_state_graph (ext_state);
+
+  gcc_assert (global_dc);
+  auto logical_loc_mgr = global_dc->get_logical_location_manager ();
+  gcc_assert (logical_loc_mgr);
+
+  auto graph = diagnostics::state_graphs::make_dot_graph (*state_graph,
+							  *logical_loc_mgr);
 
   pretty_printer pp;
   dot::writer w (pp);
diff --git a/gcc/analyzer/program-state.h b/gcc/analyzer/program-state.h
index e2076c1..4278237 100644
--- a/gcc/analyzer/program-state.h
+++ b/gcc/analyzer/program-state.h
@@ -26,8 +26,6 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/store.h"
 
-namespace xml { class document; }
-
 namespace ana {
 
 /* Data shared by all program_state instances.  */
@@ -248,11 +246,14 @@ public:
   void dump (const extrinsic_state &ext_state, bool simple) const;
   void dump () const;
 
-  std::unique_ptr<xml::document> make_xml (const extrinsic_state &ext_state) const;
-  void dump_xml_to_pp (const extrinsic_state &ext_state, pretty_printer *pp) const;
-  void dump_xml_to_file (const extrinsic_state &ext_state, FILE *outf) const;
-  void dump_xml (const extrinsic_state &ext_state) const;
-  void dump_dot (const extrinsic_state &ext_state) const;
+  std::unique_ptr<diagnostics::digraphs::digraph>
+  make_diagnostic_state_graph (const extrinsic_state &ext_state) const;
+
+  void
+  dump_sarif (const extrinsic_state &ext_state) const;
+
+  void
+  dump_dot (const extrinsic_state &ext_state) const;
 
   std::unique_ptr<json::object>
   to_json (const extrinsic_state &ext_state) const;
diff --git a/gcc/analyzer/region-model-asm.cc b/gcc/analyzer/region-model-asm.cc
index d280ae2..fe70490 100644
--- a/gcc/analyzer/region-model-asm.cc
+++ b/gcc/analyzer/region-model-asm.cc
@@ -171,7 +171,8 @@ region_model::on_asm_stmt (const gasm *stmt, region_model_context *ctxt)
 	 no point in going further.  */
       constraint = constraints[i];
       if (!parse_output_constraint (&constraint, i, ninputs, noutputs,
-				    &allows_mem, &allows_reg, &is_inout))
+				    &allows_mem, &allows_reg, &is_inout,
+				    nullptr))
 	{
 	  if (logger)
 	    logger->log ("error parsing constraint for output %i: %qs",
@@ -204,8 +205,8 @@ region_model::on_asm_stmt (const gasm *stmt, region_model_context *ctxt)
       const char *constraint = constraints[i + noutputs];
       bool allows_reg, allows_mem;
       if (! parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
-				    constraints.address (),
-				    &allows_mem, &allows_reg))
+				    constraints.address (), &allows_mem,
+				    &allows_reg, nullptr))
 	{
 	  if (logger)
 	    logger->log ("error parsing constraint for input %i: %qs",
diff --git a/gcc/analyzer/region-model.cc b/gcc/analyzer/region-model.cc
index 6df3842..618d96b 100644
--- a/gcc/analyzer/region-model.cc
+++ b/gcc/analyzer/region-model.cc
@@ -26,7 +26,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "cgraph.h"
 #include "cfg.h"
 #include "sbitmap.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "stor-layout.h"
 #include "stringpool.h"
 #include "attribs.h"
@@ -40,7 +40,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "is-a.h"
 #include "gcc-rich-location.h"
 #include "gcc-urlifier.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/sarif-sink.h"
 #include "tree-pretty-print.h"
 #include "fold-const.h"
 #include "selftest-tree.h"
@@ -907,9 +907,10 @@ public:
   }
 
   void
-  maybe_add_sarif_properties (sarif_object &result_obj) const final override
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
+    const final override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/poisoned_value_diagnostic/"
     props.set (PROPERTY_PREFIX "expr", tree_to_json (m_expr));
     props.set_string (PROPERTY_PREFIX "kind", poison_kind_to_str (m_pkind));
@@ -3835,10 +3836,11 @@ public:
     interest->add_region_creation (m_rhs);
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/dubious_allocation_size/"
     props.set (PROPERTY_PREFIX "lhs", m_lhs->to_json ());
     props.set (PROPERTY_PREFIX "rhs", m_rhs->to_json ());
@@ -7595,9 +7597,10 @@ public:
   }
 
   void
-  maybe_add_sarif_properties (sarif_object &result_obj) const final override
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
+    const final override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/-Wanalyzer-exposure-through-uninit-copy/"
     props.set (PROPERTY_PREFIX "src_region", m_src_region->to_json ());
     props.set (PROPERTY_PREFIX "dest_region", m_dest_region->to_json ());
diff --git a/gcc/analyzer/sm-fd.cc b/gcc/analyzer/sm-fd.cc
index 370d7a0..b8ae0a1 100644
--- a/gcc/analyzer/sm-fd.cc
+++ b/gcc/analyzer/sm-fd.cc
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "stringpool.h"
 #include "attribs.h"
 
@@ -392,21 +392,22 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (
       const evdesc::state_change &change) const final override
   {
+    using event = diagnostics::paths::event;
     if (change.m_old_state == m_sm.get_start_state ()
 	&& (m_sm.is_unchecked_fd_p (change.m_new_state)
 	    || change.m_new_state == m_sm.m_new_datagram_socket
 	    || change.m_new_state == m_sm.m_new_stream_socket
 	    || change.m_new_state == m_sm.m_new_unknown_socket))
-      return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					diagnostic_event::noun::resource);
+      return event::meaning (event::verb::acquire,
+			     event::noun::resource);
     if (change.m_new_state == m_sm.m_closed)
-      return diagnostic_event::meaning (diagnostic_event::verb::release,
-					diagnostic_event::noun::resource);
-    return diagnostic_event::meaning ();
+      return event::meaning (event::verb::release,
+			     event::noun::resource);
+    return event::meaning ();
   }
 
 protected:
@@ -561,7 +562,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_open_event;
+  diagnostics::paths::event_id_t m_open_event;
   std::unique_ptr<program_state> m_final_state;
 };
 
@@ -705,7 +706,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_first_close_event;
+  diagnostics::paths::event_id_t m_first_close_event;
 };
 
 class fd_use_after_close : public fd_param_diagnostic
@@ -784,7 +785,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_first_close_event;
+  diagnostics::paths::event_id_t m_first_close_event;
 };
 
 class fd_use_without_check : public fd_param_diagnostic
@@ -854,7 +855,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_first_open_event;
+  diagnostics::paths::event_id_t m_first_open_event;
 };
 
 /* Concrete pending_diagnostic subclass for -Wanalyzer-fd-phase-mismatch.  */
diff --git a/gcc/analyzer/sm-file.cc b/gcc/analyzer/sm-file.cc
index 4b1fc77..c852c02 100644
--- a/gcc/analyzer/sm-file.cc
+++ b/gcc/analyzer/sm-file.cc
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "selftest.h"
 
 #include "analyzer/analyzer-logging.h"
@@ -155,18 +155,20 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &change)
     const final override
   {
+    using event = diagnostics::paths::event;
+
     if (change.m_old_state == m_sm.get_start_state ()
 	&& change.m_new_state == m_sm.m_unchecked)
-      return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					diagnostic_event::noun::resource);
+      return event::meaning (event::verb::acquire,
+			     event::noun::resource);
     if (change.m_new_state == m_sm.m_closed)
-      return diagnostic_event::meaning (diagnostic_event::verb::release,
-					diagnostic_event::noun::resource);
-    return diagnostic_event::meaning ();
+      return event::meaning (event::verb::release,
+			     event::noun::resource);
+    return event::meaning ();
   }
 
 protected:
@@ -225,7 +227,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_first_fclose_event;
+  diagnostics::paths::event_id_t m_first_fclose_event;
 };
 
 class file_leak : public file_diagnostic
@@ -303,7 +305,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_fopen_event;
+  diagnostics::paths::event_id_t m_fopen_event;
   std::unique_ptr<program_state> m_final_state;
 };
 
diff --git a/gcc/analyzer/sm-malloc.cc b/gcc/analyzer/sm-malloc.cc
index 3581dbb..a6b1421 100644
--- a/gcc/analyzer/sm-malloc.cc
+++ b/gcc/analyzer/sm-malloc.cc
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "stringpool.h"
 #include "attribs.h"
 #include "xml-printer.h"
@@ -436,9 +436,9 @@ public:
       const extrinsic_state &ext_state) const;
 
   void
-  add_state_to_xml (xml_state &out_xml,
-		    const svalue &sval,
-		    state_machine::state_t state) const final override;
+  add_state_to_state_graph (analyzer_state_graph &out_state_graph,
+			    const svalue &sval,
+			    state_machine::state_t state) const final override;
 
   standard_deallocator_set m_free;
   standard_deallocator_set m_scalar_delete;
@@ -814,18 +814,18 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &change)
     const final override
   {
     if (change.m_old_state == m_sm.get_start_state ()
 	&& unchecked_p (change.m_new_state))
-      return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					diagnostic_event::noun::memory);
+      return diagnostics::paths::event::meaning (diagnostics::paths::event::verb::acquire,
+					diagnostics::paths::event::noun::memory);
     if (freed_p (change.m_new_state))
-      return diagnostic_event::meaning (diagnostic_event::verb::release,
-					diagnostic_event::noun::memory);
-    return diagnostic_event::meaning ();
+      return diagnostics::paths::event::meaning (diagnostics::paths::event::verb::release,
+					diagnostics::paths::event::noun::memory);
+    return diagnostics::paths::event::meaning ();
   }
 
 protected:
@@ -918,7 +918,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_alloc_event;
+  diagnostics::paths::event_id_t m_alloc_event;
   const deallocator_set *m_expected_deallocators;
   const deallocator *m_actual_dealloc;
 };
@@ -988,7 +988,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_first_free_event;
+  diagnostics::paths::event_id_t m_first_free_event;
   const char *m_funcname;
 };
 
@@ -1031,7 +1031,7 @@ public:
   }
 
 protected:
-  diagnostic_event_id_t m_origin_of_unchecked_event;
+  diagnostics::paths::event_id_t m_origin_of_unchecked_event;
 };
 
 /* Concrete subclass for describing dereference of a possible NULL
@@ -1419,7 +1419,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_free_event;
+  diagnostics::paths::event_id_t m_free_event;
   const deallocator *m_deallocator;
 };
 
@@ -1500,7 +1500,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_alloc_event;
+  diagnostics::paths::event_id_t m_alloc_event;
   std::unique_ptr<program_state> m_final_state;
 };
 
@@ -1768,7 +1768,7 @@ private:
     return result;
   }
 
-  diagnostic_event_id_t m_first_deref_event;
+  diagnostics::paths::event_id_t m_first_deref_event;
   const exploded_node *m_deref_enode;
   tree m_deref_expr;
   const exploded_node *m_check_enode;
@@ -2735,27 +2735,57 @@ malloc_state_machine::transition_ptr_sval_non_null (region_model *model,
   smap->set_state (model, new_ptr_sval, m_free.m_nonnull, nullptr, ext_state);
 }
 
+static enum diagnostics::state_graphs::node_dynalloc_state
+get_dynalloc_state_for_state (enum resource_state rs)
+{
+  switch (rs)
+    {
+    default:
+      gcc_unreachable ();
+    case RS_START:
+    case RS_NULL:
+    case RS_NON_HEAP:
+    case RS_STOP:
+      return diagnostics::state_graphs::node_dynalloc_state::unknown;
+
+    case RS_ASSUMED_NON_NULL:
+      return diagnostics::state_graphs::node_dynalloc_state::nonnull;
+
+    case RS_UNCHECKED:
+      return diagnostics::state_graphs::node_dynalloc_state::unchecked;
+    case RS_NONNULL:
+      return diagnostics::state_graphs::node_dynalloc_state::nonnull;
+    case RS_FREED:
+      return diagnostics::state_graphs::node_dynalloc_state::freed;
+    }
+}
+
 void
-malloc_state_machine::add_state_to_xml (xml_state &out_xml,
-					const svalue &sval,
-					state_machine::state_t state) const
+malloc_state_machine::
+add_state_to_state_graph (analyzer_state_graph &out_state_graph,
+			  const svalue &sval,
+			  state_machine::state_t state) const
 {
   if (const region *reg = sval.maybe_get_region ())
     {
-      auto &reg_element = out_xml.get_or_create_element (*reg);
+      auto reg_node = out_state_graph.get_or_create_state_node (*reg);
       auto alloc_state = as_a_allocation_state (state);
       gcc_assert (alloc_state);
 
-      reg_element.set_attr ("dynamic-alloc-state", state->get_name ());
+      reg_node.set_dynalloc_state
+	(get_dynalloc_state_for_state (alloc_state->m_rs));
       if (alloc_state->m_deallocators)
 	{
 	  pretty_printer pp;
 	  alloc_state->m_deallocators->dump_to_pp (&pp);
-	  reg_element.set_attr ("expected-deallocators", pp_formatted_text (&pp));
+	  reg_node.m_node.set_attr (STATE_NODE_PREFIX,
+				    "expected-deallocators",
+				    pp_formatted_text (&pp));
 	}
       if (alloc_state->m_deallocator)
-	reg_element.set_attr ("deallocator",
-			      alloc_state->m_deallocator->m_name);
+	reg_node.m_node.set_attr (STATE_NODE_PREFIX,
+				  "deallocator",
+				  alloc_state->m_deallocator->m_name);
     }
 }
 
diff --git a/gcc/analyzer/sm-pattern-test.cc b/gcc/analyzer/sm-pattern-test.cc
index 02b32ac..12449a1 100644
--- a/gcc/analyzer/sm-pattern-test.cc
+++ b/gcc/analyzer/sm-pattern-test.cc
@@ -23,7 +23,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "analyzer/common.h"
 
 #include "tree-pretty-print.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/sm.h"
diff --git a/gcc/analyzer/sm-sensitive.cc b/gcc/analyzer/sm-sensitive.cc
index f8fcded..4611b10e 100644
--- a/gcc/analyzer/sm-sensitive.cc
+++ b/gcc/analyzer/sm-sensitive.cc
@@ -21,7 +21,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 
 #include "analyzer/analyzer-logging.h"
 #include "analyzer/sm.h"
@@ -106,14 +106,15 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &change)
     const final override
   {
+    using event = diagnostics::paths::event;
+
     if (change.m_new_state == m_sm.m_sensitive)
-      return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					diagnostic_event::noun::sensitive);
-    return diagnostic_event::meaning ();
+      return event::meaning (event::verb::acquire, event::noun::sensitive);
+    return event::meaning ();
   }
   bool
   describe_call_with_state (pretty_printer &pp,
@@ -162,7 +163,7 @@ public:
 private:
   const sensitive_state_machine &m_sm;
   tree m_arg;
-  diagnostic_event_id_t m_first_sensitive_event;
+  diagnostics::paths::event_id_t m_first_sensitive_event;
 };
 
 /* sensitive_state_machine's ctor.  */
diff --git a/gcc/analyzer/sm-signal.cc b/gcc/analyzer/sm-signal.cc
index 5a4b384..35ecde1 100644
--- a/gcc/analyzer/sm-signal.cc
+++ b/gcc/analyzer/sm-signal.cc
@@ -22,7 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "analyzer/common.h"
 
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "sbitmap.h"
 #include "ordered-hash-map.h"
 #include "selftest.h"
diff --git a/gcc/analyzer/sm-taint.cc b/gcc/analyzer/sm-taint.cc
index 5c8cc7e..f2a94e8 100644
--- a/gcc/analyzer/sm-taint.cc
+++ b/gcc/analyzer/sm-taint.cc
@@ -30,7 +30,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "stringpool.h"
 #include "attribs.h"
 #include "fold-const.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/sarif-sink.h"
 #include "gcc-urlifier.h"
 
 #include "analyzer/analyzer-logging.h"
@@ -209,20 +209,22 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &change)
     const final override
   {
+    using event = diagnostics::paths::event;
     if (change.m_new_state == m_sm.m_tainted)
-      return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					diagnostic_event::noun::taint);
-    return diagnostic_event::meaning ();
+      return event::meaning (event::verb::acquire,
+			     event::noun::taint);
+    return event::meaning ();
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const override
   {
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/taint_diagnostic/"
     props.set (PROPERTY_PREFIX "arg", tree_to_json (m_arg));
     props.set_string (PROPERTY_PREFIX "has_bounds",
@@ -495,11 +497,12 @@ public:
 	}
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
     taint_diagnostic::maybe_add_sarif_properties (result_obj);
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/tainted_offset/"
     props.set (PROPERTY_PREFIX "offset", m_offset->to_json ());
 #undef PROPERTY_PREFIX
@@ -864,11 +867,12 @@ public:
 	}
   }
 
-  void maybe_add_sarif_properties (sarif_object &result_obj)
+  void
+  maybe_add_sarif_properties (diagnostics::sarif_object &result_obj)
     const final override
   {
     taint_diagnostic::maybe_add_sarif_properties (result_obj);
-    sarif_property_bag &props = result_obj.get_or_create_properties ();
+    auto &props = result_obj.get_or_create_properties ();
 #define PROPERTY_PREFIX "gcc/analyzer/tainted_allocation_size/"
     props.set (PROPERTY_PREFIX "size_in_bytes", m_size_in_bytes->to_json ());
 #undef PROPERTY_PREFIX
diff --git a/gcc/analyzer/sm.cc b/gcc/analyzer/sm.cc
index 840806a..c93e9c2 100644
--- a/gcc/analyzer/sm.cc
+++ b/gcc/analyzer/sm.cc
@@ -161,16 +161,16 @@ state_machine::to_json () const
 }
 
 void
-state_machine::add_state_to_xml (xml_state &out_xml,
-				 const svalue &sval,
-				 state_machine::state_t state) const
+state_machine::add_state_to_state_graph (analyzer_state_graph &out_state_graph,
+					 const svalue &sval,
+					 state_machine::state_t state) const
 {
   // no-op
 }
 
 void
-state_machine::add_global_state_to_xml (xml_state &out_xml,
-					state_machine::state_t state) const
+state_machine::add_global_state_to_state_graph (analyzer_state_graph &out_state_graph,
+						state_machine::state_t state) const
 {
   // no-op
 }
diff --git a/gcc/analyzer/sm.h b/gcc/analyzer/sm.h
index 6298fb6..4633fac 100644
--- a/gcc/analyzer/sm.h
+++ b/gcc/analyzer/sm.h
@@ -28,7 +28,7 @@ namespace ana {
 class state_machine;
 class sm_context;
 class pending_diagnostic;
-class xml_state;
+class analyzer_state_graph;
 
 extern bool any_pointer_p (tree expr);
 extern bool any_pointer_p (const svalue *sval);
@@ -188,13 +188,13 @@ public:
   state_t get_start_state () const { return m_start; }
 
   virtual void
-  add_state_to_xml (xml_state &out_xml,
-		    const svalue &sval,
-		    state_machine::state_t state) const;
+  add_state_to_state_graph (analyzer_state_graph &out_state_graph,
+			    const svalue &sval,
+			    state_machine::state_t state) const;
 
   virtual void
-  add_global_state_to_xml (xml_state &out_xml,
-			   state_machine::state_t state) const;
+  add_global_state_to_state_graph (analyzer_state_graph &out_state_graph,
+				   state_machine::state_t state) const;
 
 protected:
   state_t add_state (const char *name);
diff --git a/gcc/analyzer/varargs.cc b/gcc/analyzer/varargs.cc
index 4947271..1fdfd34 100644
--- a/gcc/analyzer/varargs.cc
+++ b/gcc/analyzer/varargs.cc
@@ -334,17 +334,17 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &change)
     const final override
   {
     if (change.m_new_state == m_sm.m_started)
-      return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					diagnostic_event::noun::resource);
+      return diagnostics::paths::event::meaning (diagnostics::paths::event::verb::acquire,
+					diagnostics::paths::event::noun::resource);
     if (change.m_new_state == m_sm.m_ended)
-      return diagnostic_event::meaning (diagnostic_event::verb::release,
-					diagnostic_event::noun::resource);
-    return diagnostic_event::meaning ();
+      return diagnostics::paths::event::meaning (diagnostics::paths::event::verb::release,
+					diagnostics::paths::event::noun::resource);
+    return diagnostics::paths::event::meaning ();
   }
 
 protected:
@@ -452,7 +452,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_va_end_event;
+  diagnostics::paths::event_id_t m_va_end_event;
   const char *m_usage_fnname;
 };
 
@@ -539,7 +539,7 @@ public:
   }
 
 private:
-  diagnostic_event_id_t m_start_event;
+  diagnostics::paths::event_id_t m_start_event;
   const char *m_start_event_fnname;
   std::unique_ptr<program_state> m_final_state;
 };
diff --git a/gcc/attribs.cc b/gcc/attribs.cc
index f666783..3fce9d6 100644
--- a/gcc/attribs.cc
+++ b/gcc/attribs.cc
@@ -1076,21 +1076,15 @@ apply_tm_attr (tree fndecl, tree attr)
    it to CHAIN.  */
 
 tree
-make_attribute (const char *name, const char *arg_name, tree chain)
+make_attribute (string_slice name, string_slice arg_name, tree chain)
 {
-  tree attr_name;
-  tree attr_arg_name;
-  tree attr_args;
-  tree attr;
-
-  attr_name = get_identifier (name);
-  attr_arg_name = build_string (strlen (arg_name), arg_name);
-  attr_args = tree_cons (NULL_TREE, attr_arg_name, NULL_TREE);
-  attr = tree_cons (attr_name, attr_args, chain);
+  tree attr_name = get_identifier_with_length (name.begin (), name.size ());
+  tree attr_arg_name = build_string (arg_name.size (), arg_name.begin ());
+  tree attr_args = tree_cons (NULL_TREE, attr_arg_name, NULL_TREE);
+  tree attr = tree_cons (attr_name, attr_args, chain);
   return attr;
 }
 
-
 /* Common functions used for target clone support.  */
 
 /* Comparator function to be used in qsort routine to sort attribute
diff --git a/gcc/attribs.h b/gcc/attribs.h
index 4b94639..b8b6838 100644
--- a/gcc/attribs.h
+++ b/gcc/attribs.h
@@ -45,7 +45,7 @@ extern bool cxx11_attribute_p (const_tree);
 extern tree get_attribute_name (const_tree);
 extern tree get_attribute_namespace (const_tree);
 extern void apply_tm_attr (tree, tree);
-extern tree make_attribute (const char *, const char *, tree);
+extern tree make_attribute (string_slice, string_slice, tree);
 extern bool attribute_ignored_p (tree);
 extern bool attribute_ignored_p (const attribute_spec *const);
 extern bool any_nonignored_attribute_p (tree);
diff --git a/gcc/auto-obstack.h b/gcc/auto-obstack.h
new file mode 100644
index 0000000..fe16dbd
--- /dev/null
+++ b/gcc/auto-obstack.h
@@ -0,0 +1,58 @@
+/* RAII wrapper around obstack.
+   Copyright (C) 2024-2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_AUTO_OBSTACK_H
+#define GCC_AUTO_OBSTACK_H
+
+/* RAII wrapper around obstack.  */
+
+struct auto_obstack
+{
+  auto_obstack ()
+  {
+    obstack_init (&m_obstack);
+  }
+
+  ~auto_obstack ()
+  {
+    obstack_free (&m_obstack, NULL);
+  }
+
+  operator obstack & () { return m_obstack; }
+
+  void grow (const void *src, size_t length)
+  {
+    obstack_grow (&m_obstack, src, length);
+  }
+
+  void *object_base () const
+  {
+    return m_obstack.object_base;
+  }
+
+  size_t object_size () const
+  {
+    return obstack_object_size (&m_obstack);
+  }
+
+  obstack m_obstack;
+};
+
+#endif /* GCC_AUTO_OBSTACK_H */
diff --git a/gcc/auto-profile.cc b/gcc/auto-profile.cc
index 64f4cda..7ff9526 100644
--- a/gcc/auto-profile.cc
+++ b/gcc/auto-profile.cc
@@ -53,6 +53,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "auto-profile.h"
 #include "tree-pretty-print.h"
 #include "gimple-pretty-print.h"
+#include "output.h"
 
 /* The following routines implements AutoFDO optimization.
 
@@ -239,6 +240,8 @@ public:
   /* Add new name and return its index.  */
   int add_name (char *);
 
+  /* Return cgraph node corresponding to given name index.  */
+  cgraph_node *get_cgraph_node (int);
 private:
   typedef std::map<const char *, unsigned, string_compare> string_index_map;
   string_vector vector_;
@@ -430,7 +433,8 @@ public:
   void
   set_call_location (location_t l)
   {
-    gcc_checking_assert (call_location_ == UNKNOWN_LOCATION);
+    gcc_checking_assert (call_location_ == UNKNOWN_LOCATION
+			 && l != UNKNOWN_LOCATION);
     call_location_= l;
   }
 
@@ -443,7 +447,6 @@ public:
   /* Lookup count and warn about duplicates.  */
   count_info *lookup_count (location_t loc, inline_stack &stack,
 			    cgraph_node *node);
-
 private:
   /* Callsite, represented as (decl_lineno, callee_function_name_index).  */
   typedef std::pair<unsigned, unsigned> callsite;
@@ -622,9 +625,11 @@ get_original_name (const char *name, bool alloc = true)
     }
   /* Suffixes of clones that compiler generates after auto-profile.  */
   const char *suffixes[] = {"isra", "constprop", "lto_priv", "part", "cold"};
-  for (unsigned i = 0; i < sizeof (suffixes); ++i)
+  for (unsigned i = 0; i < sizeof (suffixes) / sizeof (const char *); ++i)
     {
-      if (strncmp (next_dot + 1, suffixes[i], strlen (suffixes[i])) == 0)
+      int len = strlen (suffixes[i]);
+      if (len == last_dot - next_dot - 1
+	  && strncmp (next_dot + 1, suffixes[i], strlen (suffixes[i])) == 0)
 	{
 	  *next_dot = 0;
 	  return get_original_name (ret, false);
@@ -683,6 +688,26 @@ dump_afdo_loc (FILE *f, unsigned loc)
     fprintf (f, "%i", loc >> 16);
 }
 
+/* Return assembler name as in symbol table and DW_AT_linkage_name.  */
+
+static const char *
+raw_symbol_name (const char *asmname)
+{
+  /* If we start supporting user_label_prefixes, add_linkage_attr will also
+     need to be fixed.  */
+  if (strlen (user_label_prefix))
+    sorry ("auto-profile is not supported for targets with user label prefix");
+  return asmname + (asmname[0] == '*');
+}
+
+/* Convenience wrapper that looks up assembler name.  */
+
+static const char *
+raw_symbol_name (tree decl)
+{
+  return raw_symbol_name (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)));
+}
+
 /* Dump STACK to F.  */
 
 static void
@@ -693,7 +718,7 @@ dump_inline_stack (FILE *f, inline_stack *stack)
     {
       fprintf (f, "%s%s:",
 	       first ? "" : "; ",
-	       IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (p.decl)));
+	       raw_symbol_name (p.decl));
       dump_afdo_loc (f, p.afdo_loc);
       first = false;
     }
@@ -815,7 +840,7 @@ string_table::get_index (const char *name) const
 int
 string_table::get_index_by_decl (tree decl) const
 {
-  const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
+  const char *name = raw_symbol_name (decl);
   int ret = get_index (name);
   if (ret != -1)
     return ret;
@@ -860,10 +885,35 @@ string_table::read ()
     {
       vector_.quick_push (xstrdup (gcov_read_string ()));
       map_[vector_.last ()] = i;
+      if (gcov_is_error ())
+	return false;
     }
   return true;
 }
 
+/* Return cgraph node corresponding to given NAME_INDEX,
+   NULL if unavailable.  */
+cgraph_node *
+string_table::get_cgraph_node (int name_index)
+{
+  const char *sname = get_name (name_index);
+
+  symtab_node *n = cgraph_node::get_for_asmname (get_identifier (sname));
+  for (;n; n = n->next_sharing_asm_name)
+    if (cgraph_node *cn = dyn_cast <cgraph_node *> (n))
+      if (cn->definition && cn->has_gimple_body_p ())
+	return cn;
+  return NULL;
+}
+
+/* Return corresponding cgraph node.  */
+
+cgraph_node *
+function_instance::get_cgraph_node ()
+{
+  return afdo_string_table->get_cgraph_node (name ());
+}
+
 /* Member functions for function_instance.  */
 
 function_instance::~function_instance ()
@@ -874,20 +924,6 @@ function_instance::~function_instance ()
     delete iter->second;
 }
 
-/* Return corresponding cgraph node, NULL if unavailable.  */
-cgraph_node *
-function_instance::get_cgraph_node ()
-{
-  for (symtab_node *n = cgraph_node::get_for_asmname
-			    (get_identifier
-			       (afdo_string_table->get_name (name ())));
-       n; n = n->next_sharing_asm_name)
-    if (cgraph_node *cn = dyn_cast <cgraph_node *> (n))
-      if (cn->definition && cn->has_gimple_body_p ())
-	return cn;
-  return NULL;
-}
-
 /* Traverse callsites of the current function_instance to find one at the
    location of LINENO and callee name represented in DECL.  */
 
@@ -919,10 +955,10 @@ function_instance::get_function_instance_by_decl (unsigned lineno,
 	  dump_printf_loc (MSG_NOTE | MSG_PRIORITY_INTERNALS,
 			   dump_user_location_t::from_location_t (location),
 			   "auto-profile has mismatched function name %s"
-			   " instaed of %s at loc %i:%i",
+			   " insteed of %s at loc %i:%i",
 			   afdo_string_table->get_name (iter.first.second),
-			   IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)),
-			   lineno << 16,
+			   raw_symbol_name (decl),
+			   lineno >> 16,
 			   lineno & 65535);
     }
 
@@ -1125,10 +1161,13 @@ function_instance::offline_if_in_set (name_index_set &seen,
    Return non-zero if it correspons and 2 if renaming was done.  */
 
 static int
-match_with_target (gimple *stmt, function_instance *inlined_fn, cgraph_node *n)
+match_with_target (cgraph_node *n,
+		   gimple *stmt,
+		   function_instance *inlined_fn,
+		   cgraph_node *orig_callee)
 {
-  const char *symbol_name = IDENTIFIER_POINTER
-			       (DECL_ASSEMBLER_NAME (n->decl));
+  cgraph_node *callee = orig_callee->ultimate_alias_target ();
+  const char *symbol_name = raw_symbol_name (callee->decl);
   const char *name = afdo_string_table->get_name (inlined_fn->name ());
   if (strcmp (name, symbol_name))
     {
@@ -1142,7 +1181,7 @@ match_with_target (gimple *stmt, function_instance *inlined_fn, cgraph_node *n)
 	    in_suffix = true;
 	}
       /* Accept dwarf names and stripped suffixes.  */
-      if (!strcmp (lang_hooks.dwarf_name (n->decl, 0),
+      if (!strcmp (lang_hooks.dwarf_name (callee->decl, 0),
 		   afdo_string_table->get_name (inlined_fn->name ()))
 	  || (!name[i] && symbol_name[i] == '.')
 	  || in_suffix)
@@ -1157,10 +1196,14 @@ match_with_target (gimple *stmt, function_instance *inlined_fn, cgraph_node *n)
 	  inlined_fn->set_name (index);
 	  return 2;
 	}
-      warning_at (gimple_location (stmt), OPT_Wauto_profile,
-		  "auto-profile contains inlined "
-		  "function with symbol name %s instead of symbol name %s",
-		  name, symbol_name);
+      /* Only warn about declarations.  It is possible that the function
+	 is declared as alias in other module and we inlined cross-module.  */
+      if (callee->definition
+	  && warning (OPT_Wauto_profile,
+		      "auto-profile of %q+F contains inlined "
+		      "function with symbol name %s instead of symbol name %s",
+		      n->decl, name, symbol_name))
+	inform (gimple_location (stmt), "corresponding call");
       return 0;
     }
   return 1;
@@ -1203,13 +1246,14 @@ function_instance::lookup_count (location_t loc, inline_stack &stack,
   if (stack.length ())
     {
       int c = pos_counts.count (stack[0].afdo_loc);
-      if (c > 1)
-	warning_at (loc, OPT_Wauto_profile,
-		    "duplicated count information"
-		    " in auto-profile of %q+F"
-		    " with relative location %i discriminator %i",
-		    node->decl, stack[0].afdo_loc >> 16,
-		    stack[0].afdo_loc & 65535);
+      if (c > 1
+	  && warning (OPT_Wauto_profile,
+		      "duplicated count information"
+		      " in auto-profile of %q+F"
+		      " with relative location %i discriminator %i",
+		      node->decl, stack[0].afdo_loc >> 16,
+		      stack[0].afdo_loc & 65535))
+	  inform (loc, "corresponding source location");
       if (c)
 	return &pos_counts[stack[0].afdo_loc];
     }
@@ -1271,6 +1315,7 @@ function_instance::match (cgraph_node *node,
   hash_set<const count_info *> counts;
   hash_set<const count_info *> targets;
   hash_set<const function_instance *> functions;
+  hash_set<const function_instance *> functions_to_offline;
 
   /* We try to fill in lost disciminator if there is unique call
      with given line number.  This map is used to record them.  */
@@ -1374,20 +1419,23 @@ function_instance::match (cgraph_node *node,
 			  inlined_fn_nodisc = iter.second;
 			cnodis++;
 		      }
-		  if (c > 1 || cnodis > 1)
-		    warning_at (gimple_location (stmt), OPT_Wauto_profile,
-				"duplicated callsite in auto-profile of %q+F"
-				" with relative location %i, discriminator %i",
-				node->decl, stack[0].afdo_loc >> 16,
-				stack[0].afdo_loc & 65535);
-		  if (inlined_fn && info && info->targets.size ())
-		    warning_at (gimple_location (stmt), OPT_Wauto_profile,
-				"both call targets and inline callsite"
-				" information is present in auto-profile"
-				" of function %q+F with relative location"
-				" %i, discriminator %i",
-				node->decl, stack[0].afdo_loc >> 16,
-				stack[0].afdo_loc & 65535);
+		  if ((c > 1 || (!c && cnodis > 1))
+		      && warning (OPT_Wauto_profile,
+				  "duplicated callsite in auto-profile of %q+F"
+				  " with relative location %i,"
+				  " discriminator %i",
+				  node->decl, stack[0].afdo_loc >> 16,
+				  stack[0].afdo_loc & 65535))
+		    inform (gimple_location (stmt), "corresponding call");
+		  if (inlined_fn && info && info->targets.size ()
+		      && warning (OPT_Wauto_profile,
+				  "both call targets and inline callsite"
+				  " information is present in auto-profile"
+				  " of function %q+F with relative location"
+				  " %i, discriminator %i",
+				  node->decl, stack[0].afdo_loc >> 16,
+				  stack[0].afdo_loc & 65535))
+		    inform (gimple_location (stmt), "corresponding call");
 		  tree callee = gimple_call_fndecl (stmt);
 		  cgraph_node *callee_node;
 		  unsigned int loc = stack[0].afdo_loc;
@@ -1420,11 +1468,17 @@ function_instance::match (cgraph_node *node,
 		      if (lineno_to_call.get
 			      (stack[0].afdo_loc >> 16)->length () == 1)
 			{
-			  warning_at (gimple_location (stmt), OPT_Wauto_profile,
-				      "auto-profile of %q+F seem to contain"
-				      " lost discriminator %i for call at"
-				      " relative location %i",
-				      node->decl, loc & 65535, loc >> 16);
+			  if (warning (OPT_Wauto_profile,
+				       "auto-profile of %q+F seem to contain"
+				       " lost discriminator %i for"
+				       " call of %s at relative location %i",
+				       node->decl,
+				       loc & 65535,
+				       afdo_string_table->get_name
+					 (inlined_fn_nodisc->name ()),
+				       loc >> 16))
+			    inform (gimple_location (stmt),
+				    "corresponding call");
 			  inlined_fn = inlined_fn_nodisc;
 			  if (dump_file)
 			    fprintf (dump_file, "   Lost discriminator %i\n",
@@ -1435,11 +1489,10 @@ function_instance::match (cgraph_node *node,
 		    }
 		  if (callee && (callee_node = cgraph_node::get (callee)))
 		    {
-		      callee_node = callee_node->ultimate_alias_target ();
 		      if (inlined_fn)
 			{
 			  int old_name = inlined_fn->name ();
-			  int r = match_with_target (stmt, inlined_fn,
+			  int r = match_with_target (node, stmt, inlined_fn,
 						     callee_node);
 			  if (r == 2)
 			    {
@@ -1451,11 +1504,13 @@ function_instance::match (cgraph_node *node,
 						      == inlined_fn);
 			      callsite key2 = {stack[0].afdo_loc,
 						inlined_fn->name ()};
-			      callsites[key2] = inlined_fn;
 			      callsites.erase (iter);
+			      callsites[key2] = inlined_fn;
 			    }
 			  if (r)
 			    functions.add (inlined_fn);
+			  else
+			    functions_to_offline.add (inlined_fn);
 			}
 
 		      if (info && info->targets.size () > 1)
@@ -1473,19 +1528,24 @@ function_instance::match (cgraph_node *node,
 		    {
 		      if (inlined_fn
 			  && inlined_fn->get_call_location ()
-				  != UNKNOWN_LOCATION
-			  && warning_at (gimple_location (stmt),
-					 OPT_Wauto_profile,
-					 "%q+F contains two calls of the same"
-					 " relative location +%i,"
-					 " discrimnator %i,"
-					 " that leads to lost auto-profile",
-					 node->decl,
-					 loc << 16,
-					 loc & 65535))
+				  != UNKNOWN_LOCATION)
 			{
-			  inform (inlined_fn->get_call_location (),
-				  "location of the earlier call");
+			  if (warning (OPT_Wauto_profile,
+				       "function contains two calls of the same"
+				       " relative location +%i,"
+				       " discrimnator %i,"
+				       " that leads to lost auto-profile",
+				       loc >> 16,
+				       loc & 65535))
+			    {
+			      inform (gimple_location (stmt),
+				      "location of the first call");
+			      inform (inlined_fn->get_call_location (),
+				      "location of the second call");
+			    }
+			  if (dump_file)
+			    fprintf (dump_file,
+				     "   Duplicated call location\n");
 			  inlined_fn = NULL;
 			}
 		      if (inlined_fn)
@@ -1507,10 +1567,10 @@ function_instance::match (cgraph_node *node,
 			      callsite key2 = {stack[0].afdo_loc,
 					       newn ? *newn
 					       : inlined_fn->name ()};
+			      callsites.erase (iter);
 			      callsites[key2] = inlined_fn;
 			      inlined_fn->set_name (newn ? *newn
 						    : inlined_fn->name ());
-			      callsites.erase (iter);
 			    }
 			  functions.add (inlined_fn);
 			}
@@ -1559,12 +1619,25 @@ function_instance::match (cgraph_node *node,
     (DECL_STRUCT_FUNCTION (node->decl)->function_end_locus, node->decl);
   unsigned int start_location = get_combined_location
     (DECL_STRUCT_FUNCTION (node->decl)->function_start_locus, node->decl);
+  /* When outputting code to builtins location we use line number 0.
+     craeate_gcov is stupid and hapilly computes offsets across files.
+     Silently ignore it.  */
+  unsigned int zero_location
+	  = ((unsigned)(1-DECL_SOURCE_LINE (node->decl))) << 16;
   for (position_count_map::const_iterator iter = pos_counts.begin ();
        iter != pos_counts.end ();)
     if (!counts.contains (&iter->second))
       {
-	if (iter->first != end_location && iter->first != start_location
-	    && iter->first)
+	if (iter->first != end_location
+	    && iter->first != start_location
+	    && (iter->first & 65535) != zero_location
+	    && iter->first
+	    /* FIXME: dwarf5 does not represent inline stack of debug
+	       statements and consequently create_gcov is sometimes
+	       mixing up statements from other functions.  Do not warn
+	       user about this until this problem is solved.
+	       We still write info into dump file.  */
+	    && 0)
 	  {
 	    if (!warned)
 	      warned = warning_at (DECL_SOURCE_LOCATION (node->decl),
@@ -1601,25 +1674,38 @@ function_instance::match (cgraph_node *node,
        iter != callsites.end ();)
     if (!functions.contains (iter->second))
       {
-	if (!warned)
-	  warned = warning_at (DECL_SOURCE_LOCATION (node->decl),
-			       OPT_Wauto_profile,
-			       "auto-profile of %q+F contains extra callsites",
-			       node->decl);
-	if (warned)
-	  inform (DECL_SOURCE_LOCATION (node->decl),
-		  "call of %s with relative location +%i, discriminator %i",
-		  afdo_string_table->get_name (iter->first.second),
-		  iter->first.first >> 16, iter->first.first & 65535);
-	if ((iter->first.first >> 16) > (end_location >> 16) && warned)
-	  inform (DECL_SOURCE_LOCATION (node->decl),
-		  "location is after end of function");
-	warned = true;
 	function_instance *f = iter->second;
-	if (dump_file)
+	/* If we did not see the corresponding statement, warn.  */
+	if (!functions_to_offline.contains (iter->second))
+	  {
+	    if (!warned)
+	      warned = warning_at (DECL_SOURCE_LOCATION (node->decl),
+				   OPT_Wauto_profile,
+				   "auto-profile of %q+F contains"
+				   " extra callsites",
+				   node->decl);
+	    if (warned)
+	      inform (DECL_SOURCE_LOCATION (node->decl),
+		      "call of %s with total count %" PRId64
+		      ", relative location +%i, discriminator %i",
+		      afdo_string_table->get_name (iter->first.second),
+		      iter->second->total_count (),
+		      iter->first.first >> 16, iter->first.first & 65535);
+	    if ((iter->first.first >> 16) > (end_location >> 16) && warned)
+	      inform (DECL_SOURCE_LOCATION (node->decl),
+		      "location is after end of function");
+	    if (dump_file)
+	      {
+		fprintf (dump_file,
+			 "Offlining inline with no corresponding gimple stmt ");
+		f->dump_inline_stack (dump_file);
+		fprintf (dump_file, "\n");
+	      }
+	  }
+	else if (dump_file)
 	  {
 	    fprintf (dump_file,
-		     "Offlining inline with no corresponding gimple stmt ");
+		     "Offlining mismatched inline ");
 	    f->dump_inline_stack (dump_file);
 	    fprintf (dump_file, "\n");
 	  }
@@ -1684,9 +1770,9 @@ function_instance::remove_external_functions
       auto iter = callsites.find (key);
       callsite key2 = key;
       key2.second = *to_symbol_name.get (key.second);
-      callsites[key2] = iter->second;
       iter->second->set_name (key2.second);
       callsites.erase (iter);
+      callsites[key2] = iter->second;
     }
   auto_vec <int, 20> target_to_rename;
   for (auto &iter : pos_counts)
@@ -1883,6 +1969,7 @@ autofdo_source_profile::offline_external_functions ()
   cgraph_node *node;
   name_index_set seen;
   name_index_map to_symbol_name;
+  size_t last_name;
 
   /* Add renames erasing suffixes produced by late clones, such as
      .isra, .ipcp.  */
@@ -1918,10 +2005,10 @@ autofdo_source_profile::offline_external_functions ()
 	index = afdo_string_table->add_name (n2);
       to_symbol_name.put (i, index);
     }
+  last_name = afdo_string_table->num_entries ();
   FOR_EACH_DEFINED_FUNCTION (node)
     {
-      const char *name
-	  = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
+      const char *name = raw_symbol_name (node->decl);
       const char *dwarf_name = lang_hooks.dwarf_name (node->decl, 0);
       int index = afdo_string_table->get_index (name);
 
@@ -1958,11 +2045,17 @@ autofdo_source_profile::offline_external_functions ()
 	{
 	  if (dump_file)
 	    {
-	      fprintf (dump_file,
-		       "Node %s not in auto profile (%s neither %s)\n",
-		       node->dump_name (),
-		       name,
-		       dwarf_name);
+	      if (dwarf_name && strcmp (dwarf_name, name))
+		fprintf (dump_file,
+			 "Node %s not in auto profile (%s neither %s)\n",
+			 node->dump_name (),
+			 name,
+			 dwarf_name);
+	      else
+		fprintf (dump_file,
+			 "Node %s (symbol %s) not in auto profile\n",
+			 node->dump_name (),
+			 name);
 	    }
 	}
     }
@@ -2006,72 +2099,87 @@ autofdo_source_profile::offline_external_functions ()
      should be done on unmodified profile and merging works better if
      mismatches are already resolved both in source and destination.  */
   while (fns.length () || fns2.length ())
-    if (fns.length ())
-      {
-	function_instance *f = fns.pop ();
-	if (f->get_location () == UNKNOWN_LOCATION)
-	  {
-	    int index = f->name ();
-	    int *newn = to_symbol_name.get (index);
-	    if (newn)
-	      {
-		f->set_name (*newn);
-		if (map_.count (index)
-		    && map_[index] == f)
-		  map_.erase (index);
-		if (!map_.count (*newn))
-		  map_[*newn] = f;
-	      }
-	    if (cgraph_node *n = f->get_cgraph_node ())
-	      {
-		gcc_checking_assert (seen.contains (f->name ()));
-		f->match (n, fns, to_symbol_name);
-	      }
-	  }
-	fns2.safe_push (f);
-      }
-    else
-      {
-	function_instance *f = fns2.pop ();
-	int index = f->name ();
-	gcc_checking_assert (f->in_worklist_p ());
+    {
+      /* In case renaming introduced new name, keep seen up to date.  */
+      for (; last_name < afdo_string_table->num_entries (); last_name++)
+	{
+	  const char *name = afdo_string_table->get_name (last_name);
+	  symtab_node *n
+	    = afdo_string_table->get_cgraph_node (last_name);
+	  if (dump_file)
+	    fprintf (dump_file, "New name %s %s\n", name,
+		     n ? "wth corresponding definition"
+		     : "with no corresponding definition");
+	  if (n)
+	    seen.add (last_name);
+	}
+      if (fns.length ())
+	{
+	  function_instance *f = fns.pop ();
+	  if (f->get_location () == UNKNOWN_LOCATION)
+	    {
+	      int index = f->name ();
+	      int *newn = to_symbol_name.get (index);
+	      if (newn)
+		{
+		  f->set_name (*newn);
+		  if (map_.count (index)
+		      && map_[index] == f)
+		    map_.erase (index);
+		  if (!map_.count (*newn))
+		    map_[*newn] = f;
+		}
+	      if (cgraph_node *n = f->get_cgraph_node ())
+		{
+		  gcc_checking_assert (seen.contains (f->name ()));
+		  f->match (n, fns, to_symbol_name);
+		}
+	    }
+	  fns2.safe_push (f);
+	}
+      else
+	{
+	  function_instance *f = fns2.pop ();
+	  int index = f->name ();
+	  gcc_checking_assert (f->in_worklist_p ());
 
-	/* If map has different function_instance of same name, then
-	   this is a duplicated entry which needs to be merged.  */
-	if (map_.count (index) && map_[index] != f)
-	  {
-	    if (dump_file)
-	      {
-		fprintf (dump_file, "Merging duplicate instance: ");
-		f->dump_inline_stack (dump_file);
-		fprintf (dump_file, "\n");
-	      }
-	    map_[index]->merge (f, fns);
-	    gcc_checking_assert (!f->inlined_to ());
-	    f->clear_in_worklist ();
-	    delete f;
-	  }
-	/* If name was not seen in the symbol table, remove it.  */
-	else if (!seen.contains (index))
-	  {
-	    f->offline_if_in_set (seen, fns);
-	    f->clear_in_worklist ();
-	    if (dump_file)
-	      fprintf (dump_file, "Removing external %s\n",
-		       afdo_string_table->get_name (f->name ()));
-	    if (map_.count (index) && map_[index] == f)
-	      map_.erase (f->name ());
-	    delete f;
-	  }
-	/* If this is offline function instance seen in this
-	   translation unit offline external inlines and possibly
-	   rename from dwarf name.  */
-	else
-	  {
-	    f->remove_external_functions (seen, to_symbol_name, fns);
-	    f->clear_in_worklist ();
-	  }
-      }
+	  /* If map has different function_instance of same name, then
+	     this is a duplicated entry which needs to be merged.  */
+	  if (map_.count (index) && map_[index] != f)
+	    {
+	      if (dump_file)
+		{
+		  fprintf (dump_file, "Merging duplicate instance: ");
+		  f->dump_inline_stack (dump_file);
+		  fprintf (dump_file, "\n");
+		}
+	      map_[index]->merge (f, fns);
+	      gcc_checking_assert (!f->inlined_to ());
+	      f->clear_in_worklist ();
+	      delete f;
+	    }
+	  /* If name was not seen in the symbol table, remove it.  */
+	  else if (!seen.contains (index))
+	    {
+	      f->offline_if_in_set (seen, fns);
+	      f->clear_in_worklist ();
+	      if (dump_file)
+		fprintf (dump_file, "Removing external %s\n",
+			 afdo_string_table->get_name (f->name ()));
+	      if (map_.count (index) && map_[index] == f)
+		map_.erase (f->name ());
+	      delete f;
+	    }
+	  /* If this is offline function instance seen in this
+	     translation unit offline external inlines and possibly
+	     rename from dwarf name.  */
+	  else
+	    {
+	      f->remove_external_functions (seen, to_symbol_name, fns);
+	      f->clear_in_worklist ();
+	    }
+	}
+    }
   if (dump_file)
     for (auto const &iter : map_)
       {
@@ -2103,8 +2211,7 @@ walk_block (tree fn, function_instance *s, tree block)
 	      fprintf (dump_file, ":");
 	      dump_afdo_loc (dump_file, loc);
 	      fprintf (dump_file, " %s\n",
-		       IDENTIFIER_POINTER
-			 (DECL_ASSEMBLER_NAME (BLOCK_ABSTRACT_ORIGIN (block))));
+		       raw_symbol_name (BLOCK_ABSTRACT_ORIGIN (block)));
 	    }
 	  return;
 	}
@@ -2221,7 +2328,7 @@ autofdo_source_profile::offline_unrealized_inlines ()
 
 function_instance *
 function_instance::read_function_instance (function_instance_stack *stack,
-                                           gcov_type head_count)
+					   gcov_type head_count)
 {
   unsigned name = gcov_read_unsigned ();
   unsigned num_pos_counts = gcov_read_unsigned ();
@@ -2244,10 +2351,10 @@ function_instance::read_function_instance (function_instance_stack *stack,
         (*stack)[j]->total_count_ += count;
       for (unsigned j = 0; j < num_targets; j++)
         {
-          /* Only indirect call target histogram is supported now.  */
-          gcov_read_unsigned ();
-          gcov_type target_idx = gcov_read_counter ();
-          s->pos_counts[offset].targets[target_idx] = gcov_read_counter ();
+	  /* Only indirect call target histogram is supported now.  */
+	  gcov_read_unsigned ();
+	  gcov_type target_idx = gcov_read_counter ();
+	  s->pos_counts[offset].targets[target_idx] = gcov_read_counter ();
         }
     }
   for (unsigned i = 0; i < num_callsites; i++)
@@ -2462,7 +2569,7 @@ autofdo_source_profile::get_callsite_total_count (
     {
       if (dump_file)
 	fprintf (dump_file, "Mismatched name of callee %s and profile %s\n",
-		 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (edge->callee->decl)),
+		 raw_symbol_name (edge->callee->decl),
 		 afdo_string_table->get_name (s->name ()));
       return 0;
     }
@@ -2522,6 +2629,7 @@ autofdo_source_profile::read ()
     afdo_count_scale
       = MAX (((gcov_type)1 << (profile_count::n_bits / 2))
 	     / afdo_profile_info->sum_max, 1);
+  afdo_profile_info->cutoff *= afdo_count_scale;
   afdo_hot_bb_threshod
     = hot_frac
       ? afdo_profile_info->sum_max * afdo_count_scale / hot_frac
@@ -2531,10 +2639,12 @@ autofdo_source_profile::read ()
     fprintf (dump_file, "Max count in profile %" PRIu64 "\n"
 			"Setting scale %" PRIu64 "\n"
 			"Scaled max count %" PRIu64 "\n"
+			"Cutoff %" PRIu64 "\n"
 			"Hot count threshold %" PRIu64 "\n\n",
 	     (int64_t)afdo_profile_info->sum_max,
 	     (int64_t)afdo_count_scale,
 	     (int64_t)(afdo_profile_info->sum_max * afdo_count_scale),
+	     (int64_t)afdo_profile_info->cutoff,
 	     (int64_t)afdo_hot_bb_threshod);
   afdo_profile_info->sum_max *= afdo_count_scale;
   return true;
@@ -2553,8 +2663,7 @@ autofdo_source_profile::get_function_instance_by_inline_stack (
     {
       if (dump_file)
 	fprintf (dump_file, "No offline instance for %s\n",
-		 IDENTIFIER_POINTER
-		   (DECL_ASSEMBLER_NAME (stack[stack.length () - 1].decl)));
+		 raw_symbol_name (stack[stack.length () - 1].decl));
       return NULL;
     }
   function_instance *s = iter->second;
@@ -2576,8 +2685,7 @@ autofdo_source_profile::get_function_instance_by_inline_stack (
 			      "auto-profile has no inlined function instance "
 			      "for inlined call of %s at relative "
 			      " locaction +%i, discriminator %i\n",
-			     IDENTIFIER_POINTER
-			      (DECL_ASSEMBLER_NAME (stack[i - 1].decl)),
+			     raw_symbol_name (stack[i - 1].decl),
 			     stack[i].afdo_loc >> 16,
 			     stack[i].afdo_loc & 65535);
 	  return NULL;
@@ -2641,14 +2749,22 @@ read_profile (void)
 
   /* autofdo_source_profile.  */
   afdo_source_profile = autofdo_source_profile::create ();
-  if (afdo_source_profile == NULL)
+  if (afdo_source_profile == NULL
+      || gcov_is_error ())
     {
       error ("cannot read function profile from %s", auto_profile_file);
+      delete afdo_source_profile;
+      afdo_source_profile = NULL;
       return;
     }
 
   /* autofdo_module_profile.  */
   fake_read_autofdo_module_profile ();
+  if (gcov_is_error ())
+    {
+      error ("cannot read module profile from %s", auto_profile_file);
+      return;
+    }
 }
 
 /* From AutoFDO profiles, find values inside STMT for that we want to measure
@@ -3301,10 +3417,22 @@ cmp (const void *a, const void *b)
   return 0;
 }
 
+/* To scalle a connected component of graph we collect desired scales of
+   basic blocks on the boundary and then compute a robust average.  */
+
+struct scale
+{
+  /* Scale descired.  */
+  sreal scale;
+  /* Weight for averaging computed from execution count of the edge
+     scale originates from.  */
+  uint64_t weight;
+};
+
 /* Add scale ORIG/ANNOTATED to SCALES.  */
 
 static void
-add_scale (vec <sreal> *scales, profile_count annotated, profile_count orig)
+add_scale (vec <scale> *scales, profile_count annotated, profile_count orig)
 {
   if (dump_file)
     {
@@ -3319,9 +3447,9 @@ add_scale (vec <sreal> *scales, profile_count annotated, profile_count orig)
 	= annotated.guessed_local ()
 		.to_sreal_scale (orig);
       if (dump_file)
-	fprintf (dump_file, "    adding scale %.16f\n",
-		 scale.to_double ());
-      scales->safe_push (scale);
+	fprintf (dump_file, "    adding scale %.16f, weight %" PRId64 "\n",
+		 scale.to_double (), annotated.value () + 1);
+      scales->safe_push ({scale, annotated.value () + 1});
     }
 }
 
@@ -3367,7 +3495,7 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb)
   /* Basic blocks of connected component currently processed.  */
   auto_vec <basic_block, 20> bbs (n_basic_blocks_for_fn (cfun));
   /* Scale factors found.  */
-  auto_vec <sreal, 20> scales;
+  auto_vec <scale, 20> scales;
   auto_vec <basic_block, 20> stack (n_basic_blocks_for_fn (cfun));
 
   basic_block seed_bb;
@@ -3379,9 +3507,15 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb)
      >=2 is an id of the component BB belongs to.  */
   auto_vec <unsigned int, 20> component;
   component.safe_grow (last_basic_block_for_fn (cfun));
+  profile_count max_count_in_fn = profile_count::zero ();
   FOR_ALL_BB_FN (seed_bb, cfun)
-    component[seed_bb->index]
-	= is_bb_annotated (seed_bb, *annotated_bb) ? 1 : 0;
+    if (is_bb_annotated (seed_bb, *annotated_bb))
+      {
+	component[seed_bb->index] = 1;
+	max_count_in_fn = max_count_in_fn.max (seed_bb->count);
+      }
+    else
+      component[seed_bb->index] = 0;
   FOR_ALL_BB_FN (seed_bb, cfun)
    if (!component[seed_bb->index])
      {
@@ -3504,12 +3638,15 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb)
 		 profile_count annotated_count = e->dest->count;
 		 profile_count out_count = profile_count::zero ();
 		 bool ok = true;
+
 		 for (edge e2: e->dest->preds)
 		   if (AFDO_EINFO (e2)->is_annotated ())
 		     annotated_count -= AFDO_EINFO (e2)->get_count ();
-		   else if (component[e->src->index] == component_id)
-		     out_count += e->count ();
-		   else if (e->probability.nonzero_p ())
+		   else if (component[e2->src->index] == component_id)
+		     out_count += e2->count ();
+		   else if (is_bb_annotated (e2->src, *annotated_bb))
+		     annotated_count -= e2->count ();
+		   else if (e2->probability.nonzero_p ())
 		     {
 		       ok = false;
 		       break;
@@ -3551,12 +3688,52 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb)
 	 {
 	   if (dump_file)
 	     fprintf (dump_file,
-		      "  Can not determine count from the boundary; giving up");
+		      "  Can not determine count from the boundary; giving up\n");
 	   continue;
 	 }
        gcc_checking_assert (scales.length ());
        scales.qsort (cmp);
-       scale_bbs (bbs, scales[scales.length () / 2]);
+
+       uint64_t overall_weight = 0;
+       for (scale &e : scales)
+	 overall_weight += e.weight;
+
+       uint64_t cummulated = 0, weight_sum = 0;
+       sreal scale_sum = 0;
+       for (scale &e : scales)
+	 {
+	   uint64_t prev = cummulated;
+	   cummulated += e.weight;
+	   if (cummulated >= overall_weight / 4
+	       && prev <= 3 * overall_weight / 4)
+	     {
+	       scale_sum += e.scale * e.weight;
+	       weight_sum += e.weight;
+	       if (dump_file)
+		 fprintf (dump_file, "    accounting scale %.16f, weight %" PRId64 "\n",
+			  e.scale.to_double (), e.weight);
+	     }
+	   else if (dump_file)
+	     fprintf (dump_file, "    ignoring scale %.16f, weight %" PRId64 "\n",
+		      e.scale.to_double (), e.weight);
+	  }
+       sreal scale = scale_sum / (sreal)weight_sum;
+
+       /* Avoid scaled regions to have very large counts.
+	  Otherwise they may dominate ipa-profile's histogram computing cutoff
+	  of hot basic blocks.  */
+       if (max_count * scale > max_count_in_fn.guessed_local ())
+	 {
+	   if (dump_file)
+	     {
+	       fprintf (dump_file, "Scaling by %.16f produces max count ",
+			scale.to_double ());
+	       (max_count * scale).dump (dump_file);
+	       fprintf (dump_file, " that exceeds max count in fn; capping\n");
+	     }
+	   scale = max_count_in_fn.guessed_local ().to_sreal_scale (max_count);
+	 }
+       scale_bbs (bbs, scale);
      }
 }
 
@@ -3865,6 +4042,7 @@ read_autofdo_file (void)
   autofdo::afdo_profile_info = XNEW (gcov_summary);
   autofdo::afdo_profile_info->runs = 1;
   autofdo::afdo_profile_info->sum_max = 0;
+  autofdo::afdo_profile_info->cutoff = 1;
 
   /* Read the profile from the profile file.  */
   autofdo::read_profile ();
diff --git a/gcc/avoid-store-forwarding.cc b/gcc/avoid-store-forwarding.cc
index 37e0953..1de6fd6 100644
--- a/gcc/avoid-store-forwarding.cc
+++ b/gcc/avoid-store-forwarding.cc
@@ -119,17 +119,6 @@ generate_bit_insert_sequence (store_fwd_info *store_info, rtx dest)
   unsigned HOST_WIDE_INT bitsize = store_size * BITS_PER_UNIT;
   unsigned HOST_WIDE_INT start = store_info->offset * BITS_PER_UNIT;
 
-  /* Adjust START for machines with BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN.
-     Given that the bytes will be reversed in this case, we need to
-     calculate the starting position from the end of the destination
-     register.  */
-  if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
-    {
-      unsigned HOST_WIDE_INT load_mode_bitsize
-	= (GET_MODE_BITSIZE (GET_MODE (dest))).to_constant ();
-      start = load_mode_bitsize - bitsize - start;
-    }
-
   rtx mov_reg = store_info->mov_reg;
   store_bit_field (dest, bitsize, start, 0, 0, GET_MODE (mov_reg), mov_reg,
 		   false, false);
@@ -242,17 +231,39 @@ process_store_forwarding (vec<store_fwd_info> &stores, rtx_insn *load_insn,
 
   int move_to_front = -1;
   int total_cost = 0;
+  int base_offset_index = -1;
+
+  /* Find the last store that has the same offset the load, in the case that
+     we're eliminating the load.  We will try to use it as a base register
+     to avoid bit inserts (see second loop below).  We want the last one, as
+     it will be wider and we don't want to overwrite the base register if
+     there are many of them.  */
+  if (load_elim)
+    {
+      FOR_EACH_VEC_ELT_REVERSE (stores, i, it)
+	{
+	  const bool has_base_offset
+	    = known_eq (poly_uint64 (it->offset),
+			subreg_size_lowpart_offset (MEM_SIZE (it->store_mem),
+						    load_size));
+	  if (has_base_offset)
+	    {
+	      base_offset_index = i;
+	      break;
+	    }
+	}
+    }
 
   /* Check if we can emit bit insert instructions for all forwarded stores.  */
   FOR_EACH_VEC_ELT (stores, i, it)
     {
       it->mov_reg = gen_reg_rtx (GET_MODE (it->store_mem));
       rtx_insn *insns = NULL;
-      const bool has_zero_offset = it->offset == 0;
 
-      /* If we're eliminating the load then find the store with zero offset
-	 and use it as the base register to avoid a bit insert if possible.  */
-      if (load_elim && has_zero_offset)
+      /* Check if this is a store with base offset, if we're eliminating the
+	 load, and use it as the base register to avoid a bit insert if
+	 possible.  Load elimination is implied by base_offset_index != -1.  */
+      if (i == (unsigned) base_offset_index)
 	{
 	  start_sequence ();
 
@@ -445,9 +456,22 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
     return;
 
   auto_vec<store_fwd_info, 8> store_exprs;
+  auto_vec<rtx> store_exprs_del;
   rtx_insn *insn;
   unsigned int insn_cnt = 0;
 
+  /* We are iterating over the basic block's instructions detecting store
+     instructions.  Upon reaching a load instruction, we check if any of the
+     previously detected stores could result in store forwarding.  In that
+     case, we try to reorder the load and store instructions.
+     We skip this transformation when we encounter complex memory operations,
+     instructions that might throw an exception, instruction dependencies,
+     etc.  This is done by clearing the vector of detected stores, while
+     keeping the removed stores in another vector.  By doing so, we can check
+     if any of the removed stores operated on the load's address range, when
+     reaching a subsequent store that operates on the same address range,
+     as this would lead to incorrect values on the register that keeps the
+     loaded value.  */
   FOR_BB_INSNS (bb, insn)
     {
       if (!NONDEBUG_INSN_P (insn))
@@ -460,6 +484,10 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
 
       if (!set || insn_could_throw_p (insn))
 	{
+	  unsigned int i;
+	  store_fwd_info *it;
+	  FOR_EACH_VEC_ELT (store_exprs, i, it)
+	    store_exprs_del.safe_push (it->store_mem);
 	  store_exprs.truncate (0);
 	  continue;
 	}
@@ -483,6 +511,10 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
 	  || (load_mem && (!MEM_SIZE_KNOWN_P (load_mem)
 			   || !MEM_SIZE (load_mem).is_constant ())))
 	{
+	  unsigned int i;
+	  store_fwd_info *it;
+	  FOR_EACH_VEC_ELT (store_exprs, i, it)
+	    store_exprs_del.safe_push (it->store_mem);
 	  store_exprs.truncate (0);
 	  continue;
 	}
@@ -534,6 +566,7 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
 		    it->remove = true;
 		    removed_count++;
 		    remove_rest = true;
+		    store_exprs_del.safe_push (it->store_mem);
 		  }
 	      }
 	  }
@@ -573,23 +606,46 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
 		  it->remove = true;
 		  removed_count++;
 		  remove_rest = true;
+		  forwardings.truncate (0);
 		}
 	      else if (is_store_forwarding (store_mem, load_mem, &off_val))
 		{
+		  unsigned int j;
+		  rtx *del_it;
+		  bool same_range_as_removed = false;
+
+		  /* Check if another store in the load's address range has
+		     been deleted due to a constraint violation.  In this case
+		     we can't forward any other stores that operate in this
+		     range, as it would lead to partial update of the register
+		     that holds the loaded value.  */
+		  FOR_EACH_VEC_ELT (store_exprs_del, j, del_it)
+		    {
+		      rtx del_store_mem = *del_it;
+		      same_range_as_removed
+			= is_store_forwarding (del_store_mem, load_mem, NULL);
+		      if (same_range_as_removed)
+			break;
+		    }
+
 		  /* Check if moving this store after the load is legal.  */
 		  bool write_dep = false;
-		  for (unsigned int j = store_exprs.length () - 1; j != i; j--)
+		  if (!same_range_as_removed)
 		    {
-		      if (!store_exprs[j].forwarded
-			  && output_dependence (store_mem,
-						store_exprs[j].store_mem))
+		      unsigned int j = store_exprs.length () - 1;
+		      for (; j != i; j--)
 			{
-			  write_dep = true;
-			  break;
+			  if (!store_exprs[j].forwarded
+			      && output_dependence (store_mem,
+						    store_exprs[j].store_mem))
+			    {
+			      write_dep = true;
+			      break;
+			    }
 			}
 		    }
 
-		  if (!write_dep)
+		  if (!same_range_as_removed && !write_dep)
 		    {
 		      it->forwarded = true;
 		      it->offset = off_val;
@@ -609,6 +665,7 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
 		  it->remove = true;
 		  removed_count++;
 		  remove_rest = true;
+		  forwardings.truncate (0);
 		}
 	    }
 
@@ -616,9 +673,12 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb)
 	    process_store_forwarding (forwardings, insn, load_mem);
 	}
 
+	/* Abort in case that we encounter a memory read/write that is not a
+	   simple store/load, as we can't make safe assumptions about the
+	   side-effects of this.  */
 	if ((writes_mem && !is_simple_store)
 	     || (reads_mem && !is_simple_load))
-	   store_exprs.truncate (0);
+	  return;
 
 	if (removed_count)
 	{
diff --git a/gcc/builtins.cc b/gcc/builtins.cc
index a2ce372..7f580a3 100644
--- a/gcc/builtins.cc
+++ b/gcc/builtins.cc
@@ -7799,11 +7799,17 @@ expand_builtin_crc_table_based (internal_fn fn, scalar_mode crc_mode,
 
   rtx op1 = expand_normal (rhs1);
   rtx op2 = expand_normal (rhs2);
-  gcc_assert (TREE_CODE (rhs3) == INTEGER_CST);
-  rtx op3 = gen_int_mode (TREE_INT_CST_LOW (rhs3), crc_mode);
+  rtx op3;
+  if (TREE_CODE (rhs3) != INTEGER_CST)
+    {
+      error ("third argument to %<crc%> builtins must be a constant");
+      op3 = const0_rtx;
+    }
+  else
+    op3 = convert_to_mode (crc_mode, expand_normal (rhs3), 0);
 
   if (CONST_INT_P (op2))
-    op2 = gen_int_mode (INTVAL (op2), crc_mode);
+    op2 = convert_to_mode (crc_mode, op2, 0);
 
   if (fn == IFN_CRC)
     expand_crc_table_based (target, op1, op2, op3, data_mode);
diff --git a/gcc/c-family/ChangeLog b/gcc/c-family/ChangeLog
index dd2ae5c..c9ab153 100644
--- a/gcc/c-family/ChangeLog
+++ b/gcc/c-family/ChangeLog
@@ -1,3 +1,181 @@
+2025-08-02  Martin Uecker  <uecker@tugraz.at>
+
+	* c-attribs.cc (handle_argspec_attribute): Update.
+	(build_arg_spec): New function.
+	(build_attr_access_from_parms): Rewrite `arg spec' handling.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.cc: Make diagnostics::context::m_source_printing
+	private.
+	* c-format.cc: Likewise.
+	* c-opts.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.cc: Update usage of "diagnostic_info" to explicitly
+	refer to "diagnostics::diagnostic_info".
+	* c-opts.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-format.cc: Update for file_cache and char_span moving from
+	input.h to diagnostics/file-cache.h and into the "diagnostics::"
+	namespace.
+	* c-indentation.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.cc (c_family_tests): Add include of
+	"diagnostics/diagnostics-selftests.h".  Replace
+	c_diagnostic_cc_tests with
+	diagnostics::selftest::context_cc_tests.
+	* c-common.h: Drop c_diagnostic_cc_tests decl.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* c-format.cc: Likewise.
+	* c-lex.cc: Likewise.
+	* c-opts.cc: Likewise.
+	* c-pragma.cc: Likewise.
+	* c-warn.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.cc: Update for renaming of diagnostic_option_id to
+	diagnostics::option_id.
+	* c-common.h: Likewise.
+	* c-cppbuiltin.cc: Likewise.
+	* known-headers.cc: Likewise.
+	* known-headers.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.cc: Update comment for renaming of edit_context.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-format.cc (test_type_mismatch_range_labels): Update for
+	move of selftest::test_diagnostic_context to
+	diagnostics::selftest::test_context.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-format.cc: Update for move of selftest-diagnostic.h to
+	diagnostics/selftest-context.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-common.h: Update for diagnostic_context becoming
+	diagnostics::context.
+	* c-opts.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-opts.cc (c_diagnostic_text_finalizer): Add "m_" prefix to
+	fields of diagnostic_info.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-opts.cc: Update for move of "diagnostic-macro-unwinding.h"
+	to "diagnostics/macro-unwinding.h".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-opts.cc: Update for move of diagnostics output formats into
+	namespace "diagnostics" as "sinks".
+
+2025-07-16  Kwok Cheung Yeung  <kcyeung@baylibre.com>
+
+	* c-omp.cc (c_finish_omp_depobj): Use OMP_ITERATOR_DECL_P.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* c-attribs.cc (handle_target_clones_attribute): Change to use
+	get_clone_versions.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* c-format.cc (local_string_slice_node): New node type.
+	(asm_fprintf_char_table): New entry.
+	(init_dynamic_diag_info): Add support for string_slice.
+	* c-format.h (T_STRING_SLICE): New node type.
+
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* c-opts.cc (c_common_post_options): Change
+	warn_unused_but_set_parameter and warn_unused_but_set_variable
+	from 1 to 3 if they were set only implicitly.
+	* c-attribs.cc (build_attr_access_from_parms): Remove unused
+	but set variable nelts.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/119064
+	* c.opt (Wc++26-compat): New option.
+	* c.opt.urls: Regenerate.
+	* c-opts.cc (c_common_post_options): Clear warn_cxx26_compat for
+	C++26 or later.
+	* c-cppbuiltin.cc (c_cpp_builtins): For C++26 predefine
+	__cpp_trivial_relocatability=202502L.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/117785
+	* c-cppbuiltin.cc (c_cpp_builtins): Predefine
+	__cpp_constexpr_exceptions=202411L for C++26.
+
+2025-07-10  Qing Zhao  <qing.zhao@oracle.com>
+
+	* c-ubsan.cc (get_bound_from_access_with_size): Adjust the position
+	of the arguments per the new design.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-01  Qing Zhao  <qing.zhao@oracle.com>
+
+	* c-attribs.cc (handle_counted_by_attribute): Accept counted_by
+	attribute for pointer fields.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-01  Qing Zhao  <qing.zhao@oracle.com>
+
+	* c-gimplify.cc (is_address_with_access_with_size): New function.
+	(ubsan_walk_array_refs_r): Instrument an INDIRECT_REF whose base
+	address is .ACCESS_WITH_SIZE or an address computation whose base
+	address is .ACCESS_WITH_SIZE.
+	* c-ubsan.cc (ubsan_instrument_bounds_pointer_address): New function.
+	(struct factor_t): New structure.
+	(get_factors_from_mul_expr): New function.
+	(get_index_from_offset): New function.
+	(get_index_from_pointer_addr_expr): New function.
+	(is_instrumentable_pointer_array_address): New function.
+	(ubsan_array_ref_instrumented_p): Change prototype.
+	Handle MEM_REF in addtional to ARRAY_REF.
+	(ubsan_maybe_instrument_array_ref): Handle MEM_REF in addtional
+	to ARRAY_REF.
+
+2025-07-07  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120917
+	* c.opt: Add -Wno-abbreviated-auto-in-template-arg.
+	* c.opt.urls: Regenerate.
+
+2025-07-04  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c/120837
+	* c-common.cc (pointer_int_sum): Rewrite the intop PLUS_EXPR or
+	MINUS_EXPR optimization into extension of both intop operands,
+	their separate multiplication and then addition/subtraction followed
+	by rest of pointer_int_sum handling after the multiplication.
+
 2025-07-01  Qing Zhao  <qing.zhao@oracle.com>
 
 	* c-gimplify.cc (is_address_with_access_with_size): New function.
diff --git a/gcc/c-family/c-attribs.cc b/gcc/c-family/c-attribs.cc
index ea04ed7..a0d832b 100644
--- a/gcc/c-family/c-attribs.cc
+++ b/gcc/c-family/c-attribs.cc
@@ -2906,53 +2906,22 @@ handle_counted_by_attribute (tree *node, tree name,
 		" declaration %q+D", name, decl);
       *no_add_attrs = true;
     }
-  /* This attribute only applies to a field with array type or pointer type.  */
-  else if (TREE_CODE (TREE_TYPE (decl)) != ARRAY_TYPE
-	   && TREE_CODE (TREE_TYPE (decl)) != POINTER_TYPE)
+  /* This attribute only applies to field with array type.  */
+  else if (TREE_CODE (TREE_TYPE (decl)) != ARRAY_TYPE)
     {
       error_at (DECL_SOURCE_LOCATION (decl),
-		"%qE attribute is not allowed for a non-array"
-		" or non-pointer field", name);
+		"%qE attribute is not allowed for a non-array field",
+		name);
       *no_add_attrs = true;
     }
   /* This attribute only applies to a C99 flexible array member type.  */
-  else if (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
-	   && !c_flexible_array_member_type_p (TREE_TYPE (decl)))
+  else if (! c_flexible_array_member_type_p (TREE_TYPE (decl)))
     {
       error_at (DECL_SOURCE_LOCATION (decl),
 		"%qE attribute is not allowed for a non-flexible"
 		" array member field", name);
       *no_add_attrs = true;
     }
-  /* This attribute cannot be applied to a pointer to void type.  */
-  else if (TREE_CODE (TREE_TYPE (decl)) == POINTER_TYPE
-	   && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == VOID_TYPE)
-    {
-      error_at (DECL_SOURCE_LOCATION (decl),
-		"%qE attribute is not allowed for a pointer to void",
-		name);
-      *no_add_attrs = true;
-    }
-  /* This attribute cannot be applied to a pointer to function type.  */
-  else if (TREE_CODE (TREE_TYPE (decl)) == POINTER_TYPE
-	   && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == FUNCTION_TYPE)
-    {
-      error_at (DECL_SOURCE_LOCATION (decl),
-		"%qE attribute is not allowed for a pointer to"
-		" function", name);
-      *no_add_attrs = true;
-    }
-  /* This attribute cannot be applied to a pointer to structure or union
-     with flexible array member.  */
-  else if (TREE_CODE (TREE_TYPE (decl)) == POINTER_TYPE
-	   && RECORD_OR_UNION_TYPE_P (TREE_TYPE (TREE_TYPE (decl)))
-	   && TYPE_INCLUDES_FLEXARRAY (TREE_TYPE (TREE_TYPE (decl))))
-    {
-	error_at (DECL_SOURCE_LOCATION (decl),
-		  "%qE attribute is not allowed for a pointer to"
-		  " structure or union with flexible array member", name);
-	*no_add_attrs = true;
-    }
   /* The argument should be an identifier.  */
   else if (TREE_CODE (argval) != IDENTIFIER_NODE)
     {
@@ -2961,8 +2930,7 @@ handle_counted_by_attribute (tree *node, tree name,
       *no_add_attrs = true;
     }
   /* Issue error when there is a counted_by attribute with a different
-     field as the argument for the same flexible array member or
-     pointer field.  */
+     field as the argument for the same flexible array member field.  */
   else if (old_counted_by != NULL_TREE)
     {
       tree old_fieldname = TREE_VALUE (TREE_VALUE (old_counted_by));
@@ -4152,10 +4120,11 @@ handle_argspec_attribute (tree *, tree, tree args, int, bool *)
 {
   /* Verify the attribute has one or two arguments and their kind.  */
   gcc_assert (args && TREE_CODE (TREE_VALUE (args)) == STRING_CST);
-  for (tree next = TREE_CHAIN (args); next; next = TREE_CHAIN (next))
+  if (TREE_CHAIN (args))
     {
-      tree val = TREE_VALUE (next);
-      gcc_assert (DECL_P (val) || EXPR_P (val));
+      tree val = TREE_VALUE (TREE_CHAIN (args));
+      gcc_assert (!TREE_CHAIN (TREE_CHAIN (args)));
+      gcc_assert (TYPE_P (val));
     }
   return NULL_TREE;
 }
@@ -5768,6 +5737,71 @@ handle_access_attribute (tree node[3], tree name, tree args, int flags,
   return NULL_TREE;
 }
 
+
+/* This function builds a string which is concatenated to SPEC and returns
+   list of variably bounds corresponding to an array/VLA parameter with
+   type TYPE.  The string consists of one dollar symbol for each specified
+   variable bound, one asterisk for each unspecified variable bound,
+   a space for an array of unknown size (only possibly for the outermost),
+   and a zero for a zero-sized array.
+
+   The chainof variable bounds starts with the most significant bound.
+   For example, the TYPE T[2][m][3][n] will produce "$$" and (m, (n, nil)).  */
+
+static tree
+build_arg_spec (tree type, std::string *spec)
+{
+  while (POINTER_TYPE_P (type))
+    type = TREE_TYPE (type);
+
+  if (TREE_CODE (type) != ARRAY_TYPE)
+    return NULL_TREE;
+
+  tree list = build_arg_spec (TREE_TYPE (type), spec);
+
+  if (!COMPLETE_TYPE_P (type))
+    {
+      (*spec) += ' ';
+      return list;
+    }
+
+  tree mval = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
+
+  if (!mval)
+    {
+     (*spec) += '0';
+     return list;
+    }
+
+  if (TREE_CODE (mval) == COMPOUND_EXPR
+      && integer_zerop (TREE_OPERAND (mval, 0))
+      && integer_zerop (TREE_OPERAND (mval, 1)))
+    {
+      (*spec) += '*';
+      return list;
+    }
+
+  if (TREE_CODE (mval) == INTEGER_CST)
+    return list;
+
+  /* A variable bound.  */
+  (*spec) += '$';
+
+  mval = array_type_nelts_top (type);
+
+  /* Remove NOP_EXPR and SAVE_EXPR to uncover possible PARM_DECLS.  */
+  if (TREE_CODE (mval) == NOP_EXPR)
+    mval = TREE_OPERAND (mval, 0);
+   if (TREE_CODE (mval) == SAVE_EXPR)
+    {
+      mval = TREE_OPERAND (mval, 0);
+      if (TREE_CODE (mval) == NOP_EXPR)
+	mval = TREE_OPERAND (mval, 0);
+    }
+
+  return tree_cons (NULL_TREE, mval, list);
+}
+
 /* Extract attribute "arg spec" from each FNDECL argument that has it,
    build a single attribute access corresponding to all the arguments,
    and return the result.  SKIP_VOIDPTR set to ignore void* parameters
@@ -5844,15 +5878,16 @@ build_attr_access_from_parms (tree parms, bool skip_voidptr)
       argspec = TREE_VALUE (argspec);
 
       /* The attribute arg spec string.  */
-      tree str = TREE_VALUE (argspec);
-      const char *s = TREE_STRING_POINTER (str);
+      const char *s = TREE_STRING_POINTER (TREE_VALUE (argspec));
+      bool static_p = s && (0 == strcmp("static", s));
 
       /* Collect the list of nonnull arguments which use "[static ..]".  */
-      if (s != NULL && s[0] == '[' && s[1] == 's')
+      if (static_p)
 	nnlist = tree_cons (NULL_TREE, build_int_cst (integer_type_node,
 						      argpos + 1), nnlist);
 
-      /* Create the attribute access string from the arg spec string,
+      tree argvbs;
+      /* Create the attribute access string from the arg spec data,
 	 optionally followed by position of the VLA bound argument if
 	 it is one.  */
       {
@@ -5863,21 +5898,52 @@ build_attr_access_from_parms (tree parms, bool skip_voidptr)
 	    specend = 1;
 	  }
 
-	/* Format the access string in place.  */
-	int len = snprintf (NULL, 0, "%c%u%s",
-			    attr_access::mode_chars[access_deferred],
-			    argpos, s);
-	spec.resize (specend + len + 1);
-	sprintf (&spec[specend], "%c%u%s",
-		 attr_access::mode_chars[access_deferred],
-		 argpos, s);
+	spec += attr_access::mode_chars[access_deferred];
+	spec += std::to_string (argpos);
+	spec += '[';
+	tree type = TREE_VALUE (TREE_CHAIN (argspec));
+	argvbs = build_arg_spec (type, &spec);
+
+	/* Postprocess the string to bring it in the format expected
+	   by the code handling the access attribute.  First, we
+	   add 's' if the array was declared as [static ...].  */
+	if (static_p)
+	  {
+	    size_t send = spec.length();
+
+	    if (spec[send - 1] == '[')
+	      {
+		spec += 's';
+	      }
+	    else
+	      {
+		/* If there is a symbol, we need to swap the order.  */
+		spec += spec[send - 1];
+		spec[send - 1] = 's';
+	      }
+	  }
+
+	/* If the outermost bound is an integer constant, we need to write
+	   the size  if it is constant.  */
+	if (type && TYPE_DOMAIN (type))
+	  {
+	    tree mval = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
+	    if (mval && TREE_CODE (mval) == INTEGER_CST)
+	      {
+		char buf[40];
+		unsigned HOST_WIDE_INT n = tree_to_uhwi (mval) + 1;
+		sprintf (buf, HOST_WIDE_INT_PRINT_UNSIGNED, n);
+		spec += buf;
+	      }
+	  }
+	spec += ']';
+
 	/* Trim the trailing NUL.  */
-	spec.resize (specend + len);
+	spec.resize (spec.length ());
       }
 
       /* The (optional) list of expressions denoting the VLA bounds
 	 N in ARGTYPE <arg>[Ni]...[Nj]...[Nk].  */
-      tree argvbs = TREE_CHAIN (argspec);
       if (argvbs)
 	{
 	  spec += ',';
@@ -5890,8 +5956,7 @@ build_attr_access_from_parms (tree parms, bool skip_voidptr)
 	     order.  */
 	  vblist = tree_cons (NULL_TREE, argvbs, vblist);
 
-	  unsigned nelts = 0;
-	  for (tree vb = argvbs; vb; vb = TREE_CHAIN (vb), ++nelts)
+	  for (tree vb = argvbs; vb; vb = TREE_CHAIN (vb))
 	    {
 	      tree bound = TREE_VALUE (vb);
 	      if (const unsigned *psizpos = arg2pos.get (bound))
@@ -6173,7 +6238,9 @@ handle_target_clones_attribute (tree *node, tree name, tree ARG_UNUSED (args),
 	    }
 	}
 
-      if (get_target_clone_attr_len (args) == -1)
+      auto_vec<string_slice> versions = get_clone_attr_versions (args, NULL);
+
+      if (versions.length () == 1)
 	{
 	  warning (OPT_Wattributes,
 		   "single %<target_clones%> attribute is ignored");
diff --git a/gcc/c-family/c-common.cc b/gcc/c-family/c-common.cc
index 9d69298..e7dd460 100644
--- a/gcc/c-family/c-common.cc
+++ b/gcc/c-family/c-common.cc
@@ -56,6 +56,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-pretty-print-markup.h"
 #include "gcc-rich-location.h"
 #include "gcc-urlifier.h"
+#include "diagnostics/diagnostics-selftests.h"
 
 cpp_reader *parse_in;		/* Declared in c-pragma.h.  */
 
@@ -3438,20 +3439,41 @@ pointer_int_sum (location_t loc, enum tree_code resultcode,
 	 an overflow error if the constant is negative but INTOP is not.  */
       && (TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (intop))
 	  || (TYPE_PRECISION (TREE_TYPE (intop))
-	      == TYPE_PRECISION (TREE_TYPE (ptrop)))))
-    {
-      enum tree_code subcode = resultcode;
-      tree int_type = TREE_TYPE (intop);
-      if (TREE_CODE (intop) == MINUS_EXPR)
-	subcode = (subcode == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR);
-      /* Convert both subexpression types to the type of intop,
-	 because weird cases involving pointer arithmetic
-	 can result in a sum or difference with different type args.  */
-      ptrop = build_binary_op (EXPR_LOCATION (TREE_OPERAND (intop, 1)),
-			       subcode, ptrop,
-			       convert (int_type, TREE_OPERAND (intop, 1)),
-			       true);
-      intop = convert (int_type, TREE_OPERAND (intop, 0));
+	      == TYPE_PRECISION (TREE_TYPE (ptrop))))
+      && TYPE_PRECISION (TREE_TYPE (intop)) <= TYPE_PRECISION (sizetype))
+    {
+      tree intop0 = TREE_OPERAND (intop, 0);
+      tree intop1 = TREE_OPERAND (intop, 1);
+      if (TYPE_PRECISION (TREE_TYPE (intop)) != TYPE_PRECISION (sizetype)
+	  || TYPE_UNSIGNED (TREE_TYPE (intop)) != TYPE_UNSIGNED (sizetype))
+	{
+	  tree optype = c_common_type_for_size (TYPE_PRECISION (sizetype),
+						TYPE_UNSIGNED (sizetype));
+	  intop0 = convert (optype, intop0);
+	  intop1 = convert (optype, intop1);
+	}
+      tree t = fold_build2_loc (loc, MULT_EXPR, TREE_TYPE (intop0), intop0,
+				convert (TREE_TYPE (intop0), size_exp));
+      intop0 = convert (sizetype, t);
+      if (TREE_OVERFLOW_P (intop0) && !TREE_OVERFLOW (t))
+	intop0 = wide_int_to_tree (TREE_TYPE (intop0), wi::to_wide (intop0));
+      t = fold_build2_loc (loc, MULT_EXPR, TREE_TYPE (intop1), intop1,
+			   convert (TREE_TYPE (intop1), size_exp));
+      intop1 = convert (sizetype, t);
+      if (TREE_OVERFLOW_P (intop1) && !TREE_OVERFLOW (t))
+	intop1 = wide_int_to_tree (TREE_TYPE (intop1), wi::to_wide (intop1));
+      intop = build_binary_op (EXPR_LOCATION (intop), TREE_CODE (intop),
+			       intop0, intop1, true);
+
+      /* Create the sum or difference.  */
+      if (resultcode == MINUS_EXPR)
+	intop = fold_build1_loc (loc, NEGATE_EXPR, sizetype, intop);
+
+      ret = fold_build_pointer_plus_loc (loc, ptrop, intop);
+
+      fold_undefer_and_ignore_overflow_warnings ();
+
+      return ret;
     }
 
   /* Convert the integer argument to a type the same size as sizetype
@@ -7004,7 +7026,7 @@ c_parse_error (const char *gmsgid, enum cpp_ttype token_type,
 /* Return the gcc option code associated with the reason for a cpp
    message, or 0 if none.  */
 
-static diagnostic_option_id
+static diagnostics::option_id
 c_option_controlling_cpp_diagnostic (enum cpp_warning_reason reason)
 {
   const struct cpp_reason_option_codes_t *entry;
@@ -7046,8 +7068,8 @@ c_cpp_diagnostic (cpp_reader *pfile ATTRIBUTE_UNUSED,
 		  rich_location *richloc,
 		  const char *msg, va_list *ap)
 {
-  diagnostic_info diagnostic;
-  diagnostic_t dlevel;
+  diagnostics::diagnostic_info diagnostic;
+  enum diagnostics::kind dlevel;
   bool save_warn_system_headers = global_dc->m_warn_system_headers;
   bool ret;
 
@@ -7061,24 +7083,24 @@ c_cpp_diagnostic (cpp_reader *pfile ATTRIBUTE_UNUSED,
     case CPP_DL_WARNING:
       if (flag_no_output)
 	return false;
-      dlevel = DK_WARNING;
+      dlevel = diagnostics::kind::warning;
       break;
     case CPP_DL_PEDWARN:
       if (flag_no_output && !flag_pedantic_errors)
 	return false;
-      dlevel = DK_PEDWARN;
+      dlevel = diagnostics::kind::pedwarn;
       break;
     case CPP_DL_ERROR:
-      dlevel = DK_ERROR;
+      dlevel = diagnostics::kind::error;
       break;
     case CPP_DL_ICE:
-      dlevel = DK_ICE;
+      dlevel = diagnostics::kind::ice;
       break;
     case CPP_DL_NOTE:
-      dlevel = DK_NOTE;
+      dlevel = diagnostics::kind::note;
       break;
     case CPP_DL_FATAL:
-      dlevel = DK_FATAL;
+      dlevel = diagnostics::kind::fatal;
       break;
     default:
       gcc_unreachable ();
@@ -9942,8 +9964,11 @@ c_family_tests (void)
   c_indentation_cc_tests ();
   c_pretty_print_cc_tests ();
   c_spellcheck_cc_tests ();
-  c_diagnostic_cc_tests ();
   c_opt_problem_cc_tests ();
+
+  /* According to https://gcc.gnu.org/pipermail/gcc/2021-November/237703.html
+     this has some language-specific assumptions, so we run it here.  */
+  diagnostics::selftest::context_cc_tests ();
 }
 
 } // namespace selftest
@@ -10025,7 +10050,7 @@ try_to_locate_new_include_insertion_point (const char *file, location_t loc)
     return UNKNOWN_LOCATION;
 
   /* The "start_location" is column 0, meaning "the whole line".
-     rich_location and edit_context can't cope with this, so use
+     rich_location and diagnostics::changes can't cope with this, so use
      column 1 instead.  */
   location_t col_0 = ord_map_for_insertion->start_location;
   return linemap_position_for_loc_and_offset (line_table, col_0, 1);
@@ -10089,7 +10114,7 @@ maybe_add_include_fixit (rich_location *richloc, const char *header,
   richloc->add_fixit_insert_before (include_insert_loc, text);
   free (text);
 
-  if (override_location && global_dc->m_source_printing.enabled)
+  if (override_location && global_dc->get_source_printing_options ().enabled)
     {
       /* Replace the primary location with that of the insertion point for the
 	 fix-it hint.
diff --git a/gcc/c-family/c-common.h b/gcc/c-family/c-common.h
index 675708d..7c7e21d 100644
--- a/gcc/c-family/c-common.h
+++ b/gcc/c-family/c-common.h
@@ -979,7 +979,7 @@ extern tree build_va_arg (location_t, tree, tree);
 
 extern const unsigned int c_family_lang_mask;
 extern unsigned int c_common_option_lang_mask (void);
-extern void c_common_diagnostics_set_defaults (diagnostic_context *);
+extern void c_common_diagnostics_set_defaults (diagnostics::context *);
 extern bool c_common_complain_wrong_lang_p (const struct cl_option *);
 extern void c_common_init_options_struct (struct gcc_options *);
 extern void c_common_init_options (unsigned int, struct cl_decoded_option *);
@@ -1265,7 +1265,7 @@ extern void c_stddef_cpp_builtins (void);
 extern void fe_file_change (const line_map_ordinary *);
 extern void c_parse_error (const char *, enum cpp_ttype, tree, unsigned char,
 			   rich_location *richloc);
-extern diagnostic_option_id get_option_for_builtin_define (const char *macro_name);
+extern diagnostics::option_id get_option_for_builtin_define (const char *macro_name);
 
 /* In c-ppoutput.cc  */
 extern void init_pp_output (FILE *);
@@ -1729,7 +1729,6 @@ extern tree braced_lists_to_strings (tree, tree);
 namespace selftest {
   /* Declarations for specific families of tests within c-family,
      by source file, in alphabetical order.  */
-  extern void c_diagnostic_cc_tests (void);
   extern void c_format_cc_tests (void);
   extern void c_indentation_cc_tests (void);
   extern void c_opt_problem_cc_tests (void);
diff --git a/gcc/c-family/c-cppbuiltin.cc b/gcc/c-family/c-cppbuiltin.cc
index 459fd86..4aea902 100644
--- a/gcc/c-family/c-cppbuiltin.cc
+++ b/gcc/c-family/c-cppbuiltin.cc
@@ -1087,6 +1087,7 @@ c_cpp_builtins (cpp_reader *pfile)
 	{
 	  /* Set feature test macros for C++26.  */
 	  cpp_define (pfile, "__cpp_constexpr=202406L");
+	  cpp_define (pfile, "__cpp_constexpr_exceptions=202411L");
 	  cpp_define (pfile, "__cpp_static_assert=202306L");
 	  cpp_define (pfile, "__cpp_placeholder_variables=202306L");
 	  cpp_define (pfile, "__cpp_structured_bindings=202403L");
@@ -1095,6 +1096,7 @@ c_cpp_builtins (cpp_reader *pfile)
 	  cpp_define (pfile, "__cpp_pack_indexing=202311L");
 	  cpp_define (pfile, "__cpp_pp_embed=202502L");
 	  cpp_define (pfile, "__cpp_constexpr_virtual_inheritance=202506L");
+	  cpp_define (pfile, "__cpp_trivial_relocatability=202502L");
 	}
       if (flag_concepts && cxx_dialect > cxx14)
 	cpp_define (pfile, "__cpp_concepts=202002L");
@@ -1679,7 +1681,7 @@ c_cpp_builtins (cpp_reader *pfile)
 /* Given NAME, return the command-line option that would make it be
    a builtin define, or 0 if unrecognized.  */
 
-diagnostic_option_id
+diagnostics::option_id
 get_option_for_builtin_define (const char *name)
 {
   if (!strcmp (name, "_OPENACC"))
diff --git a/gcc/c-family/c-format.cc b/gcc/c-family/c-format.cc
index a44249a..bf144d0 100644
--- a/gcc/c-family/c-format.cc
+++ b/gcc/c-family/c-format.cc
@@ -32,7 +32,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "diagnostic.h"
 #include "substring-locations.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
+#include "diagnostics/selftest-context.h"
+#include "diagnostics/file-cache.h"
 #include "builtins.h"
 #include "attribs.h"
 #include "c-family/c-type-mismatch.h"
@@ -70,6 +71,7 @@ static GTY(()) tree local_event_ptr_node;
 static GTY(()) tree local_pp_element_ptr_node;
 static GTY(()) tree local_gimple_ptr_node;
 static GTY(()) tree local_cgraph_node_ptr_node;
+static GTY(()) tree local_string_slice_node;
 static GTY(()) tree locus;
 
 static bool decode_format_attr (const_tree, tree, tree, function_format_info *,
@@ -770,6 +772,7 @@ static const format_char_info asm_fprintf_char_table[] =
   { "p",   1, STD_C89, { T89_V,   BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN  }, "q",  "c",  NULL }, \
   { "r",   1, STD_C89, { T89_C,   BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN  }, "",    "//cR",   NULL }, \
   { "@",   1, STD_C89, { T_EVENT_PTR,   BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN  }, "", "\"",   NULL }, \
+  { "B",   1, STD_C89, { T_STRING_SLICE,   BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN  }, "q", "",   NULL }, \
   { "e",   1, STD_C89, { T_PP_ELEMENT_PTR,   BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN,  BADLEN  }, "", "\"", NULL }, \
   { "<",   0, STD_C89, NOARGUMENTS, "",      "<",   NULL }, \
   { ">",   0, STD_C89, NOARGUMENTS, "",      ">",   NULL }, \
@@ -4632,7 +4635,7 @@ get_corrected_substring (const substring_loc &fmt_loc,
   if (caret.column > finish.column)
     return NULL;
 
-  char_span line
+  diagnostics::char_span line
     = global_dc->get_file_cache ().get_source_line (start.file, start.line);
   if (!line)
     return NULL;
@@ -4644,7 +4647,8 @@ get_corrected_substring (const substring_loc &fmt_loc,
      specification, up to the (but not including) the length modifier.
      In the above example, this would be "%-+*.*".  */
   int length_up_to_type = caret.column - start.column;
-  char_span prefix_span = line.subspan (start.column - 1, length_up_to_type);
+  diagnostics::char_span prefix_span
+    = line.subspan (start.column - 1, length_up_to_type);
   char *prefix = prefix_span.xstrdup ();
 
   /* Now attempt to generate a suggestion for the rest of the specification
@@ -5211,6 +5215,11 @@ init_dynamic_diag_info (void)
       || local_cgraph_node_ptr_node == void_type_node)
     local_cgraph_node_ptr_node = get_named_type ("cgraph_node");
 
+  /* Similar to the above but for string_slice*.  */
+  if (!local_string_slice_node
+      || local_string_slice_node == void_type_node)
+    local_string_slice_node = get_named_type ("string_slice");
+
   /* Similar to the above but for diagnostic_event_id_t*.  */
   if (!local_event_ptr_node
       || local_event_ptr_node == void_type_node)
@@ -5577,10 +5586,12 @@ test_type_mismatch_range_labels ()
   gcc_rich_location richloc (fmt, &fmt_label, nullptr);
   richloc.add_range (param, SHOW_RANGE_WITHOUT_CARET, &param_label);
 
-  test_diagnostic_context dc;
+  diagnostics::selftest::test_context dc;
   diagnostic_show_locus (&dc,
-			 dc.m_source_printing,
-			 &richloc, DK_ERROR, dc.get_reference_printer ());
+			 dc.get_source_printing_options (),
+			 &richloc,
+			 diagnostics::kind::error,
+			 dc.get_reference_printer ());
   if (c_dialect_cxx ())
     /* "char*", without a space.  */
     ASSERT_STREQ ("   printf (\"msg: %i\\n\", msg);\n"
diff --git a/gcc/c-family/c-format.h b/gcc/c-family/c-format.h
index 323338c..d44d386 100644
--- a/gcc/c-family/c-format.h
+++ b/gcc/c-family/c-format.h
@@ -317,6 +317,7 @@ struct format_kind_info
 #define T89_G   { STD_C89, NULL, &local_gimple_ptr_node }
 #define T_CGRAPH_NODE   { STD_C89, NULL, &local_cgraph_node_ptr_node }
 #define T_EVENT_PTR    { STD_C89, NULL, &local_event_ptr_node }
+#define T_STRING_SLICE    { STD_C89, NULL, &local_string_slice_node }
 #define T_PP_ELEMENT_PTR    { STD_C89, NULL, &local_pp_element_ptr_node }
 #define T89_T   { STD_C89, NULL, &local_tree_type_node }
 #define T89_V	{ STD_C89, NULL, T_V }
diff --git a/gcc/c-family/c-gimplify.cc b/gcc/c-family/c-gimplify.cc
index e905059..c6fb764 100644
--- a/gcc/c-family/c-gimplify.cc
+++ b/gcc/c-family/c-gimplify.cc
@@ -66,20 +66,6 @@ along with GCC; see the file COPYING3.  If not see
     walk back up, we check that they fit our constraints, and copy them
     into temporaries if not.  */
 
-
-/* Check whether TP is an address computation whose base is a call to
-   .ACCESS_WITH_SIZE.  */
-
-static bool
-is_address_with_access_with_size (tree tp)
-{
-  if (TREE_CODE (tp) == POINTER_PLUS_EXPR
-      && (TREE_CODE (TREE_OPERAND (tp, 0)) == INDIRECT_REF)
-      && (is_access_with_size_p (TREE_OPERAND (TREE_OPERAND (tp, 0), 0))))
-	return true;
-  return false;
-}
-
 /* Callback for c_genericize.  */
 
 static tree
@@ -135,20 +121,6 @@ ubsan_walk_array_refs_r (tree *tp, int *walk_subtrees, void *data)
       walk_tree (&TREE_OPERAND (*tp, 1), ubsan_walk_array_refs_r, pset, pset);
       walk_tree (&TREE_OPERAND (*tp, 0), ubsan_walk_array_refs_r, pset, pset);
     }
-  else if (TREE_CODE (*tp) == INDIRECT_REF
-	   && is_address_with_access_with_size (TREE_OPERAND (*tp, 0)))
-    {
-      ubsan_maybe_instrument_array_ref (&TREE_OPERAND (*tp, 0), false);
-      /* Make sure ubsan_maybe_instrument_array_ref is not called again on
-	 the POINTER_PLUS_EXPR, so ensure it is not walked again and walk
-	 its subtrees manually.  */
-      tree aref = TREE_OPERAND (*tp, 0);
-      pset->add (aref);
-      *walk_subtrees = 0;
-      walk_tree (&TREE_OPERAND (aref, 0), ubsan_walk_array_refs_r, pset, pset);
-    }
-  else if (is_address_with_access_with_size (*tp))
-    ubsan_maybe_instrument_array_ref (tp, true);
   return NULL_TREE;
 }
 
diff --git a/gcc/c-family/c-indentation.cc b/gcc/c-family/c-indentation.cc
index 2e8261d..bb214fc 100644
--- a/gcc/c-family/c-indentation.cc
+++ b/gcc/c-family/c-indentation.cc
@@ -25,6 +25,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "c-indentation.h"
 #include "selftest.h"
 #include "diagnostic.h"
+#include "diagnostics/file-cache.h"
 
 /* Round up VIS_COLUMN to nearest tab stop. */
 
@@ -45,13 +46,13 @@ next_tab_stop (unsigned int vis_column, unsigned int tab_width)
    on the line (up to or before EXPLOC).  */
 
 static bool
-get_visual_column (file_cache &fc,
+get_visual_column (diagnostics::file_cache &fc,
 		   expanded_location exploc,
 		   unsigned int *out,
 		   unsigned int *first_nws,
 		   unsigned int tab_width)
 {
-  char_span line = fc.get_source_line (exploc.file, exploc.line);
+  diagnostics::char_span line = fc.get_source_line (exploc.file, exploc.line);
   if (!line)
     return false;
   if ((size_t)exploc.column > line.length ())
@@ -88,14 +89,14 @@ get_visual_column (file_cache &fc,
    Otherwise, return false, leaving *FIRST_NWS untouched.  */
 
 static bool
-get_first_nws_vis_column (file_cache &fc,
+get_first_nws_vis_column (diagnostics::file_cache &fc,
 			  const char *file, int line_num,
 			  unsigned int *first_nws,
 			  unsigned int tab_width)
 {
   gcc_assert (first_nws);
 
-  char_span line = fc.get_source_line (file, line_num);
+  diagnostics::char_span line = fc.get_source_line (file, line_num);
   if (!line)
     return false;
   unsigned int vis_column = 0;
@@ -160,7 +161,7 @@ get_first_nws_vis_column (file_cache &fc,
    Return true if such an unindent/outdent is detected.  */
 
 static bool
-detect_intervening_unindent (file_cache &fc,
+detect_intervening_unindent (diagnostics::file_cache &fc,
 			     const char *file,
 			     int body_line,
 			     int next_stmt_line,
@@ -335,7 +336,7 @@ should_warn_for_misleading_indentation (const token_indent_info &guard_tinfo,
   if (next_stmt_exploc.file != body_exploc.file)
     return false;
 
-  file_cache &fc = global_dc->get_file_cache ();
+  diagnostics::file_cache &fc = global_dc->get_file_cache ();
 
   /* If NEXT_STMT_LOC and BODY_LOC are on the same line, consider
      the location of the guard.
@@ -691,7 +692,7 @@ test_next_tab_stop ()
 
 static void
 assert_get_visual_column_succeeds (const location &loc,
-				   file_cache &fc,
+				   diagnostics::file_cache &fc,
 				   const char *file, int line, int column,
 				   const unsigned int tab_width,
 				   unsigned int expected_visual_column,
@@ -735,7 +736,7 @@ assert_get_visual_column_succeeds (const location &loc,
 
 static void
 assert_get_visual_column_fails (const location &loc,
-				file_cache &fc,
+				diagnostics::file_cache &fc,
 				const char *file, int line, int column,
 				const unsigned int tab_width)
 {
@@ -783,7 +784,7 @@ test_get_visual_column ()
 			 "\t line 2\n");
   line_table_test ltt;
   temp_source_file tmp (SELFTEST_LOCATION, ".txt", content);
-  file_cache fc;
+  diagnostics::file_cache fc;
 
   const unsigned int tab_width = 8;
   const char *file = tmp.get_filename ();
diff --git a/gcc/c-family/c-lex.cc b/gcc/c-family/c-lex.cc
index fef6ae6..b45d722 100644
--- a/gcc/c-family/c-lex.cc
+++ b/gcc/c-family/c-lex.cc
@@ -1176,7 +1176,7 @@ interpret_integer (const cpp_token *token, unsigned int flags,
 	  && (flags & CPP_N_WIDTH) != CPP_N_LARGE)
 	emit_diagnostic
 	  ((c_dialect_cxx () ? cxx_dialect == cxx98 : !flag_isoc99)
-	   ? DK_PEDWARN : DK_WARNING,
+	   ? diagnostics::kind::pedwarn : diagnostics::kind::warning,
 	   input_location, OPT_Wlong_long,
 	   (flags & CPP_N_UNSIGNED)
 	   ? "integer constant is too large for %<unsigned long%> type"
diff --git a/gcc/c-family/c-omp.cc b/gcc/c-family/c-omp.cc
index 4352214..fe272888 100644
--- a/gcc/c-family/c-omp.cc
+++ b/gcc/c-family/c-omp.cc
@@ -769,9 +769,7 @@ c_finish_omp_depobj (location_t loc, tree depobj,
 	  kind = OMP_CLAUSE_DEPEND_KIND (clause);
 	  t = OMP_CLAUSE_DECL (clause);
 	  gcc_assert (t);
-	  if (TREE_CODE (t) == TREE_LIST
-	      && TREE_PURPOSE (t)
-	      && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	  if (OMP_ITERATOR_DECL_P (t))
 	    {
 	      error_at (OMP_CLAUSE_LOCATION (clause),
 			"%<iterator%> modifier may not be specified on "
diff --git a/gcc/c-family/c-opts.cc b/gcc/c-family/c-opts.cc
index 3ee9444..c652e82 100644
--- a/gcc/c-family/c-opts.cc
+++ b/gcc/c-family/c-opts.cc
@@ -32,7 +32,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "flags.h"
 #include "toplev.h"
 #include "langhooks.h"
-#include "diagnostic-macro-unwinding.h" /* for virt_loc_aware_diagnostic_finalizer */
+#include "diagnostics/macro-unwinding.h" /* for virt_loc_aware_diagnostic_finalizer */
 #include "intl.h"
 #include "cppdefault.h"
 #include "incpath.h"
@@ -43,7 +43,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "dumpfile.h"
 #include "file-prefix-map.h"    /* add_*_prefix_map()  */
 #include "context.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 
 #ifndef DOLLARS_IN_IDENTIFIERS
 # define DOLLARS_IN_IDENTIFIERS true
@@ -169,9 +169,9 @@ c_common_option_lang_mask (void)
 
 /* Diagnostic finalizer for C/C++/Objective-C/Objective-C++.  */
 static void
-c_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
-			     const diagnostic_info *diagnostic,
-			     diagnostic_t)
+c_diagnostic_text_finalizer (diagnostics::text_sink &text_output,
+			     const diagnostics::diagnostic_info *diagnostic,
+			     enum diagnostics::kind)
 {
   pretty_printer *const pp = text_output.get_printer ();
   char *saved_prefix = pp_take_prefix (pp);
@@ -179,19 +179,19 @@ c_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
   pp_newline (pp);
   diagnostic_show_locus (&text_output.get_context (),
 			 text_output.get_source_printing_options (),
-			 diagnostic->richloc, diagnostic->kind, pp);
+			 diagnostic->m_richloc, diagnostic->m_kind, pp);
   /* By default print macro expansion contexts in the diagnostic
      finalizer -- for tokens resulting from macro expansion.  */
-  virt_loc_aware_diagnostic_finalizer (text_output, diagnostic);
+  diagnostics::virt_loc_aware_text_finalizer (text_output, diagnostic);
   pp_set_prefix (pp, saved_prefix);
   pp_flush (pp);
 }
 
 /* Common default settings for diagnostics.  */
 void
-c_common_diagnostics_set_defaults (diagnostic_context *context)
+c_common_diagnostics_set_defaults (diagnostics::context *context)
 {
-  diagnostic_text_finalizer (context) = c_diagnostic_text_finalizer;
+  diagnostics::text_finalizer (context) = c_diagnostic_text_finalizer;
   context->set_permissive_option (OPT_fpermissive);
 }
 
@@ -278,7 +278,7 @@ c_common_init_options (unsigned int decoded_options_count,
   if (c_dialect_cxx ())
     set_std_cxx17 (/*ISO*/false);
 
-  global_dc->m_source_printing.colorize_source_p = true;
+  global_dc->get_source_printing_options ().colorize_source_p = true;
 }
 
 /* Handle switch SCODE with argument ARG.  VALUE is true, unless no-
@@ -1165,6 +1165,9 @@ c_common_post_options (const char **pfilename)
       warn_cxx20_compat = 0;
       cpp_opts->cpp_warn_cxx20_compat = 0;
     }
+  if (cxx_dialect >= cxx26)
+    /* Don't warn about C++26 compatibility changes in C++26 or later.  */
+    warn_cxx26_compat = 0;
 
   /* C++17 has stricter evaluation order requirements; let's use some of them
      for earlier C++ as well, so chaining works as expected.  */
@@ -1223,6 +1226,17 @@ c_common_post_options (const char **pfilename)
   SET_OPTION_IF_UNSET (&global_options, &global_options_set,
 		       flag_range_for_ext_temps, cxx_dialect >= cxx23);
 
+  /* EnabledBy unfortunately can't specify value to use if set and
+     LangEnabledBy can't specify multiple options with &&.  For -Wunused
+     or -Wunused -Wextra we want these to default to 3 unless user specified
+     some other level explicitly.  */
+  if (warn_unused_but_set_parameter == 1)
+    SET_OPTION_IF_UNSET (&global_options, &global_options_set,
+			 warn_unused_but_set_parameter, 3);
+  if (warn_unused_but_set_variable == 1)
+    SET_OPTION_IF_UNSET (&global_options, &global_options_set,
+			 warn_unused_but_set_variable, 3);
+
   /* -fimmediate-escalation has no effect when immediate functions are not
      supported.  */
   if (flag_immediate_escalation && cxx_dialect < cxx20)
diff --git a/gcc/c-family/c-pragma.cc b/gcc/c-family/c-pragma.cc
index 137b83bf..8a1218b 100644
--- a/gcc/c-family/c-pragma.cc
+++ b/gcc/c-family/c-pragma.cc
@@ -781,7 +781,7 @@ public:
       PK_IGNORED_ATTRIBUTES,
       PK_DIAGNOSTIC,
     } pd_kind;
-  diagnostic_t diagnostic_kind;
+  enum diagnostics::kind diagnostic_kind;
   const char *kind_str;
   const char *option_str;
   bool own_option_str;
@@ -792,7 +792,7 @@ public:
     valid = false;
     loc_kind = loc_option = UNKNOWN_LOCATION;
     pd_kind = PK_INVALID;
-    diagnostic_kind = DK_UNSPECIFIED;
+    diagnostic_kind = diagnostics::kind::unspecified;
     kind_str = option_str = nullptr;
     own_option_str = false;
   }
@@ -808,7 +808,7 @@ public:
     kind_str = kind_string;
 
     pd_kind = PK_INVALID;
-    diagnostic_kind = DK_UNSPECIFIED;
+    diagnostic_kind = diagnostics::kind::unspecified;
     if (strcmp (kind_str, "push") == 0)
       pd_kind = PK_PUSH;
     else if (strcmp (kind_str, "pop") == 0)
@@ -818,17 +818,17 @@ public:
     else if (strcmp (kind_str, "error") == 0)
       {
 	pd_kind = PK_DIAGNOSTIC;
-	diagnostic_kind = DK_ERROR;
+	diagnostic_kind = diagnostics::kind::error;
       }
     else if (strcmp (kind_str, "warning") == 0)
       {
 	pd_kind = PK_DIAGNOSTIC;
-	diagnostic_kind = DK_WARNING;
+	diagnostic_kind = diagnostics::kind::warning;
       }
     else if (strcmp (kind_str, "ignored") == 0)
       {
 	pd_kind = PK_DIAGNOSTIC;
-	diagnostic_kind = DK_IGNORED;
+	diagnostic_kind = diagnostics::kind::ignored;
       }
   }
 
@@ -1016,7 +1016,8 @@ handle_pragma_diagnostic_impl ()
      what we used to do here before and changing it breaks e.g.
      PR69543 and PR69558.  */
   control_warning_option (option_index, (int) data.diagnostic_kind,
-			  arg, data.diagnostic_kind != DK_IGNORED,
+			  arg,
+			  data.diagnostic_kind != diagnostics::kind::ignored,
 			  input_location, lang_mask, &handlers,
 			  &global_options, &global_options_set,
 			  global_dc);
diff --git a/gcc/c-family/c-ubsan.cc b/gcc/c-family/c-ubsan.cc
index 38514a4..a4dc310 100644
--- a/gcc/c-family/c-ubsan.cc
+++ b/gcc/c-family/c-ubsan.cc
@@ -397,8 +397,7 @@ get_bound_from_access_with_size (tree call)
     return NULL_TREE;
 
   tree ref_to_size = CALL_EXPR_ARG (call, 1);
-  unsigned int class_of_size = TREE_INT_CST_LOW (CALL_EXPR_ARG (call, 2));
-  tree type = TREE_TYPE (CALL_EXPR_ARG (call, 3));
+  tree type = TREE_TYPE (TREE_TYPE (CALL_EXPR_ARG (call, 2)));
   tree size = fold_build2 (MEM_REF, type, unshare_expr (ref_to_size),
 			   build_int_cst (ptr_type_node, 0));
   /* If size is negative value, treat it as zero.  */
@@ -410,12 +409,7 @@ get_bound_from_access_with_size (tree call)
 			build_zero_cst (type), size);
   }
 
-  /* Only when class_of_size is 1, i.e, the number of the elements of
-     the object type, return the size.  */
-  if (class_of_size != 1)
-    return NULL_TREE;
-  else
-    size = fold_convert (sizetype, size);
+  size = fold_convert (sizetype, size);
 
   return size;
 }
@@ -554,322 +548,38 @@ ubsan_instrument_bounds (location_t loc, tree array, tree *index,
 				       *index, bound);
 }
 
-
-/* Instrument array bounds for the pointer array address which is
-   an INDIRECT_REF to the call to .ACCESS_WITH_SIZE.  We create special
-   builtin, that gets expanded in the sanopt pass, and make an array
-   dimention of it.  POINTER_ADDR is the pointer array's base address.
-   *INDEX is an index to the array.
-   IGNORE_OFF_BY_ONE is true if the POINTER_ADDR is not inside an
-   INDIRECT_REF.
-   Return NULL_TREE if no instrumentation is emitted.  */
-
-tree
-ubsan_instrument_bounds_pointer_address (location_t loc, tree pointer_addr,
-					 tree *index,
-					 bool ignore_off_by_one)
-{
-  gcc_assert (TREE_CODE (pointer_addr) == INDIRECT_REF);
-  tree call = TREE_OPERAND (pointer_addr, 0);
-  if (!is_access_with_size_p (call))
-    return NULL_TREE;
-  tree bound = get_bound_from_access_with_size (call);
-
-  if (ignore_off_by_one)
-    bound = fold_build2 (PLUS_EXPR, TREE_TYPE (bound), bound,
-			 build_int_cst (TREE_TYPE (bound),
-			 1));
-
-  /* Don't emit instrumentation in the most common cases.  */
-  tree idx = NULL_TREE;
-  if (TREE_CODE (*index) == INTEGER_CST)
-    idx = *index;
-  else if (TREE_CODE (*index) == BIT_AND_EXPR
-	   && TREE_CODE (TREE_OPERAND (*index, 1)) == INTEGER_CST)
-    idx = TREE_OPERAND (*index, 1);
-  if (idx
-      && TREE_CODE (bound) == INTEGER_CST
-      && tree_int_cst_sgn (idx) >= 0
-      && tree_int_cst_lt (idx, bound))
-    return NULL_TREE;
-
-  *index = save_expr (*index);
-
-  /* Create an array_type for the corresponding pointer array.  */
-  tree itype = build_range_type (sizetype, size_zero_node, NULL_TREE);
-  /* The array's element type can be get from the return type of the call to
-     .ACCESS_WITH_SIZE.  */
-  tree element_type = TREE_TYPE (TREE_TYPE (TREE_TYPE (call)));
-  tree array_type = build_array_type (element_type, itype);
-  /* Create a "(T *) 0" tree node to describe the array type.  */
-  tree zero_with_type = build_int_cst (build_pointer_type (array_type), 0);
-  return build_call_expr_internal_loc (loc, IFN_UBSAN_BOUNDS,
-				       void_type_node, 3, zero_with_type,
-				       *index, bound);
-}
-
-/* This structure is to combine a factor with its parent and its position
- * in its parent tree.  */
-struct factor_t
-{
-  tree factor;
-  tree parent; /* the parent tree of this factor.  */
-  int pos;     /* the position of this factor in its parent tree.  */
-};
-
-/* for a multiply expression like:
-    ((long unsigned int) m * (long unsigned int) SAVE_EXPR <n>) * 4
-
-   locate all the factors, the parents of the factor and the position of
-   the factor in its parent, and put them to VEC_FACTORS.  */
-
-static void
-get_factors_from_mul_expr (tree mult_expr, tree parent,
-			   int pos, auto_vec<factor_t> *vec_factors)
-{
-  struct factor_t mult_factor = {0, 0, -1};
-  mult_factor.factor = mult_expr;
-  mult_factor.parent = parent;
-  mult_factor.pos = pos;
-
-  while (CONVERT_EXPR_CODE_P (TREE_CODE (mult_expr)))
-    {
-      mult_factor.parent = mult_expr;
-      mult_factor.pos = 0;
-      mult_expr = TREE_OPERAND (mult_expr, 0);
-      mult_factor.factor = mult_expr;
-    }
-  if (TREE_CODE (mult_expr) != MULT_EXPR)
-    vec_factors->safe_push (mult_factor);
-  else
-    {
-      get_factors_from_mul_expr (TREE_OPERAND (mult_expr, 0), mult_expr,
-				 0, vec_factors);
-      get_factors_from_mul_expr (TREE_OPERAND (mult_expr, 1), mult_expr,
-				 1, vec_factors);
-    }
-}
-
-/* Given an OFFSET expression, and the ELEMENT_SIZE,
-   get the index expression from OFFSET and return it.
-   For example:
-   OFFSET:
-    ((long unsigned int) m * (long unsigned int) SAVE_EXPR <n>) * 4
-   ELEMENT_SIZE:
-    (sizetype) SAVE_EXPR <n> * 4
-   get the index as (long unsigned int) m, and return it.
-   The INDEX_P holds the pointer to the parent tree of the index,
-   INDEX_N holds the position of the index in its parent.  */
-
-static tree
-get_index_from_offset (tree offset, tree *index_p,
-		       int *index_n, tree element_size)
-{
-  if (TREE_CODE (offset) != MULT_EXPR)
-    return NULL_TREE;
-
-  auto_vec<factor_t> e_factors, o_factors;
-  get_factors_from_mul_expr (element_size, NULL, -1, &e_factors);
-  get_factors_from_mul_expr (offset, *index_p, *index_n, &o_factors);
-
-  if (e_factors.is_empty () || o_factors.is_empty ())
-    return NULL_TREE;
-
-  bool all_found = true;
-  for (unsigned i = 0; i < e_factors.length (); i++)
-    {
-      factor_t e_size_factor = e_factors[i];
-      bool found = false;
-      for (unsigned j = 0; j < o_factors.length ();)
-	{
-	  factor_t o_exp_factor = o_factors[j];
-	  if (operand_equal_p (e_size_factor.factor, o_exp_factor.factor))
-	    {
-	      o_factors.unordered_remove (j);
-	      found = true;
-	      break;
-	    }
-	  else
-	    j++;
-	}
-      if (!found)
-	all_found = false;
-    }
-
-  if (!all_found)
-    return NULL_TREE;
-
-  if (o_factors.length () != 1)
-    return NULL_TREE;
-
-  *index_p = o_factors[0].parent;
-  *index_n = o_factors[0].pos;
-  return o_factors[0].factor;
-}
-
-/* For an pointer + offset computation expression, such as,
-   *.ACCESS_WITH_SIZE (p->c, &p->b, 1, 0, -1, 0B)
-    + (sizetype) ((long unsigned int) index * 4
-   Return the index of this pointer array reference,
-   set the parent tree of INDEX to *INDEX_P.
-   set the operand position of the INDEX in the parent tree to *INDEX_N.
-   If failed, return NULL_TREE.  */
-
-static tree
-get_index_from_pointer_addr_expr (tree pointer, tree *index_p, int *index_n)
-{
-  *index_p = NULL_TREE;
-  *index_n = -1;
-  if (TREE_CODE (TREE_OPERAND (pointer, 0)) != INDIRECT_REF)
-    return NULL_TREE;
-  tree call = TREE_OPERAND (TREE_OPERAND (pointer, 0), 0);
-  if (!is_access_with_size_p (call))
-    return NULL_TREE;
-
-  /* Get the pointee type of the call to .ACCESS_WITH_SIZE.
-     This should be the element type of the pointer array.  */
-  tree pointee_type = TREE_TYPE (TREE_TYPE (TREE_TYPE (call)));
-  tree pointee_size = TYPE_SIZE_UNIT (pointee_type);
-
-  tree index_exp = TREE_OPERAND (pointer, 1);
-  *index_p = pointer;
-  *index_n = 1;
-
-  if (!(TREE_CODE (index_exp) != MULT_EXPR
-	&& tree_int_cst_equal (pointee_size, integer_one_node)))
-    {
-      while (CONVERT_EXPR_CODE_P (TREE_CODE (index_exp)))
-	{
-	  *index_p = index_exp;
-	  *index_n = 0;
-	  index_exp = TREE_OPERAND (index_exp, 0);
-	}
-      index_exp = get_index_from_offset (index_exp, index_p,
-					 index_n, pointee_size);
-
-      if (!index_exp)
-      return NULL_TREE;
-    }
-
-  while (CONVERT_EXPR_CODE_P (TREE_CODE (index_exp)))
-    {
-      *index_p = index_exp;
-      *index_n = 0;
-      index_exp = TREE_OPERAND (index_exp, 0);
-    }
-
-  return index_exp;
-}
-
-/* Return TRUE when the EXPR is a pointer array address that could be
-   instrumented.
-   We only instrument an address computation similar as the following:
-      *.ACCESS_WITH_SIZE (p->c, &p->b, 1, 0, -1, 0B)
-	+ (sizetype) ((long unsigned int) index * 4)
-   if the EXPR is instrumentable, return TRUE and
-   set the index to *INDEX.
-   set the *.ACCESS_WITH_SIZE to *BASE.
-   set the parent tree of INDEX to *INDEX_P.
-   set the operand position of the INDEX in the parent tree to INDEX_N.  */
-
-static bool
-is_instrumentable_pointer_array_address (tree expr, tree *base,
-					 tree *index, tree *index_p,
-					 int *index_n)
-{
-  /* For a poiner array address as:
-	(*.ACCESS_WITH_SIZE (p->c, &p->b, 1, 0, -1, 0B)
-	  + (sizetype) ((long unsigned int) index * 4)
-     op0 is the call to *.ACCESS_WITH_SIZE;
-     op1 is the index.  */
-  if (TREE_CODE (expr) != POINTER_PLUS_EXPR)
-    return false;
-
-  tree op0 = TREE_OPERAND (expr, 0);
-  if (TREE_CODE (op0) != INDIRECT_REF)
-    return false;
-  if (!is_access_with_size_p (TREE_OPERAND (op0, 0)))
-    return false;
-  tree op1 = get_index_from_pointer_addr_expr (expr, index_p, index_n);
-  if (op1 != NULL_TREE)
-    {
-      *base = op0;
-      *index = op1;
-      return true;
-    }
-  return false;
-}
-
-/* Return true iff T is an array or an indirect reference that was
-   instrumented by SANITIZE_BOUNDS.  */
+/* Return true iff T is an array that was instrumented by SANITIZE_BOUNDS.  */
 
 bool
-ubsan_array_ref_instrumented_p (tree t)
+ubsan_array_ref_instrumented_p (const_tree t)
 {
-  if (TREE_CODE (t) != ARRAY_REF
-      && TREE_CODE (t) != MEM_REF)
+  if (TREE_CODE (t) != ARRAY_REF)
     return false;
 
-  bool is_array = (TREE_CODE (t) == ARRAY_REF);
-  tree op0 = NULL_TREE;
-  tree op1 = NULL_TREE;
-  tree index_p = NULL_TREE;
-  int index_n = 0;
-  if (is_array)
-    {
-      op1 = TREE_OPERAND (t, 1);
-      return TREE_CODE (op1) == COMPOUND_EXPR
-	     && TREE_CODE (TREE_OPERAND (op1, 0)) == CALL_EXPR
-	     && CALL_EXPR_FN (TREE_OPERAND (op1, 0)) == NULL_TREE
-	     && CALL_EXPR_IFN (TREE_OPERAND (op1, 0)) == IFN_UBSAN_BOUNDS;
-    }
-  else if (is_instrumentable_pointer_array_address (t, &op0, &op1,
-						    &index_p, &index_n))
-    return TREE_CODE (op1) == COMPOUND_EXPR
-	   && TREE_CODE (TREE_OPERAND (op1, 0)) == CALL_EXPR
-	   && CALL_EXPR_FN (TREE_OPERAND (op1, 0)) == NULL_TREE
-	   && CALL_EXPR_IFN (TREE_OPERAND (op1, 0)) == IFN_UBSAN_BOUNDS;
-
-  return false;
+  tree op1 = TREE_OPERAND (t, 1);
+  return TREE_CODE (op1) == COMPOUND_EXPR
+	 && TREE_CODE (TREE_OPERAND (op1, 0)) == CALL_EXPR
+	 && CALL_EXPR_FN (TREE_OPERAND (op1, 0)) == NULL_TREE
+	 && CALL_EXPR_IFN (TREE_OPERAND (op1, 0)) == IFN_UBSAN_BOUNDS;
 }
 
-/* Instrument an ARRAY_REF or an address computation whose base address is
-   a call to .ACCESS_WITH_SIZE, if it hasn't already been instrumented.
-   IGNORE_OFF_BY_ONE is true if the ARRAY_REF is inside a ADDR_EXPR, or the
-   address computation is not inside a INDIRECT_REF.  */
+/* Instrument an ARRAY_REF, if it hasn't already been instrumented.
+   IGNORE_OFF_BY_ONE is true if the ARRAY_REF is inside a ADDR_EXPR.  */
 
 void
 ubsan_maybe_instrument_array_ref (tree *expr_p, bool ignore_off_by_one)
 {
-  tree e = NULL_TREE;
-  tree op0 = NULL_TREE;
-  tree op1 = NULL_TREE;
-  tree index_p = NULL_TREE;  /* the parent tree of INDEX.  */
-  int index_n = 0;  /* the operand position of INDEX in the parent tree.  */
-
   if (!ubsan_array_ref_instrumented_p (*expr_p)
       && sanitize_flags_p (SANITIZE_BOUNDS | SANITIZE_BOUNDS_STRICT)
       && current_function_decl != NULL_TREE)
     {
-      if (TREE_CODE (*expr_p) == ARRAY_REF)
-	{
-	  op0 = TREE_OPERAND (*expr_p, 0);
-	  op1 = TREE_OPERAND (*expr_p, 1);
-	  index_p = *expr_p;
-	  index_n = 1;
-	  e = ubsan_instrument_bounds (EXPR_LOCATION (*expr_p), op0,
-				       &op1, ignore_off_by_one);
-	}
-      else if (is_instrumentable_pointer_array_address (*expr_p, &op0, &op1,
-							&index_p, &index_n))
-	e = ubsan_instrument_bounds_pointer_address (EXPR_LOCATION (*expr_p),
-						     op0, &op1,
-						     ignore_off_by_one);
-
-      /* Replace the original INDEX with the instrumented INDEX.  */
+      tree op0 = TREE_OPERAND (*expr_p, 0);
+      tree op1 = TREE_OPERAND (*expr_p, 1);
+      tree e = ubsan_instrument_bounds (EXPR_LOCATION (*expr_p), op0, &op1,
+					ignore_off_by_one);
       if (e != NULL_TREE)
-	TREE_OPERAND (index_p, index_n)
-	  = build2 (COMPOUND_EXPR, TREE_TYPE (op1), e, op1);
+	TREE_OPERAND (*expr_p, 1) = build2 (COMPOUND_EXPR, TREE_TYPE (op1),
+					    e, op1);
     }
 }
 
diff --git a/gcc/c-family/c-warn.cc b/gcc/c-family/c-warn.cc
index d547b08..09517d2 100644
--- a/gcc/c-family/c-warn.cc
+++ b/gcc/c-family/c-warn.cc
@@ -3835,7 +3835,7 @@ do_warn_array_compare (location_t location, tree_code code, tree op0, tree op1)
     op1 = TREE_OPERAND (op1, 0);
 
   auto_diagnostic_group d;
-  diagnostic_t kind = DK_WARNING;
+  enum diagnostics::kind kind = diagnostics::kind::warning;
   const char *msg;
   if (c_dialect_cxx () && cxx_dialect >= cxx20)
     {
@@ -3843,7 +3843,7 @@ do_warn_array_compare (location_t location, tree_code code, tree op0, tree op1)
       if (cxx_dialect >= cxx26)
 	{
 	  msg = G_("comparison between two arrays is not allowed in C++26");
-	  kind = DK_PERMERROR;
+	  kind = diagnostics::kind::permerror;
 	}
       else
 	msg = G_("comparison between two arrays is deprecated in C++20");
diff --git a/gcc/c-family/c.opt b/gcc/c-family/c.opt
index 8af466d..12877eb 100644
--- a/gcc/c-family/c.opt
+++ b/gcc/c-family/c.opt
@@ -397,6 +397,10 @@ Wassign-intercept
 ObjC ObjC++ Var(warn_assign_intercept) Warning
 Warn whenever an Objective-C assignment is being intercepted by the garbage collector.
 
+Wabbreviated-auto-in-template-arg
+C++ ObjC++ Warning Var(warn_abbev_auto_targ) Init(1)
+Diagnose a placeholder type in a template argument in a function parameter type.
+
 Wbad-function-cast
 C ObjC Var(warn_bad_function_cast) Warning
 Warn about casting functions to incompatible types.
@@ -493,6 +497,10 @@ Wc++20-compat
 C++ ObjC++ Var(warn_cxx20_compat) Warning LangEnabledBy(C++ ObjC++,Wall) Init(0) CPP(cpp_warn_cxx20_compat) CppReason(CPP_W_CXX20_COMPAT)
 Warn about C++ constructs whose meaning differs between ISO C++ 2017 and ISO C++ 2020.
 
+Wc++26-compat
+C++ ObjC++ Var(warn_cxx26_compat) Warning LangEnabledBy(C++ ObjC++,Wall) Init(0)
+Warn about C++ constructs whose meaning differs between ISO C++ 2023 and ISO C++ 2026.
+
 Wc++11-extensions
 C++ ObjC++ Var(warn_cxx11_extensions) Warning Init(1)
 Warn about C++11 constructs in code compiled with an older standard.
diff --git a/gcc/c-family/c.opt.urls b/gcc/c-family/c.opt.urls
index 65d1221..5c97593 100644
--- a/gcc/c-family/c.opt.urls
+++ b/gcc/c-family/c.opt.urls
@@ -139,6 +139,9 @@ UrlSuffix(gcc/Warning-Options.html#index-Warray-parameter)
 Wassign-intercept
 UrlSuffix(gcc/Objective-C-and-Objective-C_002b_002b-Dialect-Options.html#index-Wassign-intercept)
 
+Wabbreviated-auto-in-template-arg
+UrlSuffix(gcc/C_002b_002b-Dialect-Options.html#index-Wabbreviated-auto-in-template-arg)
+
 Wbad-function-cast
 UrlSuffix(gcc/Warning-Options.html#index-Wbad-function-cast)
 
@@ -187,6 +190,9 @@ UrlSuffix(gcc/Warning-Options.html#index-Wc_002b_002b17-compat)
 Wc++20-compat
 UrlSuffix(gcc/Warning-Options.html#index-Wc_002b_002b20-compat)
 
+Wc++26-compat
+UrlSuffix(gcc/Warning-Options.html#index-Wc_002b_002b26-compat)
+
 Wc++11-extensions
 UrlSuffix(gcc/Warning-Options.html#index-Wc_002b_002b11-extensions)
 
diff --git a/gcc/c-family/known-headers.cc b/gcc/c-family/known-headers.cc
index 7e0fa95..e3c0113 100644
--- a/gcc/c-family/known-headers.cc
+++ b/gcc/c-family/known-headers.cc
@@ -329,9 +329,10 @@ suggest_missing_header::~suggest_missing_header ()
 
 /* suggest_missing_option's ctor.  */
 
-suggest_missing_option::suggest_missing_option (location_t loc,
-						const char *macro_name,
-						diagnostic_option_id option_id)
+suggest_missing_option::
+suggest_missing_option (location_t loc,
+			const char *macro_name,
+			diagnostics::option_id option_id)
 : deferred_diagnostic (loc), m_name_str (macro_name), m_option_id (option_id)
 {
   gcc_assert (macro_name);
diff --git a/gcc/c-family/known-headers.h b/gcc/c-family/known-headers.h
index b1da757f..3ffe5f3 100644
--- a/gcc/c-family/known-headers.h
+++ b/gcc/c-family/known-headers.h
@@ -48,12 +48,12 @@ class suggest_missing_option : public deferred_diagnostic
 {
  public:
   suggest_missing_option (location_t loc, const char *name,
-			  diagnostic_option_id option_id);
+			  diagnostics::option_id option_id);
   ~suggest_missing_option ();
 
  private:
   const char *m_name_str;
-  diagnostic_option_id m_option_id;
+  diagnostics::option_id m_option_id;
 };
 
 #endif /* GCC_KNOWN_HEADERS_H */
diff --git a/gcc/c/ChangeLog b/gcc/c/ChangeLog
index cb69b8c..464e5a1 100644
--- a/gcc/c/ChangeLog
+++ b/gcc/c/ChangeLog
@@ -1,3 +1,97 @@
+2025-08-02  Martin Uecker  <uecker@tugraz.at>
+
+	* c-decl.cc (get_parm_array_spec): Remove.
+	(push_parm_decl): Do not add `arg spec` attribute.
+	(build_arg_spec_attribute): New function.
+	(grokdeklarator): Add `arg spec` attribute.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-errors.cc: Update usage of "diagnostic_info" to explicitly
+	refer to "diagnostics::diagnostic_info".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-errors.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* c-parser.cc: Likewise.
+	* c-typeck.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-decl.cc: Update for renaming of diagnostic_option_id to
+	diagnostics::option_id.
+	* c-errors.cc: Likewise.
+	* c-tree.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-objc-common.cc: Update for diagnostic_context becoming
+	diagnostics::context.
+	* c-tree.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* c-errors.cc: Update to add "m_" prefix to fields of
+	diagnostic_info throughout.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* c-typeck.cc (build_asm_expr): Pass null pointer to
+	parse_{input,output}_constraint().
+
+2025-07-16  Kwok Cheung Yeung  <kcyeung@baylibre.com>
+
+	* c-typeck.cc (handle_omp_array_sections): Use OMP_ITERATOR_DECL_P.
+	(c_finish_omp_clauses): Likewise.
+
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* c-parser.cc (c_parser_unary_expression): Clear DECL_READ_P
+	after default_function_array_read_conversion for
+	-Wunused-but-set-{parameter,variable}={2,3} on
+	PRE{IN,DE}CREMENT_EXPR argument.
+	(c_parser_postfix_expression_after_primary): Similarly for
+	POST{IN,DE}CREMENT_EXPR.
+	* c-decl.cc (pop_scope): Use OPT_Wunused_but_set_variable_
+	instead of OPT_Wunused_but_set_variable.
+	(finish_function): Use OPT_Wunused_but_set_parameter_
+	instead of OPT_Wunused_but_set_parameter.
+	* c-typeck.cc (mark_exp_read): Handle {PRE,POST}{IN,DE}CREMENT_EXPR
+	and don't handle it when cast to void.
+	(build_modify_expr): Clear DECL_READ_P after build_binary_op
+	for -Wunused-but-set-{parameter,variable}=3.
+
+2025-07-10  Qing Zhao  <qing.zhao@oracle.com>
+
+	* c-typeck.cc (build_access_with_size_for_counted_by): Update comments.
+	Adjust the arguments per the new design.
+
+2025-07-10  Qing Zhao  <qing.zhao@oracle.com>
+
+	PR middle-end/121000
+	* c-typeck.cc (build_access_with_size_for_counted_by): Update comments.
+	Pass TYPE_SIZE_UNIT of the element as the 6th argument.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	* c-decl.cc (verify_counted_by_attribute): Change the 2nd argument
+	to a vector of fields with counted_by attribute. Verify all fields
+	in this vector.
+	(finish_struct): Collect all the fields with counted_by attribute
+	to a vector and pass this vector to verify_counted_by_attribute.
+	* c-typeck.cc (build_counted_by_ref): Handle pointers with counted_by.
+	Add one more argument, issue error when the pointee type is a structure
+	or union including a flexible array member.
+	(build_access_with_size_for_counted_by): Handle pointers with counted_by.
+	(handle_counted_by_for_component_ref): Call build_counted_by_ref
+	with the new prototype.
+
 2025-07-01  Qing Zhao  <qing.zhao@oracle.com>
 
 	* c-decl.cc (verify_counted_by_attribute): Change the 2nd argument
diff --git a/gcc/c/c-decl.cc b/gcc/c/c-decl.cc
index 7e1c197..7850365 100644
--- a/gcc/c/c-decl.cc
+++ b/gcc/c/c-decl.cc
@@ -1363,7 +1363,7 @@ pop_scope (void)
 	case VAR_DECL:
 	  /* Warnings for unused variables.  */
 	  if ((!TREE_USED (p) || !DECL_READ_P (p))
-	      && !warning_suppressed_p (p, OPT_Wunused_but_set_variable)
+	      && !warning_suppressed_p (p, OPT_Wunused_but_set_variable_)
 	      && !DECL_IN_SYSTEM_HEADER (p)
 	      && DECL_NAME (p)
 	      && !DECL_ARTIFICIAL (p)
@@ -1377,7 +1377,7 @@ pop_scope (void)
 		}
 	      else if (DECL_CONTEXT (p) == current_function_decl)
 		warning_at (DECL_SOURCE_LOCATION (p),
-			    OPT_Wunused_but_set_variable,
+			    OPT_Wunused_but_set_variable_,
 			    "variable %qD set but not used", p);
 	    }
 
@@ -4556,7 +4556,7 @@ lookup_name_fuzzy (tree name, enum lookup_name_fuzzy_kind kind, location_t loc)
   /* Next, look for exact matches for builtin defines that would have been
      defined if the user had passed a command-line option (e.g. -fopenmp
      for "_OPENMP").  */
-  diagnostic_option_id option_id
+  diagnostics::option_id option_id
     = get_option_for_builtin_define (IDENTIFIER_POINTER (name));
   if (option_id.m_idx > 0)
     return name_hint
@@ -6208,184 +6208,7 @@ grokparm (const struct c_parm *parm, tree *expr)
   return decl;
 }
 
-/* Return attribute "arg spec" corresponding to an array/VLA parameter
-   described by PARM, concatenated onto attributes ATTRS.
-   The spec consists of one dollar symbol for each specified variable
-   bound, one asterisk for each unspecified variable bound, followed
-   by at most one specification of the most significant bound of
-   an ordinary array parameter.  For ordinary arrays the specification
-   is either the constant bound itself, or the space character for
-   an array with an unspecified bound (the [] form).  Finally, a chain
-   of specified variable bounds is appended to the spec, starting with
-   the most significant bound.  For example, the PARM T a[2][m][3][n]
-   will produce __attribute__((arg spec ("[$$2]", m, n)).
-   For T a typedef for an array with variable bounds, the bounds are
-   included in the specification in the expected order.
-   No "arg spec"  is created for parameters of pointer types, making
-   a distinction between T(*)[N] (or, equivalently, T[][N]) and
-   the T[M][N] form, all of which have the same type and are represented
-   the same, but only the last of which gets an "arg spec" describing
-   the most significant bound M.  */
 
-static tree
-get_parm_array_spec (const struct c_parm *parm, tree attrs)
-{
-  /* The attribute specification string, minor bound first.  */
-  std::string spec;
-
-  /* A list of VLA variable bounds, major first, or null if unspecified
-     or not a VLA.  */
-  tree vbchain = NULL_TREE;
-  /* True for a pointer parameter.  */
-  bool pointer = false;
-  /* True for an ordinary array with an unpecified bound.  */
-  bool nobound = false;
-
-  /* Create a string representation for the bounds of the array/VLA.  */
-  for (c_declarator *pd = parm->declarator, *next; pd; pd = next)
-    {
-      next = pd->declarator;
-      while (next && next->kind == cdk_attrs)
-	next = next->declarator;
-
-      /* Remember if a pointer has been seen to avoid storing the constant
-	 bound.  */
-      if (pd->kind == cdk_pointer)
-	pointer = true;
-
-      if ((pd->kind == cdk_pointer || pd->kind == cdk_function)
-	  && (!next || next->kind == cdk_id))
-	{
-	  /* Do nothing for the common case of a pointer.  The fact that
-	     the parameter is one can be deduced from the absence of
-	     an arg spec for it.  */
-	  return attrs;
-	}
-
-      if (pd->kind == cdk_id)
-	{
-	  if (pointer
-	      || !parm->specs->type
-	      || TREE_CODE (parm->specs->type) != ARRAY_TYPE
-	      || !TYPE_DOMAIN (parm->specs->type)
-	      || !TYPE_MAX_VALUE (TYPE_DOMAIN (parm->specs->type)))
-	    continue;
-
-	  tree max = TYPE_MAX_VALUE (TYPE_DOMAIN (parm->specs->type));
-	  if (!vbchain
-	      && TREE_CODE (max) == INTEGER_CST)
-	    {
-	      /* Extract the upper bound from a parameter of an array type
-		 unless the parameter is an ordinary array of unspecified
-		 bound in which case a next iteration of the loop will
-		 exit.  */
-	      if (spec.empty () || spec.end ()[-1] != ' ')
-		{
-		  if (!tree_fits_shwi_p (max))
-		    continue;
-
-		  /* The upper bound is the value of the largest valid
-		     index.  */
-		  HOST_WIDE_INT n = tree_to_shwi (max) + 1;
-		  char buf[40];
-		  sprintf (buf, HOST_WIDE_INT_PRINT_UNSIGNED, n);
-		  spec += buf;
-		}
-	      continue;
-	    }
-
-	  /* For a VLA typedef, create a list of its variable bounds and
-	     append it in the expected order to VBCHAIN.  */
-	  tree tpbnds = NULL_TREE;
-	  for (tree type = parm->specs->type; TREE_CODE (type) == ARRAY_TYPE;
-	       type = TREE_TYPE (type))
-	    {
-	      tree nelts_minus_one = array_type_nelts_minus_one (type);
-	      if (error_operand_p (nelts_minus_one))
-		return attrs;
-	      if (TREE_CODE (nelts_minus_one) != INTEGER_CST)
-		{
-		  /* Each variable VLA bound is represented by the dollar
-		     sign.  */
-		  spec += "$";
-		  tpbnds = tree_cons (NULL_TREE, nelts_minus_one, tpbnds);
-		}
-	    }
-	  tpbnds = nreverse (tpbnds);
-	  vbchain = chainon (vbchain, tpbnds);
-	  continue;
-	}
-
-      if (pd->kind != cdk_array)
-	continue;
-
-      if (pd->u.array.vla_unspec_p)
-	{
-	  /* Each unspecified bound is represented by a star.  There
-	     can be any number of these in a declaration (but none in
-	     a definition).  */
-	  spec += '*';
-	  continue;
-	}
-
-      tree nelts = pd->u.array.dimen;
-      if (!nelts)
-	{
-	  /* Ordinary array of unspecified size.  There can be at most
-	     one for the most significant bound.  Exit on the next
-	     iteration which determines whether or not PARM is declared
-	     as a pointer or an array.  */
-	  nobound = true;
-	  continue;
-	}
-
-      if (pd->u.array.static_p)
-	spec += 's';
-
-      if (!INTEGRAL_TYPE_P (TREE_TYPE (nelts)))
-	/* Avoid invalid NELTS.  */
-	return attrs;
-
-      STRIP_NOPS (nelts);
-      nelts = c_fully_fold (nelts, false, nullptr);
-      if (TREE_CODE (nelts) == INTEGER_CST)
-	{
-	  /* Skip all constant bounds except the most significant one.
-	     The interior ones are included in the array type.  */
-	  if (next && (next->kind == cdk_array || next->kind == cdk_pointer))
-	    continue;
-
-	  if (!tree_fits_uhwi_p (nelts))
-	    /* Bail completely on invalid bounds.  */
-	    return attrs;
-
-	  char buf[40];
-	  unsigned HOST_WIDE_INT n = tree_to_uhwi (nelts);
-	  sprintf (buf, HOST_WIDE_INT_PRINT_UNSIGNED, n);
-	  spec += buf;
-	  break;
-	}
-
-      /* Each variable VLA bound is represented by a dollar sign.  */
-      spec += "$";
-      vbchain = tree_cons (NULL_TREE, nelts, vbchain);
-    }
-
-  if (spec.empty () && !nobound)
-    return attrs;
-
-  spec.insert (0, "[");
-  if (nobound)
-    /* Ordinary array of unspecified bound is represented by a space.
-       It must be last in the spec.  */
-    spec += ' ';
-  spec += ']';
-
-  tree acsstr = build_string (spec.length () + 1, spec.c_str ());
-  tree args = tree_cons (NULL_TREE, acsstr, vbchain);
-  tree name = get_identifier ("arg spec");
-  return tree_cons (name, args, attrs);
-}
 
 /* Given a parsed parameter declaration, decode it into a PARM_DECL
    and push that on the current scope.  EXPR is a pointer to an
@@ -6401,7 +6224,6 @@ push_parm_decl (const struct c_parm *parm, tree *expr)
   if (decl && DECL_P (decl))
     DECL_SOURCE_LOCATION (decl) = parm->loc;
 
-  attrs = get_parm_array_spec (parm, attrs);
   decl_attributes (&decl, attrs, 0);
 
   decl = pushdecl (decl);
@@ -6775,6 +6597,25 @@ add_decl_expr (location_t loc, tree type, tree *expr, bool set_name_p)
     }
 }
 
+
+/* Add attribute "arg spec" to ATTRS corresponding to an array/VLA parameter
+   declared with type TYPE.  The attribute has two arguments.  The first is
+   a string that encodes the presence of the static keyword.  The second is
+   the declared type of the array before adjustment, i.e. as an array type
+   including the outermost bound.  */
+
+static tree
+build_arg_spec_attribute (tree type, bool static_p, tree attrs)
+{
+  tree vbchain = tree_cons (NULL_TREE, type, NULL_TREE);
+  tree acsstr = static_p ? build_string (7, "static") :
+			   build_string (1, "");
+  tree args = tree_cons (NULL_TREE, acsstr, vbchain);
+  tree name = get_identifier ("arg spec");
+  return tree_cons (name, args, attrs);
+}
+
+
 /* Given declspecs and a declarator,
    determine the name and type of the object declared
    and construct a ..._DECL node for it.
@@ -6834,6 +6675,7 @@ grokdeclarator (const struct c_declarator *declarator,
   bool funcdef_flag = false;
   bool funcdef_syntax = false;
   bool size_varies = false;
+  bool size_error = false;
   tree decl_attr = declspecs->decl_attr;
   int array_ptr_quals = TYPE_UNQUALIFIED;
   tree array_ptr_attrs = NULL_TREE;
@@ -7326,6 +7168,7 @@ grokdeclarator (const struct c_declarator *declarator,
 				"size of unnamed array has non-integer type");
 		    size = integer_one_node;
 		    size_int_const = true;
+		    size_error = true;
 		  }
 		/* This can happen with enum forward declaration.  */
 		else if (!COMPLETE_TYPE_P (TREE_TYPE (size)))
@@ -7338,6 +7181,7 @@ grokdeclarator (const struct c_declarator *declarator,
 				"type");
 		    size = integer_one_node;
 		    size_int_const = true;
+		    size_error = true;
 		  }
 
 		size = c_fully_fold (size, false, &size_maybe_const);
@@ -7363,6 +7207,7 @@ grokdeclarator (const struct c_declarator *declarator,
 			  error_at (loc, "size of unnamed array is negative");
 			size = integer_one_node;
 			size_int_const = true;
+			size_error = true;
 		      }
 		    /* Handle a size folded to an integer constant but
 		       not an integer constant expression.  */
@@ -7978,6 +7823,10 @@ grokdeclarator (const struct c_declarator *declarator,
 
 	if (TREE_CODE (type) == ARRAY_TYPE)
 	  {
+	    if (!size_error)
+	      *decl_attrs = build_arg_spec_attribute (type, array_parm_static,
+						      *decl_attrs);
+
 	    /* Transfer const-ness of array into that of type pointed to.  */
 	    type = TREE_TYPE (type);
 	    if (orig_qual_type != NULL_TREE)
@@ -9432,62 +9281,56 @@ c_update_type_canonical (tree t)
     }
 }
 
-/* Verify the argument of the counted_by attribute of each of the
-   FIELDS_WITH_COUNTED_BY is a valid field of the containing structure,
-   STRUCT_TYPE, Report error and remove the corresponding attribute
-   when it's not.  */
+/* Verify the argument of the counted_by attribute of the flexible array
+   member FIELD_DECL is a valid field of the containing structure,
+   STRUCT_TYPE, Report error and remove this attribute when it's not.  */
 
 static void
-verify_counted_by_attribute (tree struct_type,
-			     auto_vec<tree> *fields_with_counted_by)
+verify_counted_by_attribute (tree struct_type, tree field_decl)
 {
-  for (tree field_decl : *fields_with_counted_by)
-    {
-      tree attr_counted_by = lookup_attribute ("counted_by",
-						DECL_ATTRIBUTES (field_decl));
+  tree attr_counted_by = lookup_attribute ("counted_by",
+					   DECL_ATTRIBUTES (field_decl));
 
-      if (!attr_counted_by)
-	continue;
+  if (!attr_counted_by)
+    return;
 
-      /* If there is an counted_by attribute attached to the field,
-	 verify it.  */
+  /* If there is an counted_by attribute attached to the field,
+     verify it.  */
 
-      tree fieldname = TREE_VALUE (TREE_VALUE (attr_counted_by));
+  tree fieldname = TREE_VALUE (TREE_VALUE (attr_counted_by));
 
-      /* Verify the argument of the attrbute is a valid field of the
-	 containing structure.  */
+  /* Verify the argument of the attrbute is a valid field of the
+     containing structure.  */
 
-      tree counted_by_field = lookup_field (struct_type, fieldname);
+  tree counted_by_field = lookup_field (struct_type, fieldname);
 
-      /* Error when the field is not found in the containing structure and
-	 remove the corresponding counted_by attribute from the field_decl.  */
-      if (!counted_by_field)
+  /* Error when the field is not found in the containing structure and
+     remove the corresponding counted_by attribute from the field_decl.  */
+  if (!counted_by_field)
+    {
+      error_at (DECL_SOURCE_LOCATION (field_decl),
+		"argument %qE to the %<counted_by%> attribute"
+		" is not a field declaration in the same structure"
+		" as %qD", fieldname, field_decl);
+      DECL_ATTRIBUTES (field_decl)
+	= remove_attribute ("counted_by", DECL_ATTRIBUTES (field_decl));
+    }
+  else
+  /* Error when the field is not with an integer type.  */
+    {
+      while (TREE_CHAIN (counted_by_field))
+	counted_by_field = TREE_CHAIN (counted_by_field);
+      tree real_field = TREE_VALUE (counted_by_field);
+
+      if (!INTEGRAL_TYPE_P (TREE_TYPE (real_field)))
 	{
 	  error_at (DECL_SOURCE_LOCATION (field_decl),
 		    "argument %qE to the %<counted_by%> attribute"
-		    " is not a field declaration in the same structure"
-		    " as %qD", fieldname, field_decl);
+		    " is not a field declaration with an integer type",
+		    fieldname);
 	  DECL_ATTRIBUTES (field_decl)
 	    = remove_attribute ("counted_by", DECL_ATTRIBUTES (field_decl));
 	}
-      else
-      /* Error when the field is not with an integer type.  */
-	{
-	  while (TREE_CHAIN (counted_by_field))
-	    counted_by_field = TREE_CHAIN (counted_by_field);
-	  tree real_field = TREE_VALUE (counted_by_field);
-
-	  if (!INTEGRAL_TYPE_P (TREE_TYPE (real_field)))
-	    {
-	      error_at (DECL_SOURCE_LOCATION (field_decl),
-			"argument %qE to the %<counted_by%> attribute"
-			" is not a field declaration with an integer type",
-			fieldname);
-	      DECL_ATTRIBUTES (field_decl)
-		= remove_attribute ("counted_by",
-				    DECL_ATTRIBUTES (field_decl));
-	    }
-	}
     }
 }
 
@@ -9562,7 +9405,7 @@ finish_struct (location_t loc, tree t, tree fieldlist, tree attributes,
      until now.)  */
 
   bool saw_named_field = false;
-  auto_vec<tree> fields_with_counted_by;
+  tree counted_by_fam_field = NULL_TREE;
   for (x = fieldlist; x; x = DECL_CHAIN (x))
     {
       /* Whether this field is the last field of the structure or union.
@@ -9643,16 +9486,9 @@ finish_struct (location_t loc, tree t, tree fieldlist, tree attributes,
 	     record it here and do more verification later after the
 	     whole structure is complete.  */
 	  if (lookup_attribute ("counted_by", DECL_ATTRIBUTES (x)))
-	    fields_with_counted_by.safe_push (x);
+	    counted_by_fam_field = x;
 	}
 
-      if (TREE_CODE (TREE_TYPE (x)) == POINTER_TYPE)
-	/* If there is a counted_by attribute attached to this field,
-	   record it here and do more verification later after the
-	   whole structure is complete.  */
-	if (lookup_attribute ("counted_by", DECL_ATTRIBUTES (x)))
-	  fields_with_counted_by.safe_push (x);
-
       if (pedantic && TREE_CODE (t) == RECORD_TYPE
 	  && flexible_array_type_p (TREE_TYPE (x)))
 	pedwarn (DECL_SOURCE_LOCATION (x), OPT_Wpedantic,
@@ -9951,8 +9787,8 @@ finish_struct (location_t loc, tree t, tree fieldlist, tree attributes,
 	struct_parse_info->struct_types.safe_push (t);
      }
 
-  if (fields_with_counted_by.length () > 0)
-    verify_counted_by_attribute (t, &fields_with_counted_by);
+  if (counted_by_fam_field)
+    verify_counted_by_attribute (t, counted_by_fam_field);
 
   return t;
 }
@@ -11478,9 +11314,9 @@ finish_function (location_t end_loc)
 	    && !DECL_READ_P (decl)
 	    && DECL_NAME (decl)
 	    && !DECL_ARTIFICIAL (decl)
-	    && !warning_suppressed_p (decl, OPT_Wunused_but_set_parameter))
+	    && !warning_suppressed_p (decl, OPT_Wunused_but_set_parameter_))
 	  warning_at (DECL_SOURCE_LOCATION (decl),
-		      OPT_Wunused_but_set_parameter,
+		      OPT_Wunused_but_set_parameter_,
 		      "parameter %qD set but not used", decl);
     }
 
diff --git a/gcc/c/c-errors.cc b/gcc/c/c-errors.cc
index 4682dca..e2c74fe 100644
--- a/gcc/c/c-errors.cc
+++ b/gcc/c/c-errors.cc
@@ -33,10 +33,10 @@ along with GCC; see the file COPYING3.  If not see
 
 bool
 pedwarn_c23 (location_t location,
-	     diagnostic_option_id option_id,
+	     diagnostics::option_id option_id,
 	     const char *gmsgid, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   bool warned = false;
   rich_location richloc (line_table, location);
@@ -48,8 +48,9 @@ pedwarn_c23 (location_t location,
     {
       diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
 			   (pedantic && !flag_isoc2y)
-			   ? DK_PEDWARN : DK_WARNING);
-      diagnostic.option_id = OPT_Wc23_c2y_compat;
+			   ? diagnostics::kind::pedwarn
+			   : diagnostics::kind::warning);
+      diagnostic.m_option_id = OPT_Wc23_c2y_compat;
       warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   /* -Wno-c23-c2y-compat suppresses even the pedwarns.  */
@@ -58,8 +59,9 @@ pedwarn_c23 (location_t location,
   /* For -pedantic outside C2Y, issue a pedwarn.  */
   else if (pedantic && !flag_isoc2y)
     {
-      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, DK_PEDWARN);
-      diagnostic.option_id = option_id;
+      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
+			   diagnostics::kind::pedwarn);
+      diagnostic.m_option_id = option_id;
       warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   va_end (ap);
@@ -77,10 +79,10 @@ pedwarn_c23 (location_t location,
 
 bool
 pedwarn_c11 (location_t location,
-	     diagnostic_option_id option_id,
+	     diagnostics::option_id option_id,
 	     const char *gmsgid, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   bool warned = false;
   rich_location richloc (line_table, location);
@@ -93,11 +95,12 @@ pedwarn_c11 (location_t location,
     {
       diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
 			   (pedantic && !flag_isoc23)
-			   ? DK_PEDWARN : DK_WARNING);
+			   ? diagnostics::kind::pedwarn
+			   : diagnostics::kind::warning);
       if (option_id == OPT_Wpedantic)
-	diagnostic.option_id = OPT_Wc11_c23_compat;
+	diagnostic.m_option_id = OPT_Wc11_c23_compat;
       else
-	diagnostic.option_id = option_id;
+	diagnostic.m_option_id = option_id;
       warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   /* -Wno-c11-c23-compat suppresses even the pedwarns.  */
@@ -106,8 +109,9 @@ pedwarn_c11 (location_t location,
   /* For -pedantic outside C23, issue a pedwarn.  */
   else if (pedantic && !flag_isoc23)
     {
-      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, DK_PEDWARN);
-      diagnostic.option_id = option_id;
+      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
+			   diagnostics::kind::pedwarn);
+      diagnostic.m_option_id = option_id;
       warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   va_end (ap);
@@ -122,10 +126,10 @@ pedwarn_c11 (location_t location,
 
 bool
 pedwarn_c99 (location_t location,
-	     diagnostic_option_id option_id,
+	     diagnostics::option_id option_id,
 	     const char *gmsgid, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   bool warned = false;
   rich_location richloc (line_table, location);
@@ -137,8 +141,9 @@ pedwarn_c99 (location_t location,
     {
       diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
 			   (pedantic && !flag_isoc11)
-			   ? DK_PEDWARN : DK_WARNING);
-      diagnostic.option_id = OPT_Wc99_c11_compat;
+			   ? diagnostics::kind::pedwarn
+			   : diagnostics::kind::warning);
+      diagnostic.m_option_id = OPT_Wc99_c11_compat;
       warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   /* -Wno-c99-c11-compat suppresses even the pedwarns.  */
@@ -147,8 +152,9 @@ pedwarn_c99 (location_t location,
   /* For -pedantic outside C11, issue a pedwarn.  */
   else if (pedantic && !flag_isoc11)
     {
-      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, DK_PEDWARN);
-      diagnostic.option_id = option_id;
+      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
+			   diagnostics::kind::pedwarn);
+      diagnostic.m_option_id = option_id;
       warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   va_end (ap);
@@ -164,10 +170,10 @@ pedwarn_c99 (location_t location,
 
 bool
 pedwarn_c90 (location_t location,
-	     diagnostic_option_id option_id,
+	     diagnostics::option_id option_id,
 	     const char *gmsgid, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   bool warned = false;
   rich_location richloc (line_table, location);
@@ -183,8 +189,9 @@ pedwarn_c90 (location_t location,
 	{
 	  diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
 			       (pedantic && !flag_isoc99)
-			       ? DK_PEDWARN : DK_WARNING);
-	  diagnostic.option_id = option_id;
+			       ? diagnostics::kind::pedwarn
+			       : diagnostics::kind::warning);
+	  diagnostic.m_option_id = option_id;
 	  diagnostic_report_diagnostic (global_dc, &diagnostic);
 	  warned = true;
 	  goto out;
@@ -196,8 +203,9 @@ pedwarn_c90 (location_t location,
     {
       diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
 			   (pedantic && !flag_isoc99)
-			   ? DK_PEDWARN : DK_WARNING);
-      diagnostic.option_id = OPT_Wc90_c99_compat;
+			   ? diagnostics::kind::pedwarn
+			   : diagnostics::kind::warning);
+      diagnostic.m_option_id = OPT_Wc90_c99_compat;
       diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   /* -Wno-c90-c99-compat suppresses the pedwarns.  */
@@ -206,8 +214,9 @@ pedwarn_c90 (location_t location,
   /* For -pedantic outside C99, issue a pedwarn.  */
   else if (pedantic && !flag_isoc99)
     {
-      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, DK_PEDWARN);
-      diagnostic.option_id = option_id;
+      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
+			   diagnostics::kind::pedwarn);
+      diagnostic.m_option_id = option_id;
       diagnostic_report_diagnostic (global_dc, &diagnostic);
       warned = true;
     }
diff --git a/gcc/c/c-objc-common.cc b/gcc/c/c-objc-common.cc
index d574bc7..5c50983 100644
--- a/gcc/c/c-objc-common.cc
+++ b/gcc/c/c-objc-common.cc
@@ -414,7 +414,7 @@ has_c_linkage (const_tree decl ATTRIBUTE_UNUSED)
 }
 
 void
-c_initialize_diagnostics (diagnostic_context *context)
+c_initialize_diagnostics (diagnostics::context *context)
 {
   context->set_pretty_printer (std::make_unique<c_pretty_printer> ());
   c_common_diagnostics_set_defaults (context);
diff --git a/gcc/c/c-parser.cc b/gcc/c/c-parser.cc
index 0c3e3e2..4a13fc0 100644
--- a/gcc/c/c-parser.cc
+++ b/gcc/c/c-parser.cc
@@ -10547,15 +10547,31 @@ c_parser_unary_expression (c_parser *parser)
       c_parser_consume_token (parser);
       exp_loc = c_parser_peek_token (parser)->location;
       op = c_parser_cast_expression (parser, NULL);
-
-      op = default_function_array_read_conversion (exp_loc, op);
+      if ((VAR_P (op.value) || TREE_CODE (op.value) == PARM_DECL)
+	  && !DECL_READ_P (op.value)
+	  && (VAR_P (op.value) ? warn_unused_but_set_variable
+			       : warn_unused_but_set_parameter) > 1)
+	{
+	  op = default_function_array_read_conversion (exp_loc, op);
+	  DECL_READ_P (op.value) = 0;
+	}
+      else
+	op = default_function_array_read_conversion (exp_loc, op);
       return parser_build_unary_op (op_loc, PREINCREMENT_EXPR, op);
     case CPP_MINUS_MINUS:
       c_parser_consume_token (parser);
       exp_loc = c_parser_peek_token (parser)->location;
       op = c_parser_cast_expression (parser, NULL);
-
-      op = default_function_array_read_conversion (exp_loc, op);
+      if ((VAR_P (op.value) || TREE_CODE (op.value) == PARM_DECL)
+	  && !DECL_READ_P (op.value)
+	  && (VAR_P (op.value) ? warn_unused_but_set_variable
+			       : warn_unused_but_set_parameter) > 1)
+	{
+	  op = default_function_array_read_conversion (exp_loc, op);
+	  DECL_READ_P (op.value) = 0;
+	}
+      else
+	op = default_function_array_read_conversion (exp_loc, op);
       return parser_build_unary_op (op_loc, PREDECREMENT_EXPR, op);
     case CPP_AND:
       c_parser_consume_token (parser);
@@ -13933,7 +13949,17 @@ c_parser_postfix_expression_after_primary (c_parser *parser,
 	  start = expr.get_start ();
 	  finish = c_parser_peek_token (parser)->get_finish ();
 	  c_parser_consume_token (parser);
-	  expr = default_function_array_read_conversion (expr_loc, expr);
+	  if ((VAR_P (expr.value) || TREE_CODE (expr.value) == PARM_DECL)
+	      && !DECL_READ_P (expr.value)
+	      && (VAR_P (expr.value) ? warn_unused_but_set_variable
+				     : warn_unused_but_set_parameter) > 1
+	      && TREE_CODE (TREE_TYPE (expr.value)) != ARRAY_TYPE)
+	    {
+	      expr = default_function_array_read_conversion (expr_loc, expr);
+	      DECL_READ_P (expr.value) = 0;
+	    }
+	  else
+	    expr = default_function_array_read_conversion (expr_loc, expr);
 	  expr.value = build_unary_op (op_loc, POSTINCREMENT_EXPR,
 				       expr.value, false);
 	  set_c_expr_source_range (&expr, start, finish);
@@ -13945,7 +13971,17 @@ c_parser_postfix_expression_after_primary (c_parser *parser,
 	  start = expr.get_start ();
 	  finish = c_parser_peek_token (parser)->get_finish ();
 	  c_parser_consume_token (parser);
-	  expr = default_function_array_read_conversion (expr_loc, expr);
+	  if ((VAR_P (expr.value) || TREE_CODE (expr.value) == PARM_DECL)
+	      && !DECL_READ_P (expr.value)
+	      && (VAR_P (expr.value) ? warn_unused_but_set_variable
+				     : warn_unused_but_set_parameter) > 1
+	      && TREE_CODE (TREE_TYPE (expr.value)) != ARRAY_TYPE)
+	    {
+	      expr = default_function_array_read_conversion (expr_loc, expr);
+	      DECL_READ_P (expr.value) = 0;
+	    }
+	  else
+	    expr = default_function_array_read_conversion (expr_loc, expr);
 	  expr.value = build_unary_op (op_loc, POSTDECREMENT_EXPR,
 				       expr.value, false);
 	  set_c_expr_source_range (&expr, start, finish);
@@ -29126,11 +29162,14 @@ c_parser_omp_error (c_parser *parser, enum pragma_context context)
       if (msg == NULL)
 	msg = _("<message unknown at compile time>");
     }
+  const enum diagnostics::kind diag_kind = (severity_fatal
+					    ? diagnostics::kind::error
+					    : diagnostics::kind::warning);
   if (msg)
-    emit_diagnostic (severity_fatal ? DK_ERROR : DK_WARNING, loc, 0,
+    emit_diagnostic (diag_kind, loc, 0,
 		     "%<pragma omp error%> encountered: %s", msg);
   else
-    emit_diagnostic (severity_fatal ? DK_ERROR : DK_WARNING, loc, 0,
+    emit_diagnostic (diag_kind, loc, 0,
 		     "%<pragma omp error%> encountered");
   return false;
 }
diff --git a/gcc/c/c-tree.h b/gcc/c/c-tree.h
index 364f51d..bb0b113 100644
--- a/gcc/c/c-tree.h
+++ b/gcc/c/c-tree.h
@@ -757,7 +757,7 @@ extern tree c_finish_bc_name (location_t, tree, bool);
 extern bool c_objc_common_init (void);
 extern bool c_missing_noreturn_ok_p (tree);
 extern bool c_warn_unused_global_decl (const_tree);
-extern void c_initialize_diagnostics (diagnostic_context *);
+extern void c_initialize_diagnostics (diagnostics::context *);
 extern bool c_var_mod_p (tree x, tree fn);
 extern alias_set_type c_get_alias_set (tree);
 extern int c_type_dwarf_attribute (const_tree, int);
@@ -960,13 +960,13 @@ extern void c_bind (location_t, tree, bool);
 extern bool tag_exists_p (enum tree_code, tree);
 
 /* In c-errors.cc */
-extern bool pedwarn_c90 (location_t, diagnostic_option_id, const char *, ...)
+extern bool pedwarn_c90 (location_t, diagnostics::option_id, const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
-extern bool pedwarn_c99 (location_t, diagnostic_option_id, const char *, ...)
+extern bool pedwarn_c99 (location_t, diagnostics::option_id, const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
-extern bool pedwarn_c11 (location_t, diagnostic_option_id, const char *, ...)
+extern bool pedwarn_c11 (location_t, diagnostics::option_id, const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
-extern bool pedwarn_c23 (location_t, diagnostic_option_id, const char *, ...)
+extern bool pedwarn_c23 (location_t, diagnostics::option_id, const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
 extern void add_note_about_new_keyword (location_t loc,
 					tree keyword_id);
diff --git a/gcc/c/c-typeck.cc b/gcc/c/c-typeck.cc
index 7948106..ed6e56e 100644
--- a/gcc/c/c-typeck.cc
+++ b/gcc/c/c-typeck.cc
@@ -2310,14 +2310,30 @@ mark_exp_read (tree exp)
     case PARM_DECL:
       DECL_READ_P (exp) = 1;
       break;
+    CASE_CONVERT:
+      if (VOID_TYPE_P (TREE_TYPE (exp)))
+	switch (TREE_CODE (TREE_OPERAND (exp, 0)))
+	  {
+	  case PREINCREMENT_EXPR:
+	  case PREDECREMENT_EXPR:
+	  case POSTINCREMENT_EXPR:
+	  case POSTDECREMENT_EXPR:
+	    return;
+	  default:
+	    break;
+	  }
+      /* FALLTHRU */
     case ARRAY_REF:
     case COMPONENT_REF:
     case MODIFY_EXPR:
     case REALPART_EXPR:
     case IMAGPART_EXPR:
-    CASE_CONVERT:
     case ADDR_EXPR:
     case VIEW_CONVERT_EXPR:
+    case PREINCREMENT_EXPR:
+    case PREDECREMENT_EXPR:
+    case POSTINCREMENT_EXPR:
+    case POSTDECREMENT_EXPR:
       mark_exp_read (TREE_OPERAND (exp, 0));
       break;
     case COMPOUND_EXPR:
@@ -2922,8 +2938,8 @@ should_suggest_deref_p (tree datum_type)
 
 /* For a SUBDATUM field of a structure or union DATUM, generate a REF to
    the object that represents its counted_by per the attribute counted_by
-   attached to this field if it's a flexible array member or a pointer
-   field, otherwise return NULL_TREE.
+   attached to this field if it's a flexible array member field, otherwise
+   return NULL_TREE.
    Set COUNTED_BY_TYPE to the TYPE of the counted_by field.
    For example, if:
 
@@ -2941,34 +2957,18 @@ should_suggest_deref_p (tree datum_type)
 
 */
 static tree
-build_counted_by_ref (location_t loc, tree datum, tree subdatum,
-		      tree *counted_by_type)
+build_counted_by_ref (tree datum, tree subdatum, tree *counted_by_type)
 {
   tree type = TREE_TYPE (datum);
-  tree sub_type = TREE_TYPE (subdatum);
-  if (!c_flexible_array_member_type_p (sub_type)
-      && TREE_CODE (sub_type) != POINTER_TYPE)
+  if (!c_flexible_array_member_type_p (TREE_TYPE (subdatum)))
     return NULL_TREE;
 
-  tree element_type = TREE_TYPE (sub_type);
-
   tree attr_counted_by = lookup_attribute ("counted_by",
 					   DECL_ATTRIBUTES (subdatum));
   tree counted_by_ref = NULL_TREE;
   *counted_by_type = NULL_TREE;
   if (attr_counted_by)
     {
-      /* Issue error when the element_type is a structure or
-	 union including a flexible array member.  */
-      if (RECORD_OR_UNION_TYPE_P (element_type)
-	  && TYPE_INCLUDES_FLEXARRAY (element_type))
-	{
-	  error_at (loc,
-		    "%<counted_by%> attribute is not allowed for a pointer to"
-		    " structure or union with flexible array member");
-	  return error_mark_node;
-	}
-
       tree field_id = TREE_VALUE (TREE_VALUE (attr_counted_by));
       counted_by_ref
 	= build_component_ref (UNKNOWN_LOCATION,
@@ -2991,29 +2991,28 @@ build_counted_by_ref (location_t loc, tree datum, tree subdatum,
 }
 
 /* Given a COMPONENT_REF REF with the location LOC, the corresponding
-   COUNTED_BY_REF, and the COUNTED_BY_TYPE, generate the corresponding
-   call to the internal function .ACCESS_WITH_SIZE.
-
-   Generate an INDIRECT_REF to a call to the internal function
-   .ACCESS_WITH_SIZE.
+   COUNTED_BY_REF, and the COUNTED_BY_TYPE, generate an INDIRECT_REF
+   to a call to the internal function .ACCESS_WITH_SIZE.
 
    REF
 
    to:
 
-   (*.ACCESS_WITH_SIZE (REF, COUNTED_BY_REF, 1, (TYPE_OF_SIZE)0, -1,
-			(TYPE_OF_ARRAY *)0))
+   (*.ACCESS_WITH_SIZE (REF, COUNTED_BY_REF, (* TYPE_OF_SIZE)0,
+			TYPE_SIZE_UNIT for element)
 
    NOTE: The return type of this function is the POINTER type pointing
-   to the original flexible array type or the original pointer type.
-   Then the type of the INDIRECT_REF is the original flexible array type
-   or the original pointer type.
+   to the original flexible array type.
+   Then the type of the INDIRECT_REF is the original flexible array type.
+
+   The type of the first argument of this function is a POINTER type
+   to the original flexible array type.
 
-   The 4th argument of the call is a constant 0 with the TYPE of the
-   object pointed by COUNTED_BY_REF.
+   The 3rd argument of the call is a constant 0 with the pointer TYPE whose
+   pointee type is the TYPE of the object pointed by COUNTED_BY_REF.
 
-   The 6th argument of the call is a constant 0 of the same TYPE as
-   the return type of the call.
+   The 4th argument of the call is the TYPE_SIZE_UNIT of the element TYPE
+   of the array.
 
   */
 static tree
@@ -3021,29 +3020,26 @@ build_access_with_size_for_counted_by (location_t loc, tree ref,
 				       tree counted_by_ref,
 				       tree counted_by_type)
 {
-  gcc_assert (c_flexible_array_member_type_p (TREE_TYPE (ref))
-	      || TREE_CODE (TREE_TYPE (ref)) == POINTER_TYPE);
-  bool is_fam = c_flexible_array_member_type_p (TREE_TYPE (ref));
-  tree first_param = is_fam ? array_to_pointer_conversion (loc, ref)
-		     : build_unary_op (loc, ADDR_EXPR, ref, false);
-
-  /* The result type of the call is a pointer to the original type
-     of the ref.  */
+  gcc_assert (c_flexible_array_member_type_p (TREE_TYPE (ref)));
+  /* The result type of the call is a pointer to the flexible array type.  */
   tree result_type = c_build_pointer_type (TREE_TYPE (ref));
-  first_param = c_fully_fold (first_param, false, NULL);
+  tree element_size = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ref)));
+
+  tree first_param
+    = c_fully_fold (array_to_pointer_conversion (loc, ref), false, NULL);
   tree second_param
     = c_fully_fold (counted_by_ref, false, NULL);
+  tree third_param = build_int_cst (build_pointer_type (counted_by_type), 0);
 
   tree call
     = build_call_expr_internal_loc (loc, IFN_ACCESS_WITH_SIZE,
-				    result_type, 6,
+				    result_type, 4,
 				    first_param,
 				    second_param,
-				    build_int_cst (integer_type_node, 1),
-				    build_int_cst (counted_by_type, 0),
-				    build_int_cst (integer_type_node, -1),
-				    build_int_cst (result_type, 0));
-  /* Wrap the call with an INDIRECT_REF with the original type of the ref.  */
+				    third_param,
+				    element_size);
+
+  /* Wrap the call with an INDIRECT_REF with the flexible array type.  */
   call = build1 (INDIRECT_REF, TREE_TYPE (ref), call);
   SET_EXPR_LOCATION (call, loc);
   return call;
@@ -3061,7 +3057,7 @@ handle_counted_by_for_component_ref (location_t loc, tree ref)
   tree datum = TREE_OPERAND (ref, 0);
   tree subdatum = TREE_OPERAND (ref, 1);
   tree counted_by_type = NULL_TREE;
-  tree counted_by_ref = build_counted_by_ref (loc, datum, subdatum,
+  tree counted_by_ref = build_counted_by_ref (datum, subdatum,
 					      &counted_by_type);
   if (counted_by_ref)
     ref = build_access_with_size_for_counted_by (loc, ref,
@@ -6437,14 +6433,16 @@ build_conditional_expr (location_t colon_loc, tree ifexp, bool ifexp_bcp,
       else
 	{
 	  int qual = ENCODE_QUAL_ADDR_SPACE (as_common);
-	  diagnostic_t kind = DK_PERMERROR;
+	  enum diagnostics::kind kind = diagnostics::kind::permerror;
 	  if (!flag_isoc99)
 	    /* This downgrade to a warning ensures that -std=gnu89
 	       -pedantic-errors does not flag these mismatches between
-	       builtins as errors (as DK_PERMERROR would).  ISO C99
-	       and later do not have implicit function declarations,
+	       builtins as errors (as diagnostics::kind::permerror would)
+	       ISO C99 and later do not have implicit function declarations,
 	       so the mismatch cannot occur naturally there.  */
-	    kind = bltin1 && bltin2 ? DK_WARNING : DK_PEDWARN;
+	    kind = (bltin1 && bltin2
+		    ? diagnostics::kind::warning
+		    : diagnostics::kind::pedwarn);
 	  if (emit_diagnostic (kind, colon_loc, OPT_Wincompatible_pointer_types,
 			       "pointer type mismatch "
 			       "in conditional expression"))
@@ -7328,8 +7326,21 @@ build_modify_expr (location_t location, tree lhs, tree lhs_origtype,
 		newrhs = build1 (EXCESS_PRECISION_EXPR, TREE_TYPE (rhs),
 				 newrhs);
 	    }
+	  bool clear_decl_read = false;
+	  if ((VAR_P (lhs) || TREE_CODE (lhs) == PARM_DECL)
+	      && !DECL_READ_P (lhs)
+	      && (VAR_P (lhs) ? warn_unused_but_set_variable
+			      : warn_unused_but_set_parameter) > 2)
+	    {
+	      mark_exp_read (newrhs);
+	      if (!DECL_READ_P (lhs))
+		clear_decl_read = true;
+	    }
+
 	  newrhs = build_binary_op (location,
 				    modifycode, lhs, newrhs, true);
+	  if (clear_decl_read)
+	    DECL_READ_P (lhs) = 0;
 
 	  /* The original type of the right hand side is no longer
 	     meaningful.  */
@@ -7589,7 +7600,8 @@ error_init (location_t loc, const char *gmsgid, ...)
   /* The gmsgid may be a format string with %< and %>. */
   va_list ap;
   va_start (ap, gmsgid);
-  bool warned = emit_diagnostic_valist (DK_ERROR, loc, -1, gmsgid, &ap);
+  bool warned = emit_diagnostic_valist (diagnostics::kind::error,
+					loc, -1, gmsgid, &ap);
   va_end (ap);
 
   ofwhat = print_spelling ((char *) alloca (spelling_length () + 1));
@@ -7601,7 +7613,7 @@ error_init (location_t loc, const char *gmsgid, ...)
 
 static bool ATTRIBUTE_GCC_DIAG (3,0)
 pedwarn_permerror_init (location_t loc, int opt, const char *gmsgid,
-			va_list *ap, diagnostic_t kind)
+			va_list *ap, enum diagnostics::kind kind)
 {
   /* Use the location where a macro was expanded rather than where
      it was defined to make sure macros defined in system headers
@@ -7628,7 +7640,8 @@ pedwarn_init (location_t loc, int opt, const char *gmsgid, ...)
 {
   va_list ap;
   va_start (ap, gmsgid);
-  bool warned = pedwarn_permerror_init (loc, opt, gmsgid, &ap, DK_PEDWARN);
+  bool warned = pedwarn_permerror_init (loc, opt, gmsgid, &ap,
+					diagnostics::kind::pedwarn);
   va_end (ap);
   return warned;
 }
@@ -7640,7 +7653,8 @@ permerror_init (location_t loc, int opt, const char *gmsgid, ...)
 {
   va_list ap;
   va_start (ap, gmsgid);
-  bool warned = pedwarn_permerror_init (loc, opt, gmsgid, &ap, DK_PERMERROR);
+  bool warned = pedwarn_permerror_init (loc, opt, gmsgid, &ap,
+					diagnostics::kind::permerror);
   va_end (ap);
   return warned;
 }
@@ -12637,7 +12651,8 @@ build_asm_expr (location_t loc, tree string, tree outputs, tree inputs,
       oconstraints[i] = constraint;
 
       if (parse_output_constraint (&constraint, i, ninputs, noutputs,
-				   &allows_mem, &allows_reg, &is_inout))
+				   &allows_mem, &allows_reg, &is_inout,
+				   nullptr))
 	{
 	  /* If the operand is going to end up in memory,
 	     mark it addressable.  */
@@ -12698,7 +12713,8 @@ build_asm_expr (location_t loc, tree string, tree outputs, tree inputs,
       input = TREE_VALUE (tail);
 
       if (parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
-				  oconstraints, &allows_mem, &allows_reg))
+				  oconstraints, &allows_mem, &allows_reg,
+				  nullptr))
 	{
 	  /* If the operand is going to end up in memory,
 	     mark it addressable.  */
@@ -12880,7 +12896,9 @@ c_finish_return (location_t loc, tree retval, tree origtype, bool musttail_p)
 	  && valtype != NULL_TREE && TREE_CODE (valtype) != VOID_TYPE)
 	{
 	  no_warning = true;
-	  if (emit_diagnostic (flag_isoc99 ? DK_PERMERROR : DK_WARNING,
+	  if (emit_diagnostic (flag_isoc99
+			       ? diagnostics::kind::permerror
+			       : diagnostics::kind::warning,
 			       loc, OPT_Wreturn_mismatch,
 			       "%<return%> with no value,"
 			       " in function returning non-void"))
@@ -15628,9 +15646,7 @@ handle_omp_array_sections (tree &c, enum c_omp_region_type ort)
   tree *tp = &OMP_CLAUSE_DECL (c);
   if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
        || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_AFFINITY)
-      && TREE_CODE (*tp) == TREE_LIST
-      && TREE_PURPOSE (*tp)
-      && TREE_CODE (TREE_PURPOSE (*tp)) == TREE_VEC)
+      && OMP_ITERATOR_DECL_P (*tp))
     tp = &TREE_VALUE (*tp);
   tree first = handle_omp_array_sections_1 (c, *tp, types,
 					    maybe_zero_len, first_non_one,
@@ -16827,9 +16843,7 @@ c_finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 	  /* FALLTHRU */
 	case OMP_CLAUSE_AFFINITY:
 	  t = OMP_CLAUSE_DECL (c);
-	  if (TREE_CODE (t) == TREE_LIST
-	      && TREE_PURPOSE (t)
-	      && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	  if (OMP_ITERATOR_DECL_P (t))
 	    {
 	      if (TREE_PURPOSE (t) != last_iterators)
 		last_iterators_remove
@@ -16929,10 +16943,7 @@ c_finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 		      break;
 		    }
 		}
-	      if (TREE_CODE (OMP_CLAUSE_DECL (c)) == TREE_LIST
-		  && TREE_PURPOSE (OMP_CLAUSE_DECL (c))
-		  && (TREE_CODE (TREE_PURPOSE (OMP_CLAUSE_DECL (c)))
-		      == TREE_VEC))
+	      if (OMP_ITERATOR_DECL_P (OMP_CLAUSE_DECL (c)))
 		TREE_VALUE (OMP_CLAUSE_DECL (c)) = t;
 	      else
 		OMP_CLAUSE_DECL (c) = t;
diff --git a/gcc/calls.cc b/gcc/calls.cc
index e16190c..2711c4e 100644
--- a/gcc/calls.cc
+++ b/gcc/calls.cc
@@ -2589,7 +2589,8 @@ can_implement_as_sibling_call_p (tree exp,
       return false;
     }
 
-  if (TYPE_VOLATILE (TREE_TYPE (TREE_TYPE (addr))))
+  if (TYPE_VOLATILE (TREE_TYPE (TREE_TYPE (addr)))
+      && !CALL_EXPR_MUST_TAIL_CALL (exp))
     {
       maybe_complain_about_tail_call (exp, _("volatile function type"));
       return false;
diff --git a/gcc/cfgexpand.cc b/gcc/cfgexpand.cc
index 33649d4..8950294 100644
--- a/gcc/cfgexpand.cc
+++ b/gcc/cfgexpand.cc
@@ -3275,44 +3275,6 @@ expand_asm_loc (tree string, int vol, location_t locus)
   emit_insn (body);
 }
 
-/* Return the number of times character C occurs in string S.  */
-static int
-n_occurrences (int c, const char *s)
-{
-  int n = 0;
-  while (*s)
-    n += (*s++ == c);
-  return n;
-}
-
-/* A subroutine of expand_asm_operands.  Check that all operands have
-   the same number of alternatives.  Return true if so.  */
-
-static bool
-check_operand_nalternatives (const vec<const char *> &constraints)
-{
-  unsigned len = constraints.length();
-  if (len > 0)
-    {
-      int nalternatives = n_occurrences (',', constraints[0]);
-
-      if (nalternatives + 1 > MAX_RECOG_ALTERNATIVES)
-	{
-	  error ("too many alternatives in %<asm%>");
-	  return false;
-	}
-
-      for (unsigned i = 1; i < len; ++i)
-	if (n_occurrences (',', constraints[i]) != nalternatives)
-	  {
-	    error ("operand constraints for %<asm%> differ "
-		   "in number of alternatives");
-	    return false;
-	  }
-    }
-  return true;
-}
-
 /* Check for overlap between registers marked in CLOBBERED_REGS and
    anything inappropriate in T.  Emit error and return the register
    variable definition for error, NULL_TREE for ok.  */
@@ -3478,10 +3440,6 @@ expand_asm_stmt (gasm *stmt)
 	= TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
     }
 
-  /* ??? Diagnose during gimplification?  */
-  if (! check_operand_nalternatives (constraints))
-    return;
-
   /* Count the number of meaningful clobbered registers, ignoring what
      we would ignore later.  */
   auto_vec<rtx> clobber_rvec;
@@ -3555,7 +3513,8 @@ expand_asm_stmt (gasm *stmt)
 	 no point in going further.  */
       constraint = constraints[i];
       if (!parse_output_constraint (&constraint, i, ninputs, noutputs,
-				    &allows_mem, &allows_reg, &is_inout))
+				    &allows_mem, &allows_reg, &is_inout,
+				    nullptr))
 	return;
 
       /* If the output is a hard register, verify it doesn't conflict with
@@ -3633,8 +3592,8 @@ expand_asm_stmt (gasm *stmt)
 
       constraint = constraints[i + noutputs];
       if (! parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
-				    constraints.address (),
-				    &allows_mem, &allows_reg))
+				    constraints.address (), &allows_mem,
+				    &allows_reg, nullptr))
 	return;
 
       if (! allows_reg && allows_mem)
@@ -3664,7 +3623,7 @@ expand_asm_stmt (gasm *stmt)
 
       ok = parse_output_constraint (&constraints[i], i, ninputs,
 				    noutputs, &allows_mem, &allows_reg,
-				    &is_inout);
+				    &is_inout, nullptr);
       gcc_assert (ok);
 
       /* If an output operand is not a decl or indirect ref and our constraint
@@ -3769,7 +3728,7 @@ expand_asm_stmt (gasm *stmt)
       constraint = constraints[i + noutputs];
       ok = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
 				   constraints.address (),
-				   &allows_mem, &allows_reg);
+				   &allows_mem, &allows_reg, nullptr);
       gcc_assert (ok);
 
       /* EXPAND_INITIALIZER will not generate code for valid initializer
@@ -5358,6 +5317,9 @@ expand_debug_expr (tree exp)
       return simplify_gen_binary (MULT, mode, op0, op1);
 
     case RDIV_EXPR:
+      gcc_assert (FLOAT_MODE_P (mode)
+		  || ALL_FIXED_POINT_MODE_P (mode));
+      /* Fall through.  */
     case TRUNC_DIV_EXPR:
     case EXACT_DIV_EXPR:
       if (unsignedp)
diff --git a/gcc/cgraph.cc b/gcc/cgraph.cc
index 94a2e6e..32071a8 100644
--- a/gcc/cgraph.cc
+++ b/gcc/cgraph.cc
@@ -1790,6 +1790,19 @@ cgraph_update_edges_for_call_stmt_node (cgraph_node *node,
 
       if (e)
 	{
+	  /* If call was devirtualized during cloning, mark edge
+	     as resolved.  */
+	  if (e->speculative)
+	    {
+	      if (new_stmt && is_gimple_call (new_stmt))
+		{
+		  tree decl = gimple_call_fndecl (new_stmt);
+		  if (decl)
+		    e = cgraph_edge::resolve_speculation (e, decl);
+		}
+	      else
+		e = cgraph_edge::resolve_speculation (e, NULL);
+	    }
 	  /* Keep calls marked as dead dead.  */
 	  if (new_stmt && is_gimple_call (new_stmt) && e->callee
 	      && fndecl_built_in_p (e->callee->decl, BUILT_IN_UNREACHABLE,
diff --git a/gcc/cgraphunit.cc b/gcc/cgraphunit.cc
index fa54a59..9f4af63 100644
--- a/gcc/cgraphunit.cc
+++ b/gcc/cgraphunit.cc
@@ -63,7 +63,7 @@ along with GCC; see the file COPYING3.  If not see
       final assembler is generated.  This is done in the following way. Note
       that with link time optimization the process is split into three
       stages (compile time, linktime analysis and parallel linktime as
-      indicated bellow).
+      indicated below).
 
       Compile time:
 
@@ -2400,9 +2400,10 @@ symbol_table::compile (void)
     if (node->alias
 	&& lookup_attribute ("weakref", DECL_ATTRIBUTES (node->decl)))
       {
-	IDENTIFIER_TRANSPARENT_ALIAS
-	   (DECL_ASSEMBLER_NAME (node->decl)) = 1;
-	TREE_CHAIN (DECL_ASSEMBLER_NAME (node->decl))
+	tree id = DECL_ASSEMBLER_NAME (node->decl);
+	gcc_assert (!IDENTIFIER_INTERNAL_P (id));
+	IDENTIFIER_TRANSPARENT_ALIAS (id) = 1;
+	TREE_CHAIN (id)
 	   = (node->alias_target ? node->alias_target
 	      : DECL_ASSEMBLER_NAME (node->get_alias_target ()->decl));
       }
diff --git a/gcc/cobol/ChangeLog b/gcc/cobol/ChangeLog
index 4b05399..35d645c 100644
--- a/gcc/cobol/ChangeLog
+++ b/gcc/cobol/ChangeLog
@@ -1,3 +1,483 @@
+2025-08-02  Jakub Jelinek  <jakub@redhat.com>
+
+	* parse.y (intrinsic): Use %td format specifier with no cast on
+	argument instead of %ld with cast to long.
+	* scan_ante.h (numstr_of): Likewise.
+	* util.cc (cbl_field_t::report_invalid_initial_value): Likewise.
+
+2025-08-01  Robert Dubner  <rdubner@symas.com>
+
+	PR cobol/119324
+	* cbldiag.h (location_dump): Inline suppression of knownConditionTrueFalse.
+	* genapi.cc (parser_statement_begin): Combine two if() statements.
+	* genutil.cc (get_binary_value): File-level suppression of duplicateBreak.
+	* symbols.cc (symbol_elem_cmp): File-level suppression of duplicateBreak.
+
+2025-07-31  Robert Dubner  <rdubner@symas.com>
+
+	PR cobol/120244
+	* genapi.cc (get_level_88_domain): Increase array size for final byte.
+	(psa_FldLiteralA): Use correct length in build_string_literal call.
+	* scan.l: Use a loop instead of std:transform to avoid EOF overrun.
+	* scan_ante.h (binary_integer_usage): Use a variable-length buffer.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* util.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* util.cc: Update for renaming of diagnostic_option_id to
+	diagnostics::option_id.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* util.cc: Remove redundant #includes
+
+2025-07-24  Robert Dubner  <rdubner@symas.com>
+
+	PR cobol/119231
+	* gcobolspec.cc: (lang_specific_driver): Pass OPT_static through.
+	Handle -static and -static-libgcobol properly.
+
+2025-07-23  Robert Dubner  <rdubner@symas.com>
+
+	* genapi.cc (leave_procedure): Adjust location_t for PERFORM.
+	(parser_perform_times): Likewise.
+	(internal_perform_through_times): Likewise.
+	(perform_outofline_before_until): Likewise.
+	(perform_outofline_after_until): Likewise.
+	(perform_outofline_testafter_varying): Likewise.
+	(perform_outofline_before_varying): Likewise.
+
+2025-07-21  Robert Dubner  <rdubner@symas.com>
+	    James K. Lowden  <jklowden@cobolworx.com>
+
+	PR cobol/120402
+	* Make-lang.in: Elminate commented-out scripting.
+	* cbldiag.h (_CBLDIAG_H): Change #if 0 to #if GCOBOL_GETENV
+	(warn_msg): Add printf attributes.
+	(location_dump): Add debugging message.
+	* cdf.y: Improved linemap tracking.
+	* genapi.cc (treeplet_fill_source): const attribute for formal parameter.
+	(insert_nop): Created to consolidate var_decl_nop writes.
+	(build_main_that_calls_something): Move generation to the end of executable.
+	(level_88_helper): Formatting.
+	(parser_call_targets_dump): Formatting.
+	(function_pointer_from_name): const attribute for formal parameter.
+	(parser_initialize_programs): const attribute for formal parameter.
+	(parser_statement_begin): Improved linemap handling.
+	(section_label):  Improved linemap handling.
+	(paragraph_label): Improved linemap handling.
+	(pseudo_return_pop): Improved linemap handling.
+	(leave_procedure): Formatting.
+	(parser_enter_section):  Improved linemap handling.
+	(parser_enter_paragraph): Improved linemap handling.
+	(parser_perform): Formatting.
+	(parser_leave_file): Move creation of main() to this routine.
+	(parser_enter_program): Move creation of main from here to leave_file.
+	(parser_accept): Formatting. const attribute for formal parameter.
+	(parser_accept_command_line): const attribute for formal parameter.
+	(parser_accept_command_line_count): const attribute for formal parameter.
+	(parser_accept_envar): Likewise.
+	(parser_set_envar): Likewise.
+	(parser_display): Likewise.
+	(get_exhibit_name): Implement EXHIBIT verb.
+	(parser_exhibit): Likewise.
+	(parser_sleep): const attribute for formal parameter.
+	(parser_division): Improved linemap handling.
+	(parser_classify): const attribute for formal parameter.
+	(create_iline_address_pairs): Improved linemap handling.
+	(parser_perform_start): Likewise.
+	(perform_inline_until): Likewise.
+	(perform_inline_testbefore_varying): Likewise.
+	(parser_perform_until): Likewise.
+	(parser_perform_inline_times): Likewise.
+	(parser_intrinsic_subst): const attribute for formal parameter.
+	(parser_file_merge): Formatting.
+	(create_and_call): Improved linemap handling.
+	(mh_identical): const attribute for formal parameter.
+	(mh_numeric_display): const attribute for formal parameter.
+	(mh_little_endian): Likewise.
+	(mh_source_is_group): Likewise.
+	(psa_FldLiteralA): Formatting.
+	* genapi.h (parser_accept): const attribute for formal parameter.
+	(parser_accept_envar): Likewise.
+	(parser_set_envar): Likewise.
+	(parser_accept_command_line): Likewise.
+	(parser_accept_command_line_count): Likewise.
+	(parser_add): Likewise.
+	(parser_classify): Likewise.
+	(parser_sleep): Likewise.
+	(parser_exhibit): Likewise.
+	(parser_display): Likewise.
+	(parser_initialize_programs): Likewise.
+	(parser_intrinsic_subst): Likewise.
+	* gengen.cc (gg_assign): Improved linemap handling.
+	(gg_add_field_to_structure): Likewise.
+	(gg_define_from_declaration): Likewise.
+	(gg_build_relational_expression): Likewise.
+	(gg_goto_label_decl): Likewise.
+	(gg_goto): Likewise.
+	(gg_printf): Likewise.
+	(gg_fprintf): Likewise.
+	(gg_memset): Likewise.
+	(gg_memchr): Likewise.
+	(gg_memcpy): Likewise.
+	(gg_memmove): Likewise.
+	(gg_strcpy): Likewise.
+	(gg_strcmp): Likewise.
+	(gg_strncmp): Likewise.
+	(gg_return): Likewise.
+	(chain_parameter_to_function): Likewise.
+	(gg_define_function): Likewise.
+	(gg_get_function_decl): Likewise.
+	(gg_call_expr): Likewise.
+	(gg_call): Likewise.
+	(gg_call_expr_list): Likewise.
+	(gg_exit): Likewise.
+	(gg_abort): Likewise.
+	(gg_strlen): Likewise.
+	(gg_strdup): Likewise.
+	(gg_malloc): Likewise.
+	(gg_realloc): Likewise.
+	(gg_free): Likewise.
+	(gg_set_current_line_number): Likewise.
+	(gg_get_current_line_number): Likewise.
+	(gg_insert_into_assembler): Likewise.
+	(token_location_override): Likewise.
+	(gg_token_location): Likewise.
+	* gengen.h (location_from_lineno): Likewise.
+	(gg_set_current_line_number): Likewise.
+	(gg_get_current_line_number): Likewise.
+	(gg_token_location): Likewise.
+	(current_token_location): Likewise.
+	(current_location_minus_one): Likewise.
+	(current_location_minus_one_clear): Likewise.
+	(token_location_override): Likewise.
+	* genmath.cc (fast_divide):  const attribute for formal parameter.
+	* genutil.cc (get_and_check_refstart_and_reflen): Likewise.
+	(get_data_offset): Likewise.
+	(refer_refmod_length): Likewise.
+	(refer_offset): Likewise.
+	(refer_size): Likewise.
+	(refer_size_dest): Likewise.
+	(refer_size_source): Likewise.
+	(qualified_data_location): Likewise.
+	* genutil.h (refer_offset): Likewise.
+	(refer_size_source): Likewise.
+	(refer_size_dest): Likewise.
+	(qualified_data_location): Likewise.
+	* parse.y: EVALUATE token; Implement EXHIBIT verb;
+	Improved linemap handling.
+	* parse_ante.h (input_file_status_notify): Improved linemap handling.
+	(location_set): Likewise.
+	* scan.l: PICTURE string validation.
+	* scan_ante.h (class picture_t): PICTURE string validation.
+	(validate_picture): Likewise.
+	* symbols.cc (symbol_currency): Revised default currency handling.
+	* symbols.h (symbol_currency): Likewise.
+	* util.cc (location_from_lineno): Improved linemap handling.
+	(current_token_location): Improved linemap handling.
+	(current_location_minus_one): Improved linemap handling.
+	(current_location_minus_one_clear): Improved linemap handling.
+	(gcc_location_set_impl): Improved linemap handling.
+	(warn_msg): Improved linemap handling.
+	* util.h (cobol_lineno): Improved linemap handling.
+
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* gcobolspec.cc (lang_specific_driver): Remove unused but set variable
+	n_cobol_files.
+
+2025-07-14  Robert Dubner  <rdubner@symas.com>
+
+	* cobol1.cc (cobol_langhook_handle_option): Eliminate cppcheck warnings.
+	* dts.h: Likewise.
+	* except.cc (cbl_enabled_exceptions_t::dump): Likewise.
+	* gcobolspec.cc (lang_specific_driver): Likewise.
+	* genapi.cc (parser_file_merge): Likewise.
+	* gengen.cc (gg_unique_in_function): Likewise.
+	(gg_declare_variable): Likewise.
+	(gg_peek_fn_decl): Likewise.
+	(gg_define_function): Likewise.
+	* genmath.cc (set_up_on_exception_label): Likewise.
+	(set_up_compute_error_label): Likewise.
+	(arithmetic_operation): Likewise.
+	(fast_divide): Likewise.
+	* genutil.cc (get_and_check_refstart_and_reflen): Likewise.
+	(get_depending_on_value_from_odo): Likewise.
+	(get_data_offset): Likewise.
+	(get_binary_value): Likewise.
+	(process_this_exception): Likewise.
+	(copy_little_endian_into_place): Likewise.
+	(refer_is_clean): Likewise.
+	(refer_fill_depends): Likewise.
+	* genutil.h (process_this_exception): Likewise.
+	(copy_little_endian_into_place): Likewise.
+	(refer_is_clean): Likewise.
+	* lexio.cc (check_push_pop_directive): Likewise.
+	(check_source_format_directive): Likewise.
+	(location_in): Likewise.
+	(lexer_input): Likewise.
+	(cdftext::lex_open): Likewise.
+	(lexio_dialect_mf): Likewise.
+	(valid_sequence_area): Likewise.
+	(cdftext::free_form_reference_format): Likewise.
+	(cdftext::segment_line): Likewise.
+	* lexio.h (struct span_t): Likewise.
+	* scan_ante.h (trim_location): Likewise.
+	* symbols.cc (symbol_elem_cmp): Likewise.
+	(symbol_alphabet): Likewise.
+	(end_of_group): Likewise.
+	(cbl_field_t::attr_str): Likewise.
+	(symbols_update): Likewise.
+	(symbol_typedef_add): Likewise.
+	(symbol_field_add): Likewise.
+	(new_temporary_impl): Likewise.
+	(symbol_label_section_exists): Likewise.
+	(symbol_program_callables): Likewise.
+	(file_status_status_of): Likewise.
+	* symfind.cc (is_data_field): Likewise.
+	(finalize_symbol_map2): Likewise.
+	(class in_scope): Likewise.
+	(symbol_match2): Likewise.
+	* util.cc (get_current_dir_name): Likewise.
+	(gb4): Likewise.
+	(class cdf_directives_t): Likewise.
+	(cbl_field_t::report_invalid_initial_value): Likewise.
+	(literal_subscript_oob): Likewise.
+	(cbl_refer_t::str): Likewise.
+	(date_time_fmt): Likewise.
+	(class unique_stack): Likewise.
+	(cobol_set_pp_option): Likewise.
+	(cobol_filename): Likewise.
+	(cobol_filename_restore): Likewise.
+	(gcc_location_set_impl): Likewise.
+	(ydferror): Likewise.
+	(error_msg_direct): Likewise.
+	(yyerror): Likewise.
+	(cbl_unimplemented_at): Likewise.
+
+2025-07-13  Robert Dubner  <rdubner@symas.com>
+
+	* Make-lang.in: Eliminate the .cc.o override.
+	* genapi.cc (level_88_helper): Eliminate cppcheck warning.
+	(get_level_88_domain): Likewise.
+	(get_class_condition_string): Likewise.
+	(parser_call_targets_dump): Likewise.
+	(parser_compile_ecs): Likewise.
+	(initialize_variable_internal): Likewise.
+	(move_tree): Likewise.
+	(combined_name): Likewise.
+	(assembler_label): Likewise.
+	(find_procedure): Likewise.
+	(parser_perform): Likewise.
+	(parser_perform_times): Likewise.
+	(internal_perform_through): Likewise.
+	(internal_perform_through_times): Likewise.
+	(psa_FldLiteralN): Likewise.
+	(psa_FldBlob): Likewise.
+	(parser_accept): Likewise.
+	(parser_accept_exception): Likewise.
+	(parser_accept_exception_end): Likewise.
+	(parser_accept_command_line): Likewise.
+	(parser_accept_envar): Likewise.
+	(parser_display_internal): Likewise.
+	(parser_display): Likewise.
+	(parser_assign): Likewise.
+	(parser_initialize_table): Likewise.
+	(parser_arith_error): Likewise.
+	(parser_arith_error_end): Likewise.
+	(parser_division): Likewise.
+	(label_fetch): Likewise.
+	(parser_label_label): Likewise.
+	(parser_label_goto): Likewise.
+	(parser_perform_start): Likewise.
+	(parser_perform_conditional): Likewise.
+	(parser_perform_conditional_end): Likewise.
+	(parser_perform_until): Likewise.
+	(parser_file_delete): Likewise.
+	(parser_intrinsic_subst): Likewise.
+	(create_lsearch_address_pairs): Likewise.
+	(parser_bsearch_start): Likewise.
+	(is_ascending_key): Likewise.
+	(parser_sort): Likewise.
+	(parser_file_sort): Likewise.
+	(parser_return_start): Likewise.
+	(parser_file_merge): Likewise.
+	(parser_string_overflow): Likewise.
+	(parser_unstring): Likewise.
+	(parser_string): Likewise.
+	(parser_call_exception): Likewise.
+	(create_and_call): Likewise.
+	(mh_identical): Likewise.
+	(move_helper): Likewise.
+	(binary_initial_from_float128): Likewise.
+	(initial_from_initial): Likewise.
+	(psa_FldLiteralA): Likewise.
+	(parser_local_add): Likewise.
+	(parser_symbol_add): Likewise.
+	* genapi.h (parser_display): Likewise.
+	* gengen.cc (gg_call_expr): Explict check for NULL_TREE.
+	(gg_call): Likewise.
+	* show_parse.h (SHOW_PARSE_LABEL_OK): Likewise.
+	(TRACE1_FIELD_VALUE): Likewise.
+	(CHECK_FIELD): Likewise.
+	(CHECK_FIELD2): Likewise.
+	(CHECK_LABEL): Likewise.
+	* util.cc (cbl_internal_error): Apply [[noreturn]] attribute.
+	* util.h (cbl_internal_error): Likewise.
+
+2025-07-11  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	PR cobol/120621
+	* lexio.cc (parse_replace_pairs): Cast mfile.lineno() to fmt_size_t.
+	* parse.y (intrinsic): Print ptrdiff_t using %ld, cast arg to long.
+	* scan_ante.h (numstr_of): Print nx using %ld, cast arg to long.
+	* util.cc (cbl_field_t::report_invalid_initial_value): Print
+	ptrdiff_t using %ld, cast arg to long.
+
+2025-07-10  James K. Lowden  <jklowden@cobolworx.com>
+
+	PR cobol/120765
+	* cdf.y: Extend grammar for new CDF syntax, relocate dictionary.
+	* cdfval.h (cdf_dictionary): Use new CDF dictionary.
+	* dts.h: Remove useless assignment, note incorrect behavior.
+	* except.cc: Remove obsolete EC state.
+	* gcobol.1: Document CDF in its own section.
+	* genapi.cc (parser_statement_begin): Use new EC state function.
+	(parser_file_merge): Same.
+	(parser_check_fatal_exception): Same.
+	* genutil.cc (get_and_check_refstart_and_reflen): Same.
+	(get_depending_on_value_from_odo): Same.
+	(get_data_offset): Same.
+	(process_this_exception): Same.
+	* lexio.cc (check_push_pop_directive): New function.
+	(check_source_format_directive): Restrict regex search to 1 line.
+	(cdftext::free_form_reference_format): Use new function.
+	* parse.y: Define new CDF tokens, use new CDF state.
+	* parse_ante.h (cdf_tokens): Use new CDF state.
+	(redefined_token): Same.
+	(class prog_descr_t): Remove obsolete CDF state.
+	(class program_stack_t): Same.
+	(current_call_convention): Same.
+	* scan.l: Recognize new CDF tokens.
+	* scan_post.h (is_cdf_token): Same.
+	* symbols.h (cdf_current_tokens): Change current_call_convention to return void.
+	* token_names.h: Regenerate.
+	* udf/stored-char-length.cbl: Use new PUSH/POP CDF functionality.
+	* util.cc (class cdf_directives_t): Define cdf_directives_t.
+	(current_call_convention): Same.
+	(cdf_current_tokens): Same.
+	(cdf_dictionary): Same.
+	(cdf_enabled_exceptions): Same.
+	(cdf_push): Same.
+	(cdf_push_call_convention): Same.
+	(cdf_push_current_tokens): Same.
+	(cdf_push_dictionary): Same.
+	(cdf_push_enabled_exceptions): Same.
+	(cdf_push_source_format): Same.
+	(cdf_pop): Same.
+	(cdf_pop_call_convention): Same.
+	(cdf_pop_current_tokens): Same.
+	(cdf_pop_dictionary): Same.
+	(cdf_pop_enabled_exceptions): Same.
+	(cdf_pop_source_format): Same.
+	* util.h (cdf_push): Declare cdf_directives_t.
+	(cdf_push_call_convention): Same.
+	(cdf_push_current_tokens): Same.
+	(cdf_push_dictionary): Same.
+	(cdf_push_enabled_exceptions): Same.
+	(cdf_push_source_format): Same.
+	(cdf_pop): Same.
+	(cdf_pop_call_convention): Same.
+	(cdf_pop_current_tokens): Same.
+	(cdf_pop_dictionary): Same.
+	(cdf_pop_source_format): Same.
+	(cdf_pop_enabled_exceptions): Same.
+
+2025-07-09  Robert Dubner  <rdubner@symas.com>
+	    James K. Lowden  <jklowden@cobolworx.com>
+
+	PR cobol/120765
+	PR cobol/119337
+	PR cobol/120794
+	* Make-lang.in: Take control of the .cc.o rule.
+	* cbldiag.h (error_msg_direct): New declaration.
+	(gcc_location_dump): Forward declaration.
+	(location_dump): Use gcc_location_dump.
+	* cdf.y: Change some tokens.
+	* gcobc: Change dialect handling.
+	* genapi.cc (parser_call_targets_dump): Temporarily remove from service.
+	(parser_compile_dcls): Combine temporary arrays.
+	(get_binary_value_from_float): Apply const to one parameter.
+	(depending_on_value): Localize a boolean variable.
+	(normal_normal_compare): Likewise.
+	(cobol_compare): Eliminate cppcheck warning.
+	(combined_name): Apply const to an input parameter.
+	(parser_perform): Apply const to a variable.
+	(parser_accept): Improve handling of special_name_t parameter and
+	the exception conditions.
+	(parser_display): Improve handling of speciat_name_t parameter; use the
+	os_filename[] string when appropriate.
+	(program_end_stuff): Rename shadowing variable.
+	(parser_division): Consolidate temporary char[] arrays.
+	(parser_file_start): Apply const to a parameter.
+	(inspect_replacing): Likewise.
+	(parser_program_hierarchy): Rename shadowing variable.
+	(mh_identical): Apply const to parameters.
+	(float_type_of): Likewise.
+	(picky_memcpy): Likewise.
+	(mh_numeric_display): Likewise.
+	(mh_little_endian): Likewise.
+	(mh_source_is_group): Apply static to a variable it.
+	(move_helper): Quiet a cppcheck warning.
+	* genapi.h (parser_accept): Add exceptions to declaration.
+	(parser_accept_under_discussion): Add declaration.
+	(parser_display): Change to std::vector; add exceptions to declaration.
+	* lexio.cc (cdf_source_format): Improve source code location handling.
+	(source_format_t::infer): Likewise.
+	(is_fixed_format): Likewise.
+	(is_reference_format): Likewise.
+	(left_margin): Likewise.
+	(right_margin): Likewise.
+	(cobol_set_indicator_column): Likewise.
+	(include_debug): Likewise.
+	(continues_at): Likewise.
+	(indicated): Likewise.
+	(check_source_format_directive): Likewise.
+	(cdftext::free_form_reference_format): Likewise.
+	* parse.y: Tokens; program and function names; DISPLAY and ACCEPT
+	handling.
+	* parse_ante.h (class tokenset_t): Removed.
+	(class current_tokens_t): Removed.
+	(field_of): Removed.
+	* scan.l: Token handling.
+	* scan_ante.h (level_found): Comment.
+	* scan_post.h (start_condition_str): Remove cast author_state:.
+	* symbols.cc (symbols_update): Change error message.
+	(symbol_table_init): Correct and reorder entries.
+	(symbol_unresolved_file_key): New function definition.
+	(cbl_file_key_t::deforward): Change error message.
+	* symbols.h (symbol_unresolved_file_key): New declaration.
+	(keyword_tok): New function.
+	(redefined_token): New function.
+	(class current_tokens_t): New class.
+	* symfind.cc (symbol_match): Revise error message.
+	* token_names.h: Reorder and change numbers in comments.
+	* util.cc (class cdf_directives_t): New class.
+	(cobol_set_indicator_column): New function.
+	(cdf_source_format): New function.
+	(gcc_location_set_impl): Improve column handling in token_location.
+	(gcc_location_dump): New function.
+	(class temp_loc_t): Modify constructor.
+	(error_msg_direct): New function.
+	* util.h (class source_format_t): New class.
+
 2025-07-01  James K. Lowden  <jklowden@cobolworx.com>
 
 	* Make-lang.in: Use && instead of semicolon between commands.
diff --git a/gcc/cobol/Make-lang.in b/gcc/cobol/Make-lang.in
index e884212..0e2a773 100644
--- a/gcc/cobol/Make-lang.in
+++ b/gcc/cobol/Make-lang.in
@@ -384,3 +384,4 @@ cobol.stagefeedback: stagefeedback-start
 selftest-cobol:
 
 lang_checks += check-cobol
+
diff --git a/gcc/cobol/cbldiag.h b/gcc/cobol/cbldiag.h
index 548b0f2..dd16190 100644
--- a/gcc/cobol/cbldiag.h
+++ b/gcc/cobol/cbldiag.h
@@ -33,7 +33,7 @@
 #else
 #define _CBLDIAG_H
 
-#if 0
+#if GCOBOL_GETENV
 #define gcobol_getenv(x) getenv(x)
 #else
 #define gcobol_getenv(x) ((char *)nullptr)
@@ -78,10 +78,19 @@ struct YDFLTYPE
 
 #endif
 
+// Diagnostic format specifiers are documented in gcc/pretty-print.cc
 // an error at a location, called from the parser for semantic errors
 void error_msg( const YYLTYPE& loc, const char gmsgid[], ... )
   ATTRIBUTE_GCOBOL_DIAG(2, 3);
 
+bool
+warn_msg( const YYLTYPE& loc, const char gmsgid[], ... )
+  ATTRIBUTE_GCOBOL_DIAG(2, 3);
+
+// an error that uses token_location, not yylloc
+void error_msg_direct( const char gmsgid[], ... )
+  ATTRIBUTE_GCOBOL_DIAG(1, 2);
+
 void dialect_error( const YYLTYPE& loc, const char term[], const char dialect[] );
 
 
@@ -104,16 +113,23 @@ void dbgmsg( const char fmt[], ... ) ATTRIBUTE_PRINTF_1;
 
 void gcc_location_set( const YYLTYPE& loc );
 
+void gcc_location_dump();
+
 // tree.h defines yy_flex_debug as a macro because options.h
 #if ! defined(yy_flex_debug)
 template <typename LOC>
 static void
 location_dump( const char func[], int line, const char tag[], const LOC& loc) {
   extern int yy_flex_debug; // cppcheck-suppress shadowVariable
-  if( yy_flex_debug && gcobol_getenv("update_location") )
-    fprintf(stderr, "%s:%d: %s location (%d,%d) to (%d,%d)\n",
-            func, line, tag,
-            loc.first_line, loc.first_column, loc.last_line, loc.last_column);
+  if( yy_flex_debug ) {
+    const char *detail = gcobol_getenv("update_location"); // cppcheck-suppress knownConditionTrueFalse
+    if( detail ) {
+      fprintf(stderr, "%s:%d: %s location (%d,%d) to (%d,%d)\n",
+              func, line, tag,
+              loc.first_line, loc.first_column, loc.last_line, loc.last_column);
+      if( *detail == '2' ) gcc_location_dump();
+    }
+  }
 }
 #endif // defined(yy_flex_debug)
 
diff --git a/gcc/cobol/cdf.y b/gcc/cobol/cdf.y
index 3344271..53fea5d 100644
--- a/gcc/cobol/cdf.y
+++ b/gcc/cobol/cdf.y
@@ -95,7 +95,7 @@ void input_file_status_notify();
         }                                                               \
       location_dump("cdf.c", __LINE__, "current", (Current));		\
       input_file_status_notify();					\
-      gcc_location_set( location_set(Current) );			\
+      location_set(Current);                                            \
   } while (0)
 
 %}
@@ -105,14 +105,14 @@ void input_file_status_notify();
 
   using std::map;
 
-  static map<std::string, cdfval_t> dictionary;
-
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-function"
   static bool
   cdfval_add( const char name[],
 	       const cdfval_t& value, bool override = false )
   {
+    cdf_values_t& dictionary( cdf_dictionary() );
+    
     if( scanner_parsing() ) {
       if( ! override ) {
 	if( dictionary.find(name) != dictionary.end() ) return false;
@@ -123,6 +123,8 @@ void input_file_status_notify();
   }
   static void
   cdfval_off( const char name[] ) {
+    cdf_values_t& dictionary( cdf_dictionary() );
+    
     if( scanner_parsing() ) {
       auto p = dictionary.find(name);
       if( p == dictionary.end() ) {
@@ -159,6 +161,8 @@ exception_turn_t exception_turn;
 			
 bool
 apply_cdf_turn( const exception_turn_t& turn ) {
+  cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
+  
   for( auto elem : turn.exception_files() ) {
     std::set<size_t> files(elem.second.begin(), elem.second.end());
     enabled_exceptions.turn_on_off(turn.enabled,
@@ -204,15 +208,17 @@ apply_cdf_turn( const exception_turn_t& turn ) {
 %type   <file>		filename
 %type   <files>         filenames
 
-%token BY 478
+%type   <number>        cdf_stackable
+
+%token BY 486
 %token COPY 362
 %token CDF_DISPLAY 384    ">>DISPLAY"
-%token IN 597
+%token IN 605
 %token NAME 286
 %token NUMSTR 305    "numeric literal"
-%token OF 678
-%token PSEUDOTEXT 713
-%token REPLACING 735
+%token OF 686
+%token PSEUDOTEXT 721
+%token REPLACING 743
 %token LITERAL 298
 %token SUPPRESS 376
 
@@ -227,25 +233,32 @@ apply_cdf_turn( const exception_turn_t& turn ) {
 %token CDF_WHEN 389    ">>WHEN"
 %token CDF_END_EVALUATE 390    ">>END-EVALUATE"
 
-%token AS 460  CONSTANT 361  DEFINED 363
+%token ALL 450
+%token CALL_CONVENTION 391    ">>CALL-CONVENTION"
+%token COBOL_WORDS 380    ">>COBOL-WORDS"
+%token CDF_PUSH 394    ">>PUSH"
+%token CDF_POP 395    ">>POP"
+%token SOURCE_FORMAT 396    ">>SOURCE FORMAT"
+
+%token AS 468  CONSTANT 361  DEFINED 363
 %type	<boolean>	     DEFINED
-%token OTHER 690  PARAMETER_kw 368    "PARAMETER"
-%token OFF 679  OVERRIDE 369
-%token THRU 931
-%token TRUE_kw 805    "True"
+%token OTHER 698  PARAMETER_kw 368    "PARAMETER"
+%token OFF 687  OVERRIDE 369
+%token THRU 939
+%token TRUE_kw 813    "True"
 
-%token CALL_COBOL 391    "CALL"
-%token CALL_VERBATIM 392    "CALL (as C)"
+%token CALL_COBOL 392    "CALL"
+%token CALL_VERBATIM 393    "CALL (as C)"
 
-%token TURN 807  CHECKING 488  LOCATION 641  ON 681  WITH 833
+%token TURN 815  CHECKING 496  LOCATION 649  ON 689  WITH 841
 
-%left OR 932
-%left AND 933
-%right NOT 934
-%left '<'  '>'  '='  NE 935  LE 936  GE 937
+%left OR 940
+%left AND 941
+%right NOT 942
+%left '<'  '>'  '='  NE 943  LE 944  GE 945
 %left '-'  '+'
 %left '*'  '/'
-%right NEG 939
+%right NEG 947
 
 %define api.prefix {ydf}
 %define api.token.prefix{YDF_}
@@ -277,6 +290,8 @@ complete:	cdf_define
 	|	cdf_display
 	|	cdf_turn
         |       cdf_call_convention
+        |       cdf_push
+        |       cdf_pop
 		;
 
 		/*
@@ -328,6 +343,7 @@ cdf_define:	CDF_DEFINE cdf_constant NAME as cdf_expr[value] override
 		  }
 		  if( !cdfval_add( $NAME, cdfval_t($value), $override) ) {
 		    error_msg(@NAME, "name already in dictionary: %s", $NAME);
+                    cdf_values_t& dictionary( cdf_dictionary() );
 		    const cdfval_t& entry = dictionary[$NAME];
 		    if( entry.filename ) {
 		      error_msg(@NAME, "%s previously defined in %s:%d",
@@ -357,7 +373,7 @@ cdf_define:	CDF_DEFINE cdf_constant NAME as cdf_expr[value] override
 		 * available regardless.
 		 */
 		{
-		  if( 0 == dictionary.count($NAME) ) {
+		  if( 0 == cdf_dictionary().count($NAME) ) {
 		    yywarn("CDF: '%s' is defined AS PARAMETER "
 			    "but was not defined", $NAME);
 		  }
@@ -400,6 +416,35 @@ cdf_call_convention:
                 }
                 ;
 
+cdf_push:       CDF_PUSH cdf_stackable {
+		  switch( $cdf_stackable ) {
+                  case YDF_ALL: 		cdf_push(); break;
+                  case YDF_CALL_CONVENTION: cdf_push_call_convention(); break;
+                  case YDF_CDF_DEFINE: 	cdf_push_dictionary(); break;
+                  case YDF_COBOL_WORDS: 	cdf_push_current_tokens(); break;
+                  case YDF_SOURCE_FORMAT:	cdf_push_source_format(); break;
+                  default: gcc_unreachable(); 
+                  }
+                }
+                ;
+cdf_pop:        CDF_POP cdf_stackable {
+		  switch( $cdf_stackable ) {
+                  case YDF_ALL: 		cdf_pop(); break;
+                  case YDF_CALL_CONVENTION: cdf_pop_call_convention(); break;
+                  case YDF_CDF_DEFINE: 	cdf_pop_dictionary(); break;
+                  case YDF_COBOL_WORDS: 	cdf_pop_current_tokens(); break;
+                  case YDF_SOURCE_FORMAT:	cdf_pop_source_format(); break; 
+                  default: gcc_unreachable(); 
+                  }
+                }
+                ;
+
+cdf_stackable:  ALL		{ $$ = YDF_ALL; }
+        |       CALL_CONVENTION	{ $$ = YDF_CALL_CONVENTION; }
+        |       COBOL_WORDS	{ $$ = YDF_COBOL_WORDS; }
+        |       CDF_DEFINE	{ $$ = YDF_CDF_DEFINE; }
+        |       SOURCE_FORMAT	{ $$ = YDF_SOURCE_FORMAT; }
+                ;
 
 except_names: 	except_name
 	|	except_names except_name
@@ -471,6 +516,7 @@ cdf_eval_obj:	cdf_cond_expr
 cdf_cond_expr:	BOOL
 	|	NAME DEFINED
 		{
+                  cdf_values_t& dictionary( cdf_dictionary() );
 		  auto p = dictionary.find($1);
 		  bool found = p != dictionary.end();
 		  if( !$DEFINED ) found = ! found;
@@ -552,6 +598,7 @@ cdf_expr:	cdf_expr '+' cdf_expr { $$ = $1(@1) + $3(@3); }
         ;
 
 cdf_factor:     NAME {
+                  cdf_values_t& dictionary( cdf_dictionary() );
 		  auto that = dictionary.find($1);
 		  if( that != dictionary.end() ) {
 		    $$ = that->second;
@@ -651,6 +698,7 @@ name_any:	namelit
 
 name_one:	NAME
 		{
+                  cdf_values_t& dictionary( cdf_dictionary() );
 		  cdf_arg_t arg = { YDF_NAME, $1 };
 		  auto p = dictionary.find($1);
 
@@ -665,6 +713,7 @@ name_one:	NAME
 
 namelit:	name
 		{
+                  cdf_values_t& dictionary( cdf_dictionary() );
 		  cdf_arg_t arg = { YDF_NAME, $1 };
 		  auto p = dictionary.find($1);
 
@@ -745,6 +794,7 @@ location_set( const YYLTYPE& loc ) {
 bool // used by cobol1.cc
 defined_cmd( const char arg[] )
 {
+  cdf_values_t& dictionary( cdf_dictionary() );
   cdfval_t value(1);
 
   char *name = xstrdup(arg);
@@ -868,6 +918,7 @@ static int ydflex(void) {
 
 bool
 cdf_value( const char name[], const cdfval_t& value ) {
+  cdf_values_t& dictionary( cdf_dictionary() );
   auto p = dictionary.find(name);
 
   if( p != dictionary.end() ) return false;
@@ -878,6 +929,7 @@ cdf_value( const char name[], const cdfval_t& value ) {
 
 const cdfval_t *
 cdf_value( const char name[] ) {
+  cdf_values_t& dictionary( cdf_dictionary() );
   auto p = dictionary.find(name);
 
   if( p == dictionary.end() ) return NULL;
diff --git a/gcc/cobol/cdfval.h b/gcc/cobol/cdfval.h
index 465bdbb..cc474a2 100644
--- a/gcc/cobol/cdfval.h
+++ b/gcc/cobol/cdfval.h
@@ -126,4 +126,8 @@ cdf_value( const char name[] );
 bool
 cdf_value( const char name[], const cdfval_t& value );
 
+typedef std::map<std::string, cdfval_t> cdf_values_t;
+
+cdf_values_t& cdf_dictionary();
+
 #endif
diff --git a/gcc/cobol/cobol1.cc b/gcc/cobol/cobol1.cc
index 4bd79f1..3146da5 100644
--- a/gcc/cobol/cobol1.cc
+++ b/gcc/cobol/cobol1.cc
@@ -357,7 +357,7 @@ cobol_langhook_handle_option (size_t scode,
             return true;
 
         case OPT_M:
-	    cobol_set_pp_option('M');
+            cobol_set_pp_option('M');
             return true;
 
         case OPT_fstatic_call:
@@ -368,16 +368,18 @@ cobol_langhook_handle_option (size_t scode,
             wsclear(cobol_default_byte);
             return true;
 
-        case OPT_fflex_debug:
+        case OPT_fflex_debug: // cppcheck-suppress syntaxError // The need for this is a mystery
             yy_flex_debug = 1;
             cobol_set_debugging( true, yy_debug == 1, cobol_trace_debug == 1 );
             return true;
+
         case OPT_fyacc_debug:
             yy_debug = 1;
             cobol_set_debugging(yy_flex_debug == 1,
                                 true,
                                 cobol_trace_debug == 1 );
             return true;
+
         case OPT_ftrace_debug:
             cobol_set_debugging( yy_flex_debug == 1, yy_debug == 1, true );
             return true;
@@ -406,11 +408,13 @@ cobol_langhook_handle_option (size_t scode,
         case OPT_fsyntax_only:
           mode_syntax_only(identification_div_e);
           break;
+
         case OPT_preprocess:
           if( ! preprocess_filter_add(arg) ) {
             cbl_errx( "could not execute preprocessor %s", arg);
           }
           return true;
+
         case OPT_include:
           if( ! include_file_add(arg) ) {
             cbl_errx( "could not include %s", arg);
diff --git a/gcc/cobol/dts.h b/gcc/cobol/dts.h
index aa3fa58..c900c45 100644
--- a/gcc/cobol/dts.h
+++ b/gcc/cobol/dts.h
@@ -86,24 +86,30 @@ namespace dts {
 #if __cpp_exceptions
       static const char msg[] = "input not NUL-terminated";
       throw std::domain_error( msg );
-#else
-      // eoinput terminates input
-      eoinput = strchr(input, '\0'); // cppcheck-suppress uselessAssignmentPtrArg
 #endif
     }
     auto ncm = re.size();
     cm.resize(ncm);
     std::vector <regmatch_t> cms(ncm);
 
-
     int erc = regexec( &re, input, ncm, cms.data(), 0 );
     if( erc != 0 ) return false;
+#if  __cpp_exceptions
+    // This is not correct at all, but current use depends on current behavior.
+    // The following line is excluded from the GCC build, which is compiled
+    // without __cpp_exceptions.  parse_copy_directive (for one) depends on
+    // regex_search returning true even if the match is beyond eoinput.
+    if( eoinput < cm[0].second ) return false;
+    // Correct behavior would return match only between input and eoinput.
+    // Because regex(3) uses a NUL terminator, it may match text between
+    // eoinput and the NUL.
+#endif
     std::transform( cms.begin(), cms.end(), cm.begin(),
                     [input]( const regmatch_t& m ) {
                       return csub_match( input, m );
                     } );
     return true;
   }
-};
+}
 
 
diff --git a/gcc/cobol/except.cc b/gcc/cobol/except.cc
index 60b8416..df1c7df 100644
--- a/gcc/cobol/except.cc
+++ b/gcc/cobol/except.cc
@@ -84,8 +84,6 @@ cbl_enabled_exception_t::dump( int i ) const {
               file );
 }
 
-cbl_enabled_exceptions_t enabled_exceptions;
-
 void
 cbl_enabled_exceptions_t::dump() const {
   extern int yydebug;
@@ -98,7 +96,7 @@ cbl_enabled_exceptions_t::dump() const {
     return;
   }
   int i = 1;
-  for( auto& elem : *this ) {
+  for( auto& elem : *this ) { // cppcheck-suppress constVariableReference
     dbgmsg("cbl_enabled_exceptions_t: %2d  {%s, %s, %lu}",
            i++,
            elem.location? "with location" : "  no location", 
diff --git a/gcc/cobol/gcobc b/gcc/cobol/gcobc
index 01c75dd..fa9f609 100755
--- a/gcc/cobol/gcobc
+++ b/gcc/cobol/gcobc
@@ -125,25 +125,24 @@ $0 recognizes the following GnuCOBOL cobc compilation options:
         -std=mvs -std=mvs-strict
         -std=mf  -std=mf-strict
         -std=cobol85 -std=cobol2002 -std=cobol2014
- Options that are the same in gcobol and cobc are passed through verbatim.
- Options that have no analog in gcobol produce a warning message.
- To produce this message, use -HELP.
+Options that are the same in gcobol and cobc are passed through verbatim.
+Options that have no analog in gcobol produce a warning message.
+To produce this message, use -HELP.
 To see the constructed cobc command-line, use -echo.
 To override the default cobc, set the "cobc" environment variable.
 By default, gcobc invokes the gcobol the same directory the gcobc resides.
 To override, set the gcobol environment variable.
- EOF
- }
+EOF
+}
 
-dialect="gnu"
+dialect="mf gnu"
 out_set=""
 first=""
 
- #
- # Simply iterate over the command-line tokens.  We can't use getopts
- # here because it's not designed for single-dash words (e.g. -shared).
 #
-
+# Iterate over the command-line tokens.  We can't use getopts here
+# because it's not designed for single-dash words (e.g. -shared).
+#
 for opt in "$@"
 do
     if [ "$skip_arg" ]
@@ -441,11 +440,13 @@ do
         -std=mvs | -std=mvs-strict | -std=ibm | -std=ibm-strict) dialect=ibm
                   ;;
         -std=mf | -std=mf-strict)  dialect=mf
-                  ;;
-        -std=default)  dialect=gnu  # that's GnuCOBOL's default and GCC's dialect for GnuCOBOL
-                   ;;
-        -std=cobol*)    dialect=""   # GCC COBOL targets COBOL2024 "mostly backward to COBOL85"
-                   ;;
+                                   ;;
+                      # GnuCOBOL's default and GCC's dialect for GnuCOBOL
+        -std=default) dialect=gnu  
+                      ;;
+                      # GCC COBOL targets COBOL2024 "mostly backward to COBOL85"
+        -std=cobol*)  dialect="" 
+                      ;;
         -std=*)
             dialect=""
             warn "$opt (unkown dialect)"
@@ -480,7 +481,8 @@ do
            opts="$opts /dev/stdin"
            ;;
 
-        *) if [ -z "$output_name" ]  # first non-option argument is source file name
+           # First file name argument is default output filename.
+        *) if [ -z "$output_name" -a -e "$opt" ]  
            then
                output_name=$(basename "${opt%.*}")
                case $mode in
@@ -512,6 +514,11 @@ fi
 # To override the default gcobol, set the "gcobol" environment variable.
 gcobol="${gcobol:-${0%/*}/gcobol}"
 
+if [ "$dialect" ]
+then
+    dialect=$(echo $dialect | sed -E 's/[[:alnum:]]+/-dialect &/g')
+fi
+
 if [ "$echo" ]
 then
     echo $gcobol $mode $opts
@@ -523,4 +530,4 @@ then
     set -x
 fi
 
-exec $gcobol $mode $opts
+exec $gcobol $mode $dialect $opts
diff --git a/gcc/cobol/gcobol.1 b/gcc/cobol/gcobol.1
index 6db5400..ebb833c 100644
--- a/gcc/cobol/gcobol.1
+++ b/gcc/cobol/gcobol.1
@@ -587,6 +587,21 @@ statement, regardless of compile-time constants.
 ..
 .
 .Ss Implemented Exception Conditions
+By default, per ISO, no EC is enabled.  Implemented ECs may be enabled
+on the command line or via the
+.Sy TURN
+directive. Any attempt to enable an EC that is not implemented is
+treated as an error.
+.Pp
+An enabled EC not handled by a
+.Sy DECLARATIVE
+is written to the system log and to standard error.  (The authors
+intend to make that an option.)  A fatal EC not handled with
+.Sy RESUME
+ends with a call to
+.Xr abort 3
+and process termination. 
+.Pp
 Not all Exception Conditions are implemented.  Any attempt to enable
 an EC that that is not implemented produces a warning message.
 The following are implemented:
@@ -913,11 +928,90 @@ or
 has no effect; the two are interchangeable.
 ..
 .
-.Ss Compiler-Directing Facility (CDF)
+.Sh COMPILER-DIRECTING FACILITY
 The CDF should be used with caution because no comprehensive test
-suite has been identified.
+suite has been identified.  
 .
-.Ss Conditional Compilation
+.Ss CDF Text Manipulation
+.Bl -tag -width >>DEFINE
+.It Sy COPY Ar copybook Li Oo OF|BY Ar library Oc Oo Sy REPLACING ... Oc
+If
+.Ar copybook
+is a literal, it treated a literal filename, which either does or does not exist.  If
+.Ar copybook
+is a \*[lang] word,
+.Nm
+looks first for an environment variable named
+.Va copybook
+and, if found, uses the contents of that variable as the name of the
+copybook file.  If that file does not exist, it continues looking for
+a file named one of:
+.sp
+.Bl -bullet -compact -offset 5n
+.It
+.Pa copybook
+(literally)
+.It
+.Pa copybook.cpy
+.It
+.Pa copybook.CPY
+.It
+.Pa copybook.cbl
+.It
+.Pa copybook.CBL
+.It
+.Pa copybook.cob
+.It
+.Pa copybook.COB
+.El
+.sp
+in that order.  It looks first in the same directory as the source
+code file, and then in any
+.Ar copybook-path
+named with the
+.Fl I
+option.
+.
+.\" FIXME: need escape mechanism for directories with ':' in the name.
+.Ar copybook-path
+may (like the shell's
+.Ev PATH
+variable) be a colon-separated list.
+The
+.Fl I
+option may occur multiple times on the command line.  Each successive
+.Ar copybook-path
+is concatenated to previous ones.
+Relative paths (having no leading
+.Ql / Ns
+\&)
+are searched relative to the compiler's current working directory.
+.Pp
+For example,
+.D1 \&
+.D1 Fl I Li /usr/local/include:include
+.D1 \&
+searches first the directory where the \*[lang] program is found, next in
+.Pa /usr/local/include ,
+and finally in an
+.Pa include
+subdirectory of the directory from which
+.Nm
+was invoked.
+.Pp
+For the
+.Sy REPLACING
+phrase, both the modern pseudo-text and the \*[lang]/85 forms are
+recognized. (The older forms are used in the NIST CCVS/85 test suite.) 
+.It Sy REPLACE ...
+.Nm
+supports the full ISO
+.Sy REPLACE
+syntax.  
+.El
+.
+.Ss CDF Directives
+.\"Bl -tag -width >>PROPAGATE
 .Bl -tag -width >>DEFINE
 .It >> Ns Sy DEFINE Ar name Sy AS Bro Ar expression Li | Sy PARAMETER Brc Op Sy OVERRIDE
 Define
@@ -959,10 +1053,6 @@ is supported. Boolean literals are not supported.
 .
 .It >> Ns Sy EVALUATE
 Not implemented.
-.El
-.
-.Ss Other CDF Directives
-.Bl -tag -width >>PROPAGATE
 .It >> Ns Sy CALL-CONVENTION Ar convention
 .Ar convention
 may be one of:
@@ -1036,6 +1126,24 @@ Not implemented.
 Not implemented.
 .It >> Ns Sy PROPAGATE
 Not implemented.
+.It >> Ns Sy PUSH Ar directive
+.It >> Ns Sy POP  Ar directive
+With
+.Sy PUSH ,
+push CDF state onto a stack.
+With
+.Sy POP ,
+return to the prior pushed state.
+.Ar directive
+may be one of
+.Bl -tag -compact
+.It Sy CALL-CONVENTION
+.It Sy COBOL-WORDS
+.It Sy DEFINE
+.It Sy SOURCE FORMAT
+.It Sy TURN
+.El
+.
 .It >> Ns Sy TURN Oo
 .Ar ec Oo Ar file Li ... Oc ...
 .Oc Sy CHECKING Bro Oo Sy ON Oc Oo Oo Sy WITH Oc Sy LOCATION Oc | Sy OFF Brc
@@ -1087,76 +1195,6 @@ the directive must appear before
 To test a feature-set variable, use
 .Dl >>IF Ar feature Li DEFINED
 .
-.Ss Copybooks
-.Nm
-supports the CDF
-.Sy COPY
-statement, with or without its
-.Sy REPLACING
-component.  For any statement
-.sp
-.D1 COPY Ar copybook
-.sp
-.Nm
-looks first for an environment variable named
-.Va copybook
-and, if found, uses the contents of that variable as the name of the
-copybook file.  If that file does not exist, it continues looking for
-a file named one of:
-.sp
-.Bl -bullet -compact -offset 5n
-.It
-.Pa copybook
-(literally)
-.It
-.Pa copybook.cpy
-.It
-.Pa copybook.CPY
-.It
-.Pa copybook.cbl
-.It
-.Pa copybook.CBL
-.It
-.Pa copybook.cob
-.It
-.Pa copybook.COB
-.El
-.sp
-in that order.  It looks first in the same directory as the source
-code file, and then in any
-.Ar copybook-path
-named with the
-.Fl I
-option.
-.
-.\" FIXME: need escape mechanism for directories with ':' in the name.
-.Ar copybook-path
-may (like the shell's
-.Ev PATH
-variable) be a colon-separated list.
-.
-The
-.Fl I
-option may occur multiple times on the command line.  Each successive
-.Ar copybook-path
-is concatenated to previous ones.
-Relative paths (having no leading
-.Ql / Ns
-\&)
-are searched relative to the compiler's current working directory.
-.Pp
-For example,
-.D1 \&
-.D1 Fl I Li /usr/local/include:include
-.D1 \&
-searches first the directory where the \*[lang] program is found, next in
-.Pa /usr/local/include ,
-and finally in an
-.Pa include
-subdirectory of the directory from which
-.Nm
-was invoked.
-.
 .Ss Intrinsic functions
 .Nm
 implements all intrinsic functions defined by \*[isostd], plus a few
diff --git a/gcc/cobol/gcobolspec.cc b/gcc/cobol/gcobolspec.cc
index 70784d7..1f1b463 100644
--- a/gcc/cobol/gcobolspec.cc
+++ b/gcc/cobol/gcobolspec.cc
@@ -142,9 +142,6 @@ lang_specific_driver (struct cl_decoded_option **in_decoded_options,
   int n_infiles = 0;
   int n_outfiles = 0;
 
-  // The number of input files when the language is "none" or "cobol"
-  int n_cobol_files = 0;
-
   // saw_OPT_no_main means "don't expect -main"
   bool saw_OPT_no_main = false;
 
@@ -234,11 +231,6 @@ lang_specific_driver (struct cl_decoded_option **in_decoded_options,
       case OPT_SPECIAL_input_file:
         no_files_error = false;
         n_infiles += 1;
-        if(    strcmp(language, "none")  == 0
-            || strcmp(language, "cobol") == 0 )
-          {
-          n_cobol_files += 1;
-          }
         if( strstr(decoded_options[i].orig_option_with_args_text, "libgcobol.a") )
           {
           // We have been given an explicit libgcobol.a.  We need to note that.
@@ -478,7 +470,10 @@ lang_specific_driver (struct cl_decoded_option **in_decoded_options,
 ////        break;
 ////#endif
       case OPT_static:
+#if defined (HAVE_LD_STATIC_DYNAMIC)
+        append_arg(decoded_options[i]);
         static_in_general = true;
+#endif        
         break;
 
       default:
@@ -506,17 +501,23 @@ lang_specific_driver (struct cl_decoded_option **in_decoded_options,
     need_libgcobol = false;
     }
 
+  if( static_in_general )
+    {
+    // These two options interfere with each other.
+    static_libgcobol = false;
+    }
+
   if( need_libgcobol )
     {
     add_arg_lib(COBOL_LIBRARY, static_libgcobol);
     }
   if( need_libdl )
     {
-    add_arg_lib(DL_LIBRARY, static_in_general);
+    add_arg_lib(DL_LIBRARY, false);
     }
   if( need_libstdc )
     {
-    add_arg_lib(STDCPP_LIBRARY, static_in_general);
+    add_arg_lib(STDCPP_LIBRARY, false);
     }
 
   if( prior_main )
@@ -529,7 +530,8 @@ lang_specific_driver (struct cl_decoded_option **in_decoded_options,
   // cl_decoded_option
 
   size_t new_option_count = new_opt.size();
-  struct cl_decoded_option *new_options = XNEWVEC (struct cl_decoded_option, new_option_count);
+  struct cl_decoded_option *new_options = XNEWVEC (struct cl_decoded_option,
+                                                    new_option_count);
 
   for(size_t i=0; i<new_option_count; i++)
     {
@@ -539,7 +541,7 @@ lang_specific_driver (struct cl_decoded_option **in_decoded_options,
 #ifdef NOISY
   verbose = true;
 #endif
-  if( verbose && new_options != original_options )
+  if( verbose && new_options != original_options ) // cppcheck-suppress knownConditionTrueFalse
     {
     fprintf(stderr, _("Driving: (" HOST_SIZE_T_PRINT_DEC ")\n"),
             (fmt_size_t)new_option_count);
diff --git a/gcc/cobol/genapi.cc b/gcc/cobol/genapi.cc
index 8017788..c9d2da4 100644
--- a/gcc/cobol/genapi.cc
+++ b/gcc/cobol/genapi.cc
@@ -27,6 +27,7 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+
 #include "cobol-system.h"
 
 #include "coretypes.h"
@@ -117,7 +118,7 @@ typedef struct TREEPLET
 
 static
 void
-treeplet_fill_source(TREEPLET &treeplet, cbl_refer_t &refer)
+treeplet_fill_source(TREEPLET &treeplet, const cbl_refer_t &refer)
   {
   treeplet.pfield = gg_get_address_of(refer.field->var_decl_node);
   treeplet.offset = refer_offset(refer);
@@ -190,6 +191,9 @@ const char *gv_trace_switch = NULL;
 char const *bTRACE1 = NULL;
 tree trace_handle;
 tree trace_indent;
+
+// This variable is set to true when the output cursor is known to be at the
+// start-of-line.
 bool cursor_at_sol = true;
 
 static void
@@ -229,6 +233,13 @@ trace1_init()
     }
   }
 
+static
+void
+insert_nop(int n)
+  {
+  gg_assign(var_decl_nop, build_int_cst_type(INT, n));
+  }
+
 static void
 create_cblc_string_variable(const char *var_name, const char *var_contents)
   {
@@ -266,8 +277,6 @@ build_main_that_calls_something(const char *something)
     SHOW_PARSE_END
     }
 
-  gg_set_current_line_number(DEFAULT_LINE_NUMBER);
-
   tree function_decl = gg_define_function( INT,
                                            "main",
                                            "main",
@@ -321,7 +330,6 @@ build_main_that_calls_something(const char *something)
                                        argc,
                                        argv,
                                        NULL_TREE)));
-  strncpy(ach_cobol_entry_point, psz, sizeof(ach_cobol_entry_point)-1);
   free(psz);
   gg_finalize_function();
   }
@@ -361,8 +369,11 @@ level_88_helper(size_t parent_capacity,
                 size_t &returned_size)
   {
   // We return a MALLOCed return value, which the caller must free.
-  char *retval  = (char *)xmalloc(parent_capacity + 64);
-  char *builder = (char *)xmalloc(parent_capacity + 64);
+  char *retval  = static_cast<char *>(xmalloc(parent_capacity + 64));
+  gcc_assert(retval);
+  char *builder = static_cast<char *>(xmalloc(parent_capacity + 64));
+  gcc_assert(builder);
+
   size_t nbuild = 0;
 
   cbl_figconst_t figconst = cbl_figconst_of( elem.name());
@@ -403,7 +414,8 @@ level_88_helper(size_t parent_capacity,
 
     // Pick up the string
     size_t first_name_length  = elem.size();
-    char *first_name = (char *)xmalloc(first_name_length + 1);
+    char *first_name = static_cast<char *>(xmalloc(first_name_length + 1));
+    gcc_assert(first_name);
     memcpy(first_name, elem.name(), first_name_length);
     first_name[first_name_length] = '\0';
 
@@ -480,7 +492,7 @@ get_level_88_domain(size_t parent_capacity, cbl_field_t *var, size_t &returned_s
   // Numerics are converted to strings, and handled as above
 
   size_t retval_capacity = 64;
-  char *retval = (char *)xmalloc(retval_capacity);
+  char *retval = static_cast<char *>(xmalloc(retval_capacity));
   size_t output_index = 0;
 
   // Loop through the provided domains:
@@ -497,8 +509,9 @@ get_level_88_domain(size_t parent_capacity, cbl_field_t *var, size_t &returned_s
     if( output_index + stream_len > retval_capacity )
       {
       retval_capacity *= 2;
-      retval = (char *)xrealloc(retval, retval_capacity);
+      retval = static_cast<char *>(xrealloc(retval, retval_capacity));
       }
+    gcc_assert(retval);
     memcpy(retval + output_index, stream, stream_len);
     output_index += stream_len;
     returned_size += stream_len;
@@ -509,14 +522,23 @@ get_level_88_domain(size_t parent_capacity, cbl_field_t *var, size_t &returned_s
     if( output_index + stream_len > retval_capacity )
       {
       retval_capacity *= 2;
-      retval = (char *)xrealloc(retval, retval_capacity);
+      retval = static_cast<char *>(xrealloc(retval, retval_capacity));
       }
+    gcc_assert(retval);
     memcpy(retval + output_index, stream, stream_len);
     output_index += stream_len;
     returned_size += stream_len;
     free(stream);
     domain += 1;
     }
+
+  if( returned_size >= retval_capacity)
+    {
+    retval_capacity *= 2;
+    retval = static_cast<char *>(xrealloc(retval, retval_capacity));
+    }
+
+  gcc_assert(returned_size < retval_capacity);
   retval[returned_size++] = '\0';
   return retval;
   }
@@ -608,13 +630,8 @@ get_class_condition_string(cbl_field_t *var)
       // Since the first.name is a single character, we can do this as
       // a single-character pair.
 
-      // Keep in mind that the single character might be a two-byte UTF-8
-      // codepoint
-        uint8_t ch1 = domain->first.name()[0];
-        uint8_t ch2 = domain->last.name()[0];
-
-        gcc_assert(first_name_length <= 2);
-        gcc_assert(last_name_length <= 2);
+      uint8_t ch1;
+      uint8_t ch2;
 
       char *p2;
       size_t one;
@@ -766,8 +783,10 @@ parser_call_target_convention( tree func )
 void
 parser_call_targets_dump()
   {
-    dbgmsg( "call targets for #" HOST_SIZE_T_PRINT_UNSIGNED,
+    dbgmsg( "call targets for #" HOST_SIZE_T_PRINT_UNSIGNED " NOT dumping",
             (fmt_size_t)current_program_index() );
+#if 0 // A change to call_targets rendered this routine useless.  Until we get
+      // around to repairing it, this code is left for reference.
     for( const auto& elem : call_targets ) {
       const auto& k = elem.first;
       const auto& v = elem.second;
@@ -781,6 +800,7 @@ parser_call_targets_dump()
       }
       fprintf(stderr, " ]\n");
     }
+#endif
   }
 
 size_t
@@ -808,8 +828,8 @@ parser_call_target_update( size_t caller,
   }
 
 static tree
-function_pointer_from_name(cbl_refer_t &name,
-                          tree function_return_type)
+function_pointer_from_name(const cbl_refer_t &name,
+                           tree function_return_type)
   {
   Analyze();
 
@@ -885,7 +905,8 @@ function_pointer_from_name(cbl_refer_t &name,
   }
 
 void
-parser_initialize_programs(size_t nprogs, struct cbl_refer_t *progs)
+parser_initialize_programs( size_t nprogs,
+                            const struct cbl_refer_t *progs)
   {
   Analyze();
   SHOW_PARSE
@@ -985,14 +1006,13 @@ parser_compile_ecs( const std::vector<uint64_t>& ecs )
     return NULL_TREE;
     }
 
-  char ach[32];
+  char ach[64];
   static int counter = 1;
   sprintf(ach, "_ecs_table_%d", counter++);
   tree retval =  array_of_long_long(ach, ecs);
   SHOW_IF_PARSE(nullptr)
     {
     SHOW_PARSE_HEADER
-    char ach[64];
     snprintf(ach, sizeof(ach), " Size is %lu; retval is %p",
              gb4(ecs.size()), as_voidp(retval));
     SHOW_PARSE_TEXT(ach)
@@ -1001,7 +1021,6 @@ parser_compile_ecs( const std::vector<uint64_t>& ecs )
   TRACE1
     {
     TRACE1_HEADER
-    char ach[64];
     snprintf(ach, sizeof(ach), " Size is %lu; retval is %p",
              gb4(ecs.size()), as_voidp(retval));
     TRACE1_TEXT_ABC("", ach, "");
@@ -1034,14 +1053,13 @@ parser_compile_dcls( const std::vector<uint64_t>& dcls )
     return NULL_TREE;
     }
 
-  char ach[32];
+  char ach[64];
   static int counter = 1;
   sprintf(ach, "_dcls_table_%d", counter++);
   tree retval =  array_of_long_long(ach, dcls);
   SHOW_IF_PARSE(nullptr)
     {
     SHOW_PARSE_HEADER
-    char ach[64];
     snprintf(ach, sizeof(ach), " Size is %lu; retval is %p",
              gb4(dcls.size()), as_voidp(retval));
     SHOW_PARSE_TEXT(ach);
@@ -1050,7 +1068,6 @@ parser_compile_dcls( const std::vector<uint64_t>& dcls )
   TRACE1
     {
     TRACE1_HEADER
-    char ach[64];
     snprintf(ach, sizeof(ach), " Size is %lu; retval is %p",
              gb4(dcls.size()), as_voidp(retval));
     TRACE1_TEXT_ABC("", ach, "");
@@ -1167,21 +1184,13 @@ parser_statement_begin( const cbl_name_t statement_name,
   //  the execution time of a program doing two-billion simple adds in an inner
   //  loop dropped from 3.8 seconds to 0.175 seconds.
 
-  bool exception_processing = enabled_exceptions.size() ;
+  bool exception_processing = cdf_enabled_exceptions().size() ;
 
   if( !exception_processing )
     {
     exception_processing = file_ops.find(statement_name) != file_ops.end();
     }
 
-  if( gg_get_current_line_number() == DEFAULT_LINE_NUMBER )
-    {
-    // This code is intended to prevert GDB anomalies when the first line of a
-    // program is a PERFORM <proc> ... TEST AFTER ... UNTIL ...
-    gg_set_current_line_number(CURRENT_LINE_NUMBER-1);
-    gg_assign(var_decl_nop, build_int_cst_type(INT, 106));
-    }
-
   // At this point, if any exception is enabled, we store the location stuff.
   // Each file I-O routine calls store_location_stuff explicitly, because
   // those exceptions can't be defeated.
@@ -1189,14 +1198,9 @@ parser_statement_begin( const cbl_name_t statement_name,
   if( exception_processing )
     {
     store_location_stuff(statement_name);
-    }
-
-  gg_set_current_line_number(CURRENT_LINE_NUMBER);
-
-  if( exception_processing )
-    {
     set_exception_environment(ecs, dcls);
     }
+
   sv_is_i_o = false;
   }
 
@@ -1210,10 +1214,9 @@ initialize_variable_internal( cbl_refer_t refer,
             // gg_string_literal(refer.field->name),
             // NULL_TREE);
   cbl_field_t *parsed_var = refer.field;
-
-  if( parsed_var->type == FldLiteralA )
+  if( !parsed_var )
     {
-    return;
+    cbl_internal_error("%s should not be null", "parsed_var");
     }
 
   if( parsed_var->is_key_name() )
@@ -1229,7 +1232,7 @@ initialize_variable_internal( cbl_refer_t refer,
     return;
     }
 
-  if( parsed_var && parsed_var->type == FldBlob )
+  if( parsed_var->type == FldBlob )
     {
     return;
     }
@@ -1347,8 +1350,6 @@ initialize_variable_internal( cbl_refer_t refer,
     SHOW_PARSE_END
     }
 
-  CHECK_FIELD(parsed_var);
-
   // When initializing a variable, we have to ignore any DEPENDING ON clause
   // that might otherwise apply
   suppress_dest_depends = true;
@@ -1588,7 +1589,7 @@ parser_initialize(const cbl_refer_t& refer, bool like_parser_symbol_add)
 
 static void
 get_binary_value_from_float(tree         value,
-                            cbl_refer_t &dest,
+                      const cbl_refer_t &dest,
                             cbl_field_t *source,
                             tree         source_offset
                             )
@@ -1682,6 +1683,7 @@ depending_on_value(tree depending_on, cbl_field_t *current_sizer)
 //  gg_assign(occurs_lower, build_int_cst_type(LONG, current_sizer->occurs.bounds.lower));
 //  gg_assign(occurs_upper, build_int_cst_type(LONG, current_sizer->occurs.bounds.upper));
 
+  gcc_assert(current_sizer);
   if( current_sizer->occurs.depending_on )
     {
     get_depending_on_value_from_odo(depending_on, current_sizer);
@@ -1825,16 +1827,12 @@ normal_normal_compare(bool debugging,
               NULL_TREE);
     }
 
-  bool needs_adjusting;
   if( !left_intermediate && !right_intermediate )
     {
     // Yay!  Both sides have fixed rdigit values.
 
-    // Flag needs_adjusting as false, because we are going to do it here:
-    needs_adjusting = false;
     int adjust =   get_scaled_rdigits(left_side_ref->field)
                  - get_scaled_rdigits(right_side_ref->field);
-
     if( adjust > 0 )
       {
       // We need to make right_side bigger to match the scale of left_side
@@ -1849,6 +1847,7 @@ normal_normal_compare(bool debugging,
   else
     {
     // At least one side is right_intermediate
+    bool needs_adjusting;
 
     tree adjust;
     if( !left_intermediate && right_intermediate )
@@ -2357,7 +2356,7 @@ cobol_compare(  tree return_int,
                 build_int_cst_type(INT, rightflags),
                 integer_zero_node,
                 NULL_TREE));
-    compared = true;
+    // compared = true;  // Commented out to quiet cppcheck
     }
 
 //  gg_printf("   result is %d\n", return_int, NULL_TREE);
@@ -2377,6 +2376,8 @@ move_tree(  cbl_field_t  *dest,
     SHOW_PARSE_END
     }
 
+  CHECK_FIELD(dest);
+
   bool moved = true;
 
   tree source_length = gg_define_size_t();
@@ -2460,7 +2461,7 @@ move_tree(  cbl_field_t  *dest,
               psz_source,
               min_length,
               member(dest->var_decl_node, "picture"),
-              NULL);
+              NULL_TREE);
       break;
       }
 
@@ -2563,7 +2564,7 @@ get_string_from(cbl_field_t *field)
   }
 
 static char *
-combined_name(cbl_label_t *label)
+combined_name(const cbl_label_t *label)
   {
   // This routine returns a pointer to a static, so make sure you use the result
   // before calling the routine again
@@ -2578,7 +2579,7 @@ combined_name(cbl_label_t *label)
     if( label->parent )
       {
       // It's possible for implicit
-      cbl_label_t *section_label = cbl_label_of(symbol_at(label->parent));
+      const cbl_label_t *section_label = cbl_label_of(symbol_at(label->parent));
       sect_name = section_label->name;
       }
     }
@@ -2588,7 +2589,7 @@ combined_name(cbl_label_t *label)
     }
 
   static size_t retval_size = 256;
-  static char *retval= (char *)xmalloc(retval_size);
+  static char *retval= static_cast<char *>(xmalloc(retval_size));
 
   char *paragraph             = cobol_name_mangler(para_name);
   char *section         = cobol_name_mangler(sect_name);
@@ -2600,8 +2601,9 @@ combined_name(cbl_label_t *label)
                   + 24 )
     {
     retval_size *= 2;
-    retval = (char *)xrealloc(retval, retval_size);
+    retval = static_cast<char *>(xrealloc(retval, retval_size));
     }
+  gcc_assert(retval);
 
   *retval = '\0';
   char ach[24];
@@ -2648,8 +2650,9 @@ assembler_label(const char *label)
     {
     length = strlen(label) + strlen(local_text) + 1;
     free(build);
-    build = (char *)xmalloc(length);
+    build = static_cast<char *>(xmalloc(length));
     }
+  gcc_assert(build);
 
   strcpy(build, label);
   strcat(build, local_text);
@@ -2663,8 +2666,6 @@ section_label(struct cbl_proc_t *procedure)
   // With nested programs, you can have multiple program/section pairs with the
   // the same names; we use a deconflictor to avoid collisions
 
-  gg_set_current_line_number(CURRENT_LINE_NUMBER);
-
   size_t deconflictor = symbol_label_id(procedure->label);
 
   cbl_label_t *label = procedure->label;
@@ -2689,7 +2690,7 @@ section_label(struct cbl_proc_t *procedure)
     }
   assembler_label(psz2);
   free(psz2);
-  gg_assign(var_decl_nop, build_int_cst_type(INT, 108));
+  insert_nop(108);
   }
 
 static void
@@ -2704,8 +2705,6 @@ paragraph_label(struct cbl_proc_t *procedure)
   //      are not referenced by the program.  We provide a deconflictor to
   //      separate such labels.
 
-  gg_set_current_line_number(CURRENT_LINE_NUMBER);
-
   cbl_label_t *paragraph  = procedure->label;
   cbl_label_t *section    = nullptr;
 
@@ -2727,6 +2726,9 @@ paragraph_label(struct cbl_proc_t *procedure)
           section_name ? section_name: "(null)" ,
           current_function->our_unmangled_name ? current_function->our_unmangled_name: "" ,
           (fmt_size_t)deconflictor );
+
+  // (0) is wrong, so back up one
+
   gg_insert_into_assembler(psz1);
 
   SHOW_PARSE
@@ -2743,7 +2745,25 @@ paragraph_label(struct cbl_proc_t *procedure)
                           combined_name(procedure->label));
   assembler_label(psz2);
   free(psz2);
-  gg_assign(var_decl_nop, build_int_cst_type(INT, 109));
+
+  // We are inserting a NOP after having created a label for the procedure.
+  // This means that when using GDC_COBOL to step into a procedure, the
+  // execution will stop there and show "123 para-name." at the stopped point.
+  //
+  // Note that because there is no user-specified executable code at that point
+  // the user can't set a working breakpoint with "break 123".  But because
+  // GDB will pick up the psz2 text and set a breakpoint there (which is the
+  // location of the NOP) "break para-name" will actually stop and show line
+  // 123.
+  //
+  // This really only makes sense when you look at the assembly language. Keep
+  // in mind as you read it that issuing a "break 123" causes GDB to set a
+  // breakpoint at the first executable machine language code following the
+  // first ".loc 123" directive.
+  //
+  // Yes, trying to understand this causes headaches for many people who read
+  // this.  Take an aspirin.
+  insert_nop(109);
   }
 
 static void
@@ -2787,6 +2807,7 @@ pseudo_return_pop(cbl_proc_t *procedure)
               NULL_TREE);
     }
 
+  token_location_override(current_location_minus_one());
   IF( var_decl_exit_address, eq_op, procedure->exit.addr )
     {
     TRACE1
@@ -2796,11 +2817,13 @@ pseudo_return_pop(cbl_proc_t *procedure)
     // The top of the stack is us!
 
     // Pick up the return address from the pseudo_return stack:
+    token_location_override(current_location_minus_one());
     gg_assign(current_function->void_star_temp,
               gg_call_expr( VOID_P,
                             "__gg__pseudo_return_pop",
                             NULL_TREE));
     // And do the return:
+    token_location_override(current_location_minus_one());
     gg_goto(current_function->void_star_temp);
     }
   ELSE
@@ -2834,11 +2857,13 @@ leave_procedure(struct cbl_proc_t *procedure, bool /*section*/)
     // procedure->bottom.label);
     // Procedure can be null, for example at the beginning of a
     // new program, or after somebody else has cleared it out.
+
     gg_append_statement(procedure->exit.label);
 
     char *psz;
     psz = xasprintf("_procret." HOST_SIZE_T_PRINT_DEC ":",
                     (fmt_size_t)symbol_label_id(procedure->label));
+    token_location_override(current_location_minus_one());
     gg_insert_into_assembler(psz);
     free(psz);
     pseudo_return_pop(procedure);
@@ -2951,7 +2976,9 @@ find_procedure(cbl_label_t *label)
     static int counter=1;
 
     // This is a new section or paragraph; we need to create its values:
-    retval = (struct cbl_proc_t *)xmalloc(sizeof(struct cbl_proc_t));
+    retval = static_cast<struct cbl_proc_t *>
+                                          (xmalloc(sizeof(struct cbl_proc_t)));
+    gcc_assert(retval);
     retval->label = label;
 
     gg_create_goto_pair(&retval->top.go_to,
@@ -3007,6 +3034,8 @@ parser_enter_section(cbl_label_t *label)
     {
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL(" ", label)
+    SHOW_PARSE_INDENT
+    linemap_dump_location( line_table, current_token_location(), stderr );
     SHOW_PARSE_END
     }
 
@@ -3014,8 +3043,7 @@ parser_enter_section(cbl_label_t *label)
 
   // This NOP is needed to give GDB a line number for the entry point of
   // paragraphs
-  gg_set_current_line_number(CURRENT_LINE_NUMBER);
-  gg_assign(var_decl_nop, build_int_cst_type(INT, 101));
+  insert_nop(101);
 
   struct cbl_proc_t *procedure = find_procedure(label);
   gg_append_statement(procedure->top.label);
@@ -3042,6 +3070,8 @@ parser_enter_paragraph(cbl_label_t *label)
     {
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL(" ", label)
+    SHOW_PARSE_INDENT
+    linemap_dump_location( line_table, current_token_location(), stderr );
     SHOW_PARSE_END
     }
 
@@ -3257,16 +3287,20 @@ parser_goto( cbl_refer_t value_ref, size_t narg, cbl_label_t * const labels[] )
 void
 parser_perform(cbl_label_t *label, bool suppress_nexting)
   {
-  label->used = yylineno;
   Analyze();
   SHOW_PARSE
     {
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL(" ", label)
     char ach[32];
-    sprintf(ach, " label is at %p", (void*)label);
+    sprintf(ach, " label is at %p", static_cast<void*>(label));
     SHOW_PARSE_TEXT(ach)
-    sprintf(ach, " label->proc is %p", (void*)label->structs.proc);
+    if( label )
+      {
+      sprintf(ach,
+              " label->proc is %p",
+              static_cast<void*>(label->structs.proc));
+      }
     SHOW_PARSE_TEXT(ach)
     SHOW_PARSE_END
     }
@@ -3279,6 +3313,7 @@ parser_perform(cbl_label_t *label, bool suppress_nexting)
     }
 
   CHECK_LABEL(label);
+  label->used = yylineno;
 
   struct cbl_proc_t *procedure = find_procedure(label);
 
@@ -3315,9 +3350,9 @@ parser_perform(cbl_label_t *label, bool suppress_nexting)
   char ach[256];
   if( label->type == LblParagraph )
     {
-    cbl_label_t *section_label = cbl_label_of(symbol_at(label->parent));
+    const cbl_label_t *sec_label = cbl_label_of(symbol_at(label->parent));
     para_name = label->name;
-    sect_name = section_label->name;
+    sect_name = sec_label->name;
     sprintf(ach,
             "%s PERFORM %s of %s of %s (" HOST_SIZE_T_PRINT_DEC ")",
             ASM_COMMENT_START,
@@ -3377,9 +3412,9 @@ parser_perform_times( cbl_label_t *proc_1, cbl_refer_t count )
     SHOW_PARSE_REF(" ", count)
     SHOW_PARSE_TEXT(" TIMES")
     char ach[32];
-    sprintf(ach, " proc_1 is at %p", (void*)proc_1);
+    sprintf(ach, " proc_1 is at %p", static_cast<void*>(proc_1));
     SHOW_PARSE_TEXT(ach)
-    sprintf(ach, " proc_1->proc is %p", (void*)proc_1->structs.proc);
+    sprintf(ach, " proc_1->proc is %p", static_cast<void*>(proc_1->structs.proc));
     SHOW_PARSE_TEXT(ach)
     SHOW_PARSE_END
     }
@@ -3416,6 +3451,7 @@ parser_perform_times( cbl_label_t *proc_1, cbl_refer_t count )
   sprintf(ach,
           "_procretb." HOST_SIZE_T_PRINT_DEC ":",
           (fmt_size_t)our_pseudo_label);
+  token_location_override(current_location_minus_one());
   gg_insert_into_assembler(ach);
   }
 
@@ -3430,17 +3466,22 @@ internal_perform_through( cbl_label_t *proc_1,
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL(" ", proc_1);
     char ach[32];
-    sprintf(ach, " proc_1 is at %p", (void*)proc_1);
+    sprintf(ach, " proc_1 is at %p", static_cast<void*>(proc_1));
     SHOW_PARSE_TEXT(ach)
-    sprintf(ach, " proc_1->proc is %p", (void*)proc_1->structs.proc);
+    if( proc_1 )
+      {
+      sprintf(ach,
+              " proc_1->proc is %p",
+              static_cast<void*>(proc_1->structs.proc));
+      }
     SHOW_PARSE_TEXT(ach)
     if( proc_2 )
       {
       SHOW_PARSE_INDENT
-      SHOW_PARSE_LABEL("", proc_2);
-      sprintf(ach, " proc_2 is at %p", (void*)proc_2);
+      SHOW_PARSE_LABEL_OK("", proc_2);
+      sprintf(ach, " proc_2 is at %p", static_cast<void*>(proc_2));
       SHOW_PARSE_TEXT(ach)
-      sprintf(ach, " proc_2->proc is %p", (void*)proc_2->structs.proc);
+      sprintf(ach, " proc_2->proc is %p", static_cast<void*>(proc_2->structs.proc));
       SHOW_PARSE_TEXT(ach)
       }
     SHOW_PARSE_END
@@ -3453,14 +3494,12 @@ internal_perform_through( cbl_label_t *proc_1,
 
   CHECK_LABEL(proc_1);
 
-  if(!proc_2)
+  if( !proc_2 )
     {
     parser_perform(proc_1, suppress_nexting);
     return;
     }
 
-  CHECK_LABEL(proc_2);
-
   struct cbl_proc_t *proc1 = find_procedure(proc_1);
   struct cbl_proc_t *proc2 = find_procedure(proc_2);
 
@@ -3515,17 +3554,22 @@ internal_perform_through_times(   cbl_label_t *proc_1,
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL(" ", proc_1);
     char ach[32];
-    sprintf(ach, " proc_1 is at %p", (void*)proc_1);
+    sprintf(ach, " proc_1 is at %p", static_cast<void*>(proc_1));
     SHOW_PARSE_TEXT(ach)
-    sprintf(ach, " proc_1->proc is %p", (void*)proc_1->structs.proc);
+    if( proc_1 )
+      {
+      sprintf(ach,
+              " proc_1->proc is %p",
+              static_cast<void*>(proc_1->structs.proc));
+      }
     SHOW_PARSE_TEXT(ach)
     if( proc_2 )
       {
       SHOW_PARSE_INDENT
-      SHOW_PARSE_LABEL("", proc_2);
-      sprintf(ach, " proc_2 is at %p", (void*)proc_2);
+      SHOW_PARSE_LABEL_OK("", proc_2);
+      sprintf(ach, " proc_2 is at %p", static_cast<void*>(proc_2));
       SHOW_PARSE_TEXT(ach)
-      sprintf(ach, " proc_2->proc is %p", (void*)proc_2->structs.proc);
+      sprintf(ach, " proc_2->proc is %p", static_cast<void*>(proc_2->structs.proc));
       SHOW_PARSE_TEXT(ach)
       }
     SHOW_PARSE_REF(" ", count);
@@ -3561,6 +3605,7 @@ internal_perform_through_times(   cbl_label_t *proc_1,
   sprintf(ach,
           "_procretb." HOST_SIZE_T_PRINT_DEC ":",
           (fmt_size_t)our_pseudo_label);
+  token_location_override(current_location_minus_one());
   gg_insert_into_assembler( ach );
   }
 
@@ -3752,6 +3797,22 @@ parser_leave_file()
     {
     // We are leaving the top-level file, which means this compilation is
     // done, done, done.
+
+    // There is, however, one thing left to do.  If the command line says
+    // that this module needs a main entry point, then this is where
+    // we create a main() function.  We build it at the end, so that all of
+    // the .loc directives associated with it appear at the end of the
+    // source code.  We used to create the main() entry point at the beginning,
+    // but that created confusion for GDB when trying to debug the generated
+    // executable.
+    if( main_entry_point )
+      {
+      next_program_is_main = false;
+      build_main_that_calls_something(main_entry_point);
+      free(main_entry_point);
+      main_entry_point = NULL;
+      }
+
     gg_leaving_the_source_code_file();
     }
   }
@@ -3861,17 +3922,8 @@ parser_enter_program( const char *funcname_,
   // The first thing we have to do is mangle this name.  This is safe even
   // though the end result will be mangled again, because the mangler doesn't
   // change a mangled name.
-   
-  char *mangled_name;
- 
-  if( current_call_convention() == cbl_call_cobol_e )
-    {
-    mangled_name = cobol_name_mangler(funcname_);
-    }
-  else
-    {
-    mangled_name = xstrdup(funcname_);
-    }
+
+  char *mangled_name = cobol_name_mangler(funcname_);
 
   size_t parent_index = current_program_index();
   char *funcname;
@@ -3899,28 +3951,25 @@ parser_enter_program( const char *funcname_,
 
   if( !is_function && !parent_index )
     {
-    // This is a top_level program, and not a function
+    // This is a top_level program-id, and not a function
     if( next_program_is_main )
       {
+      // This is the first top-level program-id.
       next_program_is_main = false;
-      if(main_entry_point)
+      if( !main_entry_point )
         {
-        build_main_that_calls_something(main_entry_point);
-        free(main_entry_point);
-        main_entry_point = NULL;
-        }
-      else
-        {
-        build_main_that_calls_something(funcname);
+        // Because no explicit main_entry_point was specified, this program-id,
+        // the first in the file, becomes the target of the main() function
+        // that will be created at parser_leave_file time.
+        main_entry_point = xstrdup(funcname);
+
+        char *psz = cobol_name_mangler(main_entry_point);
+        strncpy(ach_cobol_entry_point, psz, sizeof(ach_cobol_entry_point)-1);
+        free(psz);
         }
       }
     }
 
-  // Call this after build_main_that_calls_something, because it manipulates
-  // the current line number to DEFAULT_LINE_NUMBER.  We have to manipulate it
-  // back afterward.
-  gg_set_current_line_number(CURRENT_LINE_NUMBER);
-
   if( strcmp(funcname_, "main") == 0 && this_module_has_main )
     {
     // setting 'retval' to 1 let's the caller know that we are being told
@@ -4110,6 +4159,8 @@ psa_FldLiteralN(struct cbl_field_t *field )
   // We are constructing a completely static constant structure, based on the
   // text string in .initial
 
+  CHECK_FIELD(field);
+
   FIXED_WIDE_INT(128) value = 0;
 
   do
@@ -4302,6 +4353,8 @@ psa_FldBlob(struct cbl_field_t *var )
     SHOW_PARSE_END
     }
 
+  CHECK_FIELD(var);
+
   // We are constructing a completely static constant structure.  We know the
   // capacity.  We'll create it from the data.initial.  The var_decl_node will
   // be a pointer to the data
@@ -4339,67 +4392,182 @@ psa_FldBlob(struct cbl_field_t *var )
   }
 
 void
-parser_accept(  struct cbl_refer_t refer,
-                enum special_name_t special_e )
+parser_accept(const struct cbl_refer_t &tgt,
+              special_name_t special_e,
+              cbl_label_t *error,
+              cbl_label_t *not_error )
   {
-  Analyze();
   SHOW_PARSE
     {
     SHOW_PARSE_HEADER
-    SHOW_PARSE_REF(" ", refer);
+    if( error )
+      {
+      SHOW_PARSE_LABEL(" error ", error)
+      }
+    if( not_error )
+      {
+      SHOW_PARSE_LABEL(" not_error ", not_error)
+      }
     SHOW_PARSE_END
     }
-  TRACE1
-    {
-    TRACE1_HEADER
-    TRACE1_END
-    }
-
-  /*
-  enum special_name_t
-      {
-        SYSIN_e,
-        SYSIPT_e,
-        SYSOUT_e,
-        SYSLIST_e,
-        SYSLST_e,
-        SYSPUNCH_e,
-        SYSPCH_e,
-        CONSOLE_e,
-        C01_e, C02_e, C03_e, C04_e, C05_e, C06_e,
-        C07_e, C08_e, C09_e, C10_e, C11_e, C12_e,
-        CSP_e,
-        S01_e, S02_e, S03_e, S04_e, S05_e,
-        AFP_5A_e,
-  };
-  */
 
   // The ISO spec describes the valid special names for ACCEPT as implementation
   // dependent.  We are following IBM's lead.
 
   tree environment = build_int_cst_type(INT, special_e);
 
-  switch( special_e )
+  const char *function_to_call = NULL;
+
+  switch(special_e)
     {
+    case STDIN_e:
     case CONSOLE_e:
     case SYSIPT_e:
     case SYSIN_e:
-      break;
-    default:
-      dbgmsg("%s(): We don't know what to do with special_name_t %d,", __func__, special_e);
-      dbgmsg("%s(): so we are ignoring it.", __func__);
-      yywarn("unrecognized SPECIAL NAME ignored");
+      // This is ordinary input from from the stdin:
+      gg_call(VOID,
+              "__gg__accept",
+              environment,
+              gg_get_address_of(tgt.field->var_decl_node),
+              refer_offset(tgt),
+              refer_size_dest(tgt),
+              NULL_TREE);
       return;
       break;
-    }
 
-  gg_call(VOID,
-          "__gg__accept",
-          environment,
-          gg_get_address_of(refer.field->var_decl_node),
-          refer_offset(refer),
-          refer_size_dest(refer),
-          NULL_TREE);
+    case C01_e:
+    case C02_e:
+    case C03_e:
+    case C04_e:
+    case C05_e:
+    case C06_e:
+    case C07_e:
+    case C08_e:
+    case C09_e:
+    case C10_e:
+    case C11_e:
+    case C12_e:
+    case CSP_e:
+    case S01_e:
+    case S02_e:
+    case S03_e:
+    case S04_e:
+    case S05_e:
+    case AFP_5A_e:
+    case STDOUT_e:
+    case SYSOUT_e:
+    case SYSLIST_e:
+    case SYSLST_e:
+    case STDERR_e:
+    case SYSPUNCH_e:
+    case SYSPCH_e:
+    case SYSERR_e:
+      cbl_internal_error("Not valid for ACCEPT statement.");
+      break;
+
+    case ARG_NUM_e:
+      // This ACCEPT statement wants the number of argv values:
+      gg_call(VOID,
+              "__gg__get_argc",
+              gg_get_address_of(tgt.field->var_decl_node),
+              refer_offset(tgt),
+              refer_size_source(tgt),
+              NULL_TREE);
+       return;
+       break;
+
+    case ENV_NAME_e:
+      // This fetches the environment name set by DISPLAY... UPON ENV_NAME_e
+      gg_call(VOID,
+              "__gg__get_env_name",
+              gg_get_address_of(tgt.field->var_decl_node),
+              refer_offset(tgt),
+              refer_size_source(tgt),
+              NULL_TREE);
+       return;
+       break;
+
+    case ENV_VALUE_e:
+      // This fetches the environment value associated with the previously
+      // esablished name
+      function_to_call = "__gg__get_env_value";
+      break;
+
+    case ARG_VALUE_e:
+      // We are fetching the variable whose index was established by a prior
+      // DISPLAY UPON ARGUMENT-NUMBER.  After the fetch, the value will be
+      // incremented by one.
+      function_to_call = "__gg__accept_arg_value";
+      break;
+    }
+  if( function_to_call )
+    {
+    tree erf = gg_define_int();
+    gg_assign(erf,
+              gg_call_expr(  INT,
+                            function_to_call,
+                            gg_get_address_of(tgt.field->var_decl_node),
+                            refer_offset(tgt),
+                            refer_size_dest(tgt),
+                            NULL_TREE));
+    if( error )
+      {
+      // There is an ON EXCEPTION phrase:
+      IF( erf, ne_op, integer_zero_node )
+        {
+        SHOW_PARSE
+          {
+          SHOW_PARSE_INDENT
+          SHOW_PARSE_TEXT("Laying down GOTO     error->INTO for_argv")
+          SHOW_PARSE_LABEL_OK(" ", error)
+          }
+        gg_append_statement( error->structs.arith_error->into.go_to );
+        }
+      ELSE
+        {
+        }
+        ENDIF
+      }
+    if( not_error )
+      {
+      // There is an NOT ON EXCEPTION phrase:
+      IF( erf, eq_op, integer_zero_node )
+        {
+        SHOW_PARSE
+          {
+          SHOW_PARSE_INDENT
+          SHOW_PARSE_TEXT("Laying down GOTO not_error->INTO for_argv")
+          SHOW_PARSE_LABEL_OK(" ", not_error)
+          }
+        gg_append_statement( not_error->structs.arith_error->into.go_to );
+        }
+      ELSE
+        {
+        }
+        ENDIF
+      }
+    if( error )
+      {
+      SHOW_PARSE
+        {
+        SHOW_PARSE_INDENT
+        SHOW_PARSE_TEXT("Laying down LABEL     error->bottom")
+        SHOW_PARSE_LABEL_OK(" ", error)
+        }
+      gg_append_statement( error->structs.arith_error->bottom.label );
+      }
+    if( not_error )
+      {
+      SHOW_PARSE
+        {
+        SHOW_PARSE_INDENT
+        SHOW_PARSE_TEXT("Laying down LABEL not_error->bottom")
+        SHOW_PARSE_LABEL_OK(" ", not_error)
+        SHOW_PARSE_END
+        }
+      gg_append_statement( not_error->structs.arith_error->bottom.label );
+      }
+    }
   }
 
 // TODO: update documentation.
@@ -4411,7 +4579,6 @@ parser_accept_exception( cbl_label_t *accept_label )
 
   // We are entering either SIZE ERROR or NOT SIZE ERROR code
   RETURN_IF_PARSE_ONLY;
-  set_up_on_exception_label(accept_label);
 
   SHOW_PARSE
     {
@@ -4424,6 +4591,9 @@ parser_accept_exception( cbl_label_t *accept_label )
     SHOW_PARSE_END
     }
 
+  CHECK_LABEL(accept_label);
+  set_up_on_exception_label(accept_label);
+
   // Jump over the [NOT] ON EXCEPTION code that is about to be laid down
   gg_append_statement( accept_label->structs.arith_error->over.go_to );
   // Create the label that allows the following code to be executed at
@@ -4450,6 +4620,8 @@ parser_accept_exception_end( cbl_label_t *accept_label )
     SHOW_PARSE_END
     }
 
+  CHECK_LABEL(accept_label);
+
   // Jump to the end of the arithmetic code:
   gg_append_statement( accept_label->structs.arith_error->bottom.go_to );
   // Lay down the label that allows the ERROR/NOT ERROR instructions
@@ -4459,8 +4631,8 @@ parser_accept_exception_end( cbl_label_t *accept_label )
   }
 
 void
-parser_accept_command_line( cbl_refer_t tgt,
-                            cbl_refer_t source,
+parser_accept_command_line( const cbl_refer_t &tgt,
+                            const cbl_refer_t &source,
                             cbl_label_t *error,
                             cbl_label_t *not_error )
   {
@@ -4500,7 +4672,7 @@ parser_accept_command_line( cbl_refer_t tgt,
           {
           SHOW_PARSE_INDENT
           SHOW_PARSE_TEXT("Laying down GOTO     error->INTO for_command_line")
-          SHOW_PARSE_LABEL(" ", error)
+          SHOW_PARSE_LABEL_OK(" ", error)
           }
         gg_append_statement( error->structs.arith_error->into.go_to );
         }
@@ -4518,7 +4690,7 @@ parser_accept_command_line( cbl_refer_t tgt,
           {
           SHOW_PARSE_INDENT
           SHOW_PARSE_TEXT("Laying down GOTO not_error->INTO for command_line")
-          SHOW_PARSE_LABEL(" ", not_error)
+          SHOW_PARSE_LABEL_OK(" ", not_error)
           }
         gg_append_statement( not_error->structs.arith_error->into.go_to );
         }
@@ -4550,7 +4722,7 @@ parser_accept_command_line( cbl_refer_t tgt,
           {
           SHOW_PARSE_INDENT
           SHOW_PARSE_TEXT("Laying down GOTO     error->INTO for_argv")
-          SHOW_PARSE_LABEL(" ", error)
+          SHOW_PARSE_LABEL_OK(" ", error)
           }
         gg_append_statement( error->structs.arith_error->into.go_to );
         }
@@ -4568,7 +4740,7 @@ parser_accept_command_line( cbl_refer_t tgt,
           {
           SHOW_PARSE_INDENT
           SHOW_PARSE_TEXT("Laying down GOTO not_error->INTO for_argv")
-          SHOW_PARSE_LABEL(" ", not_error)
+          SHOW_PARSE_LABEL_OK(" ", not_error)
           }
         gg_append_statement( not_error->structs.arith_error->into.go_to );
         }
@@ -4584,7 +4756,7 @@ parser_accept_command_line( cbl_refer_t tgt,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT("Laying down LABEL     error->bottom")
-      SHOW_PARSE_LABEL(" ", error)
+      SHOW_PARSE_LABEL_OK(" ", error)
       }
     gg_append_statement( error->structs.arith_error->bottom.label );
     }
@@ -4594,7 +4766,7 @@ parser_accept_command_line( cbl_refer_t tgt,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT("Laying down LABEL not_error->bottom")
-      SHOW_PARSE_LABEL(" ", not_error)
+      SHOW_PARSE_LABEL_OK(" ", not_error)
       SHOW_PARSE_END
       }
     gg_append_statement( not_error->structs.arith_error->bottom.label );
@@ -4602,7 +4774,7 @@ parser_accept_command_line( cbl_refer_t tgt,
   }
 
 void
-parser_accept_command_line_count( cbl_refer_t tgt )
+parser_accept_command_line_count( const cbl_refer_t &tgt )
   {
   Analyze();
   SHOW_PARSE
@@ -4624,10 +4796,10 @@ parser_accept_command_line_count( cbl_refer_t tgt )
   }
 
 void
-parser_accept_envar(struct cbl_refer_t tgt,
-                    struct cbl_refer_t envar,
-                    cbl_label_t *error,
-                    cbl_label_t *not_error )
+parser_accept_envar(const struct cbl_refer_t &tgt,
+                    const struct cbl_refer_t &envar,
+                          cbl_label_t *error,
+                          cbl_label_t *not_error )
   {
   Analyze();
 
@@ -4692,7 +4864,7 @@ parser_accept_envar(struct cbl_refer_t tgt,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT("Laying down LABEL     error->bottom")
-      SHOW_PARSE_LABEL(" ", error)
+      SHOW_PARSE_LABEL_OK(" ", error)
       }
     gg_append_statement( error->structs.arith_error->bottom.label );
     }
@@ -4702,7 +4874,7 @@ parser_accept_envar(struct cbl_refer_t tgt,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT("Laying down LABEL not_error->bottom")
-      SHOW_PARSE_LABEL(" ", not_error)
+      SHOW_PARSE_LABEL_OK(" ", not_error)
       SHOW_PARSE_END
       }
     gg_append_statement( not_error->structs.arith_error->bottom.label );
@@ -4710,7 +4882,8 @@ parser_accept_envar(struct cbl_refer_t tgt,
   }
 
 void
-parser_set_envar( struct cbl_refer_t name, struct cbl_refer_t value )
+parser_set_envar( const struct cbl_refer_t &name,
+                  const struct cbl_refer_t &value )
   {
   Analyze();
   SHOW_PARSE
@@ -5112,7 +5285,6 @@ parser_display_internal(tree file_descriptor,
             build_int_cst_type(SIZE_T, refer.field->data.capacity),
             advance ? integer_one_node : integer_zero_node,
             NULL_TREE );
-    cursor_at_sol = advance;
     }
   else if( refer.field->type == FldLiteralN )
     {
@@ -5150,50 +5322,50 @@ parser_display_internal(tree file_descriptor,
         *p = 'E';
         if( exp < 0 && exp >= -9 )
           {
-            p[1] = '-';
-            p[2] = '0';
-            p[3] = '0' - exp;
-            p[4] = '\0';
+          p[1] = '-';
+          p[2] = '0';
+          p[3] = '0' - exp;
+          p[4] = '\0';
           }
         else if( exp >= 0 && exp <= 9 )
           {
-            p[1] = '+';
-            p[2] = '0';
-            p[3] = '0' + exp;
-            p[4] = '\0';
+          p[1] = '+';
+          p[2] = '0';
+          p[3] = '0' + exp;
+          p[4] = '\0';
           }
         }
       else if (exp == 0)
         {
-          p[-1] = '\0';
+        p[-1] = '\0';
         }
       else if (exp < 0)
         {
-          p[-1] = '\0';
-          char *q = strchr (ach, '.');
-          char dig = q[-1];
-          q[-1] = '\0';
-          char tem[132];
-          snprintf (tem, 132, "%s0.%0*u%c%s", ach, -exp - 1, 0, dig, q + 1);
-          strcpy (ach, tem);
+        p[-1] = '\0';
+        char *q = strchr (ach, '.');
+        char dig = q[-1];
+        q[-1] = '\0';
+        char tem[132];
+        snprintf (tem, 132, "%s0.%0*d%c%s", ach, -exp - 1, 0, dig, q + 1);
+        strcpy (ach, tem);
         }
-      else if (exp > 0)
+      else // if (exp > 0)
         {
-          p[-1] = '\0';
-          char *q = strchr (ach, '.');
-          for (int i = 0; i != exp; ++i)
-            q[i] = q[i + 1];
-          q[exp] = '.';
+        p[-1] = '\0';
+        char *q = strchr (ach, '.');
+        for (int i = 0; i != exp; ++i)
+          q[i] = q[i + 1];
+        q[exp] = '.';
         }
       __gg__remove_trailing_zeroes(ach);
       }
 
     if( symbol_decimal_point() == ',' )
       {
-      char *p = strchr(ach, '.' );
-      if( p )
+      char *pdot = strchr(ach, '.' );
+      if( pdot )
         {
-        *p = symbol_decimal_point();
+        *pdot = symbol_decimal_point();
         }
       }
 
@@ -5250,22 +5422,29 @@ parser_display_field(cbl_field_t *field)
                                 DISPLAY_NO_ADVANCE);
   }
 
-/*
- * The first parameter to parser_display is the "device" upon which to display
- * the data. Besides normal devices, these may include elements that define the
- * Unix command line and environment:
- *  1.  ARG_NUM_e, the ARGUMENT-NUMBER
- *  2.  ARG_VALUE_e, the ARGUMENT-VALUE
- *  3.  ENV_NAME_e, the ENVIRONMENT-NAME
- *  4.  ENV_VALUE_e, the ENVIRONMENT-VALUE
- * that need special care and feeding.
- */
 void
 parser_display( const struct cbl_special_name_t *upon,
-                struct cbl_refer_t refs[],
-                size_t n,
-                bool advance )
+          const std::vector<cbl_refer_t> &refs,
+                bool advance,
+          const cbl_label_t *not_error,
+          const cbl_label_t *error )
   {
+  const size_t n = refs.size();
+  /*
+   * The first parameter to parser_display is the "device" upon which to display
+   * the data. Besides normal devices, these may include elements that define the
+   * Unix command line and environment:
+   *  1.  ARG_NUM_e, the ARGUMENT-NUMBER
+   *  2.  ARG_VALUE_e, the ARGUMENT-VALUE
+   *  3.  ENV_NAME_e, the ENVIRONMENT-NAME
+   *  4.  ENV_VALUE_e, the ENVIRONMENT-VALUE
+   * that need special care and feeding.
+   */
+
+  // At the present time, I am not sure what not_error and error are for
+  gcc_assert(!not_error);
+  gcc_assert(!error);
+
   Analyze();
   SHOW_PARSE
     {
@@ -5274,7 +5453,7 @@ parser_display( const struct cbl_special_name_t *upon,
     for(size_t i=0; i<n; i++)
       {
       SHOW_PARSE_INDENT
-      SHOW_PARSE_REF("", refs[i]);
+      SHOW_PARSE_REF("", refs.at(i));
       }
     if( advance )
       {
@@ -5306,23 +5485,81 @@ parser_display( const struct cbl_special_name_t *upon,
     {
     switch(upon->id)
       {
+      // See table 5 in the IBM Cobol For Linux x86 1.2 document.
+
+      case STDIN_e:
+      case SYSIN_e:
+      case SYSIPT_e:
+        cbl_internal_error("Attempting to send to an input device.");
+        break;
+
+      case C01_e:
+      case C02_e:
+      case C03_e:
+      case C04_e:
+      case C05_e:
+      case C06_e:
+      case C07_e:
+      case C08_e:
+      case C09_e:
+      case C10_e:
+      case C11_e:
+      case C12_e:
+      case CSP_e:
+      case S01_e:
+      case S02_e:
+      case S03_e:
+      case S04_e:
+      case S05_e:
+      case AFP_5A_e:
+      case ARG_VALUE_e:
+        cbl_internal_error("Not valid for DISPLAY statement.");
+        break;
+
       case STDOUT_e:
-      case SYSOUT_e:
-      case SYSLIST_e:
-      case SYSLST_e:
       case CONSOLE_e:
+        // These are inarguably stdout
         gg_assign(file_descriptor, integer_one_node);
         break;
 
       case STDERR_e:
+      case SYSERR_e:
+        // These are inarguably stderr
+        gg_assign(file_descriptor, integer_two_node);
+        break;
+
+      case SYSOUT_e:
+      case SYSLIST_e:
+      case SYSLST_e:
       case SYSPUNCH_e:
       case SYSPCH_e:
-        gg_assign(file_descriptor, integer_two_node);
+        // In the 21st century, when there are no longer valid assumptions to
+        // be made about the existence of line printers, and where things
+        // formerly-ubiquitous card punches no longer exist, there is a need
+        // for the possibility of assigning these "devices" to externally-
+        // determined Unix gadgetry in /dev:
+        gg_assign(file_descriptor,
+                  gg_call_expr( INT,
+                                "__gg__get_file_descriptor",
+                                gg_string_literal(upon->os_filename),
+                                NULL_TREE));
+        needs_closing = true;
         break;
 
+      case ARG_NUM_e:
+        // Set the index number for a subsequent ACCEPT FROM ARG_VALUE_e
+        gg_call(VOID,
+                "__gg__set_arg_num",
+                gg_get_address_of(refs[0].field->var_decl_node),
+                refer_offset(refs[0]),
+                refer_size_source(refs[0]),
+                NULL_TREE);
+         return;
+         break;
+
       case ENV_NAME_e:
-        // This Part I of the slightly absurd method of using DISPLAY...UPON
-        // to fetch, or set, environment variables.
+        // Establish the name of an environment variable for later use with
+        // in either DISPLAY UPON or ACCEPT FROM
         gg_call(VOID,
                 "__gg__set_env_name",
                 gg_get_address_of(refs[0].field->var_decl_node),
@@ -5332,19 +5569,16 @@ parser_display( const struct cbl_special_name_t *upon,
          return;
          break;
 
-      default:
-        if( upon->os_filename[0] )
-          {
-          tree topen = gg_open( gg_string_literal(upon->os_filename),
-                                build_int_cst_type(INT, O_APPEND|O_WRONLY));
-          gg_assign(file_descriptor, topen);
-          needs_closing = true;
-          }
-        else
-          {
-          fprintf(stderr, "We don't know what to do in parser_display\n");
-          gcc_unreachable();
-          }
+      case ENV_VALUE_e:
+        // Set the contents of the environment variable named with ENV_NAME_e
+        gg_call(VOID,
+                "__gg__set_env_value",
+                gg_get_address_of(refs[0].field->var_decl_node),
+                refer_offset(refs[0]),
+                refer_size_source(refs[0]),
+                NULL_TREE);
+         return;
+         break;
       }
     }
   else
@@ -5359,17 +5593,114 @@ parser_display( const struct cbl_special_name_t *upon,
     }
   CHECK_FIELD(refs[n-1].field);
   parser_display_internal(file_descriptor, refs[n-1], advance ? DISPLAY_ADVANCE : DISPLAY_NO_ADVANCE);
-
   if( needs_closing )
     {
-    tree tclose = gg_close(file_descriptor);
-    // We are ignoring the close() return value
-    gg_append_statement(tclose);
+    gg_close(file_descriptor);
     }
 
   cursor_at_sol = advance;
   }
 
+static
+bool  // Returns false for literals; true for named variables
+get_exhibit_name(tree file_descriptor, const cbl_refer_t &arg)
+  {
+  bool retval;
+  if( is_literal(arg.field) )
+    {
+    // If something is a literal, we just display the literal value
+    parser_display_internal(file_descriptor,
+                            arg,
+                            DISPLAY_NO_ADVANCE);
+    retval = false;
+    }
+  else
+    {
+    // It's not a literal, so we show its name, and the names or literal
+    // values) of any qualifier subscripts or refmods
+    gg_write( file_descriptor,
+              gg_string_literal(arg.field->name),
+              build_int_cst_type(SIZE_T, strlen(arg.field->name)) );
+
+    if( arg.subscripts.size() )
+      {
+      // This refer has subscripts:
+      gg_write( file_descriptor,
+                gg_string_literal("("),
+                integer_one_node );
+      for(size_t i=0; i<arg.subscripts.size(); i++)
+        {
+        if( i > 0 )
+          {
+          gg_write( file_descriptor,
+                    gg_string_literal(","),
+                    integer_one_node );
+          }
+        get_exhibit_name(file_descriptor, arg.subscripts[i]);
+        }
+      gg_write( file_descriptor,
+                gg_string_literal(")"),
+                integer_one_node );
+      }
+    if( arg.refmod.from || arg.refmod.len )
+      {
+      gg_write( file_descriptor,
+                gg_string_literal("("),
+                integer_one_node );
+      if( arg.refmod.from )
+        {
+        get_exhibit_name(file_descriptor, *(arg.refmod.from));
+        }
+      gg_write( file_descriptor,
+                gg_string_literal(":"),
+                integer_one_node );
+      if( arg.refmod.len )
+        {
+        get_exhibit_name(file_descriptor, *(arg.refmod.len));
+        }
+      gg_write( file_descriptor,
+                gg_string_literal(")"),
+                integer_one_node );
+      }
+    retval = true;
+    }
+  return retval;
+  }
+
+void
+parser_exhibit( bool /*changed*/, bool /*named*/,
+                const std::vector<cbl_refer_t> &args )
+  {
+  tree file_descriptor = gg_define_int();
+  gg_assign(file_descriptor, integer_one_node);   // stdout is file descriptor 1.
+
+  for(size_t i=0; i<args.size(); i++)
+    {
+    CHECK_FIELD(args[i].field);
+    if(i > 0)
+      {
+      // When there more than one argument, the second through Nth get a space
+      // in front of them.
+      gg_write( file_descriptor,
+                gg_string_literal(" "),
+                integer_one_node);
+      }
+    if( get_exhibit_name(file_descriptor, args[i]) )
+      {
+      gg_write( file_descriptor,
+                gg_string_literal("="),
+                integer_one_node);
+      parser_display_internal(file_descriptor,
+                              args[i],
+                              DISPLAY_NO_ADVANCE);
+      }
+    }
+  gg_write( file_descriptor,
+            gg_string_literal("\n"),
+            integer_one_node);
+  cursor_at_sol = true;
+  }
+
 static tree
 get_literalN_value(cbl_field_t *var)
   {
@@ -5642,12 +5973,12 @@ parser_assign( size_t nC, cbl_num_result_t *C,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT(" Laying down on_error GOTO into")
-      SHOW_PARSE_LABEL(" ", on_error)
+      SHOW_PARSE_LABEL_OK(" ", on_error)
       }
     IF( gg_bitwise_or(error_flag,
                       compute_error->structs.compute_error->compute_error_code),
-        ne_op,
-        integer_zero_node )
+                      ne_op,
+                      integer_zero_node )
       {
       gg_append_statement( on_error->structs.arith_error->into.go_to );
       }
@@ -5673,7 +6004,7 @@ parser_assign( size_t nC, cbl_num_result_t *C,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT(" Laying down not_error GOTO into")
-      SHOW_PARSE_LABEL(" ", not_error)
+      SHOW_PARSE_LABEL_OK(" ", not_error)
       }
     IF( compute_error->structs.compute_error->compute_error_code, eq_op, integer_zero_node )
       {
@@ -5689,7 +6020,7 @@ parser_assign( size_t nC, cbl_num_result_t *C,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT(" Laying down on_error LABEL BOTTOM:")
-      SHOW_PARSE_LABEL(" ", on_error)
+      SHOW_PARSE_LABEL_OK(" ", on_error)
       }
     gg_append_statement( on_error->structs.arith_error->bottom.label );
     }
@@ -5700,7 +6031,7 @@ parser_assign( size_t nC, cbl_num_result_t *C,
       {
       SHOW_PARSE_INDENT
       SHOW_PARSE_TEXT(" Laying down not_error LABEL BOTTOM:")
-      SHOW_PARSE_LABEL(" ", not_error)
+      SHOW_PARSE_LABEL_OK(" ", not_error)
       }
     gg_append_statement( not_error->structs.arith_error->bottom.label );
     }
@@ -5975,10 +6306,18 @@ parser_initialize_table(size_t nelem,
     }
   typedef size_t span_t[2];
   static_assert(sizeof(spans[0]) == sizeof(span_t), "pair size wrong");
-  static tree tspans = gg_define_variable(SIZE_T_P, "..pit_v1", vs_file_static);
-  static tree ttbls  = gg_define_variable(SIZE_T_P, "..pit_v2", vs_file_static);
-  gg_assign(tspans, build_array_of_size_t(2*nspan, (const size_t *)spans));
-  gg_assign(ttbls,  build_array_of_size_t(2*ntbl,  (const size_t *)tbls));
+  static tree tspans = gg_define_variable(SIZE_T_P,
+                                          "..pit_v1",
+                                          vs_file_static);
+  static tree ttbls  = gg_define_variable(SIZE_T_P,
+                                          "..pit_v2",
+vs_file_static);
+  gg_assign(tspans,
+            build_array_of_size_t(2*nspan,
+                                  reinterpret_cast<const size_t *>(spans)));
+  gg_assign(ttbls,
+            build_array_of_size_t(2*ntbl,
+                                  reinterpret_cast<const size_t *>(tbls)));
 
   gg_call(VOID,
           "__gg__mirror_range",
@@ -6137,7 +6476,7 @@ is_valuable( cbl_field_type_t type ) {
   return false;
 }
 
-void parser_sleep(cbl_refer_t seconds)
+void parser_sleep(const cbl_refer_t &seconds)
   {
   if( seconds.field )
     {
@@ -6157,7 +6496,7 @@ void parser_sleep(cbl_refer_t seconds)
     // This is a naked place-holding CONTINUE.  Generate some do-nothing
     // code that will stick some .LOC information into the assembly language,
     // so that GDB-COBOL can display the CONTINUE statement.
-    gg_assign(var_decl_nop, build_int_cst_type(INT, 103));
+    insert_nop(103);
     }
   }
 
@@ -6240,12 +6579,12 @@ program_end_stuff(cbl_refer_t refer, ec_type_t ec)
 
       tree array_type = build_array_type_nelts(UCHAR,
                                     returner->data.capacity);
-      tree retval     =  gg_define_variable(array_type, vs_static);
-      gg_memcpy(gg_get_address_of(retval),
+      tree array     =  gg_define_variable(array_type, vs_static);
+      gg_memcpy(gg_get_address_of(array),
                 member(returner->var_decl_node, "data"),
                 member(returner->var_decl_node, "capacity"));
 
-      tree actual = gg_cast(COBOL_FUNCTION_RETURN_TYPE, gg_get_address_of(retval));
+      tree actual = gg_cast(COBOL_FUNCTION_RETURN_TYPE, gg_get_address_of(array));
 
       restore_local_variables();
       gg_return(actual);
@@ -6501,7 +6840,6 @@ parser_arith_error(cbl_label_t *arithmetic_label)
 
   // We are entering either SIZE ERROR or NOT SIZE ERROR code
   RETURN_IF_PARSE_ONLY;
-  set_up_on_exception_label(arithmetic_label);
 
   SHOW_PARSE
     {
@@ -6514,6 +6852,10 @@ parser_arith_error(cbl_label_t *arithmetic_label)
     SHOW_PARSE_END
     }
 
+  CHECK_LABEL(arithmetic_label);
+
+  set_up_on_exception_label(arithmetic_label);
+
   // Jump over the [NOT] ON EXCEPTION code that is about to be laid down
   gg_append_statement( arithmetic_label->structs.arith_error->over.go_to );
   // Create the label that allows the following code to be executed at
@@ -6540,6 +6882,8 @@ parser_arith_error_end(cbl_label_t *arithmetic_label)
     SHOW_PARSE_END
     }
 
+  CHECK_LABEL(arithmetic_label);
+
   // Jump to the end of the arithmetic code:
   gg_append_statement( arithmetic_label->structs.arith_error->bottom.go_to );
   // Lay down the label that allows the ERROR/NOT ERROR instructions
@@ -6723,8 +7067,6 @@ parser_division(cbl_division_t division,
     SHOW_PARSE_END
     }
 
-  gg_set_current_line_number(CURRENT_LINE_NUMBER);
-
   if( division == data_div_e )
     {
     Analyze();
@@ -6876,7 +7218,6 @@ parser_division(cbl_division_t division,
       // expected formal parameter and tacks it onto the end of the
       // function's arguments chain.
 
-      char ach[2*sizeof(cbl_name_t)];
       sprintf(ach, "_p_%s", args[i].refer.field->name);
 
       size_t nbytes = 0;
@@ -6896,14 +7237,13 @@ parser_division(cbl_division_t division,
       chain_parameter_to_function(current_function->function_decl, par_type, ach);
       }
 
-    bool check_for_parameter_count = false;
-
     if( nusing )
       {
       // During the call, we saved the parameter_count and an array of variable
       // lengths.  We need to look at those values if, and only if, one or more
       // of our USING arguments has an OPTIONAL flag or if one of our targets is
       // marked as VARYING.
+      bool check_for_parameter_count = false;
       for(size_t i=0; i<nusing; i++)
         {
         if( args[i].optional )
@@ -7099,7 +7439,6 @@ parser_division(cbl_division_t division,
           // If so, we have to give var2::data_pointer the same value as
           // var1::data_pointer
           //
-          cbl_field_t *next_var;
           size_t our_index = symbol_index(symbol_elem_of(new_var));
           size_t next_index  = our_index + 1;
           // Look ahead in the symbol table for the next LEVEL01/77
@@ -7110,7 +7449,7 @@ parser_division(cbl_division_t division,
               {
               break;
               }
-            next_var = cbl_field_of(e);
+            cbl_field_t *next_var = cbl_field_of(e);
             if( !next_var )
               {
               break;
@@ -7185,6 +7524,11 @@ parser_division(cbl_division_t division,
         ENDIF
       }
       ENDIF
+    // The first token_location that the parser establishes is caused by the
+    // parser scanning all of the lines in the source code.  This messes up the
+    // logic for backing up one line, which is needed to correctly step through
+    // COBOL code with GDB-COBOL.  So, we clear it here.
+    current_location_minus_one_clear();
     }
   }
 
@@ -7629,7 +7973,9 @@ label_fetch(struct cbl_label_t *label)
   if( !label->structs.goto_trees )
     {
     label->structs.goto_trees
-      = (cbl_label_addresses_t *)xmalloc(sizeof(struct cbl_label_addresses_t) );
+      = static_cast<cbl_label_addresses_t *>
+        (xmalloc(sizeof(struct cbl_label_addresses_t)));
+    gcc_assert(label->structs.goto_trees);
 
     gg_create_goto_pair(&label->structs.goto_trees->go_to,
                         &label->structs.goto_trees->label);
@@ -7647,15 +7993,18 @@ parser_label_label(struct cbl_label_t *label)
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL("", label)
     char ach[32];
-    sprintf(ach, " label is at %p", (void*)label);
+    sprintf(ach, " label is at %p", static_cast<void*>(label));
     SHOW_PARSE_TEXT(ach)
-    sprintf(ach, " label->proc is %p", (void*)label->structs.proc);
+    if( label )
+      {
+      sprintf(ach,
+              " label->proc is %p",
+              static_cast<void*>(label->structs.proc));
+      }
     SHOW_PARSE_TEXT(ach)
     SHOW_PARSE_END
     }
 
-  CHECK_LABEL(label);
-
   TRACE1
     {
     TRACE1_HEADER
@@ -7663,6 +8012,8 @@ parser_label_label(struct cbl_label_t *label)
     TRACE1_END
     }
 
+  CHECK_LABEL(label);
+
   if(strcmp(label->name, "_end_declaratives") == 0 )
     {
     suppress_cobol_entry_point = false;
@@ -7674,21 +8025,25 @@ void
 parser_label_goto(struct cbl_label_t *label)
   {
   label->used = yylineno;
+
   Analyze();
   SHOW_PARSE
     {
     SHOW_PARSE_HEADER
     SHOW_PARSE_LABEL(" ", label)
     char ach[32];
-    sprintf(ach, " label is at %p", (void*)label);
+    sprintf(ach, " label is at %p", static_cast<void*>(label));
     SHOW_PARSE_TEXT(ach)
-    sprintf(ach, " label->proc is %p", (void*)label->structs.proc);
+    if( label )
+      {
+      sprintf(ach,
+              " label->proc is %p",
+              static_cast<void*>(label->structs.proc));
+      }
     SHOW_PARSE_TEXT(ach)
     SHOW_PARSE_END
     }
 
-  CHECK_LABEL(label);
-
   TRACE1
     {
     TRACE1_HEADER
@@ -7696,7 +8051,9 @@ parser_label_goto(struct cbl_label_t *label)
     TRACE1_END
     }
 
-  if(strcmp(label->name, "_end_declaratives") == 0 )
+  CHECK_LABEL(label);
+
+  if( strcmp(label->name, "_end_declaratives") == 0 )
     {
     suppress_cobol_entry_point = true;
     }
@@ -7780,7 +8137,7 @@ parser_setop( struct cbl_field_t *tgt,
 
 void
 parser_classify(    cbl_field_t *tgt,
-                    cbl_refer_t  candidate,
+               const cbl_refer_t  &candidate,
                     enum classify_t type )
   {
   Analyze();
@@ -7877,14 +8234,6 @@ create_iline_address_pairs(struct cbl_perform_tgt_t *tgt)
 
   gg_create_goto_pair(&tgt->addresses.setup.go_to,
                       &tgt->addresses.setup.label);
-
-  // Even in -O0 compilations, the compiler does some elementary optimizations
-  // around JMP instructions.  We have the SETUP code for in-line performats
-  // in an island at the end of the loop code.  With this intervention, NEXTing
-  // through the code shows you the final statement of the loop before the
-  // loop actually starts.
-
-  tgt->addresses.line_number_of_setup_code = gg_get_current_line_number();
   }
 
 void
@@ -7898,7 +8247,7 @@ parser_perform_start( struct cbl_perform_tgt_t *tgt )
       {
       SHOW_PARSE_TEXT(" cbl_perform_tgt_t is at")
       char ach[32];
-      sprintf(ach, " %p", (void*)tgt);
+      sprintf(ach, " %p", static_cast<void*>(tgt));
       SHOW_PARSE_TEXT(ach);
       SHOW_PARSE_LABEL(" ", tgt->from())
       if( tgt->to() )
@@ -7947,7 +8296,7 @@ parser_perform_start( struct cbl_perform_tgt_t *tgt )
   // Give GDB-COBOL something to chew on when NEXTing.  This instruction will
   // get the line number of the PERFORM N TIMES code.
   gg_append_statement(tgt->addresses.top.label);
-  gg_assign(var_decl_nop, build_int_cst_type(INT, 104));
+  insert_nop(104);
   }
 
 void
@@ -7959,7 +8308,7 @@ parser_perform_conditional( struct cbl_perform_tgt_t *tgt )
     SHOW_PARSE_HEADER
     SHOW_PARSE_TEXT(" cbl_perform_tgt_t is at")
     char ach[32];
-    sprintf(ach, " %p", (void*)tgt);
+    sprintf(ach, " %p", static_cast<void*>(tgt));
     SHOW_PARSE_TEXT(ach);
     SHOW_PARSE_END
     }
@@ -8009,7 +8358,7 @@ parser_perform_conditional_end( struct cbl_perform_tgt_t *tgt )
     SHOW_PARSE_HEADER
     SHOW_PARSE_TEXT(" cbl_perform_tgt_t is at")
     char ach[32];
-    sprintf(ach, " %p", (void*)tgt);
+    sprintf(ach, " %p", static_cast<void*>(tgt));
     SHOW_PARSE_TEXT(ach);
     SHOW_PARSE_END
     }
@@ -8099,6 +8448,7 @@ perform_outofline_before_until(struct cbl_perform_tgt_t *tgt,
   sprintf(ach,
           "_procretb." HOST_SIZE_T_PRINT_DEC ":",
           (fmt_size_t)our_pseudo_label);
+  token_location_override(current_location_minus_one());
   gg_insert_into_assembler( ach );
   }
 
@@ -8162,6 +8512,7 @@ perform_outofline_after_until(struct cbl_perform_tgt_t *tgt,
   sprintf(ach,
           "_procretb." HOST_SIZE_T_PRINT_DEC ":",
           (fmt_size_t)our_pseudo_label);
+  token_location_override(current_location_minus_one());
   gg_insert_into_assembler( ach );
   }
 
@@ -8282,6 +8633,7 @@ perform_outofline_testafter_varying(struct cbl_perform_tgt_t *tgt,
   sprintf(ach,
           "_procretb." HOST_SIZE_T_PRINT_DEC ":",
           (fmt_size_t)our_pseudo_label);
+  token_location_override(current_location_minus_one());
   gg_insert_into_assembler( ach );
   }
 
@@ -8425,6 +8777,7 @@ perform_outofline_before_varying(   struct cbl_perform_tgt_t *tgt,
   sprintf(ach,
           "_procretb." HOST_SIZE_T_PRINT_DEC ":",
           (fmt_size_t)our_pseudo_label);
+  token_location_override(current_location_minus_one());
   gg_insert_into_assembler( ach );
   }
 
@@ -8504,8 +8857,6 @@ perform_inline_until(   struct cbl_perform_tgt_t *tgt,
                   GOTO TOP
       EXIT:
   */
-  gg_set_current_line_number(cobol_location().last_line);
-
   gg_append_statement(tgt->addresses.test.label);
 
   // Go to where the conditional is recalculated....
@@ -8620,8 +8971,6 @@ perform_inline_testbefore_varying(  struct cbl_perform_tgt_t *tgt,
     parser_move(varys[i].varying, varys[i].from);
     }
 
-  gg_set_current_line_number(cobol_location().last_line);
-
   // Lay down the testing cycle:
   for(size_t i=0; i<N; i++)
     {
@@ -8933,7 +9282,7 @@ parser_perform_until(   struct cbl_perform_tgt_t *tgt,
     SHOW_PARSE_HEADER
     SHOW_PARSE_TEXT(" cbl_perform_tgt_t is at")
     char ach[32];
-    sprintf(ach, " %p", (void*)tgt);
+    sprintf(ach, " %p", static_cast<void*>(tgt));
     SHOW_PARSE_TEXT(ach);
     SHOW_PARSE_LABEL(" ", tgt->from())
     if( tgt->to() )
@@ -8943,9 +9292,6 @@ parser_perform_until(   struct cbl_perform_tgt_t *tgt,
     SHOW_PARSE_END
     }
 
-  gg_set_current_line_number(cobol_location().last_line);
-  gg_assign(var_decl_nop, build_int_cst_type(INT, 105));
-
   if( tgt->from()->type != LblLoop )
     {
     perform_outofline( tgt, test_before, N, varys);
@@ -9012,10 +9358,6 @@ parser_perform_inline_times(struct cbl_perform_tgt_t *tgt,
   gg_append_statement( tgt->addresses.testA.label );
   gg_append_statement( tgt->addresses.test.label );
 
-  // AT this point, we want to set the line_number to the location of the
-  // END-PERFORM statement.
-  gg_set_current_line_number(cobol_location().last_line);
-
   gg_decrement(counter);
   // Do the test:
   IF( counter, gt_op, gg_cast(LONG, integer_zero_node) )
@@ -9046,8 +9388,6 @@ parser_perform_inline_times(struct cbl_perform_tgt_t *tgt,
     SHOW_PARSE_END
     }
 
-  int stash = gg_get_current_line_number();
-  gg_set_current_line_number(tgt->addresses.line_number_of_setup_code);
   gg_append_statement( tgt->addresses.setup.label );
 
   // Get the count:
@@ -9078,8 +9418,6 @@ parser_perform_inline_times(struct cbl_perform_tgt_t *tgt,
     gg_append_statement( tgt->addresses.exit.go_to );
     ENDIF
 
-  gg_set_current_line_number(stash);
-
   SHOW_PARSE
     {
     SHOW_PARSE_INDENT
@@ -9768,13 +10106,19 @@ void
 parser_file_delete( struct cbl_file_t *file, bool /*sequentially*/ )
   {
   Analyze();
+
+  if( !file )
+    {
+    cbl_internal_error("The file pointer should not be null");
+    abort();  // Because cppcheck doesn't recognize [[noerror]]
+    }
+
   bool sequentially =    file->access == file_access_seq_e
                       || file->org    == file_sequential_e
                       || file->org    == file_line_sequential_e;
   SHOW_PARSE
     {
     SHOW_PARSE_HEADER
-    if(file)
       {
       SHOW_PARSE_TEXT(" ");
       SHOW_PARSE_TEXT(file->name);
@@ -9787,10 +10131,6 @@ parser_file_delete( struct cbl_file_t *file, bool /*sequentially*/ )
         SHOW_PARSE_TEXT(" sequentially")
         }
       }
-    else
-      {
-      SHOW_PARSE_TEXT(" *file is NULL")
-      }
     SHOW_PARSE_END
     }
 
@@ -9947,8 +10287,8 @@ parser_file_start(struct cbl_file_t *file,
     // A key has a number of fields
     for(size_t ifield=0; ifield<file->keys[key_number].nfield; ifield++)
       {
-      size_t field_index = file->keys[key_number].fields[ifield];
-      cbl_field_t *field = cbl_field_of(symbol_at(field_index));
+      size_t nfield = file->keys[key_number].fields[ifield];
+      cbl_field_t *field = cbl_field_of(symbol_at(nfield));
       combined_length += field->data.capacity;
       }
     gg_assign(length, build_int_cst_type(SIZE_T, combined_length));
@@ -9975,7 +10315,7 @@ parser_file_start(struct cbl_file_t *file,
 
 static void
 inspect_tally(bool backward,
-              cbl_refer_t identifier_1,
+        const cbl_refer_t &identifier_1,
               cbl_inspect_opers_t& identifier_2)
   {
   Analyze();
@@ -10175,8 +10515,8 @@ inspect_tally(bool backward,
 
 static void
 inspect_replacing(int backward,
-                  cbl_refer_t identifier_1,
-                  cbl_inspect_opers_t& operations)
+            const cbl_refer_t &identifier_1,
+                  cbl_inspect_opers_t &operations)
   {
   Analyze();
   // This is an INSPECT FORMAT 2
@@ -10516,7 +10856,7 @@ parser_intrinsic_numval_c( cbl_field_t *f,
 
 void
 parser_intrinsic_subst( cbl_field_t *f,
-                        cbl_refer_t& ref1,
+                  const cbl_refer_t& ref1,
                         size_t argc,
                         cbl_substitute_t * argv )
   {
@@ -10541,7 +10881,9 @@ parser_intrinsic_subst( cbl_field_t *f,
 
   sv_is_i_o = true;
   store_location_stuff("SUBSTITUTE");
-  unsigned char *control_bytes = (unsigned char *)xmalloc(argc * sizeof(unsigned char));
+  unsigned char *control_bytes =
+           static_cast<unsigned char *>(xmalloc(argc * sizeof(unsigned char)));
+  gcc_assert(control_bytes);
   std::vector<cbl_refer_t> arg1(argc);
   std::vector<cbl_refer_t> arg2(argc);
 
@@ -10978,7 +11320,9 @@ static void
 create_lsearch_address_pairs(struct cbl_label_t *name)
   {
   // Create the lsearch structure
-  name->structs.lsearch = (cbl_lsearch_t *)xmalloc(sizeof(cbl_lsearch_t));
+  name->structs.lsearch =
+                  static_cast<cbl_lsearch_t *>(xmalloc(sizeof(cbl_lsearch_t)));
+  gcc_assert(name->structs.lsearch);
   cbl_lsearch_t *lsearch = name->structs.lsearch;
 
   gg_create_goto_pair(&lsearch->addresses.at_exit.go_to,
@@ -11228,7 +11572,9 @@ parser_bsearch_start(   cbl_label_t* name,
     }
 
   // We need a cbl_bsearch_t structure:
-  name->structs.bsearch = (cbl_bsearch_t *)xmalloc(sizeof(cbl_bsearch_t));
+  name->structs.bsearch =
+                  static_cast<cbl_bsearch_t *>(xmalloc(sizeof(cbl_bsearch_t)));
+  gcc_assert(name->structs.bsearch);
   cbl_bsearch_t *bsearch = name->structs.bsearch;
 
   // Create the address/label pairs we need
@@ -11261,6 +11607,8 @@ parser_bsearch_start(   cbl_label_t* name,
     current = parent_of(current);
     }
 
+  CHECK_FIELD(current);
+
   // There are a number of things we learn from the field "current"
 
   // We get the index:
@@ -11373,7 +11721,6 @@ is_ascending_key(const cbl_refer_t& key)
   bool retval = true;
 
   cbl_field_t *family_tree = key.field;
-  gcc_assert(family_tree);
   while( family_tree )
     {
     if( family_tree->occurs.nkey )
@@ -11382,7 +11729,10 @@ is_ascending_key(const cbl_refer_t& key)
       }
     family_tree = parent_of(family_tree);
     }
+
+  CHECK_FIELD(family_tree);
   gcc_assert(family_tree->occurs.nkey);
+
   for(size_t i=0; i<family_tree->occurs.nkey; i++)
     {
     for(size_t j=0; j<family_tree->occurs.keys[i].field_list.nfield; j++)
@@ -11542,8 +11892,12 @@ parser_sort(cbl_refer_t tableref,
                                          return n + key.fields.size();
                                        } );
   typedef const cbl_field_t * const_field_t;
-  const_field_t *flattened_fields = (const_field_t *)xmalloc(total_keys * sizeof(cbl_field_t *));
-  size_t *flattened_ascending       = (size_t *)xmalloc(total_keys * sizeof(size_t));
+  const_field_t *flattened_fields =
+     static_cast<const_field_t *>(xmalloc(total_keys * sizeof(cbl_field_t *)));
+  gcc_assert(flattened_fields);
+  size_t *flattened_ascending =
+                   static_cast<size_t *>(xmalloc(total_keys * sizeof(size_t)));
+  gcc_assert(flattened_ascending);
 
   size_t key_index = 0;
   for( size_t i=0; i<keys.size(); i++ )
@@ -11679,8 +12033,12 @@ parser_file_sort(   cbl_file_t *workfile,
                                          return n + key.fields.size();
                                        } );
   typedef const cbl_field_t * const_field_t;
-  auto flattened_fields = (const_field_t *)xmalloc(total_keys * sizeof(cbl_field_t *));
-  size_t *flattened_ascending    = (size_t *)      xmalloc(total_keys * sizeof(size_t));
+  auto flattened_fields
+   = static_cast<const_field_t *>(xmalloc(total_keys * sizeof(cbl_field_t *)));
+  gcc_assert(flattened_fields);
+  size_t *flattened_ascending =
+                   static_cast<size_t *>(xmalloc(total_keys * sizeof(size_t)));
+  gcc_assert(flattened_ascending);
 
   size_t key_index = 0;
   for( size_t i=0; i<keys.size(); i++ )
@@ -11839,7 +12197,9 @@ parser_return_start( cbl_file_t *workfile, cbl_refer_t into )
 
   // We assume that workfile is open.
 
-  workfile->addresses = (cbl_sortreturn_t *)xmalloc(sizeof(cbl_sortreturn_t));
+  workfile->addresses = static_cast<cbl_sortreturn_t *>
+                                          (xmalloc(sizeof(cbl_sortreturn_t)));
+  gcc_assert(workfile->addresses);
   gg_create_goto_pair(&workfile->addresses->at_end.go_to,
                       &workfile->addresses->at_end.label);
   gg_create_goto_pair(&workfile->addresses->not_at_end.go_to,
@@ -12025,9 +12385,13 @@ parser_file_merge(  cbl_file_t *workfile,
                                          return i + key.fields.size();
                                        } );
   typedef const cbl_field_t * const_field_t;
-  const_field_t *flattened_fields = (const_field_t *)xmalloc(total_keys * sizeof(cbl_field_t *));
+  const_field_t *flattened_fields
+                 = static_cast<const_field_t *>
+                   (xmalloc(total_keys * sizeof(cbl_field_t *)));
+  gcc_assert(flattened_fields);
   size_t *flattened_ascending
-            = (size_t *)xmalloc(total_keys * sizeof(size_t));
+            = static_cast<size_t *>(xmalloc(total_keys * sizeof(size_t)));
+  gcc_assert(flattened_ascending);
 
   size_t key_index = 0;
   for( size_t i=0; i<keys.size(); i++ )
@@ -12041,8 +12405,9 @@ parser_file_merge(  cbl_file_t *workfile,
     }
 
   // Create the array of cbl_field_t pointers for the keys
-  tree all_keys = gg_array_of_field_pointers(total_keys,
-                                             const_cast<cbl_field_t**>(flattened_fields));
+  tree all_keys = gg_array_of_field_pointers(
+                                 total_keys,
+                                 const_cast<cbl_field_t**>(flattened_fields));
 
   // Create the array of integers that are the flags for ASCENDING:
   tree ascending = gg_array_of_size_t(total_keys, flattened_ascending);
@@ -12066,6 +12431,9 @@ parser_file_merge(  cbl_file_t *workfile,
   ELSE
     ENDIF
 
+  const cbl_enabled_exceptions_t&
+                                enabled_exceptions( cdf_enabled_exceptions() );
+
   for(size_t i=0; i<ninputs; i++)
     {
     if( process_this_exception(ec_sort_merge_file_open_e) )
@@ -12223,7 +12591,8 @@ parser_string_overflow( cbl_label_t *name )
    */
 
   name->structs.unstring
-    = (cbl_unstring_t *)xmalloc(sizeof(struct cbl_unstring_t) );
+    = static_cast<cbl_unstring_t *>(xmalloc(sizeof(struct cbl_unstring_t)));
+  gcc_assert(name->structs.unstring);
 
   // Set up the address pairs for this clause
   gg_create_goto_pair(&name->structs.unstring->over.go_to,
@@ -12282,8 +12651,8 @@ parser_unstring(cbl_refer_t src,
     }
 
   std::vector<cbl_refer_t> delims(ndelimited);
-  char *alls = (char *)xmalloc(ndelimited+1);
-
+  char *alls = static_cast<char *>(xmalloc(ndelimited+1));
+  gcc_assert(alls);
   for(size_t i=0; i<ndelimited; i++)
     {
     delims[i] = delimiteds[i];
@@ -12374,7 +12743,8 @@ parser_string(const cbl_refer_t& tgt,
     }
 
   // We need an array of nsource+1 integers:
-  size_t *integers = (size_t *)xmalloc((nsource+1)*sizeof(size_t));
+  size_t *integers = static_cast<size_t *>(xmalloc((nsource+1)*sizeof(size_t)));
+  gcc_assert(integers);
 
   // Count up how many treeplets we are going to need:
   size_t cblc_count = 2;  // tgt and pointer
@@ -12465,8 +12835,9 @@ parser_call_exception( cbl_label_t *name )
     }
 
   name->structs.call_exception
-    = (cbl_call_exception_t *)xmalloc(sizeof(struct cbl_call_exception_t) );
-
+    = static_cast<cbl_call_exception_t *>
+                                (xmalloc(sizeof(struct cbl_call_exception_t)));
+  gcc_assert(name->structs.call_exception);
   // Set up the address pairs for this clause
   gg_create_goto_pair(&name->structs.call_exception->over.go_to,
                       &name->structs.call_exception->over.label);
@@ -12526,8 +12897,10 @@ create_and_call(size_t narg,
 
   if(narg)
     {
-    arguments = (tree *)xmalloc(2*narg * sizeof(tree));
-    allocated = (int * )xmalloc(narg * sizeof(int));
+    arguments = static_cast<tree *>(xmalloc(2*narg * sizeof(tree)));
+    gcc_assert(arguments);
+    allocated = static_cast<int *>(xmalloc(narg * sizeof(int)));
+    gcc_assert(allocated);
     }
 
   // Put the arguments onto the "stack" of calling parameters:
@@ -12759,7 +13132,7 @@ create_and_call(size_t narg,
     // Fetch the FUNCTION_DECL for that FUNCTION_TYPE
     tree function_decl = gg_build_fn_decl(funcname, fndecl_type);
     set_call_convention(function_decl, current_call_convention());
-    
+
     // Take the address of the function decl:
     tree address_of_function = gg_get_address_of(function_decl);
 
@@ -12771,7 +13144,7 @@ create_and_call(size_t narg,
     parser_call_target( funcname, assigment );
 
     // Create the call_expr from that address
-    call_expr = build_call_array_loc( location_from_lineno(),
+    call_expr = build_call_array_loc( gg_token_location(),
                                       returned_value_type,
                                       address_of_function,
                                       narg,
@@ -13510,9 +13883,9 @@ parser_program_hierarchy( const cbl_prog_hier_t& hier )
     // are also accessible by us.  Go find them.
     std::vector<const hier_node *>uncles;
     find_uncles(nodes[i], uncles);
-    for( size_t i=0; i<uncles.size(); i++ )
+    for( size_t j=0; j<uncles.size(); j++ )
       {
-      const hier_node *uncle = uncles[i];
+      const hier_node *uncle = uncles[j];
       if( map_of_sets[caller].find(uncle->name) == map_of_sets[caller].end() )
         {
         // We have a COMMON uncle or sibling we haven't seen before.
@@ -13550,7 +13923,6 @@ parser_program_hierarchy( const cbl_prog_hier_t& hier )
       if( callers.find(caller) == callers.end() )
         {
         // We haven't seen this caller before
-        callers.insert(caller);
 
         char ach[3*sizeof(cbl_name_t)];
         tree names_table_type = build_array_type_nelts(CHAR_P, mol->second.size()+1);
@@ -13617,6 +13989,8 @@ parser_program_hierarchy( const cbl_prog_hier_t& hier )
                 (fmt_size_t)caller);
         tree accessible_programs_decl = gg_trans_unit_var_decl(ach);
         gg_assign( accessible_programs_decl, gg_get_address_of(the_constructed_table) );
+
+        callers.insert(caller);
         }
       }
     }
@@ -13734,7 +14108,7 @@ parser_check_fatal_exception()
   // in its innermost loop had an execution time of 19.5 seconds.  By putting in
   // the if() statement, that was reduced to 3.8 seconds.
 
-  if( enabled_exceptions.size() || sv_is_i_o )
+  if( cdf_enabled_exceptions().size() || sv_is_i_o )
     {
     gg_call(VOID,
             "__gg__check_fatal_exception",
@@ -13882,9 +14256,9 @@ conditional_abs(tree source, const cbl_field_t *field)
   }
 
 static bool
-mh_identical(cbl_refer_t &destref,
-             cbl_refer_t &sourceref,
-             TREEPLET    &tsource)
+mh_identical(const cbl_refer_t &destref,
+             const cbl_refer_t &sourceref,
+             const TREEPLET    &tsource)
   {
   // Check to see if the two variables are identical types, thus allowing
   // for a simple byte-for-byte copy of the data areas:
@@ -13906,7 +14280,7 @@ mh_identical(cbl_refer_t &destref,
       )
     {
     // The source and destination are identical in type
-    if( (sourceref.field->attr & intermediate_e) || !symbol_find_odo(sourceref.field) )
+    if( !symbol_find_odo(sourceref.field) )
       {
       Analyze();
       // Source doesn't have a depending_on clause
@@ -14224,7 +14598,7 @@ float_type_of(const cbl_field_t *field)
   }
 
 static tree
-float_type_of(cbl_refer_t *refer)
+float_type_of(const cbl_refer_t *refer)
   {
   return float_type_of(refer->field);
   }
@@ -14456,7 +14830,7 @@ picky_memset(tree &dest_p, unsigned char value, size_t length)
   }
 
 static void
-picky_memcpy(tree &dest_p, tree &source_p, size_t length)
+picky_memcpy(tree &dest_p, const tree &source_p, size_t length)
   {
   if( length )
     {
@@ -14475,10 +14849,10 @@ picky_memcpy(tree &dest_p, tree &source_p, size_t length)
   }
 
 static bool
-mh_numeric_display( cbl_refer_t &destref,
-                    cbl_refer_t &sourceref,
-                    TREEPLET    &tsource,
-                    tree size_error)
+mh_numeric_display( const cbl_refer_t &destref,
+                    const cbl_refer_t &sourceref,
+                    const TREEPLET    &tsource,
+                          tree size_error)
   {
   bool moved = false;
 
@@ -14964,11 +15338,11 @@ mh_numeric_display( cbl_refer_t &destref,
   }
 
 static bool
-mh_little_endian( cbl_refer_t &destref,
-                  cbl_refer_t &sourceref,
-                  TREEPLET    &tsource,
-                  bool check_for_error,
-                  tree size_error)
+mh_little_endian( const cbl_refer_t &destref,
+                  const cbl_refer_t &sourceref,
+                  const TREEPLET    &tsource,
+                        bool check_for_error,
+                        tree size_error)
   {
   bool moved = false;
 
@@ -15036,9 +15410,9 @@ mh_little_endian( cbl_refer_t &destref,
   }
 
 static bool
-mh_source_is_group( cbl_refer_t &destref,
-                    cbl_refer_t &sourceref,
-                    TREEPLET    &tsrc)
+mh_source_is_group( const cbl_refer_t &destref,
+                    const cbl_refer_t &sourceref,
+                    const TREEPLET    &tsrc)
   {
   bool retval = false;
   if( sourceref.field->type == FldGroup && !(destref.field->attr & rjust_e) )
@@ -15103,7 +15477,7 @@ move_helper(tree size_error,        // This is an INT
     {
     // We are creating a copy of the original destination in case we clobber it
     // and have to restore it because of a computational error.
-    bool first_time = true;
+    static bool first_time = true;
     static size_t stash_size = 1024;
     if( first_time )
       {
@@ -15128,7 +15502,7 @@ move_helper(tree size_error,        // This is an INT
     //goto dont_be_clever; this will go through to the default.
     }
 
-  if( !moved )
+  // if( !moved ) // commented out to quiet cppcheck
     {
     moved = mh_source_is_group(destref, sourceref, tsource);
     }
@@ -15197,8 +15571,9 @@ move_helper(tree size_error,        // This is an INT
     if( buffer_size < source_length )
       {
       buffer_size = source_length;
-      buffer = (char *)xrealloc(buffer, buffer_size);
+      buffer = static_cast<char *>(xrealloc(buffer, buffer_size));
       }
+    gcc_assert(buffer);
 
     if( figconst )
       {
@@ -15341,7 +15716,7 @@ move_helper(tree size_error,        // This is an INT
       gg_attribute_bit_clear(destref.field, refmod_e);
       }
 
-    moved = true;
+    // moved = true; // commented out to quiet cppcheck
     }
 
   if( restore_on_error )
@@ -15472,7 +15847,8 @@ binary_initial_from_float128(cbl_field_t *field, int rdigits,
   FIXED_WIDE_INT(128) i
     = FIXED_WIDE_INT(128)::from (real_to_integer (&value, &fail, 128), SIGNED);
 
-  retval = (char *)xmalloc(field->data.capacity);
+  retval = static_cast<char *>(xmalloc(field->data.capacity));
+  gcc_assert(retval);
   switch(field->data.capacity)
     {
       tree type;
@@ -15483,7 +15859,7 @@ binary_initial_from_float128(cbl_field_t *field, int rdigits,
     case 16:
       type = build_nonstandard_integer_type ( field->data.capacity
                                               * BITS_PER_UNIT, 0);
-      native_encode_wide_int (type, i, (unsigned char *)retval,
+      native_encode_wide_int (type, i, PTRCAST(unsigned char, retval),
                               field->data.capacity);
       break;
     default:
@@ -15613,7 +15989,8 @@ initial_from_initial(cbl_field_t *field)
       }
     if( set_return )
       {
-      retval = (char *)xmalloc(field->data.capacity+1);
+      retval = static_cast<char *>(xmalloc(field->data.capacity+1));
+      gcc_assert(retval);
       memset(retval, const_char, field->data.capacity);
       retval[field->data.capacity] = '\0';
       return retval;
@@ -15683,7 +16060,8 @@ initial_from_initial(cbl_field_t *field)
 
     case FldNumericDisplay:
       {
-      retval = (char *)xmalloc(field->data.capacity);
+      retval = static_cast<char *>(xmalloc(field->data.capacity));
+      gcc_assert(retval);
       char *pretval = retval;
       char ach[128];
 
@@ -15763,7 +16141,8 @@ initial_from_initial(cbl_field_t *field)
 
     case FldPacked:
       {
-      retval = (char *)xmalloc(field->data.capacity);
+      retval = static_cast<char *>(xmalloc(field->data.capacity));
+      gcc_assert(retval);
       char *pretval = retval;
       char ach[128];
 
@@ -15830,7 +16209,8 @@ initial_from_initial(cbl_field_t *field)
       {
       if( field->data.initial )
         {
-        retval = (char *)xmalloc(field->data.capacity+1);
+        retval = static_cast<char *>(xmalloc(field->data.capacity+1));
+        gcc_assert(retval);
         if( field->attr & hex_encoded_e)
           {
           memcpy(retval, field->data.initial, field->data.capacity);
@@ -15838,7 +16218,7 @@ initial_from_initial(cbl_field_t *field)
         else
           {
           size_t buffer_size = 0;
-          size_t length = (size_t)field->data.capacity;
+          size_t length = field->data.capacity;
           memset(retval, internal_space, length);
           raw_to_internal(&retval, &buffer_size, field->data.initial, length);
           if( strlen(field->data.initial) < length )
@@ -15854,7 +16234,8 @@ initial_from_initial(cbl_field_t *field)
 
     case FldNumericEdited:
       {
-      retval = (char *)xmalloc(field->data.capacity+1);
+      retval = static_cast<char *>(xmalloc(field->data.capacity+1));
+      gcc_assert(retval);
       if( field->data.initial && field->attr & quoted_e )
         {
         // What the programmer says the value is, the value becomes, no
@@ -15889,7 +16270,6 @@ initial_from_initial(cbl_field_t *field)
         char ach[128];
         memset(ach, 0, sizeof(ach));
         memset(retval, 0, field->data.capacity);
-        size_t ndigits = field->data.capacity;
 
         if( (field->attr & blank_zero_e) && real_iszero (&value) )
           {
@@ -15897,6 +16277,7 @@ initial_from_initial(cbl_field_t *field)
           }
         else
           {
+          size_t ndigits = field->data.capacity;
           digits_from_float128(ach, field, ndigits, rdigits, value);
           /* ???  This resides in libgcobol valconv.cc.  */
           __gg__string_to_numeric_edited( retval,
@@ -15911,23 +16292,24 @@ initial_from_initial(cbl_field_t *field)
 
     case FldFloat:
       {
-      retval = (char *)xmalloc(field->data.capacity);
+      retval = static_cast<char *>(xmalloc(field->data.capacity));
+      gcc_assert(retval);
       switch( field->data.capacity )
         {
         case 4:
           value = real_value_truncate (TYPE_MODE (FLOAT), value);
           native_encode_real (SCALAR_FLOAT_TYPE_MODE (FLOAT), &value,
-                              (unsigned char *)retval, 4, 0);
+                              PTRCAST(unsigned char, retval), 4, 0);
           break;
         case 8:
           value = real_value_truncate (TYPE_MODE (DOUBLE), value);
           native_encode_real (SCALAR_FLOAT_TYPE_MODE (DOUBLE), &value,
-                              (unsigned char *)retval, 8, 0);
+                              PTRCAST(unsigned char, retval), 8, 0);
           break;
         case 16:
           value = real_value_truncate (TYPE_MODE (FLOAT128), value);
           native_encode_real (SCALAR_FLOAT_TYPE_MODE (FLOAT128), &value,
-                              (unsigned char *)retval, 16, 0);
+                              PTRCAST(unsigned char, retval), 16, 0);
           break;
         }
       break;
@@ -16313,12 +16695,13 @@ psa_FldLiteralA(struct cbl_field_t *field )
 
   // First make room
   static size_t buffer_size = 1024;
-  static char *buffer = (char *)xmalloc(buffer_size);
+  static char *buffer = static_cast<char *>(xmalloc(buffer_size));
   if( buffer_size < field->data.capacity+1 )
     {
     buffer_size = field->data.capacity+1;
-    buffer = (char *)xrealloc(buffer, buffer_size);
+    buffer = static_cast<char *>(xrealloc(buffer, buffer_size));
     }
+  gcc_assert(buffer);
 
   cbl_figconst_t figconst = cbl_figconst_of( field->data.initial );
   gcc_assert(figconst == normal_value_e);
@@ -16373,7 +16756,7 @@ psa_FldLiteralA(struct cbl_field_t *field )
                                                   vs_file_static);
     }
   else
-#endif    
+#endif
     {
     // We have not seen that string before
     static int nvar = 0;
@@ -16387,9 +16770,9 @@ psa_FldLiteralA(struct cbl_field_t *field )
                                                 vs_file_static);
     actually_create_the_static_field(
                 field,
-                build_string_literal(field->data.capacity+1,
+                build_string_literal(field->data.capacity,
                                      buffer),
-                field->data.capacity+1,
+                field->data.capacity,
                 field->data.initial,
                 NULL_TREE,
                 field->var_decl_node);
@@ -16417,6 +16800,8 @@ parser_local_add(struct cbl_field_t *new_var )
     SHOW_PARSE_END
     }
 
+  CHECK_FIELD(new_var);
+
   IF( member(new_var->var_decl_node, "data"),
       ne_op,
       gg_cast(UCHAR_P, null_pointer_node) )
@@ -16473,8 +16858,8 @@ parser_symbol_add(struct cbl_field_t *new_var )
       }
     while(0);
 
-    fprintf(stderr, " %2.2d %s<%s> off:" HOST_SIZE_T_PRINT_DEC " "
-                    "msiz:%d cap:%d dig:%d rdig:%d attr:0x" HOST_SIZE_T_PRINT_HEX_PURE " loc:%p",
+    fprintf(stderr, " %2.2u %s<%s> off:" HOST_SIZE_T_PRINT_UNSIGNED " "
+                    "msiz:%u cap:%u dig:%u rdig:%d attr:0x" HOST_SIZE_T_PRINT_HEX_PURE " loc:%p",
             new_var->level,
             new_var->name,
             cbl_field_type_str(new_var->type),
@@ -16484,7 +16869,7 @@ parser_symbol_add(struct cbl_field_t *new_var )
             new_var->data.digits,
             new_var->data.rdigits,
             (fmt_size_t)new_var->attr,
-            (void*)new_var);
+            static_cast<void*>(new_var));
 
     if( is_table(new_var) )
       {
@@ -16524,7 +16909,7 @@ parser_symbol_add(struct cbl_field_t *new_var )
       {
       fprintf(stderr,
               " redefines:(%p)%s",
-              (void*)symbol_redefines(new_var),
+              static_cast<void*>(symbol_redefines(new_var)),
               symbol_redefines(new_var)->name);
       }
 
@@ -16624,10 +17009,12 @@ parser_symbol_add(struct cbl_field_t *new_var )
       TRACE1_TEXT_ABC(" (", cbl_field_type_str(new_var->type) ,")")
       if( new_var->type == FldLiteralN)
         {
+        const void *p1 = (new_var->data.initial);
+        const long *pldata = static_cast<const long *>(p1);
+        long ldata = *pldata;
         gg_fprintf( trace_handle,
                     1, " [%ld]",
-                    build_int_cst_type(LONG,
-                                        *(const long *)new_var->data.initial));
+                    build_int_cst_type(LONG, ldata));
         }
       TRACE1_END
       }
diff --git a/gcc/cobol/genapi.h b/gcc/cobol/genapi.h
index 36d947b..b41b906 100644
--- a/gcc/cobol/genapi.h
+++ b/gcc/cobol/genapi.h
@@ -52,17 +52,26 @@ void parser_division( cbl_division_t division,
 void parser_enter_program(const char *funcname, bool is_function, int *retval);
 void parser_leave_program();
 
-void parser_accept( cbl_refer_t refer, special_name_t special_e);
+void parser_accept( const cbl_refer_t &refer,
+                    special_name_t special_e,
+                    cbl_label_t *error,
+                    cbl_label_t *not_error );
 void parser_accept_exception( cbl_label_t *name );
 void parser_accept_exception_end( cbl_label_t *name );
 
-void parser_accept_envar( cbl_refer_t refer, cbl_refer_t envar,
-                          cbl_label_t *error, cbl_label_t *not_error );
-void parser_set_envar( cbl_refer_t envar, cbl_refer_t refer );
+void parser_accept_under_discussion(struct cbl_refer_t tgt, special_name_t special,
+                                    cbl_label_t *error, cbl_label_t *not_error );
+void parser_accept_envar( const cbl_refer_t &refer,
+                          const cbl_refer_t &envar,
+                          cbl_label_t *error,
+                          cbl_label_t *not_error );
+void parser_set_envar( const cbl_refer_t &envar, const cbl_refer_t &refer );
 
-void parser_accept_command_line( cbl_refer_t tgt, cbl_refer_t src,
-                          cbl_label_t *error, cbl_label_t *not_error );
-void parser_accept_command_line_count( cbl_refer_t tgt );
+void parser_accept_command_line(const cbl_refer_t &tgt,
+                                const cbl_refer_t &src,
+                                cbl_label_t *error,
+                                cbl_label_t *not_error );
+void parser_accept_command_line_count( const cbl_refer_t &tgt );
 
 void parser_accept_date_yymmdd( cbl_field_t *tgt );
 void parser_accept_date_yyyymmdd( cbl_field_t *tgt );
@@ -86,8 +95,7 @@ parser_add( size_t nC, cbl_num_result_t *C,
             size_t nA, cbl_refer_t *A,
             cbl_arith_format_t format,
             cbl_label_t *error,
-            cbl_label_t *not_error,
-            void *compute_error = NULL);  // This has to be cast to a tree pointer to int
+            cbl_label_t *not_error,            void *compute_error = NULL);  // This has to be cast to a tree pointer to int
 
 void parser_arith_error( cbl_label_t *name );
 void parser_arith_error_end( cbl_label_t *name );
@@ -174,7 +182,8 @@ parser_bitwise_op(struct cbl_field_t *tgt,
 
 void
 parser_classify( struct cbl_field_t *tgt,
-                 struct cbl_refer_t  srca,  enum classify_t type );
+           const struct cbl_refer_t &srca,
+                 enum                classify_t type );
 
 void
 parser_op( struct cbl_refer_t cref,
@@ -253,7 +262,7 @@ parser_program_hierarchy( const struct cbl_prog_hier_t& hier );
 void
 parser_end_program(const char *name=NULL);
 
-void parser_sleep(cbl_refer_t seconds);
+void parser_sleep(const cbl_refer_t &seconds);
 
 void parser_exit( const cbl_refer_t& refer, ec_type_t = ec_none_e );
 void parser_exit_section(void);
@@ -262,9 +271,14 @@ void parser_exit_perform( struct cbl_perform_tgt_t *tgt, bool cycle );
 void parser_exit_program(void); // exits back to COBOL only, else continue
 
 void
+parser_exhibit( bool changed, bool named,
+                const std::vector<cbl_refer_t> &args );
+void
 parser_display( const struct cbl_special_name_t *upon,
-                struct cbl_refer_t args[], size_t n,
-                bool advance = DISPLAY_ADVANCE );
+                const std::vector<cbl_refer_t> &args,
+                bool advance = DISPLAY_ADVANCE,
+          const cbl_label_t *not_error = nullptr,
+          const cbl_label_t *compute_error = nullptr );
 
 void parser_display_field(cbl_field_t *fld);
 
@@ -300,7 +314,7 @@ void
 parser_initialize(const cbl_refer_t& refer, bool like_parser_symbol_add=false);
 
 void
-parser_initialize_programs(size_t nprog, struct cbl_refer_t *progs);
+parser_initialize_programs(size_t nprog, const struct cbl_refer_t *progs);
 
 void
 parser_label_label( struct cbl_label_t *label );
@@ -447,7 +461,7 @@ parser_intrinsic_numval_c( cbl_field_t *f,
 
 void
 parser_intrinsic_subst( cbl_field_t *f,
-                        cbl_refer_t& ref1,
+                  const cbl_refer_t& ref1,
                         size_t argc,
                         cbl_substitute_t * argv );
 
diff --git a/gcc/cobol/gengen.cc b/gcc/cobol/gengen.cc
index e42747b..3ad3344 100644
--- a/gcc/cobol/gengen.cc
+++ b/gcc/cobol/gengen.cc
@@ -107,8 +107,6 @@
 // Don't like it?  Cry me a river.
 static const int ARG_LIMIT = 512;
 
-static int sv_current_line_number;
-
 // These are globally useful constants
 tree char_nodes[256];
 
@@ -140,7 +138,7 @@ struct cbl_translation_unit_t gg_trans_unit;
 // the compiler when a source code module makes that mistake.
 static std::unordered_set<std::string> names_we_have_seen;
 
-// This vector is used to process the function_decls at the point we leave 
+// This vector is used to process the function_decls at the point we leave
 // the file.
 static std::vector<tree> finalized_function_decls;
 
@@ -452,7 +450,7 @@ gg_assign(tree dest, const tree source)
 
   if( okay )
     {
-    stmt = build2_loc(location_from_lineno(),
+    stmt = build2_loc(gg_token_location(),
                       MODIFY_EXPR,
                       TREE_TYPE(dest),
                       dest,
@@ -616,7 +614,7 @@ gg_add_field_to_structure(const tree type_of_field, const char *name_of_field, t
   tree id_of_field = get_identifier (name_of_field);
 
   // Create the new field:
-  tree new_field_decl = build_decl(   location_from_lineno(),
+  tree new_field_decl = build_decl(   gg_token_location(),
                                       FIELD_DECL,
                                       id_of_field,
                                       type_of_field);
@@ -893,7 +891,7 @@ gg_create_assembler_name(const char *cobol_name)
 static char *
 gg_unique_in_function(const char *var_name, gg_variable_scope_t vs_scope)
   {
-  char *retval = (char *)xmalloc(strlen(var_name)+32);
+  char *retval = static_cast<char *>(xmalloc(strlen(var_name)+32));
   if( (vs_scope == vs_stack || vs_scope == vs_static) )
     {
     sprintf(retval, "%s." HOST_SIZE_T_PRINT_DEC, var_name,
@@ -1028,10 +1026,7 @@ gg_declare_variable(tree type_decl,
       break;
     }
   DECL_INITIAL(var_decl) = initial_value;
-  if( unique_name )
-    {
-    free(unique_name);
-    }
+  free(unique_name);
   return var_decl;
   }
 
@@ -1046,7 +1041,7 @@ gg_define_from_declaration(tree var_decl)
     {
     // Having made sure the chain of variable declarations is nicely started,
     // it's time to actually define the storage with a decl_expression:
-    tree stmt = build1_loc (location_from_lineno(),
+    tree stmt = build1_loc (gg_token_location(),
                             DECL_EXPR,
                             TREE_TYPE(var_decl),
                             var_decl);
@@ -1777,7 +1772,7 @@ gg_build_relational_expression(tree operand_a,
       compare = LE_EXPR;
       break;
     }
-  tree relational_expression = build2_loc(location_from_lineno(),
+  tree relational_expression = build2_loc(gg_token_location(),
                                           compare,
                                           boolean_type_node,
                                           operand_a,
@@ -1894,7 +1889,7 @@ gg_create_goto_pair(tree *goto_expr,
 void
 gg_goto_label_decl(tree label_decl)
   {
-  tree goto_expr  = build1_loc( location_from_lineno(),
+  tree goto_expr  = build1_loc( gg_token_location(),
                                 GOTO_EXPR,
                                 void_type_node,
                                 label_decl);
@@ -1941,7 +1936,7 @@ gg_create_goto_pair(tree *goto_expr, tree *label_expr, const char *name)
 void
 gg_goto(tree var_decl_pointer)
   {
-  tree go_to = build1_loc(location_from_lineno(),
+  tree go_to = build1_loc(gg_token_location(),
                           GOTO_EXPR,
                           void_type_node,
                           var_decl_pointer);
@@ -2189,7 +2184,7 @@ gg_printf(const char *format_string, ...)
     function = gg_get_function_address(INT, "__gg__fprintf_stderr");
     }
 
-  tree stmt = build_call_array_loc (location_from_lineno(),
+  tree stmt = build_call_array_loc (gg_token_location(),
                                     INT,
                                     function,
                                     nargs,
@@ -2236,7 +2231,7 @@ gg_fprintf(tree fd, int nargs, const char *format_string, ...)
     function = gg_get_function_address(INT, "sprintf");
     }
 
-  tree stmt = build_call_array_loc (location_from_lineno(),
+  tree stmt = build_call_array_loc (gg_token_location(),
                                     INT,
                                     function,
                                     argc,
@@ -2283,7 +2278,7 @@ void
 gg_memset(tree dest, const tree value, tree size)
   {
   tree the_call =
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_MEMSET),
                           3,
                           dest,
@@ -2297,7 +2292,7 @@ gg_memchr(tree buf, tree ch, tree length)
   {
   tree the_call = fold_convert(
       pvoid_type_node,
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_MEMCHR),
                           3,
                           buf,
@@ -2312,7 +2307,7 @@ void
 gg_memcpy(tree dest, const tree src, tree size)
   {
   tree the_call = build_call_expr_loc(
-        location_from_lineno(),
+        gg_token_location(),
         builtin_decl_explicit (BUILT_IN_MEMCPY),
         3,
         dest,
@@ -2327,7 +2322,7 @@ void
 gg_memmove(tree dest, const tree src, tree size)
   {
   tree the_call = build_call_expr_loc(
-        location_from_lineno(),
+        gg_token_location(),
         builtin_decl_explicit (BUILT_IN_MEMMOVE),
         3,
         dest,
@@ -2360,7 +2355,7 @@ void
 gg_strcpy(tree dest, tree src)
   {
   tree the_call =
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_STRCPY),
                           2,
                           dest,
@@ -2373,7 +2368,7 @@ gg_strcmp(tree A, tree B)
   {
   tree the_call = fold_convert(
       integer_type_node,
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_STRCMP),
                           2,
                           A,
@@ -2405,7 +2400,7 @@ gg_strncmp(tree char_star_A, tree char_star_B, tree size_t_N)
   {
   tree the_call = fold_convert(
       integer_type_node,
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_STRNCMP),
                           3,
                           char_star_A,
@@ -2436,7 +2431,7 @@ gg_return(tree operand)
     {
     // When there is no operand, or if the function result is void, then
     // we just generate a return_expr.
-    stmt = build1_loc(location_from_lineno(), RETURN_EXPR, void_type_node, NULL_TREE);
+    stmt = build1_loc(gg_token_location(), RETURN_EXPR, void_type_node, NULL_TREE);
     }
   else
     {
@@ -2446,7 +2441,7 @@ gg_return(tree operand)
                             function_type,
                             DECL_RESULT(current_function->function_decl),
                             gg_cast(function_type, operand));
-    stmt = build1_loc(location_from_lineno(), RETURN_EXPR, void_type_node, modify);
+    stmt = build1_loc(gg_token_location(), RETURN_EXPR, void_type_node, modify);
     }
   gg_append_statement(stmt);
   }
@@ -2454,7 +2449,7 @@ gg_return(tree operand)
 void
 chain_parameter_to_function(tree function_decl, const tree param_type,  const char *name)
   {
-  tree parm = build_decl (location_from_lineno(),
+  tree parm = build_decl (gg_token_location(),
                           PARM_DECL,
                           get_identifier (name),
                           param_type);
@@ -2521,12 +2516,12 @@ gg_peek_fn_decl(const char *funcname, tree fndecl_type)
     }
   return retval;
   }
-  
+
 tree
 gg_build_fn_decl(const char *funcname, tree fndecl_type)
   {
   tree function_decl;
-  
+
   std::string key = function_decl_key(funcname, fndecl_type);
   std::unordered_map<std::string, tree>::const_iterator it =
           map_of_function_decls.find(key);
@@ -2617,13 +2612,13 @@ gg_define_function( tree return_type,
     }
   va_end(params);
 
+  char ach[32];
   std::unordered_set<std::string>::const_iterator it =
           names_we_have_seen.find(funcname);
   if( it != names_we_have_seen.end() )
     {
     static int bum_counter = 1;
     // We have seen this name before.  Replace it with something unique:
-    char ach[32];
     sprintf(ach, "..no_dupes.%d", bum_counter++);
     funcname = ach;
     }
@@ -2689,7 +2684,7 @@ gg_define_function( tree return_type,
     }
 
   // Establish the RESULT_DECL for the function:
-  tree resdecl = build_decl (location_from_lineno(), RESULT_DECL, NULL_TREE, return_type);
+  tree resdecl = build_decl (gg_token_location(), RESULT_DECL, NULL_TREE, return_type);
   DECL_CONTEXT (resdecl) = function_decl;
   DECL_RESULT (function_decl) = resdecl;
 
@@ -2821,7 +2816,7 @@ gg_get_function_decl(tree return_type, const char *funcname, ...)
     }
 
   // Establish the RESULT_DECL for the function:
-  tree resdecl = build_decl (location_from_lineno(), RESULT_DECL, NULL_TREE, return_type);
+  tree resdecl = build_decl (gg_token_location(), RESULT_DECL, NULL_TREE, return_type);
   DECL_CONTEXT (resdecl) = function_decl;
   DECL_RESULT (function_decl) = resdecl;
 
@@ -3058,7 +3053,7 @@ gg_call_expr(tree return_type, const char *function_name, ...)
 
     tree arg = va_arg(ap, tree);
 
-    if( !arg )
+    if( arg == NULL_TREE )
       {
       break;
       }
@@ -3079,7 +3074,7 @@ gg_call_expr(tree return_type, const char *function_name, ...)
   tree the_func_addr = build1(ADDR_EXPR,
                               build_pointer_type (TREE_TYPE(function_decl)),
                               function_decl);
-  tree the_call = build_call_array_loc(location_from_lineno(),
+  tree the_call = build_call_array_loc(gg_token_location(),
                                        return_type,
                                        the_func_addr,
                                        nargs,
@@ -3114,7 +3109,7 @@ gg_call(tree return_type, const char *function_name,  ...)
 
     tree arg = va_arg(ap, tree);
 
-    if( !arg )
+    if( arg == NULL_TREE )
       {
       break;
       }
@@ -3135,7 +3130,7 @@ gg_call(tree return_type, const char *function_name,  ...)
   tree the_func_addr = build1(ADDR_EXPR,
                               build_pointer_type (TREE_TYPE(function_decl)),
                               function_decl);
-  tree the_call = build_call_array_loc(location_from_lineno(),
+  tree the_call = build_call_array_loc(gg_token_location(),
                                        return_type,
                                        the_func_addr,
                                        nargs,
@@ -3160,7 +3155,7 @@ gg_call_expr_list(tree return_type, tree function_pointer, int param_count, tree
   // Avoid that with something like
   //      gg_assign( dest, gg_call_expr_list(...) );
 
-  tree the_call = build_call_array_loc(location_from_lineno(),
+  tree the_call = build_call_array_loc(gg_token_location(),
                                        return_type,
                                        function_pointer,
                                        param_count,
@@ -3195,7 +3190,7 @@ void
 gg_exit(tree exit_code)
   {
   tree the_call =
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_EXIT),
                           1,
                           exit_code);
@@ -3206,7 +3201,7 @@ void
 gg_abort()
   {
   tree the_call =
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_ABORT),
                           0);
   gg_append_statement(the_call);
@@ -3217,7 +3212,7 @@ gg_strlen(tree psz)
   {
   tree the_call = fold_convert(
       size_type_node,
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_STRLEN),
                           1,
                           psz));
@@ -3229,7 +3224,7 @@ gg_strdup(tree psz)
   {
   tree the_call = fold_convert(
       build_pointer_type(char_type_node),
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_STRDUP),
                           1,
                           psz));
@@ -3243,7 +3238,7 @@ gg_malloc(tree size)
   {
   tree the_call = fold_convert(
       pvoid_type_node,
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_MALLOC),
                           1,
                           size));
@@ -3255,7 +3250,7 @@ gg_realloc(tree base, tree size)
   {
   tree the_call = fold_convert(
       pvoid_type_node,
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_REALLOC),
                           2,
                           base,
@@ -3279,7 +3274,7 @@ void
 gg_free(tree pointer)
   {
   tree the_call =
-      build_call_expr_loc(location_from_lineno(),
+      build_call_expr_loc(gg_token_location(),
                           builtin_decl_explicit (BUILT_IN_FREE),
                           1,
                           pointer);
@@ -3380,18 +3375,6 @@ gg_string_literal(const char *string)
   return build_string_literal(strlen(string)+1, string);
   }
 
-void
-gg_set_current_line_number(int line_number)
-  {
-  sv_current_line_number = line_number;
-  }
-
-int
-gg_get_current_line_number()
-  {
-  return sv_current_line_number;
-  }
-
 tree
 gg_trans_unit_var_decl(const char *var_name)
   {
@@ -3413,7 +3396,7 @@ gg_insert_into_assembler(const char ach[])
   if( !optimize )
     {
     // Create the required generic tag
-    tree asm_expr = build5_loc( location_from_lineno(),
+    tree asm_expr = build5_loc( gg_token_location(),
                             ASM_EXPR,
                             VOID,
                             build_string(strlen(ach), ach),
@@ -3450,5 +3433,28 @@ gg_insert_into_assemblerf(const char *format, ...)
     gg_insert_into_assembler(ach);
     }
   }
+#pragma GCC diagnostic pop
+
+static location_t sv_token_location_override = 0;
 
-#pragma GCC diagnostic pop
\ No newline at end of file
+void
+token_location_override(location_t loc)
+  {
+  sv_token_location_override = loc;
+  }
+
+location_t
+gg_token_location()
+  {
+  location_t retval;
+  if( sv_token_location_override )
+    {
+    retval = sv_token_location_override;
+    sv_token_location_override = 0;
+    }
+  else
+    {
+    retval = current_token_location();
+    }
+  return retval;
+  }
diff --git a/gcc/cobol/gengen.h b/gcc/cobol/gengen.h
index 06b28e06..96e69dd 100644
--- a/gcc/cobol/gengen.h
+++ b/gcc/cobol/gengen.h
@@ -525,11 +525,11 @@ extern tree gg_indirect(tree pointer, tree byte_offset = NULL_TREE);
 extern tree gg_string_literal(const char *string);
 
 #define CURRENT_LINE_NUMBER (cobol_location().first_line)
-extern location_t location_from_lineno();
-
-// When set to true, use UNKNOWN_LOCATION instead of CURRENT_LINE_NUMBER
-extern void gg_set_current_line_number(int line_number);
-extern int  gg_get_current_line_number();
+extern location_t gg_token_location();
+extern location_t current_token_location();
+extern location_t current_location_minus_one();
+extern void current_location_minus_one_clear();
+extern void token_location_override(location_t loc);
 
 extern tree gg_trans_unit_var_decl(const char *var_name);
 
diff --git a/gcc/cobol/genmath.cc b/gcc/cobol/genmath.cc
index 0a1c12d..e7eb971 100644
--- a/gcc/cobol/genmath.cc
+++ b/gcc/cobol/genmath.cc
@@ -52,7 +52,8 @@ set_up_on_exception_label(cbl_label_t *arithmetic_label)
     if( !arithmetic_label->structs.arith_error )
       {
       arithmetic_label->structs.arith_error
-        = (cbl_arith_error_t *)xmalloc(sizeof(struct cbl_arith_error_t) );
+        = static_cast<cbl_arith_error_t *>
+                                  (xmalloc(sizeof(struct cbl_arith_error_t)));
       // Set up the address pairs for this clause
       gg_create_goto_pair(&arithmetic_label->structs.arith_error->over.go_to,
                           &arithmetic_label->structs.arith_error->over.label);
@@ -72,8 +73,8 @@ set_up_compute_error_label(cbl_label_t *compute_label)
     if( !compute_label->structs.compute_error )
       {
       compute_label->structs.compute_error
-        = (cbl_compute_error_t *)
-          xmalloc(sizeof(struct cbl_compute_error_t) );
+        = static_cast<cbl_compute_error_t *>
+          (xmalloc(sizeof(struct cbl_compute_error_t)));
       compute_label->structs.compute_error->compute_error_code
         = gg_define_int(0);
       }
@@ -112,7 +113,6 @@ arithmetic_operation(size_t nC, cbl_num_result_t *C,
     {
     TRACE1_HEADER
     TRACE1_TEXT_ABC("calling ", operation, "")
-    TRACE1_END
     for(size_t ii=0; ii<nA; ii++)
       {
       TRACE1_INDENT
@@ -129,7 +129,6 @@ arithmetic_operation(size_t nC, cbl_num_result_t *C,
                   build_int_cst_type(SIZE_T, ii));
       TRACE1_REFER("", B[ii], "");
       }
-    TRACE1_END
     }
 
   // We need to split up cbl_num_result_t into two arrays, one for the refer_t
@@ -223,7 +222,6 @@ arithmetic_operation(size_t nC, cbl_num_result_t *C,
     {
     for(size_t ii=0; ii<nC; ii++)
       {
-      break;  // Breaks on ADD 1 SUB2 GIVING SUB4 both PIC S9(3) COMP
       TRACE1_INDENT
       gg_fprintf( trace_handle,
                   1, "result: C[%ld]: ",
@@ -612,7 +610,7 @@ static bool
 fast_divide(size_t nC, cbl_num_result_t *C,
             size_t nA, cbl_refer_t *A,
             size_t nB, cbl_refer_t *B,
-            cbl_refer_t             remainder)
+      const cbl_refer_t             &remainder)
   {
   bool retval = false;
   if( all_results_binary(nC, C) )
@@ -663,8 +661,10 @@ fast_divide(size_t nC, cbl_num_result_t *C,
       // We now either divide into C[n] or assign dividend/divisor to C[n]:
       for(size_t i=0; i<nC; i++ )
         {
-        tree dest_type = tree_type_from_size(C[i].refer.field->data.capacity, 0);
-        tree dest_addr = gg_add(member(C[i].refer.field->var_decl_node, "data"),
+        tree dest_type =
+                       tree_type_from_size(C[i].refer.field->data.capacity, 0);
+        tree dest_addr = gg_add(member( C[i].refer.field->var_decl_node,
+                                        "data"),
                                 refer_offset(C[i].refer));
         tree ptr = gg_cast(build_pointer_type(dest_type), dest_addr);
         if( nB )
@@ -680,16 +680,15 @@ fast_divide(size_t nC, cbl_num_result_t *C,
           }
 
         // This is where we handle any remainder, keeping in mind that for
-        // nB != 0, the actual dividend is in the value we have named "divisor".
-        //
-        // And, yes, I hate comments like that, too.
+        // nB != 0, the actual dividend is in the value we have named
+        // "divisor".
 
         // We calculate the remainder by calculating
         //    dividend minus quotient * divisor
         if( remainder.field )
           {
-          tree dest_addr = gg_add(member(remainder.field->var_decl_node, "data"),
-                                  refer_offset(remainder));
+          dest_addr = gg_add( member(remainder.field->var_decl_node, "data"),
+                              refer_offset(remainder));
           dest_type = tree_type_from_size(remainder.field->data.capacity, 0);
           ptr = gg_cast(build_pointer_type(dest_type), dest_addr);
 
diff --git a/gcc/cobol/genutil.cc b/gcc/cobol/genutil.cc
index 1d921a3..a5f69a0 100644
--- a/gcc/cobol/genutil.cc
+++ b/gcc/cobol/genutil.cc
@@ -27,6 +27,9 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+
+// cppcheck-suppress-file duplicateBreak
+
 #include "cobol-system.h"
 #include "coretypes.h"
 #include "tree.h"
@@ -305,8 +308,11 @@ static
 void
 get_and_check_refstart_and_reflen(  tree         refstart,// LONG returned value
                                     tree         reflen,  // LONG returned value
-                                    cbl_refer_t &refer)
+                              const cbl_refer_t &refer)
   {
+  const cbl_enabled_exceptions_t&
+                                enabled_exceptions( cdf_enabled_exceptions() );
+
   if( !enabled_exceptions.match(ec_bound_ref_mod_e) )
     {
     // This is normal operation -- no exception checking.  Thus, we won't
@@ -458,6 +464,8 @@ get_depending_on_value_from_odo(tree retval, cbl_field_t *odo)
       declarative with a RESUME NEXT STATEMENT, or before the default_condition
       processing can do a controlled exit.
       */
+  const cbl_enabled_exceptions_t&
+                                enabled_exceptions( cdf_enabled_exceptions() );
   cbl_field_t *depending_on;
   depending_on = cbl_field_of(symbol_at(odo->occurs.depending_on));
 
@@ -471,8 +479,8 @@ get_depending_on_value_from_odo(tree retval, cbl_field_t *odo)
     return;
     }
 
-  // Bounds checking is enabled, so we test the DEPENDING ON value to be between
-  // the lower and upper OCCURS limits:
+  // Bounds checking is enabled, so we test the DEPENDING ON value to be
+  // between the lower and upper OCCURS limits:
   get_integer_value(retval,
                     depending_on,
                     NULL,
@@ -482,23 +490,28 @@ get_depending_on_value_from_odo(tree retval, cbl_field_t *odo)
     {
     // This needs to evaluate to an integer
     set_exception_code(ec_bound_odo_e);
-    gg_assign(retval, build_int_cst_type(TREE_TYPE(retval), odo->occurs.bounds.lower));
+    gg_assign(retval, build_int_cst_type( TREE_TYPE(retval),
+                                          odo->occurs.bounds.lower));
     gg_assign(var_decl_rdigits, integer_zero_node);
     }
   ELSE
     ENDIF
 
-  IF( retval, gt_op, build_int_cst_type(TREE_TYPE(retval), odo->occurs.bounds.upper) )
+  IF( retval, gt_op, build_int_cst_type(TREE_TYPE(retval),
+                                        odo->occurs.bounds.upper) )
     {
     set_exception_code(ec_bound_odo_e);
-    gg_assign(retval, build_int_cst_type(TREE_TYPE(retval), odo->occurs.bounds.lower));
+    gg_assign(retval, build_int_cst_type( TREE_TYPE(retval),
+                                          odo->occurs.bounds.lower));
     }
   ELSE
     {
-    IF( retval, lt_op, build_int_cst_type(TREE_TYPE(retval), odo->occurs.bounds.lower) )
+    IF( retval, lt_op, build_int_cst_type(TREE_TYPE(retval),
+                                          odo->occurs.bounds.lower) )
       {
       set_exception_code(ec_bound_odo_e);
-      gg_assign(retval, build_int_cst_type(TREE_TYPE(retval), odo->occurs.bounds.lower));
+      gg_assign(retval, build_int_cst_type( TREE_TYPE(retval),
+                                            odo->occurs.bounds.lower));
       }
     ELSE
       ENDIF
@@ -532,8 +545,8 @@ get_depending_on_value(tree retval, const cbl_refer_t &refer)
 
 static
 tree
-get_data_offset(cbl_refer_t &refer,
-                int *pflags = NULL)
+get_data_offset(const cbl_refer_t &refer,
+                      int *pflags = NULL)
   {
   Analyze();
   // This routine returns a tree which is the size_t offset to the data in the
@@ -548,7 +561,6 @@ get_data_offset(cbl_refer_t &refer,
   // We have a refer.
   // At the very least, we have an constant offset
   int all_flags = 0;
-  int all_flag_bit = 1;
 
   if( refer.nsubscript() )
     {
@@ -568,6 +580,7 @@ get_data_offset(cbl_refer_t &refer,
     // Establish the field_t pointer for walking up through our ancestors:
     cbl_field_t *parent = refer.field;
 
+    int all_flag_bit = 1;
     // Note the backwards test, because refer->nsubscript is an unsigned value
     for(size_t i=refer.nsubscript()-1; i<refer.nsubscript(); i-- )
       {
@@ -601,6 +614,8 @@ get_data_offset(cbl_refer_t &refer,
         }
       else
         {
+        const cbl_enabled_exceptions_t&
+                                enabled_exceptions( cdf_enabled_exceptions() );
         if( !enabled_exceptions.match(ec_bound_subscript_e) )
           {
           // With no exception testing, just pick up the value
@@ -625,21 +640,25 @@ get_data_offset(cbl_refer_t &refer,
             }
           ELSE
             {
-            IF( subscript, lt_op, gg_cast(TREE_TYPE(subscript), integer_one_node) )
+            IF( subscript, lt_op, gg_cast(TREE_TYPE(subscript),
+                                          integer_one_node) )
               {
               // The subscript is too small
               set_exception_code(ec_bound_subscript_e);
-              gg_assign(subscript, build_int_cst_type(TREE_TYPE(subscript), 1));
+              gg_assign(subscript, build_int_cst_type(TREE_TYPE(subscript),
+                                                      1));
               }
             ELSE
               {
               IF( subscript,
                   ge_op,
-                  build_int_cst_type(TREE_TYPE(subscript), parent->occurs.ntimes()) )
+                  build_int_cst_type( TREE_TYPE(subscript),
+                                      parent->occurs.ntimes()) )
                 {
                 // The subscript is too large
                 set_exception_code(ec_bound_subscript_e);
-                gg_assign(subscript, build_int_cst_type(TREE_TYPE(subscript), 1));
+                gg_assign(subscript, build_int_cst_type(TREE_TYPE(subscript),
+                                                        1));
                 }
               ELSE
                 {
@@ -654,14 +673,19 @@ get_data_offset(cbl_refer_t &refer,
 
       all_flag_bit <<= 1;
 
-      // Although we strictly don't need to look at the ODO value at this point,
-      // we do want it checked for the purposes of ec-bound-odo
+      // Although we strictly don't need to look at the ODO value at this
+      // point, we do want it checked for the purposes of ec-bound-odo
+
+      const cbl_enabled_exceptions_t&
+                                enabled_exceptions( cdf_enabled_exceptions() );
 
       if( enabled_exceptions.match(ec_bound_odo_e) )
         {
         if( parent->occurs.depending_on )
           {
-          static tree value64 = gg_define_variable(LONG, ".._gdos_value64", vs_file_static);
+          static tree value64 = gg_define_variable( LONG,
+                                                    ".._gdos_value64",
+                                                    vs_file_static);
           cbl_field_t *odo = symbol_find_odo(parent);
           get_depending_on_value_from_odo(value64, odo);
           }
@@ -1238,18 +1262,13 @@ get_binary_value( tree value,
       break;
       }
 
-    case FldAlphanumeric:
-      {
-
-      }
-
-
     default:
       {
-      fprintf(stderr, "%s(): We know not how to"
-                      " get a binary value from %s\n",
-                      __func__,
-                      cbl_field_type_str(field->type) );
+      char *err = xasprintf("%s(): We know not how to"
+                            " get a binary value from %s\n",
+                            __func__,
+                            cbl_field_type_str(field->type) );
+      cbl_internal_error("%s", err);
       abort();
       break;
       }
@@ -1667,8 +1686,9 @@ set_exception_code_func(ec_type_t ec, int /*line*/, int from_raise_statement)
   }
 
 bool
-process_this_exception(ec_type_t ec)
+process_this_exception(const ec_type_t ec)
   {
+  const cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
   bool retval;
   if( enabled_exceptions.match(ec) || !skip_exception_processing )
     {
@@ -1700,7 +1720,7 @@ copy_little_endian_into_place(cbl_field_t *dest,
                               tree value,
                               int rhs_rdigits,
                               bool check_for_error,
-                              tree &size_error)
+                        const tree &size_error)
   {
   if( check_for_error )
     {
@@ -1926,7 +1946,7 @@ get_literal_string(cbl_field_t *field)
   }
 
 bool
-refer_is_clean(cbl_refer_t &refer)
+refer_is_clean(const cbl_refer_t &refer)
   {
   if( !refer.field || refer.field->type == FldLiteralN )
     {
@@ -1957,7 +1977,7 @@ refer_is_clean(cbl_refer_t &refer)
     */
 static
 tree  // size_t
-refer_refmod_length(cbl_refer_t &refer)
+refer_refmod_length(const cbl_refer_t &refer)
   {
   Analyze();
   REFER("refstart and reflen");
@@ -1973,7 +1993,7 @@ refer_refmod_length(cbl_refer_t &refer)
 
 static
 tree // size_t
-refer_fill_depends(cbl_refer_t &refer)
+refer_fill_depends(const cbl_refer_t &refer)
   {
   REFER("");
   // This returns a positive number which is the amount a depends-limited
@@ -2000,8 +2020,8 @@ refer_fill_depends(cbl_refer_t &refer)
   }
 
 tree  // size_t
-refer_offset(cbl_refer_t &refer,
-                    int *pflags)
+refer_offset(const cbl_refer_t &refer,
+                   int *pflags)
   {
   // This routine calculates the effect of a refer offset on the
   // refer.field->data location.  When there are subscripts, the data location
@@ -2028,7 +2048,7 @@ refer_offset(cbl_refer_t &refer,
 
 static
 tree   // size_t
-refer_size(cbl_refer_t &refer, refer_type_t refer_type)
+refer_size(const cbl_refer_t &refer, refer_type_t refer_type)
   {
   Analyze();
   static tree retval = gg_define_variable(SIZE_T, "..rs_retval", vs_file_static);
@@ -2069,13 +2089,13 @@ refer_size(cbl_refer_t &refer, refer_type_t refer_type)
   }
 
 tree  // size_t
-refer_size_dest(cbl_refer_t &refer)
+refer_size_dest(const cbl_refer_t &refer)
   {
   return refer_size(refer, refer_dest);
   }
 
 tree  // size_t
-refer_size_source(cbl_refer_t &refer)
+refer_size_source(const cbl_refer_t &refer)
   {
   /*  There are oddities involved with refer_size_source and refer_size_dest.
       See the comments in refer_has_depends for some explanation.  There are
@@ -2112,7 +2132,7 @@ refer_size_source(cbl_refer_t &refer)
   }
 
 tree
-qualified_data_location(cbl_refer_t &refer)
+qualified_data_location(const cbl_refer_t &refer)
   {
   return gg_add(member(refer.field->var_decl_node, "data"),
                 refer_offset(refer));
diff --git a/gcc/cobol/genutil.h b/gcc/cobol/genutil.h
index 0d9028e..f12124e 100644
--- a/gcc/cobol/genutil.h
+++ b/gcc/cobol/genutil.h
@@ -118,7 +118,7 @@ void      set_exception_code_func(ec_type_t ec,
                                   int line,
                                   int from_raise_statement=0);
 #define set_exception_code(ec) set_exception_code_func(ec, __LINE__)
-bool      process_this_exception(ec_type_t ec);
+bool      process_this_exception(const ec_type_t ec);
 #define   CHECK_FOR_FRACTIONAL_DIGITS true
 void      get_integer_value(tree value,  // This is always a LONG
                             cbl_field_t *field,
@@ -130,7 +130,7 @@ void      copy_little_endian_into_place(cbl_field_t *dest,
                                         tree value,
                                         int rhs_rdigits,
                                         bool check_for_error,
-                                        tree &size_error);
+                                  const tree &size_error);
 tree      build_array_of_size_t( size_t  N,
                                  const size_t *values);
 void      parser_display_internal_field(tree file_descriptor,
@@ -138,14 +138,14 @@ void      parser_display_internal_field(tree file_descriptor,
                                         bool advance=DISPLAY_NO_ADVANCE);
 char     *get_literal_string(cbl_field_t *field);
 
-bool      refer_is_clean(cbl_refer_t &refer);
+bool      refer_is_clean(const cbl_refer_t &refer);
 
-tree      refer_offset(cbl_refer_t &refer,
+tree      refer_offset(const cbl_refer_t &refer,
                        int *pflags=NULL);
-tree      refer_size_source(cbl_refer_t &refer);
-tree      refer_size_dest(cbl_refer_t &refer);
+tree      refer_size_source(const cbl_refer_t &refer);
+tree      refer_size_dest(const cbl_refer_t &refer);
 
-tree      qualified_data_location(cbl_refer_t &refer);
+tree      qualified_data_location(const cbl_refer_t &refer);
 
 void      build_array_of_treeplets( int ngroup,
                                     size_t N,
diff --git a/gcc/cobol/lexio.cc b/gcc/cobol/lexio.cc
index 2d9fb72..52d1aff 100644
--- a/gcc/cobol/lexio.cc
+++ b/gcc/cobol/lexio.cc
@@ -38,29 +38,24 @@
 
 extern int yy_flex_debug;
 
-static struct {
-  bool first_file, explicitly;
-  int column, right_margin;
-  bool inference_pending() {
-    bool tf = first_file && !explicitly;
-    first_file = false;
-    return tf;
-  }
-  inline bool is_fixed() const { return column == 7; }
-  inline bool is_reffmt() const { return is_fixed() && right_margin == 73; }
-  inline bool is_free() const { return ! is_fixed(); }
-  
-  const char * description() const {
-    if( is_reffmt() ) return "REFERENCE";
-    if( is_fixed() ) return "FIXED";
-    if( is_free() ) return "FREE";
-    gcc_unreachable();
-  }    
-} indicator = { true, false, 0, 0 };
+source_format_t& cdf_source_format();
+
+void
+source_format_t::infer( const char *bol, bool want_reference_format ) {
+  if( bol ) {
+    left = 7;
+    if( want_reference_format ) {
+      right = 73;
+    }
+  }
+  dbgmsg("%s:%d: %s format detected", __func__, __LINE__,
+         description());
+}
+
 
 // public source format test functions
-bool is_fixed_format() { return indicator.is_fixed(); }
-bool is_reference_format() { return indicator.is_reffmt(); }
+bool is_fixed_format() { return cdf_source_format().is_fixed(); }
+bool is_reference_format() { return cdf_source_format().is_reffmt(); }
 
 static bool debug_mode = false;
 
@@ -76,11 +71,10 @@ static bool debug_mode = false;
 */
 
 static inline int left_margin() {
-  return indicator.column == 0? indicator.column : indicator.column - 1;
+  return cdf_source_format().left_margin();
 }
 static inline int right_margin() {
-  return indicator.right_margin == 0?
-    indicator.right_margin : indicator.right_margin - 1;
+  return cdf_source_format().right_margin();
 }
 
 /*
@@ -89,18 +83,9 @@ static inline int right_margin() {
  *   When setting back to 0 (free), the right margin is also reset to 0.
  */
 void
-cobol_set_indicator_column( int column )
-{
-  indicator.explicitly = true;
-  if( column == 0 ) indicator.right_margin = 0;
-  if( column < 0 ) {
-    column = -column;
-    indicator.right_margin = 73;
-  }
-  indicator.column = column;
-}
+cobol_set_indicator_column( int column );
 
-bool include_debug()      { return indicator.column == 7 && debug_mode; }
+bool include_debug()      { return is_fixed_format() && debug_mode; }
 bool set_debug( bool tf ) { return debug_mode = tf && is_fixed_format(); }
 
 static bool nonblank( const char ch ) { return !isblank(ch); }
@@ -114,7 +99,7 @@ start_of_line( char *bol, char *eol ) {
 
 static inline char *
 continues_at( char *bol, char *eol ) {
-  if( indicator.column == 0 ) return NULL;  // cannot continue in free format
+  if( cdf_source_format().is_free() ) return NULL;  // cannot continue in free format
   bol += left_margin();
   if( *bol != '-' ) return NULL; // not a continuation line
   return start_of_line(++bol, eol);
@@ -124,7 +109,7 @@ continues_at( char *bol, char *eol ) {
 // NULL means no indicator column or tested value not present.
 static inline char *
 indicated( char *bol, const char *eol, char ch = '\0' ) {
-  if( indicator.column == 0 && *bol != '*' ) {
+  if( cdf_source_format().left_margin() == 0 && *bol != '*' ) {
     return NULL;  // no indicator column in free format, except for comments
   }
   gcc_assert(bol != NULL);
@@ -336,7 +321,69 @@ recognize_replacements( filespan_t mfile, std::list<replace_t>& pending_replacem
 }
 
 static void
+check_push_pop_directive( filespan_t& mfile ) {
+  char eol = '\0';
+  const char *p = std::find(mfile.cur, mfile.eol, '>');
+  if( ! (p < mfile.eol && p[1] == *p ) ) return;
+
+  const char pattern[] =
+    ">>[[:blank:]]*(push|pop)[[:blank:]]+"
+    "("
+      "all|"
+      "call-convention|"
+      "cobol-words|"
+      "define|"
+      "source[[:blank:]]+format|"
+      "turn"
+    ")";
+  static regex re(pattern, extended_icase);
+
+  // show contents of marked subexpressions within each match
+  cmatch cm;
+
+  std::swap(*mfile.eol, eol); // see implementation for excuses
+  bool ok = regex_search(p, const_cast<const char *>(mfile.eol), cm, re);
+  std::swap(*mfile.eol, eol);
+
+  if( ok ) {
+    gcc_assert(cm.size() > 1);
+    bool push = TOUPPER(cm[1].first[1]) == 'U';
+    switch( TOUPPER(cm[2].first[0]) ) {
+    case 'A': // ALL
+      push? cdf_push() : cdf_pop();
+      break;
+    case 'C':
+      switch( TOUPPER(cm[2].first[1]) ) {
+      case 'A': // CALL-CONVENTION
+        push? cdf_push_call_convention() : cdf_pop_call_convention();
+        break;
+      case 'O': // COBOL-WORDS
+        push? cdf_push_current_tokens() : cdf_pop_current_tokens();
+        break;
+      default:
+        gcc_unreachable();
+      }
+      break;
+    case 'D': // DEFINE
+      push? cdf_push_dictionary() : cdf_pop_dictionary();
+      break;
+    case 'S': // SOURCE FORMAT
+      push? cdf_push_source_format() : cdf_pop_source_format();
+      break;
+    case 'T': // TURN
+      push? cdf_push_enabled_exceptions() : cdf_pop_enabled_exceptions();
+      break;
+    default:
+      gcc_unreachable();
+    }
+    erase_line(const_cast<char*>(cm[0].first),
+               const_cast<char*>(cm[0].second));
+  }
+}
+
+static void
 check_source_format_directive( filespan_t& mfile ) {
+  char eol = '\0';
   const char *p = std::find(mfile.cur, mfile.eol, '>');
   if( ! (p < mfile.eol && p[1] == *p ) ) return;
 
@@ -349,7 +396,12 @@ check_source_format_directive( filespan_t& mfile ) {
 
   // show contents of marked subexpressions within each match
   cmatch cm;
-  if( regex_search(p, const_cast<const char *>(mfile.eol), cm, re) ) {
+
+  std::swap(*mfile.eol, eol); // see implementation for excuses
+  bool ok = regex_search(p, const_cast<const char *>(mfile.eol), cm, re);
+  std::swap(*mfile.eol, eol);
+
+  if( ok ) {
     gcc_assert(cm.size() > 1);
     switch( cm[3].length() ) {
     case 4:
@@ -365,11 +417,11 @@ check_source_format_directive( filespan_t& mfile ) {
 
     dbgmsg( "%s:%d: %s format set, on line " HOST_SIZE_T_PRINT_UNSIGNED,
             __func__, __LINE__,
-            indicator.column == 7? "FIXED" : "FREE",
+            cdf_source_format().description(),
             (fmt_size_t)mfile.lineno() );
-    char *bol = indicator.is_fixed()? mfile.cur : const_cast<char*>(cm[0].first);
+    char *bol = cdf_source_format().is_fixed()? mfile.cur : const_cast<char*>(cm[0].first);
+    gcc_assert(cm[0].second <= mfile.eol);
     erase_line(bol, const_cast<char*>(cm[0].second));
-    mfile.cur = const_cast<char*>(cm[0].second);
   }
 }
 
@@ -889,7 +941,7 @@ location_in( const filespan_t& mfile, const csub_match& cm ) {
   gcc_assert(mfile.cur <= cm.first && cm.second <= mfile.eodata);
   auto nline = std::count(cm.first, cm.second, '\n');
   if( nline ) {
-    gcc_assert(loc.first_line < nline);
+    gcc_assert(nline < loc.first_line);
     loc.first_line -= nline;
     auto p = static_cast<const char*>(memrchr(cm.first, '\n', cm.length()));
     loc.last_column = (cm.second) - p;
@@ -1327,13 +1379,13 @@ lexer_input( char buf[], int max_size, FILE *input ) {
   for( auto p = mfile.cur; p < next; *output.pos++ = *p++ ) {
     static bool at_bol = false;
     if( at_bol ) {
-      auto nonblank = std::find_if( p, next,
+      auto nonblank_l = std::find_if( p, next,
                                     []( char ch ) {
                                       return !isblank(ch); } );
-      if( nonblank + 1 < next ) {
-        if( *nonblank == '\r' ) nonblank++; // Windows
-        if( *nonblank == '\n' ) {
-          p = nonblank;
+      if( nonblank_l + 1 < next ) {
+        if( *nonblank_l == '\r' ) nonblank_l++; // Windows
+        if( *nonblank_l == '\n' ) {
+          p = nonblank_l;
           continue;
         }
       }
@@ -1461,7 +1513,6 @@ cdftext::lex_open( const char filename[] ) {
 
   // Process any files supplied by the -include command-line option.
   for( auto name : included_files ) {
-    int input;
     if( -1 == (input = open(name, O_RDONLY)) ) {
       yyerrorvl(1, "", "cannot open -include file %s", name);
       continue;
@@ -1634,7 +1685,7 @@ bool lexio_dialect_mf();
  */
 static const char *
 valid_sequence_area( const char *data, const char *eodata ) {
-  
+
   for( const char *p = data;
        (p = std::find_if(p, eodata, is_p)) != eodata;
        p++ )
@@ -1657,7 +1708,7 @@ valid_sequence_area( const char *data, const char *eodata ) {
       }
     }
   }
-  return nullptr;  
+  return nullptr;
 }
 
 /*
@@ -1693,22 +1744,17 @@ cdftext::free_form_reference_format( int input ) {
   } current( mfile.data );
 
   /*
-   * Infer source code format. 
+   * Infer source code format.
    */
-  if( indicator.inference_pending()  ) {
+  if( cdf_source_format().inference_pending()  ) {
     const char *bol = valid_sequence_area(mfile.data, mfile.eodata);
     if( bol ) {
-      indicator.column = 7;
-      if( infer_reference_format(bol, mfile.eodata) ) {
-	indicator.right_margin = 73;
-      }
+      cdf_source_format().infer( bol, infer_reference_format(bol, mfile.eodata) );
     }
-
-    dbgmsg("%s:%d: %s format detected", __func__, __LINE__,
-           indicator.description());
   }
 
   while( mfile.next_line() ) {
+    check_push_pop_directive(mfile);
     check_source_format_directive(mfile);
     remove_inline_comment(mfile.cur, mfile.eol);
 
@@ -1857,7 +1903,7 @@ cdftext::process_file( filespan_t mfile, int output, bool second_pass ) {
                    []( char ch ) { return ch == '\n'; } );
       struct { int in, out; filespan_t mfile; } copy;
       dbgmsg("%s:%d: line " HOST_SIZE_T_PRINT_UNSIGNED ", opening %s on fd %d",
-             __func__, __LINE__,mfile.lineno(),
+             __func__, __LINE__, (fmt_size_t)mfile.lineno(),
              copybook.source(), copybook.current()->fd);
       copy.in = copybook.current()->fd;
       copy.mfile = free_form_reference_format( copy.in );
@@ -1936,15 +1982,15 @@ cdftext::segment_line( filespan_t& mfile ) {
     struct { unsigned long ante, post; } lineno = {
       gb4(mfile.lineno()), gb4(mfile.lineno() + segment.after.nlines())
     };
-    char *directive = lineno.ante == lineno.post?
+    const char *directive = lineno.ante == lineno.post?
       nullptr : xasprintf("\n#line %lu \"%s\"\n",
                           lineno.ante, cobol_filename());
 
-    if( directive ) 
+    if( directive )
       output.push_back( span_t(strlen(directive), directive) );
     output.push_back( span_t(mfile.cur, segment.before.p) );
     output.push_back( span_t(segment.after.p, segment.after.pend ) );
-    if( directive ) 
+    if( directive )
       output.push_back( span_t(strlen(directive), directive) );
 
     mfile.cur = const_cast<char*>(segment.before.pend);
diff --git a/gcc/cobol/lexio.h b/gcc/cobol/lexio.h
index eb41068..ba4ef0a 100644
--- a/gcc/cobol/lexio.h
+++ b/gcc/cobol/lexio.h
@@ -244,8 +244,8 @@ struct span_t {
     return span_t(output, eout);
   }
   const char * has_nul() const {
-    auto p = std::find(this->p, pend, '\0');
-    return p != pend? p : NULL;
+    auto p_l = std::find(this->p, pend, '\0');
+    return p_l != pend? p_l : NULL;
   }
 
   bool at_eol() const {
diff --git a/gcc/cobol/parse.y b/gcc/cobol/parse.y
index 74637c9..fae96ed 100644
--- a/gcc/cobol/parse.y
+++ b/gcc/cobol/parse.y
@@ -45,6 +45,7 @@
   };
 
   enum accept_func_t {
+    accept_e,
     accept_done_e,
     accept_command_line_e,
     accept_envar_e,
@@ -349,7 +350,7 @@
 %token  <string>        SECTION
 %token  <number>        STANDARD_ALPHABET "STANDARD ALPHABET"
 %token  <string>        SWITCH
-%token  <string>        UPSI
+%token  <string>        UPSI 
 %token  <number>        ZERO
 
                          /* environment names */
@@ -385,7 +386,10 @@
 			CDF_EVALUATE ">>EVALUATE"
 			CDF_WHEN ">>WHEN"
 			CDF_END_EVALUATE ">>END-EVALUATE"
+                        CALL_CONVENTION ">>CALL-CONVENTION"
 			CALL_COBOL "CALL" CALL_VERBATIM "CALL (as C)"
+                        CDF_PUSH ">>PUSH" CDF_POP ">>POP"
+                        SOURCE_FORMAT ">>SOURCE FORMAT"
 
 			IF THEN ELSE
 			SENTENCE
@@ -399,7 +403,10 @@
 			STRING_kw "STRING" STOP SUBTRACT START
 			UNSTRING WRITE WHEN
 
-			   ABS ACCESS ACOS ACTUAL ADVANCING AFTER ALL
+			ARGUMENT_NUMBER  ARGUMENT_VALUE
+			ENVIRONMENT_NAME ENVIRONMENT_VALUE
+
+			ABS ACCESS ACOS ACTUAL ADVANCING AFTER ALL
 			ALLOCATE
 			ALPHABET ALPHABETIC ALPHABETIC_LOWER "ALPHABETIC-LOWER"
 			ALPHABETIC_UPPER "ALPHABETIC-UPPER"
@@ -793,6 +800,8 @@
 %type   <error_clauses> io_invalids read_eofs write_eops
 %type   <boolean>       io_invalid  read_eof  write_eop
                         global is_global anycase backward
+                        end_display
+                        exh_changed exh_named
 %type   <number>        mistake globally first_last
 %type   <io_mode>   io_mode
 
@@ -848,7 +857,7 @@
            declarative_list_t* dcl_list_t;
            isym_list_t* isym_list;
     struct { radix_t radix; char *string; } numstr;
-    struct { int token; literal_t name; } prog_end;
+    struct { YYLTYPE loc; int token; literal_t name; } prog_end;
     struct { int token; special_name_t id; } special_type;
     struct { cbl_field_type_t type;
              uint32_t capacity; bool signable; } computational;
@@ -902,7 +911,7 @@
     struct refer_pair_t { cbl_refer_t *first, *second; } refer2;
     struct { refer_collection_t *inputs; refer_pair_t into; } str_body;
 
-    struct { accept_func_t func; cbl_refer_t *into, *from; } accept_func;
+    struct { accept_func_t func; cbl_refer_t *into, *from; special_name_t special;} accept_func;
     struct unstring_into_t *uns_into;
     struct unstring_tgt_list_t *uns_tgts;
     struct unstring_tgt_t *uns_tgt;
@@ -1004,7 +1013,9 @@
 %right                  IF THEN ELSE
                         SENTENCE
                         ACCEPT ADD ALTER CALL CANCEL CLOSE COMPUTE CONTINUE
-                        DELETE DISPLAY DIVIDE EVALUATE END EOP EXIT FILLER_kw
+                        DELETE DISPLAY DIVIDE
+                        EVALUATE END EOP EXIT
+                        FILLER_kw
                         GOBACK GOTO
                         INITIALIZE INSPECT
                         MERGE MOVE MULTIPLY OPEN OVERFLOW_kw PARAGRAPH PERFORM
@@ -1464,16 +1475,16 @@ cobol_words:	cobol_words1
 	|	cobol_words cobol_words1
 		;
 cobol_words1:	COBOL_WORDS EQUATE NAME[keyword] WITH NAME[name] {
-		  if( ! tokens.equate(@keyword, $keyword, $name) ) { YYERROR; }
+		  if( ! cdf_tokens.equate(@keyword, $keyword, $name) ) { YYERROR; }
 		}
 	|	COBOL_WORDS UNDEFINE NAME[keyword] {
-		  if( ! tokens.undefine(@keyword, $keyword) ) { YYERROR; }
+		  if( ! cdf_tokens.undefine(@keyword, $keyword) ) { YYERROR; }
 		}
 	|	COBOL_WORDS SUBSTITUTE NAME[keyword] BY NAME[name] {
-		  if( ! tokens.substitute(@keyword, $keyword, $name) ) { YYERROR; }
+		  if( ! cdf_tokens.substitute(@keyword, $keyword, $name) ) { YYERROR; }
 		}
 	|	COBOL_WORDS RESERVE NAME[name] {
-		  if( ! tokens.reserve(@name, $name) ) { YYERROR; }
+		  if( ! cdf_tokens.reserve(@name, $name) ) { YYERROR; }
 		}
 		;
 
@@ -1513,7 +1524,7 @@ program_as:     %empty     { static const literal_t empty {}; $$ = empty; }
         |       AS LITERAL { $$ = $2; }
                 ;
 
-function_id:    FUNCTION '.' NAME program_as program_attrs[attr] '.'
+function_id:    FUNCTION NAME program_as program_attrs[attr] '.'
                 {
                   internal_ebcdic_lock();
                   current_division = identification_div_e;
@@ -1547,7 +1558,7 @@ function_id:    FUNCTION '.' NAME program_as program_attrs[attr] '.'
                   current.udf_add(current_program_index());
                   if( nparse_error > 0 ) YYABORT;
                 }
-        |       FUNCTION '.' NAME program_as is PROTOTYPE '.'
+        |       FUNCTION NAME program_as is PROTOTYPE '.'
                 {
                   cbl_unimplemented("FUNCTION PROTOTYPE");
                 }
@@ -1838,7 +1849,7 @@ select:         SELECT optional NAME[name] select_clauses[clauses] '.'
                   cbl_file_t *file = $clauses.file;
 
                   file->optional = $optional;
-                  file->line = yylineno;
+                  file->line = @name.first_line;
                   if( !namcpy(@clauses, file->name, $name) ) YYERROR;
 
                   if( ! ($clauses.clauses & assign_clause_e) ) {
@@ -1911,7 +1922,7 @@ select:         SELECT optional NAME[name] select_clauses[clauses] '.'
                   cbl_file_t file = protofile;
 
                   file.optional = $optional;
-                  file.line = yylineno;
+                  file.line = @name.first_line;
                   if( !namcpy(@name, file.name, $name) ) YYERROR;
 
                   if( file_add(@name, &file) == NULL ) YYERROR;
@@ -2473,7 +2484,7 @@ special_name:   dev_mnemonic
         |       CLASS NAME is domains
                 {
                   struct cbl_field_t field = { 0,
-                    FldClass, FldInvalid, 0, 0, 0, 0, nonarray, yylineno, "",
+                    FldClass, FldInvalid, 0, 0, 0, 0, nonarray, @NAME.first_line, "",
                     0, cbl_field_t::linkage_t(),
 		    {}, NULL };
                   if( !namcpy(@NAME, field.name, $2) ) YYERROR;
@@ -2604,6 +2615,10 @@ device_name:	SYSIN           { $$.token = SYSIN; $$.id = SYSIN_e; }
         |       STDIN           { $$.token = STDIN; $$.id = STDIN_e; }
         |       STDOUT          { $$.token = STDOUT; $$.id = STDOUT_e; }
         |       STDERR          { $$.token = STDERR; $$.id = STDERR_e; }
+                /* These cannot be both ctx_name and here. *
+        /*      ARGUMENT_NUMBER   { $$.token=0; $$.id = ARG_NUM_e; } */
+        /*      ENVIRONMENT_NAME  { $$.token=0; $$.id = ENV_NAME_e; } */
+        /*      ENVIRONMENT_VALUE { $$.token=0; $$.id = ENV_VALUE_e; } */
                 ;
 
 alphabet_name:  STANDARD_ALPHABET  { $$ = alphabet_add(@1, ASCII_e); }
@@ -3164,7 +3179,7 @@ depending:      %empty
                     assert(e->type == SymField);
                     odo = symbol_index(e);
                   } else {
-		    e = symbol_field_forward_add(PROGRAM, 0, $NAME, yylineno);
+		    e = symbol_field_forward_add(PROGRAM, 0, $NAME, @NAME.first_line);
                     if( !e ) YYERROR;
 		    symbol_field_location( symbol_index(e), @NAME );
                     odo = field_index(cbl_field_of(e));
@@ -3364,7 +3379,7 @@ level_name:     LEVEL ctx_name
                   }
                   struct cbl_field_t field = { 0,
                     FldInvalid, FldInvalid, 0, 0, 0, capacity_cast($1),
-		    nonarray, yylineno, "",
+		    nonarray, @ctx_name.first_line, "",
                     0, cbl_field_t::linkage_t(),
 		    {}, NULL };
                   if( !namcpy(@ctx_name, field.name, $2) ) YYERROR;
@@ -3389,7 +3404,7 @@ level_name:     LEVEL ctx_name
                   }
                   struct cbl_field_t field = { 0,
                     FldInvalid, FldInvalid, 0, 0, 0, capacity_cast($1),
-		    nonarray, yylineno, "",
+		    nonarray, @LEVEL.first_line, "",
                     0, {}, {}, NULL };
 
                   $$ = field_add(@1, &field);
@@ -3527,7 +3542,7 @@ data_descr1:    level_name
                   }
                   struct cbl_field_t field = { 0, FldLiteralA, FldInvalid,
                                                constant_e, 0, 0, 78, nonarray,
-                                               yylineno, "", 0, {}, *$data, NULL };
+                                               @name.first_line, "", 0, {}, *$data, NULL };
                   if( !namcpy(@name, field.name, $name) ) YYERROR;
                   if( field.data.initial ) {
                     field.attr |= quoted_e;
@@ -3550,7 +3565,7 @@ data_descr1:    level_name
         |       LEVEL88 NAME /* VALUE */ NULLPTR
                 {
                   struct cbl_field_t field = { 0,
-                    FldClass, FldInvalid, 0, 0, 0, 88, nonarray, yylineno, "",
+                    FldClass, FldInvalid, 0, 0, 0, 88, nonarray, @NAME.first_line, "",
                     0, cbl_field_t::linkage_t(),
 		    {}, NULL };
                   if( !namcpy(@NAME, field.name, $2) ) YYERROR;
@@ -3576,7 +3591,7 @@ data_descr1:    level_name
         |       LEVEL88 NAME VALUE domains
                 {
                   struct cbl_field_t field = { 0,
-                    FldClass, FldInvalid, 0, 0, 0, 88, nonarray, yylineno, "",
+                    FldClass, FldInvalid, 0, 0, 0, 88, nonarray, @NAME.first_line, "",
                     0, cbl_field_t::linkage_t(),
 		    {}, NULL };
                   if( !namcpy(@NAME, field.name, $2) ) YYERROR;
@@ -4606,7 +4621,7 @@ justified_clause: is JUSTIFIED
 
 redefines_clause: REDEFINES NAME[orig]
                 {
-                  struct symbol_elem_t *e = field_of($orig);
+                  struct symbol_elem_t *e = symbol_field(PROGRAM, 0, $orig);
                   if( !e ) {
                     error_msg(@2, "REDEFINES target not defined");
                     YYERROR;
@@ -4906,6 +4921,7 @@ by_value_arg:   scalar
 declaratives:   %empty
         |       DECLARATIVES '.'
                 <label>{
+		  cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
                   current.enabled_exception_cache = enabled_exceptions;
                   enabled_exceptions.clear();
                   current.doing_declaratives(true);
@@ -4924,6 +4940,7 @@ declaratives:   %empty
                    * forward reference, because we haven't yet begun to parse
                    * nondeclarative procedures.
                    */
+		  cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
                   parser_label_label($label);
                   enabled_exceptions = current.enabled_exception_cache;
                   current.enabled_exception_cache.clear();
@@ -5038,6 +5055,7 @@ statement:      error {
         |       divide          { $$ =  DIVIDE; }
         |       entry           { $$ =  ENTRY; }
         |       evaluate        { $$ =  EVALUATE; }
+        |       exhibit_stmt    { $$ =  EXHIBIT; }
         |       exit            { $$ =  EXIT; }
         |       free            { $$ =  FREE; }
         |       go_to           { $$ =  GOTO; }
@@ -5068,9 +5086,8 @@ statement:      error {
 
 		/*
 		 * ISO defines ON EXCEPTION only for Format 3 (screen). We
-		 * implement extensions defined by MF and Fujitsu (and us) to
-		 * use ACCEPT to interact with the command line and the
-		 * environment.
+		 * implement extensions defined by MF and Fujitsu to use ACCEPT
+		 * to interact with the command line and the environment.
 		 *
 		 * ISO ACCEPT and some others are implemented in accept_body,
 		 * before the parser sees any ON EXCEPTION.  In those cases
@@ -5085,6 +5102,9 @@ accept:         accept_body end_accept {
 		  switch( $accept_body.func ) {
 		  case accept_done_e:
 		    break;
+		  case accept_e:
+                    parser_accept(*$1.into, $1.special, nullptr, nullptr);
+		    break;
 		  case accept_command_line_e:
 		    if( $1.from->field == NULL ) { // take next command-line arg
 		      parser_accept_command_line(*$1.into, argi, NULL, NULL);
@@ -5108,6 +5128,9 @@ accept:         accept_body end_accept {
 		    error_msg(@ec, "ON EXCEPTION valid only "
 			    "with ENVIRONMENT or COMMAND-LINE(n)");
 		    break;
+		  case accept_e:
+                    parser_accept(*$1.into, $1.special, $ec.on_error, $ec.not_error);
+		    break;
 		  case accept_command_line_e:
 		    if( $1.from->field == NULL ) { // take next command-line arg
 		      parser_accept_command_line(*$1.into, argi,
@@ -5139,7 +5162,7 @@ end_accept:     %empty %prec ACCEPT
 accept_body:    accept_refer
                 {
 		  $$.func = accept_done_e;
-                  parser_accept(*$1, CONSOLE_e);
+                  parser_accept(*$1, CONSOLE_e, nullptr, nullptr);
                 }
         |       accept_refer FROM DATE
                 {
@@ -5198,29 +5221,15 @@ accept_body:    accept_refer
                 }
         |       accept_refer FROM acceptable
                 {
-		  cbl_field_t *argc = register_find("_ARGI");
-		  switch( $acceptable->id ) {
-		  case ARG_NUM_e:
-		    $$.func = accept_command_line_e;
-		    $$.into = $1;
-		    $$.from = new_reference(argc);
-		    break;
-		  case ARG_VALUE_e:
-		    $$.func = accept_command_line_e;
-		    $$.into = $1;
-		    $$.from = cbl_refer_t::empty();
-		    break;
-		  default:
-		    $$.func = accept_done_e;
-		    parser_accept( *$1, $acceptable->id );
-		  }
+		  $$.func = accept_e;
+                  $$.into = $1;
+                  $$.special = $acceptable->id;
                 }
         |       accept_refer FROM ENVIRONMENT envar
                 {
 		  $$.func = accept_envar_e;
 		  $$.into = $1;
 		  $$.from = $envar;
-                  ////  parser_accept_envar( *$1, *$envar );
                 }
         |       accept_refer FROM COMMAND_LINE
                 {
@@ -5232,7 +5241,6 @@ accept_body:    accept_refer
 		  $$.func = accept_command_line_e;
 		  $$.into = $1;
 		  $$.from = $expr;
-                  //// parser_accept_command_line(*$1, $expr->field );
                 }
         |       accept_refer FROM COMMAND_LINE_COUNT {
 		  $$.func = accept_done_e;
@@ -5285,7 +5293,7 @@ accept_except:	EXCEPTION
                 {
                   $$.not_error = NULL;
                   $$.on_error = label_add(LblArith,
-                                          uniq_label("accept"), yylineno);
+                                          uniq_label("accept"), @1.first_line);
                   if( !$$.on_error ) YYERROR;
                   parser_accept_exception( $$.on_error );
 
@@ -5320,15 +5328,54 @@ acceptable:     device_name
                       error_msg(@NAME, "no such special name '%s'", $NAME);
                       YYERROR;
 		    }
+                    if( ENV_NAME_e == *special_type ) { 
+                      error_msg(@NAME, "cannot ACCEPT FROM %qs", $NAME);
+                      YYERROR;
+                    }
 		    // Add the name now, as a convenience. 
-		    cbl_special_name_t special = { 0, *special_type };
+                    int token = 0;
+                    switch(*special_type) {
+                    case ARG_NUM_e:    token = ARGUMENT_NUMBER; break;
+                    case ARG_VALUE_e:  token = ARGUMENT_VALUE; break;
+                    case ENV_VALUE_e:  token = ENVIRONMENT_VALUE; break;
+
+                    case ENV_NAME_e:
+                    default:
+                      error_msg(@NAME, "cannot ACCEPT FROM %qs", $NAME);
+                      YYERROR;
+                      break;
+                    }
+		    cbl_special_name_t special = { token, *special_type };
 		    namcpy(@NAME, special.name, $NAME);
 
 		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
 		    $$ = cbl_special_name_of(e);
+                    cbl_special_name_t& unused(*$$);
+                    assert(unused.id);
 		  }
 		  assert($$);
                 }
+        |       ENVIRONMENT_VALUE {
+		    // Add the name now, as a convenience. 
+		    cbl_special_name_t special =
+                      { ENVIRONMENT_VALUE, ENV_VALUE_e, "ENVIRONMENT-VALUE" };
+		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
+		    $$ = cbl_special_name_of(e);
+                }
+        |       ARGUMENT_NUMBER {
+		    // Add the name now, as a convenience. 
+		    cbl_special_name_t special =
+                      { ARGUMENT_NUMBER, ARG_NUM_e, "ARGUMENT-NUMBER" };
+		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
+		    $$ = cbl_special_name_of(e);
+                }
+        |       ARGUMENT_VALUE {
+		    // Add the name now, as a convenience. 
+		    cbl_special_name_t special =
+                      { ARGUMENT_VALUE, ARG_VALUE_e, "ARGUMENT-VALUE" };
+		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
+		    $$ = cbl_special_name_of(e);
+                }
                 ;
 
 add:            add_impl end_add { ast_add($1); }
@@ -5558,46 +5605,18 @@ compute_expr:   '=' {
                 }
                 ;
 
-display:        disp_body end_display
-                {
-		  std::vector <cbl_refer_t> args($1.vargs->args.size());
-		  std::copy( $1.vargs->args.begin(), $1.vargs->args.end(), args.begin() );
-		  if( $1.special && $1.special->id == ARG_NUM_e ) {
-		    if( $1.vargs->args.size() != 1 ) {
-		      error_msg(@1, "ARGUMENT-NUMBER can be set to only one value");
-		    }
-		   const  cbl_refer_t& src( $1.vargs->args.front() );
-		    cbl_field_t *dst = register_find("_ARGI");
-		    parser_move( dst, src );
-		  } else {
-		    parser_display($1.special,
-				   args.empty()? NULL : args.data(), args.size(),
-				   DISPLAY_ADVANCE);
-		  }
-		  current.declaratives_evaluate();
-                }
-        |       disp_body NO ADVANCING end_display
+display:        disp_body end_display[advance]
                 {
-                  std::vector <cbl_refer_t> args($1.vargs->args.size());
-		  std::copy( $1.vargs->args.begin(), $1.vargs->args.end(), args.begin() );
-
-		  if( $1.special && $1.special->id == ARG_NUM_e ) {
-		    if( $1.vargs->args.size() != 1 ) {
-		      error_msg(@1, "ARGUMENT-NUMBER can be set to only one value");
-		    }
-		   const cbl_refer_t& src( $1.vargs->args.front() );
-		    cbl_field_t *dst = register_find("_ARGI");
-		    parser_move( dst, src );
-		  } else {
-		    parser_display($1.special,
-				   args.empty()? NULL : args.data(), args.size(),
-				   DISPLAY_NO_ADVANCE);
-		  }
+		  std::vector <cbl_refer_t> args($1.vargs->args.begin(),
+                                                 $1.vargs->args.end());
+                  parser_display($1.special, args, $advance);
 		  current.declaratives_evaluate();
                 }
                 ;
-end_display:    %empty
-        |       END_DISPLAY
+end_display:    %empty				{ $$ = DISPLAY_ADVANCE; }
+        |                    END_DISPLAY	{ $$ = DISPLAY_ADVANCE; }
+        |       NO ADVANCING			{ $$ = DISPLAY_NO_ADVANCE; }
+        |       NO ADVANCING END_DISPLAY	{ $$ = DISPLAY_NO_ADVANCE; }
                 ;
 disp_body:      disp_vargs[vargs]
                 {
@@ -5628,14 +5647,62 @@ disp_upon:    	device_name {
                       error_msg(@NAME, "no such special name '%s'", $NAME);
                       YYERROR;
 		    }
-		    // Add the name now, as a convenience. 
-		    cbl_special_name_t special = { 0, *special_type };
+		    // Add the name now, as a convenience.
+                    // These may come through as a NAME, depending on how scanned. 
+                    int token = 0;
+                    switch(*special_type) {
+                    case ARG_NUM_e:    token = ARGUMENT_NUMBER; break;
+                    case ENV_NAME_e:   token = ENVIRONMENT_NAME; break;
+                    case ENV_VALUE_e:  token = ENVIRONMENT_VALUE; break;
+
+                    case ARG_VALUE_e:
+                    default:
+                      error_msg(@NAME, "cannot DISPLAY UPON %qs", $NAME);
+                      YYERROR;
+                      break;
+                    }
+		    cbl_special_name_t special = { token, *special_type };
 		    namcpy(@NAME, special.name, $NAME);
 
                     e = symbol_special_add(PROGRAM, &special);
 		  }
 		  $$ = cbl_special_name_of(e);
                 }
+        |       ARGUMENT_NUMBER {
+		    // Add the name now, as a convenience. 
+		    cbl_special_name_t special =
+                      { ARGUMENT_NUMBER, ARG_NUM_e, "ARGUMENT-NUMBER" };
+		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
+		    $$ = cbl_special_name_of(e);
+                }
+        |       ENVIRONMENT_NAME {
+		    // Add the name now, as a convenience. 
+		    cbl_special_name_t special =
+                      { ENVIRONMENT_NAME, ENV_NAME_e, "ENVIRONMENT-NAME" };
+		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
+		    $$ = cbl_special_name_of(e);
+                }
+        |       ENVIRONMENT_VALUE {
+		    // Add the name now, as a convenience. 
+		    cbl_special_name_t special =
+                      { ENVIRONMENT_VALUE, ENV_VALUE_e, "ENVIRONMENT-VALUE" };
+		    symbol_elem_t *e = symbol_special_add(PROGRAM, &special);
+		    $$ = cbl_special_name_of(e);
+                }
+                ;
+
+exhibit_stmt:   EXHIBIT exh_changed exh_named vargs {
+                  statement_begin(@1, EXHIBIT);
+                  std::vector<cbl_refer_t> args( $vargs->args.begin(),
+                                                 $vargs->args.end() );
+                  parser_exhibit( $exh_changed, $exh_named, args );
+                }
+                ;
+exh_changed:    %empty  { $$ = false; }
+        |       CHANGED { $$ = true; }
+                ;
+exh_named:      %empty  { $$ = false; }
+        |       NAMED   { $$ = true; }
                 ;
 
 divide:         divide_impl end_divide { ast_divide($1); }
@@ -5733,14 +5800,14 @@ end_program:    end_program1[end] '.'
                     gcc_unreachable();
                   }
                   if( !matches ) {
-                    error_msg(@end, "END %s %s does not match "
+                    error_msg($end.loc, "END %s %s does not match "
                                     "%<IDENTIFICATION DIVISION %s%>",
                               token_name, name, prog->name);
                     YYERROR;
                   }
 
                   if( 0 != strcasecmp(prog->name, name) ) {
-                    error_msg(@end, "END PROGRAM '%s' does not match PROGRAM-ID '%s'",
+                    error_msg($end.loc, "END PROGRAM '%s' does not match PROGRAM-ID '%s'",
                               name, prog->name);
                     YYERROR;
                   }
@@ -5773,20 +5840,24 @@ end_program:    end_program1[end] '.'
                 ;
 end_program1:   END_PROGRAM  namestr[name]
                 {
+                  $$.loc = @name;
                   $$.token = END_PROGRAM;
                   $$.name = $name;
                 }
         |       END_FUNCTION namestr[name]
                 {
+                  $$.loc = @name;
                   $$.token = END_FUNCTION;
                   $$.name = $name;
                 }
 	|	END_PROGRAM  '.' // error
                 {
+                  $$.loc = @1;
                   $$.token = END_PROGRAM;
                 }
         |       END_FUNCTION '.' // error
                 {
+                  $$.loc = @1;
                   $$.token = END_FUNCTION;
                 }
                 ;
@@ -6622,7 +6693,7 @@ name:           qname
                       auto name = names.front();
                       names.pop_front();
                       auto e = symbol_field_forward_add(PROGRAM, parent,
-                                                        name, yylineno);
+                                                        name, @1.first_line);
                       if( !e ) YYERROR;
 		      symbol_field_location( symbol_index(e), @qname );
                       parent = symbol_index(e);
@@ -6652,6 +6723,10 @@ ctx_name:       NAME
 
 context_word:   APPLY                   { static char s[] ="APPLY";
                                          $$ = s; } // screen description entry
+        |       ARGUMENT_NUMBER         { static char s[] ="ARGUMENT-NUMBER";
+                                         $$ = s; } // Display Upon / Accept From
+        |       ARGUMENT_VALUE         { static char s[] ="ARGUMENT-VALUE";
+                                         $$ = s; } // Accept From
         |       ARITHMETIC             { static char s[] ="ARITHMETIC";
                                          $$ = s; } // OPTIONS paragraph
         |       ATTRIBUTE              { static char s[] ="ATTRIBUTE";
@@ -6688,6 +6763,10 @@ context_word:   APPLY                   { static char s[] ="APPLY";
                                          $$ = s; } // ERASE clause in a screen description entry
         |       ENTRY_CONVENTION       { static char s[] ="ENTRY-CONVENTION";
                                          $$ = s; } // OPTIONS paragraph
+	|       ENVIRONMENT_NAME         { static char s[] ="ENVIRONMENT-NAME";
+                                         $$ = s; } // Display Upon
+        |       ENVIRONMENT_VALUE         { static char s[] ="ENVIRONMENT-VALUE";
+                                         $$ = s; } // Display Upon / Accept From
         |       ERASE                  { static char s[] ="ERASE";
                                          $$ = s; } // screen description entry
         |       EXPANDS                { static char s[] ="EXPANDS";
@@ -7036,9 +7115,9 @@ arith_err:      SIZE_ERROR
                     *ptgt = $1 == NOT?
                       current.compute_not_error() : current.compute_on_error();
                   } else {
-                    *ptgt = label_add(LblArith, uniq_label("arith"), yylineno);
+                    *ptgt = label_add(LblArith, uniq_label("arith"), @1.first_line);
                   }
-		  (*ptgt)->lain = yylineno;
+		  (*ptgt)->lain = @1.first_line;
                   parser_arith_error( *ptgt );
                 }
                 ;
@@ -7575,6 +7654,7 @@ perform_cond:   UNTIL { parser_perform_conditional( &perform_current()->tgt); }
 perform_inline: perform_start statements END_PERFORM
                 {
 		  location_set(@END_PERFORM);
+		  parser_sleep(*cbl_refer_t::empty());
 		  $$ = perform_current();
 		  if( $perform_start == LOCATION ) {
 		    error_msg(@1, "LOCATION not valid with PERFORM Format 2");
@@ -7583,6 +7663,7 @@ perform_inline: perform_start statements END_PERFORM
         |       perform_start END_PERFORM
                 {
 		  location_set(@END_PERFORM);
+		  parser_sleep(*cbl_refer_t::empty());
 		  $$ = perform_current();
 		  if( $perform_start == LOCATION ) {
 		    error_msg(@1, "LOCATION not valid with PERFORM Format 2");
@@ -8754,12 +8835,12 @@ search_1_body:  name[table] search_varying[varying]
 
                   cbl_name_t label_name;
 		  auto len = snprintf(label_name, sizeof(label_name),
-				     "linear_search_%d", yylineno);
+				     "linear_search_%d", @1.first_line);
 		  if( ! (0 < len && len < int(sizeof(label_name))) ) {
 		    gcc_unreachable();
 		  }
                   cbl_label_t *name = label_add( LblSearch,
-                                                 label_name, yylineno );
+                                                 label_name, @1.first_line );
 		  auto varying($varying);
                   if( index == varying ) varying = NULL;
                   parser_lsearch_start( name, $table, index, varying );
@@ -8812,9 +8893,9 @@ search_binary:  SEARCH ALL search_2_body search_2_cases
 search_2_body:  name[table]
                 {
                   statement_begin(@$, SEARCH);
-                  char *label_name = xasprintf("binary_search_%d", yylineno);
+                  char *label_name = xasprintf("binary_search_%d", @1.first_line);
                   cbl_label_t *name = label_add( LblSearch,
-                                                 label_name, yylineno );
+                                                 label_name, @1.first_line );
                   parser_bsearch_start( name, $table );
                   search_alloc(name);
                 }
@@ -9759,7 +9840,7 @@ call_except:    EXCEPTION
                 {
                   $$.not_error = NULL;
                   $$.on_error = label_add(LblArith,
-                                          uniq_label("call"), yylineno);
+                                          uniq_label("call"), @1.first_line);
                   if( !$$.on_error ) YYERROR;
                   parser_call_exception( $$.on_error );
 
@@ -9772,7 +9853,7 @@ call_except:    EXCEPTION
                 {
                   $$.not_error = NULL;
                   $$.on_error = label_add(LblArith,
-                                          uniq_label("call"), yylineno);
+                                          uniq_label("call"), @1.first_line);
                   if( !$$.on_error ) YYERROR;
                   parser_call_exception( $$.on_error );
 
@@ -9828,7 +9909,7 @@ go_to:          GOTO labels[args]
                   }
 
 		  for( auto& label : $args->elems ) {
-		    label->used = yylineno;
+		    label->used = @2.first_line;
 		  }
                   cbl_label_t *arg = $args->elems.front();
                   parser_goto( cbl_refer_t(), 1, &arg );
@@ -9840,7 +9921,7 @@ go_to:          GOTO labels[args]
                   std::vector <cbl_label_t *> args($args->elems.size());
 		  std::copy($args->elems.begin(), $args->elems.end(), args.begin());
 		  for( auto& label : $args->elems ) {
-		    label->used = yylineno;
+		    label->used = @2.first_line;
 		  }
                   parser_goto( *$value, args.size(), args.data() );
                 }
@@ -9860,7 +9941,7 @@ resume:         RESUME NEXT STATEMENT
                 {
                   statement_begin(@1, RESUME);
                   parser_clear_exception();
-		  $tgt->used = yylineno;
+		  $tgt->used = @1.first_line;
                   parser_goto( cbl_refer_t(), 1, &$tgt );
                 }
                 ;
@@ -10035,7 +10116,7 @@ on_overflow:    OVERFLOW_kw
                 {
                   $$.not_error = NULL;
                   $$.on_error = label_add(LblString,
-                                          uniq_label("string"), yylineno);
+                                          uniq_label("string"), @1.first_line);
                   if( !$$.on_error ) YYERROR;
                   parser_string_overflow( $$.on_error );
 
@@ -10255,7 +10336,7 @@ intrinsic:      function_udf
                   if( p != NULL ) {
 		    auto loc = symbol_field_location(field_index(p->field));
                     error_msg(loc, "FUNCTION %qs has "
-                              "inconsistent parameter type %zu (%qs)",
+                              "inconsistent parameter type %td (%qs)",
                               keyword_str($1), p - args.data(), name_of(p->field) );
                     YYERROR;
                   }
@@ -11377,6 +11458,7 @@ void ast_call( const YYLTYPE& loc, cbl_refer_t name, const cbl_refer_t& returnin
  */
 static bool
 possible_ec() {
+  cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
   bool format_1 = current.declaratives.has_format_1();
       
   bool enabled = 0xFF < (current.declaratives.status()
@@ -11399,6 +11481,7 @@ possible_ec() {
  */
 static void
 statement_epilog( int token ) {
+  cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
   if( possible_ec() && token != CONTINUE ) { 
     if( enabled_exceptions.size() ) {
       current.declaratives_evaluate();
@@ -11461,9 +11544,11 @@ keyword_str( int token ) {
     return ascii;
   }
 
-  return tokens.name_of(token);
+  return cdf_tokens.name_of(token);
 }
 
+bool iso_cobol_word( const std::string& name, bool include_context );
+
 /*
  * Return the token for the Cobol name, unless it is a function name.  The
  * lexer uses keyword_tok to determine if what appears to be a NAME is in fact
@@ -11474,15 +11559,14 @@ keyword_str( int token ) {
  */
 
 // tokens.h is generated as needed from parse.h with tokens.h.gen
-tokenset_t::tokenset_t() {
+current_tokens_t::tokenset_t::tokenset_t() {
 #include "token_names.h"
 }
 
-bool iso_cobol_word( const std::string& name, bool include_context );
 
 // Look up the lowercase form of a keyword, excluding some CDF names.
 int
-tokenset_t::find( const cbl_name_t name, bool include_intrinsics ) {
+current_tokens_t::tokenset_t::find( const cbl_name_t name, bool include_intrinsics ) {
   static const cbl_name_t non_names[] = { // including CDF NAMES, and "SWITCH"
     "CHECKING", "LIST", "LOCATION", "MAP", "SWITCH",
   }, * const eonames = non_names + COUNT_OF(non_names);
@@ -11532,7 +11616,7 @@ tokenset_t::find( const cbl_name_t name, bool include_intrinsics ) {
 
 int
 keyword_tok( const char * text, bool include_intrinsics ) {
-  return tokens.find(text, include_intrinsics);
+  return cdf_tokens.find(text, include_intrinsics);
 }
 
 static inline size_t
@@ -11724,7 +11808,7 @@ label_add( const YYLTYPE& loc,
                 name, cbl_label_of(p)->name, cbl_label_of(p)->line);
     }
   }
-  struct cbl_label_t label = { type, parent, loc.last_line };
+  struct cbl_label_t label = { type, parent, loc.first_line };
 
   if( !namcpy(loc, label.name, name) ) return NULL;
   auto p =  symbol_label_add(PROGRAM, &label);
@@ -13074,7 +13158,7 @@ cobol_dialect_set( cbl_dialect_t dialect ) {
     break;
   case dialect_gnu_e:
     if( 0 == (cbl_dialects & dialect) ) { // first time
-      tokens.equate(YYLTYPE(), "BINARY-DOUBLE", "BINARY-C-LONG");
+      cdf_tokens.equate(YYLTYPE(), "BINARY-DOUBLE", "BINARY-C-LONG");
     }
     break;
   }    
diff --git a/gcc/cobol/parse_ante.h b/gcc/cobol/parse_ante.h
index 105afe9..03cb0a0 100644
--- a/gcc/cobol/parse_ante.h
+++ b/gcc/cobol/parse_ante.h
@@ -103,7 +103,7 @@ void input_file_status_notify();
         }                                                               \
       location_dump("parse.c", __LINE__, "current", (Current));         \
       input_file_status_notify();                                       \
-      gcc_location_set( location_set(Current) );                        \
+      location_set(Current);                                            \
   } while (0)
 
 int yylex(void);
@@ -935,165 +935,11 @@ teed_up_names() {
   return name_queue_t::namelist_of( name_queue.peek() );
 }
 
-class tokenset_t {
-  // token_names is initialized from a generated header file. 
-  std::vector<const char *>token_names;  // position indicates token value
-  std::map <std::string, int> tokens;    // aliases
-  std::set<std::string> cobol_words;  // Anything in COBOL-WORDS may appear only once. 
- public:
-  static std::string
-  lowercase( const cbl_name_t name ) {
-    cbl_name_t lname;
-    std::transform(name, name + strlen(name) + 1, lname, ftolower);
-    return lname;
-  }
-  static std::string
-  uppercase( const cbl_name_t name ) {
-    cbl_name_t uname;
-    std::transform(name, name + strlen(name) + 1, uname, ftoupper);
-    return uname;
-  }
-
- public:
-  tokenset_t();
-  int find( const cbl_name_t name, bool include_intrinsics );
-
-  bool equate( const YYLTYPE& loc, int token,
-	       const cbl_name_t name, const cbl_name_t verb = "EQUATE") {
-    auto lname( lowercase(name) );
-    auto cw = cobol_words.insert(lname);
-    if( ! cw.second ) {
-      error_msg(loc, "COBOL-WORDS %s: %s may appear but once", verb, name);
-      return false;
-    }
-    auto p = tokens.find(lowercase(name));
-    bool fOK = p == tokens.end();
-    if( fOK ) { // name not already in use
-      tokens[lname] = token;
-      dbgmsg("%s:%d: %d has alias %s", __func__, __LINE__, token, name);
-    } else {
-      error_msg(loc, "%s: %s already defined as a token", verb, name);
-    }
-    return fOK;
-  }
-  bool undefine( const YYLTYPE& loc,
-		 const cbl_name_t name, const cbl_name_t verb = "UNDEFINE" ) {
-    auto lname( lowercase(name) );
-    auto cw = cobol_words.insert(lname);
-    if( ! cw.second ) {
-      error_msg(loc, "COBOL-WORDS %s: %s may appear but once", verb, name);
-      return false;
-    }
-
-    // Do not erase generic, multi-type tokens COMPUTATIONAL and BINARY_INTEGER.
-    if( binary_integer_usage_of(name) ) {
-      dbgmsg("%s:%d: generic %s remains valid as a token", __func__, __LINE__, name);
-      return true;
-    }
-
-    auto p = tokens.find(lname);
-    bool fOK = p != tokens.end();
-    if( fOK ) { // name in use
-      tokens.erase(p);
-    } else {
-      error_msg(loc, "%s: %s not defined as a token", verb, name);
-    }
-    dbgmsg("%s:%d: %s removed as a valid token name", __func__, __LINE__, name);
-    return fOK;
-  }
-  
-  bool substitute( const YYLTYPE& loc,
-		   const cbl_name_t extant, int token, const cbl_name_t name ) {
-    return
-      equate( loc, token, name, "SUBSTITUTE" )
-      &&
-      undefine( loc, extant, "SUBSTITUTE" );
-  }
-  bool reserve( const YYLTYPE& loc, const cbl_name_t name ) {
-    auto lname( lowercase(name) );
-    auto cw = cobol_words.insert(lname);
-    if( ! cw.second ) {
-      error_msg(loc, "COBOL-WORDS RESERVE: %s may appear but once", name);
-      return false;
-    }
-    tokens[lname] = -42;
-    return true;
-  }
-  int redefined_as( const cbl_name_t name ) {
-    auto lname( lowercase(name) );
-    if( cobol_words.find(lname) != cobol_words.end() ) {
-      auto p = tokens.find(lname);
-      if( p != tokens.end() ) {
-        return p->second;
-      }
-    }
-    return 0;
-  }
-  const char * name_of( int tok ) const {
-    tok -= (255 + 3);
-    gcc_assert(0 <= tok && size_t(tok) < token_names.size());
-    return tok < 0? "???" : token_names[tok];
-  }
-};
-
-class current_tokens_t {
-  tokenset_t tokens;
- public:
-  current_tokens_t() {}
-  int find( const cbl_name_t name, bool include_intrinsics ) {
-    return tokens.find(name, include_intrinsics);
-  }
-  bool equate( const YYLTYPE& loc, const cbl_name_t keyword, const cbl_name_t alias ) {
-    int token; 
-    if( 0 == (token = binary_integer_usage_of(keyword)) ) {
-      if( 0 == (token = keyword_tok(keyword)) ) {
-	error_msg(loc, "EQUATE %s: not a valid token", keyword);
-	return false;
-      }
-    }
-    auto name = keyword_alias_add(tokens.uppercase(keyword),
-				  tokens.uppercase(alias));
-    if( name != keyword ) {
-      error_msg(loc, "EQUATE: %s is already an alias for %s", alias, name.c_str());
-      return false;
-    } 
-    return tokens.equate(loc, token, alias);
-  }
-  bool undefine( const YYLTYPE& loc, cbl_name_t keyword ) {
-    return tokens.undefine(loc, keyword);
-  }
-  bool substitute( const YYLTYPE& loc, const cbl_name_t keyword, const cbl_name_t alias ) {
-    int token; 
-    if( 0 == (token = binary_integer_usage_of(keyword)) ) {
-      if( 0 == (token = keyword_tok(keyword)) ) {
-	error_msg(loc, "SUBSTITUTE %s: not a valid token", keyword);
-	return false;
-      }
-    }
-    auto name = keyword_alias_add(tokens.uppercase(keyword),
-				  tokens.uppercase(alias));
-    if( name != keyword ) {
-      error_msg(loc, "SUBSTITUTE: %s is already an alias for %s", alias, name.c_str());
-      return false;
-    } 
-
-    dbgmsg("%s:%d: %s (%d) will have alias %s", __func__, __LINE__, keyword, token, alias);
-    return tokens.substitute(loc, keyword, token, alias);
-  }
-  bool reserve( const YYLTYPE& loc, const cbl_name_t name ) {
-    return tokens.reserve(loc, name);
-  }
-  int redefined_as( const cbl_name_t name ) {
-    return tokens.redefined_as(name);
-  }
-  const char * name_of( int tok ) const {
-    return tokens.name_of(tok);
-  }
-} tokens;
+#define cdf_tokens cdf_current_tokens()
 
 int
 redefined_token( const cbl_name_t name ) {
-  return tokens.redefined_as(name);
+  return cdf_tokens.redefined_as(name);
 }
 
 struct file_list_t {
@@ -1475,7 +1321,6 @@ class prog_descr_t {
       }
     }
   } locale;
-  cbl_call_convention_t call_convention;
   cbl_options_t options;
 
   explicit prog_descr_t( size_t isymbol )
@@ -1484,9 +1329,7 @@ class prog_descr_t {
     , paragraph(NULL)
     , section(NULL)
     , collating_sequence(NULL)
-  {
-    call_convention = current_call_convention();
-  }
+  {}
 
  std::set<std::string> external_targets() {
    std::set<std::string> externals;
@@ -1575,24 +1418,13 @@ static cbl_label_t *  implicit_section();
 
 class program_stack_t : protected  std::stack<prog_descr_t> {
   struct pending_t {
-    cbl_call_convention_t call_convention;
     bool initial;
-    pending_t()
-      : call_convention(cbl_call_convention_t(0))
-      , initial(false)
-    {}
+    pending_t() : initial(false) {}
   } pending;
  public:
-  cbl_call_convention_t
-  pending_call_convention( cbl_call_convention_t convention ) {
-    return pending.call_convention = convention;
-  }
   bool pending_initial() { return pending.initial = true; }
 
   void push( prog_descr_t descr ) {
-    cbl_call_convention_t call_convention = cbl_call_cobol_e;
-    if( !empty() ) call_convention = top().call_convention;
-    descr.call_convention = call_convention;
     std::stack<prog_descr_t>& me(*this);
     me.push(descr);
   }
@@ -1618,9 +1450,6 @@ class program_stack_t : protected  std::stack<prog_descr_t> {
   }
 
   void apply_pending() {
-    if( size() == 1 && 0 != pending.call_convention ) {
-      top().call_convention = pending.call_convention;
-  }
     if( pending.initial ) {
       auto e = symbol_at(top().program_index);
       auto prog(cbl_label_of(e));
@@ -2027,19 +1856,6 @@ static class current_t {
     return programs.top().options.default_round = mode;
   }
 
-  cbl_call_convention_t
-  call_convention() {
-    return programs.empty()? cbl_call_cobol_e : programs.top().call_convention;
-  }
-  cbl_call_convention_t
-  call_convention( cbl_call_convention_t convention) {
-    if( programs.empty() ) {
-      return programs.pending_call_convention(convention);
-    }
-    auto& prog( programs.top() );
-    return prog.call_convention = convention;
-  }
-
   const char *
   locale() {
     return programs.empty()? NULL : programs.top().locale.os_name;
@@ -2137,6 +1953,7 @@ static class current_t {
    * ISO, in new_program.
    */
   std::set<std::string>  end_program() {
+    cbl_enabled_exceptions_t& enabled_exceptions( cdf_enabled_exceptions() );
     if( enabled_exceptions.size() ) {
       declaratives_evaluate();
     }
@@ -2427,15 +2244,6 @@ current_rounded_mode( cbl_round_t rounded) {
 #endif
 static cbl_round_t current_rounded_mode( int token );
 
-cbl_call_convention_t
-current_call_convention() {
-  return current.call_convention();
-}
-cbl_call_convention_t
-current_call_convention( cbl_call_convention_t convention) {
-  return current.call_convention(convention);
-}
-
 size_t program_level() { return current.program_level(); }
 
 static size_t constant_index( int token );
@@ -2909,17 +2717,6 @@ group_attr( const cbl_field_t * field ) {
   return p->attr;
 }
 
-static struct symbol_elem_t *
-field_of( const char F[], int L, const char name[] ) {
-  struct symbol_elem_t *e = symbol_field(PROGRAM, 0, name);
-  if( !e ) {
-    cbl_internal_error("%s:%d: no symbol '%s' found", F, L, name);
-  }
-  assert( procedure_div_e != current_division  );
-  return e;
-}
-#define field_of( F ) field_of(__func__, __LINE__, (F))
-
 static struct cbl_field_t *
 field_add( const YYLTYPE& loc, cbl_field_t *field ) {
   switch(current_data_section) {
@@ -3696,18 +3493,18 @@ goodnight_gracie() {
 
 // false after USE statement, to enter Declarative with EC intact. 
 static bool statement_cleanup = true;
+static YYLTYPE current_location;
 
 static void statement_epilog( int token );
 
 const char * keyword_str( int token );
 
-static YYLTYPE current_location;
-
 const YYLTYPE& cobol_location() { return current_location; }
 
-static inline YYLTYPE
+static inline void
 location_set( const YYLTYPE& loc ) {
-  return current_location = loc;
+  current_location = loc;
+  gcc_location_set(loc);
 }
 
 static void statement_begin( const YYLTYPE& loc, int token );
diff --git a/gcc/cobol/scan.l b/gcc/cobol/scan.l
index 2fc4aea..ba4c044 100644
--- a/gcc/cobol/scan.l
+++ b/gcc/cobol/scan.l
@@ -83,10 +83,13 @@ NONWORD   [^[:alnum:]$_-]+
 
 SPC  [[:space:]]+
 OSPC [[:space:]]*
+BLANK  [[:blank:]]+
+OBLANK [[:blank:]]*
 EOL  \r?\n
 BLANK_EOL   [[:blank:]]*{EOL}
 BLANK_OEOL  [[:blank:]]*{EOL}?
 
+PICTURE   [^[:space:]]+
 
 DOTSEP    [.]+[[:space:]]
 DOTEOL    [[:blank:]]*[.]{BLANK_EOL}
@@ -160,7 +163,7 @@ COMMA    [,;][[:blank:]]*
 ISNT	 (IS{SPC})?NOT
 
 
-COMMENTARY DATE-COMPILED|DATE-WRITTEN|INSTALLATION|SECURITY
+COMMENTARY AUTHOR|DATE-COMPILED|DATE-WRITTEN|INSTALLATION|SECURITY
 
 SORT_MERGE    SORT(-MERGE)?
 
@@ -174,7 +177,7 @@ SIZE_ERROR (ON[[[:space:]]+)?SIZE[[:space:]]+ERROR
 VARTYPE  NUMERIC|ALPHABETIC|ALPHABETIC_LOWER|ALPHABETIC_UPPER|DBCS|KANJI
 NAMTYP   {NAME}|{VARTYPE}
 
-NL       [[:blank:]]*\r?\n[[:blank:]]*
+NL       [[:blank:]]*{EOL}[[:blank:]]*
 
 PUSH_FILE \f?[#]FILE{SPC}PUSH{SPC}[^\f]+\f
 POP_FILE  \f?[#]FILE{SPC}POP\f
@@ -182,7 +185,7 @@ LINE_DIRECTIVE ^[#]line{SPC}[[:alnum:]]+{SPC}[""''].+\n
 
 %x procedure_div ident_state addr_of function classify
 %x program_id_state comment_entries
-%x author_state date_state field_level field_state dot_state
+%x date_state field_level field_state dot_state
 %x numeric_state name_state
 %x quoted1 quoted2 quoteq
 %x picture picture_count integer_count
@@ -238,30 +241,23 @@ WORKING-STORAGE{SPC}SECTION {
                 yy_push_state(field_state);
                 return WORKING_STORAGE_SECT; }
 LOCAL-STORAGE{SPC}SECTION {
-                yy_push_state(field_state);
-                return LOCAL_STORAGE_SECT; }
-WORKING-STORAGE {
-                return WORKING_STORAGE; }
-LOCAL-STORAGE {
-                return LOCAL_STORAGE; }
-SCREEN {
-                return SCREEN; }
+                  yy_push_state(field_state);
+                  return LOCAL_STORAGE_SECT; }
+WORKING-STORAGE	{ return WORKING_STORAGE; }
+LOCAL-STORAGE 	{ return LOCAL_STORAGE; }
+SCREEN 		{ return SCREEN; }
 
 LINKAGE{SPC}SECTION {
                 yy_push_state(field_state);
                 return LINKAGE_SECT; }
 
-FUNCTION-ID		{ yy_push_state(ident_state);
-                  	  yy_push_state(program_id_state);
-                  	  yy_push_state(name_state); return FUNCTION; }
-
-PROGRAM-ID		{ yy_push_state(ident_state);
-                  	  yy_push_state(program_id_state);
-                  	  yy_push_state(name_state); return PROGRAM_ID; }
+FUNCTION-ID{OSPC}{DOTSEP}?	{ yy_push_state(ident_state);
+                	  	  yy_push_state(program_id_state);
+                  		  yy_push_state(name_state); return FUNCTION; }
 
-PROGRAM-ID/{DOTEOL}	{ yy_push_state(ident_state);
-                  	  yy_push_state(name_state);
-                  	  yy_push_state(dot_state); return PROGRAM_ID; }
+PROGRAM-ID{OSPC}{DOTSEP}?	{ yy_push_state(ident_state);
+	                  	  yy_push_state(program_id_state);
+        	          	  yy_push_state(name_state); return PROGRAM_ID; }
 
 PROCEDURE{SPC}DIVISION	{ yy_push_state(procedure_div);
                                   return PROCEDURE_DIV; }
@@ -272,30 +268,18 @@ PROCEDURE{SPC}DIVISION	{ yy_push_state(procedure_div);
 }
 
 <ident_state>{
+  {BLANK_OEOL}
   ID(ENTIFICATION)?{SPC}DIVISION	{ myless(0); yy_pop_state(); }
+  (ENVIRONMENT|DATA|PROCEDURE){SPC}DIVISION {
+					  myless(0); yy_pop_state(); }
+  OPTIONS				{ myless(0); yy_pop_state(); }
+
   AS{SPC}[""] 	{ yy_push_state(quoted2); return AS; }
   AS{SPC}[''] 	{ yy_push_state(quoted1); return AS; }
   IS		{ pop_return IS; }
 
-  OPTIONS		{ yy_pop_state(); myless(0); }
-  [[:blank:]]*(ENVIRONMENT|DATA|PROCEDURE){SPC}DIVISION/[[:space:].] {
-                                  yy_pop_state(); myless(0); }
-  [[:blank:]]*AUTHOR[[:blank:].]+{EOL}? {
-				  // Might not have an EOL, but stop on one.
-                                  yy_push_state(author_state); }
-
-  {DOTEOL}
-
   {COMMENTARY} { BEGIN(comment_entries); }
 }
-<author_state>{
-    [[:blank:]]+
-    ^{BLANK_EOL}
-    [^\r\n]+                   { yy_pop_state();
-                                  yylval.string = xstrdup(yytext);
-                               }
-}
-
 
 <INITIAL>{
   COBOL		{ return COBOL; }
@@ -307,6 +291,15 @@ PROCEDURE{SPC}DIVISION	{ yy_push_state(procedure_div);
                             yy_push_state(field_state);
                             yy_set_bol(1);
                             myless(0); }
+
+  END{SPC}PROGRAM        { yy_push_state(name_state);
+                                  return program_level() > 1?
+                                         END_SUBPROGRAM : END_PROGRAM; }
+
+  END{SPC}FUNCTION        { yy_push_state(name_state);
+                                  return program_level() > 1?
+                                         END_SUBPROGRAM /*invalid*/ :
+                                         END_FUNCTION; }
 }
 
 <INITIAL,procedure_div,cdf_state>{
@@ -444,6 +437,11 @@ STDOUT		{ return STDOUT; }
 STDERR		{ return STDERR; }
 SYSERR		{ return STDERR; }
 
+ARGUMENT-NUMBER  	{ return ARGUMENT_NUMBER; }
+ARGUMENT-VALUE   	{ return ARGUMENT_VALUE; }
+ENVIRONMENT-NAME 	{ return ENVIRONMENT_NAME; }
+ENVIRONMENT-VALUE	{ return ENVIRONMENT_VALUE; }
+
 CANCEL		{ return CANCEL; }
 COMMIT		{ return COMMIT; }
 COMMON		{ return COMMON; }
@@ -541,7 +539,7 @@ SECTION{SPC}[+-]?{INTEGERZ}/{OSPC}{DOTSEP}	{
 				  auto eotext = yytext + yyleng;
 				  auto p = std::find_if(yytext, eotext, fisspace);
 				  p = std::find_if(p, eotext, nonspace);
-				  yylval.string = p;
+				  yylval.string = xstrdup(p);
 				  return SECTION;
 				}
 
@@ -968,7 +966,9 @@ USE({SPC}FOR)?		{ return USE; }
                               return NUMSTR;
                            }
 
-  PIC(TURE)?({SPC}IS)?[[:space:]]{BLANK_OEOL} {
+  PIC(TURE)?({SPC}IS)?{SPC}{PICTURE} {
+				  auto pos = validate_picture();
+				  myless(pos);
 				  yy_push_state(picture); return PIC; }
 
   ANY				{ return ANY; }
@@ -1150,7 +1150,7 @@ USE({SPC}FOR)?		{ return USE; }
 	    		  yy_push_state(hex_state); }
   N?X{nonseq}		{ dbgmsg("invalid hexadecimal value: %s", yytext);
 			  return NO_CONDITION; }
-  [[:blank:]]*\r?\n     {}
+  [[:blank:]]*{EOL}     {}
 
   WORKING-STORAGE{SPC}SECTION { return WORKING_STORAGE_SECT; }
   LOCAL-STORAGE{SPC}SECTION   { return LOCAL_STORAGE_SECT; }
@@ -1220,7 +1220,7 @@ USE({SPC}FOR)?		{ return USE; }
   {NP}V?/[,.]?     { yylval.number = ndigit(yyleng);     return picset(PIC_P); }
   {N9}*V/{N9}*     { yylval.number = ndigit(yyleng - 1); return picset(NINEV); }
   {N9}/{N9}*[,.]?  { yylval.number = ndigit(yyleng);     return picset(NINES); }
-  P+/[,.]?\r?\n    { yylval.number = yyleng;             return picset(PIC_P); }
+  P+/[,.]?{EOL}    { yylval.number = yyleng;             return picset(PIC_P); }
 
   1{1,31}/({COUNT}|[(]{NAME}[)]) {
                          yy_push_state(picture_count);
@@ -1319,7 +1319,7 @@ USE({SPC}FOR)?		{ return USE; }
   [""]{SPC}[&]{SPC}[""''] {
 		        if( yytext[yyleng - 1] == '\'' ) BEGIN(quoted1);
 		      }
-  [""]-{OSPC}(\r?\n{OSPC})+[""] /* continue ...  */
+  [""]-{OSPC}({EOL}{OSPC})+[""] /* continue ...  */
   [""]                {
       			char *s = xstrdup(tmpstring? tmpstring : "\0");
                         yylval.literal.set_data(strlen(s), s);
@@ -1336,7 +1336,7 @@ USE({SPC}FOR)?		{ return USE; }
   ['']{SPC}[&]{SPC}[""''] {
 		        if( yytext[yyleng - 1] == '"' ) BEGIN(quoted2);
 		      }
-  ['']-{OSPC}(\r?\n{OSPC})+[''] /* continue ...  */
+  ['']-{OSPC}({EOL}{OSPC})+[''] /* continue ...  */
   ['']                {
       			char *s = xstrdup(tmpstring? tmpstring : "\0");
                         yylval.literal.set_data(strlen(s), s);
@@ -1384,45 +1384,36 @@ USE({SPC}FOR)?		{ return USE; }
 }
 
 <program_id_state>{
-  ^[[:blank:]]+
-  ^{BLANK_EOL}
+  {BLANK_OEOL}
   (IS)?[[:space:]]
+  AS/{SPC} 	{ myless(0); yy_pop_state(); } /* => ident_state */
 
-  COMMON/[.]|{SPC}[[:alnum:].] 	{ return COMMON; }
-  INITIAL/[.]|{SPC}[[:alnum:].]	{ return INITIAL_kw; }
-  RECURSIVE			{ return RECURSIVE; }
-  PROGRAM/[.]|{SPC}[[:alnum:].] { return PROGRAM_kw; }
-
-  INITIAL	{ pop_return INITIAL_kw; }
-  COMMON	{ pop_return COMMON; }
-  PROGRAM 	{ pop_return PROGRAM; }
+  INITIAL	{ return INITIAL_kw; }
+  COMMON	{ return COMMON; }
+  RECURSIVE	{ return RECURSIVE; }
+  PROGRAM 	{ return PROGRAM_kw; }
 
-  AS/{SPC} 	{ myless(0); yy_pop_state(); } /* => ident_state */
-  [[:blank:]]*{DOTSEP}[[:blank:].]+{EOL} { pop_return '.'; }
-  {DOTEOL}		 		 { pop_return '.'; }
+  {DOTSEP}	{ pop_return '.'; }
 }
 
-<name_state>{
-  ^[[:blank:]]+
-  ^{BLANK_EOL}
-  {NAME}		|
-  {NAME}/{OSPC}[.]	{ yy_pop_state();
-    			  yylval.string = xstrdup(yytext); return NAME; }
+<name_state>{           /* Either pop from here, or let the quoted state pop */
+  {BLANK_OEOL}
 
+  {NAME}	        { yy_pop_state();
+			  yylval.string = xstrdup(yytext);
+			  return NAME; 
+			}
   Z?['']                { yylval.literal.set_prefix(yytext, yyleng-1);
-			  yy_push_state(quoted1); }
+			  BEGIN(quoted1); }
   Z?[""]                { yylval.literal.set_prefix(yytext, yyleng-1);
-			  yy_push_state(quoted2); }
-
-  [.]/[[:blank:]]+.     { return *yytext; }
+			  BEGIN(quoted2); }
 
-  [[:blank:]]*{DOTSEP}[[:blank:].]+{EOL} {
-			  yy_pop_state(); myless(0); }
-  {DOTEOL}             	{ yy_pop_state(); myless(0); }
+  .             	{ myless(0); yy_pop_state();
+			  /* Should not happen for valid inputs. */ }
 }
 <dot_state>{
   [[:blank:]]*[.][[:blank:].]+{EOL} 	{ pop_return '.'; }
-  [[:blank:]]*[.]			{ pop_return '.'; }
+  [[:blank:]]*[.]+			{ pop_return '.'; }
 }
 
 <date_state>{
@@ -1645,9 +1636,9 @@ B-SHIFT-RC
 
   FUNCTION 			{ yy_push_state(function); return FUNCTION; }
 
-  SECTION{OSPC}[.]{SPC}/USE[[:space:]]	{ yylval.string = NULL; return SECTION; }
+  SECTION{OSPC}[.]+{SPC}/USE[[:space:]]	{ yylval.string = NULL; return SECTION; }
 
-  [.]({SPC}(EJECT|SKIP[123]))*{SPC}EXIT{OSPC}/{DOTSEP} {
+  [.]+({SPC}(EJECT|SKIP[123]))*{SPC}EXIT{OSPC}/{DOTSEP} {
                    // EXIT format-1 is a "continue" statement
                  }
   {NAME}/{OSPC}{DOTSEP} {
@@ -1682,16 +1673,17 @@ B-SHIFT-RC
 		    p += 2;
 		    while( ISSPACE(*p) ) p++;
 		    cbl_name_t name2;
-		    std::transform( p, p + sizeof(name2), name2,
-				    []( char ch ) {
-				      switch(ch) {
-				      case '-':
-				      case '_': return ch;
-				      default:
-					if( ISALNUM(ch) ) return ch;
-				      }
-				      return '\0';
-				    } );
+                    const char *pend = p + sizeof(name2);
+                    char *pout = name2;
+                    while( p < pend ) {
+                      char ch = *p++;
+                      if( ISALNUM(ch) || ch == '-' || ch == '_' )  {
+                        *pout++ = ch;
+                      } else {
+                        *pout++  = '\0';
+                        break;
+                      }
+                    }
 		    symbol_elem_t *e = symbol_file(PROGRAM, name2);
 		    /*
 		     * For NAME IN FILENAME, we want the parser to handle it.
@@ -2052,7 +2044,7 @@ BASIS		{ yy_push_state(basis); return BASIS; }
 		  return symbol_file(PROGRAM, yytext)? FILENAME : NAME;
 		}
   [[:blank:]]+
-  \r?\n		{ yy_pop_state(); }
+  {EOL}		{ yy_pop_state(); }
 }
 
 <raising>{
@@ -2073,49 +2065,62 @@ BASIS		{ yy_push_state(basis); return BASIS; }
 		    if( include_debug() ) myless(7);
 		  }
 		}
-  ^[ ]*>>{OSPC}IF		{ yy_push_state(cdf_state); return CDF_IF; }
-  ^[ ]*>>{OSPC}ELSE	 	{ return CDF_ELSE; }
-  ^[ ]*>>{OSPC}END-IF	 	{ return CDF_END_IF; }
+  ^[ ]*>>{OBLANK}IF		{ yy_push_state(cdf_state); return CDF_IF; }
+  ^[ ]*>>{OBLANK}ELSE	 	{ return CDF_ELSE; }
+  ^[ ]*>>{OBLANK}END-IF	 	{ return CDF_END_IF; }
 
-  ^[ ]*[$]{OSPC}IF		{ if( ! dialect_mf() )  {
+  ^[ ]*[$]{OBLANK}IF		{ if( ! dialect_mf() )  {
 				    dialect_error(yylloc,  yytext, "mf");
 				  }
 				  yy_push_state(cdf_state); return CDF_IF; }
-  ^[ ]*[$]{OSPC}ELSE	 	{ if( ! dialect_mf() )  {
+  ^[ ]*[$]{OBLANK}ELSE	 	{ if( ! dialect_mf() )  {
 				    dialect_error(yylloc,  yytext, "mf");
 				  }
 				  return CDF_ELSE; }
-  ^[ ]*[$]{OSPC}END	 	{ if( ! dialect_mf() ) {
+  ^[ ]*[$]{OBLANK}END	 	{ if( ! dialect_mf() ) {
 				    dialect_error(yylloc,  yytext, "mf");
 				  }
 				  return CDF_END_IF; }
 
-  ^[ ]*[$]{OSPC}SET({SPC}CONSTANT)? {
+  ^[ ]*[$]{OBLANK}SET({SPC}CONSTANT)? {
 				  if( ! dialect_mf() ) dialect_error(yylloc, yytext, "mf");
 				  yy_push_state(cdf_state); return CDF_DEFINE; }
 
-  ^[ ]*>>{OSPC}EVALUATE		{ return CDF_EVALUATE; }
-  ^[ ]*>>{OSPC}WHEN		{ return CDF_WHEN; }
-  ^[ ]*>>{OSPC}END-EVALUATE	{ return CDF_END_EVALUATE; }
+  ^[ ]*>>{OBLANK}EVALUATE		{ return CDF_EVALUATE; }
+  ^[ ]*>>{OBLANK}WHEN		{ return CDF_WHEN; }
+  ^[ ]*>>{OBLANK}END-EVALUATE	{ return CDF_END_EVALUATE; }
+
+  ^[ ]*>>{OBLANK}CALL-CONVENTION{BLANK}C		{ return CALL_VERBATIM; }
+  ^[ ]*>>{OBLANK}CALL-CONVENTION{BLANK}COBOL	{ return CALL_COBOL; }
+  ^[ ]*>>{OBLANK}CALL-CONVENTION{BLANK}VERBATIM	{ return CALL_VERBATIM; }
+
+  ^[ ]*>>{OBLANK}DEFINE	{ yy_push_state(cdf_state); return CDF_DEFINE; }
+  ^[ ]*>>{OBLANK}DISPLAY 	{ return CDF_DISPLAY; }
+  ^[ ]*>>{OBLANK}TURN 	{ yy_push_state(exception); return TURN; }
+  ^[ ]*>>{OBLANK}COBOL-WORDS	{ yy_push_state(cobol_words); return COBOL_WORDS; }
 
-  ^[ ]*>>{OSPC}CALL-CONVENTION{SPC}C		{ return CALL_VERBATIM; }
-  ^[ ]*>>{OSPC}CALL-CONVENTION{SPC}COBOL	{ return CALL_COBOL; }
-  ^[ ]*>>{OSPC}CALL-CONVENTION{SPC}VERBATIM	{ return CALL_VERBATIM; }
+  ^[ ]*>>{OBLANK}SOURCE{BLANK}FORMAT	{ return SOURCE_FORMAT; }
 
-  ^[ ]*>>{OSPC}DEFINE	{ yy_push_state(cdf_state); return CDF_DEFINE; }
-  ^[ ]*>>{OSPC}DISPLAY 	{ return CDF_DISPLAY; }
-  ^[ ]*>>{OSPC}TURN 	{ yy_push_state(exception); return TURN; }
-  ^[ ]*>>{OSPC}COBOL-WORDS	{ yy_push_state(cobol_words); return COBOL_WORDS; }
+  ^[ ]*>>{OBLANK}PUSH 		{ return CDF_PUSH; }
+  ^[ ]*>>{OBLANK}POP 		{ return CDF_POP; }
 
-  ^[ ]*>>{OSPC}{NAME} 	{
+  ^[ ]*>>{OBLANK}{NAME} 	{
 			  error_msg(yylloc, "unknown CDF token: %s", yytext);
 			}
 
+
   OTHER		{ return OTHER; }
   OVERRIDE	{ return OVERRIDE; }
   PARAMETER	{ return PARAMETER_kw; }
   THRU		{ return THRU; }
   TRUE		{ return TRUE_kw; }
+
+  ALL			{ return ALL; }
+  CALL-CONVENTION	{ return CALL_CONVENTION; }
+  COBOL-WORDS		{ return COBOL_WORDS; }
+  DEFINE		{ return CDF_DEFINE; }
+  SOURCE{BLANK}FORMAT	{ return SOURCE_FORMAT; }
+
 }
 
 <cobol_words>{
@@ -2165,10 +2170,10 @@ BASIS		{ yy_push_state(basis); return BASIS; }
 
 <*>OR 				{ return OR; }
 <*>AND				{ return AND; }
-<*>{DOTSEP}[[:blank:].]+$	{ return '.'; }
+<*>{DOTSEP}			{ return '.'; }
 <*>[().=*/+&-]			{ return *yytext; }
 <*>[[:blank:]]+
-<*>\r?\n
+<*>{EOL}
 
 <*>{
   {COMMA}
@@ -2369,7 +2374,7 @@ BASIS		{ yy_push_state(basis); return BASIS; }
   POINTER	{ return POINTER; }
   POSITIVE	{ return POSITIVE; }
   PROCEDURE	{ return PROCEDURE; }
-  PROGRAM	{ return PROGRAM; }
+  PROGRAM	{ return PROGRAM_kw; }
   PROGRAM-ID	{ return PROGRAM_ID; }
   PROPERTY	{ return PROPERTY; }
   PROTOTYPE	{ return PROTOTYPE; }
@@ -2411,7 +2416,7 @@ BASIS		{ yy_push_state(basis); return BASIS; }
   SCREEN	{ return SCREEN; }
   SD		{ return SD; }
   SEARCH	{ return SEARCH; }
-  SECTION	{ return SECTION; }
+  SECTION	{ yylval.string = NULL; return SECTION; }
   SELECT	{ return SELECT; }
   SENTENCE	{ return SENTENCE; }
   SEPARATE	{ return SEPARATE; }
diff --git a/gcc/cobol/scan_ante.h b/gcc/cobol/scan_ante.h
index ea304ba..31093a6 100644
--- a/gcc/cobol/scan_ante.h
+++ b/gcc/cobol/scan_ante.h
@@ -149,7 +149,7 @@ numstr_of( const char string[], radix_t radix = decimal_e ) {
     }
     auto nx = std::count_if(input, p, fisdigit);
     if( 36 < nx ) {
-      error_msg(yylloc, "significand of %s has more than 36 digits (%zu)", input, nx);
+      error_msg(yylloc, "significand of %s has more than 36 digits (%td)", input, nx);
       return NO_CONDITION;
     }
 
@@ -356,6 +356,10 @@ static void level_found() {
   if( scanner_normal() ) parsing.need_level(false);
 }
 
+/*
+ * Trim the scanned location by the amount about to re-scanned. 
+ * Must be a macro because it expands yyless. 
+ */
 #define myless(N)				\
   do {						\
     auto n(N);					\
@@ -486,7 +490,8 @@ trim_location( int nkeep) {
          (fmt_size_t)nline, (fmt_size_t)rescan.size());
   if( nline ) {
     gcc_assert( yylloc.first_line + nline <= yylloc.last_line );
-    yylloc.last_line =- int(nline);
+    yylloc.last_line -= int(nline);
+    gcc_assert( yylloc.first_line <= yylloc.last_line );
     char *p = static_cast<char*>(memrchr(rescan.p, '\n', rescan.size()));
     yylloc.last_column = rescan.pend - ++p;
     return;
@@ -604,7 +609,9 @@ static const std::map <std::string, bint_t > binary_integers {
 
 static int
 binary_integer_usage( const char name[]) {
-  cbl_name_t uname = {};
+  // uname can't be cbl_name_t, because at this point name[] might have more
+  // than sizeof(cbl_name_t) characters.  The length check comes later.
+  char *uname = xstrdup(name);
   std::transform(name, name + strlen(name), uname, ftoupper);
 
   dbgmsg("%s:%d: checking %s in %zu keyword_aliases",
@@ -623,6 +630,7 @@ binary_integer_usage( const char name[]) {
   yylval.computational.signable = p->second.signable;
   dbgmsg("%s:%d: %s has type %d", __func__, __LINE__,
 	 uname, p->second.type );
+  free(uname);
   return p->second.token;
 }
       
@@ -689,6 +697,387 @@ picset( int token ) {
   return token;
 }
 
+/**
+## Script and data to produce picture_t::followers.
+## Based on ISO Table 10. 
+#! /usr/bin/awk -f
+
+BEGIN  {
+  str = "B0/ , . + +- +- CR/DB cs cs Z* Z* + + cs cs 9 AX S V P P 1 N E"
+  split(str, cols)
+}
+
+$1 ~ /CR|DB|cs/ { next }
+
+0 && !nlines++ {
+  for( i=0; i < length(cols); i++ ) {
+    print i, cols[i], "'" $i "'"
+  }
+}
+
+$field == "x" {
+  if( ! nout++ ) {
+    printf "%2d: %5s: \"", field, cols[field - 1]
+  }
+
+  gsub(/^ +| +$/, "", $1) 
+  printf "%s", $1
+}
+
+END {
+  if( ! nout++ ) {
+    printf "%2d: %5s: \"", field, cols[field - 1]
+  }
+  print "\""
+}
+
+B  x x x - x - - x - x x x x x x x x - x - x - x
+0  x x x - x - - x - x x x x x x x x - x - x - x
+/  x x x - x - - x - x x x x x x x x - x - x - x
+,  x x x - x - - x - x x x x x x x - - x - x
+.  x x - - x - - x - x - x - x - x
++  - - - - - - - - - - - - - - - - - - - - - - - x
++
+–
++  x x x - - - - x x x x - - x x x - - x x x
+CR x x x - - - - x x x x - - x x x - - x x x
+DB x x x - - - - x x x x - - x x x - - x x x
+cs - - - - x
+cs x x x - x - - - - x x - - - - x - - x x x
+
+Z  x x - - x - - x - x
+*  x x - - x - - x - x
+Z  x x x - x - - x - x x - - - - - - - x - x
+*  x x x - x - - x - x x - - - - - - - x - x
++  x x - - - - - x - - - x
+–  x x - - - - - x - - - x
++  x x x - - - - x - - - x x - - - - - x
+–  x x x - - - - x - - - x x - - - - - x
+cs x x - - x - - - - - - - - x
+cs x x x - x - - - - - - - - x x - - - x
+
+9  x x x x x - - x - x - x - x - x x x x - x - - x
+A  x - - - - - - - - - - - - - - x x
+X  x - - - - - - - - - - - - - - x x
+S 
+V  x x - - x - - x - x - x - x - x - x - x
+P  x x - - x - - x - x - x - x - x - x - x
+P  - - - - x - - x - - - - - - - - - x x - x
+1  - - - - - - - - - - - - - - - - - - - - - x
+N  x - - - - - - - - - - - - - - - - - - - - - x
+E  x x x - x - - - - - - - - - - x
+**/
+
+class picture_t {
+  static const char dot = '.', comma = ',';
+
+  typedef std::vector<std::string> followings_t;
+  static const std::map <char, followings_t> followers;
+  
+  const char * const begin;
+  const char *p, *pend; 
+  size_t pos;
+  struct exclusions_t { // Nonzero if set, > 1 is false.
+    // crdb means CR/DB or +/-.
+    // pluses means 2 or more consecutive '+'.
+    // minuses means 2 or more consecutive '-'.
+    // "21) The symbol 'Z' and the symbol '*' are mutually exclusive "
+    // stars means '*' or Z.
+    unsigned short int crdb, currency, dot, pluses, minuses, stars, zzz;
+    exclusions_t()
+      : crdb(0), currency(0), dot(0), pluses(0), minuses(0), stars(0)
+    {}
+  } exclusions;
+  YYLTYPE loc;
+  
+  bool is_crdb() const { // input must be uppercase for CR/DB
+    if( p[0] == 'C' || p[0] == 'D' ) {
+      char input[3] = { p[0], p[1] };
+      return ( 0 == strcmp(input, "CR") || 0 == strcmp(input, "DB") );
+    }
+    return false;
+  }
+
+  const char * match_paren( const char *paren ) const {
+    gcc_assert(paren[0] == '(');  // start with opening paren
+    paren = std::find_if( paren, pend,
+                          []( char ch ) {
+                            return ch == '(' || ch == ')';
+                          } );
+    if( *paren == '(' ) return nullptr; // no nesting
+    if( paren == pend ) return nullptr;
+    return ++paren;
+  }
+
+  const char * next_not( char ch ) const {
+    return std::find_if( p, pend,
+                         [ch = TOUPPER(ch)]( char next ) {
+                           return ch != next;
+                         } );
+  }
+
+  const char * valid_next( const char *p, const std::string& valid ) const {
+    if( p == pend || p + 1 == pend ) return pend;
+    if( p[1] == '(' ) {
+      return match_paren(++p);
+    }
+    auto pv = std::find(valid.begin(), valid.end(), TOUPPER(p[1]));
+    return pv != valid.end()? ++p : nullptr;
+  }
+  const char * valid_next( const char *p,
+                           bool first = true, char ch = '\0' ) const {
+    if( p == pend || p + 1 == pend ) return pend;
+    if( p[0] == '(' ) {
+      if( (p = match_paren(p)) == nullptr ) return nullptr;
+    }
+    if( p[0] == '(' ) return nullptr;  // consecutive parentheses
+    
+    int index = first? 0 : 1;
+    if( !ch ) ch = *p;   // use current character unless overridden
+    auto valid = followers.find(TOUPPER(ch));
+    if( valid == followers.end() ) {
+      YYLTYPE loc(yylloc);
+      loc.first_column += int(p - begin);
+      error_msg( loc, "PICTURE: strange character %qc, giving up", ch );
+      return nullptr;
+    }
+    return valid_next(p, valid->second[index]);
+  }
+
+  const char * start() { // start modifies exclusions, but not p
+    auto pnext = p;
+
+    switch(TOUPPER(p[0])) {
+    case comma: case dot:
+      // use decimal_is_comma()
+      //  4:     .: "B0/,+Z*+-9E"
+      exclusions.dot++;
+      pnext = valid_next(p, "B0/,+Z*+-9E");
+      break;
+    case '+': case '-':
+      //  6:    +-: "B0/,.Z*Z*9VPPE"
+      exclusions.crdb++;
+      pnext = next_not(p[0]);
+      if( p + 1 < pnext ) {
+        exclusions.pluses++;
+      }
+      pnext = valid_next(--pnext, "B0/,.Z*Z*9VPPE");
+      break;
+    case 'Z': case '*': 
+      exclusions.stars++;
+      pnext = next_not(p[0]);
+      break;
+    case 'S':
+      // 19:     S: "9VP"
+      pnext = valid_next(p, "9VP");
+      break;
+    }
+
+    /*
+     * "For fixed editing sign control, the currency symbol, when used, shall
+     * be either the leftmost symbol in character-string-1, optionally preceded
+     * by one of the symbols '+' or '-' "
+     */
+    if( pnext ) {
+      if( p == pnext || p[0] == '+' || p[0] == '-' ) {
+        if( symbol_currency(*pnext) ) {
+          exclusions.currency++;
+          pnext = next_not(*pnext);
+          pnext = valid_next(--pnext, true, '$');
+        }
+      }
+    }
+    
+    return pnext;
+  }
+
+  const char * next() { // modify state; do not modify position
+    auto pnext = p;
+    auto loc(picture_t::loc);
+    loc.first_column += int(p - begin);
+
+    if( is_crdb() ) {
+      if( exclusions.crdb++ ) {
+        error_msg( loc, "PICTURE: CR/DB and %c/%c may appear only once", '+', '-' );
+        return nullptr;
+      }
+      if( p + 2 != pend ) { 
+        error_msg( loc, "PICTURE: CR/DB must appear at the end" );
+        return nullptr;
+      }
+      return pend;
+    }
+
+    if( symbol_currency(p[0]) ) {
+      if( false && exclusions.currency++ ) { // not enforced
+        error_msg( loc, "PICTURE: CURRENCY SYMBOL sequence may appear at most once" );
+        return nullptr;
+      }
+      return valid_next(p, ! exclusions.dot, '$');
+    }
+
+    switch(TOUPPER(p[0])) {
+    case '(':
+      return match_paren(p);
+      break;
+    case 'B': case '0': case '/':
+      pnext = valid_next(p);
+      break;
+    case comma: 
+      if( decimal_is_comma() ) {
+        if( exclusions.dot++ ) {
+          error_msg( loc, "PICTURE: %qc: may appear at most once", p[0] );
+          return nullptr;
+        }
+        pnext = valid_next(p, true, dot);
+      } else {
+        pnext = valid_next(p);
+      }
+      break;
+    case dot: 
+      if( p + 1 == pend ) {
+        pnext = pend;
+      } else {
+        if( decimal_is_comma() ) {
+          pnext = valid_next(p, true, comma );
+        } else {
+          if( exclusions.dot++ ) {
+            error_msg( loc, "PICTURE: %qc: may appear at most once", p[0] );
+            return nullptr;
+          }
+          pnext = valid_next(p);
+        }
+      }
+      break;
+
+    case '+': case '-':
+      // 7 is trailing sign; 13 & 14 are numeric.  Leading sign handled by start(). 
+      if( p + 1 == pend ) {
+        if( exclusions.crdb++ ) {
+          error_msg( loc, "PICTURE: %c/%c may appear at most once as a sign", '+', '-' );
+          return nullptr;
+        }
+        pnext = pend;
+      } else {
+        pnext = next_not(p[0]);
+        if( p + 1 < pnext ) {
+          if( false && exclusions.pluses++ )  { // not enforced
+            error_msg( loc, "PICTURE: %qc: sequence may appear at most once", p[0] );
+            return nullptr;
+          }
+        }
+        pnext = valid_next(pnext, ! exclusions.dot);
+      }
+      break;
+
+    case 'Z': case '*':
+      if( false && exclusions.stars++ ) { // not enforced 
+        error_msg( loc, "PICTURE: %qc: sequence may appear at most once", p[0] );
+        return nullptr;
+      }
+      if( (pnext = next_not(p[0])) == nullptr ) return pnext;
+      pnext = valid_next(pnext, ! exclusions.dot);
+      break;
+    case 'P':
+      pnext = valid_next(pnext, ! exclusions.dot);
+      break;
+    case '9':
+    case 'A': case 'X':
+    case 'V':
+    case '1':
+    case 'N':
+      pnext = valid_next(p);
+      break;
+    case 'E':
+      pnext = valid_next(p, "+9");
+      if( pnext && *pnext == '+' ) {
+        pnext = valid_next(p, "9");
+      }
+      break;
+    default:
+      error_msg( loc, "PICTURE: %qc: invalid character", p[0] );
+      return nullptr;
+    }
+    return pnext;
+  }
+  
+ public:
+  picture_t( const char *p, int len )
+    : begin(p)
+    , p(p), pend(p + len)
+    , loc(yylloc)
+  { 
+    assert(TOUPPER(*p) == 'P'); // as in PICTURE (or PICTURE IS)
+    // move p to start of picture string
+    while( (p = std::find_if(p, pend, fisspace)) != pend ) {
+      this->p = p = std::find_if(p, pend,
+                                 []( char ch ) { return ! fisspace(ch); } );
+    }
+    assert(this->p != pend);
+    pos = this->p - begin;
+  }
+
+  bool is_valid() {
+    if( !p ) return false;
+    if( (p = start()) == nullptr ) {
+      return false;
+    }
+
+    while( p && p < pend) {
+      p = next();
+    }
+    return p == pend;
+  }
+
+  int starts_at() const { return pos; }
+};
+
+/*
+ * The Followers map gives 1 or 2 lists of valid characters following a
+ * character, the one in the key. If there are two lists, the correct one is
+ * determined by the caller based on the state of the picture string, i.e.,
+ * what has been seen before.
+ */
+const std::map <char, picture_t::followings_t> picture_t::followers {
+  /*   B0/ */ { 'B', {"B0/,.Z*+-9AXVPNE" } },
+  /*   B0/ */ { '0', {"B0/,.Z*+-9AXVPNE" } },
+  /*   B0/ */ { '/', {"B0/,.Z*+-9AXVPNE" } },
+  /*     , */ { ',', {"B0/,.Z*+-9VPE"} },
+  /*     . */ { '.', {"B0/,Z*+-9E"} },
+  /*     +    { '+', "9" }, */
+  /*    +- */ { '+', {"B0/,.Z*9VPE", "" } },
+  /*    +- */ { '-', {"B0/,.Z*9VPE", "" } },
+  /* CR/DB    { 'C', "" }, */
+  /*    cs    { 'c', "B0/,.Z*+-9VP" }, */
+  /*    cs    { 'c', "+" }, */
+  /*    Z* */ { 'Z', {"B0/,.+Z*9VP", "B0/,+Z*"} },
+  /*    Z* */ { '*', {"B0/,.+Z*9VP", "B0/,+Z*"} },
+  /*     + */ { '+', {"B0/,.+-9VP",  "B0/,+-"} },
+  /*    cs */ { '$', {"B0/,.+9VP",   "B0/,+"} },
+  /*     9 */ { '9', {"B0/,.+9AXVPE"} },
+  /*    AX */ { 'A', {"B0/9AX"} },
+  /*    AX */ { 'X', {"B0/9AX"} },
+  /*     S */ { 'S', {"9VP"} },
+  /*     V */ { 'V', {"B0/,+Z*+-9P"} },
+  /*     P */ { 'P', {"+VP", "B0/,+Z*9P"} },
+  /*     1 */ { '1', {"1"} },
+  /*     N */ { 'N', {"B0/N"} },
+  /*     E */ { 'E', {"+9"} },
+};
+
+/*
+ * Although picture_t::is_valid return a bool, it's not used. The validation
+ * routines emit messages where the error is detected. The entire string is
+ * subsequently parsed by the parser, which might otherwise accept an invalid
+ * string, but will usually emit a message of its own.
+ */
+static int 
+validate_picture() {
+  picture_t picture(yytext, yyleng);
+  picture.is_valid();
+  return  picture.starts_at();
+}
+
 static inline bool
 is_integer_token( int *pvalue = NULL ) {
   int v, n = 0;
diff --git a/gcc/cobol/scan_post.h b/gcc/cobol/scan_post.h
index a273da9..7cf2b98 100644
--- a/gcc/cobol/scan_post.h
+++ b/gcc/cobol/scan_post.h
@@ -34,7 +34,6 @@ start_condition_str( int sc ) {
   switch(sc) {
   case INITIAL: state = "INITIAL"; break;
   case addr_of: state = "addr_of"; break;
-  case author_state: state = "author_state"; break;
   case basis: state = "basis"; break;
   case bool_state: state = "bool_state"; break;
   case cdf_state: state = "cdf_state"; break;
@@ -159,6 +158,8 @@ is_cdf_token( int token ) {
   case CDF_DISPLAY:
   case CDF_IF:       case CDF_ELSE: case CDF_END_IF:
   case CDF_EVALUATE: case CDF_WHEN: case CDF_END_EVALUATE:
+  case CDF_PUSH:
+  case CDF_POP:
     return true;
   case CALL_COBOL:
   case CALL_VERBATIM:
diff --git a/gcc/cobol/show_parse.h b/gcc/cobol/show_parse.h
index db24807..bd0e16f 100644
--- a/gcc/cobol/show_parse.h
+++ b/gcc/cobol/show_parse.h
@@ -176,11 +176,21 @@ extern bool cursor_at_sol;
                 } \
             else \
                 { \
-		fprintf(stderr, " %p:%s (%s)", (void*)b, b->name, b->type_str()); \
+                fprintf(stderr, " %p:%s (%s)", static_cast<void*>(b), b->name, b->type_str()); \
                 } \
             show_parse_sol = false; \
             } while(0);
 
+// Use this version when b is known to be valid.  This is necessary to quiet
+// cppcheck nullPointerRedundantCheck warnings
+#define SHOW_PARSE_LABEL_OK(a, b) \
+        do \
+            { \
+            fprintf(stderr, "%s", a); \
+            fprintf(stderr, " %p:%s (%s)", static_cast<void*>(b), b->name, b->type_str()); \
+            show_parse_sol = false; \
+            } while(0);
+
 #define TRACE1 if(bTRACE1)
 #define TRACE1_HEADER do \
   { \
@@ -211,6 +221,7 @@ extern bool cursor_at_sol;
 #define TRACE1_FIELD_VALUE(a, field, b) \
         do \
             { \
+            gcc_assert(field); \
             cursor_at_sol=false; \
             if ( field->type == FldConditional ) \
               {                                       \
@@ -423,13 +434,39 @@ extern bool cursor_at_sol;
     } while(0);
 
 // Use CHECK_FIELD when a should be non-null, and a->var_decl_node also should
-// by non-null:
+// by non-null.  (The useless calls to abort() are because cppcheck doesn't
+// understand that gcc_unreachable doesn't return);
+
+// Use this after doing any SHOW_PARSE stuff, to avoid cppcheck complaints
+// about nullPointerRedundantCheck 
 #define CHECK_FIELD(a)                                                  \
-        do {                                                                  \
+        do {                                                            \
         if(!a)                                                          \
             {                                                           \
-            yywarn("%s: parameter %<" #a "%> is NULL", __func__);        \
+            yywarn("%s: parameter %<" #a "%> is NULL", __func__);       \
             gcc_unreachable();                                          \
+            abort();                                                    \
+            }                                                           \
+        if( !a->var_decl_node )                                         \
+            {                                                           \
+            yywarn("%s: parameter %<" #a "%> is variable "              \
+                   "%s<%s> with NULL %<var_decl_node%>",                \
+                __func__,                                               \
+                a->name,                                                \
+                cbl_field_type_str(a->type) );                          \
+            gcc_unreachable();                                          \
+            abort();                                                    \
+            }                                                           \
+        } while(0);
+
+// This version is a bit more lax, for special cases
+#define CHECK_FIELD2(a)                                                  \
+        do {                                                            \
+        if(!a)                                                          \
+            {                                                           \
+            yywarn("%s: parameter %<" #a "%> is NULL", __func__);       \
+            gcc_unreachable();                                          \
+            abort();                                                    \
             }                                                           \
         if( !a->var_decl_node && a->type != FldConditional && a->type != FldLiteralA) \
             {                                                           \
@@ -439,15 +476,18 @@ extern bool cursor_at_sol;
                 a->name,                                                \
                 cbl_field_type_str(a->type) );                          \
             gcc_unreachable();                                          \
+            abort();                                                    \
             }                                                           \
         } while(0);
 
+
 #define CHECK_LABEL(a)                                                  \
         do{                                                             \
         if(!a)                                                          \
             {                                                           \
             yywarn("%s: parameter %<" #a "%> is NULL", __func__);       \
             gcc_unreachable();                                          \
+            abort();                                                    \
             }                                                           \
         }while(0);
 
@@ -504,6 +544,7 @@ class ANALYZE
       }
   };
 #else
+// cppcheck-suppress ctuOneDefinitionRuleViolation
 class ANALYZE
   {
   public:
diff --git a/gcc/cobol/symbols.cc b/gcc/cobol/symbols.cc
index 089c9c1..f2cd1b5 100644
--- a/gcc/cobol/symbols.cc
+++ b/gcc/cobol/symbols.cc
@@ -28,6 +28,8 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+// cppcheck-suppress-file duplicateBreak
+
 #include "config.h"
 #include <fstream> // Before cobol-system because it uses poisoned functions
 #include "cobol-system.h"
@@ -672,7 +674,7 @@ symbol_special( size_t program, const char name[] )
 struct symbol_elem_t *
 symbol_alphabet( size_t program, const char name[] )
 {
-  cbl_alphabet_t alphabet(YYLTYPE(), custom_encoding_e);
+  cbl_alphabet_t alphabet(YYLTYPE(), custom_encoding_e); // cppcheck-suppress syntaxError
   assert(strlen(name) < sizeof alphabet.name);
   strcpy(alphabet.name, name);
 
@@ -931,7 +933,7 @@ end_of_group( size_t igroup ) {
       if( e->program != group->program ) return isym;
       if( e->type == SymLabel ) return isym; // end of data division
       if( e->type == SymField ) {
-        const auto f = cbl_field_of(e);
+        const cbl_field_t * f = cbl_field_of(e);
         if( f->level == LEVEL77 || f->level == 66 ) return isym;
         if( f->level == 1 && f->parent != igroup ) {
           return isym;
@@ -1174,7 +1176,7 @@ static struct symbol_elem_t *
         // If an 01 record exists for the FD/SD, use its capacity as the
         // default_record capacity.
         if( p != symbols_end() ) {
-          const auto record = cbl_field_of(p);
+          const cbl_field_t * record = cbl_field_of(p);
           assert(record->level == 1);
           e = calculate_capacity(p);
           auto record_size = std::max(record->data.memsize,
@@ -1262,7 +1264,7 @@ static struct symbol_elem_t *
 
     // If group has a parent that is a record area, expand it, too.
     if( 0 < group->parent ) {
-      auto redefined = symbol_redefines(group);
+      redefined = symbol_redefines(group);
       if( redefined && is_record_area(redefined) ) {
         if( redefined->data.capacity < group->data.memsize ) {
           redefined->data.capacity = group->data.memsize;
@@ -1434,11 +1436,11 @@ cbl_field_t::attr_str( const std::vector<cbl_field_attr_t>& attrs ) const
   const char *sep = "";
   char *out = NULL;
 
-  for( auto attr : attrs ) {
+  for( auto attr_l : attrs ) {
     char *part = out;
-    if( has_attr(attr) ) {
+    if( has_attr(attr_l) ) {
       int erc = asprintf(&out, "%s%s%s",
-                         part? part : "", sep, cbl_field_attr_str(attr));
+                         part? part : "", sep, cbl_field_attr_str(attr_l));
       if( -1 == erc ) return part;
       free(part);
       sep = ", ";
@@ -1745,7 +1747,7 @@ symbols_update( size_t first, bool parsed_ok ) {
 
     bool size_invalid = field->data.memsize > 0 && symbol_redefines(field);
     if( size_invalid ) { // redefine of record area is ok
-      const auto redefined = symbol_redefines(field);
+      const cbl_field_t * redefined = symbol_redefines(field);
       size_invalid = ! is_record_area(redefined);
     }
     if( !field->is_valid() || size_invalid )
@@ -1768,8 +1770,8 @@ symbols_update( size_t first, bool parsed_ok ) {
         if( e == symbols_end() ) {
           // no field redefines the file's default record
           auto file = cbl_file_of(symbol_at(field->parent));
-          ERROR_FIELD(field, "line %d: %s lacks a file description",
-                      file->line, file->name);
+          ERROR_FIELD(field, "%s lacks a file description",
+                      file->name);
           return 0;
         }
       }
@@ -1828,7 +1830,7 @@ symbols_update( size_t first, bool parsed_ok ) {
     }
 
     // Verify REDEFINing field has no ODO components
-    const auto parent = symbol_redefines(field);
+    const cbl_field_t * parent = symbol_redefines(field);
     if( parent && !is_record_area(parent) && is_variable_length(field) ) {
       ERROR_FIELD(field, "line %d: REDEFINES field %s cannot be variable length",
                field->line, field->name);
@@ -2180,14 +2182,22 @@ symbol_table_init(void) {
   }
 
   static symbol_elem_t environs[] = {
+    { symbol_elem_t{ 0, cbl_special_name_t{0, CONSOLE_e, "CONSOLE", 0, "/dev/stdout"}} }, // stdout in DISPLAY; stdin in ACCEPT
+
+    { symbol_elem_t{ 0, cbl_special_name_t{0, STDIN_e, "STDIN", 0, "/dev/stdin"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, SYSIN_e, "SYSIN", 0, "/dev/stdin"}} },
-    { symbol_elem_t{ 0, cbl_special_name_t{0, SYSIPT_e, "SYSIPT", 0, "/dev/stdout"}} },
+    { symbol_elem_t{ 0, cbl_special_name_t{0, SYSIPT_e, "SYSIPT", 0, "/dev/stdin"}} },
+
+    { symbol_elem_t{ 0, cbl_special_name_t{0, STDOUT_e, "STDOUT", 0, "/dev/stdout"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, SYSOUT_e, "SYSOUT", 0, "/dev/stdout"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, SYSLIST_e, "SYSLIST", 0, "/dev/stdout"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, SYSLST_e, "SYSLST", 0, "/dev/stdout"}} },
+
     { symbol_elem_t{ 0, cbl_special_name_t{0, SYSPUNCH_e, "SYSPUNCH", 0, "/dev/stderr"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, SYSPCH_e, "SYSPCH", 0, "/dev/stderr"}} },
-    { symbol_elem_t{ 0, cbl_special_name_t{0, CONSOLE_e, "CONSOLE", 0, "/dev/stdout"}} },
+    { symbol_elem_t{ 0, cbl_special_name_t{0, STDERR_e, "STDERR", 0, "/dev/stderr"}} },
+    { symbol_elem_t{ 0, cbl_special_name_t{0, SYSERR_e, "SYSERR", 0, "/dev/stderr"}} },
+
     { symbol_elem_t{ 0, cbl_special_name_t{0, C01_e, "C01", 0, "/dev/null"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, C02_e, "C02", 0, "/dev/null"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, C03_e, "C03", 0, "/dev/null"}} },
@@ -2207,10 +2217,6 @@ symbol_table_init(void) {
     { symbol_elem_t{ 0, cbl_special_name_t{0, S04_e, "S04", 0, "/dev/null"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, S05_e, "S05", 0, "/dev/null"}} },
     { symbol_elem_t{ 0, cbl_special_name_t{0, AFP_5A_e, "AFP-5A", 0, "/dev/null"}} },
-    { symbol_elem_t{ 0, cbl_special_name_t{0, STDIN_e, "STDIN", 0, "/dev/stdin"}} },
-    { symbol_elem_t{ 0, cbl_special_name_t{0, STDOUT_e, "STDOUT", 0, "/dev/stdout"}} },
-    { symbol_elem_t{ 0, cbl_special_name_t{0, STDERR_e, "STDERR", 0, "/dev/stderr"}} },
-    { symbol_elem_t{ 0, cbl_special_name_t{0, SYSERR_e, "SYSERR", 0, "/dev/stderr"}} },
   };
 
   struct symbol_elem_t *p = table.elems + table.nelem;
@@ -2466,7 +2472,7 @@ symbol_typedef_add( size_t program, struct cbl_field_t *field ) {
   auto e = symbols_end() - 1;
   assert( symbols_begin() < e );
   if( e->type == SymField ) {
-    const auto f = cbl_field_of(e);
+    const cbl_field_t * f = cbl_field_of(e);
     if( f == field ) return e;
   }
 
@@ -2516,7 +2522,8 @@ symbol_field_add( size_t program, struct cbl_field_t *field )
     if( is_numeric(parent->usage) && parent->data.capacity > 0 ) {
       field->type = parent->usage;
       field->data = parent->data;
-      field->data = 0;
+      field->data = 0;  // cppcheck-suppress redundantAssignment
+      //                // cppcheck doesn't understand multiple overloaded operator=
       field->data.initial = NULL;
     }
   }
@@ -3140,7 +3147,6 @@ static cbl_field_t *
 new_temporary_impl( enum cbl_field_type_t type, const cbl_name_t name = nullptr )
 {
   extern int yylineno;
-  static int nstack, nliteral;
   static const struct cbl_field_t empty_alpha = {
                                 0, FldAlphanumeric, FldInvalid,
                                 intermediate_e, 0, 0, 0, nonarray, 0, "",
@@ -3209,8 +3215,10 @@ new_temporary_impl( enum cbl_field_type_t type, const cbl_name_t name = nullptr
 
   f->line = yylineno;
   if( is_literal(type) ) {
+    static int nliteral = 0;
     snprintf(f->name, sizeof(f->name), "_literal%d",++nliteral);
   } else {
+    static int nstack = 0;
     snprintf(f->name, sizeof(f->name), "_stack%d",++nstack);
   }
 
@@ -3724,6 +3732,12 @@ symbol_label_add( size_t program, cbl_label_t *input )
 bool
 symbol_label_section_exists( size_t eval_label_index ) {
   auto eval = symbols_begin(eval_label_index);
+  /*  cppcheck warns that the following statement depends on the order of
+      evaluation of side effects.  Since this isn't my code, and since I don't
+      think the warning can be eliminated without rewriting it, I am just
+      supprressing it.
+      -- Bob Dubner, 2025-07-14 */
+  // cppcheck-suppress unknownEvaluationOrder
   bool has_section = std::any_of( ++eval, symbols_end(),
                                [program = eval->program]( const auto& sym ) {
                                  if( program == sym.program && sym.type == SymLabel ) {
@@ -4183,7 +4197,7 @@ symbol_program_callables( size_t program ) {
     if( e->type != SymLabel ) continue;
     if( e->elem.label.type != LblProgram ) continue;
 
-    const auto prog = cbl_label_of(e);
+    const cbl_label_t * prog = cbl_label_of(e);
     if( program == symbol_index(e) && !prog->recursive ) continue;
 
     if( (self->parent == prog->parent && prog->common) ||
@@ -4237,6 +4251,11 @@ symbol_currency( char sign ) {
   if( currencies.size() == 0 ) {
     currencies['$'] = "$";
   }
+  if( sign == '\0' ) { // default
+    auto result = currencies.begin();
+    gcc_assert(result != currencies.end());
+    return result->second;
+  }
   auto result = currencies.find(sign);
   return result == currencies.end()? NULL : result->second;
 }
@@ -4345,6 +4364,26 @@ cbl_occurs_t::subscript_ok( const cbl_field_t *subscript ) const {
   return bounds.lower <= (size_t)sub && (size_t)sub <= bounds.upper;
 }
 
+const cbl_field_t * 
+symbol_unresolved_file_key( const cbl_file_t * file,
+                            const cbl_name_t key_field_name ) {
+  const symbol_elem_t *file_sym = symbol_elem_of(file);
+  size_t program = file_sym->program;
+  for( const symbol_elem_t *e = file_sym - 1; e->program == program; e-- ) {
+    if( e->type == SymFile ) break;
+    if( e->type == SymField ) {
+      auto f = cbl_field_of(e);
+      if( f->type == FldLiteralA ) break;
+      if( f->type == FldForward ) {
+        if( 0 == strcmp(key_field_name, f->name) ) {
+          return f;
+        }
+      }
+    }
+  }
+  return nullptr;
+}
+
 cbl_file_key_t::
 cbl_file_key_t( cbl_name_t name,
                 const std::list<cbl_field_t *>& fields,
@@ -4486,7 +4525,7 @@ cbl_file_key_t::deforward( size_t ifile ) {
                     if( ifield == fwd ) {
                       ERROR_FIELD(field, "line %d: %s of %s "
                                "is not defined",
-                               file->line, field->name, file->name);
+                               field->line, field->name, file->name);
                       return ifield;
                     }
 
@@ -4515,9 +4554,13 @@ cbl_file_key_t::deforward( size_t ifile ) {
                     // looked-up field must have same file as parent
                     if( ! (parent != NULL &&
                            symbol_index(symbol_elem_of(parent)) == ifile) ) {
-                      ERROR_FIELD(field, "line %d: %s of %s "
-                               "is not defined in file description",
-                               file->line, field->name, file->name);
+                      const cbl_field_t *undefined =
+                        symbol_unresolved_file_key(file, field->name);
+                      int lineno = undefined? undefined->line : file->line;
+                      ERROR_FIELD(undefined? undefined : field,
+                                  "line %d: %s of %s "
+                                  "is not defined in file description",
+                                  lineno, field->name, file->name);
                     }
                     return ifield;
                   } );
@@ -4630,9 +4673,11 @@ file_status_status_of( file_status_t status ) {
   size_t n = COUNT_OF(file_status_fields);
   const file_status_field_t *fs, key { status };
 
-  fs = (file_status_field_t*)lfind( &key, file_status_fields,
-                                    &n, sizeof(*fs), cbl_file_status_cmp );
-
+  fs = static_cast<file_status_field_t*>(lfind( &key,
+                                                file_status_fields,
+                                                &n,
+                                                sizeof(*fs),
+                                                cbl_file_status_cmp ));
   return fs? (long)fs->status : -1;
 }
 
diff --git a/gcc/cobol/symbols.h b/gcc/cobol/symbols.h
index 0b72b5c..c8ae32f 100644
--- a/gcc/cobol/symbols.h
+++ b/gcc/cobol/symbols.h
@@ -1894,6 +1894,10 @@ const cbl_label_t * symbol_program_local( const char called[] );
 
 bool redefine_field( cbl_field_t *field );
 
+const cbl_field_t * 
+symbol_unresolved_file_key( const cbl_file_t * file,
+                            const cbl_name_t key_field_name );
+
 static inline struct cbl_section_t *
 cbl_section_of( struct symbol_elem_t *e ) {
   assert(e && e->type == SymDataSection);
@@ -2387,9 +2391,169 @@ enum cbl_call_convention_t {
   cbl_call_cobol_e = 'N', // native
 };
 
+int keyword_tok( const char * text, bool include_intrinsics = false );
+int redefined_token( const cbl_name_t name );
+
+class current_tokens_t {
+  class tokenset_t {
+    // token_names is initialized from a generated header file. 
+    std::vector<const char *>token_names;  // position indicates token value
+    std::map <std::string, int> tokens;    // aliases
+    std::set<std::string> cobol_words;  // Anything in COBOL-WORDS may appear only once. 
+  public:
+    static std::string
+    lowercase( const cbl_name_t name ) {
+      cbl_name_t lname;
+      std::transform(name, name + strlen(name) + 1, lname, ftolower);
+      return lname;
+    }
+    static std::string
+    uppercase( const cbl_name_t name ) {
+      cbl_name_t uname;
+      std::transform(name, name + strlen(name) + 1, uname, ftoupper);
+      return uname;
+    }
+
+  public:
+    tokenset_t();
+    int find( const cbl_name_t name, bool include_intrinsics );
+
+    bool equate( const YYLTYPE& loc, int token,
+                 const cbl_name_t name, const cbl_name_t verb = "EQUATE") {
+      auto lname( lowercase(name) );
+      auto cw = cobol_words.insert(lname);
+      if( ! cw.second ) {
+        error_msg(loc, "COBOL-WORDS %s: %s may appear but once", verb, name);
+        return false;
+      }
+      auto p = tokens.find(lowercase(name));
+      bool fOK = p == tokens.end();
+      if( fOK ) { // name not already in use
+        tokens[lname] = token;
+        dbgmsg("%s:%d: %d has alias %s", __func__, __LINE__, token, name);
+      } else {
+        error_msg(loc, "%s: %s already defined as a token", verb, name);
+      }
+      return fOK;
+    }
+    bool undefine( const YYLTYPE& loc,
+                   const cbl_name_t name, const cbl_name_t verb = "UNDEFINE" ) {
+      auto lname( lowercase(name) );
+      auto cw = cobol_words.insert(lname);
+      if( ! cw.second ) {
+        error_msg(loc, "COBOL-WORDS %s: %s may appear but once", verb, name);
+        return false;
+      }
+
+      // Do not erase generic, multi-type tokens COMPUTATIONAL and BINARY_INTEGER.
+      if( binary_integer_usage_of(name) ) {
+        dbgmsg("%s:%d: generic %s remains valid as a token", __func__, __LINE__, name);
+        return true;
+      }
+
+      auto p = tokens.find(lname);
+      bool fOK = p != tokens.end();
+      if( fOK ) { // name in use
+        tokens.erase(p);
+      } else {
+        error_msg(loc, "%s: %s not defined as a token", verb, name);
+      }
+      dbgmsg("%s:%d: %s removed as a valid token name", __func__, __LINE__, name);
+      return fOK;
+    }
+  
+    bool substitute( const YYLTYPE& loc,
+                     const cbl_name_t extant, int token, const cbl_name_t name ) {
+      return
+        equate( loc, token, name, "SUBSTITUTE" )
+        &&
+        undefine( loc, extant, "SUBSTITUTE" );
+    }
+    bool reserve( const YYLTYPE& loc, const cbl_name_t name ) {
+      auto lname( lowercase(name) );
+      auto cw = cobol_words.insert(lname);
+      if( ! cw.second ) {
+        error_msg(loc, "COBOL-WORDS RESERVE: %s may appear but once", name);
+        return false;
+      }
+      tokens[lname] = -42;
+      return true;
+    }
+    int redefined_as( const cbl_name_t name ) {
+      auto lname( lowercase(name) );
+      if( cobol_words.find(lname) != cobol_words.end() ) {
+        auto p = tokens.find(lname);
+        if( p != tokens.end() ) {
+          return p->second;
+        }
+      }
+      return 0;
+    }
+    const char * name_of( int tok ) const {
+      tok -= (255 + 3);
+      gcc_assert(0 <= tok && size_t(tok) < token_names.size());
+      return tok < 0? "???" : token_names[tok];
+    }
+  };
+
+  tokenset_t tokens;
+ public:
+  current_tokens_t() {}
+  int find( const cbl_name_t name, bool include_intrinsics ) {
+    return tokens.find(name, include_intrinsics);
+  }
+  bool equate( const YYLTYPE& loc, const cbl_name_t keyword, const cbl_name_t alias ) {
+    int token; 
+    if( 0 == (token = binary_integer_usage_of(keyword)) ) {
+      if( 0 == (token = keyword_tok(keyword)) ) {
+	error_msg(loc, "EQUATE %s: not a valid token", keyword);
+	return false;
+      }
+    }
+    auto name = keyword_alias_add(tokens.uppercase(keyword),
+				  tokens.uppercase(alias));
+    if( name != keyword ) {
+      error_msg(loc, "EQUATE: %s is already an alias for %s", alias, name.c_str());
+      return false;
+    } 
+    return tokens.equate(loc, token, alias);
+  }
+  bool undefine( const YYLTYPE& loc, cbl_name_t keyword ) {
+    return tokens.undefine(loc, keyword);
+  }
+  bool substitute( const YYLTYPE& loc, const cbl_name_t keyword, const cbl_name_t alias ) {
+    int token; 
+    if( 0 == (token = binary_integer_usage_of(keyword)) ) {
+      if( 0 == (token = keyword_tok(keyword)) ) {
+	error_msg(loc, "SUBSTITUTE %s: not a valid token", keyword);
+	return false;
+      }
+    }
+    auto name = keyword_alias_add(tokens.uppercase(keyword),
+				  tokens.uppercase(alias));
+    if( name != keyword ) {
+      error_msg(loc, "SUBSTITUTE: %s is already an alias for %s", alias, name.c_str());
+      return false;
+    } 
+
+    dbgmsg("%s:%d: %s (%d) will have alias %s", __func__, __LINE__, keyword, token, alias);
+    return tokens.substitute(loc, keyword, token, alias);
+  }
+  bool reserve( const YYLTYPE& loc, const cbl_name_t name ) {
+    return tokens.reserve(loc, name);
+  }
+  int redefined_as( const cbl_name_t name ) {
+    return tokens.redefined_as(name);
+  }
+  const char * name_of( int tok ) const {
+    return tokens.name_of(tok);
+  }
+};
+
 cbl_call_convention_t current_call_convention();
+current_tokens_t& cdf_current_tokens();
 
-cbl_call_convention_t
+void
 current_call_convention( cbl_call_convention_t convention);
 
 class procref_base_t {
@@ -2433,9 +2597,6 @@ public:
   int line_number() const { return line; }
 };
 
-int keyword_tok( const char * text, bool include_intrinsics = false );
-int redefined_token( const cbl_name_t name );
-
 void procedure_definition_add( size_t program, const cbl_label_t *procedure );
 void procedure_reference_add( const char *sect, const char *para,
                               int line, size_t context );
@@ -2452,7 +2613,7 @@ symbol_field_same_as( cbl_field_t *tgt, const cbl_field_t *src );
 size_t symbol_file_same_record_area( std::list<cbl_file_t*>& files );
 
 bool symbol_currency_add( const char symbol[], const char sign[] = NULL );
-const char * symbol_currency( char symbol );
+const char * symbol_currency( char symbol = '\0' );
 
 const char * symbol_type_str( enum symbol_type_t type );
 const char * cbl_field_type_str( enum cbl_field_type_t type );
diff --git a/gcc/cobol/symfind.cc b/gcc/cobol/symfind.cc
index 39df2a0..8141b22 100644
--- a/gcc/cobol/symfind.cc
+++ b/gcc/cobol/symfind.cc
@@ -48,7 +48,7 @@ extern int yydebug;
 static bool
 is_data_field( symbol_elem_t& e ) {
   if( e.type != SymField ) return false;
-  const auto f = cbl_field_of(&e);
+  const cbl_field_t *f = cbl_field_of(&e);
   if( f->name[0] == '\0' ) return false;
   if( is_filler(f) ) return false;
 
@@ -129,7 +129,7 @@ finalize_symbol_map2() {
   for( auto& elem : symbol_map2 ) {
     auto& fields( elem.second );
     fields.remove_if( []( auto isym ) {
-			const auto f = cbl_field_of(symbol_at(isym));
+			const cbl_field_t *f = cbl_field_of(symbol_at(isym));
 			return f->type == FldInvalid;
 		      } );
     if( fields.empty() ) empties.insert(elem.first);
@@ -316,9 +316,9 @@ public:
       if( p != item.second.end() ) {
         // Preserve symbol's index at front of ancestor list.
         symbol_map_t::mapped_type shorter(1 + ancestors->size());
-        auto p = shorter.begin();
-        *p = item.second.front();
-        shorter.insert( ++p, ancestors->begin(), ancestors->end() );
+        auto p_l = shorter.begin();
+        *p_l = item.second.front();
+        shorter.insert( ++p_l, ancestors->begin(), ancestors->end() );
         return make_pair(item.first, shorter);
       }
     }
@@ -341,7 +341,7 @@ class in_scope {
   size_t program;
 
   static size_t prog_of( size_t program ) {
-    const auto L = cbl_label_of(symbol_at(program));
+    const cbl_label_t *L = cbl_label_of(symbol_at(program));
     return L->parent;
   }
 
@@ -430,7 +430,7 @@ symbol_match2( size_t program,
   auto plist = symbol_map2.find(key);
   if( plist != symbol_map2.end() ) {
     for( auto candidate : plist->second ) {
-      const auto e = symbol_at(candidate);
+      const symbol_elem_t *e = symbol_at(candidate);
       if( name_has_names( e, names, local ) ) {
         fields.push_back( symbol_index(e) );
       }
@@ -504,7 +504,7 @@ symbol_match( size_t program, const std::list<const char *>& names ) {
     }
     auto inserted = output.insert(*p);
     if( ! inserted.second ) {
-      yyerror("%s is not a unique reference", key.name);
+      error_msg_direct("%s is not a unique reference", key.name);
     }
   }
   return output;
diff --git a/gcc/cobol/token_names.h b/gcc/cobol/token_names.h
index 4b24fc6..ca51510 100644
--- a/gcc/cobol/token_names.h
+++ b/gcc/cobol/token_names.h
@@ -1,5 +1,5 @@
 // generated by ./token_names.h.gen ../../build/gcc/cobol/parse.h
-// Thu May  8 18:53:33 EDT 2025
+// Tue Jul  8 19:21:28 EDT 2025
 tokens = {
 	{ "identification", IDENTIFICATION_DIV }, // 258
 	{ "environment", ENVIRONMENT_DIV }, // 259
@@ -137,558 +137,566 @@ tokens = {
 	{ "cdf-evaluate", CDF_EVALUATE }, // 388
 	{ "cdf-when", CDF_WHEN }, // 389
 	{ "cdf-end-evaluate", CDF_END_EVALUATE }, // 390
-	{ "call-cobol", CALL_COBOL }, // 391
-	{ "call-verbatim", CALL_VERBATIM }, // 392
-	{ "if", IF }, // 393
-	{ "then", THEN }, // 394
-	{ "else", ELSE }, // 395
-	{ "sentence", SENTENCE }, // 396
-	{ "accept", ACCEPT }, // 397
-	{ "add", ADD }, // 398
-	{ "alter", ALTER }, // 399
-	{ "call", CALL }, // 400
-	{ "cancel", CANCEL }, // 401
-	{ "close", CLOSE }, // 402
-	{ "compute", COMPUTE }, // 403
-	{ "continue", CONTINUE }, // 404
-	{ "delete", DELETE }, // 405
-	{ "display", DISPLAY }, // 406
-	{ "divide", DIVIDE }, // 407
-	{ "evaluate", EVALUATE }, // 408
-	{ "exit", EXIT }, // 409
-	{ "filler", FILLER_kw }, // 410
-	{ "goback", GOBACK }, // 411
-	{ "goto", GOTO }, // 412
-	{ "initialize", INITIALIZE }, // 413
-	{ "inspect", INSPECT }, // 414
-	{ "merge", MERGE }, // 415
-	{ "move", MOVE }, // 416
-	{ "multiply", MULTIPLY }, // 417
-	{ "open", OPEN }, // 418
-	{ "paragraph", PARAGRAPH }, // 419
-	{ "read", READ }, // 420
-	{ "release", RELEASE }, // 421
-	{ "return", RETURN }, // 422
-	{ "rewrite", REWRITE }, // 423
-	{ "search", SEARCH }, // 424
-	{ "set", SET }, // 425
-	{ "select", SELECT }, // 426
-	{ "sort", SORT }, // 427
-	{ "sort-merge", SORT_MERGE }, // 428
-	{ "string", STRING_kw }, // 429
-	{ "stop", STOP }, // 430
-	{ "subtract", SUBTRACT }, // 431
-	{ "start", START }, // 432
-	{ "unstring", UNSTRING }, // 433
-	{ "write", WRITE }, // 434
-	{ "when", WHEN }, // 435
-	{ "abs", ABS }, // 436
-	{ "access", ACCESS }, // 437
-	{ "acos", ACOS }, // 438
-	{ "actual", ACTUAL }, // 439
-	{ "advancing", ADVANCING }, // 440
-	{ "after", AFTER }, // 441
-	{ "all", ALL }, // 442
-	{ "allocate", ALLOCATE }, // 443
-	{ "alphabet", ALPHABET }, // 444
-	{ "alphabetic", ALPHABETIC }, // 445
-	{ "alphabetic-lower", ALPHABETIC_LOWER }, // 446
-	{ "alphabetic-upper", ALPHABETIC_UPPER }, // 447
-	{ "alphanumeric", ALPHANUMERIC }, // 448
-	{ "alphanumeric-edited", ALPHANUMERIC_EDITED }, // 449
-	{ "also", ALSO }, // 450
-	{ "alternate", ALTERNATE }, // 451
-	{ "annuity", ANNUITY }, // 452
-	{ "anum", ANUM }, // 453
-	{ "any", ANY }, // 454
-	{ "anycase", ANYCASE }, // 455
-	{ "apply", APPLY }, // 456
-	{ "are", ARE }, // 457
-	{ "area", AREA }, // 458
-	{ "areas", AREAS }, // 459
-	{ "as", AS }, // 460
-	{ "ascending", ASCENDING }, // 461
-	{ "activating", ACTIVATING }, // 462
-	{ "asin", ASIN }, // 463
-	{ "assign", ASSIGN }, // 464
-	{ "at", AT }, // 465
-	{ "atan", ATAN }, // 466
-	{ "based", BASED }, // 467
-	{ "baseconvert", BASECONVERT }, // 468
-	{ "before", BEFORE }, // 469
-	{ "binary", BINARY }, // 470
-	{ "bit", BIT }, // 471
-	{ "bit-of", BIT_OF }, // 472
-	{ "bit-to-char", BIT_TO_CHAR }, // 473
-	{ "blank", BLANK }, // 474
-	{ "block", BLOCK_kw }, // 475
-	{ "boolean-of-integer", BOOLEAN_OF_INTEGER }, // 476
-	{ "bottom", BOTTOM }, // 477
-	{ "by", BY }, // 478
-	{ "byte", BYTE }, // 479
-	{ "byte-length", BYTE_LENGTH }, // 480
-	{ "cf", CF }, // 481
-	{ "ch", CH }, // 482
-	{ "changed", CHANGED }, // 483
-	{ "char", CHAR }, // 484
-	{ "char-national", CHAR_NATIONAL }, // 485
-	{ "character", CHARACTER }, // 486
-	{ "characters", CHARACTERS }, // 487
-	{ "checking", CHECKING }, // 488
-	{ "class", CLASS }, // 489
-	{ "cobol", COBOL }, // 490
-	{ "code", CODE }, // 491
-	{ "code-set", CODESET }, // 492
-	{ "collating", COLLATING }, // 493
-	{ "column", COLUMN }, // 494
-	{ "combined-datetime", COMBINED_DATETIME }, // 495
-	{ "comma", COMMA }, // 496
-	{ "command-line", COMMAND_LINE }, // 497
-	{ "command-line-count", COMMAND_LINE_COUNT }, // 498
-	{ "commit", COMMIT }, // 499
-	{ "common", COMMON }, // 500
-	{ "concat", CONCAT }, // 501
-	{ "condition", CONDITION }, // 502
-	{ "configuration", CONFIGURATION_SECT }, // 503
-	{ "contains", CONTAINS }, // 504
-	{ "content", CONTENT }, // 505
-	{ "control", CONTROL }, // 506
-	{ "controls", CONTROLS }, // 507
-	{ "convert", CONVERT }, // 508
-	{ "converting", CONVERTING }, // 509
-	{ "corresponding", CORRESPONDING }, // 510
-	{ "cos", COS }, // 511
-	{ "count", COUNT }, // 512
-	{ "currency", CURRENCY }, // 513
-	{ "current", CURRENT }, // 514
-	{ "current-date", CURRENT_DATE }, // 515
-	{ "data", DATA }, // 516
-	{ "date", DATE }, // 517
-	{ "date-compiled", DATE_COMPILED }, // 518
-	{ "date-of-integer", DATE_OF_INTEGER }, // 519
-	{ "date-to-yyyymmdd", DATE_TO_YYYYMMDD }, // 520
-	{ "date-written", DATE_WRITTEN }, // 521
-	{ "day", DAY }, // 522
-	{ "day-of-integer", DAY_OF_INTEGER }, // 523
-	{ "day-of-week", DAY_OF_WEEK }, // 524
-	{ "day-to-yyyyddd", DAY_TO_YYYYDDD }, // 525
-	{ "dbcs", DBCS }, // 526
-	{ "de", DE }, // 527
-	{ "debugging", DEBUGGING }, // 528
-	{ "decimal-point", DECIMAL_POINT }, // 529
-	{ "declaratives", DECLARATIVES }, // 530
-	{ "default", DEFAULT }, // 531
-	{ "delimited", DELIMITED }, // 532
-	{ "delimiter", DELIMITER }, // 533
-	{ "depending", DEPENDING }, // 534
-	{ "descending", DESCENDING }, // 535
-	{ "detail", DETAIL }, // 536
-	{ "direct", DIRECT }, // 537
-	{ "direct-access", DIRECT_ACCESS }, // 538
-	{ "down", DOWN }, // 539
-	{ "duplicates", DUPLICATES }, // 540
-	{ "dynamic", DYNAMIC }, // 541
-	{ "e", E }, // 542
-	{ "ebcdic", EBCDIC }, // 543
-	{ "ec", EC }, // 544
-	{ "egcs", EGCS }, // 545
-	{ "entry", ENTRY }, // 546
-	{ "environment", ENVIRONMENT }, // 547
-	{ "equal", EQUAL }, // 548
-	{ "every", EVERY }, // 549
-	{ "examine", EXAMINE }, // 550
-	{ "exhibit", EXHIBIT }, // 551
-	{ "exp", EXP }, // 552
-	{ "exp10", EXP10 }, // 553
-	{ "extend", EXTEND }, // 554
-	{ "external", EXTERNAL }, // 555
-	{ "exception-file", EXCEPTION_FILE }, // 556
-	{ "exception-file-n", EXCEPTION_FILE_N }, // 557
-	{ "exception-location", EXCEPTION_LOCATION }, // 558
-	{ "exception-location-n", EXCEPTION_LOCATION_N }, // 559
-	{ "exception-statement", EXCEPTION_STATEMENT }, // 560
-	{ "exception-status", EXCEPTION_STATUS }, // 561
-	{ "factorial", FACTORIAL }, // 562
-	{ "false", FALSE_kw }, // 563
-	{ "fd", FD }, // 564
-	{ "file-control", FILE_CONTROL }, // 565
-	{ "file", FILE_KW }, // 566
-	{ "file-limit", FILE_LIMIT }, // 567
-	{ "final", FINAL }, // 568
-	{ "finally", FINALLY }, // 569
-	{ "find-string", FIND_STRING }, // 570
-	{ "first", FIRST }, // 571
-	{ "fixed", FIXED }, // 572
-	{ "footing", FOOTING }, // 573
-	{ "for", FOR }, // 574
-	{ "formatted-current-date", FORMATTED_CURRENT_DATE }, // 575
-	{ "formatted-date", FORMATTED_DATE }, // 576
-	{ "formatted-datetime", FORMATTED_DATETIME }, // 577
-	{ "formatted-time", FORMATTED_TIME }, // 578
-	{ "form-overflow", FORM_OVERFLOW }, // 579
-	{ "free", FREE }, // 580
-	{ "fraction-part", FRACTION_PART }, // 581
-	{ "from", FROM }, // 582
-	{ "function", FUNCTION }, // 583
-	{ "generate", GENERATE }, // 584
-	{ "giving", GIVING }, // 585
-	{ "global", GLOBAL }, // 586
-	{ "go", GO }, // 587
-	{ "group", GROUP }, // 588
-	{ "heading", HEADING }, // 589
-	{ "hex", HEX }, // 590
-	{ "hex-of", HEX_OF }, // 591
-	{ "hex-to-char", HEX_TO_CHAR }, // 592
-	{ "high-values", HIGH_VALUES }, // 593
-	{ "highest-algebraic", HIGHEST_ALGEBRAIC }, // 594
-	{ "hold", HOLD }, // 595
-	{ "ibm-360", IBM_360 }, // 596
-	{ "in", IN }, // 597
-	{ "include", INCLUDE }, // 598
-	{ "index", INDEX }, // 599
-	{ "indexed", INDEXED }, // 600
-	{ "indicate", INDICATE }, // 601
-	{ "initial", INITIAL_kw }, // 602
-	{ "initiate", INITIATE }, // 603
-	{ "input", INPUT }, // 604
-	{ "installation", INSTALLATION }, // 605
-	{ "interface", INTERFACE }, // 606
-	{ "integer", INTEGER }, // 607
-	{ "integer-of-boolean", INTEGER_OF_BOOLEAN }, // 608
-	{ "integer-of-date", INTEGER_OF_DATE }, // 609
-	{ "integer-of-day", INTEGER_OF_DAY }, // 610
-	{ "integer-of-formatted-date", INTEGER_OF_FORMATTED_DATE }, // 611
-	{ "integer-part", INTEGER_PART }, // 612
-	{ "into", INTO }, // 613
-	{ "intrinsic", INTRINSIC }, // 614
-	{ "invoke", INVOKE }, // 615
-	{ "i-o", IO }, // 616
-	{ "i-o-control", IO_CONTROL }, // 617
-	{ "is", IS }, // 618
-	{ "isnt", ISNT }, // 619
-	{ "kanji", KANJI }, // 620
-	{ "key", KEY }, // 621
-	{ "label", LABEL }, // 622
-	{ "last", LAST }, // 623
-	{ "leading", LEADING }, // 624
-	{ "left", LEFT }, // 625
-	{ "length", LENGTH }, // 626
-	{ "length-of", LENGTH_OF }, // 627
-	{ "limit", LIMIT }, // 628
-	{ "limits", LIMITS }, // 629
-	{ "line", LINE }, // 630
-	{ "lines", LINES }, // 631
-	{ "line-counter", LINE_COUNTER }, // 632
-	{ "linage", LINAGE }, // 633
-	{ "linkage", LINKAGE }, // 634
-	{ "locale", LOCALE }, // 635
-	{ "locale-compare", LOCALE_COMPARE }, // 636
-	{ "locale-date", LOCALE_DATE }, // 637
-	{ "locale-time", LOCALE_TIME }, // 638
-	{ "locale-time-from-seconds", LOCALE_TIME_FROM_SECONDS }, // 639
-	{ "local-storage", LOCAL_STORAGE }, // 640
-	{ "location", LOCATION }, // 641
-	{ "lock", LOCK }, // 642
-	{ "lock-on", LOCK_ON }, // 643
-	{ "log", LOG }, // 644
-	{ "log10", LOG10 }, // 645
-	{ "lower-case", LOWER_CASE }, // 646
-	{ "low-values", LOW_VALUES }, // 647
-	{ "lowest-algebraic", LOWEST_ALGEBRAIC }, // 648
-	{ "lparen", LPAREN }, // 649
-	{ "manual", MANUAL }, // 650
-	{ "maxx", MAXX }, // 651
-	{ "mean", MEAN }, // 652
-	{ "median", MEDIAN }, // 653
-	{ "midrange", MIDRANGE }, // 654
-	{ "minn", MINN }, // 655
-	{ "multiple", MULTIPLE }, // 656
-	{ "mod", MOD }, // 657
-	{ "mode", MODE }, // 658
-	{ "module-name", MODULE_NAME }, // 659
-	{ "named", NAMED }, // 660
-	{ "nat", NAT }, // 661
-	{ "national", NATIONAL }, // 662
-	{ "national-edited", NATIONAL_EDITED }, // 663
-	{ "national-of", NATIONAL_OF }, // 664
-	{ "native", NATIVE }, // 665
-	{ "nested", NESTED }, // 666
-	{ "next", NEXT }, // 667
-	{ "no", NO }, // 668
-	{ "note", NOTE }, // 669
-	{ "nulls", NULLS }, // 670
-	{ "null", NULLS }, // 670
-	{ "nullptr", NULLPTR }, // 671
-	{ "numeric", NUMERIC }, // 672
-	{ "numeric-edited", NUMERIC_EDITED }, // 673
-	{ "numval", NUMVAL }, // 674
-	{ "numval-c", NUMVAL_C }, // 675
-	{ "numval-f", NUMVAL_F }, // 676
-	{ "occurs", OCCURS }, // 677
-	{ "of", OF }, // 678
-	{ "off", OFF }, // 679
-	{ "omitted", OMITTED }, // 680
-	{ "on", ON }, // 681
-	{ "only", ONLY }, // 682
-	{ "optional", OPTIONAL }, // 683
-	{ "options", OPTIONS }, // 684
-	{ "ord", ORD }, // 685
-	{ "order", ORDER }, // 686
-	{ "ord-max", ORD_MAX }, // 687
-	{ "ord-min", ORD_MIN }, // 688
-	{ "organization", ORGANIZATION }, // 689
-	{ "other", OTHER }, // 690
-	{ "otherwise", OTHERWISE }, // 691
-	{ "output", OUTPUT }, // 692
-	{ "packed-decimal", PACKED_DECIMAL }, // 693
-	{ "padding", PADDING }, // 694
-	{ "page", PAGE }, // 695
-	{ "page-counter", PAGE_COUNTER }, // 696
-	{ "pf", PF }, // 697
-	{ "ph", PH }, // 698
-	{ "pi", PI }, // 699
-	{ "pic", PIC }, // 700
-	{ "picture", PICTURE }, // 701
-	{ "plus", PLUS }, // 702
-	{ "present-value", PRESENT_VALUE }, // 703
-	{ "print-switch", PRINT_SWITCH }, // 704
-	{ "procedure", PROCEDURE }, // 705
-	{ "procedures", PROCEDURES }, // 706
-	{ "proceed", PROCEED }, // 707
-	{ "process", PROCESS }, // 708
-	{ "program-id", PROGRAM_ID }, // 709
-	{ "program", PROGRAM_kw }, // 710
-	{ "property", PROPERTY }, // 711
-	{ "prototype", PROTOTYPE }, // 712
-	{ "pseudotext", PSEUDOTEXT }, // 713
-	{ "quotes", QUOTES }, // 714
-	{ "quote", QUOTES }, // 714
-	{ "random", RANDOM }, // 715
-	{ "random-seed", RANDOM_SEED }, // 716
-	{ "range", RANGE }, // 717
-	{ "raise", RAISE }, // 718
-	{ "raising", RAISING }, // 719
-	{ "rd", RD }, // 720
-	{ "record", RECORD }, // 721
-	{ "recording", RECORDING }, // 722
-	{ "records", RECORDS }, // 723
-	{ "recursive", RECURSIVE }, // 724
-	{ "redefines", REDEFINES }, // 725
-	{ "reel", REEL }, // 726
-	{ "reference", REFERENCE }, // 727
-	{ "relative", RELATIVE }, // 728
-	{ "rem", REM }, // 729
-	{ "remainder", REMAINDER }, // 730
-	{ "remarks", REMARKS }, // 731
-	{ "removal", REMOVAL }, // 732
-	{ "renames", RENAMES }, // 733
-	{ "replace", REPLACE }, // 734
-	{ "replacing", REPLACING }, // 735
-	{ "report", REPORT }, // 736
-	{ "reporting", REPORTING }, // 737
-	{ "reports", REPORTS }, // 738
-	{ "repository", REPOSITORY }, // 739
-	{ "rerun", RERUN }, // 740
-	{ "reserve", RESERVE }, // 741
-	{ "restricted", RESTRICTED }, // 742
-	{ "resume", RESUME }, // 743
-	{ "reverse", REVERSE }, // 744
-	{ "reversed", REVERSED }, // 745
-	{ "rewind", REWIND }, // 746
-	{ "rf", RF }, // 747
-	{ "rh", RH }, // 748
-	{ "right", RIGHT }, // 749
-	{ "rounded", ROUNDED }, // 750
-	{ "run", RUN }, // 751
-	{ "same", SAME }, // 752
-	{ "screen", SCREEN }, // 753
-	{ "sd", SD }, // 754
-	{ "seconds-from-formatted-time", SECONDS_FROM_FORMATTED_TIME }, // 755
-	{ "seconds-past-midnight", SECONDS_PAST_MIDNIGHT }, // 756
-	{ "security", SECURITY }, // 757
-	{ "separate", SEPARATE }, // 758
-	{ "sequence", SEQUENCE }, // 759
-	{ "sequential", SEQUENTIAL }, // 760
-	{ "sharing", SHARING }, // 761
-	{ "simple-exit", SIMPLE_EXIT }, // 762
-	{ "sign", SIGN }, // 763
-	{ "sin", SIN }, // 764
-	{ "size", SIZE }, // 765
-	{ "smallest-algebraic", SMALLEST_ALGEBRAIC }, // 766
-	{ "source", SOURCE }, // 767
-	{ "source-computer", SOURCE_COMPUTER }, // 768
-	{ "special-names", SPECIAL_NAMES }, // 769
-	{ "sqrt", SQRT }, // 770
-	{ "stack", STACK }, // 771
-	{ "standard", STANDARD }, // 772
-	{ "standard-1", STANDARD_1 }, // 773
-	{ "standard-deviation", STANDARD_DEVIATION }, // 774
-	{ "standard-compare", STANDARD_COMPARE }, // 775
-	{ "status", STATUS }, // 776
-	{ "strong", STRONG }, // 777
-	{ "substitute", SUBSTITUTE }, // 778
-	{ "sum", SUM }, // 779
-	{ "symbol", SYMBOL }, // 780
-	{ "symbolic", SYMBOLIC }, // 781
-	{ "synchronized", SYNCHRONIZED }, // 782
-	{ "tally", TALLY }, // 783
-	{ "tallying", TALLYING }, // 784
-	{ "tan", TAN }, // 785
-	{ "terminate", TERMINATE }, // 786
-	{ "test", TEST }, // 787
-	{ "test-date-yyyymmdd", TEST_DATE_YYYYMMDD }, // 788
-	{ "test-day-yyyyddd", TEST_DAY_YYYYDDD }, // 789
-	{ "test-formatted-datetime", TEST_FORMATTED_DATETIME }, // 790
-	{ "test-numval", TEST_NUMVAL }, // 791
-	{ "test-numval-c", TEST_NUMVAL_C }, // 792
-	{ "test-numval-f", TEST_NUMVAL_F }, // 793
-	{ "than", THAN }, // 794
-	{ "time", TIME }, // 795
-	{ "times", TIMES }, // 796
-	{ "to", TO }, // 797
-	{ "top", TOP }, // 798
-	{ "top-level", TOP_LEVEL }, // 799
-	{ "tracks", TRACKS }, // 800
-	{ "track-area", TRACK_AREA }, // 801
-	{ "trailing", TRAILING }, // 802
-	{ "transform", TRANSFORM }, // 803
-	{ "trim", TRIM }, // 804
-	{ "true", TRUE_kw }, // 805
-	{ "try", TRY }, // 806
-	{ "turn", TURN }, // 807
-	{ "type", TYPE }, // 808
-	{ "typedef", TYPEDEF }, // 809
-	{ "ulength", ULENGTH }, // 810
-	{ "unbounded", UNBOUNDED }, // 811
-	{ "unit", UNIT }, // 812
-	{ "units", UNITS }, // 813
-	{ "unit-record", UNIT_RECORD }, // 814
-	{ "until", UNTIL }, // 815
-	{ "up", UP }, // 816
-	{ "upon", UPON }, // 817
-	{ "upos", UPOS }, // 818
-	{ "upper-case", UPPER_CASE }, // 819
-	{ "usage", USAGE }, // 820
-	{ "using", USING }, // 821
-	{ "usubstr", USUBSTR }, // 822
-	{ "usupplementary", USUPPLEMENTARY }, // 823
-	{ "utility", UTILITY }, // 824
-	{ "uuid4", UUID4 }, // 825
-	{ "uvalid", UVALID }, // 826
-	{ "uwidth", UWIDTH }, // 827
-	{ "value", VALUE }, // 828
-	{ "variance", VARIANCE }, // 829
-	{ "varying", VARYING }, // 830
-	{ "volatile", VOLATILE }, // 831
-	{ "when-compiled", WHEN_COMPILED }, // 832
-	{ "with", WITH }, // 833
-	{ "working-storage", WORKING_STORAGE }, // 834
-	{ "xml", XML }, // 835
-	{ "xmlgenerate", XMLGENERATE }, // 836
-	{ "xmlparse", XMLPARSE }, // 837
-	{ "year-to-yyyy", YEAR_TO_YYYY }, // 838
-	{ "yyyyddd", YYYYDDD }, // 839
-	{ "yyyymmdd", YYYYMMDD }, // 840
-	{ "arithmetic", ARITHMETIC }, // 841
-	{ "attribute", ATTRIBUTE }, // 842
-	{ "auto", AUTO }, // 843
-	{ "automatic", AUTOMATIC }, // 844
-	{ "away-from-zero", AWAY_FROM_ZERO }, // 845
-	{ "background-color", BACKGROUND_COLOR }, // 846
-	{ "bell", BELL }, // 847
-	{ "binary-encoding", BINARY_ENCODING }, // 848
-	{ "blink", BLINK }, // 849
-	{ "capacity", CAPACITY }, // 850
-	{ "center", CENTER }, // 851
-	{ "classification", CLASSIFICATION }, // 852
-	{ "cycle", CYCLE }, // 853
-	{ "decimal-encoding", DECIMAL_ENCODING }, // 854
-	{ "entry-convention", ENTRY_CONVENTION }, // 855
-	{ "eol", EOL }, // 856
-	{ "eos", EOS }, // 857
-	{ "erase", ERASE }, // 858
-	{ "expands", EXPANDS }, // 859
-	{ "float-binary", FLOAT_BINARY }, // 860
-	{ "float-decimal", FLOAT_DECIMAL }, // 861
-	{ "foreground-color", FOREGROUND_COLOR }, // 862
-	{ "forever", FOREVER }, // 863
-	{ "full", FULL }, // 864
-	{ "highlight", HIGHLIGHT }, // 865
-	{ "high-order-left", HIGH_ORDER_LEFT }, // 866
-	{ "high-order-right", HIGH_ORDER_RIGHT }, // 867
-	{ "ignoring", IGNORING }, // 868
-	{ "implements", IMPLEMENTS }, // 869
-	{ "initialized", INITIALIZED }, // 870
-	{ "intermediate", INTERMEDIATE }, // 871
-	{ "lc-all", LC_ALL_kw }, // 872
-	{ "lc-collate", LC_COLLATE_kw }, // 873
-	{ "lc-ctype", LC_CTYPE_kw }, // 874
-	{ "lc-messages", LC_MESSAGES_kw }, // 875
-	{ "lc-monetary", LC_MONETARY_kw }, // 876
-	{ "lc-numeric", LC_NUMERIC_kw }, // 877
-	{ "lc-time", LC_TIME_kw }, // 878
-	{ "lowlight", LOWLIGHT }, // 879
-	{ "nearest-away-from-zero", NEAREST_AWAY_FROM_ZERO }, // 880
-	{ "nearest-even", NEAREST_EVEN }, // 881
-	{ "nearest-toward-zero", NEAREST_TOWARD_ZERO }, // 882
-	{ "none", NONE }, // 883
-	{ "normal", NORMAL }, // 884
-	{ "numbers", NUMBERS }, // 885
-	{ "prefixed", PREFIXED }, // 886
-	{ "previous", PREVIOUS }, // 887
-	{ "prohibited", PROHIBITED }, // 888
-	{ "relation", RELATION }, // 889
-	{ "required", REQUIRED }, // 890
-	{ "reverse-video", REVERSE_VIDEO }, // 891
-	{ "rounding", ROUNDING }, // 892
-	{ "seconds", SECONDS }, // 893
-	{ "secure", SECURE }, // 894
-	{ "short", SHORT }, // 895
-	{ "signed", SIGNED_kw }, // 896
-	{ "standard-binary", STANDARD_BINARY }, // 897
-	{ "standard-decimal", STANDARD_DECIMAL }, // 898
-	{ "statement", STATEMENT }, // 899
-	{ "step", STEP }, // 900
-	{ "structure", STRUCTURE }, // 901
-	{ "toward-greater", TOWARD_GREATER }, // 902
-	{ "toward-lesser", TOWARD_LESSER }, // 903
-	{ "truncation", TRUNCATION }, // 904
-	{ "ucs-4", UCS_4 }, // 905
-	{ "underline", UNDERLINE }, // 906
-	{ "unsigned", UNSIGNED_kw }, // 907
-	{ "utf-16", UTF_16 }, // 908
-	{ "utf-8", UTF_8 }, // 909
-	{ "address", ADDRESS }, // 910
-	{ "end-accept", END_ACCEPT }, // 911
-	{ "end-add", END_ADD }, // 912
-	{ "end-call", END_CALL }, // 913
-	{ "end-compute", END_COMPUTE }, // 914
-	{ "end-delete", END_DELETE }, // 915
-	{ "end-display", END_DISPLAY }, // 916
-	{ "end-divide", END_DIVIDE }, // 917
-	{ "end-evaluate", END_EVALUATE }, // 918
-	{ "end-multiply", END_MULTIPLY }, // 919
-	{ "end-perform", END_PERFORM }, // 920
-	{ "end-read", END_READ }, // 921
-	{ "end-return", END_RETURN }, // 922
-	{ "end-rewrite", END_REWRITE }, // 923
-	{ "end-search", END_SEARCH }, // 924
-	{ "end-start", END_START }, // 925
-	{ "end-string", END_STRING }, // 926
-	{ "end-subtract", END_SUBTRACT }, // 927
-	{ "end-unstring", END_UNSTRING }, // 928
-	{ "end-write", END_WRITE }, // 929
-	{ "end-if", END_IF }, // 930
-	{ "thru", THRU }, // 931
-	{ "through", THRU }, // 931
-	{ "or", OR }, // 932
-	{ "and", AND }, // 933
-	{ "not", NOT }, // 934
-	{ "ne", NE }, // 935
-	{ "le", LE }, // 936
-	{ "ge", GE }, // 937
-	{ "pow", POW }, // 938
-	{ "neg", NEG }, // 939
+	{ "call-convention", CALL_CONVENTION }, // 391
+	{ "call-cobol", CALL_COBOL }, // 392
+	{ "call-verbatim", CALL_VERBATIM }, // 393
+	{ "cdf-push", CDF_PUSH }, // 394
+	{ "cdf-pop", CDF_POP }, // 395
+	{ "source-format", SOURCE_FORMAT }, // 396
+	{ "if", IF }, // 397
+	{ "then", THEN }, // 398
+	{ "else", ELSE }, // 399
+	{ "sentence", SENTENCE }, // 400
+	{ "accept", ACCEPT }, // 401
+	{ "add", ADD }, // 402
+	{ "alter", ALTER }, // 403
+	{ "call", CALL }, // 404
+	{ "cancel", CANCEL }, // 405
+	{ "close", CLOSE }, // 406
+	{ "compute", COMPUTE }, // 407
+	{ "continue", CONTINUE }, // 408
+	{ "delete", DELETE }, // 409
+	{ "display", DISPLAY }, // 410
+	{ "divide", DIVIDE }, // 411
+	{ "evaluate", EVALUATE }, // 412
+	{ "exit", EXIT }, // 413
+	{ "filler", FILLER_kw }, // 414
+	{ "goback", GOBACK }, // 415
+	{ "goto", GOTO }, // 416
+	{ "initialize", INITIALIZE }, // 417
+	{ "inspect", INSPECT }, // 418
+	{ "merge", MERGE }, // 419
+	{ "move", MOVE }, // 420
+	{ "multiply", MULTIPLY }, // 421
+	{ "open", OPEN }, // 422
+	{ "paragraph", PARAGRAPH }, // 423
+	{ "read", READ }, // 424
+	{ "release", RELEASE }, // 425
+	{ "return", RETURN }, // 426
+	{ "rewrite", REWRITE }, // 427
+	{ "search", SEARCH }, // 428
+	{ "set", SET }, // 429
+	{ "select", SELECT }, // 430
+	{ "sort", SORT }, // 431
+	{ "sort-merge", SORT_MERGE }, // 432
+	{ "string", STRING_kw }, // 433
+	{ "stop", STOP }, // 434
+	{ "subtract", SUBTRACT }, // 435
+	{ "start", START }, // 436
+	{ "unstring", UNSTRING }, // 437
+	{ "write", WRITE }, // 438
+	{ "when", WHEN }, // 439
+	{ "argument-number", ARGUMENT_NUMBER }, // 440
+	{ "argument-value", ARGUMENT_VALUE }, // 441
+	{ "environment-name", ENVIRONMENT_NAME }, // 442
+	{ "environment-value", ENVIRONMENT_VALUE }, // 443
+	{ "abs", ABS }, // 444
+	{ "access", ACCESS }, // 445
+	{ "acos", ACOS }, // 446
+	{ "actual", ACTUAL }, // 447
+	{ "advancing", ADVANCING }, // 448
+	{ "after", AFTER }, // 449
+	{ "all", ALL }, // 450
+	{ "allocate", ALLOCATE }, // 451
+	{ "alphabet", ALPHABET }, // 452
+	{ "alphabetic", ALPHABETIC }, // 453
+	{ "alphabetic-lower", ALPHABETIC_LOWER }, // 454
+	{ "alphabetic-upper", ALPHABETIC_UPPER }, // 455
+	{ "alphanumeric", ALPHANUMERIC }, // 456
+	{ "alphanumeric-edited", ALPHANUMERIC_EDITED }, // 457
+	{ "also", ALSO }, // 458
+	{ "alternate", ALTERNATE }, // 459
+	{ "annuity", ANNUITY }, // 460
+	{ "anum", ANUM }, // 461
+	{ "any", ANY }, // 462
+	{ "anycase", ANYCASE }, // 463
+	{ "apply", APPLY }, // 464
+	{ "are", ARE }, // 465
+	{ "area", AREA }, // 466
+	{ "areas", AREAS }, // 467
+	{ "as", AS }, // 468
+	{ "ascending", ASCENDING }, // 469
+	{ "activating", ACTIVATING }, // 470
+	{ "asin", ASIN }, // 471
+	{ "assign", ASSIGN }, // 472
+	{ "at", AT }, // 473
+	{ "atan", ATAN }, // 474
+	{ "based", BASED }, // 475
+	{ "baseconvert", BASECONVERT }, // 476
+	{ "before", BEFORE }, // 477
+	{ "binary", BINARY }, // 478
+	{ "bit", BIT }, // 479
+	{ "bit-of", BIT_OF }, // 480
+	{ "bit-to-char", BIT_TO_CHAR }, // 481
+	{ "blank", BLANK }, // 482
+	{ "block", BLOCK_kw }, // 483
+	{ "boolean-of-integer", BOOLEAN_OF_INTEGER }, // 484
+	{ "bottom", BOTTOM }, // 485
+	{ "by", BY }, // 486
+	{ "byte", BYTE }, // 487
+	{ "byte-length", BYTE_LENGTH }, // 488
+	{ "cf", CF }, // 489
+	{ "ch", CH }, // 490
+	{ "changed", CHANGED }, // 491
+	{ "char", CHAR }, // 492
+	{ "char-national", CHAR_NATIONAL }, // 493
+	{ "character", CHARACTER }, // 494
+	{ "characters", CHARACTERS }, // 495
+	{ "checking", CHECKING }, // 496
+	{ "class", CLASS }, // 497
+	{ "cobol", COBOL }, // 498
+	{ "code", CODE }, // 499
+	{ "code-set", CODESET }, // 500
+	{ "collating", COLLATING }, // 501
+	{ "column", COLUMN }, // 502
+	{ "combined-datetime", COMBINED_DATETIME }, // 503
+	{ "comma", COMMA }, // 504
+	{ "command-line", COMMAND_LINE }, // 505
+	{ "command-line-count", COMMAND_LINE_COUNT }, // 506
+	{ "commit", COMMIT }, // 507
+	{ "common", COMMON }, // 508
+	{ "concat", CONCAT }, // 509
+	{ "condition", CONDITION }, // 510
+	{ "configuration", CONFIGURATION_SECT }, // 511
+	{ "contains", CONTAINS }, // 512
+	{ "content", CONTENT }, // 513
+	{ "control", CONTROL }, // 514
+	{ "controls", CONTROLS }, // 515
+	{ "convert", CONVERT }, // 516
+	{ "converting", CONVERTING }, // 517
+	{ "corresponding", CORRESPONDING }, // 518
+	{ "cos", COS }, // 519
+	{ "count", COUNT }, // 520
+	{ "currency", CURRENCY }, // 521
+	{ "current", CURRENT }, // 522
+	{ "current-date", CURRENT_DATE }, // 523
+	{ "data", DATA }, // 524
+	{ "date", DATE }, // 525
+	{ "date-compiled", DATE_COMPILED }, // 526
+	{ "date-of-integer", DATE_OF_INTEGER }, // 527
+	{ "date-to-yyyymmdd", DATE_TO_YYYYMMDD }, // 528
+	{ "date-written", DATE_WRITTEN }, // 529
+	{ "day", DAY }, // 530
+	{ "day-of-integer", DAY_OF_INTEGER }, // 531
+	{ "day-of-week", DAY_OF_WEEK }, // 532
+	{ "day-to-yyyyddd", DAY_TO_YYYYDDD }, // 533
+	{ "dbcs", DBCS }, // 534
+	{ "de", DE }, // 535
+	{ "debugging", DEBUGGING }, // 536
+	{ "decimal-point", DECIMAL_POINT }, // 537
+	{ "declaratives", DECLARATIVES }, // 538
+	{ "default", DEFAULT }, // 539
+	{ "delimited", DELIMITED }, // 540
+	{ "delimiter", DELIMITER }, // 541
+	{ "depending", DEPENDING }, // 542
+	{ "descending", DESCENDING }, // 543
+	{ "detail", DETAIL }, // 544
+	{ "direct", DIRECT }, // 545
+	{ "direct-access", DIRECT_ACCESS }, // 546
+	{ "down", DOWN }, // 547
+	{ "duplicates", DUPLICATES }, // 548
+	{ "dynamic", DYNAMIC }, // 549
+	{ "e", E }, // 550
+	{ "ebcdic", EBCDIC }, // 551
+	{ "ec", EC }, // 552
+	{ "egcs", EGCS }, // 553
+	{ "entry", ENTRY }, // 554
+	{ "environment", ENVIRONMENT }, // 555
+	{ "equal", EQUAL }, // 556
+	{ "every", EVERY }, // 557
+	{ "examine", EXAMINE }, // 558
+	{ "exhibit", EXHIBIT }, // 559
+	{ "exp", EXP }, // 560
+	{ "exp10", EXP10 }, // 561
+	{ "extend", EXTEND }, // 562
+	{ "external", EXTERNAL }, // 563
+	{ "exception-file", EXCEPTION_FILE }, // 564
+	{ "exception-file-n", EXCEPTION_FILE_N }, // 565
+	{ "exception-location", EXCEPTION_LOCATION }, // 566
+	{ "exception-location-n", EXCEPTION_LOCATION_N }, // 567
+	{ "exception-statement", EXCEPTION_STATEMENT }, // 568
+	{ "exception-status", EXCEPTION_STATUS }, // 569
+	{ "factorial", FACTORIAL }, // 570
+	{ "false", FALSE_kw }, // 571
+	{ "fd", FD }, // 572
+	{ "file-control", FILE_CONTROL }, // 573
+	{ "file", FILE_KW }, // 574
+	{ "file-limit", FILE_LIMIT }, // 575
+	{ "final", FINAL }, // 576
+	{ "finally", FINALLY }, // 577
+	{ "find-string", FIND_STRING }, // 578
+	{ "first", FIRST }, // 579
+	{ "fixed", FIXED }, // 580
+	{ "footing", FOOTING }, // 581
+	{ "for", FOR }, // 582
+	{ "formatted-current-date", FORMATTED_CURRENT_DATE }, // 583
+	{ "formatted-date", FORMATTED_DATE }, // 584
+	{ "formatted-datetime", FORMATTED_DATETIME }, // 585
+	{ "formatted-time", FORMATTED_TIME }, // 586
+	{ "form-overflow", FORM_OVERFLOW }, // 587
+	{ "free", FREE }, // 588
+	{ "fraction-part", FRACTION_PART }, // 589
+	{ "from", FROM }, // 590
+	{ "function", FUNCTION }, // 591
+	{ "generate", GENERATE }, // 592
+	{ "giving", GIVING }, // 593
+	{ "global", GLOBAL }, // 594
+	{ "go", GO }, // 595
+	{ "group", GROUP }, // 596
+	{ "heading", HEADING }, // 597
+	{ "hex", HEX }, // 598
+	{ "hex-of", HEX_OF }, // 599
+	{ "hex-to-char", HEX_TO_CHAR }, // 600
+	{ "high-values", HIGH_VALUES }, // 601
+	{ "highest-algebraic", HIGHEST_ALGEBRAIC }, // 602
+	{ "hold", HOLD }, // 603
+	{ "ibm-360", IBM_360 }, // 604
+	{ "in", IN }, // 605
+	{ "include", INCLUDE }, // 606
+	{ "index", INDEX }, // 607
+	{ "indexed", INDEXED }, // 608
+	{ "indicate", INDICATE }, // 609
+	{ "initial", INITIAL_kw }, // 610
+	{ "initiate", INITIATE }, // 611
+	{ "input", INPUT }, // 612
+	{ "installation", INSTALLATION }, // 613
+	{ "interface", INTERFACE }, // 614
+	{ "integer", INTEGER }, // 615
+	{ "integer-of-boolean", INTEGER_OF_BOOLEAN }, // 616
+	{ "integer-of-date", INTEGER_OF_DATE }, // 617
+	{ "integer-of-day", INTEGER_OF_DAY }, // 618
+	{ "integer-of-formatted-date", INTEGER_OF_FORMATTED_DATE }, // 619
+	{ "integer-part", INTEGER_PART }, // 620
+	{ "into", INTO }, // 621
+	{ "intrinsic", INTRINSIC }, // 622
+	{ "invoke", INVOKE }, // 623
+	{ "i-o", IO }, // 624
+	{ "i-o-control", IO_CONTROL }, // 625
+	{ "is", IS }, // 626
+	{ "isnt", ISNT }, // 627
+	{ "kanji", KANJI }, // 628
+	{ "key", KEY }, // 629
+	{ "label", LABEL }, // 630
+	{ "last", LAST }, // 631
+	{ "leading", LEADING }, // 632
+	{ "left", LEFT }, // 633
+	{ "length", LENGTH }, // 634
+	{ "length-of", LENGTH_OF }, // 635
+	{ "limit", LIMIT }, // 636
+	{ "limits", LIMITS }, // 637
+	{ "line", LINE }, // 638
+	{ "lines", LINES }, // 639
+	{ "line-counter", LINE_COUNTER }, // 640
+	{ "linage", LINAGE }, // 641
+	{ "linkage", LINKAGE }, // 642
+	{ "locale", LOCALE }, // 643
+	{ "locale-compare", LOCALE_COMPARE }, // 644
+	{ "locale-date", LOCALE_DATE }, // 645
+	{ "locale-time", LOCALE_TIME }, // 646
+	{ "locale-time-from-seconds", LOCALE_TIME_FROM_SECONDS }, // 647
+	{ "local-storage", LOCAL_STORAGE }, // 648
+	{ "location", LOCATION }, // 649
+	{ "lock", LOCK }, // 650
+	{ "lock-on", LOCK_ON }, // 651
+	{ "log", LOG }, // 652
+	{ "log10", LOG10 }, // 653
+	{ "lower-case", LOWER_CASE }, // 654
+	{ "low-values", LOW_VALUES }, // 655
+	{ "lowest-algebraic", LOWEST_ALGEBRAIC }, // 656
+	{ "lparen", LPAREN }, // 657
+	{ "manual", MANUAL }, // 658
+	{ "maxx", MAXX }, // 659
+	{ "mean", MEAN }, // 660
+	{ "median", MEDIAN }, // 661
+	{ "midrange", MIDRANGE }, // 662
+	{ "minn", MINN }, // 663
+	{ "multiple", MULTIPLE }, // 664
+	{ "mod", MOD }, // 665
+	{ "mode", MODE }, // 666
+	{ "module-name", MODULE_NAME }, // 667
+	{ "named", NAMED }, // 668
+	{ "nat", NAT }, // 669
+	{ "national", NATIONAL }, // 670
+	{ "national-edited", NATIONAL_EDITED }, // 671
+	{ "national-of", NATIONAL_OF }, // 672
+	{ "native", NATIVE }, // 673
+	{ "nested", NESTED }, // 674
+	{ "next", NEXT }, // 675
+	{ "no", NO }, // 676
+	{ "note", NOTE }, // 677
+	{ "nulls", NULLS }, // 678
+	{ "null", NULLS }, // 678
+	{ "nullptr", NULLPTR }, // 679
+	{ "numeric", NUMERIC }, // 680
+	{ "numeric-edited", NUMERIC_EDITED }, // 681
+	{ "numval", NUMVAL }, // 682
+	{ "numval-c", NUMVAL_C }, // 683
+	{ "numval-f", NUMVAL_F }, // 684
+	{ "occurs", OCCURS }, // 685
+	{ "of", OF }, // 686
+	{ "off", OFF }, // 687
+	{ "omitted", OMITTED }, // 688
+	{ "on", ON }, // 689
+	{ "only", ONLY }, // 690
+	{ "optional", OPTIONAL }, // 691
+	{ "options", OPTIONS }, // 692
+	{ "ord", ORD }, // 693
+	{ "order", ORDER }, // 694
+	{ "ord-max", ORD_MAX }, // 695
+	{ "ord-min", ORD_MIN }, // 696
+	{ "organization", ORGANIZATION }, // 697
+	{ "other", OTHER }, // 698
+	{ "otherwise", OTHERWISE }, // 699
+	{ "output", OUTPUT }, // 700
+	{ "packed-decimal", PACKED_DECIMAL }, // 701
+	{ "padding", PADDING }, // 702
+	{ "page", PAGE }, // 703
+	{ "page-counter", PAGE_COUNTER }, // 704
+	{ "pf", PF }, // 705
+	{ "ph", PH }, // 706
+	{ "pi", PI }, // 707
+	{ "pic", PIC }, // 708
+	{ "picture", PICTURE }, // 709
+	{ "plus", PLUS }, // 710
+	{ "present-value", PRESENT_VALUE }, // 711
+	{ "print-switch", PRINT_SWITCH }, // 712
+	{ "procedure", PROCEDURE }, // 713
+	{ "procedures", PROCEDURES }, // 714
+	{ "proceed", PROCEED }, // 715
+	{ "process", PROCESS }, // 716
+	{ "program-id", PROGRAM_ID }, // 717
+	{ "program", PROGRAM_kw }, // 718
+	{ "property", PROPERTY }, // 719
+	{ "prototype", PROTOTYPE }, // 720
+	{ "pseudotext", PSEUDOTEXT }, // 721
+	{ "quotes", QUOTES }, // 722
+	{ "quote", QUOTES }, // 722
+	{ "random", RANDOM }, // 723
+	{ "random-seed", RANDOM_SEED }, // 724
+	{ "range", RANGE }, // 725
+	{ "raise", RAISE }, // 726
+	{ "raising", RAISING }, // 727
+	{ "rd", RD }, // 728
+	{ "record", RECORD }, // 729
+	{ "recording", RECORDING }, // 730
+	{ "records", RECORDS }, // 731
+	{ "recursive", RECURSIVE }, // 732
+	{ "redefines", REDEFINES }, // 733
+	{ "reel", REEL }, // 734
+	{ "reference", REFERENCE }, // 735
+	{ "relative", RELATIVE }, // 736
+	{ "rem", REM }, // 737
+	{ "remainder", REMAINDER }, // 738
+	{ "remarks", REMARKS }, // 739
+	{ "removal", REMOVAL }, // 740
+	{ "renames", RENAMES }, // 741
+	{ "replace", REPLACE }, // 742
+	{ "replacing", REPLACING }, // 743
+	{ "report", REPORT }, // 744
+	{ "reporting", REPORTING }, // 745
+	{ "reports", REPORTS }, // 746
+	{ "repository", REPOSITORY }, // 747
+	{ "rerun", RERUN }, // 748
+	{ "reserve", RESERVE }, // 749
+	{ "restricted", RESTRICTED }, // 750
+	{ "resume", RESUME }, // 751
+	{ "reverse", REVERSE }, // 752
+	{ "reversed", REVERSED }, // 753
+	{ "rewind", REWIND }, // 754
+	{ "rf", RF }, // 755
+	{ "rh", RH }, // 756
+	{ "right", RIGHT }, // 757
+	{ "rounded", ROUNDED }, // 758
+	{ "run", RUN }, // 759
+	{ "same", SAME }, // 760
+	{ "screen", SCREEN }, // 761
+	{ "sd", SD }, // 762
+	{ "seconds-from-formatted-time", SECONDS_FROM_FORMATTED_TIME }, // 763
+	{ "seconds-past-midnight", SECONDS_PAST_MIDNIGHT }, // 764
+	{ "security", SECURITY }, // 765
+	{ "separate", SEPARATE }, // 766
+	{ "sequence", SEQUENCE }, // 767
+	{ "sequential", SEQUENTIAL }, // 768
+	{ "sharing", SHARING }, // 769
+	{ "simple-exit", SIMPLE_EXIT }, // 770
+	{ "sign", SIGN }, // 771
+	{ "sin", SIN }, // 772
+	{ "size", SIZE }, // 773
+	{ "smallest-algebraic", SMALLEST_ALGEBRAIC }, // 774
+	{ "source", SOURCE }, // 775
+	{ "source-computer", SOURCE_COMPUTER }, // 776
+	{ "special-names", SPECIAL_NAMES }, // 777
+	{ "sqrt", SQRT }, // 778
+	{ "stack", STACK }, // 779
+	{ "standard", STANDARD }, // 780
+	{ "standard-1", STANDARD_1 }, // 781
+	{ "standard-deviation", STANDARD_DEVIATION }, // 782
+	{ "standard-compare", STANDARD_COMPARE }, // 783
+	{ "status", STATUS }, // 784
+	{ "strong", STRONG }, // 785
+	{ "substitute", SUBSTITUTE }, // 786
+	{ "sum", SUM }, // 787
+	{ "symbol", SYMBOL }, // 788
+	{ "symbolic", SYMBOLIC }, // 789
+	{ "synchronized", SYNCHRONIZED }, // 790
+	{ "tally", TALLY }, // 791
+	{ "tallying", TALLYING }, // 792
+	{ "tan", TAN }, // 793
+	{ "terminate", TERMINATE }, // 794
+	{ "test", TEST }, // 795
+	{ "test-date-yyyymmdd", TEST_DATE_YYYYMMDD }, // 796
+	{ "test-day-yyyyddd", TEST_DAY_YYYYDDD }, // 797
+	{ "test-formatted-datetime", TEST_FORMATTED_DATETIME }, // 798
+	{ "test-numval", TEST_NUMVAL }, // 799
+	{ "test-numval-c", TEST_NUMVAL_C }, // 800
+	{ "test-numval-f", TEST_NUMVAL_F }, // 801
+	{ "than", THAN }, // 802
+	{ "time", TIME }, // 803
+	{ "times", TIMES }, // 804
+	{ "to", TO }, // 805
+	{ "top", TOP }, // 806
+	{ "top-level", TOP_LEVEL }, // 807
+	{ "tracks", TRACKS }, // 808
+	{ "track-area", TRACK_AREA }, // 809
+	{ "trailing", TRAILING }, // 810
+	{ "transform", TRANSFORM }, // 811
+	{ "trim", TRIM }, // 812
+	{ "true", TRUE_kw }, // 813
+	{ "try", TRY }, // 814
+	{ "turn", TURN }, // 815
+	{ "type", TYPE }, // 816
+	{ "typedef", TYPEDEF }, // 817
+	{ "ulength", ULENGTH }, // 818
+	{ "unbounded", UNBOUNDED }, // 819
+	{ "unit", UNIT }, // 820
+	{ "units", UNITS }, // 821
+	{ "unit-record", UNIT_RECORD }, // 822
+	{ "until", UNTIL }, // 823
+	{ "up", UP }, // 824
+	{ "upon", UPON }, // 825
+	{ "upos", UPOS }, // 826
+	{ "upper-case", UPPER_CASE }, // 827
+	{ "usage", USAGE }, // 828
+	{ "using", USING }, // 829
+	{ "usubstr", USUBSTR }, // 830
+	{ "usupplementary", USUPPLEMENTARY }, // 831
+	{ "utility", UTILITY }, // 832
+	{ "uuid4", UUID4 }, // 833
+	{ "uvalid", UVALID }, // 834
+	{ "uwidth", UWIDTH }, // 835
+	{ "value", VALUE }, // 836
+	{ "variance", VARIANCE }, // 837
+	{ "varying", VARYING }, // 838
+	{ "volatile", VOLATILE }, // 839
+	{ "when-compiled", WHEN_COMPILED }, // 840
+	{ "with", WITH }, // 841
+	{ "working-storage", WORKING_STORAGE }, // 842
+	{ "xml", XML }, // 843
+	{ "xmlgenerate", XMLGENERATE }, // 844
+	{ "xmlparse", XMLPARSE }, // 845
+	{ "year-to-yyyy", YEAR_TO_YYYY }, // 846
+	{ "yyyyddd", YYYYDDD }, // 847
+	{ "yyyymmdd", YYYYMMDD }, // 848
+	{ "arithmetic", ARITHMETIC }, // 849
+	{ "attribute", ATTRIBUTE }, // 850
+	{ "auto", AUTO }, // 851
+	{ "automatic", AUTOMATIC }, // 852
+	{ "away-from-zero", AWAY_FROM_ZERO }, // 853
+	{ "background-color", BACKGROUND_COLOR }, // 854
+	{ "bell", BELL }, // 855
+	{ "binary-encoding", BINARY_ENCODING }, // 856
+	{ "blink", BLINK }, // 857
+	{ "capacity", CAPACITY }, // 858
+	{ "center", CENTER }, // 859
+	{ "classification", CLASSIFICATION }, // 860
+	{ "cycle", CYCLE }, // 861
+	{ "decimal-encoding", DECIMAL_ENCODING }, // 862
+	{ "entry-convention", ENTRY_CONVENTION }, // 863
+	{ "eol", EOL }, // 864
+	{ "eos", EOS }, // 865
+	{ "erase", ERASE }, // 866
+	{ "expands", EXPANDS }, // 867
+	{ "float-binary", FLOAT_BINARY }, // 868
+	{ "float-decimal", FLOAT_DECIMAL }, // 869
+	{ "foreground-color", FOREGROUND_COLOR }, // 870
+	{ "forever", FOREVER }, // 871
+	{ "full", FULL }, // 872
+	{ "highlight", HIGHLIGHT }, // 873
+	{ "high-order-left", HIGH_ORDER_LEFT }, // 874
+	{ "high-order-right", HIGH_ORDER_RIGHT }, // 875
+	{ "ignoring", IGNORING }, // 876
+	{ "implements", IMPLEMENTS }, // 877
+	{ "initialized", INITIALIZED }, // 878
+	{ "intermediate", INTERMEDIATE }, // 879
+	{ "lc-all", LC_ALL_kw }, // 880
+	{ "lc-collate", LC_COLLATE_kw }, // 881
+	{ "lc-ctype", LC_CTYPE_kw }, // 882
+	{ "lc-messages", LC_MESSAGES_kw }, // 883
+	{ "lc-monetary", LC_MONETARY_kw }, // 884
+	{ "lc-numeric", LC_NUMERIC_kw }, // 885
+	{ "lc-time", LC_TIME_kw }, // 886
+	{ "lowlight", LOWLIGHT }, // 887
+	{ "nearest-away-from-zero", NEAREST_AWAY_FROM_ZERO }, // 888
+	{ "nearest-even", NEAREST_EVEN }, // 889
+	{ "nearest-toward-zero", NEAREST_TOWARD_ZERO }, // 890
+	{ "none", NONE }, // 891
+	{ "normal", NORMAL }, // 892
+	{ "numbers", NUMBERS }, // 893
+	{ "prefixed", PREFIXED }, // 894
+	{ "previous", PREVIOUS }, // 895
+	{ "prohibited", PROHIBITED }, // 896
+	{ "relation", RELATION }, // 897
+	{ "required", REQUIRED }, // 898
+	{ "reverse-video", REVERSE_VIDEO }, // 899
+	{ "rounding", ROUNDING }, // 900
+	{ "seconds", SECONDS }, // 901
+	{ "secure", SECURE }, // 902
+	{ "short", SHORT }, // 903
+	{ "signed", SIGNED_kw }, // 904
+	{ "standard-binary", STANDARD_BINARY }, // 905
+	{ "standard-decimal", STANDARD_DECIMAL }, // 906
+	{ "statement", STATEMENT }, // 907
+	{ "step", STEP }, // 908
+	{ "structure", STRUCTURE }, // 909
+	{ "toward-greater", TOWARD_GREATER }, // 910
+	{ "toward-lesser", TOWARD_LESSER }, // 911
+	{ "truncation", TRUNCATION }, // 912
+	{ "ucs-4", UCS_4 }, // 913
+	{ "underline", UNDERLINE }, // 914
+	{ "unsigned", UNSIGNED_kw }, // 915
+	{ "utf-16", UTF_16 }, // 916
+	{ "utf-8", UTF_8 }, // 917
+	{ "address", ADDRESS }, // 918
+	{ "end-accept", END_ACCEPT }, // 919
+	{ "end-add", END_ADD }, // 920
+	{ "end-call", END_CALL }, // 921
+	{ "end-compute", END_COMPUTE }, // 922
+	{ "end-delete", END_DELETE }, // 923
+	{ "end-display", END_DISPLAY }, // 924
+	{ "end-divide", END_DIVIDE }, // 925
+	{ "end-evaluate", END_EVALUATE }, // 926
+	{ "end-multiply", END_MULTIPLY }, // 927
+	{ "end-perform", END_PERFORM }, // 928
+	{ "end-read", END_READ }, // 929
+	{ "end-return", END_RETURN }, // 930
+	{ "end-rewrite", END_REWRITE }, // 931
+	{ "end-search", END_SEARCH }, // 932
+	{ "end-start", END_START }, // 933
+	{ "end-string", END_STRING }, // 934
+	{ "end-subtract", END_SUBTRACT }, // 935
+	{ "end-unstring", END_UNSTRING }, // 936
+	{ "end-write", END_WRITE }, // 937
+	{ "end-if", END_IF }, // 938
+	{ "thru", THRU }, // 939
+	{ "through", THRU }, // 939
+	{ "or", OR }, // 940
+	{ "and", AND }, // 941
+	{ "not", NOT }, // 942
+	{ "ne", NE }, // 943
+	{ "le", LE }, // 944
+	{ "ge", GE }, // 945
+	{ "pow", POW }, // 946
+	{ "neg", NEG }, // 947
 };
 
 // cppcheck-suppress useInitializationList
@@ -826,553 +834,561 @@ token_names = {
 	"CDF-EVALUATE", // 130 (388)
 	"CDF-WHEN", // 131 (389)
 	"CDF-END-EVALUATE", // 132 (390)
-	"CALL-COBOL", // 133 (391)
-	"CALL-VERBATIM", // 134 (392)
-	"IF", // 135 (393)
-	"THEN", // 136 (394)
-	"ELSE", // 137 (395)
-	"SENTENCE", // 138 (396)
-	"ACCEPT", // 139 (397)
-	"ADD", // 140 (398)
-	"ALTER", // 141 (399)
-	"CALL", // 142 (400)
-	"CANCEL", // 143 (401)
-	"CLOSE", // 144 (402)
-	"COMPUTE", // 145 (403)
-	"CONTINUE", // 146 (404)
-	"DELETE", // 147 (405)
-	"DISPLAY", // 148 (406)
-	"DIVIDE", // 149 (407)
-	"EVALUATE", // 150 (408)
-	"EXIT", // 151 (409)
-	"FILLER", // 152 (410)
-	"GOBACK", // 153 (411)
-	"GOTO", // 154 (412)
-	"INITIALIZE", // 155 (413)
-	"INSPECT", // 156 (414)
-	"MERGE", // 157 (415)
-	"MOVE", // 158 (416)
-	"MULTIPLY", // 159 (417)
-	"OPEN", // 160 (418)
-	"PARAGRAPH", // 161 (419)
-	"READ", // 162 (420)
-	"RELEASE", // 163 (421)
-	"RETURN", // 164 (422)
-	"REWRITE", // 165 (423)
-	"SEARCH", // 166 (424)
-	"SET", // 167 (425)
-	"SELECT", // 168 (426)
-	"SORT", // 169 (427)
-	"SORT-MERGE", // 170 (428)
-	"STRING", // 171 (429)
-	"STOP", // 172 (430)
-	"SUBTRACT", // 173 (431)
-	"START", // 174 (432)
-	"UNSTRING", // 175 (433)
-	"WRITE", // 176 (434)
-	"WHEN", // 177 (435)
-	"ABS", // 178 (436)
-	"ACCESS", // 179 (437)
-	"ACOS", // 180 (438)
-	"ACTUAL", // 181 (439)
-	"ADVANCING", // 182 (440)
-	"AFTER", // 183 (441)
-	"ALL", // 184 (442)
-	"ALLOCATE", // 185 (443)
-	"ALPHABET", // 186 (444)
-	"ALPHABETIC", // 187 (445)
-	"ALPHABETIC-LOWER", // 188 (446)
-	"ALPHABETIC-UPPER", // 189 (447)
-	"ALPHANUMERIC", // 190 (448)
-	"ALPHANUMERIC-EDITED", // 191 (449)
-	"ALSO", // 192 (450)
-	"ALTERNATE", // 193 (451)
-	"ANNUITY", // 194 (452)
-	"ANUM", // 195 (453)
-	"ANY", // 196 (454)
-	"ANYCASE", // 197 (455)
-	"APPLY", // 198 (456)
-	"ARE", // 199 (457)
-	"AREA", // 200 (458)
-	"AREAS", // 201 (459)
-	"AS", // 202 (460)
-	"ASCENDING", // 203 (461)
-	"ACTIVATING", // 204 (462)
-	"ASIN", // 205 (463)
-	"ASSIGN", // 206 (464)
-	"AT", // 207 (465)
-	"ATAN", // 208 (466)
-	"BASED", // 209 (467)
-	"BASECONVERT", // 210 (468)
-	"BEFORE", // 211 (469)
-	"BINARY", // 212 (470)
-	"BIT", // 213 (471)
-	"BIT-OF", // 214 (472)
-	"BIT-TO-CHAR", // 215 (473)
-	"BLANK", // 216 (474)
-	"BLOCK", // 217 (475)
-	"BOOLEAN-OF-INTEGER", // 218 (476)
-	"BOTTOM", // 219 (477)
-	"BY", // 220 (478)
-	"BYTE", // 221 (479)
-	"BYTE-LENGTH", // 222 (480)
-	"CF", // 223 (481)
-	"CH", // 224 (482)
-	"CHANGED", // 225 (483)
-	"CHAR", // 226 (484)
-	"CHAR-NATIONAL", // 227 (485)
-	"CHARACTER", // 228 (486)
-	"CHARACTERS", // 229 (487)
-	"CHECKING", // 230 (488)
-	"CLASS", // 231 (489)
-	"COBOL", // 232 (490)
-	"CODE", // 233 (491)
-	"CODE-SET", // 234 (492)
-	"COLLATING", // 235 (493)
-	"COLUMN", // 236 (494)
-	"COMBINED-DATETIME", // 237 (495)
-	"COMMA", // 238 (496)
-	"COMMAND-LINE", // 239 (497)
-	"COMMAND-LINE-COUNT", // 240 (498)
-	"COMMIT", // 241 (499)
-	"COMMON", // 242 (500)
-	"CONCAT", // 243 (501)
-	"CONDITION", // 244 (502)
-	"CONFIGURATION", // 245 (503)
-	"CONTAINS", // 246 (504)
-	"CONTENT", // 247 (505)
-	"CONTROL", // 248 (506)
-	"CONTROLS", // 249 (507)
-	"CONVERT", // 250 (508)
-	"CONVERTING", // 251 (509)
-	"CORRESPONDING", // 252 (510)
-	"COS", // 253 (511)
-	"COUNT", // 254 (512)
-	"CURRENCY", // 255 (513)
-	"CURRENT", // 256 (514)
-	"CURRENT-DATE", // 257 (515)
-	"DATA", // 258 (516)
-	"DATE", // 259 (517)
-	"DATE-COMPILED", // 260 (518)
-	"DATE-OF-INTEGER", // 261 (519)
-	"DATE-TO-YYYYMMDD", // 262 (520)
-	"DATE-WRITTEN", // 263 (521)
-	"DAY", // 264 (522)
-	"DAY-OF-INTEGER", // 265 (523)
-	"DAY-OF-WEEK", // 266 (524)
-	"DAY-TO-YYYYDDD", // 267 (525)
-	"DBCS", // 268 (526)
-	"DE", // 269 (527)
-	"DEBUGGING", // 270 (528)
-	"DECIMAL-POINT", // 271 (529)
-	"DECLARATIVES", // 272 (530)
-	"DEFAULT", // 273 (531)
-	"DELIMITED", // 274 (532)
-	"DELIMITER", // 275 (533)
-	"DEPENDING", // 276 (534)
-	"DESCENDING", // 277 (535)
-	"DETAIL", // 278 (536)
-	"DIRECT", // 279 (537)
-	"DIRECT-ACCESS", // 280 (538)
-	"DOWN", // 281 (539)
-	"DUPLICATES", // 282 (540)
-	"DYNAMIC", // 283 (541)
-	"E", // 284 (542)
-	"EBCDIC", // 285 (543)
-	"EC", // 286 (544)
-	"EGCS", // 287 (545)
-	"ENTRY", // 288 (546)
-	"ENVIRONMENT", // 289 (547)
-	"EQUAL", // 290 (548)
-	"EVERY", // 291 (549)
-	"EXAMINE", // 292 (550)
-	"EXHIBIT", // 293 (551)
-	"EXP", // 294 (552)
-	"EXP10", // 295 (553)
-	"EXTEND", // 296 (554)
-	"EXTERNAL", // 297 (555)
-	"EXCEPTION-FILE", // 298 (556)
-	"EXCEPTION-FILE-N", // 299 (557)
-	"EXCEPTION-LOCATION", // 300 (558)
-	"EXCEPTION-LOCATION-N", // 301 (559)
-	"EXCEPTION-STATEMENT", // 302 (560)
-	"EXCEPTION-STATUS", // 303 (561)
-	"FACTORIAL", // 304 (562)
-	"FALSE", // 305 (563)
-	"FD", // 306 (564)
-	"FILE-CONTROL", // 307 (565)
-	"FILE", // 308 (566)
-	"FILE-LIMIT", // 309 (567)
-	"FINAL", // 310 (568)
-	"FINALLY", // 311 (569)
-	"FIND-STRING", // 312 (570)
-	"FIRST", // 313 (571)
-	"FIXED", // 314 (572)
-	"FOOTING", // 315 (573)
-	"FOR", // 316 (574)
-	"FORMATTED-CURRENT-DATE", // 317 (575)
-	"FORMATTED-DATE", // 318 (576)
-	"FORMATTED-DATETIME", // 319 (577)
-	"FORMATTED-TIME", // 320 (578)
-	"FORM-OVERFLOW", // 321 (579)
-	"FREE", // 322 (580)
-	"FRACTION-PART", // 323 (581)
-	"FROM", // 324 (582)
-	"FUNCTION", // 325 (583)
-	"GENERATE", // 326 (584)
-	"GIVING", // 327 (585)
-	"GLOBAL", // 328 (586)
-	"GO", // 329 (587)
-	"GROUP", // 330 (588)
-	"HEADING", // 331 (589)
-	"HEX", // 332 (590)
-	"HEX-OF", // 333 (591)
-	"HEX-TO-CHAR", // 334 (592)
-	"HIGH-VALUES", // 335 (593)
-	"HIGHEST-ALGEBRAIC", // 336 (594)
-	"HOLD", // 337 (595)
-	"IBM-360", // 338 (596)
-	"IN", // 339 (597)
-	"INCLUDE", // 340 (598)
-	"INDEX", // 341 (599)
-	"INDEXED", // 342 (600)
-	"INDICATE", // 343 (601)
-	"INITIAL", // 344 (602)
-	"INITIATE", // 345 (603)
-	"INPUT", // 346 (604)
-	"INSTALLATION", // 347 (605)
-	"INTERFACE", // 348 (606)
-	"INTEGER", // 349 (607)
-	"INTEGER-OF-BOOLEAN", // 350 (608)
-	"INTEGER-OF-DATE", // 351 (609)
-	"INTEGER-OF-DAY", // 352 (610)
-	"INTEGER-OF-FORMATTED-DATE", // 353 (611)
-	"INTEGER-PART", // 354 (612)
-	"INTO", // 355 (613)
-	"INTRINSIC", // 356 (614)
-	"INVOKE", // 357 (615)
-	"I-O", // 358 (616)
-	"I-O-CONTROL", // 359 (617)
-	"IS", // 360 (618)
-	"ISNT", // 361 (619)
-	"KANJI", // 362 (620)
-	"KEY", // 363 (621)
-	"LABEL", // 364 (622)
-	"LAST", // 365 (623)
-	"LEADING", // 366 (624)
-	"LEFT", // 367 (625)
-	"LENGTH", // 368 (626)
-	"LENGTH-OF", // 369 (627)
-	"LIMIT", // 370 (628)
-	"LIMITS", // 371 (629)
-	"LINE", // 372 (630)
-	"LINES", // 373 (631)
-	"LINE-COUNTER", // 374 (632)
-	"LINAGE", // 375 (633)
-	"LINKAGE", // 376 (634)
-	"LOCALE", // 377 (635)
-	"LOCALE-COMPARE", // 378 (636)
-	"LOCALE-DATE", // 379 (637)
-	"LOCALE-TIME", // 380 (638)
-	"LOCALE-TIME-FROM-SECONDS", // 381 (639)
-	"LOCAL-STORAGE", // 382 (640)
-	"LOCATION", // 383 (641)
-	"LOCK", // 384 (642)
-	"LOCK-ON", // 385 (643)
-	"LOG", // 386 (644)
-	"LOG10", // 387 (645)
-	"LOWER-CASE", // 388 (646)
-	"LOW-VALUES", // 389 (647)
-	"LOWEST-ALGEBRAIC", // 390 (648)
-	"LPAREN", // 391 (649)
-	"MANUAL", // 392 (650)
-	"MAXX", // 393 (651)
-	"MEAN", // 394 (652)
-	"MEDIAN", // 395 (653)
-	"MIDRANGE", // 396 (654)
-	"MINN", // 397 (655)
-	"MULTIPLE", // 398 (656)
-	"MOD", // 399 (657)
-	"MODE", // 400 (658)
-	"MODULE-NAME", // 401 (659)
-	"NAMED", // 402 (660)
-	"NAT", // 403 (661)
-	"NATIONAL", // 404 (662)
-	"NATIONAL-EDITED", // 405 (663)
-	"NATIONAL-OF", // 406 (664)
-	"NATIVE", // 407 (665)
-	"NESTED", // 408 (666)
-	"NEXT", // 409 (667)
-	"NO", // 410 (668)
-	"NOTE", // 411 (669)
-	"NULLS", // 412 (670)
-	"NULLPTR", // 413 (671)
-	"NUMERIC", // 414 (672)
-	"NUMERIC-EDITED", // 415 (673)
-	"NUMVAL", // 416 (674)
-	"NUMVAL-C", // 417 (675)
-	"NUMVAL-F", // 418 (676)
-	"OCCURS", // 419 (677)
-	"OF", // 420 (678)
-	"OFF", // 421 (679)
-	"OMITTED", // 422 (680)
-	"ON", // 423 (681)
-	"ONLY", // 424 (682)
-	"OPTIONAL", // 425 (683)
-	"OPTIONS", // 426 (684)
-	"ORD", // 427 (685)
-	"ORDER", // 428 (686)
-	"ORD-MAX", // 429 (687)
-	"ORD-MIN", // 430 (688)
-	"ORGANIZATION", // 431 (689)
-	"OTHER", // 432 (690)
-	"OTHERWISE", // 433 (691)
-	"OUTPUT", // 434 (692)
-	"PACKED-DECIMAL", // 435 (693)
-	"PADDING", // 436 (694)
-	"PAGE", // 437 (695)
-	"PAGE-COUNTER", // 438 (696)
-	"PF", // 439 (697)
-	"PH", // 440 (698)
-	"PI", // 441 (699)
-	"PIC", // 442 (700)
-	"PICTURE", // 443 (701)
-	"PLUS", // 444 (702)
-	"PRESENT-VALUE", // 445 (703)
-	"PRINT-SWITCH", // 446 (704)
-	"PROCEDURE", // 447 (705)
-	"PROCEDURES", // 448 (706)
-	"PROCEED", // 449 (707)
-	"PROCESS", // 450 (708)
-	"PROGRAM-ID", // 451 (709)
-	"PROGRAM", // 452 (710)
-	"PROPERTY", // 453 (711)
-	"PROTOTYPE", // 454 (712)
-	"PSEUDOTEXT", // 455 (713)
-	"QUOTES", // 456 (714)
-	"RANDOM", // 457 (715)
-	"RANDOM-SEED", // 458 (716)
-	"RANGE", // 459 (717)
-	"RAISE", // 460 (718)
-	"RAISING", // 461 (719)
-	"RD", // 462 (720)
-	"RECORD", // 463 (721)
-	"RECORDING", // 464 (722)
-	"RECORDS", // 465 (723)
-	"RECURSIVE", // 466 (724)
-	"REDEFINES", // 467 (725)
-	"REEL", // 468 (726)
-	"REFERENCE", // 469 (727)
-	"RELATIVE", // 470 (728)
-	"REM", // 471 (729)
-	"REMAINDER", // 472 (730)
-	"REMARKS", // 473 (731)
-	"REMOVAL", // 474 (732)
-	"RENAMES", // 475 (733)
-	"REPLACE", // 476 (734)
-	"REPLACING", // 477 (735)
-	"REPORT", // 478 (736)
-	"REPORTING", // 479 (737)
-	"REPORTS", // 480 (738)
-	"REPOSITORY", // 481 (739)
-	"RERUN", // 482 (740)
-	"RESERVE", // 483 (741)
-	"RESTRICTED", // 484 (742)
-	"RESUME", // 485 (743)
-	"REVERSE", // 486 (744)
-	"REVERSED", // 487 (745)
-	"REWIND", // 488 (746)
-	"RF", // 489 (747)
-	"RH", // 490 (748)
-	"RIGHT", // 491 (749)
-	"ROUNDED", // 492 (750)
-	"RUN", // 493 (751)
-	"SAME", // 494 (752)
-	"SCREEN", // 495 (753)
-	"SD", // 496 (754)
-	"SECONDS-FROM-FORMATTED-TIME", // 497 (755)
-	"SECONDS-PAST-MIDNIGHT", // 498 (756)
-	"SECURITY", // 499 (757)
-	"SEPARATE", // 500 (758)
-	"SEQUENCE", // 501 (759)
-	"SEQUENTIAL", // 502 (760)
-	"SHARING", // 503 (761)
-	"SIMPLE-EXIT", // 504 (762)
-	"SIGN", // 505 (763)
-	"SIN", // 506 (764)
-	"SIZE", // 507 (765)
-	"SMALLEST-ALGEBRAIC", // 508 (766)
-	"SOURCE", // 509 (767)
-	"SOURCE-COMPUTER", // 510 (768)
-	"SPECIAL-NAMES", // 511 (769)
-	"SQRT", // 512 (770)
-	"STACK", // 513 (771)
-	"STANDARD", // 514 (772)
-	"STANDARD-1", // 515 (773)
-	"STANDARD-DEVIATION", // 516 (774)
-	"STANDARD-COMPARE", // 517 (775)
-	"STATUS", // 518 (776)
-	"STRONG", // 519 (777)
-	"SUBSTITUTE", // 520 (778)
-	"SUM", // 521 (779)
-	"SYMBOL", // 522 (780)
-	"SYMBOLIC", // 523 (781)
-	"SYNCHRONIZED", // 524 (782)
-	"TALLY", // 525 (783)
-	"TALLYING", // 526 (784)
-	"TAN", // 527 (785)
-	"TERMINATE", // 528 (786)
-	"TEST", // 529 (787)
-	"TEST-DATE-YYYYMMDD", // 530 (788)
-	"TEST-DAY-YYYYDDD", // 531 (789)
-	"TEST-FORMATTED-DATETIME", // 532 (790)
-	"TEST-NUMVAL", // 533 (791)
-	"TEST-NUMVAL-C", // 534 (792)
-	"TEST-NUMVAL-F", // 535 (793)
-	"THAN", // 536 (794)
-	"TIME", // 537 (795)
-	"TIMES", // 538 (796)
-	"TO", // 539 (797)
-	"TOP", // 540 (798)
-	"TOP-LEVEL", // 541 (799)
-	"TRACKS", // 542 (800)
-	"TRACK-AREA", // 543 (801)
-	"TRAILING", // 544 (802)
-	"TRANSFORM", // 545 (803)
-	"TRIM", // 546 (804)
-	"TRUE", // 547 (805)
-	"TRY", // 548 (806)
-	"TURN", // 549 (807)
-	"TYPE", // 550 (808)
-	"TYPEDEF", // 551 (809)
-	"ULENGTH", // 552 (810)
-	"UNBOUNDED", // 553 (811)
-	"UNIT", // 554 (812)
-	"UNITS", // 555 (813)
-	"UNIT-RECORD", // 556 (814)
-	"UNTIL", // 557 (815)
-	"UP", // 558 (816)
-	"UPON", // 559 (817)
-	"UPOS", // 560 (818)
-	"UPPER-CASE", // 561 (819)
-	"USAGE", // 562 (820)
-	"USING", // 563 (821)
-	"USUBSTR", // 564 (822)
-	"USUPPLEMENTARY", // 565 (823)
-	"UTILITY", // 566 (824)
-	"UUID4", // 567 (825)
-	"UVALID", // 568 (826)
-	"UWIDTH", // 569 (827)
-	"VALUE", // 570 (828)
-	"VARIANCE", // 571 (829)
-	"VARYING", // 572 (830)
-	"VOLATILE", // 573 (831)
-	"WHEN-COMPILED", // 574 (832)
-	"WITH", // 575 (833)
-	"WORKING-STORAGE", // 576 (834)
-	"XML", // 577 (835)
-	"XMLGENERATE", // 578 (836)
-	"XMLPARSE", // 579 (837)
-	"YEAR-TO-YYYY", // 580 (838)
-	"YYYYDDD", // 581 (839)
-	"YYYYMMDD", // 582 (840)
-	"ARITHMETIC", // 583 (841)
-	"ATTRIBUTE", // 584 (842)
-	"AUTO", // 585 (843)
-	"AUTOMATIC", // 586 (844)
-	"AWAY-FROM-ZERO", // 587 (845)
-	"BACKGROUND-COLOR", // 588 (846)
-	"BELL", // 589 (847)
-	"BINARY-ENCODING", // 590 (848)
-	"BLINK", // 591 (849)
-	"CAPACITY", // 592 (850)
-	"CENTER", // 593 (851)
-	"CLASSIFICATION", // 594 (852)
-	"CYCLE", // 595 (853)
-	"DECIMAL-ENCODING", // 596 (854)
-	"ENTRY-CONVENTION", // 597 (855)
-	"EOL", // 598 (856)
-	"EOS", // 599 (857)
-	"ERASE", // 600 (858)
-	"EXPANDS", // 601 (859)
-	"FLOAT-BINARY", // 602 (860)
-	"FLOAT-DECIMAL", // 603 (861)
-	"FOREGROUND-COLOR", // 604 (862)
-	"FOREVER", // 605 (863)
-	"FULL", // 606 (864)
-	"HIGHLIGHT", // 607 (865)
-	"HIGH-ORDER-LEFT", // 608 (866)
-	"HIGH-ORDER-RIGHT", // 609 (867)
-	"IGNORING", // 610 (868)
-	"IMPLEMENTS", // 611 (869)
-	"INITIALIZED", // 612 (870)
-	"INTERMEDIATE", // 613 (871)
-	"LC-ALL", // 614 (872)
-	"LC-COLLATE", // 615 (873)
-	"LC-CTYPE", // 616 (874)
-	"LC-MESSAGES", // 617 (875)
-	"LC-MONETARY", // 618 (876)
-	"LC-NUMERIC", // 619 (877)
-	"LC-TIME", // 620 (878)
-	"LOWLIGHT", // 621 (879)
-	"NEAREST-AWAY-FROM-ZERO", // 622 (880)
-	"NEAREST-EVEN", // 623 (881)
-	"NEAREST-TOWARD-ZERO", // 624 (882)
-	"NONE", // 625 (883)
-	"NORMAL", // 626 (884)
-	"NUMBERS", // 627 (885)
-	"PREFIXED", // 628 (886)
-	"PREVIOUS", // 629 (887)
-	"PROHIBITED", // 630 (888)
-	"RELATION", // 631 (889)
-	"REQUIRED", // 632 (890)
-	"REVERSE-VIDEO", // 633 (891)
-	"ROUNDING", // 634 (892)
-	"SECONDS", // 635 (893)
-	"SECURE", // 636 (894)
-	"SHORT", // 637 (895)
-	"SIGNED", // 638 (896)
-	"STANDARD-BINARY", // 639 (897)
-	"STANDARD-DECIMAL", // 640 (898)
-	"STATEMENT", // 641 (899)
-	"STEP", // 642 (900)
-	"STRUCTURE", // 643 (901)
-	"TOWARD-GREATER", // 644 (902)
-	"TOWARD-LESSER", // 645 (903)
-	"TRUNCATION", // 646 (904)
-	"UCS-4", // 647 (905)
-	"UNDERLINE", // 648 (906)
-	"UNSIGNED", // 649 (907)
-	"UTF-16", // 650 (908)
-	"UTF-8", // 651 (909)
-	"ADDRESS", // 652 (910)
-	"END-ACCEPT", // 653 (911)
-	"END-ADD", // 654 (912)
-	"END-CALL", // 655 (913)
-	"END-COMPUTE", // 656 (914)
-	"END-DELETE", // 657 (915)
-	"END-DISPLAY", // 658 (916)
-	"END-DIVIDE", // 659 (917)
-	"END-EVALUATE", // 660 (918)
-	"END-MULTIPLY", // 661 (919)
-	"END-PERFORM", // 662 (920)
-	"END-READ", // 663 (921)
-	"END-RETURN", // 664 (922)
-	"END-REWRITE", // 665 (923)
-	"END-SEARCH", // 666 (924)
-	"END-START", // 667 (925)
-	"END-STRING", // 668 (926)
-	"END-SUBTRACT", // 669 (927)
-	"END-UNSTRING", // 670 (928)
-	"END-WRITE", // 671 (929)
-	"END-IF", // 672 (930)
-	"THRU", // 673 (931)
-	"OR", // 674 (932)
-	"AND", // 675 (933)
-	"NOT", // 676 (934)
-	"NE", // 677 (935)
-	"LE", // 678 (936)
-	"GE", // 679 (937)
-	"POW", // 680 (938)
-	"NEG", // 681 (939)
+	"CALL-CONVENTION", // 133 (391)
+	"CALL-COBOL", // 134 (392)
+	"CALL-VERBATIM", // 135 (393)
+	"CDF-PUSH", // 136 (394)
+	"CDF-POP", // 137 (395)
+	"SOURCE-FORMAT", // 138 (396)
+	"IF", // 139 (397)
+	"THEN", // 140 (398)
+	"ELSE", // 141 (399)
+	"SENTENCE", // 142 (400)
+	"ACCEPT", // 143 (401)
+	"ADD", // 144 (402)
+	"ALTER", // 145 (403)
+	"CALL", // 146 (404)
+	"CANCEL", // 147 (405)
+	"CLOSE", // 148 (406)
+	"COMPUTE", // 149 (407)
+	"CONTINUE", // 150 (408)
+	"DELETE", // 151 (409)
+	"DISPLAY", // 152 (410)
+	"DIVIDE", // 153 (411)
+	"EVALUATE", // 154 (412)
+	"EXIT", // 155 (413)
+	"FILLER", // 156 (414)
+	"GOBACK", // 157 (415)
+	"GOTO", // 158 (416)
+	"INITIALIZE", // 159 (417)
+	"INSPECT", // 160 (418)
+	"MERGE", // 161 (419)
+	"MOVE", // 162 (420)
+	"MULTIPLY", // 163 (421)
+	"OPEN", // 164 (422)
+	"PARAGRAPH", // 165 (423)
+	"READ", // 166 (424)
+	"RELEASE", // 167 (425)
+	"RETURN", // 168 (426)
+	"REWRITE", // 169 (427)
+	"SEARCH", // 170 (428)
+	"SET", // 171 (429)
+	"SELECT", // 172 (430)
+	"SORT", // 173 (431)
+	"SORT-MERGE", // 174 (432)
+	"STRING", // 175 (433)
+	"STOP", // 176 (434)
+	"SUBTRACT", // 177 (435)
+	"START", // 178 (436)
+	"UNSTRING", // 179 (437)
+	"WRITE", // 180 (438)
+	"WHEN", // 181 (439)
+	"ARGUMENT-NUMBER", // 182 (440)
+	"ARGUMENT-VALUE", // 183 (441)
+	"ENVIRONMENT-NAME", // 184 (442)
+	"ENVIRONMENT-VALUE", // 185 (443)
+	"ABS", // 186 (444)
+	"ACCESS", // 187 (445)
+	"ACOS", // 188 (446)
+	"ACTUAL", // 189 (447)
+	"ADVANCING", // 190 (448)
+	"AFTER", // 191 (449)
+	"ALL", // 192 (450)
+	"ALLOCATE", // 193 (451)
+	"ALPHABET", // 194 (452)
+	"ALPHABETIC", // 195 (453)
+	"ALPHABETIC-LOWER", // 196 (454)
+	"ALPHABETIC-UPPER", // 197 (455)
+	"ALPHANUMERIC", // 198 (456)
+	"ALPHANUMERIC-EDITED", // 199 (457)
+	"ALSO", // 200 (458)
+	"ALTERNATE", // 201 (459)
+	"ANNUITY", // 202 (460)
+	"ANUM", // 203 (461)
+	"ANY", // 204 (462)
+	"ANYCASE", // 205 (463)
+	"APPLY", // 206 (464)
+	"ARE", // 207 (465)
+	"AREA", // 208 (466)
+	"AREAS", // 209 (467)
+	"AS", // 210 (468)
+	"ASCENDING", // 211 (469)
+	"ACTIVATING", // 212 (470)
+	"ASIN", // 213 (471)
+	"ASSIGN", // 214 (472)
+	"AT", // 215 (473)
+	"ATAN", // 216 (474)
+	"BASED", // 217 (475)
+	"BASECONVERT", // 218 (476)
+	"BEFORE", // 219 (477)
+	"BINARY", // 220 (478)
+	"BIT", // 221 (479)
+	"BIT-OF", // 222 (480)
+	"BIT-TO-CHAR", // 223 (481)
+	"BLANK", // 224 (482)
+	"BLOCK", // 225 (483)
+	"BOOLEAN-OF-INTEGER", // 226 (484)
+	"BOTTOM", // 227 (485)
+	"BY", // 228 (486)
+	"BYTE", // 229 (487)
+	"BYTE-LENGTH", // 230 (488)
+	"CF", // 231 (489)
+	"CH", // 232 (490)
+	"CHANGED", // 233 (491)
+	"CHAR", // 234 (492)
+	"CHAR-NATIONAL", // 235 (493)
+	"CHARACTER", // 236 (494)
+	"CHARACTERS", // 237 (495)
+	"CHECKING", // 238 (496)
+	"CLASS", // 239 (497)
+	"COBOL", // 240 (498)
+	"CODE", // 241 (499)
+	"CODE-SET", // 242 (500)
+	"COLLATING", // 243 (501)
+	"COLUMN", // 244 (502)
+	"COMBINED-DATETIME", // 245 (503)
+	"COMMA", // 246 (504)
+	"COMMAND-LINE", // 247 (505)
+	"COMMAND-LINE-COUNT", // 248 (506)
+	"COMMIT", // 249 (507)
+	"COMMON", // 250 (508)
+	"CONCAT", // 251 (509)
+	"CONDITION", // 252 (510)
+	"CONFIGURATION", // 253 (511)
+	"CONTAINS", // 254 (512)
+	"CONTENT", // 255 (513)
+	"CONTROL", // 256 (514)
+	"CONTROLS", // 257 (515)
+	"CONVERT", // 258 (516)
+	"CONVERTING", // 259 (517)
+	"CORRESPONDING", // 260 (518)
+	"COS", // 261 (519)
+	"COUNT", // 262 (520)
+	"CURRENCY", // 263 (521)
+	"CURRENT", // 264 (522)
+	"CURRENT-DATE", // 265 (523)
+	"DATA", // 266 (524)
+	"DATE", // 267 (525)
+	"DATE-COMPILED", // 268 (526)
+	"DATE-OF-INTEGER", // 269 (527)
+	"DATE-TO-YYYYMMDD", // 270 (528)
+	"DATE-WRITTEN", // 271 (529)
+	"DAY", // 272 (530)
+	"DAY-OF-INTEGER", // 273 (531)
+	"DAY-OF-WEEK", // 274 (532)
+	"DAY-TO-YYYYDDD", // 275 (533)
+	"DBCS", // 276 (534)
+	"DE", // 277 (535)
+	"DEBUGGING", // 278 (536)
+	"DECIMAL-POINT", // 279 (537)
+	"DECLARATIVES", // 280 (538)
+	"DEFAULT", // 281 (539)
+	"DELIMITED", // 282 (540)
+	"DELIMITER", // 283 (541)
+	"DEPENDING", // 284 (542)
+	"DESCENDING", // 285 (543)
+	"DETAIL", // 286 (544)
+	"DIRECT", // 287 (545)
+	"DIRECT-ACCESS", // 288 (546)
+	"DOWN", // 289 (547)
+	"DUPLICATES", // 290 (548)
+	"DYNAMIC", // 291 (549)
+	"E", // 292 (550)
+	"EBCDIC", // 293 (551)
+	"EC", // 294 (552)
+	"EGCS", // 295 (553)
+	"ENTRY", // 296 (554)
+	"ENVIRONMENT", // 297 (555)
+	"EQUAL", // 298 (556)
+	"EVERY", // 299 (557)
+	"EXAMINE", // 300 (558)
+	"EXHIBIT", // 301 (559)
+	"EXP", // 302 (560)
+	"EXP10", // 303 (561)
+	"EXTEND", // 304 (562)
+	"EXTERNAL", // 305 (563)
+	"EXCEPTION-FILE", // 306 (564)
+	"EXCEPTION-FILE-N", // 307 (565)
+	"EXCEPTION-LOCATION", // 308 (566)
+	"EXCEPTION-LOCATION-N", // 309 (567)
+	"EXCEPTION-STATEMENT", // 310 (568)
+	"EXCEPTION-STATUS", // 311 (569)
+	"FACTORIAL", // 312 (570)
+	"FALSE", // 313 (571)
+	"FD", // 314 (572)
+	"FILE-CONTROL", // 315 (573)
+	"FILE", // 316 (574)
+	"FILE-LIMIT", // 317 (575)
+	"FINAL", // 318 (576)
+	"FINALLY", // 319 (577)
+	"FIND-STRING", // 320 (578)
+	"FIRST", // 321 (579)
+	"FIXED", // 322 (580)
+	"FOOTING", // 323 (581)
+	"FOR", // 324 (582)
+	"FORMATTED-CURRENT-DATE", // 325 (583)
+	"FORMATTED-DATE", // 326 (584)
+	"FORMATTED-DATETIME", // 327 (585)
+	"FORMATTED-TIME", // 328 (586)
+	"FORM-OVERFLOW", // 329 (587)
+	"FREE", // 330 (588)
+	"FRACTION-PART", // 331 (589)
+	"FROM", // 332 (590)
+	"FUNCTION", // 333 (591)
+	"GENERATE", // 334 (592)
+	"GIVING", // 335 (593)
+	"GLOBAL", // 336 (594)
+	"GO", // 337 (595)
+	"GROUP", // 338 (596)
+	"HEADING", // 339 (597)
+	"HEX", // 340 (598)
+	"HEX-OF", // 341 (599)
+	"HEX-TO-CHAR", // 342 (600)
+	"HIGH-VALUES", // 343 (601)
+	"HIGHEST-ALGEBRAIC", // 344 (602)
+	"HOLD", // 345 (603)
+	"IBM-360", // 346 (604)
+	"IN", // 347 (605)
+	"INCLUDE", // 348 (606)
+	"INDEX", // 349 (607)
+	"INDEXED", // 350 (608)
+	"INDICATE", // 351 (609)
+	"INITIAL", // 352 (610)
+	"INITIATE", // 353 (611)
+	"INPUT", // 354 (612)
+	"INSTALLATION", // 355 (613)
+	"INTERFACE", // 356 (614)
+	"INTEGER", // 357 (615)
+	"INTEGER-OF-BOOLEAN", // 358 (616)
+	"INTEGER-OF-DATE", // 359 (617)
+	"INTEGER-OF-DAY", // 360 (618)
+	"INTEGER-OF-FORMATTED-DATE", // 361 (619)
+	"INTEGER-PART", // 362 (620)
+	"INTO", // 363 (621)
+	"INTRINSIC", // 364 (622)
+	"INVOKE", // 365 (623)
+	"I-O", // 366 (624)
+	"I-O-CONTROL", // 367 (625)
+	"IS", // 368 (626)
+	"ISNT", // 369 (627)
+	"KANJI", // 370 (628)
+	"KEY", // 371 (629)
+	"LABEL", // 372 (630)
+	"LAST", // 373 (631)
+	"LEADING", // 374 (632)
+	"LEFT", // 375 (633)
+	"LENGTH", // 376 (634)
+	"LENGTH-OF", // 377 (635)
+	"LIMIT", // 378 (636)
+	"LIMITS", // 379 (637)
+	"LINE", // 380 (638)
+	"LINES", // 381 (639)
+	"LINE-COUNTER", // 382 (640)
+	"LINAGE", // 383 (641)
+	"LINKAGE", // 384 (642)
+	"LOCALE", // 385 (643)
+	"LOCALE-COMPARE", // 386 (644)
+	"LOCALE-DATE", // 387 (645)
+	"LOCALE-TIME", // 388 (646)
+	"LOCALE-TIME-FROM-SECONDS", // 389 (647)
+	"LOCAL-STORAGE", // 390 (648)
+	"LOCATION", // 391 (649)
+	"LOCK", // 392 (650)
+	"LOCK-ON", // 393 (651)
+	"LOG", // 394 (652)
+	"LOG10", // 395 (653)
+	"LOWER-CASE", // 396 (654)
+	"LOW-VALUES", // 397 (655)
+	"LOWEST-ALGEBRAIC", // 398 (656)
+	"LPAREN", // 399 (657)
+	"MANUAL", // 400 (658)
+	"MAXX", // 401 (659)
+	"MEAN", // 402 (660)
+	"MEDIAN", // 403 (661)
+	"MIDRANGE", // 404 (662)
+	"MINN", // 405 (663)
+	"MULTIPLE", // 406 (664)
+	"MOD", // 407 (665)
+	"MODE", // 408 (666)
+	"MODULE-NAME", // 409 (667)
+	"NAMED", // 410 (668)
+	"NAT", // 411 (669)
+	"NATIONAL", // 412 (670)
+	"NATIONAL-EDITED", // 413 (671)
+	"NATIONAL-OF", // 414 (672)
+	"NATIVE", // 415 (673)
+	"NESTED", // 416 (674)
+	"NEXT", // 417 (675)
+	"NO", // 418 (676)
+	"NOTE", // 419 (677)
+	"NULLS", // 420 (678)
+	"NULLPTR", // 421 (679)
+	"NUMERIC", // 422 (680)
+	"NUMERIC-EDITED", // 423 (681)
+	"NUMVAL", // 424 (682)
+	"NUMVAL-C", // 425 (683)
+	"NUMVAL-F", // 426 (684)
+	"OCCURS", // 427 (685)
+	"OF", // 428 (686)
+	"OFF", // 429 (687)
+	"OMITTED", // 430 (688)
+	"ON", // 431 (689)
+	"ONLY", // 432 (690)
+	"OPTIONAL", // 433 (691)
+	"OPTIONS", // 434 (692)
+	"ORD", // 435 (693)
+	"ORDER", // 436 (694)
+	"ORD-MAX", // 437 (695)
+	"ORD-MIN", // 438 (696)
+	"ORGANIZATION", // 439 (697)
+	"OTHER", // 440 (698)
+	"OTHERWISE", // 441 (699)
+	"OUTPUT", // 442 (700)
+	"PACKED-DECIMAL", // 443 (701)
+	"PADDING", // 444 (702)
+	"PAGE", // 445 (703)
+	"PAGE-COUNTER", // 446 (704)
+	"PF", // 447 (705)
+	"PH", // 448 (706)
+	"PI", // 449 (707)
+	"PIC", // 450 (708)
+	"PICTURE", // 451 (709)
+	"PLUS", // 452 (710)
+	"PRESENT-VALUE", // 453 (711)
+	"PRINT-SWITCH", // 454 (712)
+	"PROCEDURE", // 455 (713)
+	"PROCEDURES", // 456 (714)
+	"PROCEED", // 457 (715)
+	"PROCESS", // 458 (716)
+	"PROGRAM-ID", // 459 (717)
+	"PROGRAM", // 460 (718)
+	"PROPERTY", // 461 (719)
+	"PROTOTYPE", // 462 (720)
+	"PSEUDOTEXT", // 463 (721)
+	"QUOTES", // 464 (722)
+	"RANDOM", // 465 (723)
+	"RANDOM-SEED", // 466 (724)
+	"RANGE", // 467 (725)
+	"RAISE", // 468 (726)
+	"RAISING", // 469 (727)
+	"RD", // 470 (728)
+	"RECORD", // 471 (729)
+	"RECORDING", // 472 (730)
+	"RECORDS", // 473 (731)
+	"RECURSIVE", // 474 (732)
+	"REDEFINES", // 475 (733)
+	"REEL", // 476 (734)
+	"REFERENCE", // 477 (735)
+	"RELATIVE", // 478 (736)
+	"REM", // 479 (737)
+	"REMAINDER", // 480 (738)
+	"REMARKS", // 481 (739)
+	"REMOVAL", // 482 (740)
+	"RENAMES", // 483 (741)
+	"REPLACE", // 484 (742)
+	"REPLACING", // 485 (743)
+	"REPORT", // 486 (744)
+	"REPORTING", // 487 (745)
+	"REPORTS", // 488 (746)
+	"REPOSITORY", // 489 (747)
+	"RERUN", // 490 (748)
+	"RESERVE", // 491 (749)
+	"RESTRICTED", // 492 (750)
+	"RESUME", // 493 (751)
+	"REVERSE", // 494 (752)
+	"REVERSED", // 495 (753)
+	"REWIND", // 496 (754)
+	"RF", // 497 (755)
+	"RH", // 498 (756)
+	"RIGHT", // 499 (757)
+	"ROUNDED", // 500 (758)
+	"RUN", // 501 (759)
+	"SAME", // 502 (760)
+	"SCREEN", // 503 (761)
+	"SD", // 504 (762)
+	"SECONDS-FROM-FORMATTED-TIME", // 505 (763)
+	"SECONDS-PAST-MIDNIGHT", // 506 (764)
+	"SECURITY", // 507 (765)
+	"SEPARATE", // 508 (766)
+	"SEQUENCE", // 509 (767)
+	"SEQUENTIAL", // 510 (768)
+	"SHARING", // 511 (769)
+	"SIMPLE-EXIT", // 512 (770)
+	"SIGN", // 513 (771)
+	"SIN", // 514 (772)
+	"SIZE", // 515 (773)
+	"SMALLEST-ALGEBRAIC", // 516 (774)
+	"SOURCE", // 517 (775)
+	"SOURCE-COMPUTER", // 518 (776)
+	"SPECIAL-NAMES", // 519 (777)
+	"SQRT", // 520 (778)
+	"STACK", // 521 (779)
+	"STANDARD", // 522 (780)
+	"STANDARD-1", // 523 (781)
+	"STANDARD-DEVIATION", // 524 (782)
+	"STANDARD-COMPARE", // 525 (783)
+	"STATUS", // 526 (784)
+	"STRONG", // 527 (785)
+	"SUBSTITUTE", // 528 (786)
+	"SUM", // 529 (787)
+	"SYMBOL", // 530 (788)
+	"SYMBOLIC", // 531 (789)
+	"SYNCHRONIZED", // 532 (790)
+	"TALLY", // 533 (791)
+	"TALLYING", // 534 (792)
+	"TAN", // 535 (793)
+	"TERMINATE", // 536 (794)
+	"TEST", // 537 (795)
+	"TEST-DATE-YYYYMMDD", // 538 (796)
+	"TEST-DAY-YYYYDDD", // 539 (797)
+	"TEST-FORMATTED-DATETIME", // 540 (798)
+	"TEST-NUMVAL", // 541 (799)
+	"TEST-NUMVAL-C", // 542 (800)
+	"TEST-NUMVAL-F", // 543 (801)
+	"THAN", // 544 (802)
+	"TIME", // 545 (803)
+	"TIMES", // 546 (804)
+	"TO", // 547 (805)
+	"TOP", // 548 (806)
+	"TOP-LEVEL", // 549 (807)
+	"TRACKS", // 550 (808)
+	"TRACK-AREA", // 551 (809)
+	"TRAILING", // 552 (810)
+	"TRANSFORM", // 553 (811)
+	"TRIM", // 554 (812)
+	"TRUE", // 555 (813)
+	"TRY", // 556 (814)
+	"TURN", // 557 (815)
+	"TYPE", // 558 (816)
+	"TYPEDEF", // 559 (817)
+	"ULENGTH", // 560 (818)
+	"UNBOUNDED", // 561 (819)
+	"UNIT", // 562 (820)
+	"UNITS", // 563 (821)
+	"UNIT-RECORD", // 564 (822)
+	"UNTIL", // 565 (823)
+	"UP", // 566 (824)
+	"UPON", // 567 (825)
+	"UPOS", // 568 (826)
+	"UPPER-CASE", // 569 (827)
+	"USAGE", // 570 (828)
+	"USING", // 571 (829)
+	"USUBSTR", // 572 (830)
+	"USUPPLEMENTARY", // 573 (831)
+	"UTILITY", // 574 (832)
+	"UUID4", // 575 (833)
+	"UVALID", // 576 (834)
+	"UWIDTH", // 577 (835)
+	"VALUE", // 578 (836)
+	"VARIANCE", // 579 (837)
+	"VARYING", // 580 (838)
+	"VOLATILE", // 581 (839)
+	"WHEN-COMPILED", // 582 (840)
+	"WITH", // 583 (841)
+	"WORKING-STORAGE", // 584 (842)
+	"XML", // 585 (843)
+	"XMLGENERATE", // 586 (844)
+	"XMLPARSE", // 587 (845)
+	"YEAR-TO-YYYY", // 588 (846)
+	"YYYYDDD", // 589 (847)
+	"YYYYMMDD", // 590 (848)
+	"ARITHMETIC", // 591 (849)
+	"ATTRIBUTE", // 592 (850)
+	"AUTO", // 593 (851)
+	"AUTOMATIC", // 594 (852)
+	"AWAY-FROM-ZERO", // 595 (853)
+	"BACKGROUND-COLOR", // 596 (854)
+	"BELL", // 597 (855)
+	"BINARY-ENCODING", // 598 (856)
+	"BLINK", // 599 (857)
+	"CAPACITY", // 600 (858)
+	"CENTER", // 601 (859)
+	"CLASSIFICATION", // 602 (860)
+	"CYCLE", // 603 (861)
+	"DECIMAL-ENCODING", // 604 (862)
+	"ENTRY-CONVENTION", // 605 (863)
+	"EOL", // 606 (864)
+	"EOS", // 607 (865)
+	"ERASE", // 608 (866)
+	"EXPANDS", // 609 (867)
+	"FLOAT-BINARY", // 610 (868)
+	"FLOAT-DECIMAL", // 611 (869)
+	"FOREGROUND-COLOR", // 612 (870)
+	"FOREVER", // 613 (871)
+	"FULL", // 614 (872)
+	"HIGHLIGHT", // 615 (873)
+	"HIGH-ORDER-LEFT", // 616 (874)
+	"HIGH-ORDER-RIGHT", // 617 (875)
+	"IGNORING", // 618 (876)
+	"IMPLEMENTS", // 619 (877)
+	"INITIALIZED", // 620 (878)
+	"INTERMEDIATE", // 621 (879)
+	"LC-ALL", // 622 (880)
+	"LC-COLLATE", // 623 (881)
+	"LC-CTYPE", // 624 (882)
+	"LC-MESSAGES", // 625 (883)
+	"LC-MONETARY", // 626 (884)
+	"LC-NUMERIC", // 627 (885)
+	"LC-TIME", // 628 (886)
+	"LOWLIGHT", // 629 (887)
+	"NEAREST-AWAY-FROM-ZERO", // 630 (888)
+	"NEAREST-EVEN", // 631 (889)
+	"NEAREST-TOWARD-ZERO", // 632 (890)
+	"NONE", // 633 (891)
+	"NORMAL", // 634 (892)
+	"NUMBERS", // 635 (893)
+	"PREFIXED", // 636 (894)
+	"PREVIOUS", // 637 (895)
+	"PROHIBITED", // 638 (896)
+	"RELATION", // 639 (897)
+	"REQUIRED", // 640 (898)
+	"REVERSE-VIDEO", // 641 (899)
+	"ROUNDING", // 642 (900)
+	"SECONDS", // 643 (901)
+	"SECURE", // 644 (902)
+	"SHORT", // 645 (903)
+	"SIGNED", // 646 (904)
+	"STANDARD-BINARY", // 647 (905)
+	"STANDARD-DECIMAL", // 648 (906)
+	"STATEMENT", // 649 (907)
+	"STEP", // 650 (908)
+	"STRUCTURE", // 651 (909)
+	"TOWARD-GREATER", // 652 (910)
+	"TOWARD-LESSER", // 653 (911)
+	"TRUNCATION", // 654 (912)
+	"UCS-4", // 655 (913)
+	"UNDERLINE", // 656 (914)
+	"UNSIGNED", // 657 (915)
+	"UTF-16", // 658 (916)
+	"UTF-8", // 659 (917)
+	"ADDRESS", // 660 (918)
+	"END-ACCEPT", // 661 (919)
+	"END-ADD", // 662 (920)
+	"END-CALL", // 663 (921)
+	"END-COMPUTE", // 664 (922)
+	"END-DELETE", // 665 (923)
+	"END-DISPLAY", // 666 (924)
+	"END-DIVIDE", // 667 (925)
+	"END-EVALUATE", // 668 (926)
+	"END-MULTIPLY", // 669 (927)
+	"END-PERFORM", // 670 (928)
+	"END-READ", // 671 (929)
+	"END-RETURN", // 672 (930)
+	"END-REWRITE", // 673 (931)
+	"END-SEARCH", // 674 (932)
+	"END-START", // 675 (933)
+	"END-STRING", // 676 (934)
+	"END-SUBTRACT", // 677 (935)
+	"END-UNSTRING", // 678 (936)
+	"END-WRITE", // 679 (937)
+	"END-IF", // 680 (938)
+	"THRU", // 681 (939)
+	"OR", // 682 (940)
+	"AND", // 683 (941)
+	"NOT", // 684 (942)
+	"NE", // 685 (943)
+	"LE", // 686 (944)
+	"GE", // 687 (945)
+	"POW", // 688 (946)
+	"NEG", // 689 (947)
 };
diff --git a/gcc/cobol/udf/stored-char-length.cbl b/gcc/cobol/udf/stored-char-length.cbl
index 9ab3b14..66889d0 100644
--- a/gcc/cobol/udf/stored-char-length.cbl
+++ b/gcc/cobol/udf/stored-char-length.cbl
@@ -1,3 +1,6 @@
+        >> PUSH source format
+        >>SOURCE format is fixed
+
       *  This function is in public domain.
       *  Contributed by James K. Lowden of Cobolworx in August 2024
 
@@ -13,3 +16,4 @@
             to Output-Value. 
         End Function STORED-CHAR-LENGTH.
 
+        >> pop source format
\ No newline at end of file
diff --git a/gcc/cobol/util.cc b/gcc/cobol/util.cc
index 23f605d..aed9483 100644
--- a/gcc/cobol/util.cc
+++ b/gcc/cobol/util.cc
@@ -34,29 +34,24 @@
  * header files.
  */
 
-#include "cobol-system.h"
-#include "coretypes.h"
-#include "tree.h"
+#include <cobol-system.h>
+#include <coretypes.h>
+#include <tree.h>
 #undef yy_flex_debug
 
 #include <langinfo.h>
 
-#include "coretypes.h"
-#include "version.h"
-#include "demangle.h"
-#include "intl.h"
-#include "backtrace.h"
-#include "diagnostic.h"
-#include "diagnostic-color.h"
-#include "diagnostic-url.h"
-#include "diagnostic-metadata.h"
-#include "diagnostic-path.h"
-#include "edit-context.h"
-#include "selftest.h"
-#include "selftest-diagnostic.h"
-#include "opts.h"
+#include <coretypes.h>
+#include <version.h>
+#include <demangle.h>
+#include <intl.h>
+#include <backtrace.h>
+#include <diagnostic.h>
+#include <opts.h>
 #include "util.h"
+
 #include "cbldiag.h"
+#include "cdfval.h"
 #include "lexio.h"
 
 #include "../../libgcobol/ec.h"
@@ -90,7 +85,7 @@ static inline char *
 get_current_dir_name ()
 {
   /* Use libiberty's allocator here.  */
-  char *buf = (char *) xmalloc (PATH_MAX);
+  char *buf = static_cast<char *>(xmalloc (PATH_MAX));
   return getcwd (buf, PATH_MAX);
 }
 #endif
@@ -98,7 +93,7 @@ get_current_dir_name ()
 /*
  * For printing messages, usually the size of the thing is some kind of string
  * length, and doesn't really need a size_t.  For message formatting, use a
- * simple unsigned long, and warn if that's no good.  "gb4" here stands for 
+ * simple unsigned long, and warn if that's no good.  "gb4" here stands for
  * "4 Gigabytes".
  */
 unsigned long
@@ -110,7 +105,159 @@ gb4( size_t input ) {
   }
   return input;
 }
-  
+
+/*
+ * Most CDF Directives -- those that have state -- can be pushed and popped.
+ * This class maintains stacks of them, with each stack having a "default
+ * value" that may be updated, without push/pop, via a CDF directive or
+ * command-line option.  A push to a stack pushes the default value onto it; a
+ * pop copies the top of the stack to the default value.
+ *
+ * Supported:
+ *   CALL-CONVENTION
+ *   COBOL-WORDS
+ *   DEFINE
+ *   DISPLAY
+ *   IF
+ *   POP
+ *   PUSH
+ *   SOURCE FORMAT
+ *   TURN
+ * not supported
+ *   EVALUATE
+ *   FLAG-02
+ *   FLAG-14
+ *   LEAP-SECOND
+ *   LISTING
+ *   PAGE
+ *   PROPAGATE
+ *   REF-MOD-ZERO-LENGTH
+ *
+ * >>PUSH ALL calls the class's push() method.
+ * >>POP ALL calls the class's pop() method.
+ */
+class cdf_directives_t
+{
+  template <typename T>
+  class cdf_stack_t : private std::stack<T> { // cppcheck-suppress noConstructor
+    T default_value;
+    const T& top() const { return std::stack<T>::top(); }
+    bool empty() const { return std::stack<T>::empty(); }
+   public:
+    void value( const T& value ) {
+      T& output( empty()? default_value : std::stack<T>::top() ); // cppcheck-suppress constVariableReference
+      output = value;
+      dbgmsg("cdf_directives_t::%s: %s", __func__, str(output).c_str());
+    }
+    T& value() {
+      return empty()? default_value : std::stack<T>::top();
+    }
+    void push() {
+      std::stack<T>::push(value());
+      dbgmsg("cdf_directives_t::%s: %s", __func__, str(top()).c_str());
+    }
+    void pop() {
+      if( empty() ) {
+        error_msg(YYLTYPE(), "CDF stack empty"); // cppcheck-suppress syntaxError
+        return;
+      }
+      default_value = top();
+      std::stack<T>::pop();
+      dbgmsg("cdf_directives_t::%s: %s", __func__, str(default_value).c_str());
+    }
+  protected:
+    static std::string str(cbl_call_convention_t arg) {
+      char output[2] = { static_cast<char>(arg) };
+      return std::string("call-convention ") + output;
+    }
+    static std::string str(current_tokens_t) {
+      return "<cobol-words>";
+    }
+    static std::string str(cdf_values_t) {
+      return "<dictionary>";
+    }
+    static std::string str(source_format_t arg) {
+      return arg.description();
+    }
+    static std::string str(cbl_enabled_exceptions_t) {
+      return "<enabled_exceptions>";
+    }
+  };
+
+ public:
+  cdf_stack_t<cbl_call_convention_t> call_convention;
+  cdf_stack_t<current_tokens_t> cobol_words;
+  cdf_stack_t<cdf_values_t> dictionary;   // DEFINE
+  cdf_stack_t<source_format_t> source_format;
+  cdf_stack_t<cbl_enabled_exceptions_t> enabled_exceptions;
+
+  cdf_directives_t() {
+    call_convention.value() = cbl_call_cobol_e;
+  }
+
+  void push() {
+    call_convention.push();
+    cobol_words.push();
+    dictionary.push();
+    source_format.push();
+    enabled_exceptions.push();
+  }
+  void pop() {
+    call_convention.pop();
+    cobol_words.pop();
+    dictionary.pop();
+    source_format.pop();
+    enabled_exceptions.pop();
+  }
+};
+static cdf_directives_t cdf_directives;
+
+void
+current_call_convention( cbl_call_convention_t convention) {
+  cdf_directives.call_convention.value(convention);
+}
+cbl_call_convention_t
+current_call_convention() {
+  return cdf_directives.call_convention.value();
+}
+
+current_tokens_t&
+cdf_current_tokens() {
+  return cdf_directives.cobol_words.value();
+}
+
+cdf_values_t&
+cdf_dictionary() {
+  return cdf_directives.dictionary.value();
+}
+
+void
+cobol_set_indicator_column( int column ) {
+  cdf_directives.source_format.value().indicator_column_set(column);
+}
+source_format_t& cdf_source_format() {
+  return cdf_directives.source_format.value();
+}
+
+cbl_enabled_exceptions_t&
+cdf_enabled_exceptions() {
+  return cdf_directives.enabled_exceptions.value();
+}
+
+void cdf_push() { cdf_directives.push(); }
+void cdf_push_call_convention() { cdf_directives.call_convention.push(); }
+void cdf_push_current_tokens() { cdf_directives.cobol_words.push(); }
+void cdf_push_dictionary() { cdf_directives.dictionary.push(); }
+void cdf_push_enabled_exceptions() { cdf_directives.enabled_exceptions.push(); }
+void cdf_push_source_format() { cdf_directives.source_format.push(); }
+
+void cdf_pop() { cdf_directives.pop(); }
+void cdf_pop_call_convention() { cdf_directives.call_convention.pop(); }
+void cdf_pop_current_tokens() { cdf_directives.cobol_words.pop(); }
+void cdf_pop_dictionary() { cdf_directives.dictionary.pop(); }
+void cdf_pop_enabled_exceptions() { cdf_directives.enabled_exceptions.pop(); }
+void cdf_pop_source_format() { cdf_directives.source_format.pop(); }
+
 const char *
 symbol_type_str( enum symbol_type_t type )
 {
@@ -839,7 +986,7 @@ cbl_field_t::report_invalid_initial_value(const YYLTYPE& loc) const {
         // 8 or more, we need do no further testing because we assume
         // everything fits.
         if( data.capacity < 8 ) {
-          const auto p = strchr(data.initial, symbol_decimal_point());
+          const char *p = strchr(data.initial, symbol_decimal_point());
           if( p && atoll(p+1) != 0 ) {
             error_msg(loc, "integer type %s VALUE '%s' "
                      "requires integer VALUE",
@@ -902,7 +1049,7 @@ cbl_field_t::report_invalid_initial_value(const YYLTYPE& loc) const {
                                                  return TOUPPER(ch) == 'E';
                                                } );
               if( !has_exponent && data.precision() < pend - p ) {
-                error_msg(loc, "%s cannot represent VALUE %qs exactly (max %c%zu)",
+                error_msg(loc, "%s cannot represent VALUE %qs exactly (max %c%td)",
                           name, data.initial, '.', pend - p);
               }
             }
@@ -985,7 +1132,7 @@ literal_subscript_oob( const cbl_refer_t& r, size_t& isub /* output */)  {
                            pdim++;
                            return ! occurs.subscript_ok(r.field);
                          } );
-  isub = psub - r.subscripts.begin(); 
+  isub = psub - r.subscripts.begin();
   return psub == r.subscripts.end()? NULL : dims[isub];
 }
 
@@ -998,12 +1145,12 @@ cbl_refer_t::subscripts_set( const std::list<cbl_refer_t>& subs ) {
 
 const char *
 cbl_refer_t::str() const {
-  static char subscripts[64];
-  sprintf(subscripts, "(%u of " HOST_SIZE_T_PRINT_UNSIGNED " dimensions)",
+  static char subscripts_l[64];
+  sprintf(subscripts_l, "(%u of " HOST_SIZE_T_PRINT_UNSIGNED " dimensions)",
           nsubscript(), (fmt_size_t)dimensions(field));
   char *output = xasprintf("%s %s %s",
                            field? field_str(field) : "(none)",
-                           0 < dimensions(field)? subscripts : "",
+                           0 < dimensions(field)? subscripts_l : "",
                            is_refmod_reference()? "(refmod)" : "" );
   return output;
 }
@@ -1705,12 +1852,13 @@ date_time_fmt( const char input[] ) {
     { regex_t(), 'd', "^(" DATE_FMT_B "|" DATE_FMT_E ")$" },
     { regex_t(), 't', "^(" TIME_FMT_B "|" TIME_FMT_E ")$" },
   };
-  int erc, cflags = REG_EXTENDED | REG_ICASE, eflags=0;
+  int cflags = REG_EXTENDED | REG_ICASE, eflags=0;
   regmatch_t m[5];
   char result = 0;
 
   if( ! compiled ) {
     for( auto& fmt : fmts ) {
+      int erc;
       if( (erc = regcomp(&fmt.reg, fmt.pattern, cflags)) != 0 ) {
         char msg[80];
         regerror(erc, &fmt.reg, msg, sizeof(msg));
@@ -1768,7 +1916,7 @@ class unique_stack : public std::stack<input_file_t>
   friend void cobol_set_pp_option(int opt);
   bool option_m;
   std::set<std::string> all_names;
-  
+
   const char *
   no_wd( const char *wd, const char *name ) {
     int i;
@@ -1779,7 +1927,7 @@ class unique_stack : public std::stack<input_file_t>
 
  public:
   unique_stack() : option_m(false) {}
-  
+
   bool push( const value_type& value ) {
     auto ok = std::none_of( c.cbegin(), c.cend(),
                             [value]( const auto& that ) {
@@ -1813,8 +1961,8 @@ class unique_stack : public std::stack<input_file_t>
   const input_file_t& peek( size_t n ) const {
     gcc_assert( n < size() );
     return c.at(size() - ++n);
-  } 
-  
+  }
+
   void option( int opt ) { // capture other preprocessor options eventually
     assert(opt == 'M');
     option_m = true;
@@ -1827,7 +1975,7 @@ class unique_stack : public std::stack<input_file_t>
     std::string input( top().name );
     printf( "%s: ", input.c_str() );
     for( const auto& name : all_names ) {
-      if( name != input ) 
+      if( name != input )
 	printf( "\\\n\t%s ", name.c_str() );
     }
     printf("\n");
@@ -1844,7 +1992,7 @@ void cobol_set_pp_option(int opt) {
   assert(opt == 'M');
   input_filenames.option_m = true;
 }
-				  
+
 /*
  * Maintain a stack of input filenames.  Ensure the files are unique (by
  * inode), to prevent copybook cycles. Before pushing a new name, Record the
@@ -1855,7 +2003,7 @@ void cobol_set_pp_option(int opt) {
  * to enforce uniqueness, and the scanner to maintain line numbers.
  */
 bool cobol_filename( const char *name, ino_t inode ) {
-  const line_map *lines = NULL;
+  //const line_map *lines = NULL;
   if( inode == 0 ) {
     auto p = old_filenames.find(name);
     if( p == old_filenames.end() ) {
@@ -1865,8 +2013,10 @@ bool cobol_filename( const char *name, ino_t inode ) {
       }
       cbl_errx( "logic error: missing inode for %s", name);
     }
-    inode = p->second;
-    assert(inode != 0);
+    else {
+      inode = p->second;
+      assert(inode != 0);
+    }
   }
   linemap_add(line_table, LC_ENTER, sysp, name, 1);
   input_filename_vestige = name;
@@ -1915,20 +2065,50 @@ cobol_filename_restore() {
   input_filenames.pop();
   if( input_filenames.empty() ) return;
 
-  auto& input = input_filenames.top();
+  const auto& input = input_filenames.top();
 
   linemap_add(line_table, LC_LEAVE, sysp, NULL, 0);
 }
 
-static location_t token_location;
+static int first_line_minus_1 = 0;
+static location_t token_location_minus_1 = 0;
+static location_t token_location = 0;
 
-location_t location_from_lineno() { return token_location; }
+location_t current_token_location() { return token_location; }
+location_t current_location_minus_one() { return token_location_minus_1; }
+void current_location_minus_one_clear()
+  {
+  first_line_minus_1 = 0;
+  }
 
 template <typename LOC>
 static void
 gcc_location_set_impl( const LOC& loc ) {
-  token_location = linemap_line_start( line_table, loc.last_line, 80 );
+  // Set the position to the first line & column in the location.
+  if( getenv("KILROY") )
+    {
+    fprintf(stderr, "********** KILROY %d\n", loc.first_line);
+    }
+
+ static location_t loc_m_1 = 0;
+
+  token_location = linemap_line_start( line_table, loc.first_line, 80 );
   token_location = linemap_position_for_column( line_table, loc.first_column);
+
+  if( loc.first_line > first_line_minus_1 )
+    {
+    // In order for GDB-COBOL to be able to step through COBOL code properly,
+    // it is sometimes necessary for the code at the beginning of a COBOL
+    // line to be using the location_t of the previous line.  This is true, for
+    // example, when laying down the infrastructure code between the last
+    // statement of a paragraph and the code created at the beginning of the
+    // following paragragh.  This code assumes that token_location values of
+    // interest are monotonic, and stores that prior value.
+    first_line_minus_1 = loc.first_line;
+    token_location_minus_1 = loc_m_1;
+    loc_m_1 = token_location;
+    }
+
   location_dump(__func__, __LINE__, "parser", loc);
 }
 
@@ -1969,9 +2149,14 @@ verify_format( const char gmsgid[] ) {
 }
 #endif
 
-static const diagnostic_option_id option_zero;
+static const diagnostics::option_id option_zero;
 size_t parse_error_inc();
 
+void gcc_location_dump() {
+    linemap_dump_location( line_table, token_location, stderr );
+}
+
+
 void ydferror( const char gmsgid[], ... ) ATTRIBUTE_GCOBOL_DIAG(1, 2);
 
 void
@@ -1982,8 +2167,9 @@ ydferror( const char gmsgid[], ... ) {
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, token_location);
-  bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_zero,
-                                         gmsgid, &ap, DK_ERROR);
+  /*bool ret =*/ global_dc->diagnostic_impl (&richloc, nullptr, option_zero,
+                                             gmsgid, &ap,
+                                             diagnostics::kind::error);
   va_end (ap);
 }
 
@@ -2008,10 +2194,7 @@ class temp_loc_t {
     gcc_location_set(loc);
   }
   explicit temp_loc_t( const YDFLTYPE& loc) : orig(token_location) {
-    YYLTYPE lloc = {
-      loc.first_line, loc.first_column,
-      loc.last_line,  loc.last_column };
-    gcc_location_set(lloc);
+    gcc_location_set(loc);
   }
   ~temp_loc_t() {
     if( orig != token_location ) {
@@ -2041,7 +2224,8 @@ class temp_loc_t {
   va_start (ap, gmsgid);                                                \
   rich_location richloc (line_table, token_location);                   \
   bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_zero,        \
-                                         gmsgid, &ap, DK_ERROR);        \
+                                         gmsgid, &ap,                   \
+                                         diagnostics::kind::error);     \
   va_end (ap);                                                          \
   global_dc->end_group();
 
@@ -2057,6 +2241,33 @@ void error_msg( const YDFLTYPE& loc, const char gmsgid[], ... ) {
   ERROR_MSG_BODY
 }
 
+bool
+warn_msg( const YYLTYPE& loc, const char gmsgid[], ... ) {
+  temp_loc_t looker(loc);
+  verify_format(gmsgid);
+  auto_diagnostic_group d;
+  va_list ap;
+  va_start (ap, gmsgid);
+  rich_location richloc (line_table, token_location);
+  auto ret = emit_diagnostic_valist( diagnostics::kind::warning,
+                                     token_location,
+                                     option_zero, gmsgid, &ap );
+  va_end (ap);
+  return ret;
+}
+
+void error_msg_direct( const char gmsgid[], ... ) {
+  verify_format(gmsgid);
+  parse_error_inc();
+  auto_diagnostic_group d;
+  va_list ap;
+  va_start (ap, gmsgid);
+  /*auto ret = */emit_diagnostic_valist( diagnostics::kind::error,
+                                         token_location,
+                                         option_zero, gmsgid, &ap );
+  va_end (ap);
+}
+
 void
 yyerror( const char gmsgid[], ... ) {
   temp_loc_t looker;
@@ -2066,8 +2277,11 @@ yyerror( const char gmsgid[], ... ) {
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, token_location);
-  bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_zero,
-                                         gmsgid, &ap, DK_ERROR);
+  /*bool ret =*/ global_dc->diagnostic_impl ( &richloc,
+                                              nullptr,
+                                              option_zero,
+                                              gmsgid,
+                                              &ap, diagnostics::kind::error);
   va_end (ap);
   global_dc->end_group();
 }
@@ -2078,7 +2292,7 @@ yywarn( const char gmsgid[], ... ) {
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
-  auto ret = emit_diagnostic_valist( DK_WARNING, token_location,
+  auto ret = emit_diagnostic_valist( diagnostics::kind::warning, token_location,
                                      option_zero, gmsgid, &ap );
   va_end (ap);
   return ret;
@@ -2276,8 +2490,11 @@ cbl_internal_error(const char *gmsgid, ...) {
   auto_diagnostic_group d;
   va_list ap;
   va_start(ap, gmsgid);
-  emit_diagnostic_valist( DK_ICE, token_location, option_zero, gmsgid, &ap );
+  emit_diagnostic_valist( diagnostics::kind::ice,
+			  token_location, option_zero, gmsgid, &ap );
   va_end(ap);
+  abort();  // This unnecessary statement is needed so that [[noreturn]]
+  //        // doesn't cause a warning.
 }
 
 void
@@ -2286,7 +2503,8 @@ cbl_unimplementedw(const char *gmsgid, ...) {
   auto_diagnostic_group d;
   va_list ap;
   va_start(ap, gmsgid);
-  emit_diagnostic_valist( DK_SORRY, token_location, option_zero, gmsgid, &ap );
+  emit_diagnostic_valist( diagnostics::kind::sorry,
+			  token_location, option_zero, gmsgid, &ap );
   va_end(ap);
 }
 
@@ -2296,7 +2514,8 @@ cbl_unimplemented(const char *gmsgid, ...) {
   auto_diagnostic_group d;
   va_list ap;
   va_start(ap, gmsgid);
-  emit_diagnostic_valist( DK_SORRY, token_location, option_zero, gmsgid, &ap );
+  emit_diagnostic_valist( diagnostics::kind::sorry,
+			  token_location, option_zero, gmsgid, &ap );
   va_end(ap);
 }
 
@@ -2307,12 +2526,13 @@ cbl_unimplemented_at( const YYLTYPE& loc, const char *gmsgid, ... ) {
   auto_diagnostic_group d;
   va_list ap;
   va_start(ap, gmsgid);
-  emit_diagnostic_valist( DK_SORRY, token_location, option_zero, gmsgid, &ap );
+  emit_diagnostic_valist( diagnostics::kind::sorry,
+			  token_location, option_zero, gmsgid, &ap );
   va_end(ap);
 }
 
-/* 
- * analogs to err(3) and errx(3). 
+/*
+ * analogs to err(3) and errx(3).
  */
 
 #pragma GCC diagnostic push
@@ -2324,7 +2544,8 @@ cbl_err(const char *fmt, ...) {
   verify_format(gmsgid);
   va_list ap;
   va_start(ap, fmt);
-  emit_diagnostic_valist( DK_FATAL, token_location, option_zero, gmsgid, &ap );
+  emit_diagnostic_valist( diagnostics::kind::fatal,
+			  token_location, option_zero, gmsgid, &ap );
   va_end(ap);
 }
 #pragma GCC diagnostic pop
@@ -2335,7 +2556,8 @@ cbl_errx(const char *gmsgid, ...) {
   auto_diagnostic_group d;
   va_list ap;
   va_start(ap, gmsgid);
-  emit_diagnostic_valist( DK_FATAL, token_location, option_zero, gmsgid, &ap );
+  emit_diagnostic_valist( diagnostics::kind::fatal,
+			  token_location, option_zero, gmsgid, &ap );
   va_end(ap);
   }
 
@@ -2447,7 +2669,7 @@ static const std::set<std::string> reserved_words = {
   "VOLATILE",
   "XML",
   "END-START",
-  
+
   // ISO 2023 keywords
   "ACCEPT",
   "ACCESS",
diff --git a/gcc/cobol/util.h b/gcc/cobol/util.h
index 165915a..d478ea2 100644
--- a/gcc/cobol/util.h
+++ b/gcc/cobol/util.h
@@ -33,7 +33,7 @@
 
 void cbl_message(int fd, const char *format_string, ...)
   ATTRIBUTE_PRINTF_2;
-void cbl_internal_error(const char *format_string, ...)
+[[noreturn]] void cbl_internal_error(const char *format_string, ...)
   ATTRIBUTE_GCOBOL_DIAG(1, 2);
 
 void cbl_err(const char *format_string, ...) ATTRIBUTE_GCOBOL_DIAG(1, 2);
@@ -49,7 +49,7 @@ void cobol_set_pp_option(int opt);
 
 void cobol_filename_restore();
 const char * cobol_lineno( int );
-int cobol_lineno();
+int cobol_lineno(void);
 
 unsigned long gb4( size_t input );
 
@@ -59,5 +59,70 @@ as_voidp( P p ) {
   return static_cast<const void *>(p);
 }
 
+/*
+ * The default source format, whether free or fixed, is determined
+ * heuristically by examining the PROGRAM-ID line, if it exists, in the first
+ * input file. If that file does not have such a line, the default is free
+ * format.  Else the format is set to fixed if anything appears on that line
+ * that would prohibit parsing it as free format,
+ */
+class source_format_t {
+  bool first_file, explicitly;
+  int left, right;
+public:
+  source_format_t()
+    : first_file(true), explicitly(false), left(0), right(0)
+  {}
+  void indicator_column_set( int column ) {
+    explicitly = true;
+    if( column == 0 ) right = 0;
+    if( column < 0 ) {
+      column = -column;
+      right = 73;
+    }
+    left = column;
+  }
+  
+  bool inference_pending() {
+    bool tf = first_file && !explicitly;
+    first_file = false;
+    return tf;
+  }
+
+  void infer( const char *bol, bool want_reference_format );
+  
+  inline bool is_fixed() const { return left == 7; }
+  inline bool is_reffmt() const { return is_fixed() && right == 73; }
+  inline bool is_free() const { return ! is_fixed(); }
+  
+  const char * description() const {
+    if( is_reffmt() ) return "REFERENCE";
+    if( is_fixed() ) return "FIXED";
+    if( is_free() ) return "FREE";
+    gcc_unreachable();
+  }    
+
+  inline int left_margin() {
+    return left == 0? left : left - 1;
+  }
+  inline int right_margin() {
+    return right == 0? right : right - 1;
+  }
+}; 
+
+
+void cdf_push();
+void cdf_push_call_convention();
+void cdf_push_current_tokens();
+void cdf_push_dictionary();
+void cdf_push_enabled_exceptions();
+void cdf_push_source_format();
+
+void cdf_pop();
+void cdf_pop_call_convention();
+void cdf_pop_current_tokens();
+void cdf_pop_dictionary();
+void cdf_pop_source_format();
+void cdf_pop_enabled_exceptions();
 
 #endif
diff --git a/gcc/common.opt b/gcc/common.opt
index 8163e03..70659fa 100644
--- a/gcc/common.opt
+++ b/gcc/common.opt
@@ -547,7 +547,7 @@ Warn if an array is accessed out of bounds.
 
 Wauto-profile
 Common Var(warn_auto_profile) Warning
-Warn about problems with auto-profile data
+Warn about problems with auto-profile data.
 
 Wuse-after-free
 Common Var(warn_use_after_free) Warning
@@ -851,11 +851,17 @@ Common Var(warn_unused) Init(0) Warning
 Enable all -Wunused- warnings.
 
 Wunused-but-set-parameter
-Common Var(warn_unused_but_set_parameter) Warning EnabledBy(Wunused && Wextra)
+Common Alias(Wunused-but-set-parameter=,3,0) Warning
+
+Wunused-but-set-parameter=
+Common Var(warn_unused_but_set_parameter) RejectNegative Joined UInteger Warning IntegerRange(0, 3) EnabledBy(Wunused && Wextra)
 Warn when a function parameter is only set, otherwise unused.
 
 Wunused-but-set-variable
-Common Var(warn_unused_but_set_variable) Warning EnabledBy(Wunused)
+Common Alias(Wunused-but-set-variable=,3,0) Warning
+
+Wunused-but-set-variable=
+Common Var(warn_unused_but_set_variable) RejectNegative Joined UInteger Warning IntegerRange(0, 3) EnabledBy(Wunused)
 Warn when a variable is only set, otherwise unused.
 
 Wunused-function
@@ -1441,7 +1447,7 @@ Driver Common Joined RejectNegative Var(flag_diagnostics_show_color) Enum(diagno
 
 ; Required for these enum values.
 SourceInclude
-diagnostic-color.h
+diagnostics/color.h
 
 Enum
 Name(diagnostic_color_rule) Type(int)
@@ -1465,7 +1471,7 @@ Driver Common Joined RejectNegative Var(flag_diagnostics_show_urls) Enum(diagnos
 
 ; Required for these enum values.
 SourceInclude
-diagnostic-url.h
+diagnostics/url.h
 
 Enum
 Name(diagnostic_url_rule) Type(int)
diff --git a/gcc/common.opt.urls b/gcc/common.opt.urls
index c705089..38dd9d3 100644
--- a/gcc/common.opt.urls
+++ b/gcc/common.opt.urls
@@ -64,6 +64,9 @@ UrlSuffix(gcc/Warning-Options.html#index-Warray-bounds)
 Warray-bounds=
 UrlSuffix(gcc/Warning-Options.html#index-Warray-bounds)
 
+Wauto-profile
+UrlSuffix(gcc/Warning-Options.html#index-Wauto-profile)
+
 Wuse-after-free
 UrlSuffix(gcc/Warning-Options.html#index-Wno-use-after-free)
 
@@ -269,9 +272,15 @@ UrlSuffix(gcc/Warning-Options.html#index-Wno-unused)
 Wunused-but-set-parameter
 UrlSuffix(gcc/Warning-Options.html#index-Wno-unused-but-set-parameter)
 
+Wunused-but-set-parameter=
+UrlSuffix(gcc/Warning-Options.html#index-Wno-unused-but-set-parameter)
+
 Wunused-but-set-variable
 UrlSuffix(gcc/Warning-Options.html#index-Wno-unused-but-set-variable)
 
+Wunused-but-set-variable=
+UrlSuffix(gcc/Warning-Options.html#index-Wno-unused-but-set-variable)
+
 Wunused-function
 UrlSuffix(gcc/Warning-Options.html#index-Wno-unused-function)
 
diff --git a/gcc/common/config/aarch64/cpuinfo.h b/gcc/common/config/aarch64/cpuinfo.h
index cd3c2b2..d329d86 100644
--- a/gcc/common/config/aarch64/cpuinfo.h
+++ b/gcc/common/config/aarch64/cpuinfo.h
@@ -39,10 +39,10 @@ enum CPUFeatures {
   FEAT_FP,
   FEAT_SIMD,
   FEAT_CRC,
-  FEAT_SHA1,
+  FEAT_CSSC,
   FEAT_SHA2,
   FEAT_SHA3,
-  FEAT_AES,
+  FEAT_unused5,
   FEAT_PMULL,
   FEAT_FP16,
   FEAT_DIT,
@@ -53,30 +53,30 @@ enum CPUFeatures {
   FEAT_RCPC,
   FEAT_RCPC2,
   FEAT_FRINTTS,
-  FEAT_DGH,
+  FEAT_unused6,
   FEAT_I8MM,
   FEAT_BF16,
-  FEAT_EBF16,
-  FEAT_RPRES,
+  FEAT_unused7,
+  FEAT_unused8,
   FEAT_SVE,
-  FEAT_SVE_BF16,
-  FEAT_SVE_EBF16,
-  FEAT_SVE_I8MM,
+  FEAT_unused9,
+  FEAT_unused10,
+  FEAT_unused11,
   FEAT_SVE_F32MM,
   FEAT_SVE_F64MM,
   FEAT_SVE2,
-  FEAT_SVE_AES,
+  FEAT_unused12,
   FEAT_SVE_PMULL128,
   FEAT_SVE_BITPERM,
   FEAT_SVE_SHA3,
   FEAT_SVE_SM4,
   FEAT_SME,
-  FEAT_MEMTAG,
+  FEAT_unused13,
   FEAT_MEMTAG2,
-  FEAT_MEMTAG3,
+  FEAT_unused14,
   FEAT_SB,
   FEAT_unused1,
-  FEAT_SSBS,
+  FEAT_unused15,
   FEAT_SSBS2,
   FEAT_BTI,
   FEAT_unused2,
@@ -87,6 +87,7 @@ enum CPUFeatures {
   FEAT_SME_I64,
   FEAT_SME2,
   FEAT_RCPC3,
+  FEAT_MOPS,
   FEAT_MAX,
   FEAT_EXT = 62, /* Reserved to indicate presence of additional features field
 		    in __aarch64_cpu_features.  */
diff --git a/gcc/common/config/avr/avr-common.cc b/gcc/common/config/avr/avr-common.cc
index 203a965..d8b982c 100644
--- a/gcc/common/config/avr/avr-common.cc
+++ b/gcc/common/config/avr/avr-common.cc
@@ -38,6 +38,7 @@ static const struct default_options avr_option_optimization_table[] =
     { OPT_LEVELS_1_PLUS, OPT_mmain_is_OS_task, NULL, 1 },
     { OPT_LEVELS_1_PLUS, OPT_mfuse_add_, NULL, 1 },
     { OPT_LEVELS_2_PLUS, OPT_mfuse_add_, NULL, 2 },
+    { OPT_LEVELS_1_PLUS, OPT_mfuse_move2, NULL, 1 },
     { OPT_LEVELS_1_PLUS_NOT_DEBUG, OPT_mfuse_move_, NULL, 3 },
     { OPT_LEVELS_2_PLUS, OPT_mfuse_move_, NULL, 23 },
     { OPT_LEVELS_2_PLUS, OPT_msplit_bit_shift, NULL, 1 },
diff --git a/gcc/common/config/i386/i386-common.cc b/gcc/common/config/i386/i386-common.cc
index dfcd4e9..9e807e4 100644
--- a/gcc/common/config/i386/i386-common.cc
+++ b/gcc/common/config/i386/i386-common.cc
@@ -131,8 +131,7 @@ along with GCC; see the file COPYING3.  If not see
 #define OPTION_MASK_ISA2_AVX10_2_SET \
   (OPTION_MASK_ISA2_AVX10_1_SET | OPTION_MASK_ISA2_AVX10_2)
 #define OPTION_MASK_ISA2_AMX_AVX512_SET \
-  (OPTION_MASK_ISA2_AMX_TILE_SET | OPTION_MASK_ISA2_AVX10_2_SET \
-   | OPTION_MASK_ISA2_AMX_AVX512)
+  (OPTION_MASK_ISA2_AMX_TILE_SET | OPTION_MASK_ISA2_AMX_AVX512)
 #define OPTION_MASK_ISA2_AMX_TF32_SET \
   (OPTION_MASK_ISA2_AMX_TILE_SET | OPTION_MASK_ISA2_AMX_TF32)
 #define OPTION_MASK_ISA2_AMX_TRANSPOSE_SET \
@@ -328,8 +327,7 @@ along with GCC; see the file COPYING3.  If not see
 #define OPTION_MASK_ISA2_USER_MSR_UNSET OPTION_MASK_ISA2_USER_MSR
 #define OPTION_MASK_ISA2_AVX10_1_UNSET \
   (OPTION_MASK_ISA2_AVX10_1 | OPTION_MASK_ISA2_AVX10_2_UNSET)
-#define OPTION_MASK_ISA2_AVX10_2_UNSET \
-  (OPTION_MASK_ISA2_AVX10_2 | OPTION_MASK_ISA2_AMX_AVX512_UNSET)
+#define OPTION_MASK_ISA2_AVX10_2_UNSET OPTION_MASK_ISA2_AVX10_2
 #define OPTION_MASK_ISA2_AMX_AVX512_UNSET OPTION_MASK_ISA2_AMX_AVX512
 #define OPTION_MASK_ISA2_AMX_TF32_UNSET OPTION_MASK_ISA2_AMX_TF32
 #define OPTION_MASK_ISA2_AMX_TRANSPOSE_UNSET OPTION_MASK_ISA2_AMX_TRANSPOSE
@@ -379,7 +377,8 @@ along with GCC; see the file COPYING3.  If not see
 #define OPTION_MASK_ISA2_AVX512F_UNSET \
   (OPTION_MASK_ISA2_AVX512BW_UNSET \
    | OPTION_MASK_ISA2_AVX512VP2INTERSECT_UNSET \
-   | OPTION_MASK_ISA2_AVX10_1_UNSET)
+   | OPTION_MASK_ISA2_AVX10_1_UNSET \
+   | OPTION_MASK_ISA2_AMX_AVX512_UNSET)
 #define OPTION_MASK_ISA2_GENERAL_REGS_ONLY_UNSET \
   OPTION_MASK_ISA2_SSE_UNSET
 #define OPTION_MASK_ISA2_AVX_UNSET \
@@ -1374,8 +1373,8 @@ ix86_handle_option (struct gcc_options *opts,
 	{
 	  opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA2_AMX_AVX512_SET;
 	  opts->x_ix86_isa_flags2_explicit |= OPTION_MASK_ISA2_AMX_AVX512_SET;
-	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX10_1_SET;
-	  opts->x_ix86_isa_flags_explicit |= OPTION_MASK_ISA_AVX10_1_SET;
+	  opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512F_SET;
+	  opts->x_ix86_isa_flags_explicit |= OPTION_MASK_ISA_AVX512F_SET;
 	}
       else
 	{
diff --git a/gcc/common/config/riscv/riscv-common.cc b/gcc/common/config/riscv/riscv-common.cc
index 82037a3..da3cb9f 100644
--- a/gcc/common/config/riscv/riscv-common.cc
+++ b/gcc/common/config/riscv/riscv-common.cc
@@ -1606,8 +1606,9 @@ bool
 riscv_ext_is_subset (struct cl_target_option *opts,
 		     struct cl_target_option *subset)
 {
-  for (const auto &[ext_name, ext_info] : riscv_ext_infos)
+  for (const auto &riscv_ext_info : riscv_ext_infos)
     {
+      const auto &ext_info = riscv_ext_info.second;
       if (ext_info.check_opts (opts) && !ext_info.check_opts (subset))
 	return false;
     }
diff --git a/gcc/config.gcc b/gcc/config.gcc
index a6f6efe..0d8dbc4 100644
--- a/gcc/config.gcc
+++ b/gcc/config.gcc
@@ -1847,11 +1847,11 @@ moxie-*-moxiebox*)
 	;;
 h8300-*-elf*)
 	tmake_file="h8300/t-h8300"
-	tm_file="h8300/h8300.h elfos.h newlib-stdint.h h8300/elf.h"
+	tm_file="elfos.h h8300/h8300.h newlib-stdint.h h8300/elf.h"
 	;;
 h8300-*-linux*)
 	tmake_file="${tmake_file} h8300/t-h8300 h8300/t-linux"
-	tm_file="h8300/h8300.h elfos.h gnu-user.h linux.h glibc-stdint.h h8300/linux.h"
+	tm_file="elfos.h h8300/h8300.h gnu-user.h linux.h glibc-stdint.h h8300/linux.h"
 	;;
 hppa*64*-*-linux*)
 	target_cpu_default="MASK_PA_11|MASK_PA_20"
@@ -3504,10 +3504,9 @@ sparc*-*-solaris2*)
 	    sparc64-*-* | sparcv9-*-*)
 		tm_file="sparc/default64.h ${tm_file}"
 		;;
-	    *)
-		test x$with_cpu != x || with_cpu=v9
-		;;
 	esac
+	# Match Studio 12.6 cc.
+	with_cpu=${with_cpu:-ultrasparc3}
 	tmake_file="${tmake_file} sparc/t-sparc sparc/t-sol2"
 	;;
 sparc-wrs-vxworks)
@@ -5894,7 +5893,7 @@ esac
 # distinguish VxWorks variants such as VxWorks 7 or 64).
 
 case ${target} in
-arm*-*-* | i[34567]86-*-* | mips*-*-* | powerpc*-*-* | sh*-*-* \
+aarch64*-*-* | arm*-*-* | i[34567]86-*-* | mips*-*-* | powerpc*-*-* | sh*-*-* \
 | sparc*-*-* | x86_64-*-*)
 	tm_file="vxworks-dummy.h ${tm_file}"
 	;;
diff --git a/gcc/config.in b/gcc/config.in
index ab62c15..353d1bc 100644
--- a/gcc/config.in
+++ b/gcc/config.in
@@ -318,6 +318,12 @@
 #endif
 
 
+/* Define to enable -mfentry by default on x86-64. */
+#ifndef USED_FOR_TARGET
+#undef ENABLE_X86_64_MFENTRY
+#endif
+
+
 /* Define to the name of a file containing a list of extra machine modes for
    this architecture. */
 #ifndef USED_FOR_TARGET
diff --git a/gcc/config/aarch64/aarch64-builtin-pairs.def b/gcc/config/aarch64/aarch64-builtin-pairs.def
new file mode 100644
index 0000000..1757e19
--- /dev/null
+++ b/gcc/config/aarch64/aarch64-builtin-pairs.def
@@ -0,0 +1,73 @@
+/* Pairs of AArch64 builtins that can be folded into each other.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+/* Pairs of single and half integer modes.  */
+#define LO_HI_PAIR_V_HSI(T, LO, HI) \
+  LO_HI_PAIR (T##_##LO##v2si, T##_##HI##v4si) \
+  LO_HI_PAIR (T##_##LO##v4hi, T##_##HI##v8hi)
+
+#define LO_HI_PAIR_V_US_HSI(T, LO, HI) \
+  LO_HI_PAIR_V_HSI (T, s##LO, s##HI) \
+  LO_HI_PAIR_V_HSI (T##U, u##LO, u##HI)
+
+/* Pairs of widenable integer modes.  */
+#define LO_HI_PAIR_V_WI(T, LO, HI) \
+  LO_HI_PAIR_V_HSI (T, LO, HI) \
+  LO_HI_PAIR (T##_##LO##v8qi, T##_##HI##v16qi)
+
+#define LO_HI_PAIR_V_US_WI(T, LO, HI) \
+  LO_HI_PAIR_V_WI (T, s##LO, s##HI) \
+  LO_HI_PAIR_V_WI (T##U, u##LO, u##HI)
+
+#define UNOP_LONG_LH_PAIRS \
+  LO_HI_PAIR (UNOP_sxtlv8hi,  UNOP_vec_unpacks_hi_v16qi) \
+  LO_HI_PAIR (UNOP_sxtlv4si,  UNOP_vec_unpacks_hi_v8hi) \
+  LO_HI_PAIR (UNOP_sxtlv2di,  UNOP_vec_unpacks_hi_v4si) \
+  LO_HI_PAIR (UNOPU_uxtlv8hi, UNOPU_vec_unpacku_hi_v16qi) \
+  LO_HI_PAIR (UNOPU_uxtlv4si, UNOPU_vec_unpacku_hi_v8hi) \
+  LO_HI_PAIR (UNOPU_uxtlv2di, UNOPU_vec_unpacku_hi_v4si)
+
+#define BINOP_LONG_LH_PAIRS \
+  LO_HI_PAIR_V_US_WI (BINOP,  addl, addl2) \
+  LO_HI_PAIR_V_US_WI (BINOP,  subl, subl2) \
+  LO_HI_PAIR_V_US_WI (BINOP,  abdl, abdl2) \
+  LO_HI_PAIR_V_WI (BINOP,  intrinsic_vec_smult_lo_, vec_widen_smult_hi_) \
+  LO_HI_PAIR_V_WI (BINOPU, intrinsic_vec_umult_lo_, vec_widen_umult_hi_) \
+  LO_HI_PAIR_V_HSI (BINOP,  sqdmull, sqdmull2) \
+  LO_HI_PAIR (BINOPP_pmullv8qi, BINOPP_pmull_hiv16qi)
+
+#define BINOP_LONG_N_LH_PAIRS \
+  LO_HI_PAIR_V_US_HSI (BINOP,  mull_n, mull_hi_n) \
+  LO_HI_PAIR_V_HSI (BINOP,  sqdmull_n, sqdmull2_n)
+
+#define BINOP_WIDE_LH_PAIRS \
+  LO_HI_PAIR_V_US_WI (BINOP,  subw, subw2) \
+  LO_HI_PAIR_V_US_WI (BINOP,  addw, addw2)
+
+#define TERNOP_LONG_LH_PAIRS \
+  LO_HI_PAIR_V_US_WI (TERNOP,  mlal, mlal_hi) \
+  LO_HI_PAIR_V_US_WI (TERNOP,  mlsl, mlsl_hi) \
+  LO_HI_PAIR_V_US_WI (TERNOP,  abal, abal2) \
+  LO_HI_PAIR_V_HSI (TERNOP, sqdmlal, sqdmlal2) \
+  LO_HI_PAIR_V_HSI (TERNOP, sqdmlsl, sqdmlsl2)
+
+#define TERNOP_LONG_N_LH_PAIRS \
+  LO_HI_PAIR_V_US_HSI (TERNOP,  mlal_n, mlal_hi_n) \
+  LO_HI_PAIR_V_US_HSI (TERNOP,  mlsl_n, mlsl_hi_n) \
+  LO_HI_PAIR_V_HSI (TERNOP,  sqdmlal_n, sqdmlal2_n)
diff --git a/gcc/config/aarch64/aarch64-builtins.cc b/gcc/config/aarch64/aarch64-builtins.cc
index 93f939a..408099a 100644
--- a/gcc/config/aarch64/aarch64-builtins.cc
+++ b/gcc/config/aarch64/aarch64-builtins.cc
@@ -49,6 +49,8 @@
 #include "attribs.h"
 #include "gimple-fold.h"
 #include "builtins.h"
+#include "tree-pass.h"
+#include "tree-vector-builder.h"
 #include "aarch64-builtins.h"
 
 using namespace aarch64;
@@ -738,6 +740,16 @@ static aarch64_simd_builtin_datum aarch64_simd_builtin_data[] = {
   VGET_HIGH_BUILTIN(u64) \
   VGET_HIGH_BUILTIN(bf16)
 
+#include "aarch64-builtin-pairs.def"
+
+#define LO_HI_PAIRINGS \
+  UNOP_LONG_LH_PAIRS \
+  BINOP_LONG_LH_PAIRS \
+  BINOP_LONG_N_LH_PAIRS \
+  BINOP_WIDE_LH_PAIRS \
+  TERNOP_LONG_LH_PAIRS \
+  TERNOP_LONG_N_LH_PAIRS
+
 typedef struct
 {
   const char *name;
@@ -5004,6 +5016,167 @@ aarch64_gimple_fold_pragma_builtin
     }
 }
 
+/* Return the fndecl of the builtin paired with FCODE_LO if one
+   exists (see aarch64-builtin-pairs.def), or NULL_TREE if not.  */
+static inline tree
+aarch64_get_highpart_builtin (unsigned int fcode_lo)
+{
+#undef LO_HI_PAIR
+#define LO_HI_PAIR(A, B) case AARCH64_SIMD_BUILTIN_##A:   \
+  return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_##B];
+
+  switch (fcode_lo)
+    {
+      LO_HI_PAIRINGS
+      default:
+	return NULL_TREE;
+    }
+}
+
+/* If REF describes the high half of a 128-bit vector, return this
+   vector.  Otherwise, return NULL_TREE.  */
+static tree
+aarch64_v128_highpart_ref (const_tree ref)
+{
+  if (TREE_CODE (ref) != SSA_NAME)
+    return NULL_TREE;
+
+  gassign *stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (ref));
+  if (!stmt || gimple_assign_rhs_code (stmt) != BIT_FIELD_REF)
+    return NULL_TREE;
+
+  /* Look for a BIT_FIELD_REF that denotes the most significant 64
+     bits of a 128-bit vector.  */
+  tree bf_ref = gimple_assign_rhs1 (stmt);
+  unsigned int offset = BYTES_BIG_ENDIAN ? 0 : 64;
+
+  if (maybe_ne (bit_field_size (bf_ref), 64u)
+      || maybe_ne (bit_field_offset (bf_ref), offset))
+    return NULL_TREE;
+
+  tree obj = TREE_OPERAND (bf_ref, 0);
+  tree type = TREE_TYPE (obj);
+
+  if (VECTOR_TYPE_P (type) && tree_fits_uhwi_p (TYPE_SIZE (type))
+      && tree_to_uhwi (TYPE_SIZE (type)) == 128)
+    return obj;
+
+  return NULL_TREE;
+}
+
+/* Build and return a new VECTOR_CST of type OUT_TY, using repeated
+   copies of the elements of VEC_IN.  */
+static tree
+aarch64_build_vector_cst (const_tree vec_in, tree out_ty)
+{
+  gcc_assert (TREE_CODE (vec_in) == VECTOR_CST
+	      && VECTOR_TYPE_P (out_ty));
+  unsigned HOST_WIDE_INT nelts
+    = VECTOR_CST_NELTS (vec_in).to_constant ();
+
+  tree_vector_builder vec_out (out_ty, nelts, 1);
+  for (unsigned i = 0; i < nelts; i++)
+    vec_out.quick_push (VECTOR_CST_ELT (vec_in, i));
+
+  return vec_out.build ();
+}
+
+/* Try to fold STMT, a call to to a lowpart-operating builtin, to
+   it's highpart-operating equivalent if doing so would save
+   unnecessary data movement instructions.
+
+   Return the new call if so, otherwise nullptr.  */
+static gcall *
+aarch64_fold_lo_call_to_hi (unsigned int fcode, gcall *stmt,
+			    gimple_stmt_iterator *gsi)
+{
+  /* Punt until as late as possible:
+    1) By folding away BIT_FIELD_REFs we remove information about the
+    operands that may be useful to other optimizers.
+
+    2) For simplicity, we'd like the expression
+
+	x = BIT_FIELD_REF<a, x, y>
+
+    to imply that A is not a VECTOR_CST.  This assumption is unlikely
+    to hold before constant prop/folding.  */
+  if (!(cfun->curr_properties & PROP_last_full_fold))
+    return nullptr;
+
+  tree builtin_hi = aarch64_get_highpart_builtin (fcode);
+  gcc_assert (builtin_hi != NULL_TREE);
+
+  /* Prefer to use the highpart builtin when at least one vector
+     argument is a reference to the high half of a 128b vector, and
+     all others are VECTOR_CSTs that we can extend to 128b.  */
+  auto_vec<unsigned int, 2> vec_constants;
+  auto_vec<unsigned int, 2> vec_highparts;
+  /* The arguments and signature of the new call.  */
+  auto_vec<tree, 4> call_args;
+  auto_vec<tree, 4> call_types;
+
+  /* The interesting args are those that differ between the lo/hi
+     builtins.  Walk the function signatures to find these.  */
+  tree types_hi = TYPE_ARG_TYPES (TREE_TYPE (builtin_hi));
+  tree types_lo = TYPE_ARG_TYPES (gimple_call_fntype (stmt));
+  unsigned int argno = 0;
+  while (types_lo != void_list_node && types_hi != void_list_node)
+    {
+      tree type_lo = TREE_VALUE (types_lo);
+      tree type_hi = TREE_VALUE (types_hi);
+      tree arg = gimple_call_arg (stmt, argno);
+      if (!types_compatible_p (type_lo, type_hi))
+	{
+	  /* Check our assumptions about this pair.  */
+	  gcc_assert (wi::to_widest (TYPE_SIZE (type_lo)) == 64
+		      && wi::to_widest (TYPE_SIZE (type_hi)) == 128);
+
+	  tree vq = aarch64_v128_highpart_ref (arg);
+	  if (vq && is_gimple_reg (vq))
+	    {
+	      vec_highparts.safe_push (argno);
+	      arg = vq;
+	    }
+	  else if (TREE_CODE (arg) == VECTOR_CST)
+	    vec_constants.safe_push (argno);
+	  else
+	    return nullptr;
+	}
+      call_args.safe_push (arg);
+      call_types.safe_push (type_hi);
+
+      argno++;
+      types_hi = TREE_CHAIN (types_hi);
+      types_lo = TREE_CHAIN (types_lo);
+    }
+  gcc_assert (types_lo == void_list_node && types_hi == void_list_node);
+
+  if (vec_highparts.is_empty ())
+    return nullptr;
+
+  /* Build and return a new call to BUILTIN_HI.  */
+  for (auto i : vec_constants)
+    call_args[i] = aarch64_build_vector_cst (call_args[i], call_types[i]);
+
+  for (auto i : vec_highparts)
+    if (!types_compatible_p (TREE_TYPE (call_args[i]), call_types[i]))
+      {
+	tree vce_ssa = make_ssa_name (call_types[i]);
+	tree vce_expr = build1 (VIEW_CONVERT_EXPR,
+				call_types[i], call_args[i]);
+	gsi_insert_before (gsi, gimple_build_assign (vce_ssa, vce_expr),
+			   GSI_SAME_STMT);
+	call_args[i] = vce_ssa;
+      }
+
+  gcall *new_call = gimple_build_call_vec (builtin_hi, call_args);
+  gimple_call_set_lhs (new_call, gimple_call_lhs (stmt));
+  return new_call;
+}
+
+#undef LO_HI_PAIR
+#define LO_HI_PAIR(A, B) case AARCH64_SIMD_BUILTIN_##A:
+
 /* Try to fold STMT, given that it's a call to the built-in function with
    subcode FCODE.  Return the new statement on success and null on
    failure.  */
@@ -5190,6 +5363,9 @@ aarch64_general_gimple_fold_builtin (unsigned int fcode, gcall *stmt,
 	    }
 	  break;
 	}
+    LO_HI_PAIRINGS
+      new_stmt = aarch64_fold_lo_call_to_hi (fcode, stmt, gsi);
+      break;
     case AARCH64_SIMD_BUILTIN_LANE_CHECK:
       if (aarch64_fold_builtin_lane_check (args[0], args[1], args[2]))
 	{
diff --git a/gcc/config/aarch64/aarch64-cores.def b/gcc/config/aarch64/aarch64-cores.def
index 8040409..6f11cc0 100644
--- a/gcc/config/aarch64/aarch64-cores.def
+++ b/gcc/config/aarch64/aarch64-cores.def
@@ -224,7 +224,7 @@ AARCH64_CORE("neoverse-v3ae", neoversev3ae, cortexa57, V9_2A, (SVE2_BITPERM, RNG
 AARCH64_CORE("demeter", demeter, cortexa57, V9A, (I8MM, BF16, SVE2_BITPERM, RNG, MEMTAG, PROFILE), neoversev2, 0x41, 0xd4f, -1)
 
 /* NVIDIA ('N') cores. */
-AARCH64_CORE("olympus", olympus, cortexa57, V9_2A, (SVE2_BITPERM, RNG, LS64, MEMTAG, PROFILE, FAMINMAX, FP8FMA, FP8DOT2, FP8DOT4, LUT, SVE2_AES, SVE2_SHA3, SVE2_SM4), neoversev3, 0x4e, 0x10, -1)
+AARCH64_CORE("olympus", olympus, cortexa57, V9_2A, (SVE2_BITPERM, RNG, LS64, MEMTAG, PROFILE, FAMINMAX, FP8FMA, FP8DOT2, FP8DOT4, LUT, SVE2_AES, SVE2_SHA3, SVE2_SM4), olympus, 0x4e, 0x10, -1)
 
 /* Armv9-A big.LITTLE processors.  */
 AARCH64_CORE("gb10",  gb10, cortexa57, V9_2A,  (SVE2_BITPERM, SVE2_AES, SVE2_SHA3, SVE2_SM4, MEMTAG, PROFILE), cortexx925, 0x41, AARCH64_BIG_LITTLE (0xd85, 0xd87), -1)
diff --git a/gcc/config/aarch64/aarch64-cost-tables.h b/gcc/config/aarch64/aarch64-cost-tables.h
index c49ff7f..e7926eb 100644
--- a/gcc/config/aarch64/aarch64-cost-tables.h
+++ b/gcc/config/aarch64/aarch64-cost-tables.h
@@ -125,9 +125,9 @@ const struct cpu_cost_table qdf24xx_extra_costs =
   {
     COSTS_N_INSNS (1),  /* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -233,9 +233,9 @@ const struct cpu_cost_table thunderx_extra_costs =
   {
     COSTS_N_INSNS (1),	/* Alu.  */
     COSTS_N_INSNS (4),	/* mult.  */
-    COSTS_N_INSNS (1),	/* movi.  */
-    COSTS_N_INSNS (2),	/* dup.  */
-    COSTS_N_INSNS (2)	/* extract.  */
+    COSTS_N_INSNS (0),	/* movi.  */
+    COSTS_N_INSNS (1),	/* dup.  */
+    COSTS_N_INSNS (1)	/* extract.  */
   }
 };
 
@@ -340,9 +340,9 @@ const struct cpu_cost_table thunderx2t99_extra_costs =
   {
     COSTS_N_INSNS (1),	/* Alu.  */
     COSTS_N_INSNS (4),	/* Mult.  */
-    COSTS_N_INSNS (1),	/* movi.  */
-    COSTS_N_INSNS (2),	/* dup.  */
-    COSTS_N_INSNS (2)	/* extract.  */
+    COSTS_N_INSNS (0),	/* movi.  */
+    COSTS_N_INSNS (1),	/* dup.  */
+    COSTS_N_INSNS (1)	/* extract.  */
   }
 };
 
@@ -447,9 +447,9 @@ const struct cpu_cost_table thunderx3t110_extra_costs =
   {
     COSTS_N_INSNS (1),	/* Alu.  */
     COSTS_N_INSNS (4),	/* Mult.  */
-    COSTS_N_INSNS (1),	/* movi.  */
-    COSTS_N_INSNS (2),	/* dup.  */
-    COSTS_N_INSNS (2)	/* extract.  */
+    COSTS_N_INSNS (0),	/* movi.  */
+    COSTS_N_INSNS (1),	/* dup.  */
+    COSTS_N_INSNS (1)	/* extract.  */
   }
 };
 
@@ -555,9 +555,9 @@ const struct cpu_cost_table tsv110_extra_costs =
   {
     COSTS_N_INSNS (1),  /* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -662,9 +662,9 @@ const struct cpu_cost_table a64fx_extra_costs =
   {
     COSTS_N_INSNS (1),  /* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -769,9 +769,9 @@ const struct cpu_cost_table ampere1_extra_costs =
   {
     COSTS_N_INSNS (3),  /* alu.  */
     COSTS_N_INSNS (3),  /* mult.  */
-    COSTS_N_INSNS (2),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (1),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -876,9 +876,9 @@ const struct cpu_cost_table ampere1a_extra_costs =
   {
     COSTS_N_INSNS (3),  /* alu.  */
     COSTS_N_INSNS (3),  /* mult.  */
-    COSTS_N_INSNS (2),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (1),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -983,9 +983,9 @@ const struct cpu_cost_table ampere1b_extra_costs =
   {
     COSTS_N_INSNS (1),  /* alu.  */
     COSTS_N_INSNS (2),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (1),  /* dup.  */
-    COSTS_N_INSNS (1)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (0),  /* dup.  */
+    COSTS_N_INSNS (0)   /* extract.  */
   }
 };
 
diff --git a/gcc/config/aarch64/aarch64-option-extensions.def b/gcc/config/aarch64/aarch64-option-extensions.def
index 1c3e697..db88df0 100644
--- a/gcc/config/aarch64/aarch64-option-extensions.def
+++ b/gcc/config/aarch64/aarch64-option-extensions.def
@@ -128,7 +128,9 @@ AARCH64_OPT_FMV_EXTENSION("sha2", SHA2, (SIMD), (), (), "sha1 sha2")
 
 AARCH64_FMV_FEATURE("sha3", SHA3, (SHA3))
 
-AARCH64_OPT_FMV_EXTENSION("aes", AES, (SIMD), (), (), "aes")
+AARCH64_OPT_EXTENSION("aes", AES, (SIMD), (), (), "aes")
+
+AARCH64_FMV_FEATURE("aes", PMULL, (AES))
 
 /* +nocrypto disables AES, SHA2 and SM4, and anything that depends on them
    (such as SHA3 and the SVE2 crypto extensions).  */
@@ -171,8 +173,6 @@ AARCH64_OPT_FMV_EXTENSION("i8mm", I8MM, (SIMD), (), (), "i8mm")
    instructions.  */
 AARCH64_OPT_FMV_EXTENSION("bf16", BF16, (FP), (SIMD), (), "bf16")
 
-AARCH64_FMV_FEATURE("rpres", RPRES, ())
-
 AARCH64_OPT_FMV_EXTENSION("sve", SVE, (SIMD, F16, FCMA), (), (), "sve")
 
 /* This specifically does not imply +sve.  */
@@ -190,7 +190,7 @@ AARCH64_OPT_FMV_EXTENSION("sve2", SVE2, (SVE), (), (), "sve2")
 
 AARCH64_OPT_EXTENSION("sve2-aes", SVE2_AES, (SVE2, AES), (), (), "sveaes")
 
-AARCH64_FMV_FEATURE("sve2-aes", SVE_AES, (SVE2_AES))
+AARCH64_FMV_FEATURE("sve2-aes", SVE_PMULL128, (SVE2_AES))
 
 AARCH64_OPT_EXTENSION("sve2-bitperm", SVE2_BITPERM, (SVE2), (), (),
 		      "svebitperm")
@@ -245,9 +245,9 @@ AARCH64_OPT_EXTENSION("sme-b16b16", SME_B16B16, (SME2, SVE_B16B16), (), (), "sme
 
 AARCH64_OPT_EXTENSION("sme-f16f16", SME_F16F16, (SME2), (), (), "smef16f16")
 
-AARCH64_OPT_EXTENSION("mops", MOPS, (), (), (), "mops")
+AARCH64_OPT_FMV_EXTENSION("mops", MOPS, (), (), (), "mops")
 
-AARCH64_OPT_EXTENSION("cssc", CSSC, (), (), (), "cssc")
+AARCH64_OPT_FMV_EXTENSION("cssc", CSSC, (), (), (), "cssc")
 
 AARCH64_OPT_EXTENSION("cmpbr", CMPBR, (), (), (), "cmpbr")
 
diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h
index e946e8d..38c307c 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -1031,6 +1031,7 @@ rtx aarch64_pfalse_reg (machine_mode);
 bool aarch64_sve_same_pred_for_ptest_p (rtx *, rtx *);
 rtx aarch64_sve_packed_pred (machine_mode);
 rtx aarch64_sve_fp_pred (machine_mode, rtx *);
+rtx aarch64_sve_emit_masked_fp_pred (machine_mode, rtx);
 void aarch64_emit_load_store_through_mode (rtx, rtx, machine_mode);
 bool aarch64_expand_maskloadstore (rtx *, machine_mode);
 void aarch64_emit_sve_pred_move (rtx, rtx, rtx);
diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md
index 8de79ca..8b75c3d 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -1190,13 +1190,16 @@
   [(set_attr "type" "neon_ins<q>, neon_from_gp<q>, neon_load1_one_lane<q>")]
 )
 
+;; Inserting from the zero register into a vector lane is treated as an
+;; expensive GP->FP move on all CPUs.  Avoid it when optimizing for speed.
 (define_insn "aarch64_simd_vec_set_zero<mode>"
   [(set (match_operand:VALL_F16 0 "register_operand" "=w")
 	(vec_merge:VALL_F16
 	    (match_operand:VALL_F16 1 "register_operand" "0")
 	    (match_operand:VALL_F16 3 "aarch64_simd_imm_zero" "")
 	    (match_operand:SI 2 "immediate_operand" "i")))]
-  "TARGET_SIMD && aarch64_exact_log2_inverse (<nunits>, operands[2]) >= 0"
+  "TARGET_SIMD && aarch64_exact_log2_inverse (<nunits>, operands[2]) >= 0
+   && optimize_function_for_size_p (cfun)"
   {
     int elt = ENDIAN_LANE_N (<nunits>,
 			     aarch64_exact_log2_inverse (<nunits>,
@@ -9180,12 +9183,12 @@
 ;; sha3
 
 (define_insn "eor3q<mode>4"
-  [(set (match_operand:VQ_I 0 "register_operand" "=w")
-	(xor:VQ_I
-	 (xor:VQ_I
-	  (match_operand:VQ_I 2 "register_operand" "w")
-	  (match_operand:VQ_I 3 "register_operand" "w"))
-	 (match_operand:VQ_I 1 "register_operand" "w")))]
+  [(set (match_operand:VDQ_I 0 "register_operand" "=w")
+	(xor:VDQ_I
+	 (xor:VDQ_I
+	  (match_operand:VDQ_I 2 "register_operand" "w")
+	  (match_operand:VDQ_I 3 "register_operand" "w"))
+	 (match_operand:VDQ_I 1 "register_operand" "w")))]
   "TARGET_SHA3"
   "eor3\\t%0.16b, %1.16b, %2.16b, %3.16b"
   [(set_attr "type" "crypto_sha3")]
@@ -9241,17 +9244,46 @@
 )
 
 (define_insn "bcaxq<mode>4"
-  [(set (match_operand:VQ_I 0 "register_operand" "=w")
-	(xor:VQ_I
-	 (and:VQ_I
-	  (not:VQ_I (match_operand:VQ_I 3 "register_operand" "w"))
-	  (match_operand:VQ_I 2 "register_operand" "w"))
-	 (match_operand:VQ_I 1 "register_operand" "w")))]
+  [(set (match_operand:VDQ_I 0 "register_operand" "=w")
+	(xor:VDQ_I
+	 (and:VDQ_I
+	  (not:VDQ_I (match_operand:VDQ_I 3 "register_operand" "w"))
+	  (match_operand:VDQ_I 2 "register_operand" "w"))
+	 (match_operand:VDQ_I 1 "register_operand" "w")))]
   "TARGET_SHA3"
   "bcax\\t%0.16b, %1.16b, %2.16b, %3.16b"
   [(set_attr "type" "crypto_sha3")]
 )
 
+(define_insn_and_split "*bcaxqdi4"
+  [(set (match_operand:DI 0 "register_operand")
+	(xor:DI
+	  (and:DI
+	    (not:DI (match_operand:DI 3 "register_operand"))
+	    (match_operand:DI 2 "register_operand"))
+	  (match_operand:DI 1 "register_operand")))]
+  "TARGET_SHA3"
+  {@ [ cons: =0, 1, 2 , 3  ; attrs: type ]
+     [ w       , w, w , w  ; crypto_sha3 ] bcax\t%0.16b, %1.16b, %2.16b, %3.16b
+     [ &r      , r, r0, r0 ; multiple    ] #
+  }
+  "&& REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
+  [(set (match_dup 4)
+	(and:DI (not:DI (match_dup 3))
+		(match_dup 2)))
+   (set (match_dup 0)
+	(xor:DI (match_dup 4)
+		(match_dup 1)))]
+  {
+    if (reload_completed)
+      operands[4] = operands[0];
+    else if (can_create_pseudo_p ())
+      operands[4] = gen_reg_rtx (DImode);
+    else
+      FAIL;
+  }
+)
+
 ;; SM3
 
 (define_insn "aarch64_sm3ss1qv4si"
diff --git a/gcc/config/aarch64/aarch64-sme.md b/gcc/config/aarch64/aarch64-sme.md
index b8bb4cc..6b1a747 100644
--- a/gcc/config/aarch64/aarch64-sme.md
+++ b/gcc/config/aarch64/aarch64-sme.md
@@ -38,6 +38,7 @@
 ;; ---- Binary arithmetic on ZA tile
 ;; ---- Binary arithmetic on ZA slice
 ;; ---- Binary arithmetic, writing to ZA slice
+;; ---- Absolute minimum/maximum
 ;;
 ;; == Ternary arithmetic
 ;; ---- [INT] Dot product
@@ -61,6 +62,10 @@
 ;; (b) they are sometimes used conditionally, particularly in streaming-
 ;; compatible code.
 ;;
+;; To prevent the latter from upsetting the assembler, we emit the literal
+;; encodings of "SMSTART SM" and "SMSTOP SM" when compiling without
+;; TARGET_SME.
+;;
 ;; =========================================================================
 
 ;; -------------------------------------------------------------------------
@@ -160,7 +165,9 @@
    (clobber (reg:VNx16BI P14_REGNUM))
    (clobber (reg:VNx16BI P15_REGNUM))]
   ""
-  "smstart\tsm"
+  {
+    return TARGET_SME ? "smstart\tsm" : ".inst 0xd503437f // smstart sm";
+  }
 )
 
 ;; Turn off streaming mode.  This clobbers all SVE state.
@@ -195,7 +202,9 @@
    (clobber (reg:VNx16BI P14_REGNUM))
    (clobber (reg:VNx16BI P15_REGNUM))]
   ""
-  "smstop\tsm"
+  {
+    return TARGET_SME ? "smstop\tsm" : ".inst 0xd503427f // smstop sm";
+  }
 )
 
 ;; -------------------------------------------------------------------------
@@ -1264,6 +1273,23 @@
   "<sme_int_op>\tza.<Vetype>[%w0, %1, vgx<vector_count>], %2, %3.<Vetype>"
 )
 
+;; -------------------------------------------------------------------------
+;; ---- Absolute minimum/maximum
+;; -------------------------------------------------------------------------
+;; Includes:
+;; - FAMIN (SME2+FAMINMAX)
+;; - FAMAX (SME2+FAMINMAX)
+;; -------------------------------------------------------------------------
+
+(define_insn "@aarch64_sme_<faminmax_uns_op><mode>"
+  [(set (match_operand:SVE_Fx24 0 "register_operand" "=Uw<vector_count>")
+	(unspec:SVE_Fx24 [(match_operand:SVE_Fx24 1 "register_operand" "%0")
+			  (match_operand:SVE_Fx24 2 "register_operand" "Uw<vector_count>")]
+	 FAMINMAX_UNS))]
+  "TARGET_STREAMING_SME2 && TARGET_FAMINMAX"
+  "<faminmax_uns_op>\t%0, %1, %2"
+)
+
 ;; =========================================================================
 ;; == Ternary arithmetic
 ;; =========================================================================
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-sme.def b/gcc/config/aarch64/aarch64-sve-builtins-sme.def
index f75c0a5..117b70e 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-sme.def
+++ b/gcc/config/aarch64/aarch64-sve-builtins-sme.def
@@ -92,6 +92,12 @@ DEF_SME_FUNCTION (svstr_zt, str_zt, none, none)
 DEF_SME_FUNCTION (svzero_zt, inherent_zt, none, none)
 #undef REQUIRED_EXTENSIONS
 
+#define REQUIRED_EXTENSIONS streaming_only (AARCH64_FL_SME2 \
+					    | AARCH64_FL_FAMINMAX)
+DEF_SME_FUNCTION_GS (svamin, binary_opt_single_n, all_float, x24, none)
+DEF_SME_FUNCTION_GS (svamax, binary_opt_single_n, all_float, x24, none)
+#undef REQUIRED_EXTENSIONS
+
 /* The d_za entries in this section just declare C _za64 overloads,
    which will then be resolved to either an integer function or a
    floating-point function.  They are needed because the integer and
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-sve2.cc b/gcc/config/aarch64/aarch64-sve-builtins-sve2.cc
index d9922de..73004a8 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-sve2.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins-sve2.cc
@@ -316,7 +316,8 @@ public:
   expand (function_expander &e) const override
   {
     e.prepare_gather_address_operands (1, false);
-    return e.use_exact_insn (CODE_FOR_aarch64_gather_ld1q);
+    auto icode = code_for_aarch64_gather_ld1q (e.tuple_mode (0));
+    return e.use_exact_insn (icode);
   }
 };
 
@@ -722,7 +723,7 @@ public:
   expand (function_expander &e) const override
   {
     rtx data = e.args.last ();
-    e.args.last () = force_lowpart_subreg (VNx2DImode, data, GET_MODE (data));
+    e.args.last () = aarch64_sve_reinterpret (VNx2DImode, data);
     e.prepare_gather_address_operands (1, false);
     return e.use_exact_insn (CODE_FOR_aarch64_scatter_st1q);
   }
@@ -929,6 +930,44 @@ public:
   unsigned int m_bits;
 };
 
+/* The same as cond_or_uncond_unspec_function but the intrinsics with vector
+   modes are SME2 extensions instead of SVE.  */
+class faminmaximpl : public function_base
+{
+public:
+  CONSTEXPR faminmaximpl (int cond_unspec, int uncond_unspec)
+    : m_cond_unspec (cond_unspec), m_uncond_unspec (uncond_unspec)
+    {}
+
+  rtx
+  expand (function_expander &e) const override
+  {
+    if (e.group_suffix ().vectors_per_tuple > 1)
+      {
+	/* SME2+faminmax intrinsics.  */
+	gcc_assert (e.pred == PRED_none);
+	auto mode = e.tuple_mode (0);
+	auto icode = (code_for_aarch64_sme (m_uncond_unspec, mode));
+	return e.use_exact_insn (icode);
+      }
+    /* SVE+faminmax intrinsics.  */
+    else if (e.pred == PRED_none)
+      {
+	auto mode = e.tuple_mode (0);
+	auto icode = (e.mode_suffix_id == MODE_single
+		      ? code_for_aarch64_sve_single (m_uncond_unspec, mode)
+		      : code_for_aarch64_sve (m_uncond_unspec, mode));
+	return e.use_exact_insn (icode);
+      }
+    return e.map_to_unspecs (m_cond_unspec, m_cond_unspec, m_cond_unspec);
+  }
+
+  /* The unspecs for the conditional and unconditional instructions,
+     respectively.  */
+  int m_cond_unspec;
+  int m_uncond_unspec;
+};
+
 } /* end anonymous namespace */
 
 namespace aarch64_sve {
@@ -957,10 +996,8 @@ FUNCTION (svaesd, fixed_insn_function, (CODE_FOR_aarch64_sve2_aesd))
 FUNCTION (svaese, fixed_insn_function, (CODE_FOR_aarch64_sve2_aese))
 FUNCTION (svaesimc, fixed_insn_function, (CODE_FOR_aarch64_sve2_aesimc))
 FUNCTION (svaesmc, fixed_insn_function, (CODE_FOR_aarch64_sve2_aesmc))
-FUNCTION (svamax, cond_or_uncond_unspec_function,
-	  (UNSPEC_COND_FAMAX, UNSPEC_FAMAX))
-FUNCTION (svamin, cond_or_uncond_unspec_function,
-	  (UNSPEC_COND_FAMIN, UNSPEC_FAMIN))
+FUNCTION (svamax, faminmaximpl, (UNSPEC_COND_FAMAX, UNSPEC_FAMAX))
+FUNCTION (svamin, faminmaximpl, (UNSPEC_COND_FAMIN, UNSPEC_FAMIN))
 FUNCTION (svandqv, reduction, (UNSPEC_ANDQV, UNSPEC_ANDQV, -1))
 FUNCTION (svbcax, CODE_FOR_MODE0 (aarch64_sve2_bcax),)
 FUNCTION (svbdep, unspec_based_function, (UNSPEC_BDEP, UNSPEC_BDEP, -1))
diff --git a/gcc/config/aarch64/aarch64-sve-builtins.cc b/gcc/config/aarch64/aarch64-sve-builtins.cc
index 2b627a9..01833a8 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins.cc
@@ -4004,7 +4004,8 @@ rtx
 function_expander::get_reg_target ()
 {
   machine_mode target_mode = result_mode ();
-  if (!possible_target || GET_MODE (possible_target) != target_mode)
+  if (!possible_target
+      || !register_operand (possible_target, target_mode))
     possible_target = gen_reg_rtx (target_mode);
   return possible_target;
 }
diff --git a/gcc/config/aarch64/aarch64-sve.md b/gcc/config/aarch64/aarch64-sve.md
index 4aecb3a..80a3288 100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -3752,9 +3752,9 @@
 
 ;; Unpredicated floating-point unary operations.
 (define_insn "@aarch64_sve_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w")
-	(unspec:SVE_FULL_F
-	  [(match_operand:SVE_FULL_F 1 "register_operand" "w")]
+  [(set (match_operand:SVE_F 0 "register_operand" "=w")
+	(unspec:SVE_F
+	  [(match_operand:SVE_F 1 "register_operand" "w")]
 	  SVE_FP_UNARY))]
   "TARGET_SVE"
   "<sve_fp_op>\t%0.<Vetype>, %1.<Vetype>"
@@ -3762,25 +3762,41 @@
 
 ;; Unpredicated floating-point unary operations.
 (define_expand "<optab><mode>2"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_dup 2)
-	   (const_int SVE_RELAXED_GP)
-	   (match_operand:SVE_FULL_F 1 "register_operand")]
+	   (match_dup 3)
+	   (match_operand:SVE_F 1 "register_operand")]
 	  SVE_COND_FP_UNARY_OPTAB))]
   "TARGET_SVE"
   {
+    operands[2] = aarch64_sve_fp_pred (<MODE>mode, &operands[3]);
+  }
+)
+
+;; FABS and FNEG are non-trapping, so we can always expand with a <VPRED>
+;; predicate.  It doesn't matter whether the padding bits of a partial
+;; vector mode are active or inactive.
+(define_expand "<optab><mode>2"
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_dup 2)
+	   (const_int SVE_RELAXED_GP)
+	   (match_operand:SVE_F 1 "register_operand")]
+	  SVE_COND_FP_UNARY_BITWISE))]
+  "TARGET_SVE"
+  {
     operands[2] = aarch64_ptrue_reg (<VPRED>mode);
   }
 )
 
 ;; Predicated floating-point unary operations.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 3 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F 2 "register_operand")]
+	   (match_operand:SVE_F 2 "register_operand")]
 	  SVE_COND_FP_UNARY))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , 2 ; attrs: movprfx ]
@@ -3806,13 +3822,13 @@
 
 ;; Predicated floating-point unary arithmetic, merging with the first input.
 (define_insn_and_rewrite "*cond_<optab><mode>_2_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 3)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")]
+	      (match_operand:SVE_F 2 "register_operand")]
 	     SVE_COND_FP_UNARY)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -3854,15 +3870,15 @@
 ;; as earlyclobber helps to make the instruction more regular to the
 ;; register allocator.
 (define_insn_and_rewrite "*cond_<optab><mode>_any_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 4)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")]
+	      (match_operand:SVE_F 2 "register_operand")]
 	     SVE_COND_FP_UNARY)
-	   (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[2], operands[3])"
   {@ [ cons: =0 , 1   , 2 , 3  ; attrs: movprfx ]
@@ -3966,7 +3982,7 @@
 )
 
 ;; Predicated predicate inverse.
-(define_insn "*one_cmpl<mode>3"
+(define_insn "@aarch64_pred_one_cmpl<mode>_z"
   [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa")
 	(and:PRED_ALL
 	  (not:PRED_ALL (match_operand:PRED_ALL 2 "register_operand" "Upa"))
@@ -5495,27 +5511,25 @@
 ;; Split a predicated instruction whose predicate is unused into an
 ;; unpredicated instruction.
 (define_split
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	   (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	   (const_int SVE_RELAXED_GP)
+	   (match_operand:SVE_F_B16B16 2 "register_operand")
+	   (match_operand:SVE_F_B16B16 3 "register_operand")]
 	  <SVE_COND_FP>))]
-  "TARGET_SVE
-   && reload_completed
-   && INTVAL (operands[4]) == SVE_RELAXED_GP"
+  "TARGET_SVE && reload_completed"
   [(set (match_dup 0)
-	(SVE_UNPRED_FP_BINARY:SVE_FULL_F_B16B16 (match_dup 2) (match_dup 3)))]
+	(SVE_UNPRED_FP_BINARY:SVE_F_B16B16 (match_dup 2) (match_dup 3)))]
 )
 
 ;; Unpredicated floating-point binary operations (post-RA only).
 ;; These are generated by the split above.
 (define_insn "*post_ra_<sve_fp_op><mode>3"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand" "=w")
-	(SVE_UNPRED_FP_BINARY:SVE_FULL_F_B16B16
-	  (match_operand:SVE_FULL_F_B16B16 1 "register_operand" "w")
-	  (match_operand:SVE_FULL_F_B16B16 2 "register_operand" "w")))]
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand" "=w")
+	(SVE_UNPRED_FP_BINARY:SVE_F_B16B16
+	  (match_operand:SVE_F_B16B16 1 "register_operand" "w")
+	  (match_operand:SVE_F_B16B16 2 "register_operand" "w")))]
   "TARGET_SVE && reload_completed"
   "<b><sve_fp_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>")
 
@@ -5547,10 +5561,10 @@
 
 ;; Unpredicated floating-point binary operations.
 (define_insn "@aarch64_sve_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand" "=w")
-	(unspec:SVE_FULL_F
-	  [(match_operand:SVE_FULL_F 1 "register_operand" "w")
-	   (match_operand:SVE_FULL_F 2 "register_operand" "w")]
+  [(set (match_operand:SVE_F 0 "register_operand" "=w")
+	(unspec:SVE_F
+	  [(match_operand:SVE_F 1 "register_operand" "w")
+	   (match_operand:SVE_F 2 "register_operand" "w")]
 	  SVE_FP_BINARY))]
   "TARGET_SVE"
   "<sve_fp_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>"
@@ -5559,27 +5573,27 @@
 ;; Unpredicated floating-point binary operations that need to be predicated
 ;; for SVE.
 (define_expand "<optab><mode>3"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_dup 3)
-	   (const_int SVE_RELAXED_GP)
-	   (match_operand:SVE_FULL_F_B16B16 1 "<sve_pred_fp_rhs1_operand>")
-	   (match_operand:SVE_FULL_F_B16B16 2 "<sve_pred_fp_rhs2_operand>")]
+	   (match_dup 4)
+	   (match_operand:SVE_F_B16B16 1 "<sve_pred_fp_rhs1_operand>")
+	   (match_operand:SVE_F_B16B16 2 "<sve_pred_fp_rhs2_operand>")]
 	  SVE_COND_FP_BINARY_OPTAB))]
   "TARGET_SVE && (<supports_bf16> || !<is_bf16>)"
   {
-    operands[3] = aarch64_ptrue_reg (<VPRED>mode);
+    operands[3] = aarch64_sve_fp_pred (<MODE>mode, &operands[4]);
   }
 )
 
 ;; Predicated floating-point binary operations that have no immediate forms.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F 2 "register_operand")
-	   (match_operand:SVE_FULL_F 3 "register_operand")]
+	   (match_operand:SVE_F 2 "register_operand")
+	   (match_operand:SVE_F 3 "register_operand")]
 	  SVE_COND_FP_BINARY_REG))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , 2 , 3 ; attrs: movprfx ]
@@ -5591,30 +5605,33 @@
 
 ;; Predicated floating-point operations with merging.
 (define_expand "@cond_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "<sve_pred_fp_rhs1_operand>")
-	      (match_operand:SVE_FULL_F_B16B16 3 "<sve_pred_fp_rhs2_operand>")]
+	      (match_operand:SVE_F_B16B16 2 "<sve_pred_fp_rhs1_operand>")
+	      (match_operand:SVE_F_B16B16 3 "<sve_pred_fp_rhs2_operand>")]
 	     SVE_COND_FP_BINARY)
-	   (match_operand:SVE_FULL_F_B16B16 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F_B16B16 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && (<supports_bf16> || !<is_bf16>)"
+  {
+    operands[1] = aarch64_sve_emit_masked_fp_pred (<MODE>mode, operands[1]);
+  }
 )
 
 ;; Predicated floating-point operations, merging with the first input.
 (define_insn_and_rewrite "*cond_<optab><mode>_2_relaxed"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_operand 4)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")]
 	     SVE_COND_FP_BINARY)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -5630,14 +5647,14 @@
 )
 
 (define_insn "*cond_<optab><mode>_2_strict"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")]
 	     SVE_COND_FP_BINARY)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -5650,14 +5667,14 @@
 
 ;; Same for operations that take a 1-bit constant.
 (define_insn_and_rewrite "*cond_<optab><mode>_2_const_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 4)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "<sve_pred_fp_rhs2_immediate>")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "<sve_pred_fp_rhs2_immediate>")]
 	     SVE_COND_FP_BINARY_I1)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -5673,14 +5690,14 @@
 )
 
 (define_insn "*cond_<optab><mode>_2_const_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "<sve_pred_fp_rhs2_immediate>")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "<sve_pred_fp_rhs2_immediate>")]
 	     SVE_COND_FP_BINARY_I1)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -5693,14 +5710,14 @@
 
 ;; Predicated floating-point operations, merging with the second input.
 (define_insn_and_rewrite "*cond_<optab><mode>_3_relaxed"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_operand 4)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")]
 	     SVE_COND_FP_BINARY)
 	   (match_dup 3)]
 	  UNSPEC_SEL))]
@@ -5716,14 +5733,14 @@
 )
 
 (define_insn "*cond_<optab><mode>_3_strict"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")]
 	     SVE_COND_FP_BINARY)
 	   (match_dup 3)]
 	  UNSPEC_SEL))]
@@ -5736,16 +5753,16 @@
 
 ;; Predicated floating-point operations, merging with an independent value.
 (define_insn_and_rewrite "*cond_<optab><mode>_any_relaxed"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_operand 5)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")]
 	     SVE_COND_FP_BINARY)
-	   (match_operand:SVE_FULL_F_B16B16 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F_B16B16 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE
    && (<supports_bf16> || !<is_bf16>)
@@ -5780,16 +5797,16 @@
 )
 
 (define_insn_and_rewrite "*cond_<optab><mode>_any_strict"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")]
 	     SVE_COND_FP_BINARY)
-	   (match_operand:SVE_FULL_F_B16B16 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F_B16B16 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE
    && (<supports_bf16> || !<is_bf16>)
@@ -5818,16 +5835,16 @@
 
 ;; Same for operations that take a 1-bit constant.
 (define_insn_and_rewrite "*cond_<optab><mode>_any_const_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 5)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "<sve_pred_fp_rhs2_immediate>")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "<sve_pred_fp_rhs2_immediate>")]
 	     SVE_COND_FP_BINARY_I1)
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[2], operands[4])"
   {@ [ cons: =0 , 1   , 2 , 4   ]
@@ -5854,16 +5871,16 @@
 )
 
 (define_insn_and_rewrite "*cond_<optab><mode>_any_const_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "<sve_pred_fp_rhs2_immediate>")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "<sve_pred_fp_rhs2_immediate>")]
 	     SVE_COND_FP_BINARY_I1)
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[2], operands[4])"
   {@ [ cons: =0 , 1   , 2 , 4   ]
@@ -5892,12 +5909,12 @@
 
 ;; Predicated floating-point addition.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F 2 "register_operand")
-	   (match_operand:SVE_FULL_F 3 "aarch64_sve_float_arith_with_sub_operand")]
+	   (match_operand:SVE_F 2 "register_operand")
+	   (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_operand")]
 	  SVE_COND_FP_ADD))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , %2 , 3   , 4   ; attrs: movprfx ]
@@ -5914,14 +5931,14 @@
 ;; Predicated floating-point addition of a constant, merging with the
 ;; first input.
 (define_insn_and_rewrite "*cond_add<mode>_2_const_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 4)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
 	     UNSPEC_COND_FADD)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -5939,14 +5956,14 @@
 )
 
 (define_insn "*cond_add<mode>_2_const_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
 	     UNSPEC_COND_FADD)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -5962,16 +5979,16 @@
 ;; Predicated floating-point addition of a constant, merging with an
 ;; independent value.
 (define_insn_and_rewrite "*cond_add<mode>_any_const_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 5)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
 	     UNSPEC_COND_FADD)
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[2], operands[4])"
   {@ [ cons: =0 , 1   , 2 , 3   , 4   ]
@@ -6001,16 +6018,16 @@
 )
 
 (define_insn_and_rewrite "*cond_add<mode>_any_const_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_immediate")]
 	     UNSPEC_COND_FADD)
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[2], operands[4])"
   {@ [ cons: =0 , 1   , 2 , 3   , 4   ]
@@ -6208,12 +6225,12 @@
 
 ;; Predicated floating-point subtraction.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F 2 "aarch64_sve_float_arith_operand")
-	   (match_operand:SVE_FULL_F 3 "register_operand")]
+	   (match_operand:SVE_F 2 "aarch64_sve_float_arith_operand")
+	   (match_operand:SVE_F 3 "register_operand")]
 	  SVE_COND_FP_SUB))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , 2   , 3 , 4   ; attrs: movprfx ]
@@ -6229,14 +6246,14 @@
 ;; Predicated floating-point subtraction from a constant, merging with the
 ;; second input.
 (define_insn_and_rewrite "*cond_sub<mode>_3_const_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 4)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "aarch64_sve_float_arith_immediate")
-	      (match_operand:SVE_FULL_F 3 "register_operand")]
+	      (match_operand:SVE_F 2 "aarch64_sve_float_arith_immediate")
+	      (match_operand:SVE_F 3 "register_operand")]
 	     UNSPEC_COND_FSUB)
 	   (match_dup 3)]
 	  UNSPEC_SEL))]
@@ -6252,14 +6269,14 @@
 )
 
 (define_insn "*cond_sub<mode>_3_const_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "aarch64_sve_float_arith_immediate")
-	      (match_operand:SVE_FULL_F 3 "register_operand")]
+	      (match_operand:SVE_F 2 "aarch64_sve_float_arith_immediate")
+	      (match_operand:SVE_F 3 "register_operand")]
 	     UNSPEC_COND_FSUB)
 	   (match_dup 3)]
 	  UNSPEC_SEL))]
@@ -6273,16 +6290,16 @@
 ;; Predicated floating-point subtraction from a constant, merging with an
 ;; independent value.
 (define_insn_and_rewrite "*cond_sub<mode>_const_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 5)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "aarch64_sve_float_arith_immediate")
-	      (match_operand:SVE_FULL_F 3 "register_operand")]
+	      (match_operand:SVE_F 2 "aarch64_sve_float_arith_immediate")
+	      (match_operand:SVE_F 3 "register_operand")]
 	     UNSPEC_COND_FSUB)
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[3], operands[4])"
   {@ [ cons: =0 , 1   , 3 , 4   ]
@@ -6309,16 +6326,16 @@
 )
 
 (define_insn_and_rewrite "*cond_sub<mode>_const_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "aarch64_sve_float_arith_immediate")
-	      (match_operand:SVE_FULL_F 3 "register_operand")]
+	      (match_operand:SVE_F 2 "aarch64_sve_float_arith_immediate")
+	      (match_operand:SVE_F 3 "register_operand")]
 	     UNSPEC_COND_FSUB)
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && !rtx_equal_p (operands[3], operands[4])"
   {@ [ cons: =0 , 1   , 3 , 4   ]
@@ -6631,12 +6648,12 @@
 
 ;; Predicated floating-point multiplication.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F 2 "register_operand")
-	   (match_operand:SVE_FULL_F 3 "aarch64_sve_float_mul_operand")]
+	   (match_operand:SVE_F 2 "register_operand")
+	   (match_operand:SVE_F 3 "aarch64_sve_float_mul_operand")]
 	  SVE_COND_FP_MUL))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , %2 , 3   , 4   ; attrs: movprfx ]
@@ -6671,12 +6688,12 @@
 ;; -------------------------------------------------------------------------
 
 (define_expand "div<mode>3"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_dup 3)
-	   (const_int SVE_RELAXED_GP)
-	   (match_operand:SVE_FULL_F 1 "nonmemory_operand")
-	   (match_operand:SVE_FULL_F 2 "register_operand")]
+	   (match_dup 4)
+	   (match_operand:SVE_F 1 "nonmemory_operand")
+	   (match_operand:SVE_F 2 "register_operand")]
 	  UNSPEC_COND_FDIV))]
   "TARGET_SVE"
   {
@@ -6684,23 +6701,23 @@
       DONE;
 
     operands[1] = force_reg (<MODE>mode, operands[1]);
-    operands[3] = aarch64_ptrue_reg (<VPRED>mode);
+    operands[3] = aarch64_sve_fp_pred (<MODE>mode, &operands[4]);
   }
 )
 
 (define_expand "@aarch64_frecpe<mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:SVE_FULL_F 1 "register_operand")]
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:SVE_F 1 "register_operand")]
 	  UNSPEC_FRECPE))]
   "TARGET_SVE"
 )
 
 (define_expand "@aarch64_frecps<mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:SVE_FULL_F 1 "register_operand")
-	   (match_operand:SVE_FULL_F 2 "register_operand")]
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:SVE_F 1 "register_operand")
+	   (match_operand:SVE_F 2 "register_operand")]
 	  UNSPEC_FRECPS))]
   "TARGET_SVE"
 )
@@ -6865,12 +6882,12 @@
 
 ;; Predicated floating-point maximum/minimum.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F 2 "register_operand")
-	   (match_operand:SVE_FULL_F 3 "aarch64_sve_float_maxmin_operand")]
+	   (match_operand:SVE_F 2 "register_operand")
+	   (match_operand:SVE_F 3 "aarch64_sve_float_maxmin_operand")]
 	  SVE_COND_FP_MAXMIN))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , %2 , 3   ; attrs: movprfx ]
@@ -6899,7 +6916,7 @@
 ;; Predicate AND.  We can reuse one of the inputs as the GP.
 ;; Doubling the second operand is the preferred implementation
 ;; of the MOV alias, so we use that instead of %1/z, %1, %2.
-(define_insn "and<mode>3"
+(define_insn "@and<mode>3"
   [(set (match_operand:PRED_ALL 0 "register_operand")
 	(and:PRED_ALL (match_operand:PRED_ALL 1 "register_operand")
 		      (match_operand:PRED_ALL 2 "register_operand")))]
@@ -7581,29 +7598,29 @@
 
 ;; Unpredicated floating-point ternary operations.
 (define_expand "<optab><mode>4"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_dup 4)
-	   (const_int SVE_RELAXED_GP)
-	   (match_operand:SVE_FULL_F_B16B16 1 "register_operand")
-	   (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	   (match_operand:SVE_FULL_F_B16B16 3 "register_operand")]
+	   (match_dup 5)
+	   (match_operand:SVE_F_B16B16 1 "register_operand")
+	   (match_operand:SVE_F_B16B16 2 "register_operand")
+	   (match_operand:SVE_F_B16B16 3 "register_operand")]
 	  SVE_COND_FP_TERNARY))]
   "TARGET_SVE && (<supports_bf16> || !<is_bf16>)"
   {
-    operands[4] = aarch64_ptrue_reg (<VPRED>mode);
+    operands[4] = aarch64_sve_fp_pred (<MODE>mode, &operands[5]);
   }
 )
 
 ;; Predicated floating-point ternary operations.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 5 "aarch64_sve_gp_strictness")
-	   (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	   (match_operand:SVE_FULL_F_B16B16 3 "register_operand")
-	   (match_operand:SVE_FULL_F_B16B16 4 "register_operand")]
+	   (match_operand:SVE_F_B16B16 2 "register_operand")
+	   (match_operand:SVE_F_B16B16 3 "register_operand")
+	   (match_operand:SVE_F_B16B16 4 "register_operand")]
 	  SVE_COND_FP_TERNARY))]
   "TARGET_SVE && (<supports_bf16> || !<is_bf16>)"
   {@ [ cons: =0 , 1   , %2  , 3 , 4 ; attrs: movprfx , is_rev ]
@@ -7617,17 +7634,17 @@
 
 ;; Predicated floating-point ternary operations with merging.
 (define_expand "@cond_<optab><mode>"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 4 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")
+	      (match_operand:SVE_F_B16B16 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
-	   (match_operand:SVE_FULL_F_B16B16 5 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F_B16B16 5 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE && (<supports_bf16> || !<is_bf16>)"
 {
@@ -7635,20 +7652,22 @@
      second of the two.  */
   if (rtx_equal_p (operands[3], operands[5]))
     std::swap (operands[2], operands[3]);
+
+  operands[1] = aarch64_sve_emit_masked_fp_pred (<MODE>mode, operands[1]);
 })
 
 ;; Predicated floating-point ternary operations, merging with the
 ;; first input.
 (define_insn_and_rewrite "*cond_<optab><mode>_2_relaxed"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+	   (unspec:SVE_F
 	     [(match_operand 5)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "register_operand")
-	      (match_operand:SVE_FULL_F 4 "register_operand")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "register_operand")
+	      (match_operand:SVE_F 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -7664,15 +7683,15 @@
 )
 
 (define_insn "*cond_<optab><mode>_2_strict"
-  [(set (match_operand:SVE_FULL_F 0 "register_operand")
-	(unspec:SVE_FULL_F
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F
+  [(set (match_operand:SVE_F 0 "register_operand")
+	(unspec:SVE_F
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F 2 "register_operand")
-	      (match_operand:SVE_FULL_F 3 "register_operand")
-	      (match_operand:SVE_FULL_F 4 "register_operand")]
+	      (match_operand:SVE_F 2 "register_operand")
+	      (match_operand:SVE_F 3 "register_operand")
+	      (match_operand:SVE_F 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
 	   (match_dup 2)]
 	  UNSPEC_SEL))]
@@ -7686,15 +7705,15 @@
 ;; Predicated floating-point ternary operations, merging with the
 ;; third input.
 (define_insn_and_rewrite "*cond_<optab><mode>_4_relaxed"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_operand 5)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 4 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")
+	      (match_operand:SVE_F_B16B16 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
 	   (match_dup 4)]
 	  UNSPEC_SEL))]
@@ -7710,15 +7729,15 @@
 )
 
 (define_insn "*cond_<optab><mode>_4_strict"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 4 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")
+	      (match_operand:SVE_F_B16B16 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
 	   (match_dup 4)]
 	  UNSPEC_SEL))]
@@ -7732,17 +7751,17 @@
 ;; Predicated floating-point ternary operations, merging with an
 ;; independent value.
 (define_insn_and_rewrite "*cond_<optab><mode>_any_relaxed"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
 	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+	   (unspec:SVE_F_B16B16
 	     [(match_operand 6)
 	      (const_int SVE_RELAXED_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 4 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")
+	      (match_operand:SVE_F_B16B16 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
-	   (match_operand:SVE_FULL_F_B16B16 5 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F_B16B16 5 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE
    && (<supports_bf16> || !<is_bf16>)
@@ -7778,17 +7797,17 @@
 )
 
 (define_insn_and_rewrite "*cond_<optab><mode>_any_strict"
-  [(set (match_operand:SVE_FULL_F_B16B16 0 "register_operand")
-	(unspec:SVE_FULL_F_B16B16
-	  [(match_operand:<VPRED> 1 "register_operand")
-	   (unspec:SVE_FULL_F_B16B16
+  [(set (match_operand:SVE_F_B16B16 0 "register_operand")
+	(unspec:SVE_F_B16B16
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
+	   (unspec:SVE_F_B16B16
 	     [(match_dup 1)
 	      (const_int SVE_STRICT_GP)
-	      (match_operand:SVE_FULL_F_B16B16 2 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 3 "register_operand")
-	      (match_operand:SVE_FULL_F_B16B16 4 "register_operand")]
+	      (match_operand:SVE_F_B16B16 2 "register_operand")
+	      (match_operand:SVE_F_B16B16 3 "register_operand")
+	      (match_operand:SVE_F_B16B16 4 "register_operand")]
 	     SVE_COND_FP_TERNARY)
-	   (match_operand:SVE_FULL_F_B16B16 5 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F_B16B16 5 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
   "TARGET_SVE
    && (<supports_bf16> || !<is_bf16>)
@@ -8187,20 +8206,23 @@
 ;;
 ;; For unpacked vectors, it doesn't really matter whether SEL uses the
 ;; the container size or the element size.  If SEL used the container size,
-;; it would ignore undefined bits of the predicate but would copy the
-;; upper (undefined) bits of each container along with the defined bits.
-;; If SEL used the element size, it would use undefined bits of the predicate
-;; to select between undefined elements in each input vector.  Thus the only
-;; difference is whether the undefined bits in a container always come from
-;; the same input as the defined bits, or whether the choice can vary
-;; independently of the defined bits.
+;; it would would copy the upper (undefined) bits of each container along
+;; with the corresponding defined bits.  If SEL used the element size,
+;; it would use separate predicate bits to select between the undefined
+;; elements in each input vector; these seperate predicate bits might
+;; themselves be undefined, depending on the mode of the predicate.
+;;
+;; Thus the only difference is whether the undefined bits in a container
+;; always come from the same input as the defined bits, or whether the
+;; choice can vary independently of the defined bits.
 ;;
 ;; For the other instructions, using the element size is more natural,
 ;; so we do that for SEL as well.
+;;
 (define_insn "*vcond_mask_<mode><vpred>"
   [(set (match_operand:SVE_ALL 0 "register_operand")
 	(unspec:SVE_ALL
-	  [(match_operand:<VPRED> 3 "register_operand")
+	  [(match_operand:<VPRED> 3 "aarch64_predicate_operand")
 	   (match_operand:SVE_ALL 1 "aarch64_sve_reg_or_dup_imm")
 	   (match_operand:SVE_ALL 2 "aarch64_simd_reg_or_zero")]
 	  UNSPEC_SEL))]
@@ -8637,8 +8659,8 @@
 (define_expand "vec_cmp<mode><vpred>"
   [(set (match_operand:<VPRED> 0 "register_operand")
 	(match_operator:<VPRED> 1 "comparison_operator"
-	  [(match_operand:SVE_FULL_F 2 "register_operand")
-	   (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero")]))]
+	  [(match_operand:SVE_F 2 "register_operand")
+	   (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero")]))]
   "TARGET_SVE"
   {
     aarch64_expand_sve_vec_cmp_float (operands[0], GET_CODE (operands[1]),
@@ -8651,10 +8673,10 @@
 (define_insn "@aarch64_pred_fcm<cmp_op><mode>"
   [(set (match_operand:<VPRED> 0 "register_operand")
 	(unspec:<VPRED>
-	  [(match_operand:<VPRED> 1 "register_operand")
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 2 "aarch64_sve_ptrue_flag")
-	   (match_operand:SVE_FULL_F 3 "register_operand")
-	   (match_operand:SVE_FULL_F 4 "aarch64_simd_reg_or_zero")]
+	   (match_operand:SVE_F 3 "register_operand")
+	   (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")]
 	  SVE_COND_FP_CMP_I0))]
   "TARGET_SVE"
   {@ [ cons: =0 , 1   , 3 , 4   ]
@@ -8667,10 +8689,10 @@
 (define_insn "@aarch64_pred_fcmuo<mode>"
   [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
 	(unspec:<VPRED>
-	  [(match_operand:<VPRED> 1 "register_operand" "Upl")
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand" "Upl")
 	   (match_operand:SI 2 "aarch64_sve_ptrue_flag")
-	   (match_operand:SVE_FULL_F 3 "register_operand" "w")
-	   (match_operand:SVE_FULL_F 4 "register_operand" "w")]
+	   (match_operand:SVE_F 3 "register_operand" "w")
+	   (match_operand:SVE_F 4 "register_operand" "w")]
 	  UNSPEC_COND_FCMUO))]
   "TARGET_SVE"
   "fcmuo\t%0.<Vetype>, %1/z, %3.<Vetype>, %4.<Vetype>"
@@ -8690,8 +8712,8 @@
 	  (unspec:<VPRED>
 	    [(match_operand:<VPRED> 1)
 	     (const_int SVE_KNOWN_PTRUE)
-	     (match_operand:SVE_FULL_F 2 "register_operand" "w, w")
-	     (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero" "Dz, w")]
+	     (match_operand:SVE_F 2 "register_operand" "w, w")
+	     (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w")]
 	    SVE_COND_FP_CMP_I0)
 	  (match_operand:<VPRED> 4 "register_operand" "Upl, Upl")))]
   "TARGET_SVE"
@@ -8713,8 +8735,8 @@
 	  (unspec:<VPRED>
 	    [(match_operand:<VPRED> 1)
 	     (const_int SVE_KNOWN_PTRUE)
-	     (match_operand:SVE_FULL_F 2 "register_operand" "w")
-	     (match_operand:SVE_FULL_F 3 "register_operand" "w")]
+	     (match_operand:SVE_F 2 "register_operand" "w")
+	     (match_operand:SVE_F 3 "register_operand" "w")]
 	    UNSPEC_COND_FCMUO)
 	  (match_operand:<VPRED> 4 "register_operand" "Upl")))]
   "TARGET_SVE"
@@ -8740,8 +8762,8 @@
 	      (unspec:<VPRED>
 	        [(match_operand:<VPRED> 1)
 	         (const_int SVE_KNOWN_PTRUE)
-	         (match_operand:SVE_FULL_F 2 "register_operand" "w")
-	         (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero" "wDz")]
+	         (match_operand:SVE_F 2 "register_operand" "w")
+	         (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "wDz")]
 	        SVE_COND_FP_CMP_I0))
 	    (match_operand:<VPRED> 4 "register_operand" "Upa"))
 	  (match_dup:<VPRED> 1)))
@@ -8777,8 +8799,8 @@
 	      (unspec:<VPRED>
 	        [(match_operand:<VPRED> 1)
 	         (const_int SVE_KNOWN_PTRUE)
-	         (match_operand:SVE_FULL_F 2 "register_operand" "w")
-	         (match_operand:SVE_FULL_F 3 "aarch64_simd_reg_or_zero" "wDz")]
+	         (match_operand:SVE_F 2 "register_operand" "w")
+	         (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "wDz")]
 	        SVE_COND_FP_CMP_I0))
 	    (not:<VPRED>
 	      (match_operand:<VPRED> 4 "register_operand" "Upa")))
@@ -8808,6 +8830,7 @@
 }
 )
 
+;; Same for unordered comparisons.
 (define_insn_and_split "*fcmuo<mode>_bic_combine"
   [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
 	(and:<VPRED>
@@ -8816,8 +8839,8 @@
 	      (unspec:<VPRED>
 	        [(match_operand:<VPRED> 1)
 	         (const_int SVE_KNOWN_PTRUE)
-	         (match_operand:SVE_FULL_F 2 "register_operand" "w")
-	         (match_operand:SVE_FULL_F 3 "register_operand" "w")]
+	         (match_operand:SVE_F 2 "register_operand" "w")
+	         (match_operand:SVE_F 3 "register_operand" "w")]
 	        UNSPEC_COND_FCMUO))
 	    (match_operand:<VPRED> 4 "register_operand" "Upa"))
 	  (match_dup:<VPRED> 1)))
@@ -8843,7 +8866,6 @@
 }
 )
 
-;; Same for unordered comparisons.
 (define_insn_and_split "*fcmuo<mode>_nor_combine"
   [(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
 	(and:<VPRED>
@@ -8852,8 +8874,8 @@
 	      (unspec:<VPRED>
 	        [(match_operand:<VPRED> 1)
 	         (const_int SVE_KNOWN_PTRUE)
-	         (match_operand:SVE_FULL_F 2 "register_operand" "w")
-	         (match_operand:SVE_FULL_F 3 "register_operand" "w")]
+	         (match_operand:SVE_F 2 "register_operand" "w")
+	         (match_operand:SVE_F 3 "register_operand" "w")]
 	        UNSPEC_COND_FCMUO))
 	    (not:<VPRED>
 	      (match_operand:<VPRED> 4 "register_operand" "Upa")))
@@ -9653,6 +9675,31 @@
   }
 )
 
+;; As above, for pairs that are used by the auto-vectorizer only.
+(define_insn_and_rewrite "*cond_<optab>_nontrunc<SVE_PARTIAL_F:mode><SVE_HSDI:mode>_relaxed"
+  [(set (match_operand:SVE_HSDI 0 "register_operand")
+	(unspec:SVE_HSDI
+	  [(match_operand:<SVE_HSDI:VPRED> 1 "register_operand")
+	   (unspec:SVE_HSDI
+	     [(match_operand 4)
+	      (const_int SVE_RELAXED_GP)
+	      (match_operand:SVE_PARTIAL_F 2 "register_operand")]
+	     SVE_COND_FCVTI)
+	   (match_operand:SVE_HSDI 3 "aarch64_simd_reg_or_zero")]
+	  UNSPEC_SEL))]
+  "TARGET_SVE
+  && (~(<SVE_HSDI:self_mask> | <SVE_HSDI:narrower_mask>) & <SVE_PARTIAL_F:self_mask>) == 0"
+  {@ [ cons: =0 , 1   , 2 , 3  ; attrs: movprfx ]
+     [ &w       , Upl , w , 0  ; *              ] fcvtz<su>\t%0.<SVE_HSDI:Vetype>, %1/m, %2.<SVE_PARTIAL_F:Vetype>
+     [ &w       , Upl , w , Dz ; yes            ] movprfx\t%0.<SVE_HSDI:Vetype>, %1/z, %2.<SVE_HSDI:Vetype>\;fcvtz<su>\t%0.<SVE_HSDI:Vetype>, %1/m, %2.<SVE_PARTIAL_F:Vetype>
+     [ ?&w      , Upl , w , w  ; yes            ] movprfx\t%0, %3\;fcvtz<su>\t%0.<SVE_HSDI:Vetype>, %1/m, %2.<SVE_PARTIAL_F:Vetype>
+  }
+  "&& !rtx_equal_p (operands[1], operands[4])"
+  {
+    operands[4] = copy_rtx (operands[1]);
+  }
+)
+
 (define_insn "*cond_<optab>_nontrunc<SVE_FULL_F:mode><SVE_FULL_HSDI:mode>_strict"
   [(set (match_operand:SVE_FULL_HSDI 0 "register_operand")
 	(unspec:SVE_FULL_HSDI
@@ -9706,6 +9753,29 @@
   }
 )
 
+(define_insn_and_rewrite "*cond_<optab>_trunc<VNx2DF_ONLY:mode><VNx2SI_ONLY:mode>_relaxed"
+  [(set (match_operand:VNx2SI_ONLY 0 "register_operand")
+	(unspec:VNx2SI_ONLY
+	  [(match_operand:VNx2BI 1 "register_operand")
+	   (unspec:VNx2SI_ONLY
+	     [(match_operand 4)
+	      (const_int SVE_RELAXED_GP)
+	      (match_operand:VNx2DF_ONLY 2 "register_operand")]
+	     SVE_COND_FCVTI)
+	   (match_operand:VNx2SI_ONLY 3 "aarch64_simd_reg_or_zero")]
+	  UNSPEC_SEL))]
+  "TARGET_SVE"
+  {@ [ cons: =0 , 1   , 2 , 3  ; attrs: movprfx ]
+     [ &w       , Upl , w , 0  ; *              ] fcvtz<su>\t%0.<VNx2SI_ONLY:Vetype>, %1/m, %2.<VNx2DF_ONLY:Vetype>
+     [ &w       , Upl , w , Dz ; yes            ] movprfx\t%0.<VNx2DF_ONLY:Vetype>, %1/z, %2.<VNx2DF_ONLY:Vetype>\;fcvtz<su>\t%0.<VNx2SI_ONLY:Vetype>, %1/m, %2.<VNx2DF_ONLY:Vetype>
+     [ ?&w      , Upl , w , w  ; yes            ] movprfx\t%0, %3\;fcvtz<su>\t%0.<VNx2SI_ONLY:Vetype>, %1/m, %2.<VNx2DF_ONLY:Vetype>
+  }
+  "&& !rtx_equal_p (operands[1], operands[4])"
+  {
+    operands[4] = copy_rtx (operands[1]);
+  }
+)
+
 ;; -------------------------------------------------------------------------
 ;; ---- [INT<-FP] Packs
 ;; -------------------------------------------------------------------------
@@ -9857,6 +9927,31 @@
   }
 )
 
+;; As above, for pairs that are used by the auto-vectorizer only.
+(define_insn_and_rewrite "*cond_<optab>_nonextend<SVE_HSDI:mode><SVE_PARTIAL_F:mode>_relaxed"
+  [(set (match_operand:SVE_PARTIAL_F 0 "register_operand")
+	(unspec:SVE_PARTIAL_F
+	  [(match_operand:<SVE_HSDI:VPRED> 1 "register_operand")
+	   (unspec:SVE_PARTIAL_F
+	     [(match_operand 4)
+	      (const_int SVE_RELAXED_GP)
+	      (match_operand:SVE_HSDI 2 "register_operand")]
+	     SVE_COND_ICVTF)
+	   (match_operand:SVE_PARTIAL_F 3 "aarch64_simd_reg_or_zero")]
+	  UNSPEC_SEL))]
+  "TARGET_SVE
+   && (~(<SVE_HSDI:self_mask> | <SVE_HSDI:narrower_mask>) & <SVE_PARTIAL_F:self_mask>) == 0"
+  {@ [ cons: =0 , 1   , 2 , 3  ; attrs: movprfx ]
+     [ &w       , Upl , w , 0  ; *              ] <su>cvtf\t%0.<SVE_PARTIAL_F:Vetype>, %1/m, %2.<SVE_HSDI:Vetype>
+     [ &w       , Upl , w , Dz ; yes            ] movprfx\t%0.<SVE_HSDI:Vetype>, %1/z, %2.<SVE_HSDI:Vetype>\;<su>cvtf\t%0.<SVE_PARTIAL_F:Vetype>, %1/m, %2.<SVE_HSDI:Vetype>
+     [ ?&w      , Upl , w , w  ; yes            ] movprfx\t%0, %3\;<su>cvtf\t%0.<SVE_PARTIAL_F:Vetype>, %1/m, %2.<SVE_HSDI:Vetype>
+  }
+  "&& !rtx_equal_p (operands[1], operands[4])"
+  {
+    operands[4] = copy_rtx (operands[1]);
+  }
+)
+
 (define_insn "*cond_<optab>_nonextend<SVE_FULL_HSDI:mode><SVE_FULL_F:mode>_strict"
   [(set (match_operand:SVE_FULL_F 0 "register_operand")
 	(unspec:SVE_FULL_F
@@ -10066,6 +10161,30 @@
   }
 )
 
+;; As above, for pairs that are used by the auto-vectorizer only.
+(define_insn_and_rewrite "*cond_<optab>_trunc<SVE_SDF:mode><SVE_PARTIAL_HSF:mode>"
+  [(set (match_operand:SVE_PARTIAL_HSF 0 "register_operand")
+	(unspec:SVE_PARTIAL_HSF
+	  [(match_operand:<SVE_SDF:VPRED> 1 "register_operand")
+	   (unspec:SVE_PARTIAL_HSF
+	     [(match_operand 4)
+	      (const_int SVE_RELAXED_GP)
+	      (match_operand:SVE_SDF 2 "register_operand")]
+	     SVE_COND_FCVT)
+	   (match_operand:SVE_PARTIAL_HSF 3 "aarch64_simd_reg_or_zero")]
+	  UNSPEC_SEL))]
+  "TARGET_SVE && (~<SVE_SDF:narrower_mask> & <SVE_PARTIAL_HSF:self_mask>) == 0"
+  {@ [ cons: =0 , 1   , 2 , 3  ; attrs: movprfx ]
+     [ w        , Upl , w , 0  ; *              ] fcvt\t%0.<SVE_PARTIAL_HSF:Vetype>, %1/m, %2.<SVE_SDF:Vetype>
+     [ ?&w      , Upl , w , Dz ; yes            ] movprfx\t%0.<SVE_SDF:Vetype>, %1/z, %2.<SVE_SDF:Vetype>\;fcvt\t%0.<SVE_PARTIAL_HSF:Vetype>, %1/m, %2.<SVE_SDF:Vetype>
+     [ ?&w      , Upl , w , w  ; yes            ] movprfx\t%0, %3\;fcvt\t%0.<SVE_PARTIAL_HSF:Vetype>, %1/m, %2.<SVE_SDF:Vetype>
+  }
+  "&& !rtx_equal_p (operands[1], operands[4])"
+  {
+    operands[4] = copy_rtx (operands[1]);
+  }
+)
+
 ;; -------------------------------------------------------------------------
 ;; ---- [FP<-FP] Packs (bfloat16)
 ;; -------------------------------------------------------------------------
@@ -10259,6 +10378,30 @@
   }
 )
 
+;; As above, for pairs that are used by the auto-vectorizer only.
+(define_insn_and_rewrite "*cond_<optab>_nontrunc<SVE_PARTIAL_HSF:mode><SVE_SDF:mode>_relaxed"
+  [(set (match_operand:SVE_SDF 0 "register_operand")
+	(unspec:SVE_SDF
+	  [(match_operand:<SVE_SDF:VPRED> 1 "register_operand")
+	   (unspec:SVE_SDF
+	     [(match_operand 4)
+	      (const_int SVE_RELAXED_GP)
+	      (match_operand:SVE_PARTIAL_HSF 2 "register_operand")]
+	     SVE_COND_FCVT)
+	   (match_operand:SVE_SDF 3 "aarch64_simd_reg_or_zero")]
+	  UNSPEC_SEL))]
+  "TARGET_SVE && (~<SVE_SDF:narrower_mask> & <SVE_PARTIAL_HSF:self_mask>) == 0"
+  {@ [ cons: =0 , 1   , 2 , 3  ; attrs: movprfx ]
+     [ w        , Upl , w , 0  ; *              ] fcvt\t%0.<SVE_SDF:Vetype>, %1/m, %2.<SVE_PARTIAL_HSF:Vetype>
+     [ ?&w      , Upl , w , Dz ; yes            ] movprfx\t%0.<SVE_SDF:Vetype>, %1/z, %2.<SVE_SDF:Vetype>\;fcvt\t%0.<SVE_SDF:Vetype>, %1/m, %2.<SVE_PARTIAL_HSF:Vetype>
+     [ ?&w      , Upl , w , w  ; yes            ] movprfx\t%0, %3\;fcvt\t%0.<SVE_SDF:Vetype>, %1/m, %2.<SVE_PARTIAL_HSF:Vetype>
+  }
+  "&& !rtx_equal_p (operands[1], operands[4])"
+  {
+    operands[4] = copy_rtx (operands[1]);
+  }
+)
+
 ;; -------------------------------------------------------------------------
 ;; ---- [PRED<-PRED] Packs
 ;; -------------------------------------------------------------------------
@@ -11437,16 +11580,12 @@
 
 (define_insn "@aarch64_sve_set_neonq_<mode>"
   [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
-      (unspec:SVE_FULL
-	[(match_operand:SVE_FULL 1 "register_operand" "w")
-	(match_operand:<V128> 2 "register_operand" "w")
-	(match_operand:<VPRED> 3 "register_operand" "Upl")]
-	UNSPEC_SET_NEONQ))]
+	(unspec:SVE_FULL
+	  [(match_operand:SVE_FULL 1 "register_operand" "w")
+	   (match_operand:<V128> 2 "register_operand" "w")
+	   (match_operand:<VPRED> 3 "register_operand" "Upl")]
+	  UNSPEC_SET_NEONQ))]
   "TARGET_SVE
    && BYTES_BIG_ENDIAN"
-  {
-    operands[2] = lowpart_subreg (<MODE>mode, operands[2],
-                                  GET_MODE (operands[2]));
-    return "sel\t%0.<Vetype>, %3, %2.<Vetype>, %1.<Vetype>";
-  }
+  "sel\t%0.<Vetype>, %3, %Z2.<Vetype>, %1.<Vetype>"
 )
diff --git a/gcc/config/aarch64/aarch64-sve2.md b/gcc/config/aarch64/aarch64-sve2.md
index 62524f3..31bdd85 100644
--- a/gcc/config/aarch64/aarch64-sve2.md
+++ b/gcc/config/aarch64/aarch64-sve2.md
@@ -334,12 +334,21 @@
 ;; - LD1Q (SVE2p1)
 ;; -------------------------------------------------------------------------
 
-;; Model this as operating on the largest valid element size, which is DI.
-;; This avoids having to define move patterns & more for VNx1TI, which would
-;; be difficult without a non-gather form of LD1Q.
-(define_insn "aarch64_gather_ld1q"
-  [(set (match_operand:VNx2DI 0 "register_operand")
-	(unspec:VNx2DI
+;; For little-endian targets, it would be enough to use a single pattern,
+;; with a subreg to bitcast the result to whatever mode is needed.
+;; However, on big-endian targets, the bitcast would need to be an
+;; aarch64_sve_reinterpret instruction.  That would interact badly
+;; with the "&" and "?" constraints in this pattern: if the result
+;; of the reinterpret needs to be in the same register as the index,
+;; the RA would tend to prefer to allocate a separate register for the
+;; intermediate (uncast) result, even if the reinterpret prefers tying.
+;;
+;; The index is logically VNx1DI rather than VNx2DI, but introducing
+;; and using VNx1DI would just create more bitcasting.  The ACLE intrinsic
+;; uses svuint64_t, which corresponds to VNx2DI.
+(define_insn "@aarch64_gather_ld1q<mode>"
+  [(set (match_operand:SVE_FULL 0 "register_operand")
+	(unspec:SVE_FULL
 	  [(match_operand:VNx2BI 1 "register_operand")
 	   (match_operand:DI 2 "aarch64_reg_or_zero")
 	   (match_operand:VNx2DI 3 "register_operand")
@@ -1337,12 +1346,12 @@
 
 ;; Predicated B16B16 binary operations.
 (define_insn "@aarch64_pred_<optab><mode>"
-  [(set (match_operand:VNx8BF_ONLY 0 "register_operand")
-	(unspec:VNx8BF_ONLY
-	  [(match_operand:<VPRED> 1 "register_operand")
+  [(set (match_operand:SVE_BF 0 "register_operand")
+	(unspec:SVE_BF
+	  [(match_operand:<VPRED> 1 "aarch64_predicate_operand")
 	   (match_operand:SI 4 "aarch64_sve_gp_strictness")
-	   (match_operand:VNx8BF_ONLY 2 "register_operand")
-	   (match_operand:VNx8BF_ONLY 3 "register_operand")]
+	   (match_operand:SVE_BF 2 "register_operand")
+	   (match_operand:SVE_BF 3 "register_operand")]
 	  SVE_COND_FP_BINARY_OPTAB))]
   "TARGET_SSVE_B16B16 && <supports_bf16>"
   {@ [ cons: =0 , 1   , 2 , 3 ; attrs: movprfx , is_rev ]
@@ -1628,7 +1637,7 @@
   "TARGET_SVE2"
   {@ [ cons: =0 , %1 , 2 ; attrs: movprfx ]
      [ w        , 0  , w ; *              ] nbsl\t%0.d, %0.d, %2.d, %0.d
-     [ ?&w      , w  , w ; yes            ] movprfx\t%0, %1\;nbsl\t%0.d, %0.d, %2.d, %0.d
+     [ ?&w      , w  , w ; yes            ] movprfx\t%0, %1\;nbsl\t%0.d, %0.d, %2.d, %1.d
   }
   "&& !CONSTANT_P (operands[3])"
   {
@@ -1636,6 +1645,20 @@
   }
 )
 
+(define_insn "*aarch64_sve2_unpred_nor<mode>"
+  [(set (match_operand:VDQ_I 0 "register_operand")
+	(and:VDQ_I
+	  (not:VDQ_I
+	    (match_operand:VDQ_I 1 "register_operand"))
+	  (not:VDQ_I
+	    (match_operand:VDQ_I 2 "register_operand"))))]
+  "TARGET_SVE2"
+  {@ [ cons: =0 , %1 , 2 ; attrs: movprfx ]
+     [ w        , 0  , w ; *              ] nbsl\t%Z0.d, %Z0.d, %Z2.d, %Z0.d
+     [ ?&w      , w  , w ; yes            ] movprfx\t%Z0, %Z1\;nbsl\t%Z0.d, %Z0.d, %Z2.d, %Z1.d
+  }
+)
+
 ;; Use NBSL for vector NAND.
 (define_insn_and_rewrite "*aarch64_sve2_nand<mode>"
   [(set (match_operand:SVE_FULL_I 0 "register_operand")
@@ -1658,6 +1681,21 @@
   }
 )
 
+;; Same as above but unpredicated and including Advanced SIMD modes.
+(define_insn "*aarch64_sve2_nand_unpred<mode>"
+  [(set (match_operand:VDQ_I 0 "register_operand")
+	(ior:VDQ_I
+	  (not:VDQ_I
+	    (match_operand:VDQ_I 1 "register_operand"))
+	  (not:VDQ_I
+	    (match_operand:VDQ_I 2 "register_operand"))))]
+  "TARGET_SVE2"
+  {@ [ cons: =0 , %1 , 2 ; attrs: movprfx ]
+     [ w        , 0  , w ; *              ] nbsl\t%Z0.d, %Z0.d, %Z2.d, %Z2.d
+     [ ?&w      , w  , w ; yes            ] movprfx\t%Z0, %Z1\;nbsl\t%Z0.d, %Z0.d, %Z2.d, %Z2.d
+  }
+)
+
 ;; Unpredicated bitwise select.
 ;; (op3 ? bsl_mov : bsl_dup) == (((bsl_mov ^ bsl_dup) & op3) ^ bsl_dup)
 (define_expand "@aarch64_sve2_bsl<mode>"
@@ -1918,6 +1956,40 @@
   }
 )
 
+;; Vector EON (~(x, y)) using BSL2N.
+(define_insn_and_rewrite "*aarch64_sve2_bsl2n_eon<mode>"
+  [(set (match_operand:SVE_FULL_I 0 "register_operand")
+	(unspec:SVE_FULL_I
+	  [(match_operand 3)
+	   (not:SVE_FULL_I
+	     (xor:SVE_FULL_I
+		(match_operand:SVE_FULL_I 1 "register_operand")
+		(match_operand:SVE_FULL_I 2 "register_operand")))]
+	    UNSPEC_PRED_X))]
+  "TARGET_SVE2"
+  {@ [ cons: =0, 1, 2 ; attrs: movprfx ]
+     [ w  ,      0, w ; *              ] bsl2n\t%0.d, %0.d, %0.d, %2.d
+     [ ?&w,      w, w ; yes            ] movprfx\t%0, %1\;bsl2n\t%0.d, %0.d, %1.d, %2.d
+  }
+  "&& !CONSTANT_P (operands[3])"
+  {
+    operands[3] = CONSTM1_RTX (<VPRED>mode);
+  }
+)
+
+(define_insn "*aarch64_sve2_eon_bsl2n_unpred<mode>"
+  [(set (match_operand:VDQ_I 0 "register_operand")
+       (not:VDQ_I
+         (xor:VDQ_I
+           (match_operand:VDQ_I 1 "register_operand")
+           (match_operand:VDQ_I 2 "register_operand"))))]
+  "TARGET_SVE2"
+  {@ [ cons: =0, 1, 2 ; attrs: movprfx ]
+     [ w  ,      0, w ; *              ] bsl2n\t%Z0.d, %Z0.d, %Z0.d, %Z2.d
+     [ ?&w,      w, w ; yes            ] movprfx\t%Z0, %Z1\;bsl2n\t%Z0.d, %Z0.d, %Z1.d, %Z2.d
+  }
+)
+
 ;; -------------------------------------------------------------------------
 ;; ---- [INT] Shift-and-accumulate operations
 ;; -------------------------------------------------------------------------
diff --git a/gcc/config/aarch64/aarch64-tuning-flags.def b/gcc/config/aarch64/aarch64-tuning-flags.def
index f2c916e..dd91324 100644
--- a/gcc/config/aarch64/aarch64-tuning-flags.def
+++ b/gcc/config/aarch64/aarch64-tuning-flags.def
@@ -44,6 +44,8 @@ AARCH64_EXTRA_TUNING_OPTION ("avoid_cross_loop_fma", AVOID_CROSS_LOOP_FMA)
 
 AARCH64_EXTRA_TUNING_OPTION ("fully_pipelined_fma", FULLY_PIPELINED_FMA)
 
+AARCH64_EXTRA_TUNING_OPTION ("avoid_ldapur", AVOID_LDAPUR)
+
 /* Enable is the target prefers to use a fresh register for predicate outputs
    rather than re-use an input predicate register.  */
 AARCH64_EXTRA_TUNING_OPTION ("avoid_pred_rmw", AVOID_PRED_RMW)
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index f3ce3a1..f4a2062 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -356,7 +356,8 @@ static int aarch64_address_cost (rtx, machine_mode, addr_space_t, bool);
 static bool aarch64_builtin_support_vector_misalignment (machine_mode mode,
 							 const_tree type,
 							 int misalignment,
-							 bool is_packed);
+							 bool is_packed,
+							 bool is_gather_scatter);
 static machine_mode aarch64_simd_container_mode (scalar_mode, poly_int64);
 static bool aarch64_print_address_internal (FILE*, machine_mode, rtx,
 					    aarch64_addr_query_type);
@@ -429,6 +430,7 @@ static const struct aarch64_flag_desc aarch64_tuning_flags[] =
 #include "tuning_models/neoversev2.h"
 #include "tuning_models/neoversev3.h"
 #include "tuning_models/neoversev3ae.h"
+#include "tuning_models/olympus.h"
 #include "tuning_models/a64fx.h"
 #include "tuning_models/fujitsu_monaka.h"
 
@@ -3931,6 +3933,33 @@ aarch64_sve_fp_pred (machine_mode data_mode, rtx *strictness)
    return aarch64_ptrue_reg (aarch64_sve_pred_mode (data_mode));
 }
 
+/* PRED is a predicate that governs an operation on DATA_MODE.  If DATA_MODE
+   is a partial vector mode, and if exceptions must be suppressed for its
+   undefined elements, convert PRED from a container-level predicate to
+   an element-level predicate and ensure that the undefined elements
+   are inactive.  Make no changes otherwise.
+
+   Return the resultant predicate.  */
+rtx
+aarch64_sve_emit_masked_fp_pred (machine_mode data_mode, rtx pred)
+{
+  unsigned int vec_flags = aarch64_classify_vector_mode (data_mode);
+  if (flag_trapping_math && (vec_flags & VEC_PARTIAL))
+    {
+      /* Generate an element-level mask.  */
+      rtx mask = aarch64_sve_packed_pred (data_mode);
+      machine_mode pmode = GET_MODE (mask);
+
+      /* Apply the existing predicate.  */
+      rtx dst = gen_reg_rtx (pmode);
+      emit_insn (gen_and3 (pmode, dst, mask,
+			   gen_lowpart (pmode, pred)));
+      return dst;
+    }
+
+  return pred;
+}
+
 /* Emit a comparison CMP between OP0 and OP1, both of which have mode
    DATA_MODE, and return the result in a predicate of mode PRED_MODE.
    Use TARGET as the target register if nonnull and convenient.  */
@@ -15854,11 +15883,14 @@ cost_plus:
 	break;
     case CONST_VECTOR:
 	{
-	  /* Load using MOVI/MVNI.  */
-	  if (aarch64_simd_valid_mov_imm (x))
-	    *cost = extra_cost->vect.movi;
-	  else /* Load using constant pool.  */
-	    *cost = extra_cost->ldst.load;
+	  if (speed)
+	    {
+	      /* Load using MOVI/MVNI.  */
+	      if (aarch64_simd_valid_mov_imm (x))
+		*cost += extra_cost->vect.movi;
+	      else /* Load using constant pool.  */
+		*cost += extra_cost->ldst.load;
+	    }
 	  break;
 	}
     case VEC_CONCAT:
@@ -15867,7 +15899,8 @@ cost_plus:
 	break;
     case VEC_DUPLICATE:
 	/* Load using a DUP.  */
-	*cost = extra_cost->vect.dup;
+	if (speed)
+	  *cost += extra_cost->vect.dup;
 	return false;
     case VEC_SELECT:
 	{
@@ -15875,13 +15908,16 @@ cost_plus:
 	  *cost = rtx_cost (op0, GET_MODE (op0), VEC_SELECT, 0, speed);
 
 	  /* cost subreg of 0 as free, otherwise as DUP */
-	  rtx op1 = XEXP (x, 1);
-	  if (vec_series_lowpart_p (mode, GET_MODE (op1), op1))
-	    ;
-	  else if (vec_series_highpart_p (mode, GET_MODE (op1), op1))
-	    *cost = extra_cost->vect.dup;
-	  else
-	    *cost = extra_cost->vect.extract;
+	  if (speed)
+	    {
+	      rtx op1 = XEXP (x, 1);
+	      if (vec_series_lowpart_p (mode, GET_MODE (op1), op1))
+		;
+	      else if (vec_series_highpart_p (mode, GET_MODE (op1), op1))
+		*cost += extra_cost->vect.dup;
+	      else
+		*cost += extra_cost->vect.extract;
+	    }
 	  return true;
 	}
     default:
@@ -17157,8 +17193,8 @@ aarch64_ld234_st234_vectors (vect_cost_for_stmt kind, stmt_vec_info stmt_info,
       && STMT_VINFO_DATA_REF (stmt_info))
     {
       stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
-      if (stmt_info
-	  && vect_mem_access_type (stmt_info, node) == VMAT_LOAD_STORE_LANES)
+      if (node
+	  && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_LOAD_STORE_LANES)
 	return DR_GROUP_SIZE (stmt_info);
     }
   return 0;
@@ -17429,8 +17465,9 @@ aarch64_detect_vector_stmt_subtype (vec_info *vinfo, vect_cost_for_stmt kind,
      for each element.  We therefore need to divide the full-instruction
      cost by the number of elements in the vector.  */
   if (kind == scalar_load
+      && node
       && sve_costs
-      && vect_mem_access_type (stmt_info, node) == VMAT_GATHER_SCATTER)
+      && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_GATHER_SCATTER)
     {
       unsigned int nunits = vect_nunits_for_cost (vectype);
       /* Test for VNx2 modes, which have 64-bit containers.  */
@@ -17442,8 +17479,9 @@ aarch64_detect_vector_stmt_subtype (vec_info *vinfo, vect_cost_for_stmt kind,
   /* Detect cases in which a scalar_store is really storing one element
      in a scatter operation.  */
   if (kind == scalar_store
+      && node
       && sve_costs
-      && vect_mem_access_type (stmt_info, node) == VMAT_GATHER_SCATTER)
+      && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_GATHER_SCATTER)
     return sve_costs->scatter_store_elt_cost;
 
   /* Detect cases in which vec_to_scalar represents an in-loop reduction.  */
@@ -17699,7 +17737,7 @@ aarch64_vector_costs::count_ops (unsigned int count, vect_cost_for_stmt kind,
   if (stmt_info
       && kind == vec_to_scalar
       && (m_vec_flags & VEC_ADVSIMD)
-      && vect_mem_access_type (stmt_info, node) == VMAT_GATHER_SCATTER)
+      && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_GATHER_SCATTER)
     {
       auto dr = STMT_VINFO_DATA_REF (stmt_info);
       tree dr_ref = DR_REF (dr);
@@ -17712,7 +17750,7 @@ aarch64_vector_costs::count_ops (unsigned int count, vect_cost_for_stmt kind,
 		{
 		  if (gimple_vuse (SSA_NAME_DEF_STMT (offset)))
 		    {
-		      if (STMT_VINFO_TYPE (stmt_info) == load_vec_info_type)
+		      if (SLP_TREE_TYPE (node) == load_vec_info_type)
 			ops->loads += count - 1;
 		      else
 			  /* Stores want to count both the index to array and data to
@@ -17814,7 +17852,7 @@ aarch64_vector_costs::count_ops (unsigned int count, vect_cost_for_stmt kind,
   if (stmt_info
       && sve_issue
       && (kind == scalar_load || kind == scalar_store)
-      && vect_mem_access_type (stmt_info, node) == VMAT_GATHER_SCATTER)
+      && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_GATHER_SCATTER)
     {
       unsigned int pairs = CEIL (count, 2);
       ops->pred_ops += sve_issue->gather_scatter_pair_pred_ops * pairs;
@@ -17932,7 +17970,7 @@ aarch64_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
 
   /* Do one-time initialization based on the vinfo.  */
   loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo);
-  if (!m_analyzed_vinfo)
+  if (!m_analyzed_vinfo && !m_costing_for_scalar)
     {
       if (loop_vinfo)
 	analyze_loop_vinfo (loop_vinfo);
@@ -17969,8 +18007,10 @@ aarch64_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
 
       /* Check if we've seen an SVE gather/scatter operation and which size.  */
       if (kind == scalar_load
+	  && node
+	  && vectype
 	  && aarch64_sve_mode_p (TYPE_MODE (vectype))
-	  && vect_mem_access_type (stmt_info, node) == VMAT_GATHER_SCATTER)
+	  && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_GATHER_SCATTER)
 	{
 	  const sve_vec_cost *sve_costs = aarch64_tune_params.vec_costs->sve;
 	  if (sve_costs)
@@ -18793,6 +18833,8 @@ aarch64_adjust_generic_arch_tuning (struct tune_params &current_tune)
   if (TARGET_SVE2)
     current_tune.extra_tuning_flags
       &= ~AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS;
+  if (!AARCH64_HAVE_ISA(V8_8A))
+    aarch64_tune_params.extra_tuning_flags |= AARCH64_EXTRA_TUNE_AVOID_LDAPUR;
 }
 
 static void
@@ -18857,7 +18899,10 @@ aarch64_override_options_internal (struct gcc_options *opts)
   /* Make a copy of the tuning parameters attached to the core, which
      we may later overwrite.  */
   aarch64_tune_params = *(tune->tune);
-  if (tune->tune == &generic_tunings)
+
+  if (tune->tune == &generic_tunings
+      || tune->tune == &generic_armv8_a_tunings
+      || tune->tune == &generic_armv9_a_tunings)
     aarch64_adjust_generic_arch_tuning (aarch64_tune_params);
 
   if (opts->x_aarch64_override_tune_string)
@@ -19953,8 +19998,9 @@ aarch64_process_one_target_attr (char *arg_str)
 	      if (valid)
 		{
 		  set_option (&global_options, NULL, p_attr->opt_num, value,
-			      NULL, DK_UNSPECIFIED, input_location,
-			      global_dc);
+			      NULL,
+			      static_cast<int> (diagnostics::kind::unspecified),
+			      input_location, global_dc);
 		}
 	      else
 		{
@@ -20466,6 +20512,8 @@ aarch64_compare_version_priority (tree decl1, tree decl2)
      unsigned long _size; // Size of the struct, so it can grow.
      unsigned long _hwcap;
      unsigned long _hwcap2;
+     unsigned long _hwcap3;
+     unsigned long _hwcap4;
    }
  */
 
@@ -20482,14 +20530,24 @@ build_ifunc_arg_type ()
   tree field3 = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
 			    get_identifier ("_hwcap2"),
 			    long_unsigned_type_node);
+  tree field4 = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
+			    get_identifier ("_hwcap3"),
+			    long_unsigned_type_node);
+  tree field5 = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
+			    get_identifier ("_hwcap4"),
+			    long_unsigned_type_node);
 
   DECL_FIELD_CONTEXT (field1) = ifunc_arg_type;
   DECL_FIELD_CONTEXT (field2) = ifunc_arg_type;
   DECL_FIELD_CONTEXT (field3) = ifunc_arg_type;
+  DECL_FIELD_CONTEXT (field4) = ifunc_arg_type;
+  DECL_FIELD_CONTEXT (field5) = ifunc_arg_type;
 
   TYPE_FIELDS (ifunc_arg_type) = field1;
   DECL_CHAIN (field1) = field2;
   DECL_CHAIN (field2) = field3;
+  DECL_CHAIN (field3) = field4;
+  DECL_CHAIN (field4) = field5;
 
   layout_type (ifunc_arg_type);
 
@@ -23074,6 +23132,58 @@ aarch64_sve_index_immediate_p (rtx base_or_step)
 	  && IN_RANGE (INTVAL (base_or_step), -16, 15));
 }
 
+/* Return true if SERIES is a constant vector that can be loaded using
+   an immediate SVE INDEX, considering both SVE and Advanced SIMD modes.
+   When returning true, store the base in *BASE_OUT and the step
+   in *STEP_OUT.  */
+
+static bool
+aarch64_sve_index_series_p (rtx series, rtx *base_out, rtx *step_out)
+{
+  rtx base, step;
+  if (!const_vec_series_p (series, &base, &step)
+      || !CONST_INT_P (base)
+      || !CONST_INT_P (step))
+    return false;
+
+  auto mode = GET_MODE (series);
+  auto elt_mode = as_a<scalar_int_mode> (GET_MODE_INNER (mode));
+  unsigned int vec_flags = aarch64_classify_vector_mode (mode);
+  if (BYTES_BIG_ENDIAN && (vec_flags & VEC_ADVSIMD))
+    {
+      /* On big-endian targets, architectural lane 0 holds the last element
+	 for Advanced SIMD and the first element for SVE; see the comment at
+	 the head of aarch64-sve.md for details.  This means that, from an SVE
+	 point of view, an Advanced SIMD series goes from the last element to
+	 the first.  */
+      auto i = GET_MODE_NUNITS (mode).to_constant () - 1;
+      base = gen_int_mode (UINTVAL (base) + i * UINTVAL (step), elt_mode);
+      step = gen_int_mode (-UINTVAL (step), elt_mode);
+    }
+
+  if (!aarch64_sve_index_immediate_p (base)
+      || !aarch64_sve_index_immediate_p (step))
+    return false;
+
+  /* If the mode spans multiple registers, check that each subseries is
+     in range.  */
+  unsigned int nvectors = aarch64_ldn_stn_vectors (mode);
+  if (nvectors != 1)
+    {
+      unsigned int nunits;
+      if (!GET_MODE_NUNITS (mode).is_constant (&nunits))
+	return false;
+      nunits /= nvectors;
+      for (unsigned int i = 1; i < nvectors; ++i)
+	if (!IN_RANGE (INTVAL (base) + i * nunits * INTVAL (step), -16, 15))
+	  return false;
+    }
+
+  *base_out = base;
+  *step_out = step;
+  return true;
+}
+
 /* Return true if X is a valid immediate for the SVE ADD and SUB instructions
    when applied to mode MODE.  Negate X first if NEGATE_P is true.  */
 
@@ -23522,13 +23632,8 @@ aarch64_simd_valid_imm (rtx op, simd_immediate_info *info,
     n_elts = CONST_VECTOR_NPATTERNS (op);
   else if (which == AARCH64_CHECK_MOV
 	   && TARGET_SVE
-	   && const_vec_series_p (op, &base, &step))
+	   && aarch64_sve_index_series_p (op, &base, &step))
     {
-      gcc_assert (GET_MODE_CLASS (mode) == MODE_VECTOR_INT);
-      if (!aarch64_sve_index_immediate_p (base)
-	  || !aarch64_sve_index_immediate_p (step))
-	return false;
-
       if (info)
 	{
 	  /* Get the corresponding container mode.  E.g. an INDEX on V2SI
@@ -23640,6 +23745,8 @@ aarch64_simd_valid_imm (rtx op, simd_immediate_info *info,
       long int as_long_ints[2];
       as_long_ints[0] = ival & 0xFFFFFFFF;
       as_long_ints[1] = (ival >> 32) & 0xFFFFFFFF;
+      if (imode == DImode && FLOAT_WORDS_BIG_ENDIAN)
+	std::swap (as_long_ints[0], as_long_ints[1]);
 
       REAL_VALUE_TYPE r;
       real_from_target (&r, as_long_ints, fmode);
@@ -24352,10 +24459,14 @@ aarch64_simd_vector_alignment_reachable (const_tree type, bool is_packed)
 static bool
 aarch64_builtin_support_vector_misalignment (machine_mode mode,
 					     const_tree type, int misalignment,
-					     bool is_packed)
+					     bool is_packed,
+					     bool is_gather_scatter)
 {
   if (TARGET_SIMD && STRICT_ALIGNMENT)
     {
+      if (is_gather_scatter)
+	return true;
+
       /* Return if movmisalign pattern is not supported for this mode.  */
       if (optab_handler (movmisalign_optab, mode) == CODE_FOR_nothing)
         return false;
@@ -24365,7 +24476,8 @@ aarch64_builtin_support_vector_misalignment (machine_mode mode,
 	return false;
     }
   return default_builtin_support_vector_misalignment (mode, type, misalignment,
-						      is_packed);
+						      is_packed,
+						      is_gather_scatter);
 }
 
 /* If VALS is a vector constant that can be loaded into a register
@@ -24830,6 +24942,13 @@ aarch64_expand_vector_init (rtx target, rtx vals)
       emit_insn (rec_seq);
     }
 
+  /* The two halves should (by induction) be individually endian-correct.
+     However, in the memory layout provided by VALS, the nth element of
+     HALVES[0] comes immediately before the nth element HALVES[1].
+     This means that, on big-endian targets, the nth element of HALVES[0]
+     is more significant than the nth element HALVES[1].  */
+  if (BYTES_BIG_ENDIAN)
+    std::swap (halves[0], halves[1]);
   rtvec v = gen_rtvec (2, halves[0], halves[1]);
   rtx_insn *zip1_insn
     = emit_set_insn (target, gen_rtx_UNSPEC (mode, v, UNSPEC_ZIP1));
@@ -26745,7 +26864,6 @@ aarch64_evpc_hvla (struct expand_vec_perm_d *d)
   machine_mode vmode = d->vmode;
   if (!TARGET_SVE2p1
       || !TARGET_NON_STREAMING
-      || BYTES_BIG_ENDIAN
       || d->vec_flags != VEC_SVE_DATA
       || GET_MODE_UNIT_BITSIZE (vmode) > 64)
     return false;
@@ -26905,12 +27023,23 @@ aarch64_evpc_tbl (struct expand_vec_perm_d *d)
 static bool
 aarch64_evpc_sve_tbl (struct expand_vec_perm_d *d)
 {
-  unsigned HOST_WIDE_INT nelt;
+  if (!d->one_vector_p)
+    {
+      /* aarch64_expand_sve_vec_perm does not yet handle variable-length
+	 vectors.  */
+      if (!d->perm.length ().is_constant ())
+	return false;
 
-  /* Permuting two variable-length vectors could overflow the
-     index range.  */
-  if (!d->one_vector_p && !d->perm.length ().is_constant (&nelt))
-    return false;
+      /* This permutation reduces to the vec_perm optab if the elements are
+	 large enough to hold all selector indices.  Do not handle that case
+	 here, since the general TBL+SUB+TBL+ORR sequence is too expensive to
+	 be considered a "native" constant permutation.
+
+	 Not doing this would undermine code that queries can_vec_perm_const_p
+	 with allow_variable_p set to false.  See PR121027.  */
+      if (selector_fits_mode_p (d->vmode, d->perm))
+	return false;
+    }
 
   if (d->testing_p)
     return true;
@@ -27300,7 +27429,7 @@ aarch64_emit_sve_fp_cond (rtx target, rtx_code code, rtx pred,
 			  bool known_ptrue_p, rtx op0, rtx op1)
 {
   rtx flag = gen_int_mode (known_ptrue_p, SImode);
-  rtx unspec = gen_rtx_UNSPEC (GET_MODE (pred),
+  rtx unspec = gen_rtx_UNSPEC (GET_MODE (target),
 			       gen_rtvec (4, pred, flag, op0, op1),
 			       aarch64_unspec_cond_code (code));
   emit_set_insn (target, unspec);
@@ -27319,10 +27448,10 @@ static void
 aarch64_emit_sve_or_fp_conds (rtx target, rtx_code code1, rtx_code code2,
 			      rtx pred, bool known_ptrue_p, rtx op0, rtx op1)
 {
-  machine_mode pred_mode = GET_MODE (pred);
-  rtx tmp1 = gen_reg_rtx (pred_mode);
+  machine_mode target_mode = GET_MODE (target);
+  rtx tmp1 = gen_reg_rtx (target_mode);
   aarch64_emit_sve_fp_cond (tmp1, code1, pred, known_ptrue_p, op0, op1);
-  rtx tmp2 = gen_reg_rtx (pred_mode);
+  rtx tmp2 = gen_reg_rtx (target_mode);
   aarch64_emit_sve_fp_cond (tmp2, code2, pred, known_ptrue_p, op0, op1);
   aarch64_emit_binop (target, ior_optab, tmp1, tmp2);
 }
@@ -27339,8 +27468,7 @@ static void
 aarch64_emit_sve_invert_fp_cond (rtx target, rtx_code code, rtx pred,
 				 bool known_ptrue_p, rtx op0, rtx op1)
 {
-  machine_mode pred_mode = GET_MODE (pred);
-  rtx tmp = gen_reg_rtx (pred_mode);
+  rtx tmp = gen_reg_rtx (GET_MODE (target));
   aarch64_emit_sve_fp_cond (tmp, code, pred, known_ptrue_p, op0, op1);
   aarch64_emit_unop (target, one_cmpl_optab, tmp);
 }
@@ -27352,10 +27480,25 @@ aarch64_emit_sve_invert_fp_cond (rtx target, rtx_code code, rtx pred,
 void
 aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
 {
-  machine_mode pred_mode = GET_MODE (target);
   machine_mode data_mode = GET_MODE (op0);
+  rtx pred = aarch64_sve_fp_pred (data_mode, nullptr);
 
-  rtx ptrue = aarch64_ptrue_reg (pred_mode);
+  /* The governing and destination modes.  */
+  machine_mode pred_mode = GET_MODE (pred);
+  machine_mode target_mode = GET_MODE (target);
+
+  /* For partial vector modes, the choice of predicate mode depends
+     on whether we need to suppress exceptions for inactive elements.
+     If we do need to suppress exceptions, the predicate mode matches
+     the element size rather than the container size and the predicate
+     marks the upper bits in each container as inactive.  The predicate
+     is then a ptrue wrt TARGET_MODE but not wrt PRED_MODE.  It is the
+     latter which matters here.
+
+     If we don't need to suppress exceptions, the predicate mode matches
+     the container size, PRED_MODE == TARGET_MODE, and the predicate is
+     thus a ptrue wrt both TARGET_MODE and PRED_MODE.  */
+  bool known_ptrue_p = pred_mode == target_mode;
   switch (code)
     {
     case UNORDERED:
@@ -27369,12 +27512,13 @@ aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
     case EQ:
     case NE:
       /* There is native support for the comparison.  */
-      aarch64_emit_sve_fp_cond (target, code, ptrue, true, op0, op1);
+      aarch64_emit_sve_fp_cond (target, code, pred, known_ptrue_p, op0, op1);
       return;
 
     case LTGT:
       /* This is a trapping operation (LT or GT).  */
-      aarch64_emit_sve_or_fp_conds (target, LT, GT, ptrue, true, op0, op1);
+      aarch64_emit_sve_or_fp_conds (target, LT, GT,
+				    pred, known_ptrue_p, op0, op1);
       return;
 
     case UNEQ:
@@ -27383,7 +27527,7 @@ aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
 	  /* This would trap for signaling NaNs.  */
 	  op1 = force_reg (data_mode, op1);
 	  aarch64_emit_sve_or_fp_conds (target, UNORDERED, EQ,
-					ptrue, true, op0, op1);
+					pred, known_ptrue_p, op0, op1);
 	  return;
 	}
       /* fall through */
@@ -27393,11 +27537,19 @@ aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
     case UNGE:
       if (flag_trapping_math)
 	{
-	  /* Work out which elements are ordered.  */
-	  rtx ordered = gen_reg_rtx (pred_mode);
 	  op1 = force_reg (data_mode, op1);
-	  aarch64_emit_sve_invert_fp_cond (ordered, UNORDERED,
-					   ptrue, true, op0, op1);
+
+	  /* Work out which elements are unordered.  */
+	  rtx uo_tmp = gen_reg_rtx (target_mode);
+	  aarch64_emit_sve_fp_cond (uo_tmp, UNORDERED,
+				    pred, known_ptrue_p, op0, op1);
+
+	  /* Invert the result.  Governered by PRED so that we only
+	     flip the active bits.  */
+	  rtx ordered = gen_reg_rtx (pred_mode);
+	  uo_tmp = gen_lowpart (pred_mode, uo_tmp);
+	  emit_insn (gen_aarch64_pred_one_cmpl_z (pred_mode, ordered,
+						  pred, uo_tmp));
 
 	  /* Test the opposite condition for the ordered elements,
 	     then invert the result.  */
@@ -27422,7 +27574,8 @@ aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
 
   /* There is native support for the inverse comparison.  */
   code = reverse_condition_maybe_unordered (code);
-  aarch64_emit_sve_invert_fp_cond (target, code, ptrue, true, op0, op1);
+  aarch64_emit_sve_invert_fp_cond (target, code,
+				   pred, known_ptrue_p, op0, op1);
 }
 
 /* Return true if:
@@ -31853,9 +32006,43 @@ aarch64_test_sysreg_encoding_clashes (void)
 static void
 aarch64_test_sve_folding ()
 {
+  aarch64_target_switcher switcher (AARCH64_FL_SVE);
+
   tree res = fold_unary (BIT_NOT_EXPR, ssizetype,
 			 ssize_int (poly_int64 (1, 1)));
   ASSERT_TRUE (operand_equal_p (res, ssize_int (poly_int64 (-2, -1))));
+
+  auto build_v16bi = [](bool a, bool b)
+    {
+      rtx_vector_builder builder (VNx16BImode, 2, 1);
+      builder.quick_push (a ? const1_rtx : const0_rtx);
+      builder.quick_push (b ? const1_rtx : const0_rtx);
+      return builder.build ();
+    };
+  rtx v16bi_10 = build_v16bi (1, 0);
+  rtx v16bi_01 = build_v16bi (0, 1);
+
+  for (auto mode : { VNx8BImode, VNx4BImode, VNx2BImode })
+    {
+      rtx reg = gen_rtx_REG (mode, LAST_VIRTUAL_REGISTER + 1);
+      rtx subreg = lowpart_subreg (VNx16BImode, reg, mode);
+      rtx and1 = simplify_gen_binary (AND, VNx16BImode, subreg, v16bi_10);
+      ASSERT_EQ (lowpart_subreg (mode, and1, VNx16BImode), reg);
+      rtx and0 = simplify_gen_binary (AND, VNx16BImode, subreg, v16bi_01);
+      ASSERT_EQ (lowpart_subreg (mode, and0, VNx16BImode), CONST0_RTX (mode));
+
+      rtx ior1 = simplify_gen_binary (IOR, VNx16BImode, subreg, v16bi_10);
+      ASSERT_EQ (lowpart_subreg (mode, ior1, VNx16BImode), CONSTM1_RTX (mode));
+      rtx ior0 = simplify_gen_binary (IOR, VNx16BImode, subreg, v16bi_01);
+      ASSERT_EQ (lowpart_subreg (mode, ior0, VNx16BImode), reg);
+
+      rtx xor1 = simplify_gen_binary (XOR, VNx16BImode, subreg, v16bi_10);
+      ASSERT_RTX_EQ (lowpart_subreg (mode, xor1, VNx16BImode),
+		     lowpart_subreg (mode, gen_rtx_NOT (VNx16BImode, subreg),
+				     VNx16BImode));
+      rtx xor0 = simplify_gen_binary (XOR, VNx16BImode, subreg, v16bi_01);
+      ASSERT_EQ (lowpart_subreg (mode, xor0, VNx16BImode), reg);
+    }
 }
 
 /* Run all target-specific selftests.  */
diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
index d5c4a42..096c853 100644
--- a/gcc/config/aarch64/aarch64.h
+++ b/gcc/config/aarch64/aarch64.h
@@ -493,6 +493,11 @@ constexpr auto AARCH64_FL_DEFAULT_ISA_MODE ATTRIBUTE_UNUSED
   (bool (aarch64_tune_params.extra_tuning_flags \
 	 & AARCH64_EXTRA_TUNE_CHEAP_FPMR_WRITE))
 
+/* Enable folding address computation into LDAPUR when RCPC2 is available.  */
+#define TARGET_ENABLE_LDAPUR (TARGET_RCPC2 \
+			      && !(aarch64_tune_params.extra_tuning_flags \
+				   & AARCH64_EXTRA_TUNE_AVOID_LDAPUR))
+
 /* Combinatorial tests.  */
 
 #define TARGET_SVE2_OR_SME2 \
diff --git a/gcc/config/aarch64/aarch64.md b/gcc/config/aarch64/aarch64.md
index 509ef4c..a4ae685 100644
--- a/gcc/config/aarch64/aarch64.md
+++ b/gcc/config/aarch64/aarch64.md
@@ -482,6 +482,8 @@
 ;; clobber for SVE predicates.
 (define_attr "pred_clobber" "any,no,yes" (const_string "any"))
 
+(define_attr "enable_ldapur" "any,no,yes" (const_string "any"))
+
 ;; [For compatibility with Arm in pipeline models]
 ;; Attribute that specifies whether or not the instruction touches fp
 ;; registers.
@@ -506,7 +508,14 @@
 	  (eq_attr "pred_clobber" "yes")
 	  (match_test "TARGET_SVE_PRED_CLOBBER"))
 	(eq_attr "pred_clobber" "any"))
-
+      (ior
+	(and
+	  (eq_attr "enable_ldapur" "yes")
+	  (match_test "TARGET_ENABLE_LDAPUR"))
+	(and
+	  (eq_attr "enable_ldapur" "no")
+	  (match_test "!TARGET_ENABLE_LDAPUR"))
+	(eq_attr "enable_ldapur" "any"))
       (ior
 	(eq_attr "arch" "any")
 
@@ -5771,6 +5780,19 @@
    (match_operand:TI 1 "register_operand")]
   "TARGET_SIMD && !TARGET_CSSC"
 {
+  /* For SVE we can do popcount on DImode chunks of the TImode argument
+     and then use a cheap ADDP reduction.  The SVE CNT instruction requires
+     materializing a PTRUE so don't do this if optimizing for size.  */
+  if (TARGET_SVE && !optimize_function_for_size_p (cfun))
+    {
+      rtx v = gen_reg_rtx (V2DImode);
+      rtx v1 = gen_reg_rtx (V2DImode);
+      emit_move_insn (v, gen_lowpart (V2DImode, operands[1]));
+      rtx p = aarch64_ptrue_reg (VNx2BImode, 16);
+      emit_insn (gen_aarch64_pred_popcountv2di (v1, p, v));
+      emit_insn (gen_reduc_plus_scal_v2di (operands[0], v1));
+      DONE;
+    }
   rtx v = gen_reg_rtx (V16QImode);
   rtx v1 = gen_reg_rtx (V16QImode);
   emit_move_insn (v, gen_lowpart (V16QImode, operands[1]));
diff --git a/gcc/config/aarch64/atomics.md b/gcc/config/aarch64/atomics.md
index 36b0dbd..ea4a936 100644
--- a/gcc/config/aarch64/atomics.md
+++ b/gcc/config/aarch64/atomics.md
@@ -679,13 +679,16 @@
 )
 
 (define_insn "aarch64_atomic_load<mode>_rcpc"
-  [(set (match_operand:ALLI 0 "register_operand" "=r")
+  [(set (match_operand:ALLI 0 "register_operand")
     (unspec_volatile:ALLI
-      [(match_operand:ALLI 1 "aarch64_sync_memory_operand" "Q")
+      [(match_operand:ALLI 1 "aarch64_rcpc_memory_operand")
        (match_operand:SI 2 "const_int_operand")]			;; model
       UNSPECV_LDAP))]
   "TARGET_RCPC"
-  "ldapr<atomic_sfx>\t%<w>0, %1"
+  {@ [ cons: =0 , 1   ; attrs: enable_ldapur  ]
+     [ r        , Q   ; any                   ] ldapr<atomic_sfx>\t%<w>0, %1
+     [ r        , Ust ; yes                   ] ldapur<atomic_sfx>\t%<w>0, %1
+  }
 )
 
 (define_insn "aarch64_atomic_load<mode>"
@@ -705,21 +708,24 @@
 )
 
 (define_insn "*aarch64_atomic_load<ALLX:mode>_rcpc_zext"
-  [(set (match_operand:SD_HSDI 0 "register_operand" "=r")
+  [(set (match_operand:SD_HSDI 0 "register_operand")
     (zero_extend:SD_HSDI
       (unspec_volatile:ALLX
-        [(match_operand:ALLX 1 "aarch64_sync_memory_operand" "Q")
+        [(match_operand:ALLX 1 "aarch64_rcpc_memory_operand")
          (match_operand:SI 2 "const_int_operand")]			;; model
        UNSPECV_LDAP)))]
   "TARGET_RCPC && (<SD_HSDI:sizen> > <ALLX:sizen>)"
-  "ldapr<ALLX:atomic_sfx>\t%w0, %1"
+  {@ [ cons: =0 , 1   ; attrs: enable_ldapur ]
+     [ r        , Q   ; any                  ] ldapr<ALLX:atomic_sfx>\t%w0, %1
+     [ r        , Ust ; yes                  ] ldapur<ALLX:atomic_sfx>\t%w0, %1
+  }
 )
 
 (define_insn "*aarch64_atomic_load<ALLX:mode>_rcpc_sext"
   [(set (match_operand:GPI  0 "register_operand" "=r")
     (sign_extend:GPI
       (unspec_volatile:ALLX
-        [(match_operand:ALLX 1 "aarch64_sync_memory_operand" "Q")
+        [(match_operand:ALLX 1 "aarch64_rcpc_memory_operand" "Ust")
          (match_operand:SI 2 "const_int_operand")]			;; model
        UNSPECV_LDAP)))]
   "TARGET_RCPC2 && (<GPI:sizen> > <ALLX:sizen>)"
diff --git a/gcc/config/aarch64/cortex-a57-fma-steering.cc b/gcc/config/aarch64/cortex-a57-fma-steering.cc
index fd6da66..f7675be 100644
--- a/gcc/config/aarch64/cortex-a57-fma-steering.cc
+++ b/gcc/config/aarch64/cortex-a57-fma-steering.cc
@@ -948,6 +948,11 @@ func_fma_steering::analyze ()
 
 	  /* Search the chain where this instruction is (one of) the root.  */
 	  dest_op_info = insn_rr[INSN_UID (insn)].op_info;
+
+	  /* Register rename could fail. */
+	  if (!dest_op_info)
+	    continue;
+
 	  dest_regno = REGNO (SET_DEST (PATTERN (insn)));
 	  for (i = 0; i < dest_op_info->n_chains; i++)
 	    {
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index c59fcd6..8533912 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -463,6 +463,7 @@
 (define_mode_iterator VNx8SI_ONLY [VNx8SI])
 (define_mode_iterator VNx8SF_ONLY [VNx8SF])
 (define_mode_iterator VNx8DI_ONLY [VNx8DI])
+(define_mode_iterator VNx2SI_ONLY [VNx2SI])
 (define_mode_iterator VNx4SI_ONLY [VNx4SI])
 (define_mode_iterator VNx4SF_ONLY [VNx4SF])
 (define_mode_iterator VNx2DI_ONLY [VNx2DI])
@@ -3366,6 +3367,10 @@
 (define_int_iterator SVE_INT_UNARY [UNSPEC_REVB
 				    UNSPEC_REVH UNSPEC_REVW])
 
+;; This iterator is currently only used for estimation instructions,
+;; which are never generated automatically when -ftrapping-math is true.
+;; The iterator is therefore applied unconditionally to partial FP modes.
+;; This might need to be revisited if new operations are added in future.
 (define_int_iterator SVE_FP_UNARY [UNSPEC_FRECPE UNSPEC_RSQRTE])
 
 (define_int_iterator SVE_FP_UNARY_INT [(UNSPEC_FEXPA "TARGET_NON_STREAMING")])
@@ -3378,6 +3383,10 @@
 (define_int_iterator SVE_INT_BINARY_MULTI [UNSPEC_SQDMULH
 					   UNSPEC_SRSHL UNSPEC_URSHL])
 
+;; This iterator is currently only used for estimation instructions,
+;; which are never generated automatically when -ftrapping-math is true.
+;; The iterator is therefore applied unconditionally to partial FP modes.
+;; This might need to be revisited if new operations are added in future.
 (define_int_iterator SVE_FP_BINARY [UNSPEC_FRECPS UNSPEC_RSQRTS])
 
 (define_int_iterator SVE_FP_BINARY_INT [UNSPEC_FTSMUL UNSPEC_FTSSEL])
@@ -3429,9 +3438,10 @@
 					   UNSPEC_FMINQV
 					   UNSPEC_FMINNMQV])
 
-(define_int_iterator SVE_COND_FP_UNARY [UNSPEC_COND_FABS
-					UNSPEC_COND_FNEG
-					UNSPEC_COND_FRECPX
+(define_int_iterator SVE_COND_FP_UNARY_BITWISE [UNSPEC_COND_FABS
+						UNSPEC_COND_FNEG])
+
+(define_int_iterator SVE_COND_FP_UNARY [UNSPEC_COND_FRECPX
 					UNSPEC_COND_FRINTA
 					UNSPEC_COND_FRINTI
 					UNSPEC_COND_FRINTM
@@ -3439,13 +3449,12 @@
 					UNSPEC_COND_FRINTP
 					UNSPEC_COND_FRINTX
 					UNSPEC_COND_FRINTZ
-					UNSPEC_COND_FSQRT])
+					UNSPEC_COND_FSQRT
+					SVE_COND_FP_UNARY_BITWISE])
 
 ;; Same as SVE_COND_FP_UNARY, but without codes that have a dedicated
 ;; <optab><mode>2 expander.
-(define_int_iterator SVE_COND_FP_UNARY_OPTAB [UNSPEC_COND_FABS
-					      UNSPEC_COND_FNEG
-					      UNSPEC_COND_FRECPX
+(define_int_iterator SVE_COND_FP_UNARY_OPTAB [UNSPEC_COND_FRECPX
 					      UNSPEC_COND_FRINTA
 					      UNSPEC_COND_FRINTI
 					      UNSPEC_COND_FRINTM
diff --git a/gcc/config/aarch64/tuning_models/cortexx925.h b/gcc/config/aarch64/tuning_models/cortexx925.h
index 7d0162e..f448493 100644
--- a/gcc/config/aarch64/tuning_models/cortexx925.h
+++ b/gcc/config/aarch64/tuning_models/cortexx925.h
@@ -222,7 +222,8 @@ static const struct tune_params cortexx925_tunings =
   (AARCH64_EXTRA_TUNE_BASE
    | AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS
    | AARCH64_EXTRA_TUNE_MATCHED_VECTOR_THROUGHPUT
-   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW),	/* tune_flags.  */
+   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW
+   | AARCH64_EXTRA_TUNE_AVOID_LDAPUR),	/* tune_flags.  */
   &generic_armv9a_prefetch_tune,
   AARCH64_LDP_STP_POLICY_ALWAYS,   /* ldp_policy_model.  */
   AARCH64_LDP_STP_POLICY_ALWAYS	   /* stp_policy_model.  */
diff --git a/gcc/config/aarch64/tuning_models/generic_armv9_a.h b/gcc/config/aarch64/tuning_models/generic_armv9_a.h
index f76a250..9eb1a20 100644
--- a/gcc/config/aarch64/tuning_models/generic_armv9_a.h
+++ b/gcc/config/aarch64/tuning_models/generic_armv9_a.h
@@ -26,7 +26,7 @@
 static const struct cpu_addrcost_table generic_armv9_a_addrcost_table =
 {
     {
-      1, /* hi  */
+      0, /* hi  */
       0, /* si  */
       0, /* di  */
       1, /* ti  */
diff --git a/gcc/config/aarch64/tuning_models/neoversev2.h b/gcc/config/aarch64/tuning_models/neoversev2.h
index b000fb4..266d8f1 100644
--- a/gcc/config/aarch64/tuning_models/neoversev2.h
+++ b/gcc/config/aarch64/tuning_models/neoversev2.h
@@ -220,7 +220,8 @@ static const struct tune_params neoversev2_tunings =
   (AARCH64_EXTRA_TUNE_BASE
    | AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS
    | AARCH64_EXTRA_TUNE_MATCHED_VECTOR_THROUGHPUT
-   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW),	/* tune_flags.  */
+   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW
+   | AARCH64_EXTRA_TUNE_AVOID_LDAPUR),	/* tune_flags.  */
   &generic_armv9a_prefetch_tune,
   AARCH64_LDP_STP_POLICY_ALWAYS,   /* ldp_policy_model.  */
   AARCH64_LDP_STP_POLICY_ALWAYS	   /* stp_policy_model.  */
diff --git a/gcc/config/aarch64/tuning_models/neoversev3.h b/gcc/config/aarch64/tuning_models/neoversev3.h
index ad3cd22..f5566d2 100644
--- a/gcc/config/aarch64/tuning_models/neoversev3.h
+++ b/gcc/config/aarch64/tuning_models/neoversev3.h
@@ -220,7 +220,8 @@ static const struct tune_params neoversev3_tunings =
   (AARCH64_EXTRA_TUNE_BASE
    | AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS
    | AARCH64_EXTRA_TUNE_MATCHED_VECTOR_THROUGHPUT
-   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW),	/* tune_flags.  */
+   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW
+   | AARCH64_EXTRA_TUNE_AVOID_LDAPUR),	/* tune_flags.  */
   &generic_armv9a_prefetch_tune,
   AARCH64_LDP_STP_POLICY_ALWAYS,   /* ldp_policy_model.  */
   AARCH64_LDP_STP_POLICY_ALWAYS	   /* stp_policy_model.  */
diff --git a/gcc/config/aarch64/tuning_models/neoversev3ae.h b/gcc/config/aarch64/tuning_models/neoversev3ae.h
index a0adef0..5796e52 100644
--- a/gcc/config/aarch64/tuning_models/neoversev3ae.h
+++ b/gcc/config/aarch64/tuning_models/neoversev3ae.h
@@ -220,7 +220,8 @@ static const struct tune_params neoversev3ae_tunings =
   (AARCH64_EXTRA_TUNE_BASE
    | AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS
    | AARCH64_EXTRA_TUNE_MATCHED_VECTOR_THROUGHPUT
-   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW),	/* tune_flags.  */
+   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW
+   | AARCH64_EXTRA_TUNE_AVOID_LDAPUR),	/* tune_flags.  */
   &generic_armv9a_prefetch_tune,
   AARCH64_LDP_STP_POLICY_ALWAYS,   /* ldp_policy_model.  */
   AARCH64_LDP_STP_POLICY_ALWAYS	   /* stp_policy_model.  */
diff --git a/gcc/config/aarch64/tuning_models/olympus.h b/gcc/config/aarch64/tuning_models/olympus.h
new file mode 100644
index 0000000..268789d
--- /dev/null
+++ b/gcc/config/aarch64/tuning_models/olympus.h
@@ -0,0 +1,210 @@
+/* Tuning model description for the NVIDIA Olympus core.
+   Copyright The GNU Toolchain Authors.
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GCC is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_AARCH64_H_OLYMPUS
+#define GCC_AARCH64_H_OLYMPUS
+
+#include "generic.h"
+
+static struct cpu_regmove_cost olympus_regmove_cost =
+{
+  1, /* GP2GP  */
+  /* Spilling to int<->fp instead of memory is recommended so set
+     realistic costs compared to memmov_cost.  */
+  3, /* GP2FP  */
+  3, /* FP2GP  */
+  2 /* FP2FP  */
+};
+
+static advsimd_vec_cost olympus_advsimd_vector_cost =
+{
+  2, /* int_stmt_cost  */
+  2, /* fp_stmt_cost  */
+  2, /* ld2_st2_permute_cost */
+  2, /* ld3_st3_permute_cost  */
+  3, /* ld4_st4_permute_cost  */
+  2, /* permute_cost  */
+  5, /* reduc_i8_cost  */
+  3, /* reduc_i16_cost  */
+  3, /* reduc_i32_cost  */
+  2, /* reduc_i64_cost  */
+  4, /* reduc_f16_cost  */
+  4, /* reduc_f32_cost  */
+  4, /* reduc_f64_cost  */
+  2, /* store_elt_extra_cost  */
+  8, /* vec_to_scalar_cost  */
+  4, /* scalar_to_vec_cost  */
+  6, /* align_load_cost  */
+  6, /* unalign_load_cost  */
+  1, /* unalign_store_cost  */
+  1  /* store_cost  */
+};
+
+static sve_vec_cost olympus_sve_vector_cost =
+{
+  {
+    2, /* int_stmt_cost  */
+    2, /* fp_stmt_cost  */
+    2, /* ld2_st2_permute_cost  */
+    3, /* ld3_st3_permute_cost  */
+    3, /* ld4_st4_permute_cost  */
+    2, /* permute_cost  */
+    9, /* reduc_i8_cost  */
+    8, /* reduc_i16_cost  */
+    6, /* reduc_i32_cost  */
+    2, /* reduc_i64_cost  */
+    8, /* reduc_f16_cost  */
+    6, /* reduc_f32_cost  */
+    4, /* reduc_f64_cost  */
+    2, /* store_elt_extra_cost  */
+    8, /* vec_to_scalar_cost  */
+    4, /* scalar_to_vec_cost  */
+    4, /* align_load_cost  */
+    6, /* unalign_load_cost  */
+    1, /* unalign_store_cost  */
+    1  /* store_cost  */
+  },
+  3, /* clast_cost  */
+  10, /* fadda_f16_cost  */
+  6, /* fadda_f32_cost  */
+  4, /* fadda_f64_cost  */
+  14, /* gather_load_x32_cost  */
+  12, /* gather_load_x64_cost  */
+  42, /* gather_load_x32_init_cost  */
+  24, /* gather_load_x64_init_cost  */
+  1 /* scatter_store_elt_cost  */
+};
+
+static aarch64_scalar_vec_issue_info olympus_scalar_issue_info =
+{
+  4, /* loads_stores_per_cycle  */
+  2, /* stores_per_cycle  */
+  8, /* general_ops_per_cycle  */
+  0, /* fp_simd_load_general_ops  */
+  1 /* fp_simd_store_general_ops  */
+};
+
+static aarch64_advsimd_vec_issue_info olympus_advsimd_issue_info =
+{
+  {
+    3, /* loads_stores_per_cycle  */
+    2, /* stores_per_cycle  */
+    6, /* general_ops_per_cycle  */
+    0, /* fp_simd_load_general_ops  */
+    1 /* fp_simd_store_general_ops  */
+  },
+  2, /* ld2_st2_general_ops  */
+  2, /* ld3_st3_general_ops  */
+  3 /* ld4_st4_general_ops  */
+};
+
+static aarch64_sve_vec_issue_info olympus_sve_issue_info =
+{
+  {
+    {
+      3, /* loads_stores_per_cycle  */
+      2, /* stores_per_cycle  */
+      6, /* general_ops_per_cycle  */
+      0, /* fp_simd_load_general_ops  */
+      1 /* fp_simd_store_general_ops  */
+    },
+    2, /* ld2_st2_general_ops  */
+    2, /* ld3_st3_general_ops  */
+    3 /* ld4_st4_general_ops  */
+  },
+  2, /* pred_ops_per_cycle  */
+  1, /* while_pred_ops  */
+  0, /* int_cmp_pred_ops  */
+  0, /* fp_cmp_pred_ops  */
+  1, /* gather_scatter_pair_general_ops  */
+  1 /* gather_scatter_pair_pred_ops  */
+};
+
+static aarch64_vec_issue_info olympus_vec_issue_info =
+{
+  &olympus_scalar_issue_info,
+  &olympus_advsimd_issue_info,
+  &olympus_sve_issue_info
+};
+
+/* Olympus costs for vector insn classes.  */
+static struct cpu_vector_cost olympus_vector_cost =
+{
+  1, /* scalar_int_stmt_cost  */
+  2, /* scalar_fp_stmt_cost  */
+  4, /* scalar_load_cost  */
+  1, /* scalar_store_cost  */
+  1, /* cond_taken_branch_cost  */
+  1, /* cond_not_taken_branch_cost  */
+  &olympus_advsimd_vector_cost, /* advsimd  */
+  &olympus_sve_vector_cost, /* sve  */
+  &olympus_vec_issue_info /* issue_info  */
+};
+
+/* Olympus prefetch settings (which disable prefetch).  */
+static cpu_prefetch_tune olympus_prefetch_tune =
+{
+  0,			/* num_slots  */
+  -1,			/* l1_cache_size  */
+  64,			/* l1_cache_line_size  */
+  -1,			/* l2_cache_size  */
+  true,			/* prefetch_dynamic_strides */
+  -1,			/* minimum_stride */
+  -1			/* default_opt_level  */
+};
+
+static struct tune_params olympus_tunings =
+{
+  &cortexa76_extra_costs,
+  &generic_armv9_a_addrcost_table,
+  &olympus_regmove_cost,
+  &olympus_vector_cost,
+  &generic_branch_cost,
+  &generic_approx_modes,
+  SVE_128, /* sve_width  */
+  { 4, /* load_int.  */
+    1, /* store_int.  */
+    6, /* load_fp.  */
+    3, /* store_fp.  */
+    5, /* load_pred.  */
+    1 /* store_pred.  */
+  }, /* memmov_cost.  */
+  10, /* issue_rate  */
+  AARCH64_FUSE_NEOVERSE_BASE, /* fusible_ops  */
+  "32:16",	/* function_align.  */
+  "4",		/* jump_align.  */
+  "32:16",	/* loop_align.  */
+  8,	/* int_reassoc_width.  */
+  6,	/* fp_reassoc_width.  */
+  4,	/* fma_reassoc_width.  */
+  6,	/* vec_reassoc_width.  */
+  2,	/* min_div_recip_mul_sf.  */
+  2,	/* min_div_recip_mul_df.  */
+  0,	/* max_case_values.  */
+  tune_params::AUTOPREFETCHER_WEAK,	/* autoprefetcher_model.  */
+  (AARCH64_EXTRA_TUNE_BASE
+   | AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS
+   | AARCH64_EXTRA_TUNE_MATCHED_VECTOR_THROUGHPUT
+   | AARCH64_EXTRA_TUNE_AVOID_PRED_RMW),	/* tune_flags.  */
+  &olympus_prefetch_tune,
+  AARCH64_LDP_STP_POLICY_ALWAYS,   /* ldp_policy_model.  */
+  AARCH64_LDP_STP_POLICY_ALWAYS	   /* stp_policy_model.  */
+};
+
+#endif /* GCC_AARCH64_H_OLYMPUS.  */
diff --git a/gcc/config/arm/aarch-cost-tables.h b/gcc/config/arm/aarch-cost-tables.h
index c7a14b3..0600e59 100644
--- a/gcc/config/arm/aarch-cost-tables.h
+++ b/gcc/config/arm/aarch-cost-tables.h
@@ -123,9 +123,9 @@ const struct cpu_cost_table generic_extra_costs =
   {
     COSTS_N_INSNS (1),	/* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -230,9 +230,9 @@ const struct cpu_cost_table cortexa53_extra_costs =
   {
     COSTS_N_INSNS (1),	/* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -337,9 +337,9 @@ const struct cpu_cost_table cortexa57_extra_costs =
   {
     COSTS_N_INSNS (1),  /* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -444,9 +444,9 @@ const struct cpu_cost_table cortexa76_extra_costs =
   {
     COSTS_N_INSNS (1),  /* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -551,9 +551,9 @@ const struct cpu_cost_table exynosm1_extra_costs =
   {
     COSTS_N_INSNS (0),  /* alu.  */
     COSTS_N_INSNS (4),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
@@ -658,9 +658,9 @@ const struct cpu_cost_table xgene1_extra_costs =
   {
     COSTS_N_INSNS (2),  /* alu.  */
     COSTS_N_INSNS (8),  /* mult.  */
-    COSTS_N_INSNS (1),  /* movi.  */
-    COSTS_N_INSNS (2),  /* dup.  */
-    COSTS_N_INSNS (2)   /* extract.  */
+    COSTS_N_INSNS (0),  /* movi.  */
+    COSTS_N_INSNS (1),  /* dup.  */
+    COSTS_N_INSNS (1)   /* extract.  */
   }
 };
 
diff --git a/gcc/config/arm/arm.cc b/gcc/config/arm/arm.cc
index bde06f3..29b45ae 100644
--- a/gcc/config/arm/arm.cc
+++ b/gcc/config/arm/arm.cc
@@ -289,7 +289,8 @@ static bool arm_vector_alignment_reachable (const_tree type, bool is_packed);
 static bool arm_builtin_support_vector_misalignment (machine_mode mode,
 						     const_tree type,
 						     int misalignment,
-						     bool is_packed);
+						     bool is_packed,
+						     bool is_gather_scatter);
 static void arm_conditional_register_usage (void);
 static enum flt_eval_method arm_excess_precision (enum excess_precision_type);
 static reg_class_t arm_preferred_rename_class (reg_class_t rclass);
@@ -30661,12 +30662,16 @@ arm_vector_alignment_reachable (const_tree type, bool is_packed)
 static bool
 arm_builtin_support_vector_misalignment (machine_mode mode,
 					 const_tree type, int misalignment,
-					 bool is_packed)
+					 bool is_packed,
+					 bool is_gather_scatter)
 {
   if (TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access)
     {
       HOST_WIDE_INT align = TYPE_ALIGN_UNIT (type);
 
+      if (is_gather_scatter)
+	return true;
+
       if (is_packed)
         return align == 1;
 
@@ -30683,7 +30688,8 @@ arm_builtin_support_vector_misalignment (machine_mode mode,
     }
 
   return default_builtin_support_vector_misalignment (mode, type, misalignment,
-						      is_packed);
+						      is_packed,
+						      is_gather_scatter);
 }
 
 static void
diff --git a/gcc/config/arm/arm_neon.h b/gcc/config/arm/arm_neon.h
index cba50de..105385f 100644
--- a/gcc/config/arm/arm_neon.h
+++ b/gcc/config/arm/arm_neon.h
@@ -20938,11 +20938,6 @@ vbfdotq_lane_f32 (float32x4_t __r, bfloat16x8_t __a, bfloat16x4_t __b,
   return __builtin_neon_vbfdot_lanev4bfv4sf (__r, __a, __b, __index);
 }
 
-#pragma GCC pop_options
-
-#pragma GCC push_options
-#pragma GCC target ("arch=armv8.2-a+bf16")
-
 typedef struct bfloat16x4x2_t
 {
   bfloat16x4_t val[2];
diff --git a/gcc/config/avr/avr-dimode.md b/gcc/config/avr/avr-dimode.md
index 903bfbf..66ba5a9 100644
--- a/gcc/config/avr/avr-dimode.md
+++ b/gcc/config/avr/avr-dimode.md
@@ -101,10 +101,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8 ACC_A)
-                   (plus:ALL8 (reg:ALL8 ACC_A)
-                              (reg:ALL8 ACC_B)))
-   (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*add<mode>3_insn"
   [(set (reg:ALL8 ACC_A)
@@ -122,10 +120,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:DI ACC_A)
-                   (plus:DI (reg:DI ACC_A)
-                            (sign_extend:DI (reg:QI REG_X))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*adddi3_const8_insn"
   [(set (reg:DI ACC_A)
@@ -146,12 +142,10 @@
                    (match_operand:ALL8 0 "const_operand" "n Ynn")))]
   "avr_have_dimode
    && !s8_operand (operands[0], VOIDmode)"
-   "#"
-   "&& reload_completed"
-   [(parallel [(set (reg:ALL8 ACC_A)
-                    (plus:ALL8 (reg:ALL8 ACC_A)
-                               (match_dup 0)))
-               (clobber (reg:CC REG_CC))])])
+  "#"
+  "&& reload_completed"
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*add<mode>3_const_insn"
   [(set (reg:ALL8 ACC_A)
@@ -211,10 +205,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8 ACC_A)
-                   (minus:ALL8 (reg:ALL8 ACC_A)
-                               (reg:ALL8 ACC_B)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sub<mode>3_insn"
   [(set (reg:ALL8 ACC_A)
@@ -236,10 +228,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8 ACC_A)
-                   (minus:ALL8 (reg:ALL8 ACC_A)
-                               (match_dup 0)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sub<mode>3_const_insn"
   [(set (reg:ALL8 ACC_A)
@@ -288,10 +278,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8S ACC_A)
-                   (ss_addsub:ALL8S (reg:ALL8S ACC_A)
-                                    (reg:ALL8S ACC_B)))
-             (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3_insn"
   [(set (reg:ALL8S ACC_A)
@@ -309,10 +297,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8S ACC_A)
-                   (ss_addsub:ALL8S (reg:ALL8S ACC_A)
-                                    (match_dup 0)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3_const_insn"
   [(set (reg:ALL8S ACC_A)
@@ -361,10 +347,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8U ACC_A)
-                   (us_addsub:ALL8U (reg:ALL8U ACC_A)
-                                    (reg:ALL8U ACC_B)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3_insn"
   [(set (reg:ALL8U ACC_A)
@@ -382,10 +366,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8U ACC_A)
-                   (us_addsub:ALL8U (reg:ALL8U ACC_A)
-                                    (match_operand:ALL8U 0 "const_operand" "n Ynn")))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3_const_insn"
   [(set (reg:ALL8U ACC_A)
@@ -421,9 +403,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:DI ACC_A)
-                   (neg:DI (reg:DI ACC_A)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*negdi2_insn"
   [(set (reg:DI ACC_A)
@@ -500,7 +481,7 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(const_int 0)]
+  [(scratch)]
   {
     emit_insn (gen_compare_<mode>2 ());
     emit_jump_insn (gen_conditional_jump (operands[0], operands[1]));
@@ -529,7 +510,7 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(const_int 0)]
+  [(scratch)]
   {
     emit_insn (gen_compare_const8_di2 ());
     emit_jump_insn (gen_conditional_jump (operands[0], operands[1]));
@@ -556,7 +537,7 @@
    && !s8_operand (operands[1], VOIDmode)"
   "#"
   "&& reload_completed"
-  [(const_int 0)]
+  [(scratch)]
   {
     emit_insn (gen_compare_const_<mode>2 (operands[1], operands[3]));
     emit_jump_insn (gen_conditional_jump (operands[0], operands[2]));
@@ -629,10 +610,8 @@
   "avr_have_dimode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL8 ACC_A)
-                   (di_shifts:ALL8 (reg:ALL8 ACC_A)
-                                   (reg:QI 16)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3_insn"
   [(set (reg:ALL8 ACC_A)
@@ -674,14 +653,10 @@
    (clobber (reg:HI REG_Z))]
   "avr_have_dimode
    && AVR_HAVE_MUL"
-   "#"
-   "&& reload_completed"
-   [(parallel [(set (reg:DI ACC_A)
-                    (mult:DI (any_extend:DI (reg:SI 18))
-                             (any_extend:DI (reg:SI 22))))
-               (clobber (reg:HI REG_X))
-               (clobber (reg:HI REG_Z))
-               (clobber (reg:CC REG_CC))])])
+  "#"
+  "&& reload_completed"
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<extend_u>mulsidi3_insn"
   [(set (reg:DI ACC_A)
diff --git a/gcc/config/avr/avr-fixed.md b/gcc/config/avr/avr-fixed.md
index ce46beb..22061fc 100644
--- a/gcc/config/avr/avr-fixed.md
+++ b/gcc/config/avr/avr-fixed.md
@@ -62,10 +62,8 @@
   "<FIXED_B:MODE>mode != <FIXED_A:MODE>mode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (fract_convert:FIXED_A
-                    (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fract<FIXED_B:mode><FIXED_A:mode>2"
   [(set (match_operand:FIXED_A 0 "register_operand" "=r")
@@ -86,10 +84,8 @@
   "<FIXED_B:MODE>mode != <FIXED_A:MODE>mode"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (unsigned_fract_convert:FIXED_A
-                    (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fractuns<FIXED_B:mode><FIXED_A:mode>2"
   [(set (match_operand:FIXED_A 0 "register_operand" "=r")
@@ -124,10 +120,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ss_addsub:ALL124S (match_dup 1)
-                                      (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3"
   [(set (match_operand:ALL124S 0 "register_operand"                          "=??d,d")
@@ -149,10 +143,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (us_addsub:ALL124U (match_dup 1)
-                                      (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>3"
   [(set (match_operand:ALL124U 0 "register_operand"                          "=??r,d")
@@ -189,9 +181,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ss_neg:QQ (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ssnegqq2"
   [(set (match_operand:QQ 0 "register_operand"            "=r")
@@ -207,9 +198,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ss_abs:QQ (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ssabsqq2"
   [(set (match_operand:QQ 0 "register_operand"            "=r")
@@ -241,9 +231,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL2S 24)
-                   (ss_abs_neg:ALL2S (reg:ALL2S 24)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>2"
   [(set (reg:ALL2S 24)
@@ -261,9 +250,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL4S 22)
-                   (ss_abs_neg:ALL4S (reg:ALL4S 22)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code_stdname><mode>2"
   [(set (reg:ALL4S 22)
@@ -296,10 +284,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:QQ (match_dup 1)
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulqq3_enh"
   [(set (match_operand:QQ 0 "register_operand"         "=r")
@@ -317,10 +303,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:UQQ (match_dup 1)
-                             (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*muluqq3_enh"
   [(set (match_operand:UQQ 0 "register_operand"          "=r")
@@ -377,12 +361,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:QQ 23)
-                   (mult:QQ (reg:QQ 24)
-                            (reg:QQ 25)))
-              (clobber (reg:QI 22))
-              (clobber (reg:HI 24))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulqq3.call"
   [(set (reg:QQ 23)
@@ -425,11 +405,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL2QA 24)
-                   (mult:ALL2QA (reg:ALL2QA 18)
-                                (reg:ALL2QA 26)))
-              (clobber (reg:HI 22))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mul<mode>3.call"
   [(set (reg:ALL2QA 24)
@@ -468,10 +445,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL4A 24)
-                   (mult:ALL4A (reg:ALL4A 16)
-                               (reg:ALL4A 20)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mul<mode>3.call"
   [(set (reg:ALL4A 24)
@@ -514,11 +489,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL1Q 24)
-                   (usdiv:ALL1Q (reg:ALL1Q 25)
-                                (reg:ALL1Q 22)))
-              (clobber (reg:QI 25))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code><mode>3.call"
   [(set (reg:ALL1Q 24)
@@ -560,12 +532,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL2QA 24)
-                   (usdiv:ALL2QA (reg:ALL2QA 26)
-                                 (reg:ALL2QA 22)))
-              (clobber (reg:HI 26))
-              (clobber (reg:QI 21))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code><mode>3.call"
   [(set (reg:ALL2QA 24)
@@ -608,12 +576,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL4A 22)
-                   (usdiv:ALL4A (reg:ALL4A 24)
-                                (reg:ALL4A 18)))
-              (clobber (reg:HI 26))
-              (clobber (reg:HI 30))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<code><mode>3.call"
   [(set (reg:ALL4A 22)
@@ -684,12 +648,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (unspec:ALL124QA [(match_dup 1)
-                                     (match_dup 2)
-                                     (const_int 0)]
-                                    UNSPEC_ROUND))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*round<mode>3_const"
   [(set (match_operand:ALL124QA 0 "register_operand"                  "=d")
@@ -714,11 +674,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL1Q 24)
-                   (unspec:ALL1Q [(reg:ALL1Q 22)
-                                  (reg:QI 24)] UNSPEC_ROUND))
-              (clobber (reg:ALL1Q 22))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*round<mode>3.libgcc"
   [(set (reg:ALL1Q 24)
@@ -740,11 +697,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL2QA 24)
-                   (unspec:ALL2QA [(reg:ALL2QA 22)
-                                   (reg:QI 24)] UNSPEC_ROUND))
-              (clobber (reg:ALL2QA 22))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*round<mode>3.libgcc"
   [(set (reg:ALL2QA 24)
@@ -766,11 +720,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:ALL4QA 22)
-                   (unspec:ALL4QA [(reg:ALL4QA 18)
-                                   (reg:QI 24)] UNSPEC_ROUND))
-              (clobber (reg:ALL4QA 18))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*round<mode>3.libgcc"
   [(set (reg:ALL4QA 22)
diff --git a/gcc/config/avr/avr-mcus.def b/gcc/config/avr/avr-mcus.def
index ad64050..2e7c8ac 100644
--- a/gcc/config/avr/avr-mcus.def
+++ b/gcc/config/avr/avr-mcus.def
@@ -313,6 +313,10 @@ AVR_MCU ("avr64da28",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR
 AVR_MCU ("avr64da32",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA32__",   0x6000, 0x0, 0x10000, 0)
 AVR_MCU ("avr64da48",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA48__",   0x6000, 0x0, 0x10000, 0)
 AVR_MCU ("avr64da64",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA64__",   0x6000, 0x0, 0x10000, 0)
+AVR_MCU ("avr64da28s",       ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA28S__",  0x6000, 0x0, 0x10000, 0)
+AVR_MCU ("avr64da32s",       ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA32S__",  0x6000, 0x0, 0x10000, 0)
+AVR_MCU ("avr64da48s",       ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA48S__",  0x6000, 0x0, 0x10000, 0)
+AVR_MCU ("avr64da64s",       ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DA64S__",  0x6000, 0x0, 0x10000, 0)
 AVR_MCU ("avr64db28",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DB28__",   0x6000, 0x0, 0x10000, 0)
 AVR_MCU ("avr64db32",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DB32__",   0x6000, 0x0, 0x10000, 0)
 AVR_MCU ("avr64db48",        ARCH_AVRXMEGA2, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR64DB48__",   0x6000, 0x0, 0x10000, 0)
@@ -389,6 +393,9 @@ AVR_MCU ("avr16du32",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR
 AVR_MCU ("avr32da28",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DA28__",   0x7000, 0x0, 0x8000, 0x8000)
 AVR_MCU ("avr32da32",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DA32__",   0x7000, 0x0, 0x8000, 0x8000)
 AVR_MCU ("avr32da48",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DA48__",   0x7000, 0x0, 0x8000, 0x8000)
+AVR_MCU ("avr32da28s",       ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DA28S__",  0x7000, 0x0, 0x8000, 0x8000)
+AVR_MCU ("avr32da32s",       ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DA32S__",  0x7000, 0x0, 0x8000, 0x8000)
+AVR_MCU ("avr32da48s",       ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DA48S__",  0x7000, 0x0, 0x8000, 0x8000)
 AVR_MCU ("avr32db28",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DB28__",   0x7000, 0x0, 0x8000, 0x8000)
 AVR_MCU ("avr32db32",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DB32__",   0x7000, 0x0, 0x8000, 0x8000)
 AVR_MCU ("avr32db48",        ARCH_AVRXMEGA3, AVR_CVT,                 "__AVR_AVR32DB48__",   0x7000, 0x0, 0x8000, 0x8000)
@@ -427,6 +434,10 @@ AVR_MCU ("avr128da28",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR
 AVR_MCU ("avr128da32",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA32__",  0x4000, 0x0, 0x20000, 0)
 AVR_MCU ("avr128da48",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA48__",  0x4000, 0x0, 0x20000, 0)
 AVR_MCU ("avr128da64",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA64__",  0x4000, 0x0, 0x20000, 0)
+AVR_MCU ("avr128da28s",      ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA28S__", 0x4000, 0x0, 0x20000, 0)
+AVR_MCU ("avr128da32s",      ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA32S__", 0x4000, 0x0, 0x20000, 0)
+AVR_MCU ("avr128da48s",      ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA48S__", 0x4000, 0x0, 0x20000, 0)
+AVR_MCU ("avr128da64s",      ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DA64S__", 0x4000, 0x0, 0x20000, 0)
 AVR_MCU ("avr128db28",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DB28__",  0x4000, 0x0, 0x20000, 0)
 AVR_MCU ("avr128db32",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DB32__",  0x4000, 0x0, 0x20000, 0)
 AVR_MCU ("avr128db48",       ARCH_AVRXMEGA4, AVR_CVT | AVR_ISA_FLMAP, "__AVR_AVR128DB48__",  0x4000, 0x0, 0x20000, 0)
diff --git a/gcc/config/avr/avr-passes.cc b/gcc/config/avr/avr-passes.cc
index 284f49d..69df6d2 100644
--- a/gcc/config/avr/avr-passes.cc
+++ b/gcc/config/avr/avr-passes.cc
@@ -4120,9 +4120,8 @@ avr_optimize_casesi (rtx_insn *insns[5], rtx *xop)
   JUMP_LABEL (cbranch) = xop[4];
   ++LABEL_NUSES (xop[4]);
 
-  rtx_insn *seq1 = get_insns ();
   rtx_insn *last1 = get_last_insn ();
-  end_sequence ();
+  rtx_insn *seq1 = end_sequence ();
 
   emit_insn_after (seq1, insns[2]);
 
@@ -4141,9 +4140,8 @@ avr_optimize_casesi (rtx_insn *insns[5], rtx *xop)
 
   emit_insn (pat_4);
 
-  rtx_insn *seq2 = get_insns ();
   rtx_insn *last2 = get_last_insn ();
-  end_sequence ();
+  rtx_insn *seq2 = end_sequence ();
 
   emit_insn_after (seq2, insns[3]);
 
@@ -4845,6 +4843,137 @@ avr_pass_fuse_add::execute1 (function *func)
 
 
 //////////////////////////////////////////////////////////////////////////////
+// Fuse 2 move insns after combine.
+
+static const pass_data avr_pass_data_2moves =
+{
+  RTL_PASS,	    // type
+  "",		    // name (will be patched)
+  OPTGROUP_NONE,    // optinfo_flags
+  TV_DF_SCAN,	    // tv_id
+  0,		    // properties_required
+  0,		    // properties_provided
+  0,		    // properties_destroyed
+  0,		    // todo_flags_start
+  0		    // todo_flags_finish
+};
+
+class avr_pass_2moves : public rtl_opt_pass
+{
+public:
+  avr_pass_2moves (gcc::context *ctxt, const char *name)
+    : rtl_opt_pass (avr_pass_data_2moves, ctxt)
+  {
+    this->name = name;
+  }
+
+  unsigned int execute (function *func) final override
+  {
+    if (optimize && avropt_fuse_move2)
+      {
+	bool changed = false;
+	basic_block bb;
+
+	FOR_EACH_BB_FN (bb, func)
+	  {
+	    changed |= optimize_2moves_bb (bb);
+	  }
+
+	if (changed)
+	  {
+	    df_note_add_problem ();
+	    df_analyze ();
+	  }
+      }
+
+    return 0;
+  }
+
+  bool optimize_2moves (rtx_insn *, rtx_insn *);
+  bool optimize_2moves_bb (basic_block);
+}; // avr_pass_2moves
+
+bool
+avr_pass_2moves::optimize_2moves_bb (basic_block bb)
+{
+  bool changed = false;
+  rtx_insn *insn1 = nullptr;
+  rtx_insn *insn2 = nullptr;
+  rtx_insn *curr;
+
+  FOR_BB_INSNS (bb, curr)
+    {
+      if (insn1 && INSN_P (insn1)
+	  && insn2 && INSN_P (insn2))
+	changed |= optimize_2moves (insn1, insn2);
+
+      insn1 = insn2;
+      insn2 = curr;
+    }
+
+  return changed;
+}
+
+bool
+avr_pass_2moves::optimize_2moves (rtx_insn *insn1, rtx_insn *insn2)
+{
+  bool good = false;
+  bool bad = false;
+  rtx set1, dest1, src1;
+  rtx set2, dest2, src2;
+
+  if ((set1 = single_set (insn1))
+      && (set2 = single_set (insn2))
+      && (src1 = SET_SRC (set1))
+      && REG_P (src2 = SET_SRC (set2))
+      && REG_P (dest1 = SET_DEST (set1))
+      && REG_P (dest2 = SET_DEST (set2))
+      && rtx_equal_p (dest1, src2)
+      // Now we have:
+      // insn1: dest1 = src1
+      // insn2: dest2 = dest1
+      && REGNO (dest1) >= FIRST_PSEUDO_REGISTER
+      // Paranoia.
+      && GET_CODE (PATTERN (insn1)) != PARALLEL
+      && GET_CODE (PATTERN (insn2)) != PARALLEL
+      && (rtx_equal_p (dest2, src1)
+	  || !reg_overlap_mentioned_p (dest2, src1)))
+    {
+      avr_dump ("\n;; Found 2moves:\n%r\n%r\n", insn1, insn2);
+      avr_dump (";; reg %d: insn uses uids:", REGNO (dest1));
+
+      // Go check that dest1 is used exactly once, namely by insn2.
+
+      df_ref use = DF_REG_USE_CHAIN (REGNO (dest1));
+      for (; use; use = DF_REF_NEXT_REG (use))
+	{
+	  rtx_insn *user = DF_REF_INSN (use);
+	  avr_dump (" %d", INSN_UID (user));
+	  good |= INSN_UID (user) == INSN_UID (insn2);
+	  bad |= INSN_UID (user) != INSN_UID (insn2);
+	}
+      avr_dump (".\n");
+
+      if (good && !bad
+	  // Propagate src1 to insn2:
+	  // insn1: # Deleted
+	  // insn2: dest2 = src1
+	  && validate_change (insn2, &SET_SRC (set2), src1, false))
+	{
+	  SET_INSN_DELETED (insn1);
+	  return true;
+	}
+    }
+
+  if (good && !bad)
+    avr_dump (";; Failed\n");
+
+  return false;
+}
+
+
+
+//////////////////////////////////////////////////////////////////////////////
 // Split insns with nonzero_bits() after combine.
 
 static const pass_data avr_pass_data_split_nzb =
@@ -5706,6 +5835,14 @@ make_avr_pass_casesi (gcc::context *ctxt)
   return new avr_pass_casesi (ctxt, "avr-casesi");
 }
 
+// Optimize 2 consecutive moves after combine.
+
+rtl_opt_pass *
+make_avr_pass_2moves (gcc::context *ctxt)
+{
+  return new avr_pass_2moves (ctxt, "avr-2moves");
+}
+
 rtl_opt_pass *
 make_avr_pass_split_nzb (gcc::context *ctxt)
 {
diff --git a/gcc/config/avr/avr-passes.def b/gcc/config/avr/avr-passes.def
index eb60a93..d668c7f 100644
--- a/gcc/config/avr/avr-passes.def
+++ b/gcc/config/avr/avr-passes.def
@@ -74,6 +74,14 @@ INSERT_PASS_BEFORE (pass_free_cfg, 1, avr_pass_recompute_notes);
 
 INSERT_PASS_AFTER (pass_expand, 1, avr_pass_casesi);
 
+/* Insn combine may come up with superfluous reg-reg moves, where the combine
+   people say that these are no problem since reg-alloc is supposed to optimize
+   them.  The issue is that the lower-subreg pass sitting between combine and
+   reg-alloc may split such moves, coming up with a zoo of subregs which are
+   only handled poorly by the register allocator.  */
+
+INSERT_PASS_AFTER (pass_combine, 1, avr_pass_2moves);
+
 /* Some combine insns have nonzero_bits() in their condition, though insns
    should not use such stuff in their condition.  Therefore, we split such
    insn into something without nonzero_bits() in their condition right after
diff --git a/gcc/config/avr/avr-protos.h b/gcc/config/avr/avr-protos.h
index ca30136..9aa00d3 100644
--- a/gcc/config/avr/avr-protos.h
+++ b/gcc/config/avr/avr-protos.h
@@ -168,6 +168,8 @@ regmask (machine_mode mode, unsigned regno)
 
 extern void avr_fix_inputs (rtx*, unsigned, unsigned);
 extern bool avr_emit3_fix_outputs (rtx (*)(rtx,rtx,rtx), rtx*, unsigned, unsigned);
+extern rtx avr_add_ccclobber (rtx_insn *);
+#define DONE_ADD_CCC emit (avr_add_ccclobber (curr_insn)); DONE;
 
 extern rtx lpm_reg_rtx;
 extern rtx lpm_addr_reg_rtx;
@@ -208,6 +210,7 @@ extern rtl_opt_pass *make_avr_pass_casesi (gcc::context *);
 extern rtl_opt_pass *make_avr_pass_ifelse (gcc::context *);
 extern rtl_opt_pass *make_avr_pass_split_nzb (gcc::context *);
 extern rtl_opt_pass *make_avr_pass_split_after_peephole2 (gcc::context *);
+extern rtl_opt_pass *make_avr_pass_2moves (gcc::context *);
 #ifdef RTX_CODE
 extern bool avr_casei_sequence_check_operands (rtx *xop);
 extern bool avr_split_fake_addressing_move (rtx_insn *insn, rtx *operands);
diff --git a/gcc/config/avr/avr.cc b/gcc/config/avr/avr.cc
index c469297..2afea95 100644
--- a/gcc/config/avr/avr.cc
+++ b/gcc/config/avr/avr.cc
@@ -411,6 +411,29 @@ avr_to_int_mode (rtx x)
 }
 
 
+/* Return the pattern of INSN, but with added (clobber (reg:CC REG_CC)).
+   The pattern of INSN must be a PARALLEL or a SET.  INSN is unchanged.  */
+
+rtx
+avr_add_ccclobber (rtx_insn *insn)
+{
+  rtx pat = PATTERN (insn);
+  gcc_assert (GET_CODE (pat) == SET || GET_CODE (pat) == PARALLEL);
+
+  int newlen = GET_CODE (pat) == SET ? 2 : 1 + XVECLEN (pat, 0);
+  rtx newpat = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (newlen));
+  rtx elt0 = GET_CODE (pat) == SET ? pat : XVECEXP (pat, 0, 0);
+
+  XVECEXP (newpat, 0, 0) = copy_rtx (elt0);
+  XVECEXP (newpat, 0, newlen - 1) = gen_rtx_CLOBBER (VOIDmode, cc_reg_rtx);
+
+  for (int i = 1; i < newlen - 1; ++i)
+    XVECEXP (newpat, 0, i) = copy_rtx (XVECEXP (pat, 0, i));
+
+  return newpat;
+}
+
+
 /* Return true if hard register REG supports the ADIW and SBIW instructions.  */
 
 bool
@@ -14418,6 +14441,13 @@ avr_output_addr_vec (rtx_insn *labl, rtx table)
   // Output the label that precedes the table.
 
   ASM_OUTPUT_ALIGN (stream, 1);
+
+  char s_labl[40];
+  targetm.asm_out.generate_internal_label (s_labl, "L",
+					   CODE_LABEL_NUMBER (labl));
+  ASM_OUTPUT_TYPE_DIRECTIVE (stream, s_labl,
+			     AVR_HAVE_JMP_CALL ? "object" : "function");
+
   targetm.asm_out.internal_label (stream, "L", CODE_LABEL_NUMBER (labl));
 
   // Output the table's content.
@@ -14984,10 +15014,11 @@ avr_addr_space_convert (rtx src, tree type_old, tree type_new)
 
       /* Linearize memory: RAM has bit 23 set.  When as_new = __flashx then
 	 this is basically UB since __flashx mistreats RAM addresses, but there
-	 is no way to bail out.  (Though -Waddr-space-convert will tell.)  */
+	 is no way to bail out.  (Though -Waddr-space-convert will tell.)
+	 ...but PR121277 is confusing, in particular when NULL is coming in. */
 
       int msb = ADDR_SPACE_GENERIC_P (as_old)
-	? 0x80
+	? as_new == ADDR_SPACE_MEMX ? 0x80 : 0x00
 	: avr_addrspace[as_old].segment;
 
       src = force_reg (Pmode, src);
@@ -15085,10 +15116,16 @@ avr_convert_to_type (tree type, tree expr)
 	  const char *name_old = avr_addrspace[as_old].name;
 	  const char *name_new = avr_addrspace[as_new].name;
 
-	  warning (OPT_Waddr_space_convert,
-		   "conversion from address space %qs to address space %qs",
-		   ADDR_SPACE_GENERIC_P (as_old) ? "generic" : name_old,
-		   ADDR_SPACE_GENERIC_P (as_new) ? "generic" : name_new);
+	  // Be relaxed when NULL is used, and when 0x0 stands for
+	  // address 0x0.
+	  bool nowarn = (expr == null_pointer_node
+			 && (as_new == ADDR_SPACE_FLASHX
+			     || as_new == ADDR_SPACE_FLASH));
+	  if (!nowarn)
+	    warning (OPT_Waddr_space_convert,
+		     "conversion from address space %qs to address space %qs",
+		     ADDR_SPACE_GENERIC_P (as_old) ? "generic" : name_old,
+		     ADDR_SPACE_GENERIC_P (as_new) ? "generic" : name_new);
 
 	  return fold_build1_loc (loc, ADDR_SPACE_CONVERT_EXPR, type, expr);
 	}
diff --git a/gcc/config/avr/avr.md b/gcc/config/avr/avr.md
index f8bbdc7..67e88c1 100644
--- a/gcc/config/avr/avr.md
+++ b/gcc/config/avr/avr.md
@@ -574,9 +574,8 @@
    && REG_Z == REGNO (XEXP (operands[0], 0))"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:MOVMODE 22)
-                   (match_dup 0))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*load_<mode>_libgcc"
   [(set (reg:MOVMODE 22)
@@ -716,14 +715,8 @@
     || avr_load_libgcc_insn_p (insn, ADDR_SPACE_FLASHX, true)"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:MOVMODE REG_22)
-                   (match_dup 0))
-              (clobber (reg:QI REG_21))
-              (clobber (reg:HI REG_Z))
-              (clobber (reg:CC REG_CC))])]
-  {
-    operands[0] = SET_SRC (single_set (curr_insn));
-  })
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fxload_<mode>_libgcc"
   [(set (reg:MOVMODE REG_22)
@@ -853,9 +846,8 @@
     || reg_or_0_operand (operands[1], <MODE>mode)"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (match_dup 1))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 ;; "movqi_insn"
 ;; "movqq_insn" "movuqq_insn"
@@ -964,9 +956,8 @@
    || reg_or_0_operand (operands[1], <MODE>mode)"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (match_dup 1))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mov<mode>"
   [(set (match_operand:ALL2 0 "nonimmediate_operand" "=r,r  ,r,m    ,d,*r,q,r")
@@ -1137,9 +1128,8 @@
    || const0_rtx == operands[1]"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (match_dup 1))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*movpsi"
   [(set (match_operand:PSI 0 "nonimmediate_operand" "=r,r,r ,Qm,!d,r")
@@ -1197,9 +1187,8 @@
    || reg_or_0_operand (operands[1], <MODE>mode)"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (match_dup 1))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mov<mode>"
   [(set (match_operand:ALL4 0 "nonimmediate_operand" "=r,r  ,r ,Qm   ,!d,r")
@@ -1245,9 +1234,8 @@
    || reg_or_0_operand (operands[1], SFmode)"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (match_dup 1))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*movsf"
   [(set (match_operand:SF 0 "nonimmediate_operand" "=r,r,r ,Qm,!d,r")
@@ -1326,16 +1314,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (mem:BLK (reg:HI REG_X))
-                   (mem:BLK (reg:HI REG_Z)))
-              (unspec [(match_dup 0)]
-                      UNSPEC_CPYMEM)
-              (use (match_dup 1))
-              (clobber (reg:HI REG_X))
-              (clobber (reg:HI REG_Z))
-              (clobber (reg:QI LPM_REGNO))
-              (clobber (match_dup 2))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*cpymem_<mode>"
   [(set (mem:BLK (reg:HI REG_X))
@@ -1382,22 +1362,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (mem:BLK (reg:HI REG_X))
-                   (match_dup 2))
-              (unspec [(match_dup 0)]
-                      UNSPEC_CPYMEM)
-              (use (reg:QIHI 24))
-              (clobber (reg:HI REG_X))
-              (clobber (reg:HI REG_Z))
-              (clobber (reg:QI LPM_REGNO))
-              (clobber (reg:HI 24))
-              (clobber (reg:QI 23))
-              (clobber (mem:QI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])]
-  {
-    rtx xset = XVECEXP (PATTERN (curr_insn), 0, 0);
-    operands[2] = SET_SRC (xset);
-  })
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*cpymemx_<mode>"
   [(set (mem:BLK (reg:HI REG_X))
@@ -1461,13 +1427,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (mem:BLK (match_dup 0))
-                   (const_int 0))
-              (use (match_dup 1))
-              (use (match_dup 2))
-              (clobber (match_dup 3))
-              (clobber (match_dup 4))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*clrmemqi"
   [(set (mem:BLK (match_operand:HI 0 "register_operand" "e"))
@@ -1492,14 +1453,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (mem:BLK (match_dup 0))
-                   (const_int 0))
-              (use (match_dup 1))
-              (use (match_dup 2))
-              (clobber (match_dup 3))
-              (clobber (match_dup 4))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "adiw,*")])
 
 
@@ -1550,13 +1505,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel
-      [(set (match_dup 0)
-            (unspec:HI [(mem:BLK (match_dup 1))
-                        (const_int 0)
-                        (match_dup 2)]
-                       UNSPEC_STRLEN))
-       (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*strlenhi"
   [(set (match_operand:HI 0 "register_operand"                      "=e")
@@ -1581,10 +1531,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:ALL1 (match_dup 1)
-                              (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*add<mode>3"
   [(set (match_operand:ALL1 0 "register_operand"            "=r,d    ,r  ,r  ,r  ,r")
@@ -1640,10 +1588,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (zero_extend:HI (match_dup 1))
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*addhi3_zero_extend"
   [(set (match_operand:HI 0 "register_operand"                         "=r,*?r")
@@ -1663,10 +1609,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (match_dup 1)
-                            (zero_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*addhi3_zero_extend1"
   [(set (match_operand:HI 0 "register_operand"                         "=r")
@@ -1684,10 +1628,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (zero_extend:HI (match_dup 1))
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*addhi3_zero_extend.const"
   [(set (match_operand:HI 0 "register_operand"                         "=d")
@@ -1723,11 +1665,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (ashift:HI (zero_extend:HI (match_dup 1))
-                                       (const_int 1))
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*addhi3_zero_extend.ashift1"
   [(set (match_operand:HI 0 "register_operand"                                    "=r")
@@ -1752,11 +1691,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (zero_extend:HI (match_dup 1))
-                            (zero_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
-
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*usum_widenqihi3"
   [(set (match_operand:HI 0 "register_operand"                          "=r")
@@ -1774,10 +1710,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:HI (zero_extend:HI (match_dup 1))
-                             (zero_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*udiff_widenqihi3"
   [(set (match_operand:HI 0 "register_operand"                           "=r")
@@ -1797,7 +1731,7 @@
     return avr_out_addto_sp (operands, NULL);
   }
   ""
-  [(const_int 0)]
+  [(scratch)]
   {
     // Do not attempt to split this pattern. This FAIL is necessary
     // to prevent the splitter from matching *add<ALL2>3_split, splitting
@@ -1909,11 +1843,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:ALL2 (match_dup 1)
-                              (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 ;; "*addhi3_clobber"
 ;; "*addhq3_clobber"  "*adduhq3_clobber"
@@ -1943,11 +1874,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:ALL4 (match_dup 1)
-                              (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*add<mode>3"
   [(set (match_operand:ALL4 0 "register_operand"          "=??r,d ,r")
@@ -1979,10 +1907,8 @@
    && (<HISI:SIZE> > 2 || <CODE> == SIGN_EXTEND)"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HISI (any_extend:HISI (match_dup 1))
-                              (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 ;;                            "*addhi3.sign_extend.qi"
 ;; "*addpsi3.zero_extend.qi"  "*addpsi3.sign_extend.qi"
@@ -2019,10 +1945,8 @@
   "<HISI:SIZE> > <QIPSI:SIZE>"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:HISI (match_dup 1)
-                               (any_extend:HISI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 ;; "*subhi3.zero_extend.qi"   "*subhi3.sign_extend.qi"
 ;; "*subpsi3.zero_extend.qi"  "*subpsi3.sign_extend.qi"
@@ -2053,11 +1977,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:PSI (match_dup 1)
-                             (match_dup 2)))
-              (clobber (match_dup 3 ))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*addpsi3"
   [(set (match_operand:PSI 0 "register_operand"         "=??r,d ,d,r")
@@ -2079,10 +2000,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:PSI (match_dup 1)
-                              (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*subpsi3"
   [(set (match_operand:PSI 0 "register_operand"           "=r")
@@ -2106,10 +2025,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:ALL1 (match_dup 1)
-                               (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sub<mode>3"
   [(set (match_operand:ALL1 0 "register_operand"                    "=??r,d    ,r  ,r  ,r  ,r")
@@ -2137,11 +2054,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:ALL2 (match_dup 1)
-                               (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sub<mode>3"
   [(set (match_operand:ALL2 0 "register_operand"                    "=??r,d    ,*r")
@@ -2167,11 +2081,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:ALL4 (match_dup 1)
-                               (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sub<mode>3"
   [(set (match_operand:ALL4 0 "register_operand"                    "=??r,d    ,r")
@@ -2209,10 +2120,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:QI (match_dup 1)
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulqi3_enh"
   [(set (match_operand:QI 0 "register_operand" "=r")
@@ -2243,10 +2152,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:QI 24)
-                   (mult:QI (reg:QI 24) (reg:QI 22)))
-              (clobber (reg:QI 22))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulqi3_call"
   [(set (reg:QI 24)
@@ -2269,12 +2176,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (truncate:QI
-                    (lshiftrt:HI (mult:HI (any_extend:HI (match_dup 1))
-                                          (any_extend:HI (match_dup 2)))
-                                 (const_int 8))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<extend_su>mulqi3_highpart"
   [(set (match_operand:QI 0 "register_operand"                                       "=r")
@@ -2361,21 +2264,21 @@
                                         (const_int 0))))
               (clobber (reg:CC REG_CC))])])
 
-;; *subqi3.lt0   *subqi3.ge0
-;; *subhi3.lt0   *subhi3.ge0
-;; *subpsi3.lt0  *subpsi3.ge0
-;; *subsi3.lt0   *subsi3.ge0
-(define_insn "*sub<QISI:mode>3.<code>0"
-  [(set (match_operand:QISI 0 "register_operand"                        "=r")
-        (minus:QISI (match_operand:QISI 1 "register_operand"             "0")
-                    (gelt:QISI (match_operand:QISI2 2 "register_operand" "r")
-                               (const_int 0))))
-   (clobber (reg:CC REG_CC))]
-  "reload_completed"
-  {
-    return avr_out_add_msb (insn, operands, <CODE>, nullptr);
-  }
-  [(set_attr "adjust_len" "add_<code>0")])
+;; *addqi3.lt0_split   *addqi3.ge0_split
+;; *addhi3.lt0_split   *addhi3.ge0_split
+;; *addpsi3.lt0_split  *addpsi3.ge0_split
+;; *addsi3.lt0_split   *addsi3.ge0_split
+(define_insn_and_split "*add<QISI:mode>3.<code>0_split"
+  [(set (match_operand:QISI 0 "register_operand"                       "=r")
+        (plus:QISI (gelt:QISI (match_operand:QISI2 1 "register_operand" "r")
+                              (const_int 0))
+                   (match_operand:QISI 2 "register_operand"             "0")))]
+  ""
+  "#"
+  "&& reload_completed"
+  ; *add<QISI:mode>3.<code>0
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 ;; *addqi3.lt0   *addqi3.ge0
 ;; *addhi3.lt0   *addhi3.ge0
@@ -2393,25 +2296,6 @@
   }
   [(set_attr "adjust_len" "add_<code>0")])
 
-;; *addqi3.lt0_split   *addqi3.ge0_split
-;; *addhi3.lt0_split   *addhi3.ge0_split
-;; *addpsi3.lt0_split  *addpsi3.ge0_split
-;; *addsi3.lt0_split   *addsi3.ge0_split
-(define_insn_and_split "*add<QISI:mode>3.<code>0_split"
-  [(set (match_operand:QISI 0 "register_operand"                       "=r")
-        (plus:QISI (gelt:QISI (match_operand:QISI2 1 "register_operand" "r")
-                              (const_int 0))
-                   (match_operand:QISI 2 "register_operand"             "0")))]
-  ""
-  "#"
-  "&& reload_completed"
-  [; *add<QISI:mode>3.<code>0
-   (parallel [(set (match_dup 0)
-                   (plus:QISI (gelt:QISI (match_dup 1)
-                                         (const_int 0))
-                              (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
-
 ;; *subqi3.lt0_split   *subqi3.ge0_split
 ;; *subhi3.lt0_split   *subhi3.ge0_split
 ;; *subpsi3.lt0_split  *subpsi3.ge0_split
@@ -2424,13 +2308,25 @@
   ""
   "#"
   "&& reload_completed"
-  [; *sub<QISI:mode>3.<code>0
-   (parallel [(set (match_dup 0)
-                   (minus:QISI (match_dup 1)
-                               (gelt:QISI (match_dup 2)
-                                          (const_int 0))))
-              (clobber (reg:CC REG_CC))])])
+  ; *sub<QISI:mode>3.<code>0
+  [(scratch)]
+  { DONE_ADD_CCC })
 
+;; *subqi3.lt0   *subqi3.ge0
+;; *subhi3.lt0   *subhi3.ge0
+;; *subpsi3.lt0  *subpsi3.ge0
+;; *subsi3.lt0   *subsi3.ge0
+(define_insn "*sub<QISI:mode>3.<code>0"
+  [(set (match_operand:QISI 0 "register_operand"                        "=r")
+        (minus:QISI (match_operand:QISI 1 "register_operand"             "0")
+                    (gelt:QISI (match_operand:QISI2 2 "register_operand" "r")
+                               (const_int 0))))
+   (clobber (reg:CC REG_CC))]
+  "reload_completed"
+  {
+    return avr_out_add_msb (insn, operands, <CODE>, nullptr);
+  }
+  [(set_attr "adjust_len" "add_<code>0")])
 
 (define_insn_and_split "*umulqihi3.call_split"
   [(set (reg:HI 24)
@@ -2441,12 +2337,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (mult:HI (zero_extend:HI (reg:QI 22))
-                            (zero_extend:HI (reg:QI 24))))
-              (clobber (reg:QI 21))
-              (clobber (reg:HI 22))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*umulqihi3.call"
   [(set (reg:HI 24)
@@ -2469,10 +2361,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (any_extend:HI (match_dup 1))
-                            (any_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "<extend_u>mulqihi3"
   [(set (match_operand:HI 0 "register_operand"                         "=r")
@@ -2492,10 +2382,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (zero_extend:HI (match_dup 1))
-                            (sign_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*usmulqihi3"
   [(set (match_operand:HI 0 "register_operand"                         "=r")
@@ -2517,10 +2405,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (sign_extend:HI (match_dup 1))
-                            (zero_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sumulqihi3"
   [(set (match_operand:HI 0 "register_operand"                         "=r")
@@ -2542,10 +2428,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (not:HI (zero_extend:HI (not:QI (match_dup 1))))
-                            (sign_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*osmulqihi3"
   [(set (match_operand:HI 0 "register_operand"                                        "=&r")
@@ -2566,10 +2450,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (not:HI (zero_extend:HI (not:QI (match_dup 1))))
-                            (zero_extend:HI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*oumulqihi3"
   [(set (match_operand:HI 0 "register_operand"                                        "=&r")
@@ -2596,11 +2478,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:QI (mult:QI (match_dup 1)
-                                     (match_dup 2))
-                            (match_dup 3)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*maddqi4"
   [(set (match_operand:QI 0 "register_operand"                  "=r")
@@ -2622,11 +2501,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:QI (match_dup 3)
-                             (mult:QI (match_dup 1)
-                                      (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*msubqi4"
   [(set (match_operand:QI 0 "register_operand"                   "=r")
@@ -2705,11 +2581,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (mult:HI (any_extend:HI (match_dup 1))
-                                     (any_extend:HI (match_dup 2)))
-                            (match_dup 3)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<extend_u>maddqihi4"
   [(set (match_operand:HI 0 "register_operand"                                  "=r")
@@ -2734,11 +2607,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:HI (match_dup 3)
-                             (mult:HI (any_extend:HI (match_dup 1))
-                                      (any_extend:HI (match_dup 2)))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<extend_u>msubqihi4"
   [(set (match_operand:HI 0 "register_operand"                                  "=r")
@@ -2765,11 +2635,8 @@
    && <any_extend:CODE> != <any_extend2:CODE>"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (plus:HI (mult:HI (any_extend:HI  (match_dup 1))
-                                     (any_extend2:HI (match_dup 2)))
-                            (match_dup 3)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<any_extend:extend_su><any_extend2:extend_su>msubqihi4"
   [(set (match_operand:HI 0 "register_operand"                                  "=r")
@@ -2800,11 +2667,8 @@
    && <any_extend:CODE> != <any_extend2:CODE>"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (minus:HI (match_dup 3)
-                             (mult:HI (any_extend:HI  (match_dup 1))
-                                      (any_extend2:HI (match_dup 2)))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<any_extend:extend_su><any_extend2:extend_su>msubqihi4"
   [(set (match_operand:HI 0 "register_operand"                                   "=r")
@@ -3072,10 +2936,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashift:HI (sign_extend:HI (match_dup 1))
-                              (const_int 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ashiftqihi2.signx.1"
   [(set (match_operand:HI 0 "register_operand"                           "=r,*r")
@@ -3153,10 +3015,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (sign_extend:HI (match_dup 1))
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulsqihi3"
   [(set (match_operand:HI 0 "register_operand"                        "=&r")
@@ -3178,10 +3038,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (zero_extend:HI (match_dup 1))
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*muluqihi3"
   [(set (match_operand:HI 0 "register_operand"                        "=&r")
@@ -3205,10 +3063,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (not:HI (zero_extend:HI (not:QI (match_dup 1))))
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*muloqihi3"
   [(set (match_operand:HI 0 "register_operand"                                        "=&r")
@@ -3277,10 +3133,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:HI (match_dup 1)
-                            (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulhi3_enh"
   [(set (match_operand:HI 0 "register_operand" "=&r")
@@ -3319,11 +3173,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (mult:HI (reg:HI 24) (reg:HI 22)))
-              (clobber (reg:HI 22))
-              (clobber (reg:QI 21))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulhi3_call"
   [(set (reg:HI 24)
@@ -3719,11 +3570,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (mult:SI (reg:SI 22)
-                            (reg:SI 18)))
-              (clobber (reg:HI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn_and_split "*mulsi3_call_pr118012_split"
   [(set (reg:SI 22)
@@ -3737,13 +3585,8 @@
    && ! AVR_TINY"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (mult:SI (reg:SI 22)
-                            (reg:SI 18)))
-              (clobber (reg:SI 18))
-              (clobber (reg:HI 26))
-              (clobber (reg:HI 30))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulsi3_call"
   [(set (reg:SI 22)
@@ -3779,10 +3622,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (mult:SI (any_extend:SI (reg:HI 18))
-                            (any_extend:SI (reg:HI 26))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<extend_u>mulhisi3_call"
   [(set (reg:SI 22)
@@ -3804,12 +3645,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (truncate:HI (lshiftrt:SI (mult:SI (any_extend:SI (reg:HI 18))
-                                                      (any_extend:SI (reg:HI 26)))
-                                             (const_int 16))))
-              (clobber (reg:HI 22))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*<extend_su>mulhi3_highpart_call"
   [(set (reg:HI 24)
@@ -3829,10 +3666,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (mult:SI (zero_extend:SI (reg:HI 18))
-                            (sign_extend:SI (reg:HI 26))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*usmulhisi3_call"
   [(set (reg:SI 22)
@@ -3850,10 +3685,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (mult:SI (any_extend:SI (reg:HI 26))
-                            (reg:SI 18)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mul<extend_su>hisi3_call"
   [(set (reg:SI 22)
@@ -3871,10 +3704,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (mult:SI (not:SI (zero_extend:SI (not:HI (reg:HI 26))))
-                            (reg:SI 18)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulohisi3_call"
   [(set (reg:SI 22)
@@ -3925,11 +3756,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:QI 24) (div:QI (reg:QI 24) (reg:QI 22)))
-              (set (reg:QI 25) (mod:QI (reg:QI 24) (reg:QI 22)))
-              (clobber (reg:QI 22))
-              (clobber (reg:QI 23))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*divmodqi4_call"
   [(set (reg:QI 24) (div:QI (reg:QI 24) (reg:QI 22)))
@@ -3969,10 +3797,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:QI 24) (udiv:QI (reg:QI 24) (reg:QI 22)))
-              (set (reg:QI 25) (umod:QI (reg:QI 24) (reg:QI 22)))
-              (clobber (reg:QI 23))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*udivmodqi4_call"
   [(set (reg:QI 24) (udiv:QI (reg:QI 24) (reg:QI 22)))
@@ -4013,11 +3839,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 22) (div:HI (reg:HI 24) (reg:HI 22)))
-              (set (reg:HI 24) (mod:HI (reg:HI 24) (reg:HI 22)))
-              (clobber (reg:HI 26))
-              (clobber (reg:QI 21))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*divmodhi4_call"
   [(set (reg:HI 22) (div:HI (reg:HI 24) (reg:HI 22)))
@@ -4059,11 +3882,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 22) (udiv:HI (reg:HI 24) (reg:HI 22)))
-              (set (reg:HI 24) (umod:HI (reg:HI 24) (reg:HI 22)))
-              (clobber (reg:HI 26))
-              (clobber (reg:QI 21))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*udivmodhi4_call"
   [(set (reg:HI 22) (udiv:HI (reg:HI 24) (reg:HI 22)))
@@ -4112,10 +3932,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (mult:PSI (zero_extend:PSI (match_dup 1))
-                             (zero_extend:PSI (match_dup 2))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*umulqihipsi3"
   [(set (match_operand:PSI 0 "register_operand"                         "=&r")
@@ -4134,31 +3952,17 @@
 
 (define_insn_and_split "*umulhiqipsi3_split"
   [(set (match_operand:PSI 0 "register_operand"                         "=&r")
-        (mult:PSI (zero_extend:PSI (match_operand:HI 2 "register_operand" "r"))
-                  (zero_extend:PSI (match_operand:QI 1 "register_operand" "r"))))]
+        (mult:PSI (zero_extend:PSI (match_operand:HI 1 "register_operand" "r"))
+                  (zero_extend:PSI (match_operand:QI 2 "register_operand" "r"))))]
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
+  ; "*umulqihipsi3"
   [(parallel [(set (match_dup 0)
                    (mult:PSI (zero_extend:PSI (match_dup 2))
                              (zero_extend:PSI (match_dup 1))))
               (clobber (reg:CC REG_CC))])])
 
-(define_insn "*umulhiqipsi3"
-  [(set (match_operand:PSI 0 "register_operand"                         "=&r")
-        (mult:PSI (zero_extend:PSI (match_operand:HI 2 "register_operand" "r"))
-                  (zero_extend:PSI (match_operand:QI 1 "register_operand" "r"))))
-   (clobber (reg:CC REG_CC))]
-  "AVR_HAVE_MUL && reload_completed"
-  "mul %1,%A2
-	movw %A0,r0
-	mul %1,%B2
-	add %B0,r0
-	mov %C0,r1
-	clr __zero_reg__
-	adc %C0,__zero_reg__"
-  [(set_attr "length" "7")])
-
 (define_expand "mulsqipsi3"
   [(parallel [(set (match_operand:PSI 0 "pseudo_register_operand" "")
                    (mult:PSI (sign_extend:PSI (match_operand:QI 1 "pseudo_register_operand" ""))
@@ -4229,10 +4033,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:PSI 18)
-                   (mult:PSI (sign_extend:PSI (reg:QI 25))
-                             (reg:PSI 22)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulsqipsi3.libgcc"
   [(set (reg:PSI 18)
@@ -4253,13 +4055,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:PSI 22)
-                   (mult:PSI (reg:PSI 22)
-                             (reg:PSI 18)))
-              (clobber (reg:QI 21))
-              (clobber (reg:QI 25))
-              (clobber (reg:HI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*mulpsi3.libgcc"
   [(set (reg:PSI 22)
@@ -4311,12 +4108,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:PSI 22) (div:PSI (reg:PSI 22) (reg:PSI 18)))
-              (set (reg:PSI 18) (mod:PSI (reg:PSI 22) (reg:PSI 18)))
-              (clobber (reg:QI 21))
-              (clobber (reg:QI 25))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*divmodpsi4_call"
   [(set (reg:PSI 22) (div:PSI (reg:PSI 22) (reg:PSI 18)))
@@ -4360,12 +4153,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:PSI 22) (udiv:PSI (reg:PSI 22) (reg:PSI 18)))
-              (set (reg:PSI 18) (umod:PSI (reg:PSI 22) (reg:PSI 18)))
-              (clobber (reg:QI 21))
-              (clobber (reg:QI 25))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*udivmodpsi4_call"
   [(set (reg:PSI 22) (udiv:PSI (reg:PSI 22) (reg:PSI 18)))
@@ -4411,11 +4200,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 18) (div:SI (reg:SI 22) (reg:SI 18)))
-              (set (reg:SI 22) (mod:SI (reg:SI 22) (reg:SI 18)))
-              (clobber (reg:HI 26))
-              (clobber (reg:HI 30))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*divmodsi4_call"
   [(set (reg:SI 18) (div:SI (reg:SI 22) (reg:SI 18)))
@@ -4458,11 +4244,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 18) (udiv:SI (reg:SI 22) (reg:SI 18)))
-              (set (reg:SI 22) (umod:SI (reg:SI 22) (reg:SI 18)))
-              (clobber (reg:HI 26))
-              (clobber (reg:HI 30))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*udivmodsi4_call"
   [(set (reg:SI 18) (udiv:SI (reg:SI 22) (reg:SI 18)))
@@ -4484,10 +4267,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (and:QI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*andqi3"
   [(set (match_operand:QI 0 "register_operand"       "=??r,d,*l ,r")
@@ -4511,11 +4292,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (and:HI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*andhi3"
   [(set (match_operand:HI 0 "register_operand"       "=??r,d,d,r  ,r  ,r")
@@ -4545,11 +4323,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (and:PSI (match_dup 1)
-                            (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*andpsi3"
   [(set (match_operand:PSI 0 "register_operand"        "=??r,d,r  ,r  ,r")
@@ -4580,11 +4355,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (and:SI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*andsi3"
   [(set (match_operand:SI 0 "register_operand"       "=??r,d,r  ,r  ,r")
@@ -4634,10 +4406,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ior:QI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*iorqi3"
   [(set (match_operand:QI 0 "register_operand"       "=??r,d,*l")
@@ -4659,11 +4429,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ior:HI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*iorhi3"
   [(set (match_operand:HI 0 "register_operand"       "=??r,d,d,r  ,r")
@@ -4691,11 +4458,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ior:PSI (match_dup 1)
-                            (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*iorpsi3"
   [(set (match_operand:PSI 0 "register_operand"        "=??r,d,r  ,r")
@@ -4723,11 +4487,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ior:SI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*iorsi3"
   [(set (match_operand:SI 0 "register_operand"       "=??r,d,r  ,r")
@@ -4758,10 +4519,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (xor:QI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*xorqi3"
   [(set (match_operand:QI 0 "register_operand" "=r")
@@ -4780,11 +4539,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (xor:HI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*xorhi3"
   [(set (match_operand:HI 0 "register_operand"       "=??r,r  ,d  ,r")
@@ -4810,11 +4566,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (xor:PSI (match_dup 1)
-                            (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*xorpsi3"
   [(set (match_operand:PSI 0 "register_operand"        "=??r,r  ,d  ,r")
@@ -4842,11 +4595,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (xor:SI (match_dup 1)
-                           (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*xorsi3"
   [(set (match_operand:SI 0 "register_operand"       "=??r,r  ,d  ,r")
@@ -4918,7 +4668,7 @@
               (clobber (reg:CC REG_CC))])]
   "optimize
    && reload_completed"
-  [(const_int 1)]
+  [(scratch)]
   {
     for (int i = 0; i < <SIZE>; i++)
       {
@@ -5026,10 +4776,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:QI (match_dup 1)
-                              (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlqi3"
   [(set (match_operand:QI 0 "register_operand"               "=r,r,r  ,r  ,r  ,r  ,r  ,r")
@@ -5099,10 +4847,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:HI (match_dup 1)
-                              (const_int 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlhi2.1"
   [(set (match_operand:HI 0 "register_operand"           "=r")
@@ -5120,10 +4866,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:HI (match_dup 1)
-                              (const_int 15)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlhi2.15"
   [(set (match_operand:HI 0 "register_operand"           "=r")
@@ -5141,10 +4885,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:PSI (match_dup 1)
-                               (const_int 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlpsi2.1"
   [(set (match_operand:PSI 0 "register_operand"            "=r")
@@ -5162,10 +4904,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:PSI (match_dup 1)
-                               (const_int 23)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlpsi2.23"
   [(set (match_operand:PSI 0 "register_operand"            "=r")
@@ -5183,10 +4923,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:SI (match_dup 1)
-                              (const_int 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlsi2.1"
   [(set (match_operand:SI 0 "register_operand"           "=r")
@@ -5204,10 +4942,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (rotate:SI (match_dup 1)
-                              (const_int 31)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*rotlsi2.31"
   [(set (match_operand:SI 0 "register_operand"           "=r")
@@ -5239,7 +4975,7 @@
    && 0 == INTVAL (operands[2]) % 16"
   "#"
   "&& reload_completed"
-  [(const_int 0)]
+  [(scratch)]
   {
     avr_rotate_bytes (operands);
     DONE;
@@ -5263,7 +4999,7 @@
            && 0 == INTVAL (operands[2]) % 16))"
   "#"
   "&& reload_completed"
-  [(const_int 0)]
+  [(scratch)]
   {
     avr_rotate_bytes (operands);
     DONE;
@@ -5363,10 +5099,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashift:ALL1 (match_dup 1)
-                                (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ashl<mode>3"
   [(set (match_operand:ALL1 0 "register_operand"              "=r,r  ,r      ,r,r")
@@ -5390,11 +5124,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashift:ALL2 (match_dup 1)
-                                (match_dup 2)))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*,*")])
 
 ;; "*ashlhi3"
@@ -5506,11 +5237,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashift:ALL4 (match_dup 1)
-                                (match_dup 2)))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*,*")])
 
 (define_insn "*ashl<mode>3"
@@ -5749,12 +5477,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashift:PSI (match_dup 1)
-                               (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*")])
 
 (define_insn "*ashlpsi3"
@@ -5808,10 +5532,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashiftrt:ALL1 (match_dup 1)
-                                  (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ashr<mode>3"
   [(set (match_operand:ALL1 0 "register_operand"                  "=r,r              ,r      ,r")
@@ -5835,11 +5557,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashiftrt:ALL2 (match_dup 1)
-                                  (match_dup 2)))
-              (clobber (reg:CC REG_CC))])]
-   ""
+  [(scratch)]
+  { DONE_ADD_CCC }
    [(set_attr "isa" "*,*,*,3op,*,*")])
 
 ;; "*ashrhi3"
@@ -5866,12 +5585,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashiftrt:PSI (match_dup 1)
-                                 (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*")])
 
 (define_insn "*ashrpsi3"
@@ -5898,11 +5613,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (ashiftrt:ALL4 (match_dup 1)
-                                  (match_dup 2)))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*,*")])
 
 (define_insn "*ashr<mode>3"
@@ -6013,10 +5725,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (lshiftrt:ALL1 (match_dup 1)
-                                  (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*lshr<mode>3"
   [(set (match_operand:ALL1 0 "register_operand"                  "=r,r  ,r      ,r,r")
@@ -6039,11 +5749,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (lshiftrt:ALL2 (match_dup 1)
-                                  (match_dup 2)))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*,*")])
 
 (define_insn "*lshr<mode>3"
@@ -6066,12 +5773,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (lshiftrt:PSI (match_dup 1)
-                                 (match_dup 2)))
-              (clobber (match_dup 3))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*")])
 
 (define_insn "*lshrpsi3"
@@ -6098,11 +5801,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (lshiftrt:ALL4 (match_dup 1)
-                                  (match_dup 2)))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,*,3op,*,*")])
 
 (define_insn "*lshr<mode>3"
@@ -6217,9 +5917,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (abs:QI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*absqi2"
   [(set (match_operand:QI 0 "register_operand" "=r")
@@ -6237,9 +5936,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (abs:SF (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*abssf2"
   [(set (match_operand:SF 0 "register_operand" "=d,r")
@@ -6260,9 +5958,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (neg:QI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*negqi2"
   [(set (match_operand:QI 0 "register_operand" "=r")
@@ -6278,9 +5975,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (neg:HI (sign_extend:HI (match_dup 1))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*negqihi2"
   [(set (match_operand:HI 0 "register_operand"                        "=r")
@@ -6296,9 +5992,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (neg:HI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*neghi2"
   [(set (match_operand:HI 0 "register_operand"        "=r,&r")
@@ -6316,9 +6011,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (neg:PSI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*negpsi2"
   [(set (match_operand:PSI 0 "register_operand"        "=!d,r,&r")
@@ -6337,10 +6031,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (neg:SI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "*,*,mov,movw")])
 
 (define_insn "*negsi2.libgcc"
@@ -6371,9 +6063,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (neg:SF (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*negsf2"
   [(set (match_operand:SF 0 "register_operand" "=d,r")
@@ -6394,9 +6085,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (not:QI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*one_cmplqi2"
   [(set (match_operand:QI 0 "register_operand" "=r")
@@ -6412,9 +6102,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (not:HI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*one_cmplhi2"
   [(set (match_operand:HI 0 "register_operand" "=r")
@@ -6431,9 +6120,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (not:PSI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*one_cmplpsi2"
   [(set (match_operand:PSI 0 "register_operand" "=r")
@@ -6449,9 +6137,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (not:SI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*one_cmplsi2"
   [(set (match_operand:SI 0 "register_operand" "=r")
@@ -6480,9 +6167,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extend:HI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extendqihi2"
   [(set (match_operand:HI 0 "register_operand" "=r,r")
@@ -6501,9 +6187,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extend:PSI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extendqipsi2"
   [(set (match_operand:PSI 0 "register_operand" "=r,r")
@@ -6522,9 +6207,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extend:SI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extendqisi2"
   [(set (match_operand:SI 0 "register_operand" "=r,r")
@@ -6543,9 +6227,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extend:PSI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extendhipsi2"
   [(set (match_operand:PSI 0 "register_operand"                               "=r,r")
@@ -6564,9 +6247,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extend:SI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extendhisi2"
   [(set (match_operand:SI 0 "register_operand"                               "=r,r")
@@ -6585,9 +6267,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extend:SI (match_dup 1)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extendpsisi2"
   [(set (match_operand:SI 0 "register_operand"                                "=r")
@@ -7032,10 +6713,11 @@
    "#"
    "reload_completed"
    [(set (reg:CC REG_CC)
-         (compare:CC (match_dup 1) (match_dup 2)))
+         (compare:CC (match_dup 1)
+                     (match_dup 2)))
     (set (pc)
-         (if_then_else (match_op_dup 0
-                         [(reg:CC REG_CC) (const_int 0)])
+         (if_then_else (match_op_dup 0 [(reg:CC REG_CC)
+                                        (const_int 0)])
                        (label_ref (match_dup 3))
                        (pc)))])
 
@@ -7054,11 +6736,12 @@
    "#"
    "reload_completed"
    [(parallel [(set (reg:CC REG_CC)
-                    (compare:CC (match_dup 1) (match_dup 2)))
+                    (compare:CC (match_dup 1)
+                                (match_dup 2)))
                (clobber (match_dup 4))])
     (set (pc)
-         (if_then_else (match_op_dup 0
-                         [(reg:CC REG_CC) (const_int 0)])
+         (if_then_else (match_op_dup 0 [(reg:CC REG_CC)
+                                        (const_int 0)])
                        (label_ref (match_dup 3))
                        (pc)))]
    {
@@ -7081,11 +6764,12 @@
    "#"
    "reload_completed"
    [(parallel [(set (reg:CC REG_CC)
-                    (compare:CC (match_dup 1) (match_dup 2)))
+                    (compare:CC (match_dup 1)
+                                (match_dup 2)))
                (clobber (match_dup 4))])
     (set (pc)
-         (if_then_else (match_op_dup 0
-                         [(reg:CC REG_CC) (const_int 0)])
+         (if_then_else (match_op_dup 0 [(reg:CC REG_CC)
+                                        (const_int 0)])
                        (label_ref (match_dup 3))
                        (pc)))]
    {
@@ -7109,11 +6793,12 @@
    "#"
    "reload_completed"
    [(parallel [(set (reg:CC REG_CC)
-                    (compare:CC (match_dup 1) (match_dup 2)))
+                    (compare:CC (match_dup 1)
+                                (match_dup 2)))
                (clobber (match_dup 4))])
     (set (pc)
-         (if_then_else (match_op_dup 0
-                         [(reg:CC REG_CC) (const_int 0)])
+         (if_then_else (match_op_dup 0 [(reg:CC REG_CC)
+                                        (const_int 0)])
                        (label_ref (match_dup 3))
                        (pc)))]
    {
@@ -7668,17 +7353,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (if_then_else
-                    (match_op_dup 0
-                                  [(zero_extract:QIDI
-                                    (match_dup 1)
-                                    (const_int 1)
-                                    (match_dup 2))
-                                   (const_int 0)])
-                    (label_ref (match_dup 3))
-                    (pc)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sbrx_branch<mode>"
   [(set (pc)
@@ -7721,13 +7397,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (if_then_else (match_op_dup 0 [(and:QISI (match_dup 1)
-                                                            (match_dup 2))
-                                                  (const_int 0)])
-                                 (label_ref (match_dup 3))
-                                 (pc)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sbrx_and_branch<mode>"
   [(set (pc)
@@ -7968,14 +7639,8 @@
   "!AVR_HAVE_EIJMP_EICALL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (unspec:HI [(match_dup 0)]
-                              UNSPEC_INDEX_JMP))
-              (use (label_ref (match_dup 1)))
-              (clobber (match_dup 2))
-              (clobber (const_int 0))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "rjmp,rjmp,jmp")])
 
 (define_insn "*tablejump"
@@ -8004,14 +7669,8 @@
   "AVR_HAVE_EIJMP_EICALL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (unspec:HI [(reg:HI REG_Z)]
-                              UNSPEC_INDEX_JMP))
-              (use (label_ref (match_dup 0)))
-              (clobber (reg:HI REG_Z))
-              (clobber (reg:QI 24))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "eijmp")])
 
 
@@ -8182,17 +7841,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (if_then_else
-                    (match_operator 0 "eqne_operator"
-                                    [(zero_extract:QIHI
-                                      (mem:QI (match_dup 1))
-                                      (const_int 1)
-                                      (match_dup 2))
-                                     (const_int 0)])
-                    (label_ref (match_dup 3))
-                    (pc)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sbix_branch"
   [(set (pc)
@@ -8230,14 +7880,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (if_then_else
-                    (match_operator 0 "gelt_operator"
-                                    [(mem:QI (match_dup 1))
-                                     (const_int 0)])
-                    (label_ref (match_dup 2))
-                    (pc)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sbix_branch_bit7"
   [(set (pc)
@@ -8277,17 +7921,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (if_then_else
-                    (match_operator 0 "eqne_operator"
-                                    [(zero_extract:QIHI
-                                      (mem:QI (match_dup 1))
-                                      (const_int 1)
-                                      (match_dup 2))
-                                     (const_int 0)])
-                    (label_ref (match_dup 3))
-                    (pc)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sbix_branch_tmp"
   [(set (pc)
@@ -8324,14 +7959,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (pc)
-                   (if_then_else
-                    (match_operator 0 "gelt_operator"
-                                    [(mem:QI (match_dup 1))
-                                     (const_int 0)])
-                    (label_ref (match_dup 2))
-                    (pc)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sbix_branch_tmp_bit7"
   [(set (pc)
@@ -8784,13 +8413,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(unspec_volatile [(match_dup 0)
-                                (const_int 1)]
-                               UNSPECV_DELAY_CYCLES)
-              (set (match_dup 1)
-                   (unspec_volatile:BLK [(match_dup 1)] UNSPECV_MEMORY_BARRIER))
-              (clobber (match_dup 2))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*delay_cycles_1"
   [(unspec_volatile [(match_operand:QI 0 "const_int_operand" "n")
@@ -8816,14 +8440,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(unspec_volatile [(match_dup 0)
-                                (const_int 2)]
-                               UNSPECV_DELAY_CYCLES)
-              (set (match_dup 1)
-                   (unspec_volatile:BLK [(match_dup 1)] UNSPECV_MEMORY_BARRIER))
-              (clobber (match_dup 2))
-              (clobber (reg:CC REG_CC))])]
-  ""
+  [(scratch)]
+  { DONE_ADD_CCC }
   [(set_attr "isa" "adiw,no_adiw")])
 
 (define_insn "*delay_cycles_2"
@@ -8853,15 +8471,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(unspec_volatile [(match_dup 0)
-                                (const_int 3)]
-                               UNSPECV_DELAY_CYCLES)
-              (set (match_dup 1)
-                   (unspec_volatile:BLK [(match_dup 1)] UNSPECV_MEMORY_BARRIER))
-              (clobber (match_dup 2))
-              (clobber (match_dup 3))
-              (clobber (match_dup 4))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*delay_cycles_3"
   [(unspec_volatile [(match_operand:SI 0 "const_int_operand" "n")
@@ -8896,16 +8507,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(unspec_volatile [(match_dup 0)
-                                (const_int 4)]
-                               UNSPECV_DELAY_CYCLES)
-              (set (match_dup 1)
-                   (unspec_volatile:BLK [(match_dup 1)] UNSPECV_MEMORY_BARRIER))
-              (clobber (match_dup 2))
-              (clobber (match_dup 3))
-              (clobber (match_dup 4))
-              (clobber (match_dup 5))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*delay_cycles_4"
   [(unspec_volatile [(match_operand:SI 0 "const_int_operand" "n")
@@ -8942,12 +8545,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (unspec:QI [(match_dup 1)
-                               (match_dup 2)
-                               (match_dup 3)]
-                              UNSPEC_INSERT_BITS))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*insert_bits"
   [(set (match_operand:QI 0 "register_operand"              "=r  ,d  ,r")
@@ -9127,12 +8726,13 @@
    "#"
    "reload_completed"
    [(set (reg:CC REG_CC)
-         (compare:CC (match_dup 0) (const_int 0)))
+         (compare:CC (match_dup 0)
+                     (const_int 0)))
     (set (pc)
-         (if_then_else (ge (reg:CC REG_CC) (const_int 0))
+         (if_then_else (ge (reg:CC REG_CC)
+                           (const_int 0))
                        (label_ref (match_dup 1))
-                       (pc)))]
-   "")
+                       (pc)))])
 
 (define_expand "flash_segment"
   [(parallel [(match_operand:QI 0 "register_operand" "")
@@ -9235,9 +8835,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (parity:HI (reg:HI 24)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*parityhi2.libgcc"
   [(set (reg:HI 24)
@@ -9253,9 +8852,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (zero_extend:HI (parity:QI (reg:QI 24))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*parityqihi2.libgcc"
   [(set (reg:HI 24)
@@ -9271,9 +8869,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (truncate:HI (parity:SI (reg:SI 22))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*paritysihi2.libgcc"
   [(set (reg:HI 24)
@@ -9329,9 +8926,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (popcount:HI (reg:HI 24)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*popcounthi2.libgcc"
   [(set (reg:HI 24)
@@ -9347,9 +8943,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (truncate:HI (popcount:SI (reg:SI 22))))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*popcountsi2.libgcc"
   [(set (reg:HI 24)
@@ -9365,9 +8960,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:QI 24)
-                   (popcount:QI (reg:QI 24)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*popcountqi2.libgcc"
   [(set (reg:QI 24)
@@ -9421,10 +9015,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (clz:HI (reg:HI 24)))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*clzhi2.libgcc"
   [(set (reg:HI 24)
@@ -9442,10 +9034,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (truncate:HI (clz:SI (reg:SI 22))))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*clzsihi2.libgcc"
   [(set (reg:HI 24)
@@ -9490,10 +9080,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (ctz:HI (reg:HI 24)))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ctzhi2.libgcc"
   [(set (reg:HI 24)
@@ -9512,11 +9100,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (truncate:HI (ctz:SI (reg:SI 22))))
-              (clobber (reg:QI 22))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ctzsihi2.libgcc"
   [(set (reg:HI 24)
@@ -9562,10 +9147,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (ffs:HI (reg:HI 24)))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ffshi2.libgcc"
   [(set (reg:HI 24)
@@ -9584,11 +9167,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 24)
-                   (truncate:HI (ffs:SI (reg:SI 22))))
-              (clobber (reg:QI 22))
-              (clobber (reg:QI 26))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*ffssihi2.libgcc"
   [(set (reg:HI 24)
@@ -9633,9 +9213,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:SI 22)
-                   (bswap:SI (reg:SI 22)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*bswapsi2.libgcc"
   [(set (reg:SI 22)
@@ -9742,11 +9321,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (unspec:HI [(match_dup 1)
-                               (match_dup 2)]
-                              UNSPEC_FMUL))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fmul_insn"
   [(set (match_operand:HI 0 "register_operand" "=r")
@@ -9768,11 +9344,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 22)
-                   (unspec:HI [(reg:QI 24)
-                               (reg:QI 25)] UNSPEC_FMUL))
-              (clobber (reg:HI 24))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fmul.call"
   [(set (reg:HI 22)
@@ -9814,11 +9387,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (unspec:HI [(match_dup 1)
-                               (match_dup 2)]
-                              UNSPEC_FMULS))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fmuls_insn"
   [(set (match_operand:HI 0 "register_operand" "=r")
@@ -9840,11 +9410,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 22)
-                   (unspec:HI [(reg:QI 24)
-                               (reg:QI 25)] UNSPEC_FMULS))
-              (clobber (reg:HI 24))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fmuls.call"
   [(set (reg:HI 22)
@@ -9886,11 +9453,8 @@
   "AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (unspec:HI [(match_dup 1)
-                               (match_dup 2)]
-                              UNSPEC_FMULSU))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fmulsu_insn"
   [(set (match_operand:HI 0 "register_operand" "=r")
@@ -9912,11 +9476,8 @@
   "!AVR_HAVE_MUL"
   "#"
   "&& reload_completed"
-  [(parallel [(set (reg:HI 22)
-                   (unspec:HI [(reg:QI 24)
-                               (reg:QI 25)] UNSPEC_FMULSU))
-              (clobber (reg:HI 24))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*fmulsu.call"
   [(set (reg:HI 22)
@@ -10037,11 +9598,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (zero_extract:QI (match_dup 0)
-                                    (const_int 1)
-                                    (match_dup 1))
-                   (match_dup 2))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*insv.reg"
   [(set (zero_extract:QI (match_operand:QI 0 "register_operand"    "+r,d,d,l,l")
@@ -10478,11 +10036,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (zero_extract:QI (not:QI (match_dup 1))
-                                    (const_int 1)
-                                    (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*extzv.not"
   [(set (match_operand:QI 0 "register_operand" "=r")
@@ -10619,11 +10174,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (and:QISI (any_shift:QISI (match_dup 1)
-                                             (match_dup 2))
-                             (match_dup 3)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*insv.any_shift.<mode>"
   [(set (match_operand:QISI 0 "register_operand" "=r")
@@ -10686,11 +10238,8 @@
   ""
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0)
-                   (sign_extract:QISI (match_dup 1)
-                                      (const_int 1)
-                                      (match_dup 2)))
-              (clobber (reg:CC REG_CC))])])
+  [(scratch)]
+  { DONE_ADD_CCC })
 
 (define_insn "*sextr.<QISI:mode>.<QISI2:mode>"
   [(set (match_operand:QISI 0 "register_operand"                    "=r")
diff --git a/gcc/config/avr/avr.opt b/gcc/config/avr/avr.opt
index 9883119..7f6f18c 100644
--- a/gcc/config/avr/avr.opt
+++ b/gcc/config/avr/avr.opt
@@ -164,6 +164,10 @@ mfuse-move=
 Target Joined RejectNegative UInteger Var(avropt_fuse_move) Init(0) Optimization IntegerRange(0, 23)
 -mfuse-move=<0,23>	Optimization. Run a post-reload pass that tweaks move instructions.
 
+mfuse-move2
+Target Var(avropt_fuse_move2) Init(0) Optimization
+Optimization. Fuse some move insns after insn combine.
+
 mabsdata
 Target Mask(ABSDATA)
 Assume that all data in static storage can be accessed by LDS / STS instructions.  This option is only useful for reduced Tiny devices like ATtiny40.
diff --git a/gcc/config/avr/avr.opt.urls b/gcc/config/avr/avr.opt.urls
index 662fdee..87c26b2 100644
--- a/gcc/config/avr/avr.opt.urls
+++ b/gcc/config/avr/avr.opt.urls
@@ -92,6 +92,9 @@ UrlSuffix(gcc/AVR-Options.html#index-mfuse-move)
 mfuse-move=
 UrlSuffix(gcc/AVR-Options.html#index-mfuse-move)
 
+mfuse-move2
+UrlSuffix(gcc/AVR-Options.html#index-mfuse-move2)
+
 mabsdata
 UrlSuffix(gcc/AVR-Options.html#index-mabsdata)
 
diff --git a/gcc/config/cris/cris.cc b/gcc/config/cris/cris.cc
index a34c9e9..4acdd1d 100644
--- a/gcc/config/cris/cris.cc
+++ b/gcc/config/cris/cris.cc
@@ -3711,9 +3711,11 @@ cris_md_asm_adjust (vec<rtx> &outputs, vec<rtx> &inputs,
   /* Determine if the source using MOF.  If it is, automatically
      clobbering MOF would cause it to have impossible constraints.  */
 
-  /* Look for a use of the MOF constraint letter: h.  */
+  /* Look for a use of the MOF constraint letter h or a hard register
+     constraint.  */
   for (unsigned i = 0, n = constraints.length(); i < n; ++i)
-    if (strchr (constraints[i], 'h') != NULL)
+    if (strchr (constraints[i], 'h') != NULL
+	|| strstr (constraints[i], "{mof}") != NULL)
       return NULL;
 
   /* Look for an output or an input that touches MOF.  */
diff --git a/gcc/config/darwin-driver.cc b/gcc/config/darwin-driver.cc
index 224e0a0..e83b7cd 100644
--- a/gcc/config/darwin-driver.cc
+++ b/gcc/config/darwin-driver.cc
@@ -64,7 +64,8 @@ validate_macosx_version_min (const char *version_str)
 
   major = strtoul (version_str, &end, 10);
 
-  /* macOS 10, 11, and 12 are known. clang accepts up to 99.  */
+  /* macOS 10, 11, 12, 13, 14, 15 and 26 are known.
+     clang accepts up to 99.  */
   if (major < 10 || major > 99)
     return NULL;
 
@@ -159,15 +160,16 @@ darwin_find_version_from_kernel (void)
   if (*version_p++ != '.')
     goto parse_failed;
 
-  /* Darwin20 sees a transition to macOS 11.  In this, it seems that the
-     mapping to macOS minor version and patch level is now always 0, 0
-     (at least for macOS 11 and 12).  */
-  if (major_vers >= 20)
-    {
-      /* Apple clang doesn't include the minor version or the patch level
-	 in the object file, nor does it pass it to ld  */
-      asprintf (&new_flag, "%d.00.00", major_vers - 9);
-    }
+  /* Darwin25 saw a transition to macOS 26.  */
+  if (major_vers >= 25)
+    /* Apple clang doesn't include the minor version or the patch level
+       in the object file, nor does it pass it to ld  */
+    asprintf (&new_flag, "%d.00.00", major_vers + 1);
+  /* Darwin20 saw a transition to macOS 11.  */
+  else if (major_vers >= 20)
+    /* Apple clang doesn't include the minor version or the patch level
+       in the object file, nor does it pass it to ld  */
+    asprintf (&new_flag, "%d.00.00", major_vers - 9);
   else if (major_vers - 4 <= 4)
     /* On 10.4 and earlier, the old linker is used which does not
        support three-component system versions.
diff --git a/gcc/config/epiphany/epiphany.cc b/gcc/config/epiphany/epiphany.cc
index 16626f8..f53a643 100644
--- a/gcc/config/epiphany/epiphany.cc
+++ b/gcc/config/epiphany/epiphany.cc
@@ -2816,12 +2816,16 @@ epiphany_vector_alignment_reachable (const_tree type, bool is_packed)
 
 static bool
 epiphany_support_vector_misalignment (machine_mode mode, const_tree type,
-				      int misalignment, bool is_packed)
+				      int misalignment, bool is_packed,
+				      bool is_gather_scatter)
 {
+  if (is_gather_scatter)
+    return true;
   if (GET_MODE_SIZE (mode) == 8 && misalignment % 4 == 0)
     return true;
   return default_builtin_support_vector_misalignment (mode, type, misalignment,
-						      is_packed);
+						      is_packed,
+						      is_gather_scatter);
 }
 
 /* STRUCTURE_SIZE_BOUNDARY seems a bit crude in how it enlarges small
diff --git a/gcc/config/gcn/gcn-opts.h b/gcc/config/gcn/gcn-opts.h
index 0bfc786..0287400 100644
--- a/gcc/config/gcn/gcn-opts.h
+++ b/gcc/config/gcn/gcn-opts.h
@@ -82,11 +82,18 @@ enum hsaco_attr_type
 #define TARGET_DPP_FULL !TARGET_RDNA2_PLUS
 #define TARGET_DPP16 TARGET_RDNA2_PLUS
 #define TARGET_DPP8 TARGET_RDNA2_PLUS
+/* Device requires no manually inserted wait states; that's the
+   case for RDNA 2, 3 and 3.5 (but not for RNDA 4).  */
+#define TARGET_NO_MANUAL_NOPS TARGET_RDNA2_PLUS
 /* Device requires CDNA1-style manually inserted wait states for AVGPRs.  */
 #define TARGET_AVGPR_CDNA1_NOPS TARGET_CDNA1
+/* Device requires CDNA3-style manually inserted wait states.  */
+#define TARGET_CDNA3_NOPS TARGET_CDNA3
 /* Whether to use the 'globally coherent' (glc) or the 'scope' (sc0) flag
    for non-scalar memory operations. The string starts on purpose with a space.
    Note: for scalar memory operations (i.e. 's_...'), 'glc' is still used.
+   Note: on atomics, glc/sc0 denotes whether the pre-op operation should
+   be used.
    CDNA3 also uses 'nt' instead of 'slc' and 'sc1' instead of 'scc'; however,
    there is no non-scalar user so far.  */
 #define TARGET_GLC_NAME (TARGET_CDNA3 ? " sc0" : " glc")
diff --git a/gcc/config/gcn/gcn-valu.md b/gcc/config/gcn/gcn-valu.md
index 4b21302..a34d2e3 100644
--- a/gcc/config/gcn/gcn-valu.md
+++ b/gcc/config/gcn/gcn-valu.md
@@ -811,7 +811,7 @@
   [(set_attr "type" "vop3a")
    (set_attr "length" "8")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "write")])
 
 ; FIXME: 64bit operations really should be splitters, but I am not sure how
 ; to represent vertical subregs.
@@ -828,7 +828,7 @@
   [(set_attr "type" "vmult")
    (set_attr "length" "16")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "write")])
 
 (define_expand "vec_set<mode>"
   [(set (match_operand:V_MOV 0 "register_operand")
@@ -854,7 +854,7 @@
   [(set_attr "type" "vop3a")
    (set_attr "length" "8")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "write")])
 
 (define_insn "*vec_set<mode>_1"
   [(set (match_operand:V_2REG 0 "register_operand"		   "=v")
@@ -871,7 +871,7 @@
   [(set_attr "type" "vmult")
    (set_attr "length" "16")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "write")])
 
 (define_insn "vec_duplicate<mode><exec>"
   [(set (match_operand:V_1REG 0 "register_operand"	   "=v")
@@ -910,7 +910,7 @@
   [(set_attr "type" "vop3a")
    (set_attr "length" "8")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "read")])
 
 (define_insn "vec_extract<mode><scalar_mode>"
   [(set (match_operand:<SCALAR_MODE> 0 "register_operand"  "=&Sg")
@@ -922,7 +922,7 @@
   [(set_attr "type" "vmult")
    (set_attr "length" "16")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "read")])
 
 (define_insn "vec_extract<mode><scalar_mode>"
   [(set (match_operand:<SCALAR_MODE> 0 "register_operand"  "=&Sg")
@@ -934,7 +934,7 @@
   [(set_attr "type" "vmult")
    (set_attr "length" "32")
    (set_attr "exec" "none")
-   (set_attr "laneselect" "yes")])
+   (set_attr "laneselect" "read")])
 
 (define_insn "vec_extract<V_1REG:mode><V_1REG_ALT:mode>_nop"
   [(set (match_operand:V_1REG_ALT 0 "register_operand" "=v,v")
@@ -1133,6 +1133,23 @@
     DONE;
   })
 
+(define_expand "gather_load<mode><vndi>"
+  [(match_operand:V_MOV 0 "register_operand")
+   (match_operand:DI 1 "register_operand")
+   (match_operand:<VnDI> 2 "register_operand")
+   (match_operand 3 "immediate_operand")
+   (match_operand:SI 4 "gcn_alu_operand")]
+  ""
+  {
+    rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[1],
+					  operands[2], operands[4],
+					  INTVAL (operands[3]), NULL);
+
+    emit_insn (gen_gather<mode>_insn_1offset (operands[0], addr, const0_rtx,
+					      const0_rtx, const0_rtx));
+    DONE;
+  })
+
 ; Allow any address expression
 (define_expand "gather<mode>_expr<exec>"
   [(set (match_operand:V_MOV 0 "register_operand")
@@ -1175,6 +1192,7 @@
     return buf;
   }
   [(set_attr "type" "flat")
+   (set_attr "flatmemaccess" "load")
    (set_attr "length" "12")
    (set_attr "cdna" "*,cdna2,*,cdna2")
    (set_attr "xnack" "off,off,on,on")])
@@ -1233,6 +1251,7 @@
     return buf;
   }
   [(set_attr "type" "flat")
+   (set_attr "flatmemaccess" "load")
    (set_attr "length" "12")
    (set_attr "cdna" "*,cdna2,*,cdna2")
    (set_attr "xnack" "off,off,on,on")])
@@ -1259,6 +1278,23 @@
     DONE;
   })
 
+(define_expand "scatter_store<mode><vndi>"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:<VnDI> 1 "register_operand")
+   (match_operand 2 "immediate_operand")
+   (match_operand:SI 3 "gcn_alu_operand")
+   (match_operand:V_MOV 4 "register_operand")]
+  ""
+  {
+    rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[0],
+					  operands[1], operands[3],
+					  INTVAL (operands[2]), NULL);
+
+    emit_insn (gen_scatter<mode>_insn_1offset (addr, const0_rtx, operands[4],
+					       const0_rtx, const0_rtx));
+    DONE;
+  })
+
 ; Allow any address expression
 (define_expand "scatter<mode>_expr<exec_scatter>"
   [(set (mem:BLK (scratch))
@@ -1301,6 +1337,7 @@
     return buf;
   }
   [(set_attr "type" "flat")
+   (set_attr "flatmemaccess" "store")
    (set_attr "length" "12")
    (set_attr "cdna" "*,cdna2")])
 
@@ -1356,6 +1393,7 @@
     return buf;
   }
   [(set_attr "type" "flat")
+   (set_attr "flatmemaccess" "store")
    (set_attr "length" "12")
    (set_attr "cdna" "*,cdna2")])
 
@@ -1455,28 +1493,26 @@
 ;; }}}
 ;; {{{ ALU special case: add/sub
 
-(define_insn "add<mode>3<exec_clobber>"
+(define_insn "add<mode>3<exec>"
   [(set (match_operand:V_INT_1REG 0 "register_operand")
 	(plus:V_INT_1REG
 	  (match_operand:V_INT_1REG 1 "register_operand")
-	  (match_operand:V_INT_1REG 2 "gcn_alu_operand")))
-   (clobber (reg:DI VCC_REG))]
+	  (match_operand:V_INT_1REG 2 "gcn_alu_operand")))]
   ""
   {@ [cons: =0, %1, 2; attrs: type, length]
-  [v,v,vSvA;vop2,4] v_add_co_u32\t%0, vcc, %2, %1
+  [v,v,vSvA;vop2,4] {v_add_u32|v_add_nc_u32}\t%0, %2, %1
   [v,v,vSvB;vop2,8] ^
   })
 
-(define_insn "add<mode>3_dup<exec_clobber>"
+(define_insn "add<mode>3_dup<exec>"
   [(set (match_operand:V_INT_1REG 0 "register_operand")
 	(plus:V_INT_1REG
 	  (vec_duplicate:V_INT_1REG
 	    (match_operand:<SCALAR_MODE> 2 "gcn_alu_operand"))
-	  (match_operand:V_INT_1REG 1 "register_operand")))
-   (clobber (reg:DI VCC_REG))]
+	  (match_operand:V_INT_1REG 1 "register_operand")))]
   ""
   {@ [cons: =0, 1, 2; attrs: type, length]
-  [v,v,SvA;vop2,4] v_add_co_u32\t%0, vcc, %2, %1
+  [v,v,SvA;vop2,4] {v_add_u32|v_add_nc_u32}\t%0, %2, %1
   [v,v,SvB;vop2,8] ^
   })
 
@@ -1503,16 +1539,16 @@
 	(plus:V_SI
 	  (vec_duplicate:V_SI
 	    (match_operand:SI 1 "gcn_alu_operand"))
-	  (match_operand:V_SI 2 "register_operand")))
+	  (match_operand:V_SI 2 "gcn_alu_operand")))
    (set (match_operand:DI 3 "register_operand")
-	(ltu:DI (plus:V_SI (vec_duplicate:V_SI (match_dup 2))
-			   (match_dup 1))
-		(vec_duplicate:V_SI (match_dup 2))))]
+	(ltu:DI (plus:V_SI (vec_duplicate:V_SI (match_dup 1))
+			   (match_dup 2))
+		(match_dup 2)))]
   ""
   {@ [cons: =0, 1, 2, =3; attrs: type, length]
-  [v,SvA,v,cV;vop2 ,4] v_add_co_u32\t%0, %3, %1, %2
-  [v,SvB,v,cV;vop2 ,8] ^
-  [v,SvA,v,Sg;vop3b,8] ^
+  [v,SvA,vA,cV;vop2 ,4] v_add_co_u32\t%0, %3, %1, %2
+  [v,SvB,vA,cV;vop2 ,8] ^
+  [v,SvA,vA,Sg;vop3b,8] ^
   })
 
 ; v_addc does not accept an SGPR because the VCC read already counts as an
@@ -1551,16 +1587,15 @@
   [(set_attr "type" "vop2,vop3b")
    (set_attr "length" "4,8")])
 
-(define_insn "sub<mode>3<exec_clobber>"
+(define_insn "sub<mode>3<exec>"
   [(set (match_operand:V_INT_1REG 0 "register_operand"  "=  v,   v")
 	(minus:V_INT_1REG
 	  (match_operand:V_INT_1REG 1 "gcn_alu_operand" "vSvB,   v")
-	  (match_operand:V_INT_1REG 2 "gcn_alu_operand" "   v,vSvB")))
-   (clobber (reg:DI VCC_REG))]
+	  (match_operand:V_INT_1REG 2 "gcn_alu_operand" "   v,vSvB")))]
   ""
   "@
-   v_sub_co_u32\t%0, vcc, %1, %2
-   v_subrev_co_u32\t%0, vcc, %2, %1"
+   {v_sub_u32|v_sub_nc_u32}\t%0, %1, %2
+   {v_subrev_u32|v_subrev_nc_u32}\t%0, %2, %1"
   [(set_attr "type" "vop2")
    (set_attr "length" "8,8")])
 
@@ -1648,6 +1683,39 @@
   [(set_attr "type" "vmult")
    (set_attr "length" "8")])
 
+(define_insn_and_split "add<mode>3_dup"
+  [(set (match_operand:V_DI 0 "register_operand" "= v")
+	(plus:V_DI
+	  (vec_duplicate:V_DI
+	    (match_operand:DI 1 "register_operand"   "SvB"))
+	  (match_operand:V_DI 2 "gcn_alu_operand"    "vDb")))
+   (clobber (reg:DI VCC_REG))
+   (clobber (match_scratch:<VnSI> 3 "=&v"))]
+  ""
+  "#"
+  "gcn_can_split_p  (<MODE>mode, operands[0])
+   && gcn_can_split_p (<MODE>mode, operands[1])
+   && gcn_can_split_p (<MODE>mode, operands[2])"
+  [(const_int 0)]
+  {
+    rtx vcc = gen_rtx_REG (DImode, VCC_REG);
+    emit_insn (gen_add<vnsi>3_vcc_dup
+		(gcn_operand_part (<MODE>mode, operands[0], 0),
+		 gcn_operand_part (DImode, operands[1], 0),
+		 gcn_operand_part (<MODE>mode, operands[2], 0),
+		 vcc));
+    emit_insn (gen_vec_duplicate<vnsi> (operands[3],
+				  gcn_operand_part (DImode, operands[1], 1)));
+    emit_insn (gen_addc<vnsi>3
+		(gcn_operand_part (<MODE>mode, operands[0], 1),
+		 operands[3],
+		 gcn_operand_part (<MODE>mode, operands[2], 1),
+		 vcc, vcc));
+    DONE;
+  }
+  [(set_attr "type" "vmult")
+   (set_attr "length" "8")])
+
 (define_insn_and_split "add<mode>3_exec"
   [(set (match_operand:V_DI 0 "register_operand"		 "=  v")
 	(vec_merge:V_DI
@@ -1685,6 +1753,49 @@
   [(set_attr "type" "vmult")
    (set_attr "length" "8")])
 
+(define_insn_and_split "add<mode>3_dup_exec"
+  [(set (match_operand:V_DI 0 "register_operand"		     "= v")
+	(vec_merge:V_DI
+	  (plus:V_DI
+	    (vec_duplicate:V_DI
+	      (match_operand:DI 1 "register_operand"		     "SvB"))
+	    (match_operand:V_DI 2 "gcn_alu_operand"		         "vDb"))
+	  (match_operand:V_DI 3 "gcn_register_or_unspec_operand" " U0")
+	  (match_operand:DI 4 "gcn_exec_reg_operand"		     "  e")))
+   (clobber (reg:DI VCC_REG))
+   (clobber (match_scratch:<VnSI> 5 "=&v"))]
+  ""
+  "#"
+  "gcn_can_split_p  (<MODE>mode, operands[0])
+   && gcn_can_split_p (<MODE>mode, operands[1])
+   && gcn_can_split_p (<MODE>mode, operands[2])
+   && gcn_can_split_p (<MODE>mode, operands[4])"
+  [(const_int 0)]
+  {
+    rtx vcc = gen_rtx_REG (DImode, VCC_REG);
+    emit_insn (gen_add<vnsi>3_vcc_dup_exec
+		(gcn_operand_part (<MODE>mode, operands[0], 0),
+		 gcn_operand_part (DImode, operands[1], 0),
+		 gcn_operand_part (<MODE>mode, operands[2], 0),
+		 vcc,
+		 gcn_operand_part (<MODE>mode, operands[3], 0),
+		 operands[4]));
+    emit_insn (gen_vec_duplicate<vnsi>_exec (operands[5],
+				gcn_operand_part (DImode, operands[1], 1),
+				gcn_gen_undef (<VnSI>mode),
+				operands[4]));
+    emit_insn (gen_addc<vnsi>3_exec
+		(gcn_operand_part (<MODE>mode, operands[0], 1),
+		 operands[5],
+		 gcn_operand_part (<MODE>mode, operands[2], 1),
+		 vcc, vcc,
+		 gcn_operand_part (<MODE>mode, operands[3], 1),
+		 operands[4]));
+    DONE;
+  }
+  [(set_attr "type" "vmult")
+   (set_attr "length" "8")])
+
 (define_insn_and_split "sub<mode>3"
   [(set (match_operand:V_DI 0 "register_operand"  "= v,  v")
 	(minus:V_DI                                        
@@ -1827,7 +1938,7 @@
 	(ltu:DI (plus:V_DI 
 		  (zero_extend:V_DI (vec_duplicate:<VnSI> (match_dup 1)))
 		  (match_dup 2))
-		(match_dup 1)))]
+		(match_dup 2)))]
   ""
   {@ [cons: =0, 1, 2, =3]
   [v,ASv,v,&Sg] #
@@ -1878,7 +1989,7 @@
 	  (ltu:DI (plus:V_DI 
 		    (zero_extend:V_DI (vec_duplicate:<VnSI> (match_dup 1)))
 		    (match_dup 2))
-		  (match_dup 1))
+		  (match_dup 2))
 	  (match_dup 5)))]
   ""
   {@ [cons: =0, 1, 2, =3, 4, 5]
@@ -1932,7 +2043,7 @@
 	(ltu:DI (plus:V_DI 
 		  (zero_extend:V_DI (match_dup 1))
 		  (vec_duplicate:V_DI (match_dup 2)))
-		(match_dup 1)))]
+		(vec_duplicate:V_DI (match_dup 2))))]
   ""
   {@ [cons: =0, 1, 2, =3]
   [v,v,DbSv,&cV] #
@@ -1981,7 +2092,7 @@
 	  (ltu:DI (plus:V_DI 
 		    (zero_extend:V_DI (match_dup 1))
 		    (vec_duplicate:V_DI (match_dup 2)))
-		  (match_dup 1))
+		  (vec_duplicate:V_DI (match_dup 2)))
 	  (match_dup 5)))]
   ""
   {@ [cons: =0, 1, 2, =3, 4, 5]
@@ -2190,6 +2301,22 @@
   [(set_attr "type" "vop3a")
    (set_attr "length" "8")])
 
+(define_insn "<su>mul<mode>3_highpart_dup<exec>"
+  [(set (match_operand:V_SI 0 "register_operand" "= v")
+	(truncate:V_SI
+	  (lshiftrt:<VnDI>
+	    (mult:<VnDI>
+	      (any_extend:<VnDI>
+		(vec_duplicate:V_SI
+		  (match_operand:SI 1 "gcn_alu_operand"  "SvA")))
+	      (any_extend:<VnDI>
+		(match_operand:V_SI 2 "gcn_alu_operand"  " vA")))
+	    (const_int 32))))]
+  ""
+  "v_mul_hi<sgnsuffix>0\t%0, %2, %1"
+  [(set_attr "type" "vop3a")
+   (set_attr "length" "8")])
+
 (define_insn "mul<mode>3<exec>"
   [(set (match_operand:V_INT_1REG 0 "register_operand"  "=   v")
 	(mult:V_INT_1REG
@@ -2201,11 +2328,11 @@
    (set_attr "length" "8")])
 
 (define_insn "mul<mode>3_dup<exec>"
-  [(set (match_operand:V_INT_1REG 0 "register_operand"	     "=   v")
+  [(set (match_operand:V_INT_1REG 0 "register_operand"	 "= v")
 	(mult:V_INT_1REG
-	  (match_operand:V_INT_1REG 1 "gcn_alu_operand"	     "%vSvA")
 	  (vec_duplicate:V_INT_1REG
-	    (match_operand:<SCALAR_MODE> 2 "gcn_alu_operand" "  SvA"))))]
+	    (match_operand:<SCALAR_MODE> 1 "gcn_alu_operand" "SvA"))
+	  (match_operand:V_INT_1REG 2 "gcn_alu_operand"	     " vA")))]
   ""
   "v_mul_lo_u32\t%0, %1, %2"
   [(set_attr "type" "vop3a")
@@ -2241,6 +2368,37 @@
     DONE;
   })
 
+(define_insn_and_split "mul<mode>3_dup"
+  [(set (match_operand:V_DI 0 "register_operand" "=&v")
+	(mult:V_DI
+	  (vec_duplicate:V_DI
+	    (match_operand:DI 1 "gcn_alu_operand"    " Sv"))
+	  (match_operand:V_DI 2 "gcn_alu_operand"    "vDA")))
+   (clobber (match_scratch:<VnSI> 3 "=&v"))]
+  ""
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+  {
+    rtx out_lo = gcn_operand_part (<MODE>mode, operands[0], 0);
+    rtx out_hi = gcn_operand_part (<MODE>mode, operands[0], 1);
+    rtx left_lo = gcn_operand_part (DImode, operands[1], 0);
+    rtx left_hi = gcn_operand_part (DImode, operands[1], 1);
+    rtx right_lo = gcn_operand_part (<MODE>mode, operands[2], 0);
+    rtx right_hi = gcn_operand_part (<MODE>mode, operands[2], 1);
+    rtx tmp = operands[3];
+
+    emit_insn (gen_mul<vnsi>3_dup (out_lo, left_lo, right_lo));
+    emit_insn (gen_umul<vnsi>3_highpart_dup (out_hi, left_lo, right_lo));
+    emit_insn (gen_mul<vnsi>3_dup (tmp, left_hi, right_lo));
+    emit_insn (gen_add<vnsi>3 (out_hi, out_hi, tmp));
+    emit_insn (gen_mul<vnsi>3_dup (tmp, left_lo, right_hi));
+    emit_insn (gen_add<vnsi>3 (out_hi, out_hi, tmp));
+    emit_insn (gen_mul<vnsi>3_dup (tmp, left_hi, right_hi));
+    emit_insn (gen_add<vnsi>3 (out_hi, out_hi, tmp));
+    DONE;
+  })
+
 (define_insn_and_split "mul<mode>3_exec"
   [(set (match_operand:V_DI 0 "register_operand"		 "=&v")
 	(vec_merge:V_DI
@@ -2289,6 +2447,56 @@
     DONE;
   })
 
+(define_insn_and_split "mul<mode>3_dup_exec"
+  [(set (match_operand:V_DI 0 "register_operand"		 "=&v")
+	(vec_merge:V_DI
+	  (mult:V_DI
+	    (vec_duplicate:V_DI
+	      (match_operand:DI 1 "gcn_alu_operand"		 " Sv"))
+	    (match_operand:V_DI 2 "gcn_alu_operand"		 "vDA"))
+	  (match_operand:V_DI 3 "gcn_register_or_unspec_operand" " U0")
+	  (match_operand:DI 4 "gcn_exec_reg_operand"		 "  e")))
+   (clobber (match_scratch:<VnSI> 5 "=&v"))]
+  ""
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+  {
+    rtx out_lo = gcn_operand_part (<MODE>mode, operands[0], 0);
+    rtx out_hi = gcn_operand_part (<MODE>mode, operands[0], 1);
+    rtx left_lo = gcn_operand_part (DImode, operands[1], 0);
+    rtx left_hi = gcn_operand_part (DImode, operands[1], 1);
+    rtx right_lo = gcn_operand_part (<MODE>mode, operands[2], 0);
+    rtx right_hi = gcn_operand_part (<MODE>mode, operands[2], 1);
+    rtx exec = operands[4];
+    rtx tmp = operands[5];
+
+    rtx old_lo, old_hi;
+    if (GET_CODE (operands[3]) == UNSPEC)
+      {
+	old_lo = old_hi = gcn_gen_undef (<VnSI>mode);
+      }
+    else
+      {
+	old_lo = gcn_operand_part (<MODE>mode, operands[3], 0);
+	old_hi = gcn_operand_part (<MODE>mode, operands[3], 1);
+      }
+
+    rtx undef = gcn_gen_undef (<VnSI>mode);
+
+    emit_insn (gen_mul<vnsi>3_dup_exec (out_lo, left_lo, right_lo, old_lo,
+					exec));
+    emit_insn (gen_umul<vnsi>3_highpart_dup_exec (out_hi, left_lo, right_lo,
+						  old_hi, exec));
+    emit_insn (gen_mul<vnsi>3_dup_exec (tmp, left_hi, right_lo, undef, exec));
+    emit_insn (gen_add<vnsi>3_exec (out_hi, out_hi, tmp, out_hi, exec));
+    emit_insn (gen_mul<vnsi>3_dup_exec (tmp, left_lo, right_hi, undef, exec));
+    emit_insn (gen_add<vnsi>3_exec (out_hi, out_hi, tmp, out_hi, exec));
+    emit_insn (gen_mul<vnsi>3_dup_exec (tmp, left_hi, right_hi, undef, exec));
+    emit_insn (gen_add<vnsi>3_exec (out_hi, out_hi, tmp, out_hi, exec));
+    DONE;
+  })
+
 (define_insn_and_split "mul<mode>3_zext"
   [(set (match_operand:V_DI 0 "register_operand"      "=&v")
 	(mult:V_DI
@@ -3056,7 +3264,8 @@
   "flag_unsafe_math_optimizations"
   "v_sqrt%i0\t%0, %1"
   [(set_attr "type" "vop1")
-   (set_attr "length" "8")])
+   (set_attr "length" "8")
+   (set_attr "transop" "yes")])
 
 (define_insn "sqrt<mode>2"
   [(set (match_operand:FP 0 "register_operand"  "=  v")
@@ -3065,7 +3274,8 @@
   "flag_unsafe_math_optimizations"
   "v_sqrt%i0\t%0, %1"
   [(set_attr "type" "vop1")
-   (set_attr "length" "8")])
+   (set_attr "length" "8")
+   (set_attr "transop" "yes")])
 
 ; These FP unops have f64, f32 and f16 versions.
 (define_int_iterator MATH_UNOP_1OR2REG
@@ -3355,7 +3565,8 @@
   ""
   "v_rcp%i0\t%0, %1"
   [(set_attr "type" "vop1")
-   (set_attr "length" "8")])
+   (set_attr "length" "8")
+   (set_attr "transop" "yes")])
 
 ;; v_div_scale takes a numerator (op2) and denominator (op1) and returns the
 ;; one that matches op3 adjusted for best results in reciprocal division.
@@ -3727,6 +3938,7 @@
    v_cmpx%E1\t%2, %3
    v_cmpx%E1\t%2, %3"
   [(set_attr "type" "vopc,vopc,vopc,vopc,vop3a,vop3a,vopc,vopc")
+   (set_attr "vcmp" "vcmp,vcmp,vcmpx,vcmpx,vcmp,vcmp,vcmpx,vcmpx")
    (set_attr "length" "4,8,4,8,8,8,4,8")
    (set_attr "rdna" "*,*,no,no,*,*,yes,yes")])
 
@@ -3781,6 +3993,7 @@
    v_cmpx%E1\t%2, %3
    v_cmpx%E1\t%2, %3"
   [(set_attr "type" "vopc,vopc,vopc,vopc,vop3a,vop3a,vopc,vopc")
+   (set_attr "vcmp" "vcmp,vcmp,vcmpx,vcmpx,vcmp,vcmp,vcmpx,vcmpx")
    (set_attr "length" "4,8,4,8,8,8,4,8")
    (set_attr "rdna" "*,*,no,no,*,*,yes,yes")])
 
@@ -3795,9 +4008,9 @@
     /* Unsigned comparisons use the same patterns as signed comparisons,
        except that they use unsigned operators (e.g. LTU vs LT).
        The '%E1' directive then does the Right Thing.  */
-    emit_insn (gen_vec_cmpu<mode>di_exec (operands[0], operands[1],
-					  operands[2], operands[3],
-					  operands[4]));
+    emit_insn (gen_vec_cmp<mode>di_exec (operands[0], operands[1],
+					 operands[2], operands[3],
+					 operands[4]));
     DONE;
   })
 
@@ -3839,6 +4052,7 @@
    v_cmpx%E1\t%2, %3
    v_cmpx%E1\t%2, %3"
   [(set_attr "type" "vopc,vopc,vopc,vopc,vop3a,vopc,vopc")
+   (set_attr "vcmp" "vcmp,vcmp,vcmpx,vcmpx,vcmp,vcmpx,vcmpx")
    (set_attr "length" "4,8,4,8,8,4,8")
    (set_attr "rdna" "*,*,no,no,*,yes,yes")])
 
@@ -3862,6 +4076,7 @@
    v_cmpx%E1\t%2, %3
    v_cmpx%E1\t%2, %3"
   [(set_attr "type" "vopc,vopc,vopc,vopc,vop3a,vopc,vopc")
+   (set_attr "vcmp" "vcmp,vcmp,vcmpx,vcmpx,vcmp,vcmpx,vcmpx")
    (set_attr "length" "4,8,4,8,8,4,8")
    (set_attr "rdna" "*,*,no,no,*,yes,yes")])
 
@@ -4052,6 +4267,32 @@
     DONE;
   })
 
+(define_expand "mask_gather_load<mode><vndi>"
+  [(set:V_MOV (match_operand:V_MOV 0 "register_operand")
+	      (unspec:V_MOV
+		[(match_operand:DI 1 "register_operand")
+		 (match_operand:<VnDI> 2 "register_operand")
+		 (match_operand 3 "immediate_operand")
+		 (match_operand:SI 4 "gcn_alu_operand")
+		 (match_operand:DI 5 "")
+		 (match_operand:V_MOV 6 "maskload_else_operand")]
+		UNSPEC_GATHER))]
+  ""
+  {
+    rtx exec = force_reg (DImode, operands[5]);
+
+    rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[1],
+					  operands[2], operands[4],
+					  INTVAL (operands[3]), exec);
+
+    emit_insn (gen_gather<mode>_insn_1offset_exec (operands[0], addr,
+						   const0_rtx, const0_rtx,
+						   const0_rtx,
+						   gcn_gen_undef (<MODE>mode),
+						   exec));
+    DONE;
+  })
+
 (define_expand "mask_scatter_store<mode><vnsi>"
   [(match_operand:DI 0 "register_operand")
    (match_operand:<VnSI> 1 "register_operand")
@@ -4080,6 +4321,27 @@
     DONE;
   })
 
+(define_expand "mask_scatter_store<mode><vndi>"
+  [(match_operand:DI 0 "register_operand")
+   (match_operand:<VnDI> 1 "register_operand")
+   (match_operand 2 "immediate_operand")
+   (match_operand:SI 3 "gcn_alu_operand")
+   (match_operand:V_MOV 4 "register_operand")
+   (match_operand:DI 5 "")]
+  ""
+  {
+    rtx exec = force_reg (DImode, operands[5]);
+
+    rtx addr = gcn_expand_scaled_offsets (DEFAULT_ADDR_SPACE, operands[0],
+					  operands[1], operands[3],
+					  INTVAL (operands[2]), exec);
+
+    emit_insn (gen_scatter<mode>_insn_1offset_exec (addr, const0_rtx,
+						    operands[4], const0_rtx,
+						    const0_rtx, exec));
+    DONE;
+  })
+
 (define_code_iterator cond_op [plus minus mult])
 
 (define_expand "cond_<expander><mode>"
@@ -4400,7 +4662,7 @@
     rtx tmp = gen_reg_rtx (<MODE>mode);
     rtx v1 = gen_rtx_REG (<MODE>mode, VGPR_REGNO (1));
 
-    emit_insn (gen_mul<mode>3_dup (tmp, v1, operands[2]));
+    emit_insn (gen_mul<mode>3_dup (tmp, operands[2], v1));
     emit_insn (gen_add<mode>3_dup (operands[0], tmp, operands[1]));
     DONE;
   })
diff --git a/gcc/config/gcn/gcn.cc b/gcc/config/gcn/gcn.cc
index 0ce5a29..5ffeb23 100644
--- a/gcc/config/gcn/gcn.cc
+++ b/gcc/config/gcn/gcn.cc
@@ -54,6 +54,7 @@
 #include "gimple.h"
 #include "cgraph.h"
 #include "case-cfn-macros.h"
+#include "opts.h"
 
 /* This file should be included last.  */
 #include "target-def.h"
@@ -183,6 +184,11 @@ gcn_option_override (void)
 
   if (flag_sram_ecc == HSACO_ATTR_DEFAULT)
     flag_sram_ecc = gcn_devices[gcn_arch].sramecc_default;
+
+  /* TODO: This seems to produce tighter loops, but the testsuites expects it
+     to be set to '2', so I'll leave it default for now.
+  SET_OPTION_IF_UNSET (&global_options, &global_options_set,
+		       param_vect_partial_vector_usage, 1);  */
 }
 
 /* }}}  */
@@ -1275,13 +1281,13 @@ gen_##PREFIX##vN##SUFFIX (PARAMS) \
 }
 
 #define GEN_VNM_NOEXEC(PREFIX, SUFFIX, PARAMS, ARGS) \
-GEN_VN_NOEXEC (PREFIX, qi##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN_NOEXEC (PREFIX, hi##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN_NOEXEC (PREFIX, hf##SUFFIX, A(PARAMS), A(ARGS)) \
+USE_QHF (GEN_VN_NOEXEC (PREFIX, qi##SUFFIX, A(PARAMS), A(ARGS))) \
+USE_QHF (GEN_VN_NOEXEC (PREFIX, hi##SUFFIX, A(PARAMS), A(ARGS))) \
+USE_QHF (GEN_VN_NOEXEC (PREFIX, hf##SUFFIX, A(PARAMS), A(ARGS))) \
 GEN_VN_NOEXEC (PREFIX, si##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN_NOEXEC (PREFIX, sf##SUFFIX, A(PARAMS), A(ARGS)) \
+USE_QHF (GEN_VN_NOEXEC (PREFIX, sf##SUFFIX, A(PARAMS), A(ARGS))) \
 GEN_VN_NOEXEC (PREFIX, di##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN_NOEXEC (PREFIX, df##SUFFIX, A(PARAMS), A(ARGS)) \
+USE_QHF (GEN_VN_NOEXEC (PREFIX, df##SUFFIX, A(PARAMS), A(ARGS))) \
 static rtx \
 gen_##PREFIX##vNm##SUFFIX (PARAMS) \
 { \
@@ -1289,13 +1295,13 @@ gen_##PREFIX##vNm##SUFFIX (PARAMS) \
   \
   switch (mode) \
     { \
-    case E_QImode: return gen_##PREFIX##vNqi##SUFFIX (ARGS); \
-    case E_HImode: return gen_##PREFIX##vNhi##SUFFIX (ARGS); \
-    case E_HFmode: return gen_##PREFIX##vNhf##SUFFIX (ARGS); \
+    USE_QHF (case E_QImode: return gen_##PREFIX##vNqi##SUFFIX (ARGS);) \
+    USE_QHF (case E_HImode: return gen_##PREFIX##vNhi##SUFFIX (ARGS);) \
+    USE_QHF (case E_HFmode: return gen_##PREFIX##vNhf##SUFFIX (ARGS);) \
     case E_SImode: return gen_##PREFIX##vNsi##SUFFIX (ARGS); \
-    case E_SFmode: return gen_##PREFIX##vNsf##SUFFIX (ARGS); \
+    USE_QHF (case E_SFmode: return gen_##PREFIX##vNsf##SUFFIX (ARGS);) \
     case E_DImode: return gen_##PREFIX##vNdi##SUFFIX (ARGS); \
-    case E_DFmode: return gen_##PREFIX##vNdf##SUFFIX (ARGS); \
+    USE_QHF (case E_DFmode: return gen_##PREFIX##vNdf##SUFFIX (ARGS);) \
     default: \
       break; \
     } \
@@ -1340,13 +1346,13 @@ gen_##PREFIX##vN##SUFFIX (PARAMS, rtx merge_src=NULL, rtx exec=NULL) \
 }
 
 #define GEN_VNM(PREFIX, SUFFIX, PARAMS, ARGS) \
-GEN_VN (PREFIX, qi##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN (PREFIX, hi##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN (PREFIX, hf##SUFFIX, A(PARAMS), A(ARGS)) \
+USE_QHF (GEN_VN (PREFIX, qi##SUFFIX, A(PARAMS), A(ARGS))) \
+USE_QHF (GEN_VN (PREFIX, hi##SUFFIX, A(PARAMS), A(ARGS))) \
+USE_QHF (GEN_VN (PREFIX, hf##SUFFIX, A(PARAMS), A(ARGS))) \
 GEN_VN (PREFIX, si##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN (PREFIX, sf##SUFFIX, A(PARAMS), A(ARGS)) \
+USE_QHF (GEN_VN (PREFIX, sf##SUFFIX, A(PARAMS), A(ARGS))) \
 GEN_VN (PREFIX, di##SUFFIX, A(PARAMS), A(ARGS)) \
-GEN_VN (PREFIX, df##SUFFIX, A(PARAMS), A(ARGS)) \
+USE_QHF (GEN_VN (PREFIX, df##SUFFIX, A(PARAMS), A(ARGS))) \
 USE_TI (GEN_VN (PREFIX, ti##SUFFIX, A(PARAMS), A(ARGS))) \
 static rtx \
 gen_##PREFIX##vNm##SUFFIX (PARAMS, rtx merge_src=NULL, rtx exec=NULL) \
@@ -1355,15 +1361,22 @@ gen_##PREFIX##vNm##SUFFIX (PARAMS, rtx merge_src=NULL, rtx exec=NULL) \
   \
   switch (mode) \
     { \
-    case E_QImode: return gen_##PREFIX##vNqi##SUFFIX (ARGS, merge_src, exec); \
-    case E_HImode: return gen_##PREFIX##vNhi##SUFFIX (ARGS, merge_src, exec); \
-    case E_HFmode: return gen_##PREFIX##vNhf##SUFFIX (ARGS, merge_src, exec); \
-    case E_SImode: return gen_##PREFIX##vNsi##SUFFIX (ARGS, merge_src, exec); \
-    case E_SFmode: return gen_##PREFIX##vNsf##SUFFIX (ARGS, merge_src, exec); \
-    case E_DImode: return gen_##PREFIX##vNdi##SUFFIX (ARGS, merge_src, exec); \
-    case E_DFmode: return gen_##PREFIX##vNdf##SUFFIX (ARGS, merge_src, exec); \
-    case E_TImode: \
-	USE_TI (return gen_##PREFIX##vNti##SUFFIX (ARGS, merge_src, exec);) \
+    USE_QHF (case E_QImode: \
+	return gen_##PREFIX##vNqi##SUFFIX (ARGS, merge_src, exec);) \
+    USE_QHF (case E_HImode: \
+	return gen_##PREFIX##vNhi##SUFFIX (ARGS, merge_src, exec);) \
+    USE_QHF (case E_HFmode: \
+	return gen_##PREFIX##vNhf##SUFFIX (ARGS, merge_src, exec);) \
+    case E_SImode: \
+	return gen_##PREFIX##vNsi##SUFFIX (ARGS, merge_src, exec); \
+    USE_QHF (case E_SFmode: \
+	return gen_##PREFIX##vNsf##SUFFIX (ARGS, merge_src, exec);) \
+    case E_DImode: \
+	return gen_##PREFIX##vNdi##SUFFIX (ARGS, merge_src, exec); \
+    USE_QHF (case E_DFmode: \
+	return gen_##PREFIX##vNdf##SUFFIX (ARGS, merge_src, exec);) \
+    USE_TI (case E_TImode: \
+	return gen_##PREFIX##vNti##SUFFIX (ARGS, merge_src, exec);) \
     default: \
       break; \
     } \
@@ -1372,7 +1385,8 @@ gen_##PREFIX##vNm##SUFFIX (PARAMS, rtx merge_src=NULL, rtx exec=NULL) \
   return NULL_RTX; \
 }
 
-/* These have TImode support.  */
+/* These support everything.  */
+#define USE_QHF(ARGS) ARGS
 #define USE_TI(ARGS) ARGS
 GEN_VNM (mov,, A(rtx dest, rtx src), A(dest, src))
 GEN_VNM (vec_duplicate,, A(rtx dest, rtx src), A(dest, src))
@@ -1382,6 +1396,7 @@ GEN_VNM (vec_duplicate,, A(rtx dest, rtx src), A(dest, src))
 #define USE_TI(ARGS)
 GEN_VNM (add,3, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
 GEN_VN (add,si3_dup, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
+GEN_VN (add,di3_dup, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
 GEN_VN (add,si3_vcc_dup, A(rtx dest, rtx src1, rtx src2, rtx vcc),
 	A(dest, src1, src2, vcc))
 GEN_VN (add,di3_sext_dup2, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
@@ -1393,15 +1408,20 @@ GEN_VN (add,di3_vcc_zext_dup2, A(rtx dest, rtx src1, rtx src2, rtx vcc),
 GEN_VN (addc,si3, A(rtx dest, rtx src1, rtx src2, rtx vccout, rtx vccin),
 	A(dest, src1, src2, vccout, vccin))
 GEN_VN (and,si3, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
-GEN_VN (ashl,si3, A(rtx dest, rtx src, rtx shift), A(dest, src, shift))
 GEN_VNM_NOEXEC (ds_bpermute,, A(rtx dest, rtx addr, rtx src, rtx exec),
 		A(dest, addr, src, exec))
 GEN_VNM (gather,_expr, A(rtx dest, rtx addr, rtx as, rtx vol),
 	 A(dest, addr, as, vol))
-GEN_VN (mul,si3_dup, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
 GEN_VN (sub,si3, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
 GEN_VN_NOEXEC (vec_series,si, A(rtx dest, rtx x, rtx c), A(dest, x, c))
 
+/* These do not have QI, HI, or any FP support.  */
+#undef USE_QHF
+#define USE_QHF(ARGS)
+GEN_VNM (ashl,3, A(rtx dest, rtx src, rtx shift), A(dest, src, shift))
+GEN_VNM (mul,3_dup, A(rtx dest, rtx src1, rtx src2), A(dest, src1, src2))
+
+#undef USE_QHF
 #undef USE_TI
 #undef GEN_VNM
 #undef GEN_VN
@@ -1995,8 +2015,8 @@ gcn_expand_vector_init (rtx op0, rtx vec)
   rtx addr = gen_reg_rtx (addrmode);
 
   int unit_size = GET_MODE_SIZE (GET_MODE_INNER (GET_MODE (op0)));
-  emit_insn (gen_mulvNsi3_dup (ramp, gen_rtx_REG (offsetmode, VGPR_REGNO (1)),
-			       GEN_INT (unit_size)));
+  emit_insn (gen_mulvNsi3_dup (ramp, GEN_INT (unit_size),
+			       gen_rtx_REG (offsetmode, VGPR_REGNO (1))));
 
   bool simple_repeat = true;
 
@@ -2293,36 +2313,46 @@ gcn_expand_scalar_to_vector_address (machine_mode mode, rtx exec, rtx mem,
 
    Return values.
      ADDR_SPACE_FLAT   - return VnDImode vector of absolute addresses.
-     ADDR_SPACE_GLOBAL - return VnSImode vector of offsets.  */
+     ADDR_SPACE_GLOBAL - return VnSImode vector of offsets.
+     64-bit offsets    - return VnDImode vector of absolute addresses. */
 
 rtx
 gcn_expand_scaled_offsets (addr_space_t as, rtx base, rtx offsets, rtx scale,
 			   bool unsigned_p, rtx exec)
 {
   int vf = GET_MODE_NUNITS (GET_MODE (offsets));
-  rtx tmpsi = gen_reg_rtx (VnMODE (vf, SImode));
-  rtx tmpdi = gen_reg_rtx (VnMODE (vf, DImode));
+  rtx scaled_offsets = gen_reg_rtx (GET_MODE (offsets));
+  rtx abs_addr = gen_reg_rtx (VnMODE (vf, DImode));
+  bool use_di = GET_MODE_INNER (GET_MODE (scaled_offsets)) == DImode;
 
   if (CONST_INT_P (scale)
       && INTVAL (scale) > 0
       && exact_log2 (INTVAL (scale)) >= 0)
-    emit_insn (gen_ashlvNsi3 (tmpsi, offsets,
-			      GEN_INT (exact_log2 (INTVAL (scale))),
-			      NULL, exec));
+    emit_insn (gen_ashlvNm3 (scaled_offsets, offsets,
+			     GEN_INT (exact_log2 (INTVAL (scale))),
+			     NULL, exec));
   else
-     emit_insn (gen_mulvNsi3_dup (tmpsi, offsets, scale, NULL, exec));
+     emit_insn (gen_mulvNm3_dup (scaled_offsets, scale, offsets, NULL, exec));
 
+  /* No instructions support DImode offsets.  */
+  if (use_di)
+    {
+      emit_insn (gen_addvNdi3_dup (abs_addr, base, scaled_offsets, NULL, exec));
+      return abs_addr;
+    }
   /* "Global" instructions do not support negative register offsets.  */
-  if (as == ADDR_SPACE_FLAT || !unsigned_p)
+  else if (as == ADDR_SPACE_FLAT || !unsigned_p)
     {
       if (unsigned_p)
-	 emit_insn (gen_addvNdi3_zext_dup2 (tmpdi, tmpsi, base, NULL, exec));
+	emit_insn (gen_addvNdi3_zext_dup2 (abs_addr, scaled_offsets, base,
+					   NULL, exec));
       else
-	 emit_insn (gen_addvNdi3_sext_dup2 (tmpdi, tmpsi, base, NULL, exec));
-      return tmpdi;
+	emit_insn (gen_addvNdi3_sext_dup2 (abs_addr, scaled_offsets, base,
+					   NULL, exec));
+      return abs_addr;
     }
   else if (as == ADDR_SPACE_GLOBAL)
-    return tmpsi;
+    return scaled_offsets;
 
   gcc_unreachable ();
 }
@@ -5315,8 +5345,12 @@ gcn_preferred_vector_alignment (const_tree type)
 static bool
 gcn_vectorize_support_vector_misalignment (machine_mode ARG_UNUSED (mode),
 					   const_tree type, int misalignment,
-					   bool is_packed)
+					   bool is_packed,
+					   bool is_gather_scatter)
 {
+  if (is_gather_scatter)
+    return true;
+
   if (is_packed)
     return false;
 
@@ -5761,6 +5795,16 @@ gcn_libc_has_function (enum function_class fn_class,
   return bsd_libc_has_function (fn_class, type);
 }
 
+/* Implement TARGET_VECTORIZE_PREFER_GATHER_SCATTER. */
+
+static bool
+gcn_prefer_gather_scatter (machine_mode ARG_UNUSED (mode),
+			   int ARG_UNUSED (scale),
+			   unsigned int ARG_UNUSED (group_size))
+{
+  return true;
+}
+
 /* }}}  */
 /* {{{ md_reorg pass.  */
 
@@ -6124,12 +6168,22 @@ gcn_md_reorg (void)
      detects the missed cases, and inserts the documented number of NOPs
      required for correct execution.  */
 
+  /* RDNA4 (not yet implemented) differs from RNDA 2/3/3.5 and requires some
+     s_nop, see 5.7 and esp. 5.7.2. in its ISA manual.
+     The assert here is a reminder to add those.  */
+  STATIC_ASSERT (ISA_CDNA1 - ISA_RDNA3 == 1);
+
+  if (TARGET_NO_MANUAL_NOPS)
+    return;
+
   const int max_waits = 5;
   struct ilist
   {
     rtx_insn *insn;
     attr_unit unit;
-    attr_delayeduse delayeduse;
+    attr_type type;
+    attr_flatmemaccess flatmemaccess;
+    bool delayeduse;
     HARD_REG_SET writes;
     HARD_REG_SET reads;
     int age;
@@ -6150,7 +6204,29 @@ gcn_md_reorg (void)
 
       attr_type itype = get_attr_type (insn);
       attr_unit iunit = get_attr_unit (insn);
-      attr_delayeduse idelayeduse = get_attr_delayeduse (insn);
+      attr_flatmemaccess iflatmemaccess = get_attr_flatmemaccess (insn);
+      bool delayeduse;
+      if (TARGET_CDNA3_NOPS)
+	switch (iflatmemaccess)
+	  {
+	  case FLATMEMACCESS_STORE:
+	  case FLATMEMACCESS_STOREX34:
+	  case FLATMEMACCESS_ATOMIC:
+	  case FLATMEMACCESS_CMPSWAPX2:
+	    delayeduse = true;
+	    break;
+	  case FLATMEMACCESS_LOAD:
+	  case FLATMEMACCESS_ATOMICWAIT:
+	  case FLATMEMACCESS_NO:
+	    delayeduse = false;
+	    break;
+	  default:
+	    gcc_unreachable ();
+	  }
+      else
+	delayeduse = (iflatmemaccess == FLATMEMACCESS_CMPSWAPX2
+		      || iflatmemaccess == FLATMEMACCESS_STOREX34);
+
       int ivccwait = get_attr_vccwait (insn);
       HARD_REG_SET ireads, iwrites;
       CLEAR_HARD_REG_SET (ireads);
@@ -6195,16 +6271,26 @@ gcn_md_reorg (void)
 		   && TEST_HARD_REG_BIT (ireads, VCCZ_REG))))
 	    nops_rqd = 5 - prev_insn->age;
 
-	  /* VALU writes SGPR/VCC followed by v_{read,write}lane using
-	     SGPR/VCC as lane select requires 4 wait states.  */
+	  /* VALU writes SGPR/VCC followed by
+	     - v_{read,write}lane using SGPR/VCC as lane select requires
+	       4 wait states
+	     - [CDNA3] VALU reads SGPR as constant requires 1 wait state
+	     - [CDNA3] VALU reads SGPR as carry-in requires no wait states  */
 	  if ((prev_insn->age + nops_rqd) < 4
 	      && prev_insn->unit == UNIT_VECTOR
-	      && get_attr_laneselect (insn) == LANESELECT_YES
+	      && get_attr_laneselect (insn) != LANESELECT_NO
 	      && (hard_reg_set_intersect_p
 		    (depregs, reg_class_contents[(int) SGPR_REGS])
 		  || hard_reg_set_intersect_p
 		       (depregs, reg_class_contents[(int) VCC_CONDITIONAL_REG])))
 	    nops_rqd = 4 - prev_insn->age;
+	  else if (TARGET_CDNA3_NOPS
+		   && (prev_insn->age + nops_rqd) < 1
+		   && prev_insn->unit == UNIT_VECTOR
+		   && iunit == UNIT_VECTOR
+		   && hard_reg_set_intersect_p
+			(depregs, reg_class_contents[(int) SGPR_REGS]))
+	    nops_rqd = 1 - prev_insn->age;
 
 	  /* VALU writes VGPR followed by VALU_DPP reading that VGPR
 	     requires 2 wait states.  */
@@ -6217,22 +6303,128 @@ gcn_md_reorg (void)
 		nops_rqd = 2 - prev_insn->age;
 	    }
 
+	  /* VALU writes EXEC followed by VALU DPP op requires 5 nop.  */
+	  if ((prev_insn->age + nops_rqd) < 5
+	      && itype == TYPE_VOP_DPP
+	      && prev_insn->unit == UNIT_VECTOR
+	      && TEST_HARD_REG_BIT (prev_insn->writes, EXECZ_REG))
+	    nops_rqd = 5 - prev_insn->age;
+
 	  /* Store that requires input registers are not overwritten by
-	     following instruction.  */
-	  if ((prev_insn->age + nops_rqd) < 1
-	      && prev_insn->delayeduse == DELAYEDUSE_YES
+	     following instruction.
+	     For CDNA3, only, VALU writes require 2 not 1 nop.
+	     CDNA3 additionally requires that 1 or 2 nop for global & scatch
+	     store/atomic.  */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 2
+	      && prev_insn->delayeduse
+	      && iunit == UNIT_VECTOR
+	      && ((hard_reg_set_intersect_p
+		   (prev_insn->reads, iwrites))))
+	    nops_rqd = 2 - prev_insn->age;
+	  else if ((prev_insn->age + nops_rqd) < 1
+	      && prev_insn->delayeduse
 	      && ((hard_reg_set_intersect_p
 		   (prev_insn->reads, iwrites))))
 	    nops_rqd = 1 - prev_insn->age;
 
-	  /* Instruction that requires VCC is not written too close before
-	     using it.  */
+	  /* Instruction (such as v_div_fmas) that requires VCC is not written
+	     too close before using it  */
 	  if (prev_insn->age < ivccwait
 	      && (hard_reg_set_intersect_p
 		  (prev_insn->writes,
 		   reg_class_contents[(int)VCC_CONDITIONAL_REG])))
 	    nops_rqd = ivccwait - prev_insn->age;
 
+	  /* NOTE: The following condition for adding wait state exists, but
+	     GCC does not access the special registers using their SGPR#.
+	     Thus, no action is required here.  The following wait-state
+	     condition exists at least for VEGA/gfx900+ to CDNA3:
+		Mixed use of VCC: alias vs. SGPR# - v_readlane,
+		v_readfirstlane, v_cmp, v_add_*i/u, v_sub_*i/u, v_div_*scale
+		followed by VALU reads VCC as constant requires 1 wait state.
+		(As carry-in, it requires none.)
+		[VCC can be accessed by name or logical SGPR that holds it.]  */
+
+	  /* Testing indicates that CDNA3 requires an s_nop between
+	     e.g. 'v_cmp_eq_u64 vcc, v[4:5], v[8:9]' and 'v_mov_b32 v0, vcc_lo'.
+	     Thus: add it between v_cmp writing VCC and VALU read of VCC.  */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 1
+	      && iunit == UNIT_VECTOR
+	      && (hard_reg_set_intersect_p
+		  (depregs, reg_class_contents[(int)VCC_CONDITIONAL_REG]))
+	      && get_attr_vcmp (prev_insn->insn) == VCMP_VCMP)
+	    nops_rqd = 1 - prev_insn->age;
+
+	  /* CDNA3: VALU writes SGPR/VCC: v_readlane, v_readfirstlane, v_cmp,
+	     v_add_*i/u, v_sub_*i/u, v_div_*scale - followed by:
+	     - VALU reads SGPR as constant requires 1 waite state
+	     - VALU reads SGPR as carry-in requires no waite state
+	     - v_readlane/v_writelane reads SGPR as lane select requires 4 wait
+	       states.  */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 4
+	      && iunit == UNIT_VECTOR
+	      && prev_insn->unit == UNIT_VECTOR
+	      && hard_reg_set_intersect_p
+		   (depregs, reg_class_contents[(int) SGPR_SRC_REGS]))
+	    {
+	      if (get_attr_laneselect (insn) != LANESELECT_NO)
+		nops_rqd = 4 - prev_insn->age;
+	      else if ((prev_insn->age + nops_rqd) < 1)
+		nops_rqd = 1 - prev_insn->age;
+	    }
+
+	  /* CDNA3: v_cmpx followed by
+	     - V_readlane, v_readfirstlane, v_writelane requires 4 wait states
+	     - VALU reads EXEC as constant requires 2 wait states
+	     - other VALU requires no wait state  */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 4
+	      && get_attr_vcmp (prev_insn->insn) == VCMP_VCMPX
+	      && get_attr_laneselect (insn) != LANESELECT_NO)
+	    nops_rqd = 4 - prev_insn->age;
+	  else if (TARGET_CDNA3_NOPS
+		   && (prev_insn->age + nops_rqd) < 2
+		   && iunit == UNIT_VECTOR
+		   && get_attr_vcmp (prev_insn->insn) == VCMP_VCMPX
+		   && TEST_HARD_REG_BIT (ireads, EXECZ_REG))
+	    nops_rqd = 2 - prev_insn->age;
+
+	  /* CDNA3: VALU writes VGPR followed by v_readlane vsrc0 reads VGPRn
+	     requires 1 wait state. */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 1
+	      && prev_insn->unit == UNIT_VECTOR
+	      && prev_insn->flatmemaccess != FLATMEMACCESS_LOAD
+	      && get_attr_laneselect (insn) == LANESELECT_READ
+	      && hard_reg_set_intersect_p
+		    (depregs, reg_class_contents[(int) VGPR_REGS]))
+	    nops_rqd = 1 - prev_insn->age;
+
+	  /* CDNA3: VALU op which uses OPSEL or SDWA with changes the result's
+	     bit position followed by VALU op consumes result of that op
+	     requires 1 wait state.
+	     FIXME: Handle OPSEL, once used.  */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 1
+	      && prev_insn->unit == UNIT_VECTOR
+	      && prev_insn->type == TYPE_VOP_SDWA
+	      && !hard_reg_set_empty_p (depregs))
+	    nops_rqd = 1 - prev_insn->age;
+
+	  /* CNDA3: VALU Trans Op (such as v_rcp_f64) followed by non-trans VALU
+	     op consumes result of that op requires 1 wait state.  */
+	  if (TARGET_CDNA3_NOPS
+	      && (prev_insn->age + nops_rqd) < 1
+	      && prev_insn->unit == UNIT_VECTOR
+	      && iunit == UNIT_VECTOR
+	      && get_attr_transop (prev_insn->insn) == TRANSOP_YES
+	      && get_attr_transop (insn) == TRANSOP_NO
+	      && !hard_reg_set_empty_p (depregs))
+	    nops_rqd = 1 - prev_insn->age;
+
 	  /* CDNA1: write VGPR before v_accvgpr_write reads it.  */
 	  if (TARGET_AVGPR_CDNA1_NOPS
 	      && (prev_insn->age + nops_rqd) < 2
@@ -6264,8 +6456,8 @@ gcn_md_reorg (void)
 	}
 
       /* Insert the required number of NOPs.  */
-      for (int i = nops_rqd; i > 0; i--)
-	emit_insn_after (gen_nop (), last_insn);
+      if (nops_rqd > 0)
+	emit_insn_after (gen_nops (GEN_INT (nops_rqd-1)), last_insn);
 
       /* Age the previous instructions.  We can also ignore writes to
          registers subsequently overwritten.  */
@@ -6288,7 +6480,9 @@ gcn_md_reorg (void)
       /* Track the current instruction as a previous instruction.  */
       back[oldest].insn = insn;
       back[oldest].unit = iunit;
-      back[oldest].delayeduse = idelayeduse;
+      back[oldest].type = itype;
+      back[oldest].flatmemaccess = iflatmemaccess;
+      back[oldest].delayeduse = delayeduse;
       back[oldest].writes = iwrites;
       back[oldest].reads = ireads;
       back[oldest].age = 0;
@@ -7109,6 +7303,11 @@ print_operand_address (FILE *file, rtx mem)
    H - print second part of a multi-reg value (high-part of 2-reg value)
    J - print third part of a multi-reg value
    K - print fourth part of a multi-reg value
+   R   Print a scalar register number as an integer.  Temporary hack.
+   V - Print a vector register number as an integer.  Temporary hack.
+
+   Additionally, the standard builtin c, n, a, and l exist; see gccint's
+   "Output Templates and Operand Substitution" for details.
  */
 
 void
@@ -7957,6 +8156,8 @@ gcn_dwarf_register_span (rtx rtl)
   gcn_vectorize_builtin_vectorized_function
 #undef  TARGET_VECTORIZE_GET_MASK_MODE
 #define TARGET_VECTORIZE_GET_MASK_MODE gcn_vectorize_get_mask_mode
+#undef  TARGET_VECTORIZE_PREFER_GATHER_SCATTER
+#define TARGET_VECTORIZE_PREFER_GATHER_SCATTER gcn_prefer_gather_scatter
 #undef  TARGET_VECTORIZE_PREFERRED_SIMD_MODE
 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE gcn_vectorize_preferred_simd_mode
 #undef  TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT
diff --git a/gcc/config/gcn/gcn.md b/gcc/config/gcn/gcn.md
index 2ce2e05..4130cf6 100644
--- a/gcc/config/gcn/gcn.md
+++ b/gcc/config/gcn/gcn.md
@@ -312,18 +312,33 @@
 ; We need to be able to identify v_readlane and v_writelane with
 ; SGPR lane selection in order to handle "Manually Inserted Wait States".
 
-(define_attr "laneselect" "yes,no" (const_string "no"))
+(define_attr "laneselect" "write,read,no" (const_string "no"))
 
-; Identify instructions that require a "Manually Inserted Wait State" if
-; their inputs are overwritten by subsequent instructions.
+; Global or flat memory access using store or load followed by waitcnt
+; and using flat/global atomic access, possibly followed by a waitcnt.
+; 'storex34' denotes FLAT_STORE_X{3,4}.
+; 'cmpswapx2' denotes FLAT_ATOMIC_{F}CMPSWAP_X2
+; Used to handle "Manually Inserted Wait State".
 
-(define_attr "delayeduse" "yes,no" (const_string "no"))
+(define_attr "flatmemaccess"
+             "store,storex34,load,atomic,atomicwait,cmpswapx2,no"
+             (const_string "no"))
+
+; Identify v_cmp and v_cmpx instructions for "Manually Inserted Wait State"
+; handling.
+
+(define_attr "vcmp" "vcmp,vcmpx,no" (const_string "no"))
 
 ; Identify instructions that require "Manually Inserted Wait State" if
 ; a previous instruction writes to VCC.  The number gives the number of NOPs.
 
 (define_attr "vccwait" "" (const_int 0))
 
+; Mark trans ops such as v_{exp,rsq,sqrt,sin,cos,log,...}_F{16,32,64}
+; for later conditional s_nop insertion.
+
+(define_attr "transop" "yes,no" (const_string "no"))
+
 ;; }}}
 ;; {{{ Iterators useful across the wole machine description
 
@@ -414,6 +429,15 @@
   "s_nop\t0x0"
   [(set_attr "type" "sopp")])
 
+; Variant of 'nop' that accepts a count argument.
+; s_nop accepts 0x0 to 0xf for 1 to 16 nops; however,
+; as %0 prints decimals, only 0 to 9 (= 1 to 10 nops) can be used.
+(define_insn "nops"
+  [(match_operand 0 "const_int_operand")]
+  ""
+  "s_nop\t0x%0"
+  [(set_attr "type" "sopp")])
+
 ; FIXME: What should the value of the immediate be? Zero is disallowed, so
 ; pick 1 for now.
 (define_insn "trap"
@@ -555,9 +579,12 @@
   }
   [(set_attr "type" "sop1,vop1,vop3a,sopk,vopc,mult,smem,smem,smem,flat,flat,
 		     flat,flat,flat,flat")
+   (set_attr "flatmemaccess" "*,*,*,*,*,*,*,*,*,load,load,store,load,load,store")
+   (set_attr "vcmp" "*,*,*,*,vcmp,*,*,*,*,*,*,*,*,*,*")
    (set_attr "exec" "*,*,none,*,*,*,*,*,*,*,*,*,*,*,*")
    (set_attr "length" "4,4,4,4,4,8,12,12,12,12,12,12,12,12,12")
-   (set_attr "xnack" "*,*,*,*,*,*,off,on,*,off,on,*,off,on,*")])
+   (set_attr "xnack" "*,*,*,*,*,*,off,on,*,off,on,*,off,on,*")
+   (set_attr "laneselect" "*,*,read,*,*,*,*,*,*,*,*,*,*,*,*")])
 
 ; 32bit move pattern
 
@@ -565,38 +592,38 @@
   [(set (match_operand:SISF 0 "nonimmediate_operand")
 	(match_operand:SISF 1 "gcn_load_operand"))]
   ""
-  {@ [cons: =0, 1; attrs: type, exec, length, cdna, xnack]
-   [SD  ,SSA ;sop1 ,*   ,4 ,*    ,*  ] s_mov_b32\t%0, %1
-   [SD  ,J   ;sopk ,*   ,4 ,*    ,*  ] s_movk_i32\t%0, %1
-   [SD  ,B   ;sop1 ,*   ,8 ,*    ,*  ] s_mov_b32\t%0, %1
-   [SD  ,RB  ;smem ,*   ,12,*    ,off] s_buffer_load%s0\t%0, s[0:3], %1\;s_waitcnt\tlgkmcnt(0)
-   [&SD ,RB  ;smem ,*   ,12,*    ,on ] ^
-   [RB  ,Sm  ;smem ,*   ,12,*    ,*  ] s_buffer_store%s1\t%1, s[0:3], %0
-   [Sm  ,RS  ;smem ,*   ,12,*    ,off] s_load_dword\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
-   [&Sm ,RS  ;smem ,*   ,12,*    ,on ] ^
-   [RS  ,Sm  ;smem ,*   ,12,*    ,*  ] s_store_dword\t%1, %A0
-   [v   ,v   ;vop1 ,*   ,4 ,*    ,*  ] v_mov_b32\t%0, %1
-   [Sg  ,v   ;vop3a,none,8 ,*    ,*  ] v_readlane_b32\t%0, %1, 0
-   [v   ,Sv  ;vop3a,none,8 ,*    ,*  ] v_writelane_b32\t%0, %1, 0
-   [v   ,^a  ;vop3p_mai,*,8,*    ,*  ] v_accvgpr_read_b32\t%0, %1
-   [a   ,v   ;vop3p_mai,*,8,*    ,*  ] v_accvgpr_write_b32\t%0, %1
-   [a   ,a   ;vop1 ,*    ,4,cdna2,*  ] v_accvgpr_mov_b32\t%0, %1
-   [v   ,RF  ;flat ,*   ,12,*    ,off] flat_load_dword\t%0, %A1%O1%g1\;s_waitcnt\t0
-   [&v  ,RF  ;flat ,*   ,12,*    ,on ] ^
-   [^a  ,RF  ;flat ,*   ,12,cdna2,off] ^
-   [&^a ,RF  ;flat ,*   ,12,cdna2,on ] ^
-   [RF  ,v   ;flat ,*   ,12,*    ,*  ] flat_store_dword\t%A0, %1%O0%g0
-   [RF  ,a   ;flat ,*   ,12,cdna2,*  ] ^
-   [v   ,B   ;vop1 ,*   ,8 ,*    ,*  ] v_mov_b32\t%0, %1
-   [RLRG,v   ;ds   ,*   ,12,*    ,*  ] ds_write_b32\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
-   [v   ,RLRG;ds   ,*   ,12,*    ,*  ] ds_read_b32\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
-   [SD  ,Y   ;sop1 ,*   ,8 ,*    ,*  ] s_mov_b32\t%0, %1
-   [v   ,RM  ;flat ,*   ,12,*    ,off] global_load_dword\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
-   [&v  ,RM  ;flat ,*   ,12,*    ,on ] ^
-   [^a  ,RM  ;flat ,*   ,12,cdna2,off] ^
-   [&^a ,RM  ;flat ,*   ,12,cdna2,on ] ^
-   [RM  ,v   ;flat ,*   ,12,*    ,*  ] global_store_dword\t%A0, %1%O0%g0
-   [RM  ,a   ;flat ,*   ,12,cdna2,*  ] ^
+  {@ [cons: =0, 1; attrs: type, exec, length, cdna, xnack, laneselect, flatmemaccess]
+   [SD  ,SSA ;sop1 ,*   ,4 ,*    ,*  ,*    ,*    ] s_mov_b32\t%0, %1
+   [SD  ,J   ;sopk ,*   ,4 ,*    ,*  ,*    ,*    ] s_movk_i32\t%0, %1
+   [SD  ,B   ;sop1 ,*   ,8 ,*    ,*  ,*    ,*    ] s_mov_b32\t%0, %1
+   [SD  ,RB  ;smem ,*   ,12,*    ,off,*    ,*    ] s_buffer_load%s0\t%0, s[0:3], %1\;s_waitcnt\tlgkmcnt(0)
+   [&SD ,RB  ;smem ,*   ,12,*    ,on ,*    ,*    ] ^
+   [RB  ,Sm  ;smem ,*   ,12,*    ,*  ,*    ,*    ] s_buffer_store%s1\t%1, s[0:3], %0
+   [Sm  ,RS  ;smem ,*   ,12,*    ,off,*    ,*    ] s_load_dword\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
+   [&Sm ,RS  ;smem ,*   ,12,*    ,on ,*    ,*    ] ^
+   [RS  ,Sm  ;smem ,*   ,12,*    ,*  ,*    ,*    ] s_store_dword\t%1, %A0
+   [v   ,v   ;vop1 ,*   ,4 ,*    ,*  ,*    ,*    ] v_mov_b32\t%0, %1
+   [Sg  ,v   ;vop3a,none,8 ,*    ,*  ,read ,*    ] v_readlane_b32\t%0, %1, 0
+   [v   ,Sv  ;vop3a,none,8 ,*    ,*  ,write,*    ] v_writelane_b32\t%0, %1, 0
+   [v   ,^a  ;vop3p_mai,*,8,*    ,*  ,*    ,*    ] v_accvgpr_read_b32\t%0, %1
+   [a   ,v   ;vop3p_mai,*,8,*    ,*  ,*    ,*    ] v_accvgpr_write_b32\t%0, %1
+   [a   ,a   ;vop1 ,*    ,4,cdna2,*  ,*    ,*    ] v_accvgpr_mov_b32\t%0, %1
+   [v   ,RF  ;flat ,*   ,12,*    ,off,*    ,load ] flat_load_dword\t%0, %A1%O1%g1\;s_waitcnt\t0
+   [&v  ,RF  ;flat ,*   ,12,*    ,on ,*    ,load ] ^
+   [^a  ,RF  ;flat ,*   ,12,cdna2,off,*    ,load ] ^
+   [&^a ,RF  ;flat ,*   ,12,cdna2,on ,*    ,load ] ^
+   [RF  ,v   ;flat ,*   ,12,*    ,*  ,*    ,store] flat_store_dword\t%A0, %1%O0%g0
+   [RF  ,a   ;flat ,*   ,12,cdna2,*  ,*    ,store] ^
+   [v   ,B   ;vop1 ,*   ,8 ,*    ,*  ,*    ,*    ] v_mov_b32\t%0, %1
+   [RLRG,v   ;ds   ,*   ,12,*    ,*  ,*    ,*    ] ds_write_b32\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+   [v   ,RLRG;ds   ,*   ,12,*    ,*  ,*    ,*    ] ds_read_b32\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+   [SD  ,Y   ;sop1 ,*   ,8 ,*    ,*  ,*    ,*    ] s_mov_b32\t%0, %1
+   [v   ,RM  ;flat ,*   ,12,*    ,off,*    ,load ] global_load_dword\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
+   [&v  ,RM  ;flat ,*   ,12,*    ,on ,*    ,load ] ^
+   [^a  ,RM  ;flat ,*   ,12,cdna2,off,*    ,load ] ^
+   [&^a ,RM  ;flat ,*   ,12,cdna2,on ,*    ,load ] ^
+   [RM  ,v   ;flat ,*   ,12,*    ,*  ,*    ,store] global_store_dword\t%A0, %1%O0%g0
+   [RM  ,a   ;flat ,*   ,12,cdna2,*  ,*    ,store] ^
   })
 
 ; 8/16bit move pattern
@@ -606,31 +633,31 @@
   [(set (match_operand:QIHI 0 "nonimmediate_operand")
 	(match_operand:QIHI 1 "gcn_load_operand"))]
   "gcn_valid_move_p (<MODE>mode, operands[0], operands[1])"
-  {@ [cons: =0, 1; attrs: type, exec, length, cdna, xnack]
-  [SD  ,SSA ;sop1 ,*   ,4 ,*    ,*  ] s_mov_b32\t%0, %1
-  [SD  ,J   ;sopk ,*   ,4 ,*    ,*  ] s_movk_i32\t%0, %1
-  [SD  ,B   ;sop1 ,*   ,8 ,*    ,*  ] s_mov_b32\t%0, %1
-  [v   ,v   ;vop1 ,*   ,4 ,*    ,*  ] v_mov_b32\t%0, %1
-  [Sg  ,v   ;vop3a,none,4 ,*    ,*  ] v_readlane_b32\t%0, %1, 0
-  [v   ,Sv  ;vop3a,none,4 ,*    ,*  ] v_writelane_b32\t%0, %1, 0
-  [v   ,^a  ;vop3p_mai,*,8,*    ,*  ] v_accvgpr_read_b32\t%0, %1
-  [a   ,v   ;vop3p_mai,*,8,*    ,*  ] v_accvgpr_write_b32\t%0, %1
-  [a   ,a   ;vop1 ,*    ,8,cdna2,*  ] v_accvgpr_mov_b32\t%0, %1
-  [v   ,RF  ;flat ,*   ,12,*    ,off] flat_load%o1\t%0, %A1%O1%g1\;s_waitcnt\t0
-  [&v  ,RF  ;flat ,*   ,12,*    ,on ] ^
-  [^a  ,RF  ;flat ,*   ,12,cdna2,off] ^
-  [&^a ,RF  ;flat ,*   ,12,cdna2,on ] ^
-  [RF  ,v   ;flat ,*   ,12,*    ,*  ] flat_store%s0\t%A0, %1%O0%g0
-  [RF  ,a   ;flat ,*   ,12,cdna2,*  ] ^
-  [v   ,B   ;vop1 ,*   ,8 ,*    ,*  ] v_mov_b32\t%0, %1
-  [RLRG,v   ;ds   ,*   ,12,*    ,*  ] ds_write%b0\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
-  [v   ,RLRG;ds   ,*   ,12,*    ,*  ] ds_read%u1\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
-  [v   ,RM  ;flat ,*   ,12,*    ,off] global_load%o1\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
-  [&v  ,RM  ;flat ,*   ,12,*    ,on ] ^
-  [^a  ,RM  ;flat ,*   ,12,cdna2,off] ^
-  [&^a ,RM  ;flat ,*   ,12,cdna2,on ] ^
-  [RM  ,v   ;flat ,*   ,12,*    ,*  ] global_store%s0\t%A0, %1%O0%g0
-  [RM  ,a   ;flat ,*   ,12,cdna2,*  ] ^
+  {@ [cons: =0, 1; attrs: type, exec, length, cdna, xnack, laneselect, flatmemaccess]
+  [SD  ,SSA ;sop1 ,*   ,4 ,*    ,*  ,*    ,*    ] s_mov_b32\t%0, %1
+  [SD  ,J   ;sopk ,*   ,4 ,*    ,*  ,*    ,*    ] s_movk_i32\t%0, %1
+  [SD  ,B   ;sop1 ,*   ,8 ,*    ,*  ,*    ,*    ] s_mov_b32\t%0, %1
+  [v   ,v   ;vop1 ,*   ,4 ,*    ,*  ,*    ,*    ] v_mov_b32\t%0, %1
+  [Sg  ,v   ;vop3a,none,4 ,*    ,*  ,read ,*    ] v_readlane_b32\t%0, %1, 0
+  [v   ,Sv  ;vop3a,none,4 ,*    ,*  ,write,*    ] v_writelane_b32\t%0, %1, 0
+  [v   ,^a  ;vop3p_mai,*,8,*    ,*  ,*    ,*    ] v_accvgpr_read_b32\t%0, %1
+  [a   ,v   ;vop3p_mai,*,8,*    ,*  ,*    ,*    ] v_accvgpr_write_b32\t%0, %1
+  [a   ,a   ;vop1 ,*    ,8,cdna2,*  ,*    ,*    ] v_accvgpr_mov_b32\t%0, %1
+  [v   ,RF  ;flat ,*   ,12,*    ,off,*    ,load ] flat_load%o1\t%0, %A1%O1%g1\;s_waitcnt\t0
+  [&v  ,RF  ;flat ,*   ,12,*    ,on ,*    ,load ] ^
+  [^a  ,RF  ;flat ,*   ,12,cdna2,off,*    ,load ] ^
+  [&^a ,RF  ;flat ,*   ,12,cdna2,on ,*    ,load ] ^
+  [RF  ,v   ;flat ,*   ,12,*    ,*  ,*    ,store] flat_store%s0\t%A0, %1%O0%g0
+  [RF  ,a   ;flat ,*   ,12,cdna2,*  ,*    ,store] ^
+  [v   ,B   ;vop1 ,*   ,8 ,*    ,*  ,*    ,*    ] v_mov_b32\t%0, %1
+  [RLRG,v   ;ds   ,*   ,12,*    ,*  ,*    ,*    ] ds_write%b0\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+  [v   ,RLRG;ds   ,*   ,12,*    ,*  ,*    ,*    ] ds_read%u1\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+  [v   ,RM  ;flat ,*   ,12,*    ,off,*    ,load ] global_load%o1\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
+  [&v  ,RM  ;flat ,*   ,12,*    ,on ,*    ,load ] ^
+  [^a  ,RM  ;flat ,*   ,12,cdna2,off,*    ,load ] ^
+  [&^a ,RM  ;flat ,*   ,12,cdna2,on ,*    ,load ] ^
+  [RM  ,v   ;flat ,*   ,12,*    ,*  ,*    ,store] global_store%s0\t%A0, %1%O0%g0
+  [RM  ,a   ;flat ,*   ,12,cdna2,*  ,*    ,store] ^
   })
 
 ; 64bit move pattern
@@ -639,34 +666,34 @@
   [(set (match_operand:DIDF 0 "nonimmediate_operand")
 	(match_operand:DIDF 1 "general_operand"))]
   "GET_CODE(operands[1]) != SYMBOL_REF"
-  {@ [cons: =0, 1; attrs: type, length, cdna, xnack]
-  [SD  ,SSA ;sop1 ,4 ,*    ,*  ] s_mov_b64\t%0, %1
-  [SD  ,C   ;sop1 ,8 ,*    ,*  ] ^
-  [SD  ,DB  ;mult ,* ,*    ,*  ] #
-  [RS  ,Sm  ;smem ,12,*    ,*  ] s_store_dwordx2\t%1, %A0
-  [Sm  ,RS  ;smem ,12,*    ,off] s_load_dwordx2\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
-  [&Sm ,RS  ;smem ,12,*    ,on ] ^
-  [v   ,v   ;vmult,* ,*    ,*  ] #
-  [v   ,DB  ;vmult,* ,*    ,*  ] #
-  [Sg  ,v   ;vmult,* ,*    ,*  ] #
-  [v   ,Sv  ;vmult,* ,*    ,*  ] #
-  [v   ,^a  ;vmult,* ,*    ,*  ] #
-  [a   ,v   ;vmult,* ,*    ,*  ] #
-  [a   ,a   ;vmult,* ,cdna2,*  ] #
-  [v   ,RF  ;flat ,12,*    ,off] flat_load_dwordx2\t%0, %A1%O1%g1\;s_waitcnt\t0
-  [&v  ,RF  ;flat ,12,*    ,on ] ^
-  [^a  ,RF  ;flat ,12,cdna2,off] ^
-  [&^a ,RF  ;flat ,12,cdna2,on ] ^
-  [RF  ,v   ;flat ,12,*    ,*  ] flat_store_dwordx2\t%A0, %1%O0%g0
-  [RF  ,a   ;flat ,12,cdna2,*  ] ^
-  [RLRG,v   ;ds   ,12,*    ,*  ] ds_write_b64\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
-  [v   ,RLRG;ds   ,12,*    ,*  ] ds_read_b64\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
-  [v   ,RM  ;flat ,12,*    ,off] global_load_dwordx2\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
-  [&v  ,RM  ;flat ,12,*    ,on ] ^
-  [^a  ,RM  ;flat ,12,cdna2,off] ^
-  [&^a ,RM  ;flat ,12,cdna2,on ] ^
-  [RM  ,v   ;flat ,12,*    ,*  ] global_store_dwordx2\t%A0, %1%O0%g0
-  [RM  ,a   ;flat ,12,cdna2,*  ] ^
+  {@ [cons: =0, 1; attrs: type, length, cdna, xnack, flatmemaccess]
+  [SD  ,SSA ;sop1 ,4 ,*    ,*  ,*    ] s_mov_b64\t%0, %1
+  [SD  ,C   ;sop1 ,8 ,*    ,*  ,*    ] ^
+  [SD  ,DB  ;mult ,* ,*    ,*  ,*    ] #
+  [RS  ,Sm  ;smem ,12,*    ,*  ,*    ] s_store_dwordx2\t%1, %A0
+  [Sm  ,RS  ;smem ,12,*    ,off,*    ] s_load_dwordx2\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
+  [&Sm ,RS  ;smem ,12,*    ,on ,*    ] ^
+  [v   ,v   ;vmult,* ,*    ,*  ,*    ] #
+  [v   ,DB  ;vmult,* ,*    ,*  ,*    ] #
+  [Sg  ,v   ;vmult,* ,*    ,*  ,*    ] #
+  [v   ,Sv  ;vmult,* ,*    ,*  ,*    ] #
+  [v   ,^a  ;vmult,* ,*    ,*  ,*    ] #
+  [a   ,v   ;vmult,* ,*    ,*  ,*    ] #
+  [a   ,a   ;vmult,* ,cdna2,*  ,*    ] #
+  [v   ,RF  ;flat ,12,*    ,off,load ] flat_load_dwordx2\t%0, %A1%O1%g1\;s_waitcnt\t0
+  [&v  ,RF  ;flat ,12,*    ,on ,load ] ^
+  [^a  ,RF  ;flat ,12,cdna2,off,load ] ^
+  [&^a ,RF  ;flat ,12,cdna2,on ,load ] ^
+  [RF  ,v   ;flat ,12,*    ,*  ,store] flat_store_dwordx2\t%A0, %1%O0%g0
+  [RF  ,a   ;flat ,12,cdna2,*  ,store] ^
+  [RLRG,v   ;ds   ,12,*    ,*  ,*    ] ds_write_b64\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+  [v   ,RLRG;ds   ,12,*    ,*  ,*    ] ds_read_b64\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+  [v   ,RM  ;flat ,12,*    ,off,load ] global_load_dwordx2\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
+  [&v  ,RM  ;flat ,12,*    ,on ,load ] ^
+  [^a  ,RM  ;flat ,12,cdna2,off,load ] ^
+  [&^a ,RM  ;flat ,12,cdna2,on ,load ] ^
+  [RM  ,v   ;flat ,12,*    ,*  ,store] global_store_dwordx2\t%A0, %1%O0%g0
+  [RM  ,a   ;flat ,12,cdna2,*  ,store] ^
   }
   "reload_completed
    && ((!MEM_P (operands[0]) && !MEM_P (operands[1])
@@ -704,31 +731,31 @@
   [(set (match_operand:TI 0 "nonimmediate_operand")
 	(match_operand:TI 1 "general_operand"  ))]
   ""
-  {@ [cons: =0, 1; attrs: type, delayeduse, length, cdna, xnack]
-  [SD ,SSB;mult ,*  ,* ,*    ,*  ] #
-  [RS ,Sm ;smem ,*  ,12,*    ,*  ] s_store_dwordx4\t%1, %A0
-  [Sm ,RS ;smem ,yes,12,*    ,off] s_load_dwordx4\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
-  [&Sm,RS ;smem ,yes,12,*    ,on ] ^
-  [RF ,v  ;flat ,*  ,12,*    ,*  ] flat_store_dwordx4\t%A0, %1%O0%g0
-  [RF ,a  ;flat ,*  ,12,cdna2,*  ] ^
-  [v  ,RF ;flat ,*  ,12,*    ,off] flat_load_dwordx4\t%0, %A1%O1%g1\;s_waitcnt\t0
-  [&v ,RF ;flat ,*  ,12,*    ,on ] ^
-  [^a ,RF ;flat ,*  ,12,cdna2,off] ^
-  [&^a,RF ;flat ,*  ,12,cdna2,on ] ^
-  [v  ,v  ;vmult,*  ,* ,*    ,*  ] #
-  [v  ,Sv ;vmult,*  ,* ,*    ,*  ] #
-  [SD ,v  ;vmult,*  ,* ,*    ,*  ] #
-  [RM ,v  ;flat ,yes,12,*    ,*  ] global_store_dwordx4\t%A0, %1%O0%g0
-  [RM ,a  ;flat ,yes,12,cdna2,*  ] ^
-  [v  ,RM ;flat ,*  ,12,*    ,off] global_load_dwordx4\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
-  [&v ,RM ;flat ,*  ,12,*    ,on ] ^
-  [^a ,RM ;flat ,*  ,12,cdna2,off] ^
-  [&^a,RM ;flat ,*  ,12,cdna2,on ] ^
-  [RL ,v  ;ds   ,*  ,12,*    ,*  ] ds_write_b128\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
-  [v  ,RL ;ds   ,*  ,12,*    ,*  ] ds_read_b128\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
-  [v  ,^a ;vmult,*  ,* ,*    ,*  ] #
-  [a  ,v  ;vmult,*  ,* ,*    ,*  ] #
-  [a  ,a  ;vmult,*  ,* ,cdna2,*  ] #
+  {@ [cons: =0, 1; attrs: type, length, cdna, xnack, flatmemaccess]
+  [SD ,SSB;mult ,* ,*    ,*  ,*       ] #
+  [RS ,Sm ;smem ,12,*    ,*  ,*       ] s_store_dwordx4\t%1, %A0
+  [Sm ,RS ;smem ,12,*    ,off,*       ] s_load_dwordx4\t%0, %A1\;s_waitcnt\tlgkmcnt(0)
+  [&Sm,RS ;smem ,12,*    ,on ,*       ] ^
+  [RF ,v  ;flat ,12,*    ,*  ,storex34] flat_store_dwordx4\t%A0, %1%O0%g0
+  [RF ,a  ;flat ,12,cdna2,*  ,storex34] ^
+  [v  ,RF ;flat ,12,*    ,off,load    ] flat_load_dwordx4\t%0, %A1%O1%g1\;s_waitcnt\t0
+  [&v ,RF ;flat ,12,*    ,on ,load    ] ^
+  [^a ,RF ;flat ,12,cdna2,off,load    ] ^
+  [&^a,RF ;flat ,12,cdna2,on ,load    ] ^
+  [v  ,v  ;vmult,* ,*    ,*  ,*       ] #
+  [v  ,Sv ;vmult,* ,*    ,*  ,*       ] #
+  [SD ,v  ;vmult,* ,*    ,*  ,*       ] #
+  [RM ,v  ;flat ,12,*    ,*  ,storex34] global_store_dwordx4\t%A0, %1%O0%g0
+  [RM ,a  ;flat ,12,cdna2,*  ,storex34] ^
+  [v  ,RM ;flat ,12,*    ,off,load    ] global_load_dwordx4\t%0, %A1%O1%g1\;s_waitcnt\tvmcnt(0)
+  [&v ,RM ;flat ,12,*    ,on ,load    ] ^
+  [^a ,RM ;flat ,12,cdna2,off,load    ] ^
+  [&^a,RM ;flat ,12,cdna2,on ,load    ] ^
+  [RL ,v  ;ds   ,12,*    ,*  ,*       ] ds_write_b128\t%A0, %1%O0\;s_waitcnt\tlgkmcnt(0)
+  [v  ,RL ;ds   ,12,*    ,*  ,*       ] ds_read_b128\t%0, %A1%O1\;s_waitcnt\tlgkmcnt(0)
+  [v  ,^a ;vmult,* ,*    ,*  ,*       ] #
+  [a  ,v  ;vmult,* ,*    ,*  ,*       ] #
+  [a  ,a  ;vmult,* ,cdna2,*  ,*       ] #
   }
   "reload_completed
    && REG_P (operands[0])
@@ -1077,6 +1104,7 @@
    s_cmp%D1\t%2, %3
    v_cmp%E1\tvcc, %2, %3"
   [(set_attr "type" "sopc,vopc")
+   (set_attr "vcmp" "vcmp")
    (set_attr "length" "8")])
 
 (define_insn "cstoredi4_vector"
@@ -1087,6 +1115,7 @@
   ""
   "v_cmp%E1\tvcc, %2, %3"
   [(set_attr "type" "vopc")
+   (set_attr "vcmp" "vcmp")
    (set_attr "length" "8")])
 
 (define_expand "cbranchdi4"
@@ -1113,6 +1142,7 @@
   ""
   "v_cmp%E1\tvcc, %2, %3"
   [(set_attr "type" "vopc")
+   (set_attr "vcmp" "vcmp")
    (set_attr "length" "8")])
 
 (define_expand "cbranch<mode>4"
@@ -1136,14 +1166,13 @@
   [(set (match_operand:SI 0 "register_operand"         "= Sg, Sg, Sg,   v")
         (plus:SI (match_operand:SI 1 "gcn_alu_operand" "%SgA,  0,SgA,   v")
 		 (match_operand:SI 2 "gcn_alu_operand" " SgA,SgJ,  B,vBSv")))
-   (clobber (match_scratch:BI 3			       "= cs, cs, cs,   X"))
-   (clobber (match_scratch:DI 4			       "=  X,  X,  X,  cV"))]
+   (clobber (match_scratch:BI 3			       "= cs, cs, cs,   X"))]
   ""
   "@
    s_add_i32\t%0, %1, %2
    s_addk_i32\t%0, %2
    s_add_i32\t%0, %1, %2
-   v_add_co_u32\t%0, vcc, %2, %1"
+   {v_add_u32|v_add_nc_u32}\t%0, %2, %1"
   [(set_attr "type" "sop2,sopk,sop2,vop2")
    (set_attr "length" "4,4,8,8")])
 
@@ -1151,8 +1180,7 @@
   [(parallel [(set (match_operand:SI 0 "register_operand")
 		   (plus:SI (match_operand:SI 1 "gcn_alu_operand")
 			    (match_operand:SI 2 "gcn_alu_operand")))
-	      (clobber (reg:BI SCC_REG))
-	      (clobber (scratch:DI))])]
+	      (clobber (reg:BI SCC_REG))])]
   ""
   {})
 
@@ -1332,14 +1360,13 @@
   [(set (match_operand:SI 0 "register_operand"          "=Sg, Sg,    v,   v")
 	(minus:SI (match_operand:SI 1 "gcn_alu_operand" "SgA,SgA,    v,vBSv")
 		  (match_operand:SI 2 "gcn_alu_operand" "SgA,  B, vBSv,   v")))
-   (clobber (match_scratch:BI 3				"=cs, cs,    X,   X"))
-   (clobber (match_scratch:DI 4				"= X,  X,   cV,  cV"))]
+   (clobber (match_scratch:BI 3				"=cs, cs,    X,   X"))]
   ""
   "@
    s_sub_i32\t%0, %1, %2
    s_sub_i32\t%0, %1, %2
-   v_subrev_co_u32\t%0, vcc, %2, %1
-   v_sub_co_u32\t%0, vcc, %1, %2"
+   {v_subrev_u32|v_subrev_nc_u32}\t%0, %2, %1
+   {v_sub_u32|v_sub_nc_u32}\t%0, %1, %2"
   [(set_attr "type" "sop2,sop2,vop2,vop2")
    (set_attr "length" "4,8,8,8")])
 
@@ -1569,8 +1596,7 @@
 	(mult:DI (match_operand:DI 1 "register_operand" "%Sg, Sg,  v, v")
 		 (match_operand:DI 2 "nonmemory_operand" "Sg,  i,vSv, A")))
    (clobber (match_scratch:SI 3 "=&Sg,&Sg,&v,&v"))
-   (clobber (match_scratch:BI 4  "=cs, cs, X, X"))
-   (clobber (match_scratch:DI 5   "=X,  X,cV,cV"))]
+   (clobber (match_scratch:BI 4  "=cs, cs, X, X"))]
   ""
   "#"
   "reload_completed"
@@ -1585,15 +1611,13 @@
     emit_insn (gen_umulsidi3 (operands[0], op1lo, op2lo));
     emit_insn (gen_mulsi3 (tmp, op1lo, op2hi));
     rtx add = gen_rtx_SET (dsthi, gen_rtx_PLUS (SImode, dsthi, tmp));
-    rtx clob1 = gen_rtx_CLOBBER (VOIDmode, operands[4]);
-    rtx clob2 = gen_rtx_CLOBBER (VOIDmode, operands[5]);
-    add = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (3, add, clob1, clob2));
+    rtx clob = gen_rtx_CLOBBER (VOIDmode, operands[4]);
+    add = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, add, clob));
     emit_insn (add);
     emit_insn (gen_mulsi3 (tmp, op1hi, op2lo));
     add = gen_rtx_SET (dsthi, gen_rtx_PLUS (SImode, dsthi, tmp));
-    clob1 = gen_rtx_CLOBBER (VOIDmode, operands[4]);
-    clob2 = gen_rtx_CLOBBER (VOIDmode, operands[5]);
-    add = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (3, add, clob1, clob2));
+    clob = gen_rtx_CLOBBER (VOIDmode, operands[4]);
+    add = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, add, clob));
     emit_insn (add);
     DONE;
   })
@@ -1991,6 +2015,7 @@
    flat_atomic_<bare_mnemonic><X>\t%0, %1, %2 %G2\;s_waitcnt\t0
    global_atomic_<bare_mnemonic><X>\t%0, %A1, %2%O1 %G2\;s_waitcnt\tvmcnt(0)"
   [(set_attr "type" "smem,flat,flat")
+   (set_attr "flatmemaccess" "*,atomicwait,atomicwait")
    (set_attr "length" "12")])
 
 ; FIXME: These patterns are disabled because the instructions don't
@@ -2012,6 +2037,7 @@
    flat_atomic_<bare_mnemonic><X>\t%0, %1\;s_waitcnt\t0
    global_atomic_<bare_mnemonic><X>\t%A0, %1%O0\;s_waitcnt\tvmcnt(0)"
   [(set_attr "type" "smem,flat,flat")
+   (set_attr "flatmemaccess" "*,atomicwait,atomicwait")
    (set_attr "length" "12")])
 
 (define_mode_attr x2 [(SI "DI") (DI "TI")])
@@ -2059,7 +2085,7 @@
    global_atomic_cmpswap<X>\t%0, %A1, %2%O1 %G2\;s_waitcnt\tvmcnt(0)"
   [(set_attr "type" "smem,flat,flat")
    (set_attr "length" "12")
-   (set_attr "delayeduse" "*,yes,yes")])
+   (set_attr "flatmemaccess" "*,cmpswapx2,cmpswapx2")])
 
 (define_insn "sync_compare_and_swap<mode>_lds_insn"
   [(set (match_operand:SIDI 0 "register_operand"    "= v")
@@ -2157,7 +2183,7 @@
 		    ? "buffer_gl1_inv\;buffer_gl0_inv\;flat_load%o0\t%0, %A1%O1 %G1\;"
 		      "s_waitcnt\t0\;buffer_gl1_inv\;buffer_gl0_inv"
 		    : TARGET_TARGET_SC_CACHE
-		    ? "buffer_inv sc1\;flat_load%o0\t%0, %A1%O1 %G1\;"
+		    ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;flat_load%o0\t%0, %A1%O1 %G1\;"
 		      "s_waitcnt\t0\;buffer_inv sc1"
 		    : "buffer_wbinvl1_vol\;flat_load%o0\t%0, %A1%O1 %G1\;"
 		      "s_waitcnt\t0\;buffer_wbinvl1_vol");
@@ -2169,7 +2195,7 @@
 		    ? "buffer_gl1_inv\;buffer_gl0_inv\;global_load%o0\t%0, %A1%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_gl1_inv\;buffer_gl0_inv"
 		    : TARGET_TARGET_SC_CACHE
-		    ? "buffer_inv sc1\;global_load%o0\t%0, %A1%O1 %G1\;"
+		    ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;global_load%o0\t%0, %A1%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_inv sc1"
 		    : "buffer_wbinvl1_vol\;global_load%o0\t%0, %A1%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_wbinvl1_vol");
@@ -2179,6 +2205,7 @@
     gcc_unreachable ();
   }
   [(set_attr "type" "smem,flat,flat")
+   (set_attr "flatmemaccess" "*,load,load")
    (set_attr "length" "28")
    (set_attr "rdna" "no,*,*")])
 
@@ -2215,7 +2242,7 @@
 		    : TARGET_WBINVL1_CACHE
 		    ? "buffer_wbinvl1_vol\;flat_store%o1\t%A0, %1%O0 %G1"
 		    : TARGET_TARGET_SC_CACHE
-		    ? "buffer_inv sc1\;flat_store%o1\t%A0, %1%O0 %G1"
+		    ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;flat_store%o1\t%A0, %1%O0 %G1"
 		    : "error: cache architectire unspecified");
 	  case 2:
 	    return (TARGET_GLn_CACHE
@@ -2223,7 +2250,7 @@
 		    : TARGET_WBINVL1_CACHE
 		    ? "buffer_wbinvl1_vol\;global_store%o1\t%A0, %1%O0 %G1"
 		    : TARGET_TARGET_SC_CACHE
-		    ? "buffer_inv sc1\;global_store%o1\t%A0, %1%O0 %G1"
+		    ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;global_store%o1\t%A0, %1%O0 %G1"
 		    : "error: cache architecture unspecified");
 	  }
 	break;
@@ -2243,7 +2270,8 @@
 		    ? "buffer_wbinvl1_vol\;flat_store%o1\t%A0, %1%O0 %G1\;"
 		      "s_waitcnt\t0\;buffer_wbinvl1_vol"
 		    : TARGET_TARGET_SC_CACHE
-		    ? "buffer_inv sc1\;flat_store%o1\t%A0, %1%O0 %G1\;"
+		    ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;"
+		      "flat_store%o1\t%A0, %1%O0 %G1\;"
 		      "s_waitcnt\t0\;buffer_inv sc1"
 		    : "error: cache architecture unspecified");
 	  case 2:
@@ -2254,7 +2282,8 @@
 		    ? "buffer_wbinvl1_vol\;global_store%o1\t%A0, %1%O0 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_wbinvl1_vol"
 		    : TARGET_TARGET_SC_CACHE
-		    ? "buffer_inv sc1\;global_store%o1\t%A0, %1%O0 %G1\;"
+		    ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;"
+		      "global_store%o1\t%A0, %1%O0 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_inv sc1"
 		    : "error: cache architecture unspecified");
 	  }
@@ -2263,6 +2292,7 @@
     gcc_unreachable ();
   }
   [(set_attr "type" "smem,flat,flat")
+   (set_attr "flatmemaccess" "*,store,store")
    (set_attr "length" "28")
    (set_attr "rdna" "no,*,*")])
 
@@ -2337,7 +2367,7 @@
             ? "buffer_wbinvl1_vol\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
 		      "s_waitcnt\t0"
 	    : TARGET_TARGET_SC_CACHE
-            ? "buffer_inv sc1\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
+            ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
 		      "s_waitcnt\t0"
             : "error: cache architecture unspecified");
 	  case 2:
@@ -2350,7 +2380,7 @@
 		      "global_atomic_swap<X>\t%0, %A1, %2%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)"
 	    : TARGET_TARGET_SC_CACHE
-            ? "buffer_inv sc1\;"
+            ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;"
 		      "global_atomic_swap<X>\t%0, %A1, %2%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)"
             : "error: cache architecture unspecified");
@@ -2372,7 +2402,7 @@
             ? "buffer_wbinvl1_vol\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
 		      "s_waitcnt\t0\;buffer_wbinvl1_vol"
 	    : TARGET_TARGET_SC_CACHE
-            ? "buffer_inv sc1\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
+            ? "buffer_wbl2\tsc0\;s_waitcnt\t0\;flat_atomic_swap<X>\t%0, %1, %2 %G1\;"
 		      "s_waitcnt\t0\;buffer_inv sc1"
             : "error: cache architecture unspecified");
 	  case 2:
@@ -2385,7 +2415,7 @@
 		      "global_atomic_swap<X>\t%0, %A1, %2%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_wbinvl1_vol"
 	    : TARGET_TARGET_SC_CACHE
-            ? "buffer_inv sc1\;"
+            ? "buffer_wbl2\tsc0\;s_waitcnt\tvmcnt(0)\;"
 		      "global_atomic_swap<X>\t%0, %A1, %2%O1 %G1\;"
 		      "s_waitcnt\tvmcnt(0)\;buffer_inv sc1"
             : "error: cache architecture unspecified");
@@ -2395,6 +2425,7 @@
     gcc_unreachable ();
   }
   [(set_attr "type" "smem,flat,flat")
+   (set_attr "flatmemaccess" "*,atomicwait,atomicwait")
    (set_attr "length" "28")
    (set_attr "rdna" "no,*,*")])
 
diff --git a/gcc/config/h8300/h8300.h b/gcc/config/h8300/h8300.h
index 6b2dd65..8018b63 100644
--- a/gcc/config/h8300/h8300.h
+++ b/gcc/config/h8300/h8300.h
@@ -610,6 +610,12 @@ struct cum_arg
 #define DATA_SECTION_ASM_OP "\t.section .data"
 #define BSS_SECTION_ASM_OP "\t.section .bss"
 
+/* Override default definitions from elfos.h. */
+#undef INIT_SECTION_ASM_OP
+#define INIT_SECTION_ASM_OP "\t.section\t.init,\"ax\""
+#undef FINI_SECTION_ASM_OP
+#define FINI_SECTION_ASM_OP "\t.section\t.fini,\"ax\""
+
 #undef DO_GLOBAL_CTORS_BODY
 #define DO_GLOBAL_CTORS_BODY			\
 {						\
@@ -647,19 +653,11 @@ struct cum_arg
 /* Globalizing directive for a label.  */
 #define GLOBAL_ASM_OP "\t.global "
 
+/* Override default definition from elfos.h. */
+#undef ASM_DECLARE_FUNCTION_NAME
 #define ASM_DECLARE_FUNCTION_NAME(FILE, NAME, DECL) \
    ASM_OUTPUT_FUNCTION_LABEL (FILE, NAME, DECL)
 
-/* This is how to store into the string LABEL
-   the symbol_ref name of an internal numbered label where
-   PREFIX is the class of label and NUM is the number within the class.
-   This is suitable for output with `assemble_name'.
-
-   N.B.: The h8300.md branch_true and branch_false patterns also know
-   how to generate internal labels.  */
-#define ASM_GENERATE_INTERNAL_LABEL(LABEL, PREFIX, NUM)	\
-  sprintf (LABEL, "*.%s%lu", PREFIX, (unsigned long)(NUM))
-
 /* This is how to output an insn to push a register on the stack.
    It need not be very fast code.  */
 
@@ -690,9 +688,6 @@ struct cum_arg
   if ((LOG) != 0)				\
     fprintf (FILE, "\t.align %d\n", (LOG))
 
-#define ASM_OUTPUT_SKIP(FILE, SIZE) \
-  fprintf (FILE, "\t.space %d\n", (int)(SIZE))
-
 /* This says how to output an assembler line
    to define a global common symbol.  */
 
diff --git a/gcc/config/i386/i386-expand.cc b/gcc/config/i386/i386-expand.cc
index 83076ad..09aa9b1 100644
--- a/gcc/config/i386/i386-expand.cc
+++ b/gcc/config/i386/i386-expand.cc
@@ -387,7 +387,7 @@ ix86_expand_move (machine_mode mode, rtx operands[])
       tmp = XEXP (op1, 0);
 
       if (GET_CODE (tmp) != PLUS
-	  || GET_CODE (XEXP (tmp, 0)) != SYMBOL_REF)
+	  || !SYMBOL_REF_P (XEXP (tmp, 0)))
 	break;
 
       op1 = XEXP (tmp, 0);
@@ -487,7 +487,7 @@ ix86_expand_move (machine_mode mode, rtx operands[])
 		op1 = machopic_legitimize_pic_address (op1, mode,
 						       tmp == op1 ? 0 : tmp);
 	    }
-	  if (op0 != op1 && GET_CODE (op0) != MEM)
+	  if (op0 != op1 && !MEM_P (op0))
 	    {
 	      rtx insn = gen_rtx_SET (op0, op1);
 	      emit_insn (insn);
@@ -1396,11 +1396,11 @@ ix86_expand_vector_logical_operator (enum rtx_code code, machine_mode mode,
      to cast them temporarily to integer vectors.  */
   if (op1
       && !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
-      && (SUBREG_P (op2) || GET_CODE (op2) == CONST_VECTOR)
+      && (SUBREG_P (op2) || CONST_VECTOR_P (op2))
       && GET_MODE_CLASS (GET_MODE (SUBREG_REG (op1))) == MODE_VECTOR_FLOAT
       && GET_MODE_SIZE (GET_MODE (SUBREG_REG (op1))) == GET_MODE_SIZE (mode)
       && SUBREG_BYTE (op1) == 0
-      && (GET_CODE (op2) == CONST_VECTOR
+      && (CONST_VECTOR_P (op2)
 	  || (GET_MODE (SUBREG_REG (op1)) == GET_MODE (SUBREG_REG (op2))
 	      && SUBREG_BYTE (op2) == 0))
       && can_create_pseudo_p ())
@@ -1415,7 +1415,7 @@ ix86_expand_vector_logical_operator (enum rtx_code code, machine_mode mode,
 	case E_V4DFmode:
 	case E_V8DFmode:
 	  dst = gen_reg_rtx (GET_MODE (SUBREG_REG (op1)));
-	  if (GET_CODE (op2) == CONST_VECTOR)
+	  if (CONST_VECTOR_P (op2))
 	    {
 	      op2 = gen_lowpart (GET_MODE (dst), op2);
 	      op2 = force_reg (GET_MODE (dst), op2);
@@ -4918,7 +4918,7 @@ ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
 	case LEU:
 	  /* x <= cst can be handled as x < cst + 1 unless there is
 	     wrap around in cst + 1.  */
-	  if (GET_CODE (cop1) == CONST_VECTOR
+	  if (CONST_VECTOR_P (cop1)
 	      && GET_MODE_INNER (mode) != TImode)
 	    {
 	      unsigned int n_elts = GET_MODE_NUNITS (mode), i;
@@ -4962,7 +4962,7 @@ ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
 	case GEU:
 	  /* x >= cst can be handled as x > cst - 1 unless there is
 	     wrap around in cst - 1.  */
-	  if (GET_CODE (cop1) == CONST_VECTOR
+	  if (CONST_VECTOR_P (cop1)
 	      && GET_MODE_INNER (mode) != TImode)
 	    {
 	      unsigned int n_elts = GET_MODE_NUNITS (mode), i;
@@ -5033,9 +5033,9 @@ ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
 	    }
 	}
 
-      if (GET_CODE (cop0) == CONST_VECTOR)
+      if (CONST_VECTOR_P (cop0))
 	cop0 = force_reg (mode, cop0);
-      else if (GET_CODE (cop1) == CONST_VECTOR)
+      else if (CONST_VECTOR_P (cop1))
 	cop1 = force_reg (mode, cop1);
 
       rtx optrue = op_true ? op_true : CONSTM1_RTX (data_mode);
@@ -5234,7 +5234,7 @@ ix86_expand_int_sse_cmp (rtx dest, enum rtx_code code, rtx cop0, rtx cop1,
   if (*negate)
     std::swap (op_true, op_false);
 
-  if (GET_CODE (cop1) == CONST_VECTOR)
+  if (CONST_VECTOR_P (cop1))
     cop1 = force_reg (mode, cop1);
 
   /* Allow the comparison to be done in one mode, but the movcc to
@@ -6188,7 +6188,7 @@ ix86_extract_perm_from_pool_constant (int* perm, rtx mem)
 
   rtx constant = get_pool_constant (XEXP (mem, 0));
 
-  if (GET_CODE (constant) != CONST_VECTOR)
+  if (!CONST_VECTOR_P (constant))
     return false;
 
   /* There could be some rtx like
@@ -6198,7 +6198,7 @@ ix86_extract_perm_from_pool_constant (int* perm, rtx mem)
     {
       constant = simplify_subreg (mode, constant, GET_MODE (constant), 0);
 
-      if (constant == nullptr || GET_CODE (constant) != CONST_VECTOR)
+      if (constant == nullptr || !CONST_VECTOR_P (constant))
 	return false;
     }
 
@@ -6244,7 +6244,7 @@ ix86_split_to_parts (rtx operand, rtx *parts, machine_mode mode)
       return size;
     }
 
-  if (GET_CODE (operand) == CONST_VECTOR)
+  if (CONST_VECTOR_P (operand))
     {
       scalar_int_mode imode = int_mode_for_mode (mode).require ();
       /* Caution: if we looked through a constant pool memory above,
@@ -6378,7 +6378,7 @@ ix86_split_long_move (rtx operands[])
 	 fp moves, that force all constants to memory to allow combining.  */
 
       if (MEM_P (operands[1])
-	  && GET_CODE (XEXP (operands[1], 0)) == SYMBOL_REF
+	  && SYMBOL_REF_P (XEXP (operands[1], 0))
 	  && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0)))
 	operands[1] = get_pool_constant (XEXP (operands[1], 0));
       if (push_operand (operands[0], VOIDmode))
@@ -7899,7 +7899,8 @@ expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
 			       rtx count, machine_mode mode, int unroll,
 			       int expected_size, bool issetmem)
 {
-  rtx_code_label *out_label, *top_label;
+  rtx_code_label *out_label = nullptr;
+  rtx_code_label *top_label = nullptr;
   rtx iter, tmp;
   machine_mode iter_mode = counter_mode (count);
   int piece_size_n = GET_MODE_SIZE (mode) * unroll;
@@ -7907,9 +7908,19 @@ expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
   rtx piece_size_mask = GEN_INT (~((GET_MODE_SIZE (mode) * unroll) - 1));
   rtx size;
   int i;
+  int loop_count;
+
+  if (expected_size != -1 && CONST_INT_P (count))
+    loop_count = INTVAL (count) / GET_MODE_SIZE (mode) / unroll;
+  else
+    loop_count = -1;
 
-  top_label = gen_label_rtx ();
-  out_label = gen_label_rtx ();
+  /* Don't generate the loop if the loop count is 1.  */
+  if (loop_count != 1)
+    {
+      top_label = gen_label_rtx ();
+      out_label = gen_label_rtx ();
+    }
   iter = gen_reg_rtx (iter_mode);
 
   size = expand_simple_binop (iter_mode, AND, count, piece_size_mask,
@@ -7923,7 +7934,8 @@ expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
     }
   emit_move_insn (iter, const0_rtx);
 
-  emit_label (top_label);
+  if (loop_count != 1)
+    emit_label (top_label);
 
   tmp = convert_modes (Pmode, iter_mode, iter, true);
 
@@ -7991,21 +8003,25 @@ expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
   if (tmp != iter)
     emit_move_insn (iter, tmp);
 
-  emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
-			   true, top_label);
-  if (expected_size != -1)
+  if (loop_count != 1)
     {
-      expected_size /= GET_MODE_SIZE (mode) * unroll;
-      if (expected_size == 0)
-	predict_jump (0);
-      else if (expected_size > REG_BR_PROB_BASE)
-	predict_jump (REG_BR_PROB_BASE - 1);
+      emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
+			       true, top_label);
+      if (expected_size != -1)
+	{
+	  expected_size /= GET_MODE_SIZE (mode) * unroll;
+	  if (expected_size == 0)
+	    predict_jump (0);
+	  else if (expected_size > REG_BR_PROB_BASE)
+	    predict_jump (REG_BR_PROB_BASE - 1);
+	  else
+	    predict_jump (REG_BR_PROB_BASE
+			  - (REG_BR_PROB_BASE + expected_size / 2)
+			    / expected_size);
+	}
       else
-        predict_jump (REG_BR_PROB_BASE - (REG_BR_PROB_BASE + expected_size / 2)
-		      / expected_size);
+	predict_jump (REG_BR_PROB_BASE * 80 / 100);
     }
-  else
-    predict_jump (REG_BR_PROB_BASE * 80 / 100);
   iter = ix86_zero_extend_to_Pmode (iter);
   tmp = expand_simple_binop (Pmode, PLUS, destptr, iter, destptr,
 			     true, OPTAB_LIB_WIDEN);
@@ -8018,7 +8034,8 @@ expand_set_or_cpymem_via_loop (rtx destmem, rtx srcmem,
       if (tmp != srcptr)
 	emit_move_insn (srcptr, tmp);
     }
-  emit_label (out_label);
+  if (loop_count != 1)
+    emit_label (out_label);
 }
 
 /* Divide COUNTREG by SCALE.  */
@@ -8221,19 +8238,11 @@ expand_cpymem_epilogue (rtx destmem, rtx srcmem,
   rtx src, dest;
   if (CONST_INT_P (count))
     {
-      HOST_WIDE_INT countval = INTVAL (count);
-      HOST_WIDE_INT epilogue_size = countval % max_size;
-      int i;
-
-      /* For now MAX_SIZE should be a power of 2.  This assert could be
-	 relaxed, but it'll require a bit more complicated epilogue
-	 expanding.  */
-      gcc_assert ((max_size & (max_size - 1)) == 0);
-      for (i = max_size; i >= 1; i >>= 1)
-	{
-	  if (epilogue_size & i)
-	    destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
-	}
+      unsigned HOST_WIDE_INT countval = UINTVAL (count);
+      unsigned HOST_WIDE_INT epilogue_size = countval % max_size;
+      unsigned int destalign = MEM_ALIGN (destmem);
+      move_by_pieces (destmem, srcmem, epilogue_size, destalign,
+		      RETURN_BEGIN);
       return;
     }
   if (max_size > 8)
@@ -8394,6 +8403,81 @@ expand_setmem_epilogue_via_loop (rtx destmem, rtx destptr, rtx value,
 				 1, max_size / 2, true);
 }
 
+/* Callback routine for store_by_pieces.  Return the RTL of a register
+   containing GET_MODE_SIZE (MODE) bytes in the RTL register op_p which
+   is a word or a word vector register.  If PREV_P isn't nullptr, it
+   has the RTL info from the previous iteration.  */
+
+static rtx
+setmem_epilogue_gen_val (void *op_p, void *prev_p, HOST_WIDE_INT,
+			 fixed_size_mode mode)
+{
+  rtx target;
+  by_pieces_prev *prev = (by_pieces_prev *) prev_p;
+  if (prev)
+    {
+      rtx prev_op = prev->data;
+      if (prev_op)
+	{
+	  machine_mode prev_mode = GET_MODE (prev_op);
+	  if (prev_mode == mode)
+	    return prev_op;
+	  if (VECTOR_MODE_P (prev_mode)
+	      && VECTOR_MODE_P (mode)
+	      && GET_MODE_INNER (prev_mode) == GET_MODE_INNER (mode))
+	    {
+	      target = gen_rtx_SUBREG (mode, prev_op, 0);
+	      return target;
+	    }
+	}
+    }
+
+  rtx op = (rtx) op_p;
+  machine_mode op_mode = GET_MODE (op);
+
+  gcc_assert (op_mode == word_mode
+	      || (VECTOR_MODE_P (op_mode)
+		  && GET_MODE_INNER (op_mode) == word_mode));
+
+  if (VECTOR_MODE_P (mode))
+    {
+      gcc_assert (GET_MODE_INNER (mode) == QImode);
+
+      unsigned int op_size = GET_MODE_SIZE (op_mode);
+      unsigned int size = GET_MODE_SIZE (mode);
+      unsigned int nunits = op_size / GET_MODE_SIZE (QImode);
+      machine_mode vec_mode
+	= mode_for_vector (QImode, nunits).require ();
+      target = gen_reg_rtx (vec_mode);
+      op = gen_rtx_SUBREG (vec_mode, op, 0);
+      emit_move_insn (target, op);
+      if (op_size == size)
+	return target;
+
+      rtx tmp = gen_reg_rtx (mode);
+      target = gen_rtx_SUBREG (mode, target, 0);
+      emit_move_insn (tmp, target);
+      return tmp;
+    }
+
+  target = gen_reg_rtx (word_mode);
+  if (VECTOR_MODE_P (op_mode))
+    {
+      op = gen_rtx_SUBREG (word_mode, op, 0);
+      emit_move_insn (target, op);
+    }
+  else
+    target = op;
+
+  if (mode == word_mode)
+    return target;
+
+  rtx tmp = gen_reg_rtx (mode);
+  target = gen_rtx_SUBREG (mode, target, 0);
+  emit_move_insn (tmp, target);
+  return tmp;
+}
+
 /* Output code to set at most count & (max_size - 1) bytes starting by DEST.  */
 static void
 expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx vec_value,
@@ -8403,24 +8487,12 @@ expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx vec_value,
 
   if (CONST_INT_P (count))
     {
-      HOST_WIDE_INT countval = INTVAL (count);
-      HOST_WIDE_INT epilogue_size = countval % max_size;
-      int i;
-
-      /* For now MAX_SIZE should be a power of 2.  This assert could be
-	 relaxed, but it'll require a bit more complicated epilogue
-	 expanding.  */
-      gcc_assert ((max_size & (max_size - 1)) == 0);
-      for (i = max_size; i >= 1; i >>= 1)
-	{
-	  if (epilogue_size & i)
-	    {
-	      if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
-		destmem = emit_memset (destmem, destptr, vec_value, i);
-	      else
-		destmem = emit_memset (destmem, destptr, value, i);
-	    }
-	}
+      unsigned HOST_WIDE_INT countval = UINTVAL (count);
+      unsigned HOST_WIDE_INT epilogue_size = countval % max_size;
+      unsigned int destalign = MEM_ALIGN (destmem);
+      store_by_pieces (destmem, epilogue_size, setmem_epilogue_gen_val,
+		       vec_value ? vec_value : value, destalign, true,
+		       RETURN_BEGIN);
       return;
     }
   if (max_size > 32)
@@ -8552,6 +8624,7 @@ expand_small_cpymem_or_setmem (rtx destmem, rtx srcmem,
   rtx_code_label *label = ix86_expand_aligntest (count, size, false);
   machine_mode mode = int_mode_for_size (size * BITS_PER_UNIT, 1).else_blk ();
   rtx modesize;
+  rtx scalar_value = value;
   int n;
 
   /* If we do not have vector value to copy, we must reduce size.  */
@@ -8571,11 +8644,57 @@ expand_small_cpymem_or_setmem (rtx destmem, rtx srcmem,
     {
       /* Choose appropriate vector mode.  */
       if (size >= 32)
-	mode = TARGET_AVX ? V32QImode : TARGET_SSE ? V16QImode : DImode;
+	switch (MOVE_MAX)
+	  {
+	  case 64:
+	    if (size >= 64)
+	      {
+		mode = V64QImode;
+		break;
+	      }
+	    /* FALLTHRU */
+	  case 32:
+	    mode = V32QImode;
+	    break;
+	  case 16:
+	    mode = V16QImode;
+	    break;
+	  case 8:
+	    mode = DImode;
+	    break;
+	  default:
+	    gcc_unreachable ();
+	  }
       else if (size >= 16)
 	mode = TARGET_SSE ? V16QImode : DImode;
       srcmem = change_address (srcmem, mode, srcptr);
     }
+  if (issetmem && vec_value && GET_MODE_SIZE (mode) > size)
+    {
+      /* For memset with vector and the size is smaller than the vector
+	 size, first try the narrower vector, otherwise, use the
+	 original value. */
+      machine_mode inner_mode = GET_MODE_INNER (mode);
+      unsigned int nunits = size / GET_MODE_SIZE (inner_mode);
+      if (nunits > 1)
+	{
+	  mode = mode_for_vector (GET_MODE_INNER (mode),
+				  nunits).require ();
+	  value = gen_rtx_SUBREG (mode, value, 0);
+	}
+      else
+	{
+	  scalar_int_mode smode
+	    = smallest_int_mode_for_size (size * BITS_PER_UNIT).require ();
+	  gcc_assert (GET_MODE_SIZE (GET_MODE (scalar_value))
+		      >= GET_MODE_SIZE (smode));
+	  mode = smode;
+	  if (GET_MODE (scalar_value) == mode)
+	    value = scalar_value;
+	  else
+	    value = gen_rtx_SUBREG (mode, scalar_value, 0);
+	}
+    }
   destmem = change_address (destmem, mode, destptr);
   modesize = GEN_INT (GET_MODE_SIZE (mode));
   gcc_assert (GET_MODE_SIZE (mode) <= size);
@@ -9179,13 +9298,26 @@ decide_alignment (int align,
 static rtx
 promote_duplicated_reg (machine_mode mode, rtx val)
 {
+  if (val == const0_rtx)
+    return copy_to_mode_reg (mode, CONST0_RTX (mode));
+
   machine_mode valmode = GET_MODE (val);
+  if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
+    {
+      /* Duplicate the scalar value for integer vector.  */
+      gcc_assert ((val == const0_rtx || val == constm1_rtx)
+		  || GET_MODE_INNER (mode) == valmode);
+      rtx dup = gen_reg_rtx (mode);
+      bool ok = ix86_expand_vector_init_duplicate (false, mode, dup,
+						   val);
+      gcc_assert (ok);
+      return dup;
+    }
+
   rtx tmp;
   int nops = mode == DImode ? 3 : 2;
 
-  gcc_assert (mode == SImode || mode == DImode || val == const0_rtx);
-  if (val == const0_rtx)
-    return copy_to_mode_reg (mode, CONST0_RTX (mode));
+  gcc_assert (mode == SImode || mode == DImode);
   if (CONST_INT_P (val))
     {
       HOST_WIDE_INT v = INTVAL (val) & 255;
@@ -9413,11 +9545,6 @@ ix86_expand_set_or_cpymem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
     return false;
   gcc_assert (alg != no_stringop);
 
-  /* For now vector-version of memset is generated only for memory zeroing, as
-     creating of promoted vector value is very cheap in this case.  */
-  if (issetmem && alg == vector_loop && val_exp != const0_rtx)
-    alg = unrolled_loop;
-
   if (!count)
     count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp);
   destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
@@ -9510,20 +9637,41 @@ ix86_expand_set_or_cpymem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
        && ((desired_align > align && !align_bytes)
 	   || (!count && epilogue_size_needed > 1)));
 
+  /* Destination is aligned after the misaligned prologue.  */
+  bool aligned_dstmem = misaligned_prologue_used;
+
+  if (noalign && !misaligned_prologue_used)
+    {
+      /* Also use misaligned prologue if alignment isn't needed and
+	 destination isn't aligned.   Since alignment isn't needed,
+	 the destination after prologue won't be aligned.  */
+      aligned_dstmem = (GET_MODE_ALIGNMENT (move_mode)
+			<= MEM_ALIGN (dst));
+      if (!aligned_dstmem)
+	misaligned_prologue_used = true;
+    }
+
   /* Do the cheap promotion to allow better CSE across the
      main loop and epilogue (ie one load of the big constant in the
      front of all code.
      For now the misaligned move sequences do not have fast path
      without broadcasting.  */
-  if (issetmem && ((CONST_INT_P (val_exp) || misaligned_prologue_used)))
+  if (issetmem
+      && (alg == vector_loop
+	  || CONST_INT_P (val_exp)
+	  || misaligned_prologue_used))
     {
       if (alg == vector_loop)
 	{
-	  gcc_assert (val_exp == const0_rtx);
-	  vec_promoted_val = promote_duplicated_reg (move_mode, val_exp);
 	  promoted_val = promote_duplicated_reg_to_size (val_exp,
 							 GET_MODE_SIZE (word_mode),
 							 desired_align, align);
+	  /* Duplicate the promoted scalar value if not 0 nor -1.  */
+	  vec_promoted_val
+	    = promote_duplicated_reg (move_mode,
+				      (val_exp == const0_rtx
+				       || val_exp == constm1_rtx)
+				      ? val_exp : promoted_val);
 	}
       else
 	{
@@ -9548,7 +9696,8 @@ ix86_expand_set_or_cpymem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
       if (!issetmem)
         src = change_address (src, BLKmode, srcreg);
       dst = change_address (dst, BLKmode, destreg);
-      set_mem_align (dst, desired_align * BITS_PER_UNIT);
+      if (aligned_dstmem)
+	set_mem_align (dst, desired_align * BITS_PER_UNIT);
       epilogue_size_needed = 0;
       if (need_zero_guard
 	  && min_size < (unsigned HOST_WIDE_INT) size_needed)
@@ -10096,7 +10245,7 @@ construct_plt_address (rtx symbol)
 {
   rtx tmp, unspec;
 
-  gcc_assert (GET_CODE (symbol) == SYMBOL_REF);
+  gcc_assert (SYMBOL_REF_P (symbol));
   gcc_assert (ix86_cmodel == CM_LARGE_PIC && !TARGET_PECOFF);
   gcc_assert (Pmode == DImode);
 
@@ -10130,7 +10279,7 @@ ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
   tree fndecl;
   bool call_no_callee_saved_registers = false;
 
-  if (GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
+  if (SYMBOL_REF_P (XEXP (fnaddr, 0)))
     {
       fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
       if (fndecl)
@@ -10167,7 +10316,7 @@ ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
   if (TARGET_MACHO && !TARGET_64BIT)
     {
 #if TARGET_MACHO
-      if (flag_pic && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
+      if (flag_pic && SYMBOL_REF_P (XEXP (fnaddr, 0)))
 	fnaddr = machopic_indirect_call_target (fnaddr);
 #endif
     }
@@ -10177,7 +10326,7 @@ ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
 	 check if PLT was explicitly avoided via no-plt or "noplt" attribute, making
 	 it an indirect call.  */
       if (flag_pic
-	  && GET_CODE (addr) == SYMBOL_REF
+	  && SYMBOL_REF_P (addr)
 	  && ix86_call_use_plt_p (addr))
 	{
 	  if (flag_plt
@@ -10251,7 +10400,7 @@ ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
   if (ix86_cmodel == CM_LARGE_PIC
       && !TARGET_PECOFF
       && MEM_P (fnaddr)
-      && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF
+      && SYMBOL_REF_P (XEXP (fnaddr, 0))
       && !local_symbolic_operand (XEXP (fnaddr, 0), VOIDmode))
     fnaddr = gen_rtx_MEM (QImode, construct_plt_address (XEXP (fnaddr, 0)));
   /* Since x32 GOT slot is 64 bit with zero upper 32 bits, indirect
@@ -10354,7 +10503,7 @@ ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
     }
 
   if (TARGET_MACHO && TARGET_64BIT && !sibcall
-      && ((GET_CODE (addr) == SYMBOL_REF && !SYMBOL_REF_LOCAL_P (addr))
+      && ((SYMBOL_REF_P (addr) && !SYMBOL_REF_LOCAL_P (addr))
 	  || !fndecl || TREE_PUBLIC (fndecl)))
     {
       /* We allow public functions defined in a TU to bind locally for PIC
@@ -12463,7 +12612,7 @@ ix86_expand_args_builtin (const struct builtin_description *d,
 static rtx
 ix86_erase_embedded_rounding (rtx pat)
 {
-  if (GET_CODE (pat) == INSN)
+  if (NONJUMP_INSN_P (pat))
     pat = PATTERN (pat);
 
   gcc_assert (GET_CODE (pat) == SET);
@@ -25178,7 +25327,7 @@ const_vector_equal_evenodd_p (rtx op)
 {
   machine_mode mode = GET_MODE (op);
   int i, nunits = GET_MODE_NUNITS (mode);
-  if (GET_CODE (op) != CONST_VECTOR
+  if (!CONST_VECTOR_P (op)
       || nunits != CONST_VECTOR_NUNITS (op))
     return false;
   for (i = 0; i < nunits; i += 2)
@@ -25521,7 +25670,7 @@ ix86_notrack_prefixed_insn_p (rtx_insn *insn)
 
       /* Do not emit 'notrack' if it's not an indirect call.  */
       if (MEM_P (addr)
-	  && GET_CODE (XEXP (addr, 0)) == SYMBOL_REF)
+	  && SYMBOL_REF_P (XEXP (addr, 0)))
 	return false;
       else
 	return find_reg_note (insn, REG_CALL_NOCF_CHECK, 0);
@@ -26338,8 +26487,8 @@ do_mem_operand:
       if (rtx_equal_p (op, args[2]))
 	return 0xaa;
       /* Check if CONST_VECTOR is the ones-complement of args[2].  */
-      if (GET_CODE (op) == CONST_VECTOR
-	  && GET_CODE (args[2]) == CONST_VECTOR
+      if (CONST_VECTOR_P (op)
+	  && CONST_VECTOR_P (args[2])
 	  && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
 							  op, GET_MODE (op)),
 			  args[2]))
@@ -26352,8 +26501,8 @@ do_mem_operand:
       if (rtx_equal_p (op, args[0]))
 	return 0xf0;
       /* Check if CONST_VECTOR is the ones-complement of args[0].  */
-      if (GET_CODE (op) == CONST_VECTOR
-	  && GET_CODE (args[0]) == CONST_VECTOR
+      if (CONST_VECTOR_P (op)
+	  && CONST_VECTOR_P (args[0])
 	  && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
 							  op, GET_MODE (op)),
 			  args[0]))
@@ -26366,8 +26515,8 @@ do_mem_operand:
       if (rtx_equal_p (op, args[1]))
 	return 0xcc;
       /* Check if CONST_VECTOR is the ones-complement of args[1].  */
-      if (GET_CODE (op) == CONST_VECTOR
-	  && GET_CODE (args[1]) == CONST_VECTOR
+      if (CONST_VECTOR_P (op)
+	  && CONST_VECTOR_P (args[1])
 	  && rtx_equal_p (simplify_const_unary_operation (NOT, GET_MODE (op),
 							  op, GET_MODE (op)),
 			  args[1]))
@@ -26597,15 +26746,6 @@ ix86_expand_ternlog (machine_mode mode, rtx op0, rtx op1, rtx op2, int idx,
 	  && (!op2 || !side_effects_p (op2))
 	  && op0)
 	{
-	  if (GET_MODE (op0) != mode)
-	    op0 = gen_lowpart (mode, op0);
-	  if (!TARGET_64BIT && !register_operand (op0, mode))
-	    {
-	      /* Avoid force_reg (mode, op0).  */
-	      rtx reg = gen_reg_rtx (mode);
-	      emit_move_insn (reg, op0);
-	      op0 = reg;
-	    }
 	  emit_move_insn (target, gen_rtx_XOR (mode, op0, CONSTM1_RTX (mode)));
 	  return target;
 	}
@@ -26630,15 +26770,6 @@ ix86_expand_ternlog (machine_mode mode, rtx op0, rtx op1, rtx op2, int idx,
 	  && (!op2 || !side_effects_p (op2))
 	  && op1)
 	{
-	  if (GET_MODE (op1) != mode)
-	    op1 = gen_lowpart (mode, op1);
-	  if (!TARGET_64BIT && !register_operand (op1, mode))
-	    {
-	      /* Avoid force_reg (mode, op1).  */
-	      rtx reg = gen_reg_rtx (mode);
-	      emit_move_insn (reg, op1);
-	      op1 = reg;
-	    }
 	  emit_move_insn (target, gen_rtx_XOR (mode, op1, CONSTM1_RTX (mode)));
 	  return target;
 	}
@@ -26670,15 +26801,6 @@ ix86_expand_ternlog (machine_mode mode, rtx op0, rtx op1, rtx op2, int idx,
 	  && (!op1 || !side_effects_p (op1))
 	  && op2)
 	{
-	  if (GET_MODE (op2) != mode)
-	    op2 = gen_lowpart (mode, op2);
-	  if (!TARGET_64BIT && !register_operand (op2, mode))
-	    {
-	      /* Avoid force_reg (mode, op2).  */
-	      rtx reg = gen_reg_rtx (mode);
-	      emit_move_insn (reg, op2);
-	      op2 = reg;
-	    }
 	  emit_move_insn (target, gen_rtx_XOR (mode, op2, CONSTM1_RTX (mode)));
 	  return target;
 	}
diff --git a/gcc/config/i386/i386-features.cc b/gcc/config/i386/i386-features.cc
index 054f8d5..53e86c8 100644
--- a/gcc/config/i386/i386-features.cc
+++ b/gcc/config/i386/i386-features.cc
@@ -2141,7 +2141,7 @@ convertible_comparison_p (rtx_insn *insn, enum machine_mode mode)
 
   gcc_assert (GET_CODE (src) == COMPARE);
 
-  if (GET_CODE (dst) != REG
+  if (!REG_P (dst)
       || REGNO (dst) != FLAGS_REG
       || GET_MODE (dst) != CCZmode)
     return false;
@@ -2953,7 +2953,7 @@ rest_of_insert_endbr_and_patchable_area (bool need_endbr,
 
 		  /* Also generate ENDBRANCH for non-tail call which
 		     may return via indirect branch.  */
-		  if (GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
+		  if (SYMBOL_REF_P (XEXP (fnaddr, 0)))
 		    fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
 		  if (fndecl == NULL_TREE)
 		    fndecl = MEM_EXPR (fnaddr);
@@ -3226,7 +3226,7 @@ remove_partial_avx_dependency (void)
 	      break;
 	    }
 
-	  /* Only hanlde conversion here.  */
+	  /* Only handle conversion here.  */
 	  machine_mode src_mode
 	    = convert_p ? GET_MODE (XEXP (src, 0)) : VOIDmode;
 	  switch (src_mode)
@@ -3534,22 +3534,20 @@ ix86_broadcast_inner (rtx op, machine_mode mode,
 		      machine_mode *scalar_mode_p,
 		      x86_cse_kind *kind_p, rtx_insn **insn_p)
 {
-  if (op == const0_rtx || op == CONST0_RTX (mode))
+  switch (standard_sse_constant_p (op, mode))
     {
+    case 1:
       *scalar_mode_p = QImode;
       *kind_p = X86_CSE_CONST0_VECTOR;
       *insn_p = nullptr;
       return const0_rtx;
-    }
-  else if ((GET_MODE_CLASS (mode) == MODE_VECTOR_INT
-	    && (op == constm1_rtx || op == CONSTM1_RTX (mode)))
-	    || (GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT
-		&& float_vector_all_ones_operand (op, mode)))
-    {
+    case 2:
       *scalar_mode_p = QImode;
       *kind_p = X86_CSE_CONSTM1_VECTOR;
       *insn_p = nullptr;
       return constm1_rtx;
+    default:
+      break;
     }
 
   mode = GET_MODE (op);
diff --git a/gcc/config/i386/i386-modes.def b/gcc/config/i386/i386-modes.def
index 2fedbeb..c2db305 100644
--- a/gcc/config/i386/i386-modes.def
+++ b/gcc/config/i386/i386-modes.def
@@ -91,7 +91,6 @@ VECTOR_MODES (FLOAT, 16);     /*         V8HF V4SF V2DF */
 VECTOR_MODES (FLOAT, 32);     /*   V16HF V8SF V4DF V2TF */
 VECTOR_MODES (FLOAT, 64);     /*  V32HF V16SF V8DF V4TF */
 VECTOR_MODES (FLOAT, 128);    /* V64HF V32SF V16DF V8TF */
-VECTOR_MODES (FLOAT, 256);    /* V128HF V64SF V32DF V16TF */
 VECTOR_MODE (FLOAT, HF, 2);   /* 	      	   V2HF */
 VECTOR_MODE (FLOAT, BF, 2);   /* 	      	   V2BF */
 VECTOR_MODE (FLOAT, HF, 6);   /*		   V6HF */
@@ -102,7 +101,6 @@ VECTOR_MODE (INT, QI, 2);     /*                   V2QI */
 VECTOR_MODE (INT, QI, 12);    /*                  V12QI */
 VECTOR_MODE (INT, QI, 14);    /*                  V14QI */
 VECTOR_MODE (INT, HI, 6);     /*                   V6HI */
-VECTOR_MODE (INT, SI, 64);    /* 		  V64SI */
 
 INT_MODE (OI, 32);
 INT_MODE (XI, 64);
diff --git a/gcc/config/i386/i386-options.cc b/gcc/config/i386/i386-options.cc
index 09cb133..09a35ef 100644
--- a/gcc/config/i386/i386-options.cc
+++ b/gcc/config/i386/i386-options.cc
@@ -1362,7 +1362,9 @@ ix86_valid_target_attribute_inner_p (tree fndecl, tree args, char *p_strings[],
 	  arg_ok = opt_enum_arg_to_value (opt, p + opt_len, &value, CL_TARGET);
 	  if (arg_ok)
 	    set_option (opts, enum_opts_set, opt, value,
-			p + opt_len, DK_UNSPECIFIED, input_location,
+			p + opt_len,
+			static_cast<int> (diagnostics::kind::unspecified),
+			input_location,
 			global_dc);
 	  else
 	    {
@@ -2839,7 +2841,9 @@ ix86_option_override_internal (bool main_args_p,
 
   /* Set the default value for -mfentry.  */
   if (!opts_set->x_flag_fentry)
-    opts->x_flag_fentry = TARGET_SEH;
+    opts->x_flag_fentry = (TARGET_SEH
+			   || (TARGET_64BIT_P (opts->x_ix86_isa_flags)
+			       && ENABLE_X86_64_MFENTRY));
   else
     {
       if (!TARGET_64BIT_P (opts->x_ix86_isa_flags) && opts->x_flag_pic
@@ -2850,6 +2854,17 @@ ix86_option_override_internal (bool main_args_p,
 	sorry ("%<-mno-fentry%> isn%'t compatible with SEH");
     }
 
+#ifdef OPTION_GLIBC_P
+  /* -mfentry is supported only on glibc targets.  */
+  if (!opts->x_flag_fentry
+      && OPTION_GLIBC_P (opts)
+      && (TARGET_64BIT_P (opts->x_ix86_isa_flags) || !opts->x_flag_pic)
+      && opts->x_flag_shrink_wrap
+      && opts->x_profile_flag)
+    warning (0, "%<-pg%> without %<-mfentry%> may be unreliable with "
+	     "shrink wrapping");
+#endif
+
   if (TARGET_SEH && TARGET_CALL_MS2SYSV_XLOGUES)
     sorry ("%<-mcall-ms2sysv-xlogues%> isn%'t currently supported with SEH");
 
@@ -3600,6 +3615,18 @@ ix86_handle_cconv_attribute (tree *node, tree name, tree args, int,
       return NULL_TREE;
     }
 
+  if (TARGET_64BIT)
+    {
+      /* Do not warn when emulating the MS ABI.  */
+      if ((TREE_CODE (*node) != FUNCTION_TYPE
+	   && TREE_CODE (*node) != METHOD_TYPE)
+	  || ix86_function_type_abi (*node) != MS_ABI)
+	warning (OPT_Wattributes, "%qE attribute ignored",
+		 name);
+      *no_add_attrs = true;
+      return NULL_TREE;
+    }
+
   /* Can combine regparm with all attributes but fastcall, and thiscall.  */
   if (is_attribute_p ("regparm", name))
     {
@@ -3612,7 +3639,7 @@ ix86_handle_cconv_attribute (tree *node, tree name, tree args, int,
 
       if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node)))
 	{
-	  error ("regparam and thiscall attributes are not compatible");
+	  error ("regparm and thiscall attributes are not compatible");
 	}
 
       cst = TREE_VALUE (args);
@@ -3633,19 +3660,7 @@ ix86_handle_cconv_attribute (tree *node, tree name, tree args, int,
       return NULL_TREE;
     }
 
-  if (TARGET_64BIT)
-    {
-      /* Do not warn when emulating the MS ABI.  */
-      if ((TREE_CODE (*node) != FUNCTION_TYPE
-	   && TREE_CODE (*node) != METHOD_TYPE)
-	  || ix86_function_type_abi (*node) != MS_ABI)
-	warning (OPT_Wattributes, "%qE attribute ignored",
-	         name);
-      *no_add_attrs = true;
-      return NULL_TREE;
-    }
-
-  /* Can combine fastcall with stdcall (redundant) and sseregparm.  */
+  /* Can combine fastcall with sseregparm.  */
   if (is_attribute_p ("fastcall", name))
     {
       if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
@@ -3666,8 +3681,7 @@ ix86_handle_cconv_attribute (tree *node, tree name, tree args, int,
 	}
     }
 
-  /* Can combine stdcall with fastcall (redundant), regparm and
-     sseregparm.  */
+  /* Can combine stdcall with regparm and sseregparm.  */
   else if (is_attribute_p ("stdcall", name))
     {
       if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
@@ -3717,6 +3731,10 @@ ix86_handle_cconv_attribute (tree *node, tree name, tree args, int,
 	{
 	  error ("cdecl and thiscall attributes are not compatible");
 	}
+      if (lookup_attribute ("regparm", TYPE_ATTRIBUTES (*node)))
+	{
+	  error ("regparm and thiscall attributes are not compatible");
+	}
     }
 
   /* Can combine sseregparm with all attributes.  */
diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index b64175d..65e04d3 100644
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -5979,7 +5979,7 @@ symbolic_reference_mentioned_p (rtx op)
   const char *fmt;
   int i;
 
-  if (GET_CODE (op) == SYMBOL_REF || GET_CODE (op) == LABEL_REF)
+  if (SYMBOL_REF_P (op) || LABEL_REF_P (op))
     return true;
 
   fmt = GET_RTX_FORMAT (GET_CODE (op));
@@ -6526,7 +6526,7 @@ output_set_got (rtx dest, rtx label)
 
   xops[0] = dest;
 
-  if (TARGET_VXWORKS_RTP && flag_pic)
+  if (TARGET_VXWORKS_GOTTPIC && TARGET_VXWORKS_RTP && flag_pic)
     {
       /* Load (*VXWORKS_GOTT_BASE) into the PIC register.  */
       xops[2] = gen_rtx_MEM (Pmode,
@@ -10724,8 +10724,7 @@ split_stack_prologue_scratch_regno (void)
 
 static GTY(()) rtx split_stack_fn;
 
-/* A SYMBOL_REF for the more stack function when using the large
-   model.  */
+/* A SYMBOL_REF for the more stack function when using the large model.  */
 
 static GTY(()) rtx split_stack_fn_large;
 
@@ -11413,7 +11412,7 @@ ix86_force_load_from_GOT_p (rtx x, bool call_p)
 	  && (!flag_pic || this_is_asm_operands)
 	  && ix86_cmodel != CM_LARGE
 	  && ix86_cmodel != CM_LARGE_PIC
-	  && GET_CODE (x) == SYMBOL_REF
+	  && SYMBOL_REF_P (x)
 	  && ((!call_p
 	       && (!ix86_direct_extern_access
 		   || (SYMBOL_REF_DECL (x)
@@ -11459,23 +11458,23 @@ ix86_legitimate_constant_p (machine_mode mode, rtx x)
 	  case UNSPEC_TPOFF:
 	  case UNSPEC_NTPOFF:
 	    x = XVECEXP (x, 0, 0);
-	    return (GET_CODE (x) == SYMBOL_REF
+	    return (SYMBOL_REF_P (x)
 		    && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
 	  case UNSPEC_DTPOFF:
 	    x = XVECEXP (x, 0, 0);
-	    return (GET_CODE (x) == SYMBOL_REF
+	    return (SYMBOL_REF_P (x)
 		    && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_DYNAMIC);
 	  case UNSPEC_SECREL32:
 	    x = XVECEXP (x, 0, 0);
-	    return GET_CODE (x) == SYMBOL_REF;
+	    return SYMBOL_REF_P (x);
 	  default:
 	    return false;
 	  }
 
       /* We must have drilled down to a symbol.  */
-      if (GET_CODE (x) == LABEL_REF)
+      if (LABEL_REF_P (x))
 	return true;
-      if (GET_CODE (x) != SYMBOL_REF)
+      if (!SYMBOL_REF_P (x))
 	return false;
       /* FALLTHRU */
 
@@ -11602,11 +11601,11 @@ legitimate_pic_operand_p (rtx x)
 	    return TARGET_64BIT;
 	  case UNSPEC_TPOFF:
 	    x = XVECEXP (inner, 0, 0);
-	    return (GET_CODE (x) == SYMBOL_REF
+	    return (SYMBOL_REF_P (x)
 		    && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
 	  case UNSPEC_SECREL32:
 	    x = XVECEXP (inner, 0, 0);
-	    return GET_CODE (x) == SYMBOL_REF;
+	    return SYMBOL_REF_P (x);
 	  case UNSPEC_MACHOPIC_OFFSET:
 	    return legitimate_pic_address_disp_p (x);
 	  default:
@@ -11657,7 +11656,7 @@ legitimate_pic_address_disp_p (rtx disp)
 	  if (INTVAL (op1) >= 16*1024*1024
 	      || INTVAL (op1) < -16*1024*1024)
 	    break;
-	  if (GET_CODE (op0) == LABEL_REF)
+	  if (LABEL_REF_P (op0))
 	    return true;
 	  if (GET_CODE (op0) == CONST
 	      && GET_CODE (XEXP (op0, 0)) == UNSPEC
@@ -11666,7 +11665,7 @@ legitimate_pic_address_disp_p (rtx disp)
 	  if (GET_CODE (op0) == UNSPEC
 	      && XINT (op0, 1) == UNSPEC_PCREL)
 	    return true;
-	  if (GET_CODE (op0) != SYMBOL_REF)
+	  if (!SYMBOL_REF_P (op0))
 	    break;
 	  /* FALLTHRU */
 
@@ -11731,8 +11730,8 @@ legitimate_pic_address_disp_p (rtx disp)
 	      && XINT (disp, 1) != UNSPEC_PLTOFF))
 	return false;
 
-      if (GET_CODE (XVECEXP (disp, 0, 0)) != SYMBOL_REF
-	  && GET_CODE (XVECEXP (disp, 0, 0)) != LABEL_REF)
+      if (!SYMBOL_REF_P (XVECEXP (disp, 0, 0))
+	  && !LABEL_REF_P (XVECEXP (disp, 0, 0)))
 	return false;
       return true;
     }
@@ -11760,14 +11759,14 @@ legitimate_pic_address_disp_p (rtx disp)
       /* We need to check for both symbols and labels because VxWorks loads
 	 text labels with @GOT rather than @GOTOFF.  See gotoff_operand for
 	 details.  */
-      return (GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
-	      || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF);
+      return (SYMBOL_REF_P (XVECEXP (disp, 0, 0))
+	      || LABEL_REF_P (XVECEXP (disp, 0, 0)));
     case UNSPEC_GOTOFF:
       /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
 	 While ABI specify also 32bit relocation but we don't produce it in
 	 small PIC model at all.  */
-      if ((GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
-	   || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF)
+      if ((SYMBOL_REF_P (XVECEXP (disp, 0, 0))
+	   || LABEL_REF_P (XVECEXP (disp, 0, 0)))
 	  && !TARGET_64BIT)
         return !TARGET_PECOFF && gotoff_operand (XVECEXP (disp, 0, 0), Pmode);
       return false;
@@ -11777,19 +11776,19 @@ legitimate_pic_address_disp_p (rtx disp)
       if (saw_plus)
 	return false;
       disp = XVECEXP (disp, 0, 0);
-      return (GET_CODE (disp) == SYMBOL_REF
+      return (SYMBOL_REF_P (disp)
 	      && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_INITIAL_EXEC);
     case UNSPEC_NTPOFF:
       disp = XVECEXP (disp, 0, 0);
-      return (GET_CODE (disp) == SYMBOL_REF
+      return (SYMBOL_REF_P (disp)
 	      && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_EXEC);
     case UNSPEC_DTPOFF:
       disp = XVECEXP (disp, 0, 0);
-      return (GET_CODE (disp) == SYMBOL_REF
+      return (SYMBOL_REF_P (disp)
 	      && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_DYNAMIC);
     case UNSPEC_SECREL32:
       disp = XVECEXP (disp, 0, 0);
-      return GET_CODE (disp) == SYMBOL_REF;
+      return SYMBOL_REF_P (disp);
     }
 
   return false;
@@ -12131,11 +12130,11 @@ ix86_legitimate_address_p (machine_mode, rtx addr, bool strict,
 	     that never results in lea, this seems to be easier and
 	     correct fix for crash to disable this test.  */
 	}
-      else if (GET_CODE (disp) != LABEL_REF
+      else if (!LABEL_REF_P (disp)
 	       && !CONST_INT_P (disp)
 	       && (GET_CODE (disp) != CONST
 		   || !ix86_legitimate_constant_p (Pmode, disp))
-	       && (GET_CODE (disp) != SYMBOL_REF
+	       && (!SYMBOL_REF_P (disp)
 		   || !ix86_legitimate_constant_p (Pmode, disp)))
 	/* Displacement is not constant.  */
 	return false;
@@ -12242,10 +12241,10 @@ legitimize_pic_address (rtx orig, rtx reg)
       else
 	new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
     }
-  else if ((GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_TLS_MODEL (addr) == 0)
+  else if ((SYMBOL_REF_P (addr) && SYMBOL_REF_TLS_MODEL (addr) == 0)
 	   /* We can't always use @GOTOFF for text labels
 	      on VxWorks, see gotoff_operand.  */
-	   || (TARGET_VXWORKS_RTP && GET_CODE (addr) == LABEL_REF))
+	   || (TARGET_VXWORKS_VAROFF && LABEL_REF_P (addr)))
     {
 #if TARGET_PECOFF
       rtx tmp = legitimize_pe_coff_symbol (addr, true);
@@ -12380,8 +12379,8 @@ legitimize_pic_address (rtx orig, rtx reg)
 		  /* For %rip addressing, we have to use
 		     just disp32, not base nor index.  */
 		  if (TARGET_64BIT
-		      && (GET_CODE (base) == SYMBOL_REF
-			  || GET_CODE (base) == LABEL_REF))
+		      && (SYMBOL_REF_P (base)
+			  || LABEL_REF_P (base)))
 		    base = force_reg (mode, base);
 		  if (GET_CODE (new_rtx) == PLUS
 		      && CONSTANT_P (XEXP (new_rtx, 1)))
@@ -12443,6 +12442,28 @@ static GTY(()) rtx ix86_tls_symbol;
 static rtx
 ix86_tls_get_addr (void)
 {
+  if (cfun->machine->call_saved_registers
+      == TYPE_NO_CALLER_SAVED_REGISTERS)
+    {
+      /* __tls_get_addr doesn't preserve vector registers.  When a
+	 function with no_caller_saved_registers attribute calls
+	 __tls_get_addr, YMM and ZMM registers will be clobbered.
+	 Issue an error and suggest -mtls-dialect=gnu2 in this case.  */
+      if (cfun->machine->func_type == TYPE_NORMAL)
+	error (G_("%<-mtls-dialect=gnu2%> must be used with a function"
+		  " with the %<no_caller_saved_registers%> attribute"));
+      else
+	error (cfun->machine->func_type == TYPE_EXCEPTION
+	       ? G_("%<-mtls-dialect=gnu2%> must be used with an"
+		    " exception service routine")
+	       : G_("%<-mtls-dialect=gnu2%> must be used with an"
+		    " interrupt service routine"));
+      /* Don't issue the same error twice.  */
+      cfun->machine->func_type = TYPE_NORMAL;
+      cfun->machine->call_saved_registers
+	= TYPE_DEFAULT_CALL_SAVED_REGISTERS;
+    }
+
   if (!ix86_tls_symbol)
     {
       const char *sym
@@ -12883,12 +12904,12 @@ ix86_legitimize_address (rtx x, rtx, machine_mode mode)
   bool changed = false;
   unsigned log;
 
-  log = GET_CODE (x) == SYMBOL_REF ? SYMBOL_REF_TLS_MODEL (x) : 0;
+  log = SYMBOL_REF_P (x) ? SYMBOL_REF_TLS_MODEL (x) : 0;
   if (log)
     return legitimize_tls_address (x, (enum tls_model) log, false);
   if (GET_CODE (x) == CONST
       && GET_CODE (XEXP (x, 0)) == PLUS
-      && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
+      && SYMBOL_REF_P (XEXP (XEXP (x, 0), 0))
       && (log = SYMBOL_REF_TLS_MODEL (XEXP (XEXP (x, 0), 0))))
     {
       rtx t = legitimize_tls_address (XEXP (XEXP (x, 0), 0),
@@ -13305,7 +13326,7 @@ ix86_delegitimize_tls_address (rtx orig_x)
   if (GET_CODE (unspec) != UNSPEC || XINT (unspec, 1) != UNSPEC_NTPOFF)
     return orig_x;
   x = XVECEXP (unspec, 0, 0);
-  gcc_assert (GET_CODE (x) == SYMBOL_REF);
+  gcc_assert (SYMBOL_REF_P (x));
   if (unspec != XEXP (addr.disp, 0))
     x = gen_rtx_PLUS (Pmode, x, XEXP (XEXP (addr.disp, 0), 1));
   if (addr.index)
@@ -13472,7 +13493,7 @@ ix86_delegitimize_address_1 (rtx x, bool base_term_p)
       else if (base_term_p
 	       && pic_offset_table_rtx
 	       && !TARGET_MACHO
-	       && !TARGET_VXWORKS_RTP)
+	       && !TARGET_VXWORKS_VAROFF)
 	{
 	  rtx tmp = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
 	  tmp = gen_rtx_MINUS (Pmode, copy_rtx (addend), tmp);
@@ -14669,7 +14690,7 @@ ix86_print_operand (FILE *file, rtx x, int code)
       /* We have patterns that allow zero sets of memory, for instance.
 	 In 64-bit mode, we should probably support all 8-byte vectors,
 	 since we can in fact encode that into an immediate.  */
-      if (GET_CODE (x) == CONST_VECTOR)
+      if (CONST_VECTOR_P (x))
 	{
 	  if (x != CONST0_RTX (GET_MODE (x)))
 	    output_operand_lossage ("invalid vector immediate");
@@ -14699,8 +14720,8 @@ ix86_print_operand (FILE *file, rtx x, int code)
 	      if (ASSEMBLER_DIALECT == ASM_ATT)
 		putc ('$', file);
 	    }
-	  else if (GET_CODE (x) == CONST || GET_CODE (x) == SYMBOL_REF
-		   || GET_CODE (x) == LABEL_REF)
+	  else if (GET_CODE (x) == CONST || SYMBOL_REF_P (x)
+		   || LABEL_REF_P (x))
 	    {
 	      if (ASSEMBLER_DIALECT == ASM_ATT)
 		putc ('$', file);
@@ -14795,8 +14816,8 @@ ix86_print_operand_address_as (FILE *file, rtx addr,
 	  && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
 	symbol = XEXP (XEXP (disp, 0), 0);
 
-      if (GET_CODE (symbol) == LABEL_REF
-	  || (GET_CODE (symbol) == SYMBOL_REF
+      if (LABEL_REF_P (symbol)
+	  || (SYMBOL_REF_P (symbol)
 	      && SYMBOL_REF_TLS_MODEL (symbol) == 0))
 	base = pc_rtx;
     }
@@ -14884,7 +14905,7 @@ ix86_print_operand_address_as (FILE *file, rtx addr,
 	    {
 	      if (flag_pic)
 		output_pic_addr_const (file, disp, 0);
-	      else if (GET_CODE (disp) == LABEL_REF)
+	      else if (LABEL_REF_P (disp))
 		output_asm_label (disp);
 	      else
 		output_addr_const (file, disp);
@@ -14920,7 +14941,7 @@ ix86_print_operand_address_as (FILE *file, rtx addr,
 
 	      if (flag_pic)
 		output_pic_addr_const (file, disp, 0);
-	      else if (GET_CODE (disp) == LABEL_REF)
+	      else if (LABEL_REF_P (disp))
 		output_asm_label (disp);
 	      else if (CONST_INT_P (disp))
 		offset = disp;
@@ -15872,7 +15893,7 @@ ix86_output_addr_diff_elt (FILE *file, int value, int rel)
   gcc_assert (!TARGET_64BIT);
 #endif
   /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand.  */
-  if (TARGET_64BIT || TARGET_VXWORKS_RTP)
+  if (TARGET_64BIT || TARGET_VXWORKS_VAROFF)
     fprintf (file, "%s%s%d-%s%d\n",
 	     directive, LPREFIX, value, LPREFIX, rel);
 #if TARGET_MACHO
@@ -16704,6 +16725,10 @@ ix86_convert_const_vector_to_integer (rtx op, machine_mode mode)
 	  val = wi::insert (val, wv, innermode_bits * i, innermode_bits);
 	}
       break;
+    case E_V1SImode:
+    case E_V1DImode:
+      op = CONST_VECTOR_ELT (op, 0);
+      return INTVAL (op);
     case E_V2HFmode:
     case E_V2BFmode:
     case E_V4HFmode:
@@ -17679,8 +17704,8 @@ ix86_rip_relative_addr_p (struct ix86_address *parts)
 	      && CONST_INT_P (XEXP (symbol, 1)))
 	    symbol = XEXP (symbol, 0);
 
-	  if (GET_CODE (symbol) == LABEL_REF
-	      || (GET_CODE (symbol) == SYMBOL_REF
+	  if (LABEL_REF_P (symbol)
+	      || (SYMBOL_REF_P (symbol)
 		  && SYMBOL_REF_TLS_MODEL (symbol) == 0)
 	      || (GET_CODE (symbol) == UNSPEC
 		  && (XINT (symbol, 1) == UNSPEC_GOTPCREL
@@ -20004,7 +20029,7 @@ ix86_gimple_fold_builtin (gimple_stmt_iterator *gsi)
 	tree utype, ures, vce;
 	utype = unsigned_type_for (TREE_TYPE (arg0));
 	/* PABSB/W/D/Q store the unsigned result in dst, use ABSU_EXPR
-	   instead of ABS_EXPR to hanlde overflow case(TYPE_MIN).  */
+	   instead of ABS_EXPR to handle overflow case(TYPE_MIN).  */
 	ures = gimple_build (&stmts, ABSU_EXPR, utype, arg0);
 	gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
 	loc = gimple_location (stmt);
@@ -21488,8 +21513,7 @@ ix86_hard_regno_nregs (unsigned int regno, machine_mode mode)
   /* Register pair for mask registers.  */
   if (mode == P2QImode || mode == P2HImode)
     return 2;
-  if (mode == V64SFmode || mode == V64SImode)
-    return 4;
+
   return 1;
 }
 
@@ -23087,7 +23111,7 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
 	  /* Make (subreg:V4SI (not:V16QI (reg:V16QI ..)) 0)
 	     cost the same as register.
 	     This is used by avx_cmp<mode>3_ltint_not.  */
-	  if (GET_CODE (unsop0) == SUBREG)
+	  if (SUBREG_P (unsop0))
 	    unsop0 = XEXP (unsop0, 0);
 	  if (GET_CODE (unsop0) == NOT)
 	    unsop0 = XEXP (unsop0, 0);
@@ -23129,7 +23153,7 @@ ix86_rtx_costs (rtx x, machine_mode mode, int outer_code_i, int opno,
 	     So current solution is make constant disp as cheap as possible.  */
 	  if (GET_CODE (addr) == PLUS
 	      && x86_64_immediate_operand (XEXP (addr, 1), Pmode)
-	      /* Only hanlde (reg + disp) since other forms of addr are mostly LEA,
+	      /* Only handle (reg + disp) since other forms of addr are mostly LEA,
 		 there's no additional cost for the plus of disp.  */
 	      && register_operand (XEXP (addr, 0), Pmode))
 	    {
@@ -24785,6 +24809,12 @@ static void map_egpr_constraints (vec<const char *> &constraints)
 	      buf.safe_push (cur[j + 1]);
 	      j++;
 	      break;
+	    case '{':
+	      do
+		{
+		  buf.safe_push (cur[j]);
+		} while (cur[j++] != '}');
+	      break;
 	    default:
 	      buf.safe_push (cur[j]);
 	      break;
@@ -25202,20 +25232,14 @@ asm_preferred_eh_data_format (int code, int global)
   return DW_EH_PE_absptr;
 }
 
-/* Implement targetm.vectorize.builtin_vectorization_cost.  */
+/* Worker for ix86_builtin_vectorization_cost and the fallback calls
+   from ix86_vector_costs::add_stmt_cost.  */
 static int
-ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
-                                 tree vectype, int)
+ix86_default_vector_cost (enum vect_cost_for_stmt type_of_cost,
+			  machine_mode mode)
 {
-  bool fp = false;
-  machine_mode mode = TImode;
+  bool fp = FLOAT_MODE_P (mode);
   int index;
-  if (vectype != NULL)
-    {
-      fp = FLOAT_TYPE_P (vectype);
-      mode = TYPE_MODE (vectype);
-    }
-
   switch (type_of_cost)
     {
       case scalar_stmt:
@@ -25274,14 +25298,14 @@ ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
 			      COSTS_N_INSNS
 				 (ix86_cost->gather_static
 				  + ix86_cost->gather_per_elt
-				    * TYPE_VECTOR_SUBPARTS (vectype)) / 2);
+				    * GET_MODE_NUNITS (mode)) / 2);
 
       case vector_scatter_store:
         return ix86_vec_cost (mode,
 			      COSTS_N_INSNS
 				 (ix86_cost->scatter_static
 				  + ix86_cost->scatter_per_elt
-				    * TYPE_VECTOR_SUBPARTS (vectype)) / 2);
+				    * GET_MODE_NUNITS (mode)) / 2);
 
       case cond_branch_taken:
         return ix86_cost->cond_taken_branch_cost;
@@ -25299,7 +25323,7 @@ ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
 
       case vec_construct:
 	{
-	  int n = TYPE_VECTOR_SUBPARTS (vectype);
+	  int n = GET_MODE_NUNITS (mode);
 	  /* N - 1 element inserts into an SSE vector, the possible
 	     GPR -> XMM move is accounted for in add_stmt_cost.  */
 	  if (GET_MODE_BITSIZE (mode) <= 128)
@@ -25327,6 +25351,17 @@ ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
     }
 }
 
+/* Implement targetm.vectorize.builtin_vectorization_cost.  */
+static int
+ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
+				 tree vectype, int)
+{
+  machine_mode mode = TImode;
+  if (vectype != NULL)
+    mode = TYPE_MODE (vectype);
+  return ix86_default_vector_cost (type_of_cost, mode);
+}
+
 
 /* This function returns the calling abi specific va_list type node.
    It returns  the FNDECL specific va_list type.  */
@@ -25780,7 +25815,7 @@ ix86_vectorize_create_costs (vec_info *vinfo, bool costing_for_scalar)
 unsigned
 ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
 				  stmt_vec_info stmt_info, slp_tree node,
-				  tree vectype, int misalign,
+				  tree vectype, int,
 				  vect_cost_model_location where)
 {
   unsigned retval = 0;
@@ -25798,6 +25833,14 @@ ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
       if (scalar_p)
 	mode = TYPE_MODE (TREE_TYPE (vectype));
     }
+  /* When we are costing a scalar stmt use the scalar stmt to get at the
+     type of the operation.  */
+  else if (scalar_p && stmt_info)
+    if (tree lhs = gimple_get_lhs (stmt_info->stmt))
+      {
+	fp = FLOAT_TYPE_P (TREE_TYPE (lhs));
+	mode = TYPE_MODE (TREE_TYPE (lhs));
+      }
 
   if ((kind == vector_stmt || kind == scalar_stmt)
       && stmt_info
@@ -26111,32 +26154,24 @@ ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
      (AGU and load ports).  Try to account for this by scaling the
      construction cost by the number of elements involved.  */
   if ((kind == vec_construct || kind == vec_to_scalar)
-      && ((stmt_info
-	   && (STMT_VINFO_TYPE (stmt_info) == load_vec_info_type
-	       || STMT_VINFO_TYPE (stmt_info) == store_vec_info_type)
-	   && ((STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) == VMAT_ELEMENTWISE
-		&& (TREE_CODE (DR_STEP (STMT_VINFO_DATA_REF (stmt_info)))
+      && ((node
+	   && (((SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_ELEMENTWISE
+		 || (SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_STRIDED_SLP
+		     && SLP_TREE_LANES (node) == 1))
+		&& (TREE_CODE (DR_STEP (STMT_VINFO_DATA_REF
+					(SLP_TREE_REPRESENTATIVE (node))))
 		    != INTEGER_CST))
-	       || (STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info)
-		   == VMAT_GATHER_SCATTER)))
-	  || (node
-	      && (((SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_ELEMENTWISE
-		    || (SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_STRIDED_SLP
-			&& SLP_TREE_LANES (node) == 1))
-		   && (TREE_CODE (DR_STEP (STMT_VINFO_DATA_REF
-					     (SLP_TREE_REPRESENTATIVE (node))))
-		      != INTEGER_CST))
-		  || (SLP_TREE_MEMORY_ACCESS_TYPE (node)
-		      == VMAT_GATHER_SCATTER)))))
-    {
-      stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign);
+	       || (SLP_TREE_MEMORY_ACCESS_TYPE (node)
+		   == VMAT_GATHER_SCATTER)))))
+    {
+      stmt_cost = ix86_default_vector_cost (kind, mode);
       stmt_cost *= (TYPE_VECTOR_SUBPARTS (vectype) + 1);
     }
   else if ((kind == vec_construct || kind == scalar_to_vec)
 	   && node
 	   && SLP_TREE_DEF_TYPE (node) == vect_external_def)
     {
-      stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign);
+      stmt_cost = ix86_default_vector_cost (kind, mode);
       unsigned i;
       tree op;
       FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
@@ -26200,7 +26235,7 @@ ix86_vector_costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
 	  TREE_VISITED (op) = 0;
     }
   if (stmt_cost == -1)
-    stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign);
+    stmt_cost = ix86_default_vector_cost (kind, mode);
 
   if (kind == vec_perm && vectype
       && GET_MODE_SIZE (TYPE_MODE (vectype)) == 32)
@@ -26295,6 +26330,65 @@ ix86_vector_costs::finish_cost (const vector_costs *scalar_costs)
       && LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant () >= 16)
     m_suggested_epilogue_mode = V8QImode;
 
+  /* When X86_TUNE_AVX512_MASKED_EPILOGUES is enabled try to use
+     a masked epilogue if that doesn't seem detrimental.  */
+  if (loop_vinfo
+      && !LOOP_VINFO_EPILOGUE_P (loop_vinfo)
+      && LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant () > 2
+      && ix86_tune_features[X86_TUNE_AVX512_MASKED_EPILOGUES]
+      && !OPTION_SET_P (param_vect_partial_vector_usage))
+    {
+      bool avoid = false;
+      if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+	  && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) >= 0)
+	{
+	  unsigned int peel_niter
+	    = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
+	  if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
+	    peel_niter += 1;
+	  /* When we know the number of scalar iterations of the epilogue,
+	     avoid masking when a single vector epilog iteration handles
+	     it in full.  */
+	  if (pow2p_hwi ((LOOP_VINFO_INT_NITERS (loop_vinfo) - peel_niter)
+			 % LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant ()))
+	    avoid = true;
+	}
+      if (!avoid && loop_outer (loop_outer (LOOP_VINFO_LOOP (loop_vinfo))))
+	for (auto ddr : LOOP_VINFO_DDRS (loop_vinfo))
+	  {
+	    if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
+	      ;
+	    else if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
+	      ;
+	    else
+	      {
+		int loop_depth
+		    = index_in_loop_nest (LOOP_VINFO_LOOP (loop_vinfo)->num,
+					  DDR_LOOP_NEST (ddr));
+		if (DDR_NUM_DIST_VECTS (ddr) == 1
+		    && DDR_DIST_VECTS (ddr)[0][loop_depth] == 0)
+		  {
+		    /* Avoid the case when there's an outer loop that might
+		       traverse a multi-dimensional array with the inner
+		       loop just executing the masked epilogue with a
+		       read-write where the next outer iteration might
+		       read from the masked part of the previous write,
+		       'n' filling half a vector.
+			 for (j = 0; j < m; ++j)
+			   for (i = 0; i < n; ++i)
+			     a[j][i] = c * a[j][i];  */
+		    avoid = true;
+		    break;
+		  }
+	      }
+	  }
+      if (!avoid)
+	{
+	  m_suggested_epilogue_mode = loop_vinfo->vector_mode;
+	  m_masked_epilogue = 1;
+	}
+    }
+
   vector_costs::finish_cost (scalar_costs);
 }
 
@@ -26731,7 +26825,7 @@ ix86_reloc_rw_mask (void)
 static bool
 symbolic_base_address_p (rtx addr)
 {
-  if (GET_CODE (addr) == SYMBOL_REF)
+  if (SYMBOL_REF_P (addr))
     return true;
 
   if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_GOTOFF)
diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
index 3f7ad68..791f3b9 100644
--- a/gcc/config/i386/i386.h
+++ b/gcc/config/i386/i386.h
@@ -1842,8 +1842,8 @@ typedef struct ix86_args {
 #define STRIP_UNARY(X) (UNARY_P (X) ? XEXP (X, 0) : X)
 
 #define SYMBOLIC_CONST(X)	\
-  (GET_CODE (X) == SYMBOL_REF						\
-   || GET_CODE (X) == LABEL_REF						\
+  (SYMBOL_REF_P (X)							\
+   || LABEL_REF_P (X)							\
    || (GET_CODE (X) == CONST && symbolic_reference_mentioned_p (X)))
 
 /* Max number of args passed in registers.  If this is more than 3, we will
@@ -2465,10 +2465,11 @@ constexpr wide_int_bitmask PTA_ARROWLAKE = PTA_ALDERLAKE | PTA_AVXIFMA
   | PTA_AVXVNNIINT8 | PTA_AVXNECONVERT | PTA_CMPCCXADD | PTA_UINTR;
 constexpr wide_int_bitmask PTA_ARROWLAKE_S = PTA_ARROWLAKE | PTA_AVXVNNIINT16
   | PTA_SHA512 | PTA_SM3 | PTA_SM4;
-constexpr wide_int_bitmask PTA_CLEARWATERFOREST = PTA_SIERRAFOREST
-  | PTA_AVXVNNIINT16 | PTA_SHA512 | PTA_SM3 | PTA_SM4 | PTA_USER_MSR
-  | PTA_PREFETCHI;
-constexpr wide_int_bitmask PTA_PANTHERLAKE = PTA_ARROWLAKE_S | PTA_PREFETCHI;
+constexpr wide_int_bitmask PTA_CLEARWATERFOREST =
+  (PTA_SIERRAFOREST & (~(PTA_KL | PTA_WIDEKL))) | PTA_AVXVNNIINT16 | PTA_SHA512
+  | PTA_SM3 | PTA_SM4 | PTA_USER_MSR | PTA_PREFETCHI;
+constexpr wide_int_bitmask PTA_PANTHERLAKE =
+  (PTA_ARROWLAKE_S & (~(PTA_KL | PTA_WIDEKL))) | PTA_PREFETCHI;
 constexpr wide_int_bitmask PTA_DIAMONDRAPIDS = PTA_GRANITERAPIDS_D
   | PTA_AVXIFMA | PTA_AVXNECONVERT | PTA_AVXVNNIINT16 | PTA_AVXVNNIINT8
   | PTA_CMPCCXADD | PTA_SHA512 | PTA_SM3 | PTA_SM4 | PTA_AVX10_2
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index 21b9f5c..a50475b 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -2968,7 +2968,8 @@
 	(match_operand:SWI248 1 "const_int_operand"))]
   "optimize_insn_for_size_p () && optimize_size > 1
    && operands[1] != const0_rtx
-   && operands[1] != constm1_rtx
+   && (operands[1] != constm1_rtx
+       || (<MODE>mode == DImode && LEGACY_INT_REG_P (operands[0])))
    && IN_RANGE (INTVAL (operands[1]), -128, 127)
    && !ix86_red_zone_used
    && REGNO (operands[0]) != SP_REG"
@@ -12810,8 +12811,8 @@
 		   (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))
 	      (clobber (reg:CC FLAGS_REG))])]
 {
-  if (GET_CODE (operands[2]) == SYMBOL_REF
-      || GET_CODE (operands[2]) == LABEL_REF)
+  if (SYMBOL_REF_P (operands[2])
+      || LABEL_REF_P (operands[2]))
     {
       operands[2] = shallow_copy_rtx (operands[2]);
       PUT_MODE (operands[2], SImode);
@@ -14769,6 +14770,17 @@
 		   (compare:CCZ (neg:SWI (match_dup 0)) (const_int 0)))
 	      (set (match_dup 0) (neg:SWI (match_dup 0)))])])
 
+;; Optimize *negsi_1 followed by *cmpsi_ccno_1 (PR target/91384) with APX_F
+(define_peephole2
+  [(parallel [(set (match_operand:SWI 0 "general_reg_operand")
+		   (neg:SWI (match_operand:SWI 1 "general_reg_operand")))
+	      (clobber (reg:CC FLAGS_REG))])
+   (set (reg:CCZ FLAGS_REG) (compare:CCZ (match_dup 1) (const_int 0)))]
+  "TARGET_APX_NDD"
+  [(parallel [(set (reg:CCZ FLAGS_REG)
+		   (compare:CCZ (neg:SWI (match_dup 1)) (const_int 0)))
+	      (set (match_dup 0) (neg:SWI (match_dup 1)))])])
+
 ;; Special expand pattern to handle integer mode abs
 
 (define_expand "abs<mode>2"
@@ -20102,7 +20114,7 @@
 
       /* We can't use @GOTOFF for text labels on VxWorks;
 	 see gotoff_operand.  */
-      if (TARGET_64BIT || TARGET_VXWORKS_RTP)
+      if (TARGET_64BIT || TARGET_VXWORKS_VAROFF)
 	{
 	  code = PLUS;
 	  op0 = operands[0];
@@ -20970,7 +20982,7 @@
       (clobber (reg:CC FLAGS_REG))])]
   "!TARGET_64BIT"
 {
-  if (flag_pic && !TARGET_VXWORKS_RTP)
+  if (flag_pic && !TARGET_VXWORKS_GOTTPIC)
     ix86_pc_thunk_call_expanded = true;
 })
 
@@ -20991,7 +21003,7 @@
       (clobber (reg:CC FLAGS_REG))])]
   "!TARGET_64BIT"
 {
-  if (flag_pic && !TARGET_VXWORKS_RTP)
+  if (flag_pic && !TARGET_VXWORKS_GOTTPIC)
     ix86_pc_thunk_call_expanded = true;
 })
 
diff --git a/gcc/config/i386/mmx.md b/gcc/config/i386/mmx.md
index 7920232..1f97993 100644
--- a/gcc/config/i386/mmx.md
+++ b/gcc/config/i386/mmx.md
@@ -81,12 +81,13 @@
 ;; 4-byte and 2-byte QImode vector modes
 (define_mode_iterator VI1_16_32 [V4QI V2QI])
 
-;; All 2-byte, 4-byte and 8-byte vector modes with more than 1 element
+;; All 2-byte, 4-byte and 8-byte vector modes.
 (define_mode_iterator V_16_32_64
-   [V2QI V4QI V2HI V2HF
+   [V2QI V4QI V2HI V1SI V2HF V2BF
     (V8QI "TARGET_64BIT") (V4HI "TARGET_64BIT")
     (V4HF "TARGET_64BIT") (V4BF "TARGET_64BIT")
-    (V2SI "TARGET_64BIT") (V2SF "TARGET_64BIT")])
+    (V2SI "TARGET_64BIT") (V2SF "TARGET_64BIT")
+    (V1DI "TARGET_64BIT")])
 
 ;; V2S* modes
 (define_mode_iterator V2FI [V2SF V2SI])
@@ -107,6 +108,7 @@
   [(V8QI "DI") (V4QI "SI") (V2QI "HI")
    (V4HI "DI") (V2HI "SI")
    (V2SI "DI")
+   (V1DI "DI") (V1SI "SI")
    (V4HF "DI") (V2HF "SI")
    (V4BF "DI") (V2BF "SI")
    (V2SF "DI")])
@@ -329,7 +331,7 @@
 
 (define_expand "mov<mode>"
   [(set (match_operand:V_32 0 "nonimmediate_operand")
-	(match_operand:V_32 1 "nonimmediate_operand"))]
+	(match_operand:V_32 1 "nonimm_or_0_operand"))]
   ""
 {
   ix86_expand_vector_move (<MODE>mode, operands);
@@ -339,7 +341,7 @@
 (define_insn "*mov<mode>_internal"
   [(set (match_operand:V_32 0 "nonimmediate_operand"
     "=r ,m ,v,v,v,m,r,v")
-	(match_operand:V_32 1 "general_operand"
+	(match_operand:V_32 1 "nonimm_or_0_operand"
     "rmC,rC,C,v,m,v,v,r"))]
   "!(MEM_P (operands[0]) && MEM_P (operands[1]))
    && ix86_hardreg_mov_ok (operands[0], operands[1])"
@@ -407,22 +409,6 @@
 	   ]
 	   (symbol_ref "true")))])
 
-;; 16-bit, 32-bit and 64-bit constant vector stores.  After reload,
-;; convert them to immediate integer stores.
-(define_insn_and_split "*mov<mode>_imm"
-  [(set (match_operand:V_16_32_64 0 "memory_operand" "=m")
-	(match_operand:V_16_32_64 1 "x86_64_const_vector_operand" "i"))]
-  ""
-  "#"
-  "&& reload_completed"
-  [(set (match_dup 0) (match_dup 1))]
-{
-  HOST_WIDE_INT val = ix86_convert_const_vector_to_integer (operands[1],
-							    <MODE>mode);
-  operands[1] = GEN_INT (val);
-  operands[0] = lowpart_subreg (<mmxinsnmode>mode, operands[0], <MODE>mode);
-})
-
 ;; For TARGET_64BIT we always round up to 8 bytes.
 (define_insn "*push<mode>2_rex64"
   [(set (match_operand:V_32 0 "push_operand" "=X,X")
@@ -457,7 +443,7 @@
 
 (define_expand "movv2qi"
   [(set (match_operand:V2QI 0 "nonimmediate_operand")
-	(match_operand:V2QI 1 "nonimmediate_operand"))]
+	(match_operand:V2QI 1 "nonimm_or_0_operand"))]
   ""
 {
   ix86_expand_vector_move (V2QImode, operands);
@@ -467,9 +453,10 @@
 (define_insn "*movv2qi_internal"
   [(set (match_operand:V2QI 0 "nonimmediate_operand"
     "=r,r,r,m ,v,v,v,jm,m,r,v")
-	(match_operand:V2QI 1 "general_operand"
+	(match_operand:V2QI 1 "nonimm_or_0_operand"
     "r ,C,m,rC,C,v,m,x,v,v,r"))]
-  "!(MEM_P (operands[0]) && MEM_P (operands[1]))"
+  "!(MEM_P (operands[0]) && MEM_P (operands[1]))
+   && ix86_hardreg_mov_ok (operands[0], operands[1])"
 {
   switch (get_attr_type (insn))
     {
@@ -587,6 +574,42 @@
 	   ]
 	   (symbol_ref "true")))])
 
+(define_split
+  [(set (match_operand:V_16_32_64 0 "general_reg_operand")
+	(match_operand:V_16_32_64 1 "memory_operand"))]
+  "reload_completed
+   && SYMBOL_REF_P (XEXP (operands[1], 0))
+   && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0))"
+  [(set (match_dup 0) (match_dup 1))]
+{
+  rtx op1 = avoid_constant_pool_reference (operands[1]);
+
+  if (!CONST_VECTOR_P (op1))
+    FAIL;
+
+  HOST_WIDE_INT val = ix86_convert_const_vector_to_integer (op1, <MODE>mode);
+
+  operands[0] = lowpart_subreg (<mmxinsnmode>mode, operands[0], <MODE>mode);
+  operands[1] = GEN_INT (val);
+})
+
+;; 16-bit, 32-bit and 64-bit constant vector stores.  After reload,
+;; convert them to immediate integer stores.
+(define_insn_and_split "*mov<mode>_imm"
+  [(set (match_operand:V_16_32_64 0 "memory_operand" "=m")
+	(match_operand:V_16_32_64 1 "x86_64_const_vector_operand" "i"))]
+  ""
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0) (match_dup 1))]
+{
+  rtx op1 = operands[1];
+  HOST_WIDE_INT val = ix86_convert_const_vector_to_integer (op1, <MODE>mode);
+
+  operands[0] = adjust_address (operands[0], <mmxinsnmode>mode, 0);
+  operands[1] = GEN_INT (val);
+})
+
 ;; We always round up to UNITS_PER_WORD bytes.
 (define_insn "*pushv2qi2"
   [(set (match_operand:V2QI 0 "push_operand" "=X,X")
diff --git a/gcc/config/i386/predicates.md b/gcc/config/i386/predicates.md
index 1bd63b2..b2d2eec 100644
--- a/gcc/config/i386/predicates.md
+++ b/gcc/config/i386/predicates.md
@@ -573,8 +573,8 @@
 
     case CONST:
       op = XEXP (op, 0);
-      if (GET_CODE (op) == SYMBOL_REF
-	  || GET_CODE (op) == LABEL_REF
+      if (SYMBOL_REF_P (op)
+	  || LABEL_REF_P (op)
 	  || (GET_CODE (op) == UNSPEC
 	      && (XINT (op, 1) == UNSPEC_GOT
 		  || XINT (op, 1) == UNSPEC_GOTOFF
@@ -586,8 +586,8 @@
 	return false;
 
       op = XEXP (op, 0);
-      if (GET_CODE (op) == SYMBOL_REF
-	  || GET_CODE (op) == LABEL_REF)
+      if (SYMBOL_REF_P (op)
+	  || LABEL_REF_P (op))
 	return true;
       /* Only @GOTOFF gets offsets.  */
       if (GET_CODE (op) != UNSPEC
@@ -595,8 +595,8 @@
 	return false;
 
       op = XVECEXP (op, 0, 0);
-      if (GET_CODE (op) == SYMBOL_REF
-	  || GET_CODE (op) == LABEL_REF)
+      if (SYMBOL_REF_P (op)
+	  || LABEL_REF_P (op))
 	return true;
       return false;
 
@@ -614,10 +614,10 @@
       && CONST_INT_P (XEXP (XEXP (op, 0), 1)))
     op = XEXP (XEXP (op, 0), 0);
 
-  if (GET_CODE (op) == LABEL_REF)
+  if (LABEL_REF_P (op))
     return true;
 
-  if (GET_CODE (op) != SYMBOL_REF)
+  if (!SYMBOL_REF_P (op))
     return false;
 
   if (SYMBOL_REF_TLS_MODEL (op))
@@ -649,7 +649,7 @@
       && CONST_INT_P (XEXP (XEXP (op, 0), 1)))
     op = XEXP (XEXP (op, 0), 0);
 
-  if (GET_CODE (op) == SYMBOL_REF
+  if (SYMBOL_REF_P (op)
       && !SYMBOL_REF_FUNCTION_P (op))
     return false;
 
@@ -664,8 +664,9 @@
 ;; same segment as the GOT.  Unfortunately, the flexibility of linker
 ;; scripts means that we can't be sure of that in general, so assume
 ;; @GOTOFF is not valid on VxWorks, except with the large code model.
+;; The comments above seem to apply only to VxWorks releases before 7.
 (define_predicate "gotoff_operand"
-  (and (ior (not (match_test "TARGET_VXWORKS_RTP"))
+  (and (ior (not (match_test "TARGET_VXWORKS_VAROFF"))
             (match_test "ix86_cmodel == CM_LARGE")
             (match_test "ix86_cmodel == CM_LARGE_PIC"))
        (match_operand 0 "local_symbolic_operand")))
@@ -1144,7 +1145,7 @@
   unsigned n_elts;
   op = avoid_constant_pool_reference (op);
 
-  if (GET_CODE (op) != CONST_VECTOR)
+  if (!CONST_VECTOR_P (op))
     return false;
 
   n_elts = CONST_VECTOR_NUNITS (op);
@@ -1172,7 +1173,7 @@
   if (MEM_P (op))
     {
       op = get_pool_constant (XEXP (op, 0));
-      if (GET_CODE (op) != CONST_VECTOR)
+      if (!CONST_VECTOR_P (op))
 	return false;
 
       if (GET_MODE (op) != mode
@@ -1421,8 +1422,8 @@
 	}
       if (TARGET_64BIT
 	  && flag_pic
-	  && (GET_CODE (disp) == SYMBOL_REF
-	      || GET_CODE (disp) == LABEL_REF))
+	  && (SYMBOL_REF_P (disp)
+	      || LABEL_REF_P (disp)))
 	return false;
     }
 
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index 252ba07..ec74f93 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -12756,7 +12756,7 @@
 	     lo insns have =m and 0C constraints.  */
 	  : (operands[2] != const0_rtx
 	     || (!rtx_equal_p (dest, operands[3])
-		 && GET_CODE (operands[3]) != CONST_VECTOR))))
+		 && !CONST_VECTOR_P (operands[3])))))
     dest = gen_reg_rtx (<ssehalfvecmode>mode);
   switch (INTVAL (operands[2]))
     {
@@ -21729,6 +21729,19 @@
 	   (const_string "orig")))
    (set_attr "mode" "TI,TI,TI,TI,TI,TI,V4SF,V2SF,V2SF")])
 
+;; Eliminate redundancy caused by
+;; /* Special case TImode to 128-bit vector conversions via V2DI.  */
+;; in ix86_expand_vector_move
+
+(define_split
+  [(set (match_operand:V2DI 0 "register_operand")
+	(vec_concat:V2DI
+	  (subreg:DI (match_operand:TI 1 "register_operand") 0)
+	  (subreg:DI (match_dup 1) 8)))]
+  "TARGET_SSE2 && ix86_pre_reload_split ()"
+  [(set (match_dup 0)
+	(subreg:V2DI (match_dup 1) 0))])
+
 (define_insn "*vec_concatv2di_0"
   [(set (match_operand:V2DI 0 "register_operand"     "=v,v ,x")
 	(vec_concat:V2DI
diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
index 91cdca7..a86cbad 100644
--- a/gcc/config/i386/x86-tune.def
+++ b/gcc/config/i386/x86-tune.def
@@ -31,7 +31,7 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 	- Updating ix86_issue_rate and ix86_adjust_cost in i386.md
 	- possibly updating ia32_multipass_dfa_lookahead, ix86_sched_reorder
 	  and ix86_sched_init_global if those tricks are needed.
-    - Tunning the flags bellow. Those are split into sections and each
+    - tuning flags below; those are split into sections and each
       section is very roughly ordered by importance.  */
 
 /*****************************************************************************/
@@ -639,6 +639,11 @@ DEF_TUNE (X86_TUNE_AVX512_STORE_BY_PIECES, "avx512_store_by_pieces",
 DEF_TUNE (X86_TUNE_AVX512_TWO_EPILOGUES, "avx512_two_epilogues",
 	  m_ZNVER4 | m_ZNVER5)
 
+/* X86_TUNE_AVX512_MAKED_EPILOGUES: Use two masked vector epilogues
+   when fit.  */
+DEF_TUNE (X86_TUNE_AVX512_MASKED_EPILOGUES, "avx512_masked_epilogues",
+	  m_ZNVER4 | m_ZNVER5)
+
 /*****************************************************************************/
 /*****************************************************************************/
 /* Historical relics: tuning flags that helps a specific old CPU designs     */
diff --git a/gcc/config/loongarch/lasx.md b/gcc/config/loongarch/lasx.md
index 43e3ab0..3d71f30 100644
--- a/gcc/config/loongarch/lasx.md
+++ b/gcc/config/loongarch/lasx.md
@@ -2060,9 +2060,9 @@
   [(set_attr "type" "simd_int_arith")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "lasx_xvshuf_<lasxfmt_f>"
+(define_insn "@lasx_xvshuf_<lasxfmt_f>"
   [(set (match_operand:LASX_DWH 0 "register_operand" "=f")
-	(unspec:LASX_DWH [(match_operand:LASX_DWH 1 "register_operand" "0")
+	(unspec:LASX_DWH [(match_operand:<VIMODE> 1 "register_operand" "0")
 			  (match_operand:LASX_DWH 2 "register_operand" "f")
 			  (match_operand:LASX_DWH 3 "register_operand" "f")]
 			UNSPEC_LASX_XVSHUF))]
diff --git a/gcc/config/loongarch/loongarch.cc b/gcc/config/loongarch/loongarch.cc
index f62e416..493f95e 100644
--- a/gcc/config/loongarch/loongarch.cc
+++ b/gcc/config/loongarch/loongarch.cc
@@ -8380,7 +8380,7 @@ static bool
 loongarch_try_expand_lsx_vshuf_const (struct expand_vec_perm_d *d)
 {
   int i;
-  rtx target, op0, op1, sel, tmp;
+  rtx target, op0, op1;
   rtx rperm[MAX_VECT_LEN];
 
   if (GET_MODE_SIZE (d->vmode) == 16)
@@ -8399,47 +8399,23 @@ loongarch_try_expand_lsx_vshuf_const (struct expand_vec_perm_d *d)
       for (i = 0; i < d->nelt; i += 1)
 	  rperm[i] = GEN_INT (d->perm[i]);
 
-      if (d->vmode == E_V2DFmode)
-	{
-	  sel = gen_rtx_CONST_VECTOR (E_V2DImode, gen_rtvec_v (d->nelt, rperm));
-	  tmp = simplify_gen_subreg (E_V2DImode, d->target, d->vmode, 0);
-	  emit_move_insn (tmp, sel);
-	}
-      else if (d->vmode == E_V4SFmode)
-	{
-	  sel = gen_rtx_CONST_VECTOR (E_V4SImode, gen_rtvec_v (d->nelt, rperm));
-	  tmp = simplify_gen_subreg (E_V4SImode, d->target, d->vmode, 0);
-	  emit_move_insn (tmp, sel);
-	}
+      machine_mode sel_mode = related_int_vector_mode (d->vmode)
+	.require ();
+      rtvec sel_v = gen_rtvec_v (d->nelt, rperm);
+
+      /* Despite vshuf.* (except vshuf.b) needs sel == target, we cannot
+	 load sel into target right now: here we are dealing with
+	 pseudo regs, and target may be the same pseudo as one of op0
+	 or op1.  Then we'd clobber the input.  Instead, we use a new
+	 pseudo reg here.  The reload pass will look at the constraint
+	 of vshuf.* and move sel into target first if needed.  */
+      rtx sel = force_reg (sel_mode,
+			   gen_rtx_CONST_VECTOR (sel_mode, sel_v));
+
+      if (d->vmode == E_V16QImode)
+	emit_insn (gen_lsx_vshuf_b (target, op1, op0, sel));
       else
-	{
-	  sel = gen_rtx_CONST_VECTOR (d->vmode, gen_rtvec_v (d->nelt, rperm));
-	  emit_move_insn (d->target, sel);
-	}
-
-      switch (d->vmode)
-	{
-	case E_V2DFmode:
-	  emit_insn (gen_lsx_vshuf_d_f (target, target, op1, op0));
-	  break;
-	case E_V2DImode:
-	  emit_insn (gen_lsx_vshuf_d (target, target, op1, op0));
-	  break;
-	case E_V4SFmode:
-	  emit_insn (gen_lsx_vshuf_w_f (target, target, op1, op0));
-	  break;
-	case E_V4SImode:
-	  emit_insn (gen_lsx_vshuf_w (target, target, op1, op0));
-	  break;
-	case E_V8HImode:
-	  emit_insn (gen_lsx_vshuf_h (target, target, op1, op0));
-	  break;
-	case E_V16QImode:
-	  emit_insn (gen_lsx_vshuf_b (target, op1, op0, target));
-	  break;
-	default:
-	  break;
-	}
+	emit_insn (gen_lsx_vshuf (d->vmode, target, sel, op1, op0));
 
       return true;
     }
@@ -9435,7 +9411,7 @@ loongarch_expand_vec_perm_const (struct expand_vec_perm_d *d)
   bool flag = false;
   unsigned int i;
   unsigned char idx;
-  rtx target, op0, op1, sel, tmp;
+  rtx target, op0, op1;
   rtx rperm[MAX_VECT_LEN];
   unsigned int remapped[MAX_VECT_LEN];
   unsigned char perm2[MAX_VECT_LEN];
@@ -9615,63 +9591,23 @@ loongarch_expand_vec_perm_const (struct expand_vec_perm_d *d)
 expand_perm_const_end:
       if (flag)
 	{
-	  /* Copy selector vector from memory to vector register for later insn
-	     gen function.
-	     If vector's element in floating point value, we cannot fit
-	     selector argument into insn gen function directly, because of the
-	     insn template definition.  As a solution, generate a integral mode
-	     subreg of target, then copy selector vector (that is in integral
-	     mode) to this subreg.  */
-	  switch (d->vmode)
-	    {
-	    case E_V4DFmode:
-	      sel = gen_rtx_CONST_VECTOR (E_V4DImode, gen_rtvec_v (d->nelt,
-								   rperm));
-	      tmp = simplify_gen_subreg (E_V4DImode, d->target, d->vmode, 0);
-	      emit_move_insn (tmp, sel);
-	      break;
-	    case E_V8SFmode:
-	      sel = gen_rtx_CONST_VECTOR (E_V8SImode, gen_rtvec_v (d->nelt,
-								   rperm));
-	      tmp = simplify_gen_subreg (E_V8SImode, d->target, d->vmode, 0);
-	      emit_move_insn (tmp, sel);
-	      break;
-	    default:
-	      sel = gen_rtx_CONST_VECTOR (d->vmode, gen_rtvec_v (d->nelt,
-								 rperm));
-	      emit_move_insn (d->target, sel);
-	      break;
-	    }
-
 	  target = d->target;
 	  op0 = d->op0;
 	  op1 = d->one_vector_p ? d->op0 : d->op1;
 
-	  /* We FINALLY can generate xvshuf.* insn.  */
-	  switch (d->vmode)
-	    {
-	    case E_V4DFmode:
-	      emit_insn (gen_lasx_xvshuf_d_f (target, target, op1, op0));
-	      break;
-	    case E_V4DImode:
-	      emit_insn (gen_lasx_xvshuf_d (target, target, op1, op0));
-	      break;
-	    case E_V8SFmode:
-	      emit_insn (gen_lasx_xvshuf_w_f (target, target, op1, op0));
-	      break;
-	    case E_V8SImode:
-	      emit_insn (gen_lasx_xvshuf_w (target, target, op1, op0));
-	      break;
-	    case E_V16HImode:
-	      emit_insn (gen_lasx_xvshuf_h (target, target, op1, op0));
-	      break;
-	    case E_V32QImode:
-	      emit_insn (gen_lasx_xvshuf_b (target, op1, op0, target));
-	      break;
-	    default:
-	      gcc_unreachable ();
-	      break;
-	    }
+	  machine_mode sel_mode = related_int_vector_mode (d->vmode)
+	    .require ();
+	  rtvec sel_v = gen_rtvec_v (d->nelt, rperm);
+
+	  /* See the comment in loongarch_expand_lsx_shuffle for why
+	     we don't simply use a SUBREG to pun target.  */
+	  rtx sel = force_reg (sel_mode,
+			       gen_rtx_CONST_VECTOR (sel_mode, sel_v));
+
+	  if (d->vmode == E_V32QImode)
+	    emit_insn (gen_lasx_xvshuf_b (target, op1, op0, sel));
+	  else
+	    emit_insn (gen_lasx_xvshuf (d->vmode, target, sel, op1, op0));
 
 	  return true;
 	}
@@ -11116,17 +11052,21 @@ static bool
 loongarch_builtin_support_vector_misalignment (machine_mode mode,
 					       const_tree type,
 					       int misalignment,
-					       bool is_packed)
+					       bool is_packed,
+					       bool is_gather_scatter)
 {
   if ((ISA_HAS_LSX || ISA_HAS_LASX) && STRICT_ALIGNMENT)
     {
+      if (is_gather_scatter)
+	return true;
       if (optab_handler (movmisalign_optab, mode) == CODE_FOR_nothing)
 	return false;
       if (misalignment == -1)
 	return false;
     }
   return default_builtin_support_vector_misalignment (mode, type, misalignment,
-						      is_packed);
+						      is_packed,
+						      is_gather_scatter);
 }
 
 /* Return a PARALLEL containing NELTS elements, with element I equal
diff --git a/gcc/config/loongarch/loongarch.h b/gcc/config/loongarch/loongarch.h
index d897763..5fc8665 100644
--- a/gcc/config/loongarch/loongarch.h
+++ b/gcc/config/loongarch/loongarch.h
@@ -823,8 +823,6 @@ typedef struct {
 
 #define CASE_VECTOR_MODE Pmode
 
-#define CASE_VECTOR_SHORTEN_MODE(MIN, MAX, BODY) Pmode
-
 /* Define this as 1 if `char' should by default be signed; else as 0.  */
 #ifndef DEFAULT_SIGNED_CHAR
 #define DEFAULT_SIGNED_CHAR 1
diff --git a/gcc/config/loongarch/loongarch.md b/gcc/config/loongarch/loongarch.md
index f7005de..32ef980 100644
--- a/gcc/config/loongarch/loongarch.md
+++ b/gcc/config/loongarch/loongarch.md
@@ -4573,9 +4573,10 @@
   "&& true"
   [(set (match_dup 3) (match_dup 2))
    (set (match_dup 0)
-	(unspec:SI [(match_dup 3) (subreg:SI (match_dup 1) 0)] CRC))]
+	(unspec:SI [(match_dup 3) (match_dup 1)] CRC))]
   {
     operands[3] = gen_reg_rtx (<MODE>mode);
+    operands[1] = lowpart_subreg (SImode, operands[1], DImode);
   })
 
 ;; With normal or medium code models, if the only use of a pc-relative
diff --git a/gcc/config/loongarch/lsx.md b/gcc/config/loongarch/lsx.md
index 407c868..fb0236b 100644
--- a/gcc/config/loongarch/lsx.md
+++ b/gcc/config/loongarch/lsx.md
@@ -535,9 +535,9 @@
   DONE;
 })
 
-(define_insn "lsx_vshuf_<lsxfmt_f>"
+(define_insn "@lsx_vshuf_<lsxfmt_f>"
   [(set (match_operand:LSX_DWH 0 "register_operand" "=f")
-	(unspec:LSX_DWH [(match_operand:LSX_DWH 1 "register_operand" "0")
+	(unspec:LSX_DWH [(match_operand:<VIMODE> 1 "register_operand" "0")
 			 (match_operand:LSX_DWH 2 "register_operand" "f")
 			 (match_operand:LSX_DWH 3 "register_operand" "f")]
 			UNSPEC_LSX_VSHUF))]
diff --git a/gcc/config/nvptx/nvptx.opt b/gcc/config/nvptx/nvptx.opt
index d326ca4..9796839 100644
--- a/gcc/config/nvptx/nvptx.opt
+++ b/gcc/config/nvptx/nvptx.opt
@@ -120,6 +120,51 @@ Target RejectNegative Alias(misa=,sm_89)
 march-map=sm_90a
 Target RejectNegative Alias(misa=,sm_89)
 
+march-map=sm_100
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_100f
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_100a
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_101
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_101f
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_101a
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_103
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_103f
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_103a
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_120
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_120f
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_120a
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_121
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_121f
+Target RejectNegative Alias(misa=,sm_89)
+
+march-map=sm_121a
+Target RejectNegative Alias(misa=,sm_89)
+
 Enum
 Name(ptx_version) Type(enum ptx_version)
 Known PTX ISA versions (for use with the -mptx= option):
diff --git a/gcc/config/pru/pru-pragma.cc b/gcc/config/pru/pru-pragma.cc
index c3f3d33..9338780 100644
--- a/gcc/config/pru/pru-pragma.cc
+++ b/gcc/config/pru/pru-pragma.cc
@@ -46,21 +46,24 @@ pru_pragma_ctable_entry (cpp_reader *)
   enum cpp_ttype type;
 
   type = pragma_lex (&ctable_index);
-  if (type == CPP_NUMBER && tree_fits_uhwi_p (ctable_index))
+  if (type == CPP_NUMBER && tree_fits_shwi_p (ctable_index))
     {
       type = pragma_lex (&base_addr);
-      if (type == CPP_NUMBER  && tree_fits_uhwi_p (base_addr))
+      if (type == CPP_NUMBER && tree_fits_shwi_p (base_addr))
 	{
-	  unsigned HOST_WIDE_INT i = tree_to_uhwi (ctable_index);
-	  unsigned HOST_WIDE_INT base = tree_to_uhwi (base_addr);
+	  HOST_WIDE_INT i = tree_to_shwi (ctable_index);
+	  HOST_WIDE_INT base = sext_hwi (tree_to_shwi (base_addr),
+					 POINTER_SIZE);
 
 	  type = pragma_lex (&base_addr);
 	  if (type != CPP_EOF)
 	    error ("junk at end of %<#pragma CTABLE_ENTRY%>");
-	  else if (i >= ARRAY_SIZE (pru_ctable))
+	  else if (!IN_RANGE (i, 0, ARRAY_SIZE (pru_ctable) - 1))
 	    error ("%<CTABLE_ENTRY%> index %wd is not valid", i);
 	  else if (pru_ctable[i].valid && pru_ctable[i].base != base)
 	    error ("redefinition of %<CTABLE_ENTRY %wd%>", i);
+	  else if (!IN_RANGE (base, INT32_MIN, INT32_MAX))
+	    error ("%<CTABLE_ENTRY%> base address does not fit in 32-bits");
 	  else
 	    {
 	      if (base & 0xff)
diff --git a/gcc/config/pru/pru-protos.h b/gcc/config/pru/pru-protos.h
index c73fad8..4750f0e 100644
--- a/gcc/config/pru/pru-protos.h
+++ b/gcc/config/pru/pru-protos.h
@@ -23,7 +23,7 @@
 
 struct pru_ctable_entry {
     bool valid;
-    unsigned HOST_WIDE_INT base;
+    HOST_WIDE_INT base;
 };
 
 extern struct pru_ctable_entry pru_ctable[32];
@@ -66,9 +66,9 @@ pru_regno_ok_for_index_p (int regno, bool strict_p)
   return pru_regno_ok_for_base_p (regno, strict_p);
 }
 
-extern int pru_get_ctable_exact_base_index (unsigned HOST_WIDE_INT caddr);
-extern int pru_get_ctable_base_index (unsigned HOST_WIDE_INT caddr);
-extern int pru_get_ctable_base_offset (unsigned HOST_WIDE_INT caddr);
+extern int pru_get_ctable_exact_base_index (HOST_WIDE_INT caddr);
+extern int pru_get_ctable_base_index (HOST_WIDE_INT caddr);
+extern int pru_get_ctable_base_offset (HOST_WIDE_INT caddr);
 
 extern int pru_symref2ioregno (rtx op);
 
diff --git a/gcc/config/pru/pru.cc b/gcc/config/pru/pru.cc
index 47e5f24..322e319 100644
--- a/gcc/config/pru/pru.cc
+++ b/gcc/config/pru/pru.cc
@@ -1428,7 +1428,7 @@ pru_valid_const_ubyte_offset (machine_mode mode, HOST_WIDE_INT offset)
 /* Recognize a CTABLE base address.  Return CTABLE entry index, or -1 if
    base was not found in the pragma-filled pru_ctable.  */
 int
-pru_get_ctable_exact_base_index (unsigned HOST_WIDE_INT caddr)
+pru_get_ctable_exact_base_index (HOST_WIDE_INT caddr)
 {
   unsigned int i;
 
@@ -1444,7 +1444,7 @@ pru_get_ctable_exact_base_index (unsigned HOST_WIDE_INT caddr)
 /* Check if the given address can be addressed via CTABLE_BASE + UBYTE_OFFS,
    and return the base CTABLE index if possible.  */
 int
-pru_get_ctable_base_index (unsigned HOST_WIDE_INT caddr)
+pru_get_ctable_base_index (HOST_WIDE_INT caddr)
 {
   unsigned int i;
 
@@ -1461,7 +1461,7 @@ pru_get_ctable_base_index (unsigned HOST_WIDE_INT caddr)
 
 /* Return the offset from some CTABLE base for this address.  */
 int
-pru_get_ctable_base_offset (unsigned HOST_WIDE_INT caddr)
+pru_get_ctable_base_offset (HOST_WIDE_INT caddr)
 {
   int i;
 
@@ -2004,7 +2004,7 @@ pru_print_operand_address (FILE *file, machine_mode mode, rtx op)
 
     case CONST_INT:
       {
-	unsigned HOST_WIDE_INT caddr = INTVAL (op);
+	HOST_WIDE_INT caddr = INTVAL (op);
 	int base = pru_get_ctable_base_index (caddr);
 	int offs = pru_get_ctable_base_offset (caddr);
 	if (base < 0)
diff --git a/gcc/config/riscv/arch-canonicalize b/gcc/config/riscv/arch-canonicalize
index fd55255..34dad45 100755
--- a/gcc/config/riscv/arch-canonicalize
+++ b/gcc/config/riscv/arch-canonicalize
@@ -32,7 +32,7 @@ import itertools
 from functools import reduce
 
 SUPPORTED_ISA_SPEC = ["2.2", "20190608", "20191213"]
-CANONICAL_ORDER = "imafdgqlcbkjtpvn"
+CANONICAL_ORDER = "imafdqlcbkjtpvnh"
 LONG_EXT_PREFIXES = ['z', 's', 'h', 'x']
 
 #
diff --git a/gcc/config/riscv/autovec-opt.md b/gcc/config/riscv/autovec-opt.md
index 8df7f64..6531996 100644
--- a/gcc/config/riscv/autovec-opt.md
+++ b/gcc/config/riscv/autovec-opt.md
@@ -1714,6 +1714,74 @@
   }
   [(set_attr "type" "vialu")])
 
+(define_insn_and_split "*<sat_op_v_vdup>_vx_<mode>"
+ [(set (match_operand:V_VLSI   0 "register_operand")
+   (if_then_else:V_VLSI
+    (unspec:<VM>
+     [(match_operand:<VM>      1 "vector_mask_operand")
+      (match_operand           5 "vector_length_operand")
+      (match_operand           6 "const_int_operand")
+      (match_operand           7 "const_int_operand")
+      (match_operand           8 "const_int_operand")
+      (match_operand           9 "const_int_operand")
+      (reg:SI VL_REGNUM)
+      (reg:SI VTYPE_REGNUM)
+      (reg:SI VXRM_REGNUM)] UNSPEC_VPREDICATE)
+    (unspec:V_VLSI
+     [(match_operand:V_VLSI    3 "register_operand")
+      (vec_duplicate:V_VLSI
+       (match_operand:<VEL>    4 "reg_or_int_operand"))] VSAT_VX_OP_V_VDUP)
+    (unspec:V_VLSI
+     [(match_operand:DI        2 "register_operand")] UNSPEC_VUNDEF)))]
+  "TARGET_VECTOR && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+  {
+    int vxrm_val = INTVAL (operands[9]);
+    riscv_vector::expand_vx_binary_vxrm_vec_vec_dup (operands[0], operands[3],
+						     operands[4],
+						     <VSAT_VX_OP_V_VDUP>,
+						     vxrm_val, <MODE>mode);
+
+    DONE;
+  }
+  [(set_attr "type" "vaalu")])
+
+(define_insn_and_split "*<sat_op_vdup_v>_vx_<mode>"
+ [(set (match_operand:V_VLSI   0 "register_operand")
+   (if_then_else:V_VLSI
+    (unspec:<VM>
+     [(match_operand:<VM>      1 "vector_mask_operand")
+      (match_operand           5 "vector_length_operand")
+      (match_operand           6 "const_int_operand")
+      (match_operand           7 "const_int_operand")
+      (match_operand           8 "const_int_operand")
+      (match_operand           9 "const_int_operand")
+      (reg:SI VL_REGNUM)
+      (reg:SI VTYPE_REGNUM)
+      (reg:SI VXRM_REGNUM)] UNSPEC_VPREDICATE)
+    (unspec:V_VLSI
+     [(vec_duplicate:V_VLSI
+       (match_operand:<VEL>    4 "reg_or_int_operand"))
+      (match_operand:V_VLSI    3 "register_operand")] VSAT_VX_OP_VDUP_V)
+    (unspec:V_VLSI
+     [(match_operand:DI        2 "register_operand")] UNSPEC_VUNDEF)))]
+  "TARGET_VECTOR && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+  {
+    int vxrm_val = INTVAL (operands[9]);
+    riscv_vector::expand_vx_binary_vxrm_vec_dup_vec (operands[0], operands[3],
+						     operands[4],
+						     <VSAT_VX_OP_VDUP_V>,
+						     vxrm_val, <MODE>mode);
+
+    DONE;
+  }
+  [(set_attr "type" "vaalu")])
+
 ;; =============================================================================
 ;; Combine vec_duplicate + op.vv to op.vf
 ;; Include
@@ -1725,6 +1793,8 @@
 ;; - vfmsac.vf
 ;; - vfnmacc.vf
 ;; - vfnmsac.vf
+;; - vfwmacc.vf
+;; - vfwmsac.vf
 ;; =============================================================================
 
 ;; vfmadd.vf, vfmsub.vf, vfmacc.vf, vfmsac.vf
@@ -1796,3 +1866,99 @@
   }
   [(set_attr "type" "vfmuladd")]
 )
+
+;; vfwmacc.vf, vfwmsac.vf
+(define_insn_and_split "*vfwmacc_vf_<mode>"
+  [(set (match_operand:VWEXTF 0 "register_operand")
+    (plus_minus:VWEXTF
+	    (mult:VWEXTF
+	      (float_extend:VWEXTF
+	        (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand"))
+	      (vec_duplicate:VWEXTF
+	        (float_extend:<VEL>
+		  (match_operand:<VSUBEL> 2 "register_operand"))))
+	    (match_operand:VWEXTF 1 "register_operand")))]
+  "TARGET_VECTOR && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+  {
+    rtx ops[] = {operands[0], operands[1], operands[2], operands[3]};
+    riscv_vector::emit_vlmax_insn (code_for_pred_widen_mul_scalar (<CODE>, <MODE>mode),
+				   riscv_vector::WIDEN_TERNARY_OP_FRM_DYN, ops);
+    DONE;
+  }
+  [(set_attr "type" "vfwmuladd")]
+)
+
+;; Intermediate pattern for vfwmacc.vf and vfwmsac.vf used by combine
+(define_insn_and_split "*extend_vf_<mode>"
+ [(set (match_operand:VWEXTF 0 "register_operand")
+    (vec_duplicate:VWEXTF
+      (float_extend:<VEL>
+        (match_operand:<VSUBEL> 1 "register_operand"))))]
+  "TARGET_VECTOR && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+  {
+    rtx tmp = gen_reg_rtx (<VEL>mode);
+    emit_insn (gen_extend<vsubel><vel>2(tmp, operands[1]));
+
+    rtx ops[] = {operands[0], tmp};
+    riscv_vector::expand_broadcast (<MODE>mode, ops);
+    DONE;
+  }
+  [(set_attr "type" "vfwmuladd")]
+)
+
+;; vfwnmacc.vf
+(define_insn_and_split "*vfwnmacc_vf_<mode>"
+  [(set (match_operand:VWEXTF 0 "register_operand")
+    (minus:VWEXTF
+      (mult:VWEXTF
+	(neg:VWEXTF
+	  (vec_duplicate:VWEXTF
+	    (float_extend:<VEL>
+	      (match_operand:<VSUBEL> 2 "register_operand"))))
+	(float_extend:VWEXTF
+	  (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand")))
+      (match_operand:VWEXTF 1 "register_operand")))]
+  "TARGET_VECTOR && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+  {
+    rtx ops[] = {operands[0], operands[1], operands[2], operands[3]};
+    riscv_vector::emit_vlmax_insn(
+	code_for_pred_widen_mul_neg_scalar(MINUS, <MODE>mode),
+	riscv_vector::WIDEN_TERNARY_OP_FRM_DYN, ops);
+    DONE;
+  }
+  [(set_attr "type" "vfwmuladd")]
+)
+
+;; vfwnmsac.vf
+(define_insn_and_split "*vfwnmsac_vf_<mode>"
+  [(set (match_operand:VWEXTF 0 "register_operand")
+    (minus:VWEXTF
+      (match_operand:VWEXTF 1 "register_operand")
+      (mult:VWEXTF
+	(float_extend:VWEXTF
+	  (match_operand:<V_DOUBLE_TRUNC> 3 "register_operand"))
+	(vec_duplicate:VWEXTF
+	  (float_extend:<VEL>
+	    (match_operand:<VSUBEL> 2 "register_operand"))))))]
+  "TARGET_VECTOR && can_create_pseudo_p ()"
+  "#"
+  "&& 1"
+  [(const_int 0)]
+  {
+    rtx ops[] = {operands[0], operands[1], operands[2], operands[3]};
+    riscv_vector::emit_vlmax_insn(
+	code_for_pred_widen_mul_neg_scalar (PLUS, <MODE>mode),
+	riscv_vector::WIDEN_TERNARY_OP_FRM_DYN, ops);
+    DONE;
+  }
+  [(set_attr "type" "vfwmuladd")]
+)
diff --git a/gcc/config/riscv/autovec.md b/gcc/config/riscv/autovec.md
index 94a61bd..48de5ef 100644
--- a/gcc/config/riscv/autovec.md
+++ b/gcc/config/riscv/autovec.md
@@ -1359,9 +1359,7 @@
   if (operands[2] == const0_rtx)
     {
       rtx ops[] = {operands[0], operands[0], operands[1]};
-      riscv_vector::emit_nonvlmax_insn (code_for_pred_broadcast (<MODE>mode),
-					riscv_vector::SCALAR_MOVE_MERGED_OP_TU,
-					ops, CONST1_RTX (Pmode));
+      riscv_vector::expand_set_first_tu (<MODE>mode, ops);
     }
   else
     {
@@ -1385,8 +1383,7 @@
 	 VL we need for the slide.  */
       rtx tmp = gen_reg_rtx (<MODE>mode);
       rtx ops1[] = {tmp, operands[1]};
-      emit_nonvlmax_insn (code_for_pred_broadcast (<MODE>mode),
-                           riscv_vector::UNARY_OP, ops1, length);
+      riscv_vector::expand_broadcast (<MODE>mode, ops1, length);
 
       /* Slide exactly one element up leaving the tail elements
 	 unchanged.  */
@@ -2489,7 +2486,8 @@
       (sign_extend:VWEXTI
        (match_operand:<V_DOUBLE_TRUNC> 1 "register_operand"))
       (sign_extend:VWEXTI
-       (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand"))))))]
+       (match_operand:<V_DOUBLE_TRUNC> 2 "register_operand")))
+     (const_int 1))))]
   "TARGET_VECTOR"
   {
     insn_code icode = code_for_pred (UNSPEC_VAADD, <V_DOUBLE_TRUNC>mode);
@@ -2499,6 +2497,19 @@
   }
 )
 
+(define_expand "avg<mode>3_floor"
+ [(match_operand:V_VLSI_D 0 "register_operand")
+  (match_operand:V_VLSI_D 1 "register_operand")
+  (match_operand:V_VLSI_D 2 "register_operand")]
+  "TARGET_VECTOR"
+  {
+    insn_code icode = code_for_pred (UNSPEC_VAADD, <MODE>mode);
+    riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP_VXRM_RDN,
+				   operands);
+    DONE;
+  }
+)
+
 (define_expand "avg<v_double_trunc>3_ceil"
  [(set (match_operand:<V_DOUBLE_TRUNC> 0 "register_operand")
    (truncate:<V_DOUBLE_TRUNC>
@@ -2509,7 +2520,8 @@
 	(match_operand:<V_DOUBLE_TRUNC> 1 "register_operand"))
        (sign_extend:VWEXTI
 	(match_operand:<V_DOUBLE_TRUNC> 2 "register_operand")))
-      (const_int 1)))))]
+      (const_int 1))
+     (const_int 1))))]
   "TARGET_VECTOR"
   {
     insn_code icode = code_for_pred (UNSPEC_VAADD, <V_DOUBLE_TRUNC>mode);
@@ -2519,6 +2531,19 @@
   }
 )
 
+(define_expand "avg<mode>3_ceil"
+ [(match_operand:V_VLSI_D 0 "register_operand")
+  (match_operand:V_VLSI_D 1 "register_operand")
+  (match_operand:V_VLSI_D 2 "register_operand")]
+  "TARGET_VECTOR"
+  {
+    insn_code icode = code_for_pred (UNSPEC_VAADD, <MODE>mode);
+    riscv_vector::emit_vlmax_insn (icode, riscv_vector::BINARY_OP_VXRM_RNU,
+				   operands);
+    DONE;
+  }
+)
+
 ;; csrwi vxrm, 2
 ;; vaaddu.vv vd, vs2, vs1
 (define_expand "uavg<mode>3_floor"
diff --git a/gcc/config/riscv/constraints.md b/gcc/config/riscv/constraints.md
index ccab1a2..5ecaa19 100644
--- a/gcc/config/riscv/constraints.md
+++ b/gcc/config/riscv/constraints.md
@@ -237,10 +237,11 @@
  (and (match_code "const_vector")
       (match_test "rtx_equal_p (op, riscv_vector::gen_scalar_move_mask (GET_MODE (op)))")))
 
-(define_memory_constraint "Wdm"
+(define_constraint "Wdm"
   "Vector duplicate memory operand"
-  (and (match_code "mem")
-       (match_code "reg" "0")))
+  (and (match_test "strided_load_broadcast_p ()")
+       (and (match_code "mem")
+	    (match_code "reg" "0"))))
 
 ;; Vendor ISA extension constraints.
 
diff --git a/gcc/config/riscv/gen-riscv-mcpu-texi.cc b/gcc/config/riscv/gen-riscv-mcpu-texi.cc
new file mode 100644
index 0000000..9681438
--- /dev/null
+++ b/gcc/config/riscv/gen-riscv-mcpu-texi.cc
@@ -0,0 +1,43 @@
+#include <string>
+#include <vector>
+#include <stdio.h>
+
+int
+main ()
+{
+  puts ("@c Copyright (C) 2025 Free Software Foundation, Inc.");
+  puts ("@c This is part of the GCC manual.");
+  puts ("@c For copying conditions, see the file gcc/doc/include/fdl.texi.");
+  puts ("");
+  puts ("@c This file is generated automatically using");
+  puts ("@c  gcc/config/riscv/gen-riscv-mcpu-texi.cc from:");
+  puts ("@c       gcc/config/riscv/riscv-cores.def");
+  puts ("");
+  puts ("@c Please *DO NOT* edit manually.");
+  puts ("");
+  puts ("@samp{Core Name}");
+  puts ("");
+  puts ("@opindex mcpu");
+  puts ("@item -mcpu=@var{processor-string}");
+  puts ("Use architecture of and optimize the output for the given processor, specified");
+  puts ("by particular CPU name. Permissible values for this option are:");
+  puts ("");
+  puts ("");
+
+  std::vector<std::string> coreNames;
+
+#define RISCV_CORE(CORE_NAME, ARCH, MICRO_ARCH) \
+  coreNames.push_back (CORE_NAME);
+#include "riscv-cores.def"
+#undef RISCV_CORE
+
+  for (size_t i = 0; i < coreNames.size(); ++i) {
+    if (i == coreNames.size() - 1) {
+      printf("@samp{%s}.\n", coreNames[i].c_str());
+    } else {
+      printf("@samp{%s},\n\n", coreNames[i].c_str());
+    }
+  }
+
+  return 0;
+}
diff --git a/gcc/config/riscv/gen-riscv-mtune-texi.cc b/gcc/config/riscv/gen-riscv-mtune-texi.cc
new file mode 100644
index 0000000..1bdfe2a
--- /dev/null
+++ b/gcc/config/riscv/gen-riscv-mtune-texi.cc
@@ -0,0 +1,41 @@
+#include <string>
+#include <vector>
+#include <stdio.h>
+
+int
+main ()
+{
+  puts ("@c Copyright (C) 2025 Free Software Foundation, Inc.");
+  puts ("@c This is part of the GCC manual.");
+  puts ("@c For copying conditions, see the file gcc/doc/include/fdl.texi.");
+  puts ("");
+  puts ("@c This file is generated automatically using");
+  puts ("@c  gcc/config/riscv/gen-riscv-mtune-texi.cc from:");
+  puts ("@c       gcc/config/riscv/riscv-cores.def");
+  puts ("");
+  puts ("@c Please *DO NOT* edit manually.");
+  puts ("");
+  puts ("@samp{Tune Name}");
+  puts ("");
+  puts ("@opindex mtune");
+  puts ("@item -mtune=@var{processor-string}");
+  puts ("Optimize the output for the given processor, specified by microarchitecture or");
+  puts ("particular CPU name.  Permissible values for this option are:");
+  puts ("");
+  puts ("");
+
+  std::vector<std::string> tuneNames;
+
+#define RISCV_TUNE(TUNE_NAME, PIPELINE_MODEL, TUNE_INFO) \
+  tuneNames.push_back (TUNE_NAME);
+#include "riscv-cores.def"
+#undef RISCV_TUNE
+
+  for (size_t i = 0; i < tuneNames.size(); ++i) {
+    printf("@samp{%s},\n\n", tuneNames[i].c_str());
+  }
+
+  puts ("and all valid options for @option{-mcpu=}.");
+
+  return 0;
+}
diff --git a/gcc/config/riscv/generic-vector-ooo.md b/gcc/config/riscv/generic-vector-ooo.md
index ab9e57f..773003b 100644
--- a/gcc/config/riscv/generic-vector-ooo.md
+++ b/gcc/config/riscv/generic-vector-ooo.md
@@ -17,6 +17,9 @@
 ;; <http://www.gnu.org/licenses/>.
 ;; Vector load/store
 
+;; The insn reservations include "generic" as we won't have a in-order
+;; generic definition for vector instructions.
+
 (define_automaton "vector_ooo")
 
 ;; Separate issue queue for vector instructions.
@@ -29,119 +32,141 @@
 (define_cpu_unit "vxu_ooo_multicycle" "vector_ooo")
 
 (define_insn_reservation "vec_load" 6
-  (eq_attr "type" "vlde,vldm,vlds,vldux,vldox,vldff,vldr")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vlde,vldm,vlds,vldux,vldox,vldff,vldr"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 (define_insn_reservation "vec_store" 6
-  (eq_attr "type" "vste,vstm,vsts,vstux,vstox,vstr")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vste,vstm,vsts,vstux,vstox,vstr"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector segment loads/stores.
 (define_insn_reservation "vec_loadstore_seg" 10
-  (eq_attr "type" "vlsegde,vlsegds,vlsegdux,vlsegdox,vlsegdff,\
-		   vssegte,vssegts,vssegtux,vssegtox")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vlsegde,vlsegds,vlsegdux,vlsegdox,vlsegdff,\
+		        vssegte,vssegts,vssegtux,vssegtox"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Regular vector operations and integer comparisons.
 (define_insn_reservation "vec_alu" 3
-  (eq_attr "type" "vialu,viwalu,vext,vicalu,vshift,vnshift,viminmax,vicmp,\
-		   vimov,vsalu,vaalu,vsshift,vnclip,vmov,vfmov,vector,\
-		   vandn,vbrev,vbrev8,vrev8,vclz,vctz,vrol,vror,vwsll")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vialu,viwalu,vext,vicalu,vshift,vnshift,viminmax,vicmp,\
+		        vimov,vsalu,vaalu,vsshift,vnclip,vmov,vfmov,vector,\
+		        vandn,vbrev,vbrev8,vrev8,vclz,vctz,vrol,vror,vwsll"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector float comparison, conversion etc.
 (define_insn_reservation "vec_fcmp" 3
-  (eq_attr "type" "vfrecp,vfminmax,vfcmp,vfsgnj,vfclass,vfcvtitof,\
-                   vfcvtftoi,vfwcvtitof,vfwcvtftoi,vfwcvtftof,vfncvtitof,\
-                   vfncvtftoi,vfncvtftof,vfncvtbf16,vfwcvtbf16")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vfrecp,vfminmax,vfcmp,vfsgnj,vfclass,vfcvtitof,\
+                        vfcvtftoi,vfwcvtitof,vfwcvtftoi,vfwcvtftof,vfncvtitof,\
+                        vfncvtftoi,vfncvtftof,vfncvtbf16,vfwcvtbf16"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector integer multiplication.
 (define_insn_reservation "vec_imul" 4
-  (eq_attr "type" "vimul,viwmul,vimuladd,viwmuladd,vsmul,vclmul,vclmulh,\
-                   vghsh,vgmul")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vimul,viwmul,vimuladd,viwmuladd,vsmul,vclmul,vclmulh,\
+                        vghsh,vgmul"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector float addition.
 (define_insn_reservation "vec_fadd" 4
-  (eq_attr "type" "vfalu,vfwalu")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vfalu,vfwalu"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector float multiplication and FMA.
 (define_insn_reservation "vec_fmul" 6
-  (eq_attr "type" "vfmul,vfwmul,vfmuladd,vfwmuladd,vfwmaccbf16,sf_vqmacc,sf_vfnrclip")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vfmul,vfwmul,vfmuladd,vfwmuladd,vfwmaccbf16,sf_vqmacc,sf_vfnrclip"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector crypto, assumed to be a generic operation for now.
 (define_insn_reservation "vec_crypto" 4
-  (eq_attr "type" "crypto,vclz,vctz,vcpop")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "crypto,vclz,vctz,vcpop"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector crypto, AES
 (define_insn_reservation "vec_crypto_aes" 4
-  (eq_attr "type" "vaesef,vaesem,vaesdf,vaesdm,vaeskf1,vaeskf2,vaesz")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vaesef,vaesem,vaesdf,vaesdm,vaeskf1,vaeskf2,vaesz"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector crypto, sha
 (define_insn_reservation "vec_crypto_sha" 4
-  (eq_attr "type" "vsha2ms,vsha2ch,vsha2cl")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vsha2ms,vsha2ch,vsha2cl"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector crypto, SM3/4
 (define_insn_reservation "vec_crypto_sm" 4
-  (eq_attr "type" "vsm4k,vsm4r,vsm3me,vsm3c")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vsm4k,vsm4r,vsm3me,vsm3c"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector permute.
 (define_insn_reservation "vec_perm" 3
-  (eq_attr "type" "vimerge,vfmerge,vslideup,vslidedown,vislide1up,\
-                   vislide1down,vfslide1up,vfslide1down,vgather,vcompress")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vimerge,vfmerge,vslideup,vslidedown,vislide1up,\
+                        vislide1down,vfslide1up,vfslide1down,vgather,vcompress"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector reduction.
 (define_insn_reservation "vec_reduction" 8
-  (eq_attr "type" "vired,viwred,vfredu,vfwredu")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vired,viwred,vfredu,vfwredu"))
   "vxu_ooo_issue,vxu_ooo_multicycle")
 
 ;; Vector ordered reduction, assume the latency number is for
 ;; a 128-bit vector.  It is scaled in riscv_sched_adjust_cost
 ;; for larger vectors.
 (define_insn_reservation "vec_ordered_reduction" 10
-  (eq_attr "type" "vfredo,vfwredo")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vfredo,vfwredo"))
   "vxu_ooo_issue,vxu_ooo_multicycle*3")
 
 ;; Vector integer division, assume not pipelined.
 (define_insn_reservation "vec_idiv" 16
-  (eq_attr "type" "vidiv")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vidiv"))
   "vxu_ooo_issue,vxu_ooo_multicycle*3")
 
 ;; Vector float divisions and sqrt, assume not pipelined.
 (define_insn_reservation "vec_float_divsqrt" 16
-  (eq_attr "type" "vfdiv,vfsqrt")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vfdiv,vfsqrt"))
   "vxu_ooo_issue,vxu_ooo_multicycle*3")
 
 ;; Vector mask operations.
 (define_insn_reservation "vec_mask" 2
-  (eq_attr "type" "vmalu,vmpop,vmffs,vmsfs,vmiota,vmidx,vimovvx,vimovxv,\
-                   vfmovvf,vfmovfv")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vmalu,vmpop,vmffs,vmsfs,vmiota,vmidx,vimovvx,vimovxv,\
+                        vfmovvf,vfmovfv"))
   "vxu_ooo_issue,vxu_ooo_alu")
 
 ;; Vector vsetvl.
 (define_insn_reservation "vec_vesetvl" 1
-  (eq_attr "type" "vsetvl,vsetvl_pre")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "vsetvl,vsetvl_pre"))
   "vxu_ooo_issue")
 
 ;; Vector rounding mode setters, assume pipeline barrier.
 (define_insn_reservation "vec_setrm" 20
-  (eq_attr "type" "wrvxrm,wrfrm")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "wrvxrm,wrfrm"))
   "vxu_ooo_issue,vxu_ooo_issue*3")
 
 ;; Vector read vlen/vlenb.
 (define_insn_reservation "vec_readlen" 4
-  (eq_attr "type" "rdvlenb,rdvl")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "rdvlenb,rdvl"))
   "vxu_ooo_issue,vxu_ooo_issue")
 
 ;; Vector sf_vcp.
 (define_insn_reservation "vec_sf_vcp" 2
-  (eq_attr "type" "sf_vc,sf_vc_se")
+  (and (eq_attr "tune" "generic_ooo,generic")
+       (eq_attr "type" "sf_vc,sf_vc_se"))
   "vxu_ooo_issue")
diff --git a/gcc/config/riscv/mips-insn.md b/gcc/config/riscv/mips-insn.md
new file mode 100644
index 0000000..ad46026
--- /dev/null
+++ b/gcc/config/riscv/mips-insn.md
@@ -0,0 +1,35 @@
+;; Machine description for MIPS custom instructions.
+;; Copyright (C) 2025 Free Software Foundation, Inc.
+
+;; This file is part of GCC.
+
+;; GCC is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 3, or (at your option)
+;; any later version.
+
+;; GCC is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GCC; see the file COPYING3.  If not see
+;; <http://www.gnu.org/licenses/>.
+
+(define_insn "*mov<GPR:mode><X:mode>cc_bitmanip"
+  [(set (match_operand:GPR 0 "register_operand" "=r")
+	(if_then_else:GPR (any_eq:X (match_operand:X 1 "register_operand" "r")
+				    (match_operand:X 2 "const_0_operand" "J"))
+			  (match_operand:GPR 3 "reg_or_0_operand" "rJ")
+			  (match_operand:GPR 4 "reg_or_0_operand" "rJ")))]
+  "TARGET_XMIPSCMOV"
+{
+  enum rtx_code code = <CODE>;
+  if (code == NE)
+    return "mips.ccmov\t%0,%1,%z3,%z4";
+  else
+    return "mips.ccmov\t%0,%1,%z4,%z3";
+}
+[(set_attr "type" "condmove")
+ (set_attr "mode" "<GPR:MODE>")])
diff --git a/gcc/config/riscv/mips-p8700.md b/gcc/config/riscv/mips-p8700.md
index ae0ea8d..fac9abb 100644
--- a/gcc/config/riscv/mips-p8700.md
+++ b/gcc/config/riscv/mips-p8700.md
@@ -163,5 +163,5 @@
    vgather,vcompress,vmov,vector,vandn,vbrev,vbrev8,vrev8,vclz,vctz,vcpop,vrol,vror,vwsll,
    vclmul,vclmulh,vghsh,vgmul,vaesef,vaesem,vaesdf,vaesdm,vaeskf1,vaeskf2,vaesz,
    vsha2ms,vsha2ch,vsha2cl,vsm4k,vsm4r,vsm3me,vsm3c,vfncvtbf16,vfwcvtbf16,vfwmaccbf16,
-   sf_vc,sf_vc_se"))
+   sf_vc,sf_vc_se,ghost"))
   "mips_p8700_dummies")
diff --git a/gcc/config/riscv/predicates.md b/gcc/config/riscv/predicates.md
index 061904b..381f96c 100644
--- a/gcc/config/riscv/predicates.md
+++ b/gcc/config/riscv/predicates.md
@@ -33,11 +33,11 @@
 (define_predicate "prefetch_operand"
   (ior (match_operand 0 "register_operand")
        (and (match_test "const_arith_operand (op, VOIDmode)")
-	    (match_test "(INTVAL (op) & 0xf) == 0"))
+	    (match_test "(INTVAL (op) & 0x1f) == 0"))
        (and (match_code "plus")
 	    (match_test "register_operand (XEXP (op, 0), word_mode)")
 	    (match_test "const_arith_operand (XEXP (op, 1), VOIDmode)")
-	    (match_test "(INTVAL (XEXP (op, 1)) & 0xf) == 0"))))
+	    (match_test "(INTVAL (XEXP (op, 1)) & 0x1f) == 0"))))
 
 (define_predicate "lui_operand"
   (and (match_code "const_int")
@@ -518,6 +518,10 @@
 
 (define_predicate "vector_broadcast_mask_operand"
   (ior (match_operand 0 "vector_least_significant_set_mask_operand")
+       (match_operand 0 "vector_all_trues_mask_operand")))
+
+(define_predicate "strided_broadcast_mask_operand"
+  (ior (match_operand 0 "vector_least_significant_set_mask_operand")
     (ior (match_operand 0 "register_operand")
          (match_operand 0 "vector_all_trues_mask_operand"))))
 
@@ -617,7 +621,16 @@
 
 ;; The scalar operand can be directly broadcast by RVV instructions.
 (define_predicate "direct_broadcast_operand"
-  (match_test "riscv_vector::can_be_broadcasted_p (op)"))
+  (match_test "riscv_vector::can_be_broadcast_p (op)"))
+
+;; A strided broadcast is just a fallback pattern that loads from
+;; memory.
+(define_predicate "strided_broadcast_operand"
+  (match_test "riscv_vector::strided_broadcast_p (op)"))
+
+(define_predicate "any_broadcast_operand"
+  (ior (match_operand 0 "direct_broadcast_operand")
+       (match_operand 0 "strided_broadcast_operand")))
 
 ;; A CONST_INT operand that has exactly two bits cleared.
 (define_predicate "const_nottwobits_operand"
diff --git a/gcc/config/riscv/riscv-avlprop.cc b/gcc/config/riscv/riscv-avlprop.cc
index bb4aceb..3031c29 100644
--- a/gcc/config/riscv/riscv-avlprop.cc
+++ b/gcc/config/riscv/riscv-avlprop.cc
@@ -508,7 +508,7 @@ pass_avlprop::execute (function *fn)
       simplify_replace_vlmax_avl (rinsn, prop.second);
     }
 
-  if (rvv_vector_bits == RVV_VECTOR_BITS_ZVL)
+  if (rvv_vector_bits == RVV_VECTOR_BITS_ZVL && !TARGET_XTHEADVECTOR)
     {
       /* Simplify VLMAX AVL into immediate AVL.
 	 E.g. Simplify this following case:
diff --git a/gcc/config/riscv/riscv-cores.def b/gcc/config/riscv/riscv-cores.def
index 2096c00..98f3470 100644
--- a/gcc/config/riscv/riscv-cores.def
+++ b/gcc/config/riscv/riscv-cores.def
@@ -169,7 +169,6 @@ RISCV_CORE("xiangshan-kunminghu",   "rv64imafdcbvh_sdtrig_sha_shcounterenw_"
 			      "zvfhmin_zvkt_zvl128b_zvl32b_zvl64b",
 			      "xiangshan-kunminghu")
 
-RISCV_CORE("mips-p8700",     	"rv64imafd_zicsr_zmmul_"
-			      "zaamo_zalrsc_zba_zbb",
+RISCV_CORE("mips-p8700",      "rv64imfd_zicsr_zifencei_zalrsc_zba_zbb",
 			      "mips-p8700")
 #undef RISCV_CORE
diff --git a/gcc/config/riscv/riscv-ext-mips.def b/gcc/config/riscv/riscv-ext-mips.def
new file mode 100644
index 0000000..5d7836d
--- /dev/null
+++ b/gcc/config/riscv/riscv-ext-mips.def
@@ -0,0 +1,35 @@
+/* MIPS extension definition file for RISC-V.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.
+
+Please run `make riscv-regen` in build folder to make sure updated anything.
+
+Format of DEFINE_RISCV_EXT, please refer to riscv-ext.def.  */
+
+DEFINE_RISCV_EXT (
+  /* NAME.  */ xmipscmov,
+  /* UPPERCASE_NAME.  */ XMIPSCMOV,
+  /* FULL_NAME.  */ "Mips conditional move extension",
+  /* DESC.  */ "",
+  /* URL.  */ ,
+  /* DEP_EXTS.  */ ({}),
+  /* SUPPORTED_VERSIONS.  */ ({{1, 0}}),
+  /* FLAG_GROUP.  */ xmips,
+  /* BITMASK_GROUP_ID.  */ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_BIT_POSITION.  */ BITMASK_NOT_YET_ALLOCATED,
+  /* EXTRA_EXTENSION_FLAGS.  */ 0)
diff --git a/gcc/config/riscv/riscv-ext.def b/gcc/config/riscv/riscv-ext.def
index 816acaa..09f18ad 100644
--- a/gcc/config/riscv/riscv-ext.def
+++ b/gcc/config/riscv/riscv-ext.def
@@ -80,8 +80,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({}),
   /* SUPPORTED_VERSIONS */ ({{2, 0}}),
   /* FLAG_GROUP */ base,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 0,
+  /* BITMASK_BIT_POSITION*/ 4,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -190,8 +190,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({"zba", "zbb", "zbs"}),
   /* SUPPORTED_VERSIONS */ ({{1, 0}}),
   /* FLAG_GROUP */ base,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 0,
+  /* BITMASK_BIT_POSITION*/ 1,
   /* EXTRA_EXTENSION_FLAGS */ EXT_FLAG_MACRO)
 
 DEFINE_RISCV_EXT(
@@ -216,8 +216,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({}),
   /* SUPPORTED_VERSIONS */ ({{1, 0}}),
   /* FLAG_GROUP */ base,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 0,
+  /* BITMASK_BIT_POSITION*/ 7,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -398,8 +398,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({}),
   /* SUPPORTED_VERSIONS */ ({{2, 0}}),
   /* FLAG_GROUP */ zi,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 1,
+  /* BITMASK_BIT_POSITION*/ 11,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -464,7 +464,7 @@ DEFINE_RISCV_EXT(
   /* SUPPORTED_VERSIONS */ ({{1, 0}}),
   /* FLAG_GROUP */ zi,
   /* BITMASK_GROUP_ID */ 1,
-  /* BITMASK_BIT_POSITION*/ 1,
+  /* BITMASK_BIT_POSITION*/ 8,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -476,8 +476,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({}),
   /* SUPPORTED_VERSIONS */ ({{1, 0}}),
   /* FLAG_GROUP */ zm,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 1,
+  /* BITMASK_BIT_POSITION*/ 12,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -787,8 +787,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({"zca"}),
   /* SUPPORTED_VERSIONS */ ({{1, 0}}),
   /* FLAG_GROUP */ zc,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 1,
+  /* BITMASK_BIT_POSITION*/ 10,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -813,8 +813,8 @@ DEFINE_RISCV_EXT(
   /* DEP_EXTS */ ({"zca", "zilsd"}),
   /* SUPPORTED_VERSIONS */ ({{1, 0}}),
   /* FLAG_GROUP */ zc,
-  /* BITMASK_GROUP_ID */ BITMASK_NOT_YET_ALLOCATED,
-  /* BITMASK_BIT_POSITION*/ BITMASK_NOT_YET_ALLOCATED,
+  /* BITMASK_GROUP_ID */ 1,
+  /* BITMASK_BIT_POSITION*/ 9,
   /* EXTRA_EXTENSION_FLAGS */ 0)
 
 DEFINE_RISCV_EXT(
@@ -2082,3 +2082,4 @@ DEFINE_RISCV_EXT(
 #include "riscv-ext-sifive.def"
 #include "riscv-ext-thead.def"
 #include "riscv-ext-ventana.def"
+#include "riscv-ext-mips.def"
diff --git a/gcc/config/riscv/riscv-ext.opt b/gcc/config/riscv/riscv-ext.opt
index 9f8c545..26d6e68 100644
--- a/gcc/config/riscv/riscv-ext.opt
+++ b/gcc/config/riscv/riscv-ext.opt
@@ -47,6 +47,9 @@ TargetVariable
 int riscv_xcv_subext
 
 TargetVariable
+int riscv_xmips_subext
+
+TargetVariable
 int riscv_xsf_subext
 
 TargetVariable
@@ -445,3 +448,4 @@ Mask(XTHEADVECTOR) Var(riscv_xthead_subext)
 
 Mask(XVENTANACONDOPS) Var(riscv_xventana_subext)
 
+Mask(XMIPSCMOV) Var(riscv_xmips_subext)
diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h
index a033120..539321f 100644
--- a/gcc/config/riscv/riscv-protos.h
+++ b/gcc/config/riscv/riscv-protos.h
@@ -137,6 +137,7 @@ extern void riscv_expand_usadd (rtx, rtx, rtx);
 extern void riscv_expand_ssadd (rtx, rtx, rtx);
 extern void riscv_expand_ussub (rtx, rtx, rtx);
 extern void riscv_expand_sssub (rtx, rtx, rtx);
+extern void riscv_expand_usmul (rtx, rtx, rtx);
 extern void riscv_expand_ustrunc (rtx, rtx);
 extern void riscv_expand_sstrunc (rtx, rtx);
 extern int riscv_register_move_cost (machine_mode, reg_class_t, reg_class_t);
@@ -413,8 +414,14 @@ enum insn_flags : unsigned int
   /* Means INSN has VXRM operand and the value is VXRM_RNU.  */
   VXRM_RNU_P = 1 << 20,
 
+  /* Means INSN has VXRM operand and the value is VXRM_RNE.  */
+  VXRM_RNE_P = 1 << 21,
+
   /* Means INSN has VXRM operand and the value is VXRM_RDN.  */
-  VXRM_RDN_P = 1 << 21,
+  VXRM_RDN_P = 1 << 22,
+
+  /* Means INSN has VXRM operand and the value is VXRM_ROD.  */
+  VXRM_ROD_P = 1 << 23,
 };
 
 enum insn_type : unsigned int
@@ -476,7 +483,9 @@ enum insn_type : unsigned int
   BINARY_OP_TUMA = __MASK_OP_TUMA | BINARY_OP_P,
   BINARY_OP_FRM_DYN = BINARY_OP | FRM_DYN_P,
   BINARY_OP_VXRM_RNU = BINARY_OP | VXRM_RNU_P,
+  BINARY_OP_VXRM_RNE = BINARY_OP | VXRM_RNE_P,
   BINARY_OP_VXRM_RDN = BINARY_OP | VXRM_RDN_P,
+  BINARY_OP_VXRM_ROD = BINARY_OP | VXRM_ROD_P,
 
   /* Ternary operator. Always have real merge operand.  */
   TERNARY_OP = HAS_DEST_P | HAS_MASK_P | USE_ALL_TRUES_MASK_P | HAS_MERGE_P
@@ -603,6 +612,7 @@ void emit_vlmax_vsetvl (machine_mode, rtx);
 void emit_hard_vlmax_vsetvl (machine_mode, rtx);
 void emit_vlmax_insn (unsigned, unsigned, rtx *);
 void emit_nonvlmax_insn (unsigned, unsigned, rtx *, rtx);
+void emit_avltype_insn (unsigned, unsigned, rtx *, avl_type, rtx = nullptr);
 void emit_vlmax_insn_lra (unsigned, unsigned, rtx *, rtx);
 enum vlmul_type get_vlmul (machine_mode);
 rtx get_vlmax_rtx (machine_mode);
@@ -670,6 +680,8 @@ void expand_vec_oct_sstrunc (rtx, rtx, machine_mode, machine_mode,
 			     machine_mode);
 void expand_vx_binary_vec_dup_vec (rtx, rtx, rtx, rtx_code, machine_mode);
 void expand_vx_binary_vec_vec_dup (rtx, rtx, rtx, rtx_code, machine_mode);
+void expand_vx_binary_vxrm_vec_vec_dup (rtx, rtx, rtx, int, int, machine_mode);
+void expand_vx_binary_vxrm_vec_dup_vec (rtx, rtx, rtx, int, int, machine_mode);
 #endif
 bool sew64_scalar_helper (rtx *, rtx *, rtx, machine_mode,
 			  bool, void (*)(rtx *, rtx), enum avl_type);
@@ -693,6 +705,9 @@ bool expand_block_move (rtx, rtx, rtx, bool);
 machine_mode preferred_simd_mode (scalar_mode);
 machine_mode get_mask_mode (machine_mode);
 void expand_vec_series (rtx, rtx, rtx, rtx = 0);
+void expand_broadcast (machine_mode, rtx *, rtx = 0);
+void expand_set_first (machine_mode, rtx *, rtx = 0);
+void expand_set_first_tu (machine_mode, rtx *, rtx = 0);
 void expand_vec_init (rtx, rtx);
 void expand_vec_perm (rtx, rtx, rtx, rtx);
 void expand_select_vl (rtx *);
@@ -759,7 +774,8 @@ uint8_t get_sew (rtx_insn *);
 enum vlmul_type get_vlmul (rtx_insn *);
 int count_regno_occurrences (rtx_insn *, unsigned int);
 bool imm_avl_p (machine_mode);
-bool can_be_broadcasted_p (rtx);
+bool can_be_broadcast_p (rtx);
+bool strided_broadcast_p (rtx);
 bool gather_scatter_valid_offset_p (machine_mode);
 HOST_WIDE_INT estimated_poly_value (poly_int64, unsigned int);
 bool whole_reg_to_reg_move_p (rtx *, machine_mode, int);
@@ -812,6 +828,7 @@ extern const char *th_output_move (rtx, rtx);
 extern bool th_print_operand_address (FILE *, machine_mode, rtx);
 #endif
 
+extern bool strided_load_broadcast_p (void);
 extern bool riscv_use_divmod_expander (void);
 void riscv_init_cumulative_args (CUMULATIVE_ARGS *, tree, rtx, tree, int);
 extern bool
diff --git a/gcc/config/riscv/riscv-selftests.cc b/gcc/config/riscv/riscv-selftests.cc
index 34d01ac..9ca1ffe 100644
--- a/gcc/config/riscv/riscv-selftests.cc
+++ b/gcc/config/riscv/riscv-selftests.cc
@@ -342,9 +342,13 @@ run_broadcast_selftests (void)
 	  expand_vector_broadcast (mode, mem);                                 \
 	  insn = get_last_insn ();                                             \
 	  src = SET_SRC (PATTERN (insn));                                      \
-	  ASSERT_TRUE (MEM_P (XEXP (src, 0)));                                 \
-	  ASSERT_TRUE (                                                        \
-	    rtx_equal_p (src, gen_rtx_VEC_DUPLICATE (mode, XEXP (src, 0))));   \
+	  if (strided_load_broadcast_p ())                                     \
+	    {                                                                  \
+	      ASSERT_TRUE (MEM_P (XEXP (src, 0)));                             \
+	      ASSERT_TRUE (                                                    \
+		rtx_equal_p (src,                                              \
+			     gen_rtx_VEC_DUPLICATE (mode, XEXP (src, 0))));    \
+	    }                                                                  \
 	  end_sequence ();                                                     \
 	  /* Test vmv.v.x or vfmv.v.f.  */                                     \
 	  start_sequence ();                                                   \
diff --git a/gcc/config/riscv/riscv-string.cc b/gcc/config/riscv/riscv-string.cc
index 9080189..61c4a09 100644
--- a/gcc/config/riscv/riscv-string.cc
+++ b/gcc/config/riscv/riscv-string.cc
@@ -1625,16 +1625,14 @@ expand_vec_setmem (rtx dst_in, rtx length_in, rtx fill_value_in)
      Otherwise, use a predicated store.  */
   if (known_eq (GET_MODE_SIZE (info.vmode), INTVAL (info.avl)))
     {
-      emit_vlmax_insn (code_for_pred_broadcast (info.vmode), UNARY_OP,
-		       broadcast_ops);
+      riscv_vector::expand_broadcast (info.vmode, broadcast_ops);
       emit_move_insn (dst, fill_value);
     }
   else
     {
       if (!satisfies_constraint_vl (info.avl))
 	info.avl = force_reg (Pmode, info.avl);
-      emit_nonvlmax_insn (code_for_pred_broadcast (info.vmode),
-			  riscv_vector::UNARY_OP, broadcast_ops, info.avl);
+      riscv_vector::expand_broadcast (info.vmode, broadcast_ops, info.avl);
       machine_mode mask_mode
 	= riscv_vector::get_vector_mode (BImode, GET_MODE_NUNITS (info.vmode))
 	  .require ();
diff --git a/gcc/config/riscv/riscv-v.cc b/gcc/config/riscv/riscv-v.cc
index ce1633c..c9c8328 100644
--- a/gcc/config/riscv/riscv-v.cc
+++ b/gcc/config/riscv/riscv-v.cc
@@ -351,9 +351,12 @@ public:
       add_rounding_mode_operand (FRM_RNE);
     else if (m_insn_flags & VXRM_RNU_P)
       add_rounding_mode_operand (VXRM_RNU);
+    else if (m_insn_flags & VXRM_RNE_P)
+      add_rounding_mode_operand (VXRM_RNE);
     else if (m_insn_flags & VXRM_RDN_P)
       add_rounding_mode_operand (VXRM_RDN);
-
+    else if (m_insn_flags & VXRM_ROD_P)
+      add_rounding_mode_operand (VXRM_ROD);
 
     if (insn_data[(int) icode].n_operands != m_opno)
       internal_error ("invalid number of operands for insn %s, "
@@ -408,7 +411,7 @@ emit_vlmax_insn_lra (unsigned icode, unsigned insn_flags, rtx *ops, rtx vl)
   gcc_assert (!can_create_pseudo_p ());
   machine_mode mode = GET_MODE (ops[0]);
 
-  if (imm_avl_p (mode))
+  if (imm_avl_p (mode) && !TARGET_XTHEADVECTOR)
     {
       /* Even though VL is a real hardreg already allocated since
 	 it is post-RA now, we still gain benefits that we emit
@@ -437,6 +440,26 @@ emit_nonvlmax_insn (unsigned icode, unsigned insn_flags, rtx *ops, rtx vl)
   e.emit_insn ((enum insn_code) icode, ops);
 }
 
+/* Emit either a VLMAX insn or a non-VLMAX insn depending on TYPE.  For a
+   non-VLMAX insn, the length must be specified in VL.  */
+
+void
+emit_avltype_insn (unsigned icode, unsigned insn_flags, rtx *ops,
+		   avl_type type, rtx vl)
+{
+  if (type != avl_type::VLMAX && vl != NULL_RTX)
+    {
+      insn_expander<RVV_INSN_OPERANDS_MAX> e (insn_flags, false);
+      e.set_vl (vl);
+      e.emit_insn ((enum insn_code) icode, ops);
+    }
+  else
+    {
+      insn_expander<RVV_INSN_OPERANDS_MAX> e (insn_flags, true);
+      e.emit_insn ((enum insn_code) icode, ops);
+    }
+}
+
 /* Return true if the vector duplicated by a super element which is the fusion
    of consecutive elements.
 
@@ -1170,6 +1193,59 @@ expand_vector_init_trailing_same_elem (rtx target,
   return false;
 }
 
+/* Helper function to emit a vmv.vx/vi and float variants.
+   If VL is not given a VLMAX insn will be emitted, otherwise
+   a non-VLMAX insn with length VL.
+   If the value to be broadcast is not suitable for vmv.vx
+   fall back to a vlse with zero stride.  This itself has a
+   fallback if the uarch prefers not to use a strided load
+   for broadcast.  */
+
+void
+expand_broadcast (machine_mode mode, rtx *ops, rtx vl)
+{
+  rtx elt = ops[1];
+  avl_type type = vl ? NONVLMAX : VLMAX;
+  if (can_be_broadcast_p (elt))
+    emit_avltype_insn (code_for_pred_broadcast (mode), UNARY_OP, ops,
+		       type, vl);
+  else
+    emit_avltype_insn (code_for_pred_strided_broadcast (mode),
+		       UNARY_OP, ops, type, vl);
+}
+
+/* Similar to expand_broadcast but emits a vmv.s.x/vfmv.s.f instead.  */
+
+void
+expand_set_first (machine_mode mode, rtx *ops, rtx vl)
+{
+  rtx elt = ops[1];
+  avl_type type = vl ? NONVLMAX : VLMAX;
+  if (can_be_broadcast_p (elt))
+    emit_avltype_insn (code_for_pred_broadcast (mode),
+			SCALAR_MOVE_OP, ops, type, vl);
+  else
+    emit_avltype_insn (code_for_pred_strided_broadcast (mode),
+			SCALAR_MOVE_OP, ops, type, vl);
+}
+
+/* Similar to expand_set_first but keeping the tail elements
+   unchanged (TU) */
+
+void
+expand_set_first_tu (machine_mode mode, rtx *ops, rtx vl)
+{
+  rtx elt = ops[2];
+  if (!vl)
+    vl = const1_rtx;
+  if (can_be_broadcast_p (elt))
+    emit_nonvlmax_insn (code_for_pred_broadcast (mode),
+			SCALAR_MOVE_MERGED_OP_TU, ops, vl);
+  else
+    emit_nonvlmax_insn (code_for_pred_strided_broadcast (mode),
+			SCALAR_MOVE_MERGED_OP_TU, ops, vl);
+}
+
 static void
 expand_const_vec_duplicate (rtx target, rtx src, rtx elt)
 {
@@ -1206,7 +1282,7 @@ expand_const_vec_duplicate (rtx target, rtx src, rtx elt)
       if (lra_in_progress)
 	{
 	  rtx ops[] = {result, elt};
-	  emit_vlmax_insn (code_for_pred_broadcast (mode), UNARY_OP, ops);
+	  expand_broadcast (mode, ops);
 	}
       else
 	{
@@ -1258,8 +1334,7 @@ expand_const_vector_duplicate_repeating (rtx target, rvv_builder *builder)
     {
       dup = gen_reg_rtx (builder->new_mode ());
       rtx ops[] = {dup, ele};
-      emit_vlmax_insn (code_for_pred_broadcast (builder->new_mode ()),
-		       UNARY_OP, ops);
+      expand_broadcast (builder->new_mode (), ops);
     }
   else
     dup = expand_vector_broadcast (builder->new_mode (), ele);
@@ -1302,8 +1377,7 @@ expand_const_vector_duplicate_default (rtx target, rvv_builder *builder)
 
   rtx tmp1 = gen_reg_rtx (builder->mode ());
   rtx dup_ops[] = {tmp1, builder->elt (0)};
-  emit_vlmax_insn (code_for_pred_broadcast (builder->mode ()), UNARY_OP,
-		   dup_ops);
+  expand_broadcast (builder->mode (), dup_ops);
 
   for (unsigned int i = 1; i < builder->npatterns (); i++)
     {
@@ -2116,18 +2190,32 @@ has_vi_variant_p (rtx_code code, rtx x)
     }
 }
 
+/* This is a helper for binary ops with DImode scalar operands that are
+   broadcast (like vadd.vx v1, a1).
+   Instead of having similar code for all the expanders this function
+   unifies the handling.  For 64-bit targets all we do is choose
+   between the vi variant (if available) and the register variant.
+   For 32-bit targets we either create the sign-extending variant
+   of vop.vx (when the immediate fits 32 bits) or emit a vector
+   broadcast of the 64-bit register/immediate and switch to a
+   vop.vv (replacing the scalar op with the broadcast vector.  */
+
 bool
 sew64_scalar_helper (rtx *operands, rtx *scalar_op, rtx vl,
 		     machine_mode vector_mode, bool has_vi_variant_p,
 		     void (*emit_vector_func) (rtx *, rtx), enum avl_type type)
 {
   machine_mode scalar_mode = GET_MODE_INNER (vector_mode);
+
+  /* If the scalar broadcast op fits an immediate, use the
+     vop.vi variant if there is one.  */
   if (has_vi_variant_p)
     {
       *scalar_op = force_reg (scalar_mode, *scalar_op);
       return false;
     }
 
+  /* On a 64-bit target we can always use the vop.vx variant.  */
   if (TARGET_64BIT)
     {
       if (!rtx_equal_p (*scalar_op, const0_rtx))
@@ -2135,6 +2223,8 @@ sew64_scalar_helper (rtx *operands, rtx *scalar_op, rtx vl,
       return false;
     }
 
+  /* For 32 bit and if there is no vop.vi variant for a 32-bit immediate
+     we need to use the sign-extending (SI -> DI) vop.vx variants.  */
   if (immediate_operand (*scalar_op, Pmode))
     {
       if (!rtx_equal_p (*scalar_op, const0_rtx))
@@ -2144,6 +2234,17 @@ sew64_scalar_helper (rtx *operands, rtx *scalar_op, rtx vl,
       return false;
     }
 
+  /* Now we're left with a 64-bit immediate or a register.
+     We cannot use a vop.vx variant but must broadcast the value first
+     and switch to a vop.vv variant.
+     Broadcast can either be done via vlse64.v v1, reg, zero
+     or by loading one 64-bit element (vle64.v) and using a
+     broadcast vrgather.vi.  This is decided when splitting
+     the strided broadcast insn.  */
+  gcc_assert (!TARGET_64BIT
+	      && (CONST_INT_P (*scalar_op)
+		  || register_operand (*scalar_op, scalar_mode)));
+
   if (CONST_INT_P (*scalar_op))
     {
       if (maybe_gt (GET_MODE_SIZE (scalar_mode), GET_MODE_SIZE (Pmode)))
@@ -2154,11 +2255,8 @@ sew64_scalar_helper (rtx *operands, rtx *scalar_op, rtx vl,
 
   rtx tmp = gen_reg_rtx (vector_mode);
   rtx ops[] = {tmp, *scalar_op};
-  if (type == VLMAX)
-    emit_vlmax_insn (code_for_pred_broadcast (vector_mode), UNARY_OP, ops);
-  else
-    emit_nonvlmax_insn (code_for_pred_broadcast (vector_mode), UNARY_OP, ops,
-			vl);
+  emit_avltype_insn (code_for_pred_strided_broadcast (vector_mode),
+		     UNARY_OP, ops, type, vl);
   emit_vector_func (operands, tmp);
 
   return true;
@@ -2552,8 +2650,7 @@ expand_vector_init_merge_repeating_sequence (rtx target,
 
   /* Step 1: Broadcast the first pattern.  */
   rtx ops[] = {target, force_reg (builder.inner_mode (), builder.elt (0))};
-  emit_vlmax_insn (code_for_pred_broadcast (builder.mode ()),
-		    UNARY_OP, ops);
+  expand_broadcast (builder.mode (), ops);
   /* Step 2: Merge the rest iteration of pattern.  */
   for (unsigned int i = 1; i < builder.npatterns (); i++)
     {
@@ -2566,8 +2663,7 @@ expand_vector_init_merge_repeating_sequence (rtx target,
       if (full_nelts <= builder.inner_bits_size ()) /* vmv.s.x.  */
 	{
 	  rtx ops[] = {dup, merge_mask};
-	  emit_nonvlmax_insn (code_for_pred_broadcast (GET_MODE (dup)),
-			       SCALAR_MOVE_OP, ops, CONST1_RTX (Pmode));
+	  expand_set_first (GET_MODE (dup), ops);
 	}
       else /* vmv.v.x.  */
 	{
@@ -2575,8 +2671,7 @@ expand_vector_init_merge_repeating_sequence (rtx target,
 		       force_reg (GET_MODE_INNER (mask_int_mode), merge_mask)};
 	  rtx vl = gen_int_mode (CEIL (full_nelts, builder.inner_bits_size ()),
 				 Pmode);
-	  emit_nonvlmax_insn (code_for_pred_broadcast (mask_int_mode), UNARY_OP,
-			       ops, vl);
+	  expand_broadcast (mask_int_mode, ops, vl);
 	}
 
       emit_move_insn (mask, gen_lowpart (mask_bit_mode, dup));
@@ -4667,20 +4762,20 @@ expand_reduction (unsigned unspec, unsigned unspec_for_vl0_safe,
 
   rtx m1_tmp = gen_reg_rtx (m1_mode);
   rtx scalar_move_ops[] = {m1_tmp, init};
-  insn_code icode = code_for_pred_broadcast (m1_mode);
   if (need_mask_operand_p (insn_flags))
     {
       if (need_vl0_safe)
-	emit_nonvlmax_insn (icode, SCALAR_MOVE_OP, scalar_move_ops, const1_rtx);
+	expand_set_first (m1_mode, scalar_move_ops, const1_rtx);
       else
-	emit_nonvlmax_insn (icode, SCALAR_MOVE_OP, scalar_move_ops, vl_op);
+	expand_set_first (m1_mode, scalar_move_ops, vl_op);
     }
   else
-    emit_vlmax_insn (icode, SCALAR_MOVE_OP, scalar_move_ops);
+    expand_set_first (m1_mode, scalar_move_ops);
 
   rtx m1_tmp2 = gen_reg_rtx (m1_mode);
   rtx reduc_ops[] = {m1_tmp2, vector_src, m1_tmp};
 
+  insn_code icode;
   if (need_vl0_safe)
     icode = code_for_pred (unspec_for_vl0_safe, vmode);
   else
@@ -5544,6 +5639,7 @@ expand_vx_binary_vec_dup_vec (rtx op_0, rtx op_1, rtx op_2,
     case SMIN:
     case UMIN:
     case US_PLUS:
+    case SS_PLUS:
       icode = code_for_pred_scalar (code, mode);
       break;
     case MINUS:
@@ -5557,6 +5653,82 @@ expand_vx_binary_vec_dup_vec (rtx op_0, rtx op_1, rtx op_2,
   emit_vlmax_insn (icode, riscv_vector::BINARY_OP, ops);
 }
 
+static enum insn_type
+get_insn_type_by_vxrm_val (int vxrm_val)
+{
+  enum insn_type itype;
+
+  switch (vxrm_val)
+    {
+    case VXRM_RNU:
+      itype = BINARY_OP_VXRM_RNU;
+      break;
+    case VXRM_RNE:
+      itype = BINARY_OP_VXRM_RNE;
+      break;
+    case VXRM_RDN:
+      itype = BINARY_OP_VXRM_RDN;
+      break;
+    case VXRM_ROD:
+      itype = BINARY_OP_VXRM_ROD;
+      break;
+    default:
+      gcc_unreachable ();
+    }
+
+  return itype;
+}
+
+/* Expand the binary vx combine with the format like v2 = vop(v1, vec_dup(x))
+   and its' vxrm value.  Aka the second op comes from the vec_duplicate,
+   and the first op is the vector reg.  */
+
+void
+expand_vx_binary_vxrm_vec_vec_dup (rtx op_0, rtx op_1, rtx op_2, int unspec,
+				   int vxrm_val, machine_mode mode)
+{
+  enum insn_code icode;
+  enum insn_type itype = get_insn_type_by_vxrm_val (vxrm_val);
+  rtx ops[] = {op_0, op_1, op_2};
+
+  switch (unspec)
+    {
+    case UNSPEC_VAADD:
+    case UNSPEC_VAADDU:
+      icode = code_for_pred_scalar (unspec, mode);
+      break;
+    default:
+      gcc_unreachable ();
+    }
+
+  emit_vlmax_insn (icode, itype, ops);
+}
+
+/* Expand the binary vx combine with the format like v2 = vop(vec_dup(x), v1)
+   and its' vxrm value.  Aka the second op comes from the vec_duplicate,
+   and the first op is the vector reg.  */
+
+void
+expand_vx_binary_vxrm_vec_dup_vec (rtx op_0, rtx op_1, rtx op_2, int unspec,
+				   int vxrm_val, machine_mode mode)
+{
+  enum insn_code icode;
+  enum insn_type itype = get_insn_type_by_vxrm_val (vxrm_val);
+  rtx ops[] = {op_0, op_1, op_2};
+
+  switch (unspec)
+    {
+    case UNSPEC_VAADD:
+    case UNSPEC_VAADDU:
+      icode = code_for_pred_scalar (unspec, mode);
+      break;
+    default:
+      gcc_unreachable ();
+    }
+
+  emit_vlmax_insn (icode, itype, ops);
+}
+
 /* Expand the binary vx combine with the format like v2 = vop(v1, vec_dup(x)).
    Aka the second op comes from the vec_duplicate, and the first op is
    the vector reg.  */
@@ -5584,6 +5756,8 @@ expand_vx_binary_vec_vec_dup (rtx op_0, rtx op_1, rtx op_2,
     case UMIN:
     case US_PLUS:
     case US_MINUS:
+    case SS_PLUS:
+    case SS_MINUS:
       icode = code_for_pred_scalar (code, mode);
       break;
     default:
@@ -5766,24 +5940,84 @@ count_regno_occurrences (rtx_insn *rinsn, unsigned int regno)
   return count;
 }
 
-/* Return true if the OP can be directly broadcasted.  */
+/* Return true if the OP can be broadcast with a
+   v[f]mv.v.[xif] instruction.  */
+
+bool
+can_be_broadcast_p (rtx op)
+{
+  machine_mode mode = GET_MODE (op);
+
+  /* Zero always works and we can always put an immediate into a
+     register.
+     What's tricky is that for an immediate we don't know the
+     register's mode it will end up in, i.e. what element size
+     we want to broadcast.  So even if the immediate is small it might
+     still end up in a DImode register that we cannot broadcast.
+     vmv.s.x, i.e. a single-element set can handle this, though,
+     because it implicitly sign-extends to SEW.  */
+  if (rtx_equal_p (op, CONST0_RTX (mode))
+      || const_int_operand (op, Xmode))
+    return true;
+
+  /* Do not accept DImode broadcasts on !TARGET_64BIT.  Those
+     are handled by strided broadcast.  */
+  if (INTEGRAL_MODE_P (mode)
+      && maybe_gt (GET_MODE_SIZE (mode), UNITS_PER_WORD))
+    return false;
+
+  /* Non-register operands that can be forced into a register we can
+     handle.  These don't need to use strided broadcast. */
+  if (INTEGRAL_MODE_P (mode)
+      && (memory_operand (op, mode) || CONST_POLY_INT_P (op))
+      && can_create_pseudo_p ())
+    return true;
+
+  /* Likewise, do not accept HFmode broadcast if we don't have
+     vfmv.v.f for 16-bit registers available.  */
+  if (mode == HFmode && !TARGET_ZVFH)
+    return false;
+
+  /* Same for float, just that we can always handle 64-bit doubles
+     even on !TARGET_64BIT.  We have ruled out 16-bit HF already
+     above.  */
+  if (FLOAT_MODE_P (mode)
+      && (memory_operand (op, mode) || CONSTANT_P (op))
+      && can_create_pseudo_p ())
+    return true;
+
+  /* After excluding all the cases we cannot handle the register types
+     that remain can always be broadcast.  */
+  if (register_operand (op, mode))
+    return true;
+
+  return false;
+}
+
+/* Returns true for all operands that cannot use vmv.vx, vfmv.vf,
+   vmv.s.x, or vfmv.s.f but rather need to go via memory.  */
+
 bool
-can_be_broadcasted_p (rtx op)
+strided_broadcast_p (rtx op)
 {
   machine_mode mode = GET_MODE (op);
-  /* We don't allow RA (register allocation) reload generate
-    (vec_duplicate:DI reg) in RV32 system wheras we allow
-    (vec_duplicate:DI mem) in RV32 system.  */
-  if (!can_create_pseudo_p () && !FLOAT_MODE_P (mode)
-      && maybe_gt (GET_MODE_SIZE (mode), GET_MODE_SIZE (Pmode))
-      && !satisfies_constraint_Wdm (op))
+  if (!memory_operand (op, mode)
+      && !register_operand (op, mode)
+      && !rtx_equal_p (op, CONST0_RTX (mode))
+      && !const_int_operand (op, mode))
     return false;
 
-  if (satisfies_constraint_K (op) || register_operand (op, mode)
-      || satisfies_constraint_Wdm (op) || rtx_equal_p (op, CONST0_RTX (mode)))
+  /* !TARGET64_BIT does not have a vmv.v.x/vmv.s.x for 64-bit
+     DImode elements.  */
+  if (INTEGRAL_MODE_P (mode)
+      && maybe_gt (GET_MODE_SIZE (mode), UNITS_PER_WORD))
+    return true;
+
+  /* Zvfhmin does not have a vfmv.v.f/vfmv.s.f.  for 16-bit elements.  */
+  if (!TARGET_ZVFH && mode == HFmode)
     return true;
 
-  return can_create_pseudo_p () && nonmemory_operand (op, mode);
+  return false;
 }
 
 void
@@ -5898,7 +6132,10 @@ whole_reg_to_reg_move_p (rtx *ops, machine_mode mode, int avl_type_index)
   return false;
 }
 
-/* Return true if we can transform vmv.v.x/vfmv.v.f to vmv.s.x/vfmv.s.f.  */
+/* Return true if we can transform vmv.v.x/vfmv.v.f to vmv.s.x/vfmv.s.f.
+   That's the case if we're dealing with a scalar broadcast that
+   has VL = 1.  */
+
 bool
 splat_to_scalar_move_p (rtx *ops)
 {
diff --git a/gcc/config/riscv/riscv-vector-builtins-bases.cc b/gcc/config/riscv/riscv-vector-builtins-bases.cc
index bf5172c..7e4d396 100644
--- a/gcc/config/riscv/riscv-vector-builtins-bases.cc
+++ b/gcc/config/riscv/riscv-vector-builtins-bases.cc
@@ -643,7 +643,8 @@ public:
 	return e.use_exact_insn (code_for_pred_mov (e.vector_mode ()));
       case OP_TYPE_x:
       case OP_TYPE_f:
-	return e.use_exact_insn (code_for_pred_broadcast (e.vector_mode ()));
+	return e.use_scalar_broadcast_insn
+	  (code_for_pred_broadcast (e.vector_mode ()));
       default:
 	gcc_unreachable ();
       }
diff --git a/gcc/config/riscv/riscv-vector-builtins.cc b/gcc/config/riscv/riscv-vector-builtins.cc
index f652a12..0db7549 100644
--- a/gcc/config/riscv/riscv-vector-builtins.cc
+++ b/gcc/config/riscv/riscv-vector-builtins.cc
@@ -4753,7 +4753,10 @@ function_expander::use_ternop_insn (bool vd_accum_p, insn_code icode)
 }
 
 /* Implement the call using instruction ICODE, with a 1:1 mapping between
-   arguments and input operands.  */
+   arguments and input operands.
+   There are operands that cannot be broadcast using v[f]mv.  In that case
+   we switch to a strided broadcast.  */
+
 rtx
 function_expander::use_widen_ternop_insn (insn_code icode)
 {
@@ -4794,7 +4797,10 @@ function_expander::use_widen_ternop_insn (insn_code icode)
 }
 
 /* Implement the call using instruction ICODE, with a 1:1 mapping between
-   arguments and input operands.  */
+   arguments and input operands.
+   There are operands that cannot be broadcast using v[f]mv.  In that case
+   we switch to a strided broadcast.  */
+
 rtx
 function_expander::use_scalar_move_insn (insn_code icode)
 {
@@ -4812,6 +4818,37 @@ function_expander::use_scalar_move_insn (insn_code icode)
   for (int argno = arg_offset; argno < call_expr_nargs (exp); argno++)
     add_input_operand (argno);
 
+  if (!can_be_broadcast_p (m_ops[3].value))
+    icode = code_for_pred_strided_broadcast (vector_mode ());
+
+  add_input_operand (Pmode, get_tail_policy_for_pred (pred));
+  add_input_operand (Pmode, get_mask_policy_for_pred (pred));
+  add_input_operand (Pmode, get_avl_type_rtx (avl_type::NONVLMAX));
+  return generate_insn (icode);
+}
+
+/* Implement the call using instruction ICODE, with a 1:1 mapping between
+   arguments and input operands.  */
+rtx
+function_expander::use_scalar_broadcast_insn (insn_code icode)
+{
+  machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
+
+  /* Record the offset to get the argument.  */
+  int arg_offset = 0;
+  add_all_one_mask_operand (mask_mode ());
+
+  if (use_real_merge_p (pred))
+    add_input_operand (arg_offset++);
+  else
+    add_vundef_operand (mode);
+
+  for (int argno = arg_offset; argno < call_expr_nargs (exp); argno++)
+    add_input_operand (argno);
+
+  if (!can_be_broadcast_p (m_ops[3].value))
+    icode = code_for_pred_strided_broadcast (vector_mode ());
+
   add_input_operand (Pmode, get_tail_policy_for_pred (pred));
   add_input_operand (Pmode, get_mask_policy_for_pred (pred));
   add_input_operand (Pmode, get_avl_type_rtx (avl_type::NONVLMAX));
@@ -4977,6 +5014,12 @@ registered_function::overloaded_hash () const
   for (unsigned int i = 0; i < argument_types.length (); i++)
     {
       type = argument_types[i];
+
+      /* If we're passed something entirely unreasonable, just ignore here.
+	 We'll warn later anyway.  */
+      if (TREE_CODE_CLASS (TREE_CODE (type)) != tcc_type)
+	continue;
+
       unsigned_p = POINTER_TYPE_P (type) ? TYPE_UNSIGNED (TREE_TYPE (type))
 					 : TYPE_UNSIGNED (type);
       mode_p = POINTER_TYPE_P (type) ? TYPE_MODE (TREE_TYPE (type))
diff --git a/gcc/config/riscv/riscv-vector-builtins.h b/gcc/config/riscv/riscv-vector-builtins.h
index 1f2587a..86d8115 100644
--- a/gcc/config/riscv/riscv-vector-builtins.h
+++ b/gcc/config/riscv/riscv-vector-builtins.h
@@ -497,6 +497,7 @@ public:
   rtx use_ternop_insn (bool, insn_code);
   rtx use_widen_ternop_insn (insn_code);
   rtx use_scalar_move_insn (insn_code);
+  rtx use_scalar_broadcast_insn (insn_code);
   rtx generate_insn (insn_code);
 
   /* The function call expression.  */
diff --git a/gcc/config/riscv/riscv-vector-costs.cc b/gcc/config/riscv/riscv-vector-costs.cc
index 4d8170d..44ef44a 100644
--- a/gcc/config/riscv/riscv-vector-costs.cc
+++ b/gcc/config/riscv/riscv-vector-costs.cc
@@ -178,8 +178,8 @@ get_live_range (hash_map<tree, pair> *live_ranges, tree arg)
        STMT 5 (be vectorized)      -- point 2
        ...
 */
-static void
-compute_local_program_points (
+void
+costs::compute_local_program_points (
   vec_info *vinfo,
   hash_map<basic_block, vec<stmt_point>> &program_points_per_bb)
 {
@@ -274,14 +274,14 @@ loop_invariant_op_p (class loop *loop,
 
 /* Return true if the variable should be counted into liveness.  */
 static bool
-variable_vectorized_p (class loop *loop, stmt_vec_info stmt_info, tree var,
-		       bool lhs_p)
+variable_vectorized_p (class loop *loop, stmt_vec_info stmt_info,
+		       slp_tree node, tree var, bool lhs_p)
 {
   if (!var)
     return false;
   gimple *stmt = STMT_VINFO_STMT (stmt_info);
-  enum stmt_vec_info_type type
-    = STMT_VINFO_TYPE (vect_stmt_to_vectorize (stmt_info));
+  stmt_info = vect_stmt_to_vectorize (stmt_info);
+  enum stmt_vec_info_type type = SLP_TREE_TYPE (node);
   if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
     {
       if (gimple_call_internal_fn (stmt) == IFN_MASK_STORE
@@ -357,8 +357,8 @@ variable_vectorized_p (class loop *loop, stmt_vec_info stmt_info, tree var,
 
    The live range of SSA 1 is [1, 3] in bb 2.
    The live range of SSA 2 is [0, 4] in bb 3.  */
-static machine_mode
-compute_local_live_ranges (
+machine_mode
+costs::compute_local_live_ranges (
   loop_vec_info loop_vinfo,
   const hash_map<basic_block, vec<stmt_point>> &program_points_per_bb,
   hash_map<basic_block, hash_map<tree, pair>> &live_ranges_per_bb)
@@ -388,8 +388,11 @@ compute_local_live_ranges (
 	      unsigned int point = program_point.point;
 	      gimple *stmt = program_point.stmt;
 	      tree lhs = gimple_get_lhs (stmt);
-	      if (variable_vectorized_p (loop, program_point.stmt_info, lhs,
-					 true))
+	      slp_tree *node = vinfo_slp_map.get (program_point.stmt_info);
+	      if (!node)
+		continue;
+	      if (variable_vectorized_p (loop, program_point.stmt_info,
+					 *node, lhs, true))
 		{
 		  biggest_mode = get_biggest_mode (biggest_mode,
 						   TYPE_MODE (TREE_TYPE (lhs)));
@@ -397,7 +400,7 @@ compute_local_live_ranges (
 		  pair &live_range
 		    = live_ranges->get_or_insert (lhs, &existed_p);
 		  gcc_assert (!existed_p);
-		  if (STMT_VINFO_MEMORY_ACCESS_TYPE (program_point.stmt_info)
+		  if (SLP_TREE_MEMORY_ACCESS_TYPE (*node)
 		      == VMAT_LOAD_STORE_LANES)
 		    point = get_first_lane_point (program_points,
 						  program_point.stmt_info);
@@ -406,8 +409,8 @@ compute_local_live_ranges (
 	      for (i = 0; i < gimple_num_args (stmt); i++)
 		{
 		  tree var = gimple_arg (stmt, i);
-		  if (variable_vectorized_p (loop, program_point.stmt_info, var,
-					     false))
+		  if (variable_vectorized_p (loop, program_point.stmt_info,
+					     *node, var, false))
 		    {
 		      biggest_mode
 			= get_biggest_mode (biggest_mode,
@@ -415,8 +418,7 @@ compute_local_live_ranges (
 		      bool existed_p = false;
 		      pair &live_range
 			= live_ranges->get_or_insert (var, &existed_p);
-		      if (STMT_VINFO_MEMORY_ACCESS_TYPE (
-			    program_point.stmt_info)
+		      if (SLP_TREE_MEMORY_ACCESS_TYPE (*node)
 			  == VMAT_LOAD_STORE_LANES)
 			point = get_last_lane_point (program_points,
 						     program_point.stmt_info);
@@ -597,15 +599,15 @@ get_store_value (gimple *stmt)
 }
 
 /* Return true if additional vector vars needed.  */
-static bool
-need_additional_vector_vars_p (stmt_vec_info stmt_info)
+bool
+costs::need_additional_vector_vars_p (stmt_vec_info stmt_info,
+				      slp_tree node)
 {
-  enum stmt_vec_info_type type
-    = STMT_VINFO_TYPE (vect_stmt_to_vectorize (stmt_info));
+  enum stmt_vec_info_type type = SLP_TREE_TYPE (node);
   if (type == load_vec_info_type || type == store_vec_info_type)
     {
       if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
-	  && STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) == VMAT_GATHER_SCATTER)
+	  && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_GATHER_SCATTER)
 	return true;
 
       machine_mode mode = TYPE_MODE (STMT_VINFO_VECTYPE (stmt_info));
@@ -657,8 +659,8 @@ compute_estimated_lmul (loop_vec_info loop_vinfo, machine_mode mode)
 
    Then, after this function, we update SSA 1 live range in bb 2
    into [2, 4] since SSA 1 is live out into bb 3.  */
-static void
-update_local_live_ranges (
+void
+costs::update_local_live_ranges (
   vec_info *vinfo,
   hash_map<basic_block, vec<stmt_point>> &program_points_per_bb,
   hash_map<basic_block, hash_map<tree, pair>> &live_ranges_per_bb,
@@ -685,8 +687,13 @@ update_local_live_ranges (
 	{
 	  gphi *phi = psi.phi ();
 	  stmt_vec_info stmt_info = vinfo->lookup_stmt (phi);
-	  if (STMT_VINFO_TYPE (vect_stmt_to_vectorize (stmt_info))
-	      == undef_vec_info_type)
+	  stmt_info = vect_stmt_to_vectorize (stmt_info);
+	  slp_tree *node = vinfo_slp_map.get (stmt_info);
+
+	  if (!node)
+	    continue;
+
+	  if (SLP_TREE_TYPE (*node) == undef_vec_info_type)
 	    continue;
 
 	  for (j = 0; j < gimple_phi_num_args (phi); j++)
@@ -761,9 +768,12 @@ update_local_live_ranges (
 	  if (!is_gimple_assign_or_call (gsi_stmt (si)))
 	    continue;
 	  stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
-	  enum stmt_vec_info_type type
-	    = STMT_VINFO_TYPE (vect_stmt_to_vectorize (stmt_info));
-	  if (need_additional_vector_vars_p (stmt_info))
+	  stmt_info = vect_stmt_to_vectorize (stmt_info);
+	  slp_tree *node = vinfo_slp_map.get (stmt_info);
+	  if (!node)
+	    continue;
+	  enum stmt_vec_info_type type = SLP_TREE_TYPE (*node);
+	  if (need_additional_vector_vars_p (stmt_info, *node))
 	    {
 	      /* For non-adjacent load/store STMT, we will potentially
 		 convert it into:
@@ -816,8 +826,8 @@ update_local_live_ranges (
 }
 
 /* Compute the maximum live V_REGS.  */
-static bool
-has_unexpected_spills_p (loop_vec_info loop_vinfo)
+bool
+costs::has_unexpected_spills_p (loop_vec_info loop_vinfo)
 {
   /* Compute local program points.
      It's a fast and effective computation.  */
@@ -899,7 +909,11 @@ costs::analyze_loop_vinfo (loop_vec_info loop_vinfo)
   /* Detect whether we're vectorizing for VLA and should apply the unrolling
      heuristic described above m_unrolled_vls_niters.  */
   record_potential_vls_unrolling (loop_vinfo);
+}
 
+void
+costs::record_lmul_spills (loop_vec_info loop_vinfo)
+{
   /* Detect whether the LOOP has unexpected spills.  */
   record_potential_unexpected_spills (loop_vinfo);
 }
@@ -1071,7 +1085,7 @@ costs::better_main_loop_than_p (const vector_costs *uncast_other) const
    load/store.  */
 static int
 segment_loadstore_group_size (enum vect_cost_for_stmt kind,
-			      stmt_vec_info stmt_info)
+			      stmt_vec_info stmt_info, slp_tree node)
 {
   if (stmt_info
       && (kind == vector_load || kind == vector_store)
@@ -1079,7 +1093,7 @@ segment_loadstore_group_size (enum vect_cost_for_stmt kind,
     {
       stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
       if (stmt_info
-	  && STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) == VMAT_LOAD_STORE_LANES)
+	  && SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_LOAD_STORE_LANES)
 	return DR_GROUP_SIZE (stmt_info);
     }
   return 0;
@@ -1093,7 +1107,7 @@ segment_loadstore_group_size (enum vect_cost_for_stmt kind,
 unsigned
 costs::adjust_stmt_cost (enum vect_cost_for_stmt kind, loop_vec_info loop,
 			 stmt_vec_info stmt_info,
-			 slp_tree, tree vectype, int stmt_cost)
+			 slp_tree node, tree vectype, int stmt_cost)
 {
   const cpu_vector_cost *costs = get_vector_costs ();
   switch (kind)
@@ -1116,7 +1130,8 @@ costs::adjust_stmt_cost (enum vect_cost_for_stmt kind, loop_vec_info loop,
 		 each vector in the group.  Here we additionally add permute
 		 costs for each.  */
 	      /* TODO: Indexed and ordered/unordered cost.  */
-	      int group_size = segment_loadstore_group_size (kind, stmt_info);
+	      int group_size = segment_loadstore_group_size (kind, stmt_info,
+							     node);
 	      if (group_size > 1)
 		{
 		  switch (group_size)
@@ -1239,8 +1254,12 @@ costs::add_stmt_cost (int count, vect_cost_for_stmt kind,
   int stmt_cost
     = targetm.vectorize.builtin_vectorization_cost (kind, vectype, misalign);
 
+  if (stmt_info && node)
+    vinfo_slp_map.put (stmt_info, node);
+
   /* Do one-time initialization based on the vinfo.  */
   loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo);
+
   if (!m_analyzed_vinfo)
     {
       if (loop_vinfo)
@@ -1326,6 +1345,8 @@ costs::finish_cost (const vector_costs *scalar_costs)
 {
   if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo))
     {
+      record_lmul_spills (loop_vinfo);
+
       adjust_vect_cost_per_loop (loop_vinfo);
     }
   vector_costs::finish_cost (scalar_costs);
diff --git a/gcc/config/riscv/riscv-vector-costs.h b/gcc/config/riscv/riscv-vector-costs.h
index de546a6..b84ceb1 100644
--- a/gcc/config/riscv/riscv-vector-costs.h
+++ b/gcc/config/riscv/riscv-vector-costs.h
@@ -91,7 +91,10 @@ private:
   typedef pair_hash <tree_operand_hash, tree_operand_hash> tree_pair_hash;
   hash_set <tree_pair_hash> memrefs;
 
+  hash_map <stmt_vec_info, slp_tree> vinfo_slp_map;
+
   void analyze_loop_vinfo (loop_vec_info);
+  void record_lmul_spills (loop_vec_info loop_vinfo);
   void record_potential_vls_unrolling (loop_vec_info);
   bool prefer_unrolled_loop () const;
 
@@ -103,6 +106,19 @@ private:
   bool m_has_unexpected_spills_p = false;
   void record_potential_unexpected_spills (loop_vec_info);
 
+  void compute_local_program_points (vec_info *,
+				     hash_map<basic_block, vec<stmt_point>> &);
+  void update_local_live_ranges (vec_info *,
+				 hash_map<basic_block, vec<stmt_point>> &,
+				 hash_map<basic_block, hash_map<tree, pair>> &,
+				 machine_mode *);
+  machine_mode compute_local_live_ranges
+    (loop_vec_info, const hash_map<basic_block, vec<stmt_point>> &,
+     hash_map<basic_block, hash_map<tree, pair>> &);
+
+  bool has_unexpected_spills_p (loop_vec_info);
+  bool need_additional_vector_vars_p (stmt_vec_info, slp_tree);
+
   void adjust_vect_cost_per_loop (loop_vec_info);
   unsigned adjust_stmt_cost (enum vect_cost_for_stmt kind,
 			     loop_vec_info,
diff --git a/gcc/config/riscv/riscv-vsetvl.def b/gcc/config/riscv/riscv-vsetvl.def
index d7a5ada..0f999d2 100644
--- a/gcc/config/riscv/riscv-vsetvl.def
+++ b/gcc/config/riscv/riscv-vsetvl.def
@@ -79,7 +79,7 @@ DEF_SEW_LMUL_RULE (sew_only, sew_only, sew_only, sew_eq_p, sew_eq_p, nop)
 DEF_SEW_LMUL_RULE (sew_only, ge_sew, sew_only,
 		   sew_ge_and_prev_sew_le_next_max_sew_p, sew_ge_p, nop)
 DEF_SEW_LMUL_RULE (
-  sew_only, ratio_and_ge_sew, sew_lmul,
+  sew_only, ratio_and_ge_sew, ratio_and_ge_sew,
   sew_ge_and_prev_sew_le_next_max_sew_and_next_ratio_valid_for_prev_sew_p,
   always_false, modify_lmul_with_next_ratio)
 
@@ -104,9 +104,9 @@ DEF_SEW_LMUL_RULE (ratio_and_ge_sew, sew_lmul, sew_lmul,
 DEF_SEW_LMUL_RULE (ratio_and_ge_sew, ratio_only, ratio_and_ge_sew, ratio_eq_p,
 		   ratio_eq_p, use_max_sew_and_lmul_with_prev_ratio)
 DEF_SEW_LMUL_RULE (
-  ratio_and_ge_sew, sew_only, sew_only,
+  ratio_and_ge_sew, sew_only, ratio_and_ge_sew,
   sew_le_and_next_sew_le_prev_max_sew_and_prev_ratio_valid_for_next_sew_p,
-  always_false, use_next_sew_with_prev_ratio)
+  sew_eq_p, use_next_sew_with_prev_ratio)
 DEF_SEW_LMUL_RULE (ratio_and_ge_sew, ge_sew, ratio_and_ge_sew,
 		   max_sew_overlap_and_prev_ratio_valid_for_next_sew_p,
 		   sew_ge_p, use_max_sew_and_lmul_with_prev_ratio)
diff --git a/gcc/config/riscv/riscv.cc b/gcc/config/riscv/riscv.cc
index 8fa1082..0a9fcef 100644
--- a/gcc/config/riscv/riscv.cc
+++ b/gcc/config/riscv/riscv.cc
@@ -87,6 +87,10 @@ along with GCC; see the file COPYING3.  If not see
 #include "riscv-vector-costs.h"
 #include "riscv-subset.h"
 
+/* Target variants that support full conditional move.  */
+#define	TARGET_COND_MOV						\
+   (TARGET_SFB_ALU || TARGET_XTHEADCONDMOV || TARGET_XMIPSCMOV)
+
 /* True if X is an UNSPEC wrapper around a SYMBOL_REF or LABEL_REF.  */
 #define UNSPEC_ADDRESS_P(X)					\
   (GET_CODE (X) == UNSPEC					\
@@ -166,7 +170,7 @@ struct GTY(())  riscv_frame_info {
 };
 
 enum riscv_privilege_levels {
-  UNKNOWN_MODE, USER_MODE, SUPERVISOR_MODE, MACHINE_MODE
+  UNKNOWN_MODE, SUPERVISOR_MODE, MACHINE_MODE, RNMI_MODE
 };
 
 struct GTY(()) mode_switching_info {
@@ -283,6 +287,10 @@ enum riscv_fusion_pairs
   RISCV_FUSE_AUIPC_LD = (1 << 7),
   RISCV_FUSE_LDPREINCREMENT = (1 << 8),
   RISCV_FUSE_ALIGNED_STD = (1 << 9),
+  RISCV_FUSE_CACHE_ALIGNED_STD = (1 << 10),
+  RISCV_FUSE_BFEXT = (1 << 11),
+  RISCV_FUSE_EXPANDED_LD = (1 << 12),
+  RISCV_FUSE_B_ALUI = (1 << 13),
 };
 
 /* Costs of various operations on the different architectures.  */
@@ -302,6 +310,7 @@ struct riscv_tune_param
   bool vector_unaligned_access;
   bool use_divmod_expansion;
   bool overlap_op_by_pieces;
+  bool use_zero_stride_load;
   bool speculative_sched_vsetvl;
   unsigned int fusible_ops;
   const struct cpu_vector_cost *vec_costs;
@@ -465,6 +474,7 @@ static const struct riscv_tune_param generic_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   NULL,						/* vector cost */
@@ -488,6 +498,7 @@ static const struct riscv_tune_param rocket_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   NULL,						/* vector cost */
@@ -511,6 +522,7 @@ static const struct riscv_tune_param sifive_7_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   NULL,						/* vector cost */
@@ -534,6 +546,7 @@ static const struct riscv_tune_param sifive_p400_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_LUI_ADDI | RISCV_FUSE_AUIPC_ADDI,  /* fusible_ops */
   &generic_vector_cost,				/* vector cost */
@@ -557,6 +570,7 @@ static const struct riscv_tune_param sifive_p600_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_LUI_ADDI | RISCV_FUSE_AUIPC_ADDI,  /* fusible_ops */
   &generic_vector_cost,				/* vector cost */
@@ -580,6 +594,7 @@ static const struct riscv_tune_param thead_c906_tune_info = {
   false,					/* vector_unaligned_access */
   false,	/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   NULL,						/* vector cost */
@@ -603,6 +618,7 @@ static const struct riscv_tune_param xiangshan_nanhu_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_ZEXTW | RISCV_FUSE_ZEXTH,          /* fusible_ops */
   NULL,						/* vector cost */
@@ -626,6 +642,7 @@ static const struct riscv_tune_param generic_ooo_tune_info = {
   true,						/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   true,						/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   &generic_vector_cost,				/* vector cost */
@@ -649,6 +666,7 @@ static const struct riscv_tune_param tt_ascalon_d8_tune_info = {
   true,						/* vector_unaligned_access */
   true,						/* use_divmod_expansion */
   true,						/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   &generic_vector_cost,				/* vector cost */
@@ -672,6 +690,7 @@ static const struct riscv_tune_param optimize_size_tune_info = {
   false,					/* vector_unaligned_access */
   false,					/* use_divmod_expansion */
   false,					/* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,                           /* fusible_ops */
   NULL,						/* vector cost */
@@ -695,6 +714,7 @@ static const struct riscv_tune_param mips_p8700_tune_info = {
   false,        /* vector_unaligned_access */
   true,         /* use_divmod_expansion */
   false,        /* overlap_op_by_pieces */
+  true,						/* use_zero_stride_load */
   false,					/* speculative_sched_vsetvl */
   RISCV_FUSE_NOTHING,				/* fusible_ops */
   NULL,         /* vector cost */
@@ -3947,13 +3967,27 @@ get_vector_binary_rtx_cost (rtx x, int scalar2vr_cost)
 {
   gcc_assert (riscv_v_ext_mode_p (GET_MODE (x)));
 
-  rtx op_0 = XEXP (x, 0);
-  rtx op_1 = XEXP (x, 1);
+  rtx neg;
+  rtx op_0;
+  rtx op_1;
+
+  if (GET_CODE (x) == UNSPEC)
+    {
+      op_0 = XVECEXP (x, 0, 0);
+      op_1 = XVECEXP (x, 0, 1);
+    }
+  else
+    {
+      op_0 = XEXP (x, 0);
+      op_1 = XEXP (x, 1);
+    }
 
   if (GET_CODE (op_0) == VEC_DUPLICATE
       || GET_CODE (op_1) == VEC_DUPLICATE)
     return (scalar2vr_cost + 1) * COSTS_N_INSNS (1);
-  else if (GET_CODE (op_0) == NEG && GET_CODE (op_1) == VEC_DUPLICATE)
+  else if (GET_CODE (neg = op_0) == NEG
+	   && (GET_CODE (op_1) == VEC_DUPLICATE
+	       || GET_CODE (XEXP (neg, 0)) == VEC_DUPLICATE))
     return (scalar2vr_cost + 1) * COSTS_N_INSNS (1);
   else
     return COSTS_N_INSNS (1);
@@ -3997,8 +4031,25 @@ riscv_rtx_costs (rtx x, machine_mode mode, int outer_code, int opno ATTRIBUTE_UN
 		    case UMOD:
 		    case US_PLUS:
 		    case US_MINUS:
+		    case SS_PLUS:
+		    case SS_MINUS:
 		      *total = get_vector_binary_rtx_cost (op, scalar2vr_cost);
 		      break;
+		    case UNSPEC:
+		      {
+			switch (XINT (op, 1))
+			  {
+			  case UNSPEC_VAADDU:
+			  case UNSPEC_VAADD:
+			    *total
+			      = get_vector_binary_rtx_cost (op, scalar2vr_cost);
+			    break;
+			  default:
+			    *total = COSTS_N_INSNS (1);
+			    break;
+			  }
+		      }
+		      break;
 		    default:
 		      *total = COSTS_N_INSNS (1);
 		      break;
@@ -4151,7 +4202,7 @@ riscv_rtx_costs (rtx x, machine_mode mode, int outer_code, int opno ATTRIBUTE_UN
       return false;
 
     case IF_THEN_ELSE:
-      if ((TARGET_SFB_ALU || TARGET_XTHEADCONDMOV)
+      if (TARGET_COND_MOV
 	  && reg_or_0_operand (XEXP (x, 1), mode)
 	  && sfb_alu_operand (XEXP (x, 2), mode)
 	  && comparison_operator (XEXP (x, 0), VOIDmode))
@@ -5469,6 +5520,68 @@ riscv_expand_conditional_branch (rtx label, rtx_code code, rtx op0, rtx op1)
   emit_jump_insn (gen_condjump (condition, label));
 }
 
+/* canonicalization of the comparands.  */
+void
+canonicalize_comparands (rtx_code code, rtx *op0, rtx *op1)
+{
+  /* An integer comparison must be comparing WORD_MODE objects.
+     Extend the comparison arguments as necessary.  */
+  if ((INTEGRAL_MODE_P (GET_MODE (*op0)) && GET_MODE (*op0) != word_mode)
+      || (INTEGRAL_MODE_P (GET_MODE (*op1)) && GET_MODE (*op1) != word_mode))
+    riscv_extend_comparands (code, op0, op1);
+
+  /* We might have been handed back a SUBREG.  Just to make things
+     easy, force it into a REG.  */
+  if (!REG_P (*op0) && !CONST_INT_P (*op0))
+    *op0 = force_reg (word_mode, *op0);
+  if (!REG_P (*op1) && !CONST_INT_P (*op1))
+    *op1 = force_reg (word_mode, *op1);
+}
+
+/* Emit target specific conditional move like TARGET_XMIPSCMOV etc.  */
+bool
+riscv_target_conditional_move (rtx dest, rtx op0, rtx op1, rtx_code code,
+				rtx cons, rtx alt)
+{
+  machine_mode dst_mode = GET_MODE (dest);
+  rtx target;
+
+  /* force the operands to the register.  */
+  cons = force_reg (dst_mode, cons);
+  alt = force_reg (dst_mode, alt);
+
+  if (TARGET_XMIPSCMOV)
+    {
+      if (code == EQ || code == NE)
+	{
+	  op0 = riscv_zero_if_equal (op0, op1);
+	  op1 = const0_rtx;
+	}
+      else
+	{
+	  target = gen_reg_rtx (GET_MODE (op0));
+	  riscv_emit_int_order_test (code, 0, target, op0, op1);
+	  op0 = target;
+	  op1 = const0_rtx;
+	  code = NE;
+	}
+      riscv_emit_int_compare (&code, &op0, &op1);
+      rtx cond = gen_rtx_fmt_ee (code, GET_MODE (op0), op0, op1);
+      emit_insn (gen_rtx_SET (dest, gen_rtx_IF_THEN_ELSE (dst_mode,
+							   cond, cons, alt)));
+      return true;
+    }
+  /* TARGET_SFB_ALU || TARGET_XTHEADCONDMOV.  */
+  else
+   {
+     riscv_emit_int_compare (&code, &op0, &op1, !TARGET_SFB_ALU);
+     rtx cond = gen_rtx_fmt_ee (code, GET_MODE (op0), op0, op1);
+     emit_insn (gen_rtx_SET (dest, gen_rtx_IF_THEN_ELSE (dst_mode, cond,
+							 cons, alt)));
+     return true;
+   }
+}
+
 /* Emit a cond move: If OP holds, move CONS to DEST; else move ALT to DEST.
    Return 0 if expansion failed.  */
 
@@ -5521,34 +5634,22 @@ riscv_expand_conditional_move (rtx dest, rtx op, rtx cons, rtx alt)
       /* If we need more special cases, add them here.  */
     }
 
+
   if (((TARGET_ZICOND_LIKE
 	|| (arith_operand (cons, dst_mode) && arith_operand (alt, dst_mode)))
        && GET_MODE_CLASS (dst_mode) == MODE_INT
        && GET_MODE_CLASS (cond_mode) == MODE_INT)
-      || TARGET_SFB_ALU || TARGET_XTHEADCONDMOV)
+       || TARGET_COND_MOV)
     {
       machine_mode mode0 = GET_MODE (op0);
-      machine_mode mode1 = GET_MODE (op1);
-
-      /* An integer comparison must be comparing WORD_MODE objects.
-	 Extend the comparison arguments as necessary.  */
-      if ((INTEGRAL_MODE_P (mode0) && mode0 != word_mode)
-	  || (INTEGRAL_MODE_P (mode1) && mode1 != word_mode))
-	riscv_extend_comparands (code, &op0, &op1);
 
-      /* We might have been handed back a SUBREG.  Just to make things
-	 easy, force it into a REG.  */
-      if (!REG_P (op0) && !CONST_INT_P (op0))
-	op0 = force_reg (word_mode, op0);
-      if (!REG_P (op1) && !CONST_INT_P (op1))
-	op1 = force_reg (word_mode, op1);
+      canonicalize_comparands (code,&op0,&op1);
 
       /* In the fallback generic case use DST_MODE rather than WORD_MODE
 	 for the output of the SCC instruction, to match the mode of the NEG
 	 operation below.  The output of SCC is 0 or 1 boolean, so it is
 	 valid for input in any scalar integer mode.  */
-      rtx tmp = gen_reg_rtx ((TARGET_ZICOND_LIKE
-			      || TARGET_SFB_ALU || TARGET_XTHEADCONDMOV)
+      rtx tmp = gen_reg_rtx ((TARGET_ZICOND_LIKE || TARGET_COND_MOV)
 			     ? word_mode : dst_mode);
       bool invert = false;
 
@@ -5585,25 +5686,12 @@ riscv_expand_conditional_move (rtx dest, rtx op, rtx cons, rtx alt)
 	  op0 = XEXP (op, 0);
 	  op1 = XEXP (op, 1);
 	}
-      else if (!TARGET_ZICOND_LIKE && !TARGET_SFB_ALU && !TARGET_XTHEADCONDMOV)
+      else if (!TARGET_ZICOND_LIKE && !TARGET_COND_MOV)
 	riscv_expand_int_scc (tmp, code, op0, op1, &invert);
 
-      if (TARGET_SFB_ALU || TARGET_XTHEADCONDMOV)
-	{
-	  riscv_emit_int_compare (&code, &op0, &op1, !TARGET_SFB_ALU);
-	  rtx cond = gen_rtx_fmt_ee (code, GET_MODE (op0), op0, op1);
+      if (TARGET_COND_MOV)
+	return riscv_target_conditional_move (dest, op0, op1, code, cons, alt);
 
-	  /* The expander is a bit loose in its specification of the true
-	     arm of the conditional move.  That allows us to support more
-	     cases for extensions which are more general than SFB.  But
-	     does mean we need to force CONS into a register at this point.  */
-	  cons = force_reg (dst_mode, cons);
-	  /* With XTheadCondMov we need to force ALT into a register too.  */
-	  alt = force_reg (dst_mode, alt);
-	  emit_insn (gen_rtx_SET (dest, gen_rtx_IF_THEN_ELSE (dst_mode, cond,
-							      cons, alt)));
-	  return true;
-	}
       else if (!TARGET_ZICOND_LIKE)
 	{
 	  if (invert)
@@ -6837,12 +6925,18 @@ riscv_handle_type_attribute (tree *node ATTRIBUTE_UNUSED, tree name, tree args,
 	    }
 
 	  string = TREE_STRING_POINTER (cst);
-	  if (strcmp (string, "user") && strcmp (string, "supervisor")
-	      && strcmp (string, "machine"))
+	  if (!strcmp (string, "rnmi") && !TARGET_SMRNMI)
+	    {
+	      error ("attribute 'rnmi' requires the Smrnmi ISA extension");
+	      *no_add_attrs = true;
+	    }
+	  else if (strcmp (string, "supervisor")
+		   && strcmp (string, "machine")
+		   && strcmp (string, "rnmi"))
 	    {
 	      warning (OPT_Wattributes,
-		       "argument to %qE attribute is not %<\"user\"%>, %<\"supervisor\"%>, "
-		       "or %<\"machine\"%>", name);
+		       "argument to %qE attribute is not %<\"supervisor\"%>, "
+		       "%<\"machine\"%>, or %<\"rnmi\"%>", name);
 	      *no_add_attrs = true;
 	    }
 	}
@@ -8990,7 +9084,7 @@ riscv_allocate_and_probe_stack_space (rtx temp1, HOST_WIDE_INT size)
 	  /* We want the CFA independent of the stack pointer for the
 	     duration of the loop.  */
 	  add_reg_note (insn, REG_CFA_DEF_CFA,
-			plus_constant (Pmode, temp1,
+			plus_constant (Pmode, temp2,
 				       initial_cfa_offset + rounded_size));
 	  RTX_FRAME_RELATED_P (insn) = 1;
 	}
@@ -9623,12 +9717,12 @@ riscv_expand_epilogue (int style)
 
       if (th_int_mask && TH_INT_INTERRUPT (cfun))
 	emit_jump_insn (gen_th_int_pop ());
-      else if (mode == MACHINE_MODE)
-	emit_jump_insn (gen_riscv_mret ());
       else if (mode == SUPERVISOR_MODE)
 	emit_jump_insn (gen_riscv_sret ());
-      else
-	emit_jump_insn (gen_riscv_uret ());
+      else if (mode == RNMI_MODE)
+	emit_jump_insn (gen_riscv_mnret ());
+      else /* Must be MACHINE_MODE.  */
+	emit_jump_insn (gen_riscv_mret ());
     }
   else if (style != SIBCALL_RETURN)
     {
@@ -10203,6 +10297,81 @@ riscv_fusion_enabled_p(enum riscv_fusion_pairs op)
   return tune_param->fusible_ops & op;
 }
 
+/* Matches an add:
+   (set (reg:DI rd) (plus:SI (reg:SI rs1) (reg:SI rs2))) */
+
+static bool
+riscv_set_is_add (rtx set)
+{
+  return (GET_CODE (SET_SRC (set)) == PLUS
+	  && REG_P (XEXP (SET_SRC (set), 0))
+	  && REG_P (XEXP (SET_SRC (set), 1))
+	  && REG_P (SET_DEST (set)));
+}
+
+/* Matches an addi:
+   (set (reg:DI rd) (plus:SI (reg:SI rs1) (const_int imm))) */
+
+static bool
+riscv_set_is_addi (rtx set)
+{
+  return (GET_CODE (SET_SRC (set)) == PLUS
+	  && REG_P (XEXP (SET_SRC (set), 0))
+	  && CONST_INT_P (XEXP (SET_SRC (set), 1))
+	  && REG_P (SET_DEST (set)));
+}
+
+/* Matches an add.uw:
+  (set (reg:DI rd)
+    (plus:DI (zero_extend:DI (reg:SI rs1)) (reg:DI rs2))) */
+
+static bool
+riscv_set_is_adduw (rtx set)
+{
+  return (GET_CODE (SET_SRC (set)) == PLUS
+	  && GET_CODE (XEXP (SET_SRC (set), 0)) == ZERO_EXTEND
+	  && REG_P (XEXP (XEXP (SET_SRC (set), 0), 0))
+	  && REG_P (XEXP (SET_SRC (set), 1))
+	  && REG_P (SET_DEST (set)));
+}
+
+/* Matches a shNadd:
+  (set (reg:DI rd)
+       (plus:DI (ashift:DI (reg:DI rs1) (const_int N)) (reg:DI rS2)) */
+
+static bool
+riscv_set_is_shNadd (rtx set)
+{
+  return (GET_CODE (SET_SRC (set)) == PLUS
+	  && GET_CODE (XEXP (SET_SRC (set), 0)) == ASHIFT
+	  && REG_P (XEXP (XEXP (SET_SRC (set), 0), 0))
+	  && CONST_INT_P (XEXP (XEXP (SET_SRC (set), 0), 1))
+	  && (INTVAL (XEXP (XEXP (SET_SRC (set), 0), 1)) == 1
+	      || INTVAL (XEXP (XEXP (SET_SRC (set), 0), 1)) == 2
+	      || INTVAL (XEXP (XEXP (SET_SRC (set), 0), 1)) == 3)
+	  && REG_P (SET_DEST (set)));
+}
+
+/* Matches a shNadd.uw:
+  (set (reg:DI rd)
+       (plus:DI (and:DI (ashift:DI (reg:DI rs1) (const_int N))
+			(const_int N))
+		(reg:DI rs2)) */
+
+static bool
+riscv_set_is_shNadduw (rtx set)
+{
+  return (GET_CODE (SET_SRC (set)) == PLUS
+	  && GET_CODE (XEXP (SET_SRC (set), 0)) == AND
+	  && GET_CODE (XEXP (XEXP (SET_SRC (set), 0), 0)) == ASHIFT
+	  && REG_P (XEXP (XEXP (XEXP (SET_SRC (set), 0), 0), 0))
+	  && CONST_INT_P (XEXP (XEXP (XEXP (SET_SRC (set), 0), 0), 1))
+	  && (INTVAL (XEXP (XEXP (XEXP (SET_SRC (set), 0), 0), 1)) == 1
+	      || INTVAL (XEXP (XEXP (XEXP (SET_SRC (set), 0), 0), 1)) == 2
+	      || INTVAL (XEXP (XEXP (XEXP (SET_SRC (set), 0), 0), 1)) == 3)
+	  && REG_P (SET_DEST (set)));
+}
+
 /* Implement TARGET_SCHED_MACRO_FUSION_PAIR_P.  Return true if PREV and CURR
    should be kept together during scheduling.  */
 
@@ -10225,10 +10394,10 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
   bool simple_sets_p = prev_set && curr_set && !any_condjump_p (curr);
   bool sched1 = can_create_pseudo_p ();
 
-  unsigned int prev_dest_regno = (REG_P (SET_DEST (prev_set))
+  unsigned int prev_dest_regno = (prev_set && REG_P (SET_DEST (prev_set))
 				  ? REGNO (SET_DEST (prev_set))
 				  : FIRST_PSEUDO_REGISTER);
-  unsigned int curr_dest_regno = (REG_P (SET_DEST (curr_set))
+  unsigned int curr_dest_regno = (curr_set && REG_P (SET_DEST (curr_set))
 				  ? REGNO (SET_DEST (curr_set))
 				  : FIRST_PSEUDO_REGISTER);
 
@@ -10253,11 +10422,15 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && CONST_INT_P (XEXP (SET_SRC (prev_set), 1))
 	  && CONST_INT_P (XEXP (SET_SRC (curr_set), 1))
 	  && INTVAL (XEXP (SET_SRC (prev_set), 1)) == 32
-	  && (( INTVAL (XEXP (SET_SRC (curr_set), 1)) == 32
-		&& riscv_fusion_enabled_p(RISCV_FUSE_ZEXTW) )
-	      || ( INTVAL (XEXP (SET_SRC (curr_set), 1)) < 32
-		   && riscv_fusion_enabled_p(RISCV_FUSE_ZEXTWS))))
-	return true;
+	  && ((INTVAL (XEXP (SET_SRC (curr_set), 1)) == 32
+	       && riscv_fusion_enabled_p (RISCV_FUSE_ZEXTW) )
+	      || (INTVAL (XEXP (SET_SRC (curr_set), 1)) < 32
+		  && riscv_fusion_enabled_p (RISCV_FUSE_ZEXTWS))))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_ZEXTWS\n");
+	  return true;
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_ZEXTH)
@@ -10278,7 +10451,11 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && CONST_INT_P (XEXP (SET_SRC (curr_set), 1))
 	  && INTVAL (XEXP (SET_SRC (prev_set), 1)) == 48
 	  && INTVAL (XEXP (SET_SRC (curr_set), 1)) == 48)
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file,"RISCV_FUSE_ZEXTH\n");
+	  return true;
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_LDINDEXED)
@@ -10297,7 +10474,11 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && GET_CODE (SET_SRC (prev_set)) == PLUS
 	  && REG_P (XEXP (SET_SRC (prev_set), 0))
 	  && REG_P (XEXP (SET_SRC (prev_set), 1)))
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LDINDEXED\n");
+	  return true;
+	}
 
       /* We are trying to match the following:
 	   prev (add) == (set (reg:DI rD)
@@ -10313,7 +10494,144 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && GET_CODE (SET_SRC (prev_set)) == PLUS
 	  && REG_P (XEXP (SET_SRC (prev_set), 0))
 	  && REG_P (XEXP (SET_SRC (prev_set), 1)))
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LDINDEXED\n");
+	  return true;
+	}
+    }
+
+  if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_EXPANDED_LD)
+      && (sched1 || prev_dest_regno == curr_dest_regno))
+    {
+      /* For the "expanded add/load fusion" family we have 2 main
+	 categories: memory loads with displacement (i.e. with imm offset)
+	 and loads without displacement (i.e. with offset = x0).
+
+	 For loads without displacement we'll need:
+	 - add + ld (done in RISCV_FUSE_LDINDEXED)
+	 - addi + ld (done in RISCV_FUSE_LDPREINCREMENT)
+	 - shNadd + ld
+	 - add.uw + lw
+	 - shNadd.uw + lw
+
+	 For loads with displacement/immediates:
+	 with lw with immediate):
+	 - add + ld with displacement
+	 - addi + ld with displacement
+	 - shNadd + ld with displacement
+	 - add.uw + lw with displacement
+	 - shNadd.uw + lw with displacement */
+
+      /* We're trying to match a curr_set ld with displacement:
+	  prev (add|addi) = (set (reg:DI rd) (...))
+	  curr (ld)  == (set (reg:DI rD)
+		(mem:DI (plus:DI (reg:DI rD) (const_int IMM12)))) */
+      if (MEM_P (SET_SRC (curr_set))
+	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
+	  && GET_CODE (XEXP (SET_SRC (curr_set), 0)) == PLUS
+	  && REGNO (XEXP (XEXP (SET_SRC (curr_set), 0), 0)) == prev_dest_regno)
+	{
+	  if (riscv_set_is_add (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+
+	  if (riscv_set_is_addi (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+
+	  if (riscv_set_is_shNadd (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+	}
+
+      /* We're trying to match a ld without displacement:
+	  prev (addi|shNadd) = (reg:DI rD) (...))
+	  curr (ld)  == (set (reg:DI rD)
+			     (mem:DI (reg:DI rD))) */
+      if (MEM_P (SET_SRC (curr_set))
+	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
+	  && REG_P (XEXP (SET_SRC (curr_set), 0))
+	  && REGNO (XEXP (SET_SRC (curr_set), 0)) == prev_dest_regno)
+	{
+	  if (riscv_set_is_addi (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+
+	  if (riscv_set_is_shNadd (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+	}
+
+      /* We're trying to match a curr_set lw with displacement:
+	  prev (add.uw|shNadd.uw) = (set (reg:DI rd) (...))
+	  curr (lw)  == (set (reg:DI rd)
+		(any_extend:DI (mem:SUBX (plus:DI ((reg:DI rd)
+						   (const_int IMM)))) */
+      if ((GET_CODE (SET_SRC (curr_set)) == SIGN_EXTEND
+	   || (GET_CODE (SET_SRC (curr_set)) == ZERO_EXTEND))
+	  && MEM_P (XEXP (SET_SRC (curr_set), 0))
+	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
+	  && GET_CODE (XEXP (XEXP (SET_SRC (curr_set), 0), 0)) == PLUS
+	  && REG_P (XEXP (XEXP (XEXP (SET_SRC (curr_set), 0), 0),0))
+	  && (REGNO (XEXP (XEXP (XEXP (SET_SRC (curr_set), 0), 0),0))
+	      == prev_dest_regno))
+	{
+	  if (riscv_set_is_adduw (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+
+	  if (riscv_set_is_shNadduw (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+	}
+
+      /* We're trying to match a curr_set lw without displacement:
+	  prev (add.uw|shNadd.uw) = (set (reg:DI rd) (...))
+	  curr (ld|lh|lw)  == (set (reg:DI rd)
+		(any_extend:DI (mem:SUBX (reg:DI rsd)))) */
+      if ((GET_CODE (SET_SRC (curr_set)) == SIGN_EXTEND
+	   || (GET_CODE (SET_SRC (curr_set)) == ZERO_EXTEND))
+	  && MEM_P (XEXP (SET_SRC (curr_set), 0))
+	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
+	  && REG_P (XEXP (XEXP (SET_SRC (curr_set), 0), 0))
+	  && REGNO (XEXP (XEXP (SET_SRC (curr_set), 0), 0)) == prev_dest_regno)
+	{
+	  if (riscv_set_is_adduw (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+
+	  if (riscv_set_is_shNadduw (prev_set))
+	    {
+	      if (dump_file)
+		fprintf (dump_file, "RISCV_FUSE_EXPANDED_LD\n");
+	      return true;
+	    }
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_LDPREINCREMENT)
@@ -10332,7 +10650,11 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && GET_CODE (SET_SRC (prev_set)) == PLUS
 	  && REG_P (XEXP (SET_SRC (prev_set), 0))
 	  && CONST_INT_P (XEXP (SET_SRC (prev_set), 1)))
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LDPREINCREMENT\n");
+	  return true;
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_LUI_ADDI)
@@ -10350,7 +10672,11 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && (GET_CODE (SET_SRC (prev_set)) == HIGH
 	      || (CONST_INT_P (SET_SRC (prev_set))
 		  && LUI_OPERAND (INTVAL (SET_SRC (prev_set))))))
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LUI_ADDI\n");
+	  return true;
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_AUIPC_ADDI)
@@ -10372,7 +10698,11 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 		  && CONST_INT_P (XEXP (SET_SRC (curr_set), 1))
 		  && SMALL_OPERAND (INTVAL (XEXP (SET_SRC (curr_set), 1))))))
 
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_AUIPC_ADDI\n");
+	  return true;
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_LUI_LD)
@@ -10392,14 +10722,22 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
 	  && GET_CODE (XEXP (SET_SRC (curr_set), 0)) == PLUS
 	  && REGNO (XEXP (XEXP (SET_SRC (curr_set), 0), 0)) == prev_dest_regno)
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LUI_LD\n");
+	  return true;
+	}
 
       if (GET_CODE (SET_SRC (prev_set)) == HIGH
 	  && MEM_P (SET_SRC (curr_set))
 	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
 	  && GET_CODE (XEXP (SET_SRC (curr_set), 0)) == LO_SUM
 	  && REGNO (XEXP (XEXP (SET_SRC (curr_set), 0), 0)) == prev_dest_regno)
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LUI_LD\n");
+	  return true;
+	}
 
       if (GET_CODE (SET_SRC (prev_set)) == HIGH
 	  && (GET_CODE (SET_SRC (curr_set)) == SIGN_EXTEND
@@ -10409,7 +10747,11 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && (GET_CODE (XEXP (XEXP (SET_SRC (curr_set), 0), 0)) == LO_SUM
 	      && (REGNO (XEXP (XEXP (XEXP (SET_SRC (curr_set), 0), 0), 0))
 		  == prev_dest_regno)))
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_LUI_LD\n");
+	  return true;
+	}
     }
 
   if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_AUIPC_LD)
@@ -10425,10 +10767,14 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 	  && MEM_P (SET_SRC (curr_set))
 	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
 	  && GET_CODE (XEXP (SET_SRC (curr_set), 0)) == PLUS)
-	return true;
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_AUIPC_LD\n");
+	  return true;
+	}
     }
 
-  if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_ALIGNED_STD))
+  if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_CACHE_ALIGNED_STD))
     {
       /* We are trying to match the following:
 	   prev (sd) == (set (mem (plus (reg sp|fp) (const_int)))
@@ -10473,11 +10819,184 @@ riscv_macro_fusion_pair_p (rtx_insn *prev, rtx_insn *curr)
 		 lower offset.  */
 	      if ((INTVAL (offset_prev) % 16) == 0
 		  && (INTVAL (offset_prev) + 8 == INTVAL (offset_curr)))
-		return true;
+		{
+		  if (dump_file)
+		    fprintf (dump_file, "RISCV_FUSE_ALIGNED_STD\n");
+		  return true;
+		}
 	    }
 	}
     }
 
+  /* More general form of the RISCV_FUSE_CACHE_ALIGNED_STD.  The
+     major difference is the dependency on the stores being opposite
+     halves of a cache line is dropped.  Instead the lowest address
+     needs 2X the alignment of the object and the higher address
+     immediately followed the first object.  */
+  if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_ALIGNED_STD))
+    {
+      /* We are trying to match the following:
+	prev (sd) == (set (mem (plus (reg rS1) (const_int)))
+			  (reg rS2))
+	curr (sd) == (set (mem (plus (reg rS1) (const_int)))
+			  (reg rS3)) */
+
+      if (MEM_P (SET_DEST (prev_set))
+	  && SCALAR_INT_MODE_P (GET_MODE (SET_DEST (curr_set)))
+	  && MEM_P (SET_DEST (curr_set))
+	  /* Stores must have the same width */
+	  && GET_MODE (SET_DEST (curr_set)) == GET_MODE (SET_DEST (prev_set)))
+	{
+	  rtx base_prev, base_curr, offset_prev, offset_curr;
+	  unsigned mode_size;
+
+	  extract_base_offset_in_addr (SET_DEST (prev_set),
+				       &base_prev, &offset_prev);
+	  extract_base_offset_in_addr (SET_DEST (curr_set),
+				       &base_curr, &offset_curr);
+
+	  /* Proceed only if we find both bases, both bases
+	     are registers and bases are the same register.  */
+	  if (base_prev != NULL_RTX && base_curr != NULL_RTX
+	      && REG_P (base_prev) && REG_P (base_curr)
+	      && REGNO (base_prev) == REGNO (base_curr))
+	    {
+	      machine_mode mode = GET_MODE (SET_DEST (curr_set));
+	      mode_size = estimated_poly_value (GET_MODE_SIZE (mode));
+
+	      HOST_WIDE_INT offset_prev_int = INTVAL (offset_prev);
+	      HOST_WIDE_INT offset_curr_int = INTVAL (offset_curr);
+
+	      /* Get the smaller offset into OFFSET_PREV_INT.  */
+	      if (offset_prev_int > offset_curr_int)
+		std::swap (offset_prev_int, offset_curr_int);
+
+	      /* We've normalized, so we need to check that the lower
+		 address is aligned to 2X the size of the object.  The
+		 higher address must be the lower address plus the
+		 size of the object.  */
+	      if (((offset_prev_int % (2 * mode_size)) == 0)
+		  && offset_prev_int + mode_size == offset_curr_int)
+		{
+		  if (dump_file)
+		    fprintf (dump_file, "RISCV_FUSE_ALIGNED_STD\n");
+		  return true;
+		}
+	    }
+	}
+    }
+
+  if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_BFEXT)
+      && (sched1 || prev_dest_regno == curr_dest_regno))
+    {
+      /* We are trying to match the following:
+	   prev (slli) == (set (reg:DI rD)
+			       (ashift:DI (reg:DI rS) (const_int)))
+	   curr (srli) == (set (reg:DI rD)
+			       (lshiftrt:DI (reg:DI rD) (const_int))) */
+
+      if (GET_CODE (SET_SRC (prev_set)) == ASHIFT
+	  && (GET_CODE (SET_SRC (curr_set)) == LSHIFTRT
+	      || GET_CODE (SET_SRC (curr_set)) == ASHIFTRT)
+	  && REG_P (SET_DEST (prev_set))
+	  && REG_P (SET_DEST (curr_set))
+	  && REGNO (XEXP (SET_SRC (curr_set), 0)) == prev_dest_regno
+	  && CONST_INT_P (XEXP (SET_SRC (prev_set), 1))
+	  && CONST_INT_P (XEXP (SET_SRC (curr_set), 1)))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_BFEXT\n");
+	  return true;
+	}
+    }
+
+  if (simple_sets_p && riscv_fusion_enabled_p (RISCV_FUSE_B_ALUI)
+      && (sched1 || prev_dest_regno == curr_dest_regno))
+    {
+      /* We are trying to match the following:
+	  prev (orc.b) == (set (reg rD)
+				(unspec (reg rS1)))
+	  curr (not) == (set (reg rD2) (not (reg rD))) */
+
+      if (GET_CODE (SET_SRC (prev_set)) == UNSPEC
+	  && GET_CODE (SET_SRC (curr_set)) == NOT
+	  && XINT (SET_SRC (prev_set), 1) == UNSPEC_ORC_B
+	  && REG_P (SET_DEST (prev_set))
+	  && REG_P (SET_DEST (curr_set))
+	  && REG_P (XEXP (SET_SRC (curr_set), 0))
+	  && REGNO (XEXP (SET_SRC (curr_set), 0)) == prev_dest_regno)
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_B_ALUI\n");
+	  return true;
+	}
+
+      /* We are trying to match the following:
+	  prev (ctz) == (set (reg rD) (ctz (reg rS1)))
+	  curr (andi) == (set (reg rD)
+				(and (reg rD) (const_int 63))) */
+
+      if (GET_CODE (SET_SRC (prev_set)) == CTZ
+	  && GET_CODE (SET_SRC (curr_set)) == AND
+	  && CONST_INT_P (XEXP (SET_SRC (curr_set), 1))
+	  && INTVAL (XEXP (SET_SRC (curr_set), 1)) == 63
+	  && REG_P (SET_DEST (prev_set))
+	  && REG_P (SET_DEST (curr_set))
+	  && REG_P (XEXP (SET_SRC (curr_set), 0))
+	  && REGNO (XEXP (SET_SRC (curr_set), 0)) == prev_dest_regno)
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_B_ALUI\n");
+	  return true;
+	}
+
+      /* We are trying to match the following:
+	  prev (sub) == (set (reg rD)
+				(minus (const_int 0) (reg rS2))
+	  curr (max) == (set (reg rD)
+				(smax (reg rD) (reg rS2))) */
+
+      if (GET_CODE (SET_SRC (prev_set)) == MINUS
+	  && (XEXP (SET_SRC (prev_set), 0)
+	      == CONST0_RTX (GET_MODE (SET_SRC (prev_set))))
+	  && CONST_INT_P (XEXP (SET_SRC (prev_set), 0))
+	  && GET_CODE (SET_SRC (curr_set)) == SMAX
+	  && REG_P (SET_DEST (prev_set))
+	  && REG_P (SET_DEST (curr_set))
+	  && REG_P (XEXP (SET_SRC (curr_set), 0))
+	  && REGNO (XEXP (SET_SRC (curr_set), 0)) == prev_dest_regno
+	  && REG_P (XEXP (SET_SRC (prev_set), 1))
+	  && REG_P (XEXP (SET_SRC (curr_set), 1))
+	  && (REGNO (XEXP (SET_SRC (prev_set), 1))
+	      == REGNO (XEXP (SET_SRC (curr_set), 1))))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_B_ALUI\n");
+	  return true;
+	}
+
+      /* We are trying to match the following:
+	  prev (neg) == (set (reg rD) (neg (reg rS1)))
+	  curr (max) == (set (reg rD)
+				(smax (reg rD) (reg rS1))) */
+
+      if (GET_CODE (SET_SRC (prev_set)) == NEG
+	  && GET_CODE (SET_SRC (curr_set)) == SMAX
+	  && REG_P (SET_DEST (prev_set))
+	  && REG_P (SET_DEST (curr_set))
+	  && REG_P (XEXP (SET_SRC (curr_set), 0))
+	  && REGNO (XEXP (SET_SRC (curr_set), 0)) == prev_dest_regno
+	  && REG_P (XEXP (SET_SRC (prev_set), 0))
+	  && REG_P (XEXP (SET_SRC (curr_set), 1))
+	  && (REGNO (XEXP (SET_SRC (prev_set), 0))
+	      == REGNO (XEXP (SET_SRC (curr_set), 1))))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "RISCV_FUSE_B_ALUI\n");
+	  return true;
+	}
+    }
+
   return false;
 }
 
@@ -11545,10 +12064,10 @@ riscv_get_interrupt_type (tree decl)
     {
       const char *string = TREE_STRING_POINTER (TREE_VALUE (attr_args));
 
-      if (!strcmp (string, "user"))
-	return USER_MODE;
-      else if (!strcmp (string, "supervisor"))
+      if (!strcmp (string, "supervisor"))
 	return SUPERVISOR_MODE;
+      else if (!strcmp (string, "rnmi"))
+	return RNMI_MODE;
       else /* Must be "machine".  */
 	return MACHINE_MODE;
     }
@@ -12165,14 +12684,31 @@ riscv_estimated_poly_value (poly_int64 val,
 /* Return true if the vector misalignment factor is supported by the
    target.  */
 bool
-riscv_support_vector_misalignment (machine_mode mode,
-				   const_tree type ATTRIBUTE_UNUSED,
-				   int misalignment,
-				   bool is_packed ATTRIBUTE_UNUSED)
+riscv_support_vector_misalignment (machine_mode mode, const_tree type,
+				   int misalignment, bool is_packed,
+				   bool is_gather_scatter)
 {
-  /* Depend on movmisalign pattern.  */
+  /* IS_PACKED is true if the corresponding scalar element is not naturally
+     aligned.  If the misalignment is unknown and the the access is packed
+     we defer to the default hook which will check if movmisalign is present.
+     Movmisalign, in turn, depends on TARGET_VECTOR_MISALIGN_SUPPORTED.  */
+  if (misalignment == DR_MISALIGNMENT_UNKNOWN)
+    {
+      if (!is_packed)
+	return true;
+    }
+  else
+    {
+      /* If we know that misalignment is a multiple of the element size, we're
+	 good.  */
+      if (misalignment % TYPE_ALIGN_UNIT (type) == 0)
+	return true;
+    }
+
+  /* Otherwise fall back to movmisalign again.  */
   return default_builtin_support_vector_misalignment (mode, type, misalignment,
-						      is_packed);
+						      is_packed,
+						      is_gather_scatter);
 }
 
 /* Implement TARGET_VECTORIZE_GET_MASK_MODE.  */
@@ -12334,6 +12870,14 @@ riscv_lshift_subword (machine_mode mode ATTRIBUTE_UNUSED, rtx value, rtx shift,
 						  gen_lowpart (QImode, shift)));
 }
 
+/* Return TRUE if we should use the zero stride load, FALSE otherwise. */
+
+bool
+strided_load_broadcast_p ()
+{
+  return tune_param->use_zero_stride_load;
+}
+
 /* Return TRUE if we should use the divmod expander, FALSE otherwise.  This
    allows the behavior to be tuned for specific implementations as well as
    when optimizing for size.  */
@@ -13298,6 +13842,88 @@ riscv_expand_sssub (rtx dest, rtx x, rtx y)
   emit_move_insn (dest, gen_lowpart (mode, xmode_dest));
 }
 
+/* Implement the Xmode usmul.
+
+   b = SAT_MUL (a, b);
+   =>
+   _1 = a * b;
+   _2 = mulhu (a, b);
+   _overflow_p = _2 == 0;
+   _mask = - _overflow_p;
+   b = _1 | _mask;
+ */
+
+static void
+riscv_expand_xmode_usmul (rtx dest, rtx x, rtx y)
+{
+  machine_mode mode = GET_MODE (dest);
+
+  gcc_assert (mode == Xmode);
+
+  rtx mul = gen_reg_rtx (Xmode);
+  rtx mulhu = gen_reg_rtx (Xmode);
+  rtx overflow_p = gen_reg_rtx (Xmode);
+
+  riscv_emit_binary (MULT, mul, x, y);
+
+  if (TARGET_64BIT)
+    emit_insn (gen_usmuldi3_highpart (mulhu, x, y));
+  else
+    emit_insn (gen_usmulsi3_highpart (mulhu, x, y));
+
+  riscv_emit_binary (NE, overflow_p, mulhu, CONST0_RTX (Xmode));
+  riscv_emit_unary (NEG, overflow_p, overflow_p);
+  riscv_emit_binary (IOR, dest, mul, overflow_p);
+}
+
+/* Implement the non-Xmode usmul.
+
+   b = SAT_MUL (a, b);
+   =>
+   _1 = a * b;
+   _max = (T)-1
+   _overflow_p = _1 > _max;
+   _mask = - _overflow_p;
+   b = _1 | _mask;
+ */
+
+static void
+riscv_expand_non_xmode_usmul (rtx dest, rtx x, rtx y)
+{
+  machine_mode mode = GET_MODE (dest);
+  unsigned bitsize = GET_MODE_BITSIZE (mode).to_constant ();
+
+  gcc_assert (mode != Xmode);
+
+  rtx xmode_x = riscv_extend_to_xmode_reg (x, mode, ZERO_EXTEND);
+  rtx xmode_y = riscv_extend_to_xmode_reg (y, mode, ZERO_EXTEND);
+  rtx xmode_mul = gen_reg_rtx (Xmode);
+  rtx mul_max = gen_reg_rtx (Xmode);
+  rtx overflow_p = gen_reg_rtx (Xmode);
+
+  uint64_t max = ((uint64_t)1 << bitsize) - 1;
+
+  emit_move_insn (mul_max, GEN_INT (max));
+  riscv_emit_binary (MULT, xmode_mul, xmode_x, xmode_y);
+
+  riscv_emit_binary (LTU, overflow_p, mul_max, xmode_mul);
+  riscv_emit_unary (NEG, overflow_p, overflow_p);
+  riscv_emit_binary (IOR, xmode_mul, xmode_mul, overflow_p);
+
+  emit_move_insn (dest, gen_lowpart (mode, xmode_mul));
+}
+
+/* Implements the unsigned saturation mult standard name usmul for int mode.  */
+
+void
+riscv_expand_usmul (rtx dest, rtx x, rtx y)
+{
+  if (GET_MODE (dest) == Xmode)
+    return riscv_expand_xmode_usmul (dest, x, y) ;
+  else
+    return riscv_expand_non_xmode_usmul (dest, x, y);
+}
+
 /* Implement the unsigned saturation truncation for int mode.
 
    b = SAT_TRUNC (a);
diff --git a/gcc/config/riscv/riscv.md b/gcc/config/riscv/riscv.md
index 893c925..578dd43 100644
--- a/gcc/config/riscv/riscv.md
+++ b/gcc/config/riscv/riscv.md
@@ -120,7 +120,7 @@
   ;; Interrupt handler instructions.
   UNSPECV_MRET
   UNSPECV_SRET
-  UNSPECV_URET
+  UNSPECV_MNRET
 
   ;; Blockage and synchronization.
   UNSPECV_BLOCKAGE
@@ -3298,7 +3298,7 @@
 			  (match_operand:GPR 2 "movcc_operand")
 			  (match_operand:GPR 3 "movcc_operand")))]
   "TARGET_SFB_ALU || TARGET_XTHEADCONDMOV || TARGET_ZICOND_LIKE
-   || TARGET_MOVCC"
+   || TARGET_MOVCC || TARGET_XMIPSCMOV"
 {
   if (riscv_expand_conditional_move (operands[0], operands[1],
 				     operands[2], operands[3]))
@@ -4166,11 +4166,11 @@
   "sret"
   [(set_attr "type" "ret")])
 
-(define_insn "riscv_uret"
+(define_insn "riscv_mnret"
   [(return)
-   (unspec_volatile [(const_int 0)] UNSPECV_URET)]
-  ""
-  "uret"
+   (unspec_volatile [(const_int 0)] UNSPECV_MNRET)]
+  "TARGET_SMRNMI"
+  "mnret"
   [(set_attr "type" "ret")])
 
 (define_insn "stack_tie<mode>"
@@ -4402,7 +4402,7 @@
 )
 
 (define_insn "prefetch"
-  [(prefetch (match_operand 0 "prefetch_operand" "Q")
+  [(prefetch (match_operand 0 "prefetch_operand" "Qr")
              (match_operand 1 "imm5_operand" "i")
              (match_operand 2 "const_int_operand" "n"))]
   "TARGET_ZICBOP"
@@ -4634,6 +4634,17 @@
   }
 )
 
+(define_expand "usmul<mode>3"
+  [(match_operand:ANYI 0 "register_operand")
+   (match_operand:ANYI 1 "register_operand")
+   (match_operand:ANYI 2 "register_operand")]
+  ""
+  {
+    riscv_expand_usmul (operands[0], operands[1], operands[2]);
+    DONE;
+  }
+)
+
 (define_expand "ustrunc<mode><anyi_double_truncated>2"
   [(match_operand:<ANYI_DOUBLE_TRUNCATED> 0 "register_operand")
    (match_operand:ANYI_DOUBLE_TRUNC       1 "register_operand")]
@@ -4872,6 +4883,7 @@
 (include "vector-crypto.md")
 (include "vector-bfloat16.md")
 (include "zicond.md")
+(include "mips-insn.md")
 (include "sfb.md")
 (include "zc.md")
 ;; Vendor extensions
diff --git a/gcc/config/riscv/sync.md b/gcc/config/riscv/sync.md
index a75ea68..50ec8b3 100644
--- a/gcc/config/riscv/sync.md
+++ b/gcc/config/riscv/sync.md
@@ -627,7 +627,7 @@
 	(match_operand:SHORT 1 "memory_operand" "+A"))				;; memory
    (set (match_dup 1)
 	(unspec_volatile:SHORT [(match_operand:SHORT 2 "register_operand" "0")  ;; expected_val
-				(match_operand:SHORT 3 "register_operand" "rJ") ;; desired_val
+				(match_operand:SHORT 3 "reg_or_0_operand" "rJ") ;; desired_val
 				(match_operand:SI 4 "const_int_operand")	;; mod_s
 				(match_operand:SI 5 "const_int_operand")]	;; mod_f
 	 UNSPEC_COMPARE_AND_SWAP))]
diff --git a/gcc/config/riscv/t-riscv b/gcc/config/riscv/t-riscv
index 32092d8..a7eaa8b 100644
--- a/gcc/config/riscv/t-riscv
+++ b/gcc/config/riscv/t-riscv
@@ -194,7 +194,8 @@ RISCV_EXT_DEFS = \
   $(srcdir)/config/riscv/riscv-ext.def \
   $(srcdir)/config/riscv/riscv-ext-sifive.def \
   $(srcdir)/config/riscv/riscv-ext-thead.def \
-  $(srcdir)/config/riscv/riscv-ext-ventana.def
+  $(srcdir)/config/riscv/riscv-ext-ventana.def \
+  $(srcdir)/config/riscv/riscv-ext-mips.def
 
 $(srcdir)/config/riscv/riscv-ext.opt: $(RISCV_EXT_DEFS)
 
@@ -228,8 +229,41 @@ s-riscv-ext.texi: build/gen-riscv-ext-texi$(build_exeext)
 	$(SHELL) $(srcdir)/../move-if-change tmp-riscv-ext.texi $(srcdir)/doc/riscv-ext.texi
 	$(STAMP) s-riscv-ext.texi
 
-# Run `riscv-regen' after you changed or added anything from riscv-ext*.def
+RISCV_CORES_DEFS = \
+  $(srcdir)/config/riscv/riscv-cores.def
+
+build/gen-riscv-mtune-texi.o: $(srcdir)/config/riscv/gen-riscv-mtune-texi.cc \
+	$(RISCV_CORES_DEFS)
+	$(CXX_FOR_BUILD) $(CXXFLAGS_FOR_BUILD) -c $< -o $@
+
+build/gen-riscv-mcpu-texi.o: $(srcdir)/config/riscv/gen-riscv-mcpu-texi.cc \
+	$(RISCV_CORES_DEFS)
+	$(CXX_FOR_BUILD) $(CXXFLAGS_FOR_BUILD) -c $< -o $@
+
+build/gen-riscv-mtune-texi$(build_exeext): build/gen-riscv-mtune-texi.o
+	$(LINKER_FOR_BUILD) $(BUILD_LINKERFLAGS) $(BUILD_LDFLAGS) -o $@ $<
+
+build/gen-riscv-mcpu-texi$(build_exeext): build/gen-riscv-mcpu-texi.o
+	$(LINKER_FOR_BUILD) $(BUILD_LINKERFLAGS) $(BUILD_LDFLAGS) -o $@ $<
+
+$(srcdir)/doc/riscv-mtune.texi: $(RISCV_CORES_DEFS)
+$(srcdir)/doc/riscv-mtune.texi: s-riscv-mtune.texi ; @true
+
+$(srcdir)/doc/riscv-mcpu.texi: $(RISCV_CORES_DEFS)
+$(srcdir)/doc/riscv-mcpu.texi: s-riscv-mcpu.texi ; @true
+
+s-riscv-mtune.texi: build/gen-riscv-mtune-texi$(build_exeext)
+	$(RUN_GEN) build/gen-riscv-mtune-texi$(build_exeext) > tmp-riscv-mtune.texi
+	$(SHELL) $(srcdir)/../move-if-change tmp-riscv-mtune.texi $(srcdir)/doc/riscv-mtune.texi
+	$(STAMP) s-riscv-mtune.texi
+
+s-riscv-mcpu.texi: build/gen-riscv-mcpu-texi$(build_exeext)
+	$(RUN_GEN) build/gen-riscv-mcpu-texi$(build_exeext) > tmp-riscv-mcpu.texi
+	$(SHELL) $(srcdir)/../move-if-change tmp-riscv-mcpu.texi $(srcdir)/doc/riscv-mcpu.texi
+	$(STAMP) s-riscv-mcpu.texi
+
+# Run `riscv-regen' after you changed or added anything from riscv-ext*.def and riscv-cores*.def
 
 .PHONY: riscv-regen
 
-riscv-regen: s-riscv-ext.texi s-riscv-ext.opt
+riscv-regen: s-riscv-ext.texi s-riscv-ext.opt s-riscv-mtune.texi s-riscv-mcpu.texi
diff --git a/gcc/config/riscv/vector-iterators.md b/gcc/config/riscv/vector-iterators.md
index 7825444..aa3b6fb 100644
--- a/gcc/config/riscv/vector-iterators.md
+++ b/gcc/config/riscv/vector-iterators.md
@@ -2418,6 +2418,47 @@
   (RVVM1x2DF "rvvm1df")
 ])
 
+(define_mode_attr vsubel [
+  (RVVM8HI "qi") (RVVM4HI "qi") (RVVM2HI "qi") (RVVM1HI "qi") (RVVMF2HI "qi") (RVVMF4HI "qi")
+
+  (RVVM8SI "hi") (RVVM4SI "hi") (RVVM2SI "hi") (RVVM1SI "hi") (RVVMF2SI "hi")
+
+  (RVVM8SF "hf") (RVVM4SF "hf") (RVVM2SF "hf") (RVVM1SF "hf") (RVVMF2SF "hf")
+
+  (RVVM8DI "si") (RVVM4DI "si") (RVVM2DI "si") (RVVM1DI "si")
+
+  (RVVM8DF "sf") (RVVM4DF "sf") (RVVM2DF "sf") (RVVM1DF "sf")
+
+  ;; VLS modes.
+  (V1HI "qi") (V2HI "qi") (V4HI "qi") (V8HI "qi") (V16HI "qi") (V32HI "qi") (V64HI "qi") (V128HI "qi") (V256HI "qi")
+  (V512HI "qi") (V1024HI "qi") (V2048HI "qi")
+  (V1SI "hi") (V2SI "hi") (V4SI "hi") (V8SI "hi") (V16SI "hi") (V32SI "hi") (V64SI "hi") (V128SI "hi") (V256SI "hi")
+  (V512SI "hi") (V1024SI "hi")
+  (V1DI "si") (V2DI "si") (V4DI "si") (V8DI "si") (V16DI "si") (V32DI "si") (V64DI "si") (V128DI "si") (V256DI "si") (V512DI "si")
+
+  (V1SF "hf")
+  (V2SF "hf")
+  (V4SF "hf")
+  (V8SF "hf")
+  (V16SF "hf")
+  (V32SF "hf")
+  (V64SF "hf")
+  (V128SF "hf")
+  (V256SF "hf")
+  (V512SF "hf")
+  (V1024SF "hf")
+  (V1DF "sf")
+  (V2DF "sf")
+  (V4DF "sf")
+  (V8DF "sf")
+  (V16DF "sf")
+  (V32DF "sf")
+  (V64DF "sf")
+  (V128DF "sf")
+  (V256DF "sf")
+  (V512DF "sf")
+])
+
 (define_mode_attr VSUBEL [
   (RVVM8HI "QI") (RVVM4HI "QI") (RVVM2HI "QI") (RVVM1HI "QI") (RVVMF2HI "QI") (RVVMF4HI "QI")
 
@@ -3972,6 +4013,14 @@
 			      UNSPEC_VASUBU UNSPEC_VASUB UNSPEC_VSMUL
 			      UNSPEC_VSSRL UNSPEC_VSSRA])
 
+(define_int_iterator VSAT_VX_OP_V_VDUP [
+  UNSPEC_VAADDU UNSPEC_VAADD
+])
+
+(define_int_iterator VSAT_VX_OP_VDUP_V [
+  UNSPEC_VAADDU UNSPEC_VAADD
+])
+
 (define_int_iterator VSAT_ARITH_OP [UNSPEC_VAADDU UNSPEC_VAADD
 			      	    UNSPEC_VASUBU UNSPEC_VASUB UNSPEC_VSMUL])
 (define_int_iterator VSAT_SHIFT_OP [UNSPEC_VSSRL UNSPEC_VSSRA])
@@ -4006,6 +4055,14 @@
 			 	(UNSPEC_VSSRA "vsshift") (UNSPEC_VNCLIP "vnclip")
 				(UNSPEC_VNCLIPU "vnclip")])
 
+(define_int_attr sat_op_v_vdup [
+  (UNSPEC_VAADDU "aaddu") (UNSPEC_VAADD "aadd")
+])
+
+(define_int_attr sat_op_vdup_v [
+  (UNSPEC_VAADDU "aaddu") (UNSPEC_VAADD "aadd")
+])
+
 (define_int_attr misc_op [(UNSPEC_VMSBF "sbf") (UNSPEC_VMSIF "sif") (UNSPEC_VMSOF "sof")
 			  (UNSPEC_VFRSQRT7 "rsqrt7")])
 
@@ -4042,11 +4099,12 @@
 ])
 
 (define_code_iterator any_int_binop_no_shift_v_vdup [
-  plus minus and ior xor mult div udiv mod umod smax umax smin umin us_plus us_minus
+  plus minus and ior xor mult div udiv mod umod smax umax smin umin us_plus
+  us_minus ss_plus ss_minus
 ])
 
 (define_code_iterator any_int_binop_no_shift_vdup_v [
-  plus minus and ior xor mult smax umax smin umin us_plus
+  plus minus and ior xor mult smax umax smin umin us_plus ss_plus
 ])
 
 (define_code_iterator any_int_unop [neg not])
diff --git a/gcc/config/riscv/vector.md b/gcc/config/riscv/vector.md
index 6753b01..66b7670 100644
--- a/gcc/config/riscv/vector.md
+++ b/gcc/config/riscv/vector.md
@@ -1551,20 +1551,44 @@
 (define_expand "vec_duplicate<mode>"
   [(set (match_operand:V_VLS 0 "register_operand")
         (vec_duplicate:V_VLS
-          (match_operand:<VEL> 1 "direct_broadcast_operand")))]
+          (match_operand:<VEL> 1 "any_broadcast_operand")))]
   "TARGET_VECTOR"
   {
-    /* Early expand DImode broadcast in RV32 system to avoid RA reload
-       generate (set (reg) (vec_duplicate:DI)).  */
+    /* Don't keep a DImode broadcast for RV32 in the vec_duplicate form.
+       Otherwise combine or late combine could end up doing
+	      "64-bit broadcast" (!= vmv.v.x)
+            + vadd.vv
+	    = vadd.vx
+       which would be invalid.  */
     bool gt_p = maybe_gt (GET_MODE_SIZE (<VEL>mode), GET_MODE_SIZE (Pmode));
     if (!FLOAT_MODE_P (<VEL>mode) && gt_p)
       {
-        riscv_vector::emit_vlmax_insn (code_for_pred_broadcast (<MODE>mode),
-				       riscv_vector::UNARY_OP, operands);
-	DONE;
+        riscv_vector::emit_vlmax_insn
+	  (code_for_pred_strided_broadcast
+	    (<MODE>mode), riscv_vector::UNARY_OP, operands);
+       DONE;
       }
-    /* Otherwise, allow it fall into general vec_duplicate pattern
-       which allow us to have vv->vx combine optimization in later pass.  */
+
+    /* Even though we can eventually broadcast any permissible
+       constant by moving it into a register we need to force
+       any non-immediate one into a register here.
+       If we didn't do that we couldn't fwprop/late-combine
+	      vec_duplicate 123.45f
+	    + vfadd.vv
+	    = vfadd.vf
+       because the constant is valid for vec_duplicate but not
+       for vfadd.vf.  Therefore we need to do
+	      fa0 = 123.45f
+	      vec_duplicate fa0
+	    + vfadd.vv
+	    = vfadd.vf  */
+    if (!satisfies_constraint_P (operands[1])
+	&& !satisfies_constraint_J (operands[1])
+	&& !rtx_equal_p (operands[1], CONST0_RTX (<VEL>mode))
+	&& !memory_operand (operands[1], <VEL>mode))
+      operands[1] = force_reg (<VEL>mode, operands[1]);
+
+    /* Otherwise keep the vec_duplicate pattern until split.  */
   })
 
 ;; According to GCC internal:
@@ -1574,14 +1598,20 @@
 (define_insn_and_split "*vec_duplicate<mode>"
   [(set (match_operand:V_VLS 0 "register_operand")
         (vec_duplicate:V_VLS
-          (match_operand:<VEL> 1 "direct_broadcast_operand")))]
+          (match_operand:<VEL> 1 "any_broadcast_operand")))]
   "TARGET_VECTOR && can_create_pseudo_p ()"
   "#"
   "&& 1"
   [(const_int 0)]
   {
-    riscv_vector::emit_vlmax_insn (code_for_pred_broadcast (<MODE>mode),
-                                   riscv_vector::UNARY_OP, operands);
+    if (riscv_vector::can_be_broadcast_p (operands[1]))
+      riscv_vector::emit_vlmax_insn (code_for_pred_broadcast (<MODE>mode),
+				     riscv_vector::UNARY_OP, operands);
+    else
+      riscv_vector::emit_vlmax_insn (code_for_pred_strided_broadcast
+				     (<MODE>mode), riscv_vector::UNARY_OP,
+				     operands);
+
     DONE;
   }
   [(set_attr "type" "vector")]
@@ -1783,7 +1813,7 @@
   [(set_attr "type" "vsetvl")
    (set_attr "mode" "SI")])
 
-;; This pattern use to combine bellow two insns and then further remove
+;; This pattern use to combine below two insns and then further remove
 ;; unnecessary sign_extend operations:
 ;;   (set (reg:DI 134 [ _1 ])
 ;;        (unspec:DI [
@@ -2127,69 +2157,45 @@
 	  (match_operand:V_VLS 2 "vector_merge_operand")))]
   "TARGET_VECTOR"
 {
-  /* Transform vmv.v.x/vfmv.v.f (avl = 1) into vmv.s.x since vmv.s.x/vfmv.s.f
-     has better chances to do vsetvl fusion in vsetvl pass.  */
   bool wrap_vec_dup = true;
   rtx vec_cst = NULL_RTX;
-  if (riscv_vector::splat_to_scalar_move_p (operands))
-    {
-      operands[1] = riscv_vector::gen_scalar_move_mask (<VM>mode);
-      operands[3] = force_reg (<VEL>mode, operands[3]);
-    }
-  else if (immediate_operand (operands[3], <VEL>mode)
-	   && (vec_cst = gen_const_vec_duplicate (<MODE>mode, operands[3]))
-	   && (/* -> pred_broadcast<mode>_zero */
-	       (vector_least_significant_set_mask_operand (operands[1],
-							   <VM>mode)
-		&& vector_const_0_operand (vec_cst, <MODE>mode))
-	       || (/* pred_broadcast<mode>_imm */
-		   vector_all_trues_mask_operand (operands[1], <VM>mode)
-		   && vector_const_int_or_double_0_operand (vec_cst,
-							    <MODE>mode))))
+  if (immediate_operand (operands[3], <VEL>mode)
+      && (vec_cst = gen_const_vec_duplicate (<MODE>mode, operands[3]))
+      && (/* -> pred_broadcast<mode>_zero */
+	  (vector_least_significant_set_mask_operand (operands[1],
+						      <VM>mode)
+	   && vector_const_0_operand (vec_cst, <MODE>mode))
+	  || (/* pred_broadcast<mode>_imm */
+	      vector_all_trues_mask_operand (operands[1], <VM>mode)
+	      && vector_const_int_or_double_0_operand (vec_cst,
+						       <MODE>mode))))
     {
       operands[3] = vec_cst;
       wrap_vec_dup = false;
     }
-  /* Handle vmv.s.x instruction (Wb1 mask) which has memory scalar.  */
-  else if (satisfies_constraint_Wdm (operands[3]))
-    {
-      if (satisfies_constraint_Wb1 (operands[1]))
-	{
-	  /* Case 1: vmv.s.x (TA, x == memory) ==> vlse.v (TA)  */
-	  if (satisfies_constraint_vu (operands[2]))
-	    operands[1] = CONSTM1_RTX (<VM>mode);
-	  else if (GET_MODE_BITSIZE (<VEL>mode) > GET_MODE_BITSIZE (Pmode))
-	    {
-	      /* Case 2: vmv.s.x (TU, x == memory) ==>
-			   vl = 0 or 1; + vlse.v (TU) in RV32 system  */
-	      operands[4] = riscv_vector::gen_avl_for_scalar_move (operands[4]);
-	      operands[1] = CONSTM1_RTX (<VM>mode);
-	    }
-	  else
-	    /* Case 3: load x (memory) to register.  */
-	    operands[3] = force_reg (<VEL>mode, operands[3]);
-	}
-    }
-  else if (GET_MODE_BITSIZE (<VEL>mode) > GET_MODE_BITSIZE (Pmode)
-           && (immediate_operand (operands[3], Pmode)
+  else if (GET_MODE_SIZE (<VEL>mode) > UNITS_PER_WORD
+	   && satisfies_constraint_Wb1 (operands[1])
+	   && (immediate_operand (operands[3], Xmode)
 	       || (CONST_POLY_INT_P (operands[3])
 	           && known_ge (rtx_to_poly_int64 (operands[3]), 0U)
-		   && known_le (rtx_to_poly_int64 (operands[3]), GET_MODE_SIZE (<MODE>mode)))))
+		   && known_le (rtx_to_poly_int64 (operands[3]),
+				GET_MODE_SIZE (<MODE>mode)))))
     {
       rtx tmp = gen_reg_rtx (Pmode);
       poly_int64 value = rtx_to_poly_int64 (operands[3]);
-      emit_move_insn (tmp, gen_int_mode (value, Pmode));
+      emit_move_insn (tmp, gen_int_mode (value, Xmode));
       operands[3] = gen_rtx_SIGN_EXTEND (<VEL>mode, tmp);
     }
-  /* Never load (const_int 0) into a register, that's silly.  */
-  else if (operands[3] == CONST0_RTX (<VEL>mode))
+
+  /* For a vmv.v.x never load (const_int 0) or valid immediate operands
+     into a register, because we can use vmv.v.i.  */
+  else if (satisfies_constraint_Wc1 (operands[1])
+      && (satisfies_constraint_P (operands[3])
+	  || operands[3] == CONST0_RTX (<VEL>mode)))
     ;
-  /* If we're broadcasting [-16..15] across more than just
-     element 0, then we can use vmv.v.i directly, thus avoiding
-     the load of the constant into a GPR.  */
-  else if (CONST_INT_P (operands[3])
-	   && IN_RANGE (INTVAL (operands[3]), -16, 15)
-	   && !satisfies_constraint_Wb1 (operands[1]))
+  /* For vmv.s.x we have vmv.s.x v1, zero.  */
+  else if (satisfies_constraint_Wb1 (operands[1])
+	   && operands[3] == CONST0_RTX (<VEL>mode))
     ;
   else
     operands[3] = force_reg (<VEL>mode, operands[3]);
@@ -2197,113 +2203,68 @@
     operands[3] = gen_rtx_VEC_DUPLICATE (<MODE>mode, operands[3]);
 })
 
-(define_insn_and_split "*pred_broadcast<mode>"
-  [(set (match_operand:V_VLSI 0 "register_operand"                 "=vr, vr, vd, vd, vr, vr, vr, vr")
+(define_insn_and_rewrite "*pred_broadcast<mode>"
+  [(set (match_operand:V_VLSI 0 "register_operand"                 "=vr, vr, vr, vr")
 	(if_then_else:V_VLSI
 	  (unspec:<VM>
-	    [(match_operand:<VM> 1 "vector_broadcast_mask_operand" "Wc1,Wc1, vm, vm,Wc1,Wc1,Wb1,Wb1")
-	     (match_operand 4 "vector_length_operand"              "rvl,rvl,rvl,rvl,rvl,rvl,rvl,rvl")
-	     (match_operand 5 "const_int_operand"                  "  i,  i,  i,  i,  i,  i,  i,  i")
-	     (match_operand 6 "const_int_operand"                  "  i,  i,  i,  i,  i,  i,  i,  i")
-	     (match_operand 7 "const_int_operand"                  "  i,  i,  i,  i,  i,  i,  i,  i")
+	    [(match_operand:<VM> 1 "vector_broadcast_mask_operand" "Wc1,Wc1,Wb1,Wb1")
+	     (match_operand 4 "vector_length_operand"		   "rvl,rvl,rvl,rvl")
+	     (match_operand 5 "const_int_operand"                  "  i,  i,  i,  i")
+	     (match_operand 6 "const_int_operand"                  "  i,  i,  i,  i")
+	     (match_operand 7 "const_int_operand"                  "  i,  i,  i,  i")
 	     (reg:SI VL_REGNUM)
 	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
 	  (vec_duplicate:V_VLSI
-	    (match_operand:<VEL> 3 "direct_broadcast_operand"       "rP,rP,Wdm,Wdm,Wdm,Wdm, rJ, rJ"))
-	  (match_operand:V_VLSI 2 "vector_merge_operand"            "vu, 0, vu,  0, vu,  0, vu,  0")))]
+	    (match_operand:<VEL> 3 "direct_broadcast_operand"      " rP, rP, rJ, rJ"))
+	  (match_operand:V_VLSI 2 "vector_merge_operand"           " vu,  0, vu,  0")))]
   "TARGET_VECTOR"
   "@
    vmv.v.%o3\t%0,%3
    vmv.v.%o3\t%0,%3
-   vlse<sew>.v\t%0,%3,zero,%1.t
-   vlse<sew>.v\t%0,%3,zero,%1.t
-   vlse<sew>.v\t%0,%3,zero
-   vlse<sew>.v\t%0,%3,zero
    vmv.s.x\t%0,%z3
    vmv.s.x\t%0,%z3"
-  "(register_operand (operands[3], <VEL>mode)
-  || CONST_POLY_INT_P (operands[3]))
-  && GET_MODE_BITSIZE (<VEL>mode) > GET_MODE_BITSIZE (Pmode)"
-  [(set (match_dup 0)
-	(if_then_else:V_VLSI (unspec:<VM> [(match_dup 1) (match_dup 4)
-	     (match_dup 5) (match_dup 6) (match_dup 7)
-	     (reg:SI VL_REGNUM) (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
-	  (vec_duplicate:V_VLSI (match_dup 3))
-	  (match_dup 2)))]
-  {
-    gcc_assert (can_create_pseudo_p ());
-    if (CONST_POLY_INT_P (operands[3]))
-      {
-	rtx tmp = gen_reg_rtx (<VEL>mode);
-	emit_move_insn (tmp, operands[3]);
-	operands[3] = tmp;
-      }
-    rtx m = assign_stack_local (<VEL>mode, GET_MODE_SIZE (<VEL>mode),
-				GET_MODE_ALIGNMENT (<VEL>mode));
-    m = validize_mem (m);
-    emit_move_insn (m, operands[3]);
-    m = gen_rtx_MEM (<VEL>mode, force_reg (Pmode, XEXP (m, 0)));
-    operands[3] = m;
-
-    /* For SEW = 64 in RV32 system, we expand vmv.s.x:
-       andi a2,a2,1
-       vsetvl zero,a2,e64
-       vlse64.v  */
-    if (satisfies_constraint_Wb1 (operands[1]))
-      {
-	operands[4] = riscv_vector::gen_avl_for_scalar_move (operands[4]);
-	operands[1] = CONSTM1_RTX (<VM>mode);
-      }
-  }
-  [(set_attr "type" "vimov,vimov,vlds,vlds,vlds,vlds,vimovxv,vimovxv")
+  "&& (operands[1] == CONSTM1_RTX (<VM>mode)
+       && operands[4] == CONST1_RTX (Pmode)
+       && (register_operand (operands[3], <VEL>mode)
+           || satisfies_constraint_J (operands[3])))"
+{
+  /* A broadcast of a single element is just a vmv.s.x.  */
+  operands[1] = riscv_vector::gen_scalar_move_mask (<VM>mode);
+}
+  [(set_attr "type" "vimov,vimov,vimovxv,vimovxv")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "*pred_broadcast<mode>_zvfh"
-  [(set (match_operand:V_VLSF    0 "register_operand"              "=vr,  vr,  vr,  vr")
+(define_insn_and_rewrite "pred_broadcast<mode>_zvfh"
+  [(set (match_operand:V_VLSF    0 "register_operand"              "=vr, vr, vr, vr")
 	(if_then_else:V_VLSF
 	  (unspec:<VM>
-	    [(match_operand:<VM> 1 "vector_broadcast_mask_operand" "Wc1, Wc1, Wb1, Wb1")
-	     (match_operand      4 "vector_length_operand"         "rvl, rvl, rvl, rvl")
-	     (match_operand      5 "const_int_operand"             "  i,   i,   i,   i")
-	     (match_operand      6 "const_int_operand"             "  i,   i,   i,   i")
-	     (match_operand      7 "const_int_operand"             "  i,   i,   i,   i")
+	    [(match_operand:<VM> 1 "vector_broadcast_mask_operand" "Wc1,Wc1,Wb1,Wb1")
+	     (match_operand      4 "vector_length_operand"	   "rvl,rvl,rvl,rvl")
+	     (match_operand      5 "const_int_operand"             "  i,  i,  i,  i")
+	     (match_operand      6 "const_int_operand"             "  i,  i,  i,  i")
+	     (match_operand      7 "const_int_operand"             "  i,  i,  i,  i")
 	     (reg:SI VL_REGNUM)
 	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
 	  (vec_duplicate:V_VLSF
-	    (match_operand:<VEL> 3 "direct_broadcast_operand"      "  f,   f,   f,   f"))
-	  (match_operand:V_VLSF  2 "vector_merge_operand"          " vu,   0,  vu,   0")))]
+	    (match_operand:<VEL> 3 "direct_broadcast_operand"      "  f,  f,  f,  f"))
+	  (match_operand:V_VLSF  2 "vector_merge_operand"          " vu,  0, vu,  0")))]
   "TARGET_VECTOR"
   "@
    vfmv.v.f\t%0,%3
    vfmv.v.f\t%0,%3
    vfmv.s.f\t%0,%3
    vfmv.s.f\t%0,%3"
+  "&& (operands[1] == CONSTM1_RTX (<VM>mode)
+       && operands[4] == CONST1_RTX (Pmode)
+       && (register_operand (operands[3], <VEL>mode)
+           || satisfies_constraint_J (operands[3])))"
+{
+  /* A broadcast of a single element is just a vfmv.s.f.  */
+  operands[1] = riscv_vector::gen_scalar_move_mask (<VM>mode);
+}
   [(set_attr "type" "vfmov,vfmov,vfmovfv,vfmovfv")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "*pred_broadcast<mode>_zvfhmin"
-  [(set (match_operand:V_VLSF_ZVFHMIN   0 "register_operand"              "=vr,  vr,  vr,  vr")
-	(if_then_else:V_VLSF_ZVFHMIN
-	  (unspec:<VM>
-	    [(match_operand:<VM>        1 "vector_broadcast_mask_operand" " vm,  vm, Wc1, Wc1")
-	     (match_operand             4 "vector_length_operand"         "rvl, rvl, rvl, rvl")
-	     (match_operand             5 "const_int_operand"             "  i,   i,   i,   i")
-	     (match_operand             6 "const_int_operand"             "  i,   i,   i,   i")
-	     (match_operand             7 "const_int_operand"             "  i,   i,   i,   i")
-	     (reg:SI VL_REGNUM)
-	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
-	  (vec_duplicate:V_VLSF_ZVFHMIN
-	    (match_operand:<VEL>        3 "direct_broadcast_operand"      "Wdm, Wdm, Wdm, Wdm"))
-	  (match_operand:V_VLSF_ZVFHMIN 2 "vector_merge_operand"          " vu,   0,  vu,   0")))]
-  "TARGET_VECTOR"
-  "@
-   vlse<sew>.v\t%0,%3,zero,%1.t
-   vlse<sew>.v\t%0,%3,zero,%1.t
-   vlse<sew>.v\t%0,%3,zero
-   vlse<sew>.v\t%0,%3,zero"
-  [(set_attr "type" "vlds,vlds,vlds,vlds")
-   (set_attr "mode" "<MODE>")])
-
 (define_insn "*pred_broadcast<mode>_extended_scalar"
   [(set (match_operand:V_VLSI_D 0 "register_operand"               "=vr, vr, vr, vr")
 	(if_then_else:V_VLSI_D
@@ -2366,6 +2327,117 @@
   [(set_attr "type" "vimov,vimov")
    (set_attr "mode" "<MODE>")])
 
+(define_expand "@pred_strided_broadcast<mode>"
+  [(set (match_operand:V_VLS 0 "register_operand")
+	(if_then_else:V_VLS
+	  (unspec:<VM>
+	    [(match_operand:<VM> 1 "strided_broadcast_mask_operand")
+	     (match_operand 4 "vector_length_operand")
+	     (match_operand 5 "const_int_operand")
+	     (match_operand 6 "const_int_operand")
+	     (match_operand 7 "const_int_operand")
+	     (reg:SI VL_REGNUM)
+	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+	  (vec_duplicate:V_VLS
+	    (match_operand:<VEL> 3 "strided_broadcast_operand"))
+	  (match_operand:V_VLS 2 "vector_merge_operand")))]
+  "TARGET_VECTOR"
+{
+  if (satisfies_constraint_Wb1 (operands[1]))
+    {
+      /* If we're asked to set a single element (like vmv.s.x but we
+	 need to go via memory here) and the tail policy is agnostic
+	 we can overwrite all elements.
+	 Thus, set the mask to broadcast.  */
+      operands[1] = CONSTM1_RTX (<VM>mode);
+      if (!satisfies_constraint_vu (operands[2])
+	  && GET_MODE_SIZE (<VEL>mode) > UNITS_PER_WORD)
+	{
+	  /* Case 2: vmv.s.x (TU, x == memory) ==>
+	     vl = 0 or 1; + vlse.v (TU) in RV32 system  */
+	  /* In this case we must not overwrite the residual elements,
+	     so set the vector length to 0/1.  */
+	  operands[4] = riscv_vector::gen_avl_for_scalar_move (operands[4]);
+	}
+    }
+})
+
+(define_insn_and_split "*pred_strided_broadcast<mode>"
+  [(set (match_operand:V_VLSI 0 "register_operand"                  "=vd, vd, vr, vr")
+	(if_then_else:V_VLSI
+	  (unspec:<VM>
+	    [(match_operand:<VM> 1 "strided_broadcast_mask_operand" " vm, vm,Wc1,Wc1")
+	     (match_operand 4 "vector_length_operand"               "rvl,rvl,rvl,rvl")
+	     (match_operand 5 "const_int_operand"                   "  i,  i,  i,  i")
+	     (match_operand 6 "const_int_operand"                   "  i,  i,  i,  i")
+	     (match_operand 7 "const_int_operand"                   "  i,  i,  i,  i")
+	     (reg:SI VL_REGNUM)
+	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+	  (vec_duplicate:V_VLSI
+	    (match_operand:<VEL> 3 "strided_broadcast_operand"	    "  A,  A,  A,  A"))
+	  (match_operand:V_VLSI 2 "vector_merge_operand"            " vu,  0, vu,  0")))]
+  "TARGET_VECTOR"
+  "@
+   vlse<sew>.v\t%0,%3,zero,%1.t
+   vlse<sew>.v\t%0,%3,zero,%1.t
+   vlse<sew>.v\t%0,%3,zero
+   vlse<sew>.v\t%0,%3,zero"
+  "&& !strided_load_broadcast_p () && can_create_pseudo_p ()"
+  [(const_int 0)]
+  {
+    rtx tmp = gen_reg_rtx (V1DImode);
+    emit_move_insn (tmp, gen_lowpart (V1DImode, operands[3]));
+    tmp = lowpart_subreg (<MODE>mode, tmp, V1DImode);
+
+    emit_insn
+      (gen_pred_gather<mode>_scalar
+       (operands[0], operands[1], operands[2], tmp, CONST0_RTX (Pmode),
+	operands[4], operands[5], operands[6], operands[7]));
+    DONE;
+  }
+  [(set_attr "type" "vlds,vlds,vlds,vlds")
+   (set_attr "mode" "<MODE>")])
+
+(define_insn_and_split "*pred_strided_broadcast<mode>_zvfhmin"
+  [(set (match_operand:V_VLSF_ZVFHMIN   0 "register_operand"               "=vr,  vr,  vr,  vr")
+	(if_then_else:V_VLSF_ZVFHMIN
+	  (unspec:<VM>
+	    [(match_operand:<VM>        1 "strided_broadcast_mask_operand" " vm,  vm, Wc1, Wc1")
+	     (match_operand             4 "vector_length_operand"          "rvl, rvl, rvl, rvl")
+	     (match_operand             5 "const_int_operand"              "  i,   i,   i,   i")
+	     (match_operand             6 "const_int_operand"              "  i,   i,   i,   i")
+	     (match_operand             7 "const_int_operand"              "  i,   i,   i,   i")
+	     (reg:SI VL_REGNUM)
+	     (reg:SI VTYPE_REGNUM)] UNSPEC_VPREDICATE)
+	  (vec_duplicate:V_VLSF_ZVFHMIN
+	    (match_operand:<VEL>        3 "strided_broadcast_operand"	   "  A,   A,   A,   A"))
+	  (match_operand:V_VLSF_ZVFHMIN 2 "vector_merge_operand"           " vu,   0,  vu,   0")))]
+  "TARGET_VECTOR"
+  "@
+   vlse<sew>.v\t%0,%3,zero,%1.t
+   vlse<sew>.v\t%0,%3,zero,%1.t
+   vlse<sew>.v\t%0,%3,zero
+   vlse<sew>.v\t%0,%3,zero"
+  "&& !strided_load_broadcast_p ()
+   && <VEL>mode == HFmode
+   && can_create_pseudo_p ()"
+  [(const_int 0)]
+  {
+    poly_uint64 nunits = GET_MODE_NUNITS (<MODE>mode);
+    machine_mode vmodehi
+      = riscv_vector::get_vector_mode (HImode, nunits).require ();
+    rtx ops[] = {gen_lowpart (vmodehi, operands[0]),
+		 gen_lowpart (HImode, operands[3])};
+    riscv_vector::emit_avltype_insn (code_for_pred_broadcast (vmodehi),
+				     riscv_vector::UNARY_OP, ops,
+				     (riscv_vector::avl_type) INTVAL (operands[7]),
+				     operands[4]);
+    DONE;
+  }
+  [(set_attr "type" "vlds,vlds,vlds,vlds")
+   (set_attr "mode" "<MODE>")])
+
+
 ;; -------------------------------------------------------------------------------
 ;; ---- Predicated Strided loads/stores
 ;; -------------------------------------------------------------------------------
@@ -4607,8 +4679,8 @@
 ;; Handle GET_MODE_INNER (mode) = DImode. We need to split them since
 ;; we need to deal with SEW = 64 in RV32 system.
 (define_expand "@pred_<sat_op><mode>_scalar"
-  [(set (match_operand:VI_D 0 "register_operand")
-	(if_then_else:VI_D
+  [(set (match_operand:V_VLSI_D 0 "register_operand")
+	(if_then_else:V_VLSI_D
 	  (unspec:<VM>
 	    [(match_operand:<VM> 1 "vector_mask_operand")
 	     (match_operand 5 "vector_length_operand")
@@ -4619,10 +4691,10 @@
 	     (reg:SI VL_REGNUM)
 	     (reg:SI VTYPE_REGNUM)
 	     (reg:SI VXRM_REGNUM)] UNSPEC_VPREDICATE)
-	  (unspec:VI_D
-	    [(match_operand:VI_D 3 "register_operand")
+	  (unspec:V_VLSI_D
+	    [(match_operand:V_VLSI_D 3 "register_operand")
 	     (match_operand:<VEL> 4 "reg_or_int_operand")] VSAT_ARITH_OP)
-	  (match_operand:VI_D 2 "vector_merge_operand")))]
+	  (match_operand:V_VLSI_D 2 "vector_merge_operand")))]
   "TARGET_VECTOR"
 {
   if (riscv_vector::sew64_scalar_helper (
@@ -4641,8 +4713,8 @@
 })
 
 (define_insn "*pred_<sat_op><mode>_scalar"
-  [(set (match_operand:VI_D 0 "register_operand"         "=vd, vr, vd, vr")
-	(if_then_else:VI_D
+  [(set (match_operand:V_VLSI_D 0 "register_operand"         "=vd, vr, vd, vr")
+	(if_then_else:V_VLSI_D
 	  (unspec:<VM>
 	    [(match_operand:<VM> 1 "vector_mask_operand" " vm, vm,Wc1,Wc1")
 	     (match_operand 5 "vector_length_operand"    "rvl,rvl,rvl,rvl")
@@ -4653,18 +4725,18 @@
 	     (reg:SI VL_REGNUM)
 	     (reg:SI VTYPE_REGNUM)
 	     (reg:SI VXRM_REGNUM)] UNSPEC_VPREDICATE)
-	  (unspec:VI_D
-	    [(match_operand:VI_D 3 "register_operand"    " vr, vr, vr, vr")
+	  (unspec:V_VLSI_D
+	    [(match_operand:V_VLSI_D 3 "register_operand"    " vr, vr, vr, vr")
 	     (match_operand:<VEL> 4 "reg_or_0_operand"   " rJ, rJ, rJ, rJ")] VSAT_ARITH_OP)
-	  (match_operand:VI_D 2 "vector_merge_operand"   " vu,  0, vu,  0")))]
+	  (match_operand:V_VLSI_D 2 "vector_merge_operand"   " vu,  0, vu,  0")))]
   "TARGET_VECTOR"
   "v<sat_op>.vx\t%0,%3,%z4%p1"
   [(set_attr "type" "<sat_insn_type>")
    (set_attr "mode" "<MODE>")])
 
 (define_insn "*pred_<sat_op><mode>_extended_scalar"
-  [(set (match_operand:VI_D 0 "register_operand"            "=vd, vr, vd, vr")
-	(if_then_else:VI_D
+  [(set (match_operand:V_VLSI_D 0 "register_operand"            "=vd, vr, vd, vr")
+	(if_then_else:V_VLSI_D
 	  (unspec:<VM>
 	    [(match_operand:<VM> 1 "vector_mask_operand"    " vm, vm,Wc1,Wc1")
 	     (match_operand 5 "vector_length_operand"       "rvl,rvl,rvl,rvl")
@@ -4675,11 +4747,11 @@
 	     (reg:SI VL_REGNUM)
 	     (reg:SI VTYPE_REGNUM)
 	     (reg:SI VXRM_REGNUM)] UNSPEC_VPREDICATE)
-	  (unspec:VI_D
-	    [(match_operand:VI_D 3 "register_operand"       " vr, vr, vr, vr")
+	  (unspec:V_VLSI_D
+	    [(match_operand:V_VLSI_D 3 "register_operand"       " vr, vr, vr, vr")
 	     (sign_extend:<VEL>
 	       (match_operand:<VSUBEL> 4 "reg_or_0_operand" " rJ, rJ, rJ, rJ"))] VSAT_ARITH_OP)
-	  (match_operand:VI_D 2 "vector_merge_operand"      " vu,  0, vu,  0")))]
+	  (match_operand:V_VLSI_D 2 "vector_merge_operand"      " vu,  0, vu,  0")))]
   "TARGET_VECTOR && !TARGET_64BIT"
   "v<sat_op>.vx\t%0,%3,%z4%p1"
   [(set_attr "type" "<sat_insn_type>")
@@ -7267,10 +7339,10 @@
 	  (plus_minus:VWEXTF
 	    (mult:VWEXTF
 	      (float_extend:VWEXTF
-	        (vec_duplicate:<V_DOUBLE_TRUNC>
-	          (match_operand:<VSUBEL> 3 "register_operand"       "    f")))
-	      (float_extend:VWEXTF
-	        (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr")))
+	        (match_operand:<V_DOUBLE_TRUNC> 4 "register_operand" "   vr"))
+	      (vec_duplicate:VWEXTF
+	        (float_extend:<VEL>
+	          (match_operand:<VSUBEL> 3 "register_operand"       "    f"))))
 	    (match_operand:VWEXTF 2 "register_operand"               "    0"))
 	  (match_dup 2)))]
   "TARGET_VECTOR"
diff --git a/gcc/config/riscv/xiangshan.md b/gcc/config/riscv/xiangshan.md
index 5ed6bac..34b4a8f 100644
--- a/gcc/config/riscv/xiangshan.md
+++ b/gcc/config/riscv/xiangshan.md
@@ -107,7 +107,8 @@
 ;; they are just dummies like this one.
 (define_insn_reservation "xiangshan_alu_unknown" 1
   (and (eq_attr "tune" "xiangshan")
-       (eq_attr "type" "zicond,min,max,minu,maxu,clz,ctz,cpop,ghost,rotate,clmul,condmove,crypto,mvpair,rdvlenb,rdvl,wrvxrm,wrfrm,rdfrm,vsetvl,vsetvl_pre,vlde,vste,vldm,vstm,vlds,vsts,vldux,vldox,vstux,vstox,vldff,vldr,vstr,vlsegde,vssegte,vlsegds,vssegts,vlsegdux,vlsegdox,vssegtux,vssegtox,vlsegdff,vialu,viwalu,vext,vicalu,vshift,vnshift,vicmp,viminmax,vimul,vidiv,viwmul,vimuladd,sf_vqmacc,viwmuladd,vimerge,vimov,vsalu,vaalu,vsmul,vsshift,vnclip,sf_vfnrclip,vfalu,vfwalu,vfmul,vfdiv,vfwmul,vfmuladd,vfwmuladd,vfsqrt,vfrecp,vfcmp,vfminmax,vfsgnj,vfclass,vfmerge,vfmov,vfcvtitof,vfcvtftoi,vfwcvtitof,vfwcvtftoi,vfwcvtftof,vfncvtitof,vfncvtftoi,vfncvtftof,vired,viwred,vfredu,vfredo,vfwredu,vfwredo,vmalu,vmpop,vmffs,vmsfs,vmiota,vmidx,vimovvx,vimovxv,vfmovvf,vfmovfv,vslideup,vslidedown,vislide1up,vislide1down,vfslide1up,vfslide1down,vgather,vcompress,vmov,vector,vandn,vbrev,vbrev8,vrev8,vclz,vctz,vcpop,vrol,vror,vwsll,vclmul,vclmulh,vghsh,vgmul,vaesef,vaesem,vaesdf,vaesdm,vaeskf1,vaeskf2,vaesz,vsha2ms,vsha2ch,vsha2cl,vsm4k,vsm4r,vsm3me,vsm3c,vfncvtbf16,vfwcvtbf16,vfwmaccbf16"))
+       (eq_attr "type" "zicond,min,max,minu,maxu,clz,ctz,cpop,ghost,rotate,clmul,condmove,crypto,mvpair,rdvlenb,rdvl,wrvxrm,wrfrm,rdfrm,vsetvl,vsetvl_pre,vlde,vste,vldm,vstm,vlds,vsts,vldux,vldox,vstux,vstox,vldff,vldr,vstr,vlsegde,vssegte,vlsegds,vssegts,vlsegdux,vlsegdox,vssegtux,vssegtox,vlsegdff,vialu,viwalu,vext,vicalu,vshift,vnshift,vicmp,viminmax,vimul,vidiv,viwmul,vimuladd,sf_vqmacc,viwmuladd,vimerge,vimov,vsalu,vaalu,vsmul,vsshift,vnclip,sf_vfnrclip,vfalu,vfwalu,vfmul,vfdiv,vfwmul,vfmuladd,vfwmuladd,vfsqrt,vfrecp,vfcmp,vfminmax,vfsgnj,vfclass,vfmerge,vfmov,vfcvtitof,vfcvtftoi,vfwcvtitof,vfwcvtftoi,vfwcvtftof,vfncvtitof,vfncvtftoi,vfncvtftof,vired,viwred,vfredu,vfredo,vfwredu,vfwredo,vmalu,vmpop,vmffs,vmsfs,vmiota,vmidx,vimovvx,vimovxv,vfmovvf,vfmovfv,vslideup,vslidedown,vislide1up,vislide1down,vfslide1up,vfslide1down,vgather,vcompress,vmov,vector,vandn,vbrev,vbrev8,vrev8,vclz,vctz,vcpop,vrol,vror,vwsll,vclmul,vclmulh,vghsh,vgmul,vaesef,vaesem,vaesdf,vaesdm,vaeskf1,vaeskf2,vaesz,vsha2ms,vsha2ch,vsha2cl,vsm4k,vsm4r,vsm3me,vsm3c,vfncvtbf16,vfwcvtbf16,vfwmaccbf16,sf_vc,sf_vc_se"))
+
   "xs_alu_rs")
 
 ;; ----------------------------------------------------
diff --git a/gcc/config/rs6000/rs6000.cc b/gcc/config/rs6000/rs6000.cc
index 7ee26e5..764b499 100644
--- a/gcc/config/rs6000/rs6000.cc
+++ b/gcc/config/rs6000/rs6000.cc
@@ -4951,10 +4951,19 @@ static bool
 rs6000_builtin_support_vector_misalignment (machine_mode mode,
 					    const_tree type,
 					    int misalignment,
-					    bool is_packed)
+					    bool is_packed,
+					    bool is_gather_scatter)
 {
   if (TARGET_VSX)
     {
+      if (is_gather_scatter)
+	{
+	  if (TARGET_ALTIVEC && is_packed)
+	    return false;
+	  else
+	    return true;
+	}
+
       if (TARGET_EFFICIENT_UNALIGNED_VSX)
 	return true;
 
@@ -5165,6 +5174,7 @@ public:
 
 protected:
   void update_target_cost_per_stmt (vect_cost_for_stmt, stmt_vec_info,
+				    slp_tree node,
 				    vect_cost_model_location, unsigned int);
   void density_test (loop_vec_info);
   void adjust_vect_cost_per_loop (loop_vec_info);
@@ -5312,6 +5322,7 @@ rs6000_adjust_vect_cost_per_stmt (enum vect_cost_for_stmt kind,
 void
 rs6000_cost_data::update_target_cost_per_stmt (vect_cost_for_stmt kind,
 					       stmt_vec_info stmt_info,
+					       slp_tree node,
 					       vect_cost_model_location where,
 					       unsigned int orig_count)
 {
@@ -5372,12 +5383,12 @@ rs6000_cost_data::update_target_cost_per_stmt (vect_cost_for_stmt kind,
 	 or may not need to apply.  When finalizing the cost of the loop,
 	 the extra penalty is applied when the load density heuristics
 	 are satisfied.  */
-      if (kind == vec_construct && stmt_info
-	  && STMT_VINFO_TYPE (stmt_info) == load_vec_info_type
-	  && (STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) == VMAT_ELEMENTWISE
-	      || STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) == VMAT_STRIDED_SLP))
+      if (kind == vec_construct && node
+	  && SLP_TREE_TYPE (node) == load_vec_info_type
+	  && (SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_ELEMENTWISE
+	      || SLP_TREE_MEMORY_ACCESS_TYPE (node) == VMAT_STRIDED_SLP))
 	{
-	  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+	  tree vectype = SLP_TREE_VECTYPE (node);
 	  unsigned int nunits = vect_nunits_for_cost (vectype);
 	  /* As PR103702 shows, it's possible that vectorizer wants to do
 	     costings for only one unit here, it's no need to do any
@@ -5406,7 +5417,7 @@ rs6000_cost_data::update_target_cost_per_stmt (vect_cost_for_stmt kind,
 
 unsigned
 rs6000_cost_data::add_stmt_cost (int count, vect_cost_for_stmt kind,
-				 stmt_vec_info stmt_info, slp_tree,
+				 stmt_vec_info stmt_info, slp_tree node,
 				 tree vectype, int misalign,
 				 vect_cost_model_location where)
 {
@@ -5424,7 +5435,7 @@ rs6000_cost_data::add_stmt_cost (int count, vect_cost_for_stmt kind,
       retval = adjust_cost_for_freq (stmt_info, where, count * stmt_cost);
       m_costs[where] += retval;
 
-      update_target_cost_per_stmt (kind, stmt_info, where, orig_count);
+      update_target_cost_per_stmt (kind, stmt_info, node, where, orig_count);
     }
 
   return retval;
@@ -10309,15 +10320,18 @@ can_be_rotated_to_negative_lis (HOST_WIDE_INT c, int *rot)
 
   /* case b. xx0..01..1xx: some of 15 x's (and some of 16 0's) are
      rotated over the highest bit.  */
-  int pos_one = clz_hwi ((c << 16) >> 16);
-  middle_zeros = ctz_hwi (c >> (HOST_BITS_PER_WIDE_INT - pos_one));
-  int middle_ones = clz_hwi (~(c << pos_one));
-  if (middle_zeros >= 16 && middle_ones >= 33)
+  unsigned HOST_WIDE_INT uc = c;
+  int pos_one = clz_hwi ((HOST_WIDE_INT) (uc << 16) >> 16);
+  if (pos_one != 0)
     {
-      *rot = pos_one;
-      return true;
+      middle_zeros = ctz_hwi (c >> (HOST_BITS_PER_WIDE_INT - pos_one));
+      int middle_ones = clz_hwi (~(uc << pos_one));
+      if (middle_zeros >= 16 && middle_ones >= 33)
+	{
+	  *rot = pos_one;
+	  return true;
+	}
     }
-
   return false;
 }
 
@@ -10434,7 +10448,8 @@ can_be_built_by_li_and_rldic (HOST_WIDE_INT c, int *shift, HOST_WIDE_INT *mask)
   if (lz >= HOST_BITS_PER_WIDE_INT)
     return false;
 
-  int middle_ones = clz_hwi (~(c << lz));
+  unsigned HOST_WIDE_INT uc = c;
+  int middle_ones = clz_hwi (~(uc << lz));
   if (tz + lz + middle_ones >= ones
       && (tz - lz) < HOST_BITS_PER_WIDE_INT
       && tz < HOST_BITS_PER_WIDE_INT)
@@ -10468,7 +10483,7 @@ can_be_built_by_li_and_rldic (HOST_WIDE_INT c, int *shift, HOST_WIDE_INT *mask)
   if (!IN_RANGE (pos_first_1, 1, HOST_BITS_PER_WIDE_INT-1))
     return false;
 
-  middle_ones = clz_hwi (~c << pos_first_1);
+  middle_ones = clz_hwi ((~(unsigned HOST_WIDE_INT) c) << pos_first_1);
   middle_zeros = ctz_hwi (c >> (HOST_BITS_PER_WIDE_INT - pos_first_1));
   if (pos_first_1 < HOST_BITS_PER_WIDE_INT
       && middle_ones + middle_zeros < HOST_BITS_PER_WIDE_INT
@@ -10570,7 +10585,8 @@ rs6000_emit_set_long_const (rtx dest, HOST_WIDE_INT c, int *num_insns)
     {
       /* li/lis; rldicX */
       unsigned HOST_WIDE_INT imm = (c | ~mask);
-      imm = (imm >> shift) | (imm << (HOST_BITS_PER_WIDE_INT - shift));
+      if (shift != 0)
+	imm = (imm >> shift) | (imm << (HOST_BITS_PER_WIDE_INT - shift));
 
       count_or_emit_insn (temp, GEN_INT (imm));
       if (shift != 0)
diff --git a/gcc/config/rs6000/rs6000.md b/gcc/config/rs6000/rs6000.md
index 9c718ca..e31ee40 100644
--- a/gcc/config/rs6000/rs6000.md
+++ b/gcc/config/rs6000/rs6000.md
@@ -1969,7 +1969,7 @@
   [(set (match_dup 0) (plus:GPR (match_dup 1) (match_dup 3)))
    (set (match_dup 0) (plus:GPR (match_dup 0) (match_dup 4)))]
 {
-  HOST_WIDE_INT val = INTVAL (operands[2]);
+  unsigned HOST_WIDE_INT val = UINTVAL (operands[2]);
   HOST_WIDE_INT low = sext_hwi (val, 16);
   HOST_WIDE_INT rest = trunc_int_for_mode (val - low, <MODE>mode);
 
diff --git a/gcc/config/rs6000/vxworks.h b/gcc/config/rs6000/vxworks.h
index fa2c837b..e77247b 100644
--- a/gcc/config/rs6000/vxworks.h
+++ b/gcc/config/rs6000/vxworks.h
@@ -34,6 +34,21 @@ along with GCC; see the file COPYING3.  If not see
 /* Common definitions first.                                   */
 /*-------------------------------------------------------------*/
 
+/* Default to 64 bits when the target is powerpc64*-wrs-vxworks*,
+   and to 32 bits otherwise.  */
+#undef SUBTARGET_DRIVER_SELF_SPECS
+#if TARGET_VXWORKS64
+#define SUBTARGET_DRIVER_SELF_SPECS "%{!m64:%{!m32:-m64}}"
+#else
+#define SUBTARGET_DRIVER_SELF_SPECS "%{!m32:%{!m64:-m32}}"
+#endif
+
+/* Having used the build-time TARGET_VXWORKS64 to choose the default ABI above,
+   redefine it so that it matches whichever ABI is selected for each
+   compilation.  */
+#undef TARGET_VXWORKS64
+#define TARGET_VXWORKS64 TARGET_64BIT
+
 /* CPP predefined macros.  */
 
 #undef TARGET_OS_CPP_BUILTINS
diff --git a/gcc/config/s390/s390-protos.h b/gcc/config/s390/s390-protos.h
index d760a7e..6becad1 100644
--- a/gcc/config/s390/s390-protos.h
+++ b/gcc/config/s390/s390-protos.h
@@ -128,6 +128,8 @@ extern void s390_expand_vcond (rtx, rtx, rtx, enum rtx_code, rtx, rtx);
 extern void s390_expand_vec_init (rtx, rtx);
 extern rtx s390_expand_merge_perm_const (machine_mode, bool);
 extern void s390_expand_merge (rtx, rtx, rtx, bool);
+extern void s390_expand_int_spaceship (rtx, rtx, rtx, rtx);
+extern void s390_expand_fp_spaceship (rtx, rtx, rtx, rtx);
 extern rtx s390_build_signbit_mask (machine_mode);
 extern rtx s390_return_addr_rtx (int, rtx);
 extern rtx s390_back_chain_rtx (void);
diff --git a/gcc/config/s390/s390.cc b/gcc/config/s390/s390.cc
index 3826720..012b6db 100644
--- a/gcc/config/s390/s390.cc
+++ b/gcc/config/s390/s390.cc
@@ -3862,7 +3862,21 @@ s390_register_move_cost (machine_mode mode,
 {
   /* On s390, copy between fprs and gprs is expensive.  */
 
-  /* It becomes somewhat faster having ldgr/lgdr.  */
+  /* With vector extensions any GPR<->VR load up to 8 bytes is supported.  */
+  if (TARGET_VX && GET_MODE_SIZE (mode) <= 8)
+    {
+      /* ldgr/vlvgg take one cycle and vlvg[bhf] take two cycles. */
+      if (reg_classes_intersect_p (from, GENERAL_REGS)
+	  && reg_classes_intersect_p (to, VEC_REGS))
+	return GET_MODE_SIZE (mode) == 8 ? 1 : 2;
+      /* lgdr/vlgv[fg] take three cycles and vlgv[bh] take five cycles. */
+      if (reg_classes_intersect_p (to, GENERAL_REGS)
+	  && reg_classes_intersect_p (from, VEC_REGS))
+	return GET_MODE_SIZE (mode) >= 4 ? 3 : 4;
+    }
+
+  /* Without vector extensions it still becomes somewhat faster having
+     ldgr/lgdr.  */
   if (TARGET_Z10 && GET_MODE_SIZE (mode) == 8)
     {
       /* ldgr is single cycle. */
@@ -8199,6 +8213,167 @@ s390_expand_atomic (machine_mode mode, enum rtx_code code,
 					       NULL_RTX, 1, OPTAB_DIRECT), 1);
 }
 
+/* Expand integer op0 = op1 <=> op2, i.e.,
+   op0 = op1 == op2 ? 0 : op1 < op2 ? -1 : 1.
+
+   Signedness is specified by op3.  If op3 equals 1, then perform an unsigned
+   comparison, and if op3 equals -1, then perform a signed comparison.
+
+   For integer comparisons we strive for a sequence like
+   CR[L] ; LHI ; LOCHIL ; LOCHIH
+   where the first three instructions fit into a group.  */
+
+void
+s390_expand_int_spaceship (rtx op0, rtx op1, rtx op2, rtx op3)
+{
+  gcc_assert (op3 == const1_rtx || op3 == constm1_rtx);
+
+  rtx cc, cond_lt, cond_gt;
+  machine_mode cc_mode;
+  machine_mode mode = GET_MODE (op1);
+
+  /* Prior VXE3 emulate a 128-bit comparison by breaking it up into three
+     comparisons.  First test the high halfs.  In case they equal, then test
+     the low halfs.  Finally, test for equality.  Depending on the results
+     make use of LOCs.  */
+  if (mode == TImode && !TARGET_VXE3)
+    {
+      gcc_assert (TARGET_VX);
+      op1
+	= force_reg (V2DImode, simplify_gen_subreg (V2DImode, op1, TImode, 0));
+      op2
+	= force_reg (V2DImode, simplify_gen_subreg (V2DImode, op2, TImode, 0));
+      rtx lab = gen_label_rtx ();
+      rtx ccz = gen_rtx_REG (CCZmode, CC_REGNUM);
+      /* Compare high halfs for equality.
+	 VEC[L]G op1, op2 sets
+	   CC1 if high(op1) < high(op2)
+	 and
+	   CC2 if high(op1) > high(op2).  */
+      machine_mode cc_mode = op3 == const1_rtx ? CCUmode : CCSmode;
+      rtx lane0 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
+      emit_insn (gen_rtx_SET (
+	gen_rtx_REG (cc_mode, CC_REGNUM),
+	gen_rtx_COMPARE (cc_mode,
+			 gen_rtx_VEC_SELECT (DImode, op1, lane0),
+			 gen_rtx_VEC_SELECT (DImode, op2, lane0))));
+      s390_emit_jump (lab, gen_rtx_NE (CCZmode, ccz, const0_rtx));
+      /* At this point we know that the high halfs equal.
+	 VCHLGS op2, op1 sets CC1 if low(op1) < low(op2)  */
+      emit_insn (gen_rtx_PARALLEL (
+	VOIDmode,
+	gen_rtvec (2,
+		   gen_rtx_SET (gen_rtx_REG (CCVIHUmode, CC_REGNUM),
+				gen_rtx_COMPARE (CCVIHUmode, op2, op1)),
+		   gen_rtx_CLOBBER (VOIDmode, gen_rtx_SCRATCH (V2DImode)))));
+      emit_label (lab);
+      emit_insn (gen_rtx_SET (op0, const1_rtx));
+      emit_insn (
+	gen_movsicc (op0,
+		     gen_rtx_LTU (CCUmode, gen_rtx_REG (CCUmode, CC_REGNUM),
+				  const0_rtx),
+		     constm1_rtx, op0));
+      /* Deal with the case where both halfs equal.  */
+      emit_insn (gen_rtx_PARALLEL (
+	VOIDmode,
+	gen_rtvec (2,
+		   gen_rtx_SET (gen_rtx_REG (CCVEQmode, CC_REGNUM),
+				gen_rtx_COMPARE (CCVEQmode, op1, op2)),
+		   gen_rtx_SET (gen_reg_rtx (V2DImode),
+				gen_rtx_EQ (V2DImode, op1, op2)))));
+      emit_insn (gen_movsicc (op0, gen_rtx_EQ (CCZmode, ccz, const0_rtx),
+			      const0_rtx, op0));
+      return;
+    }
+
+  if (mode == QImode || mode == HImode)
+    {
+      rtx_code extend = op3 == const1_rtx ? ZERO_EXTEND : SIGN_EXTEND;
+      op1 = simplify_gen_unary (extend, SImode, op1, mode);
+      op1 = force_reg (SImode, op1);
+      op2 = simplify_gen_unary (extend, SImode, op2, mode);
+      op2 = force_reg (SImode, op2);
+      mode = SImode;
+    }
+
+  if (op3 == const1_rtx)
+    {
+      cc_mode = CCUmode;
+      cc = gen_rtx_REG (cc_mode, CC_REGNUM);
+      cond_lt = gen_rtx_LTU (mode, cc, const0_rtx);
+      cond_gt = gen_rtx_GTU (mode, cc, const0_rtx);
+    }
+  else
+    {
+      cc_mode = CCSmode;
+      cc = gen_rtx_REG (cc_mode, CC_REGNUM);
+      cond_lt = gen_rtx_LT (mode, cc, const0_rtx);
+      cond_gt = gen_rtx_GT (mode, cc, const0_rtx);
+    }
+
+  emit_insn (gen_rtx_SET (cc, gen_rtx_COMPARE (cc_mode, op1, op2)));
+  emit_move_insn (op0, const0_rtx);
+  emit_insn (gen_movsicc (op0, cond_lt, constm1_rtx, op0));
+  emit_insn (gen_movsicc (op0, cond_gt, const1_rtx, op0));
+}
+
+/* Expand floating-point op0 = op1 <=> op2, i.e.,
+   op0 = op1 == op2 ? 0 : op1 < op2 ? -1 : op1 > op2 ? 1 : 2.
+
+   If op3 equals const0_rtx, then we are interested in the compare only (see
+   test spaceship-fp-4.c).  Otherwise, op3 is a CONST_INT different than
+   const1_rtx and constm1_rtx which is used in order to set op0 for unordered.
+
+   Emit a branch-only solution, i.e., let if-convert fold the branches into
+   LOCs if applicable.  This has the benefit that the solution is also
+   applicable if we are only interested in the compare, i.e., if op3 equals
+   const0_rtx.
+ */
+
+void
+s390_expand_fp_spaceship (rtx op0, rtx op1, rtx op2, rtx op3)
+{
+  gcc_assert (op3 != const1_rtx && op3 != constm1_rtx);
+
+  machine_mode mode = GET_MODE (op1);
+  machine_mode cc_mode = s390_select_ccmode (LTGT, op1, op2);
+  rtx cc_reg = gen_rtx_REG (cc_mode, CC_REGNUM);
+  rtx cond_unordered = gen_rtx_UNORDERED (mode, cc_reg, const0_rtx);
+  rtx cond_eq = gen_rtx_EQ (mode, cc_reg, const0_rtx);
+  rtx cond_gt = gen_rtx_GT (mode, cc_reg, const0_rtx);
+  rtx_insn *insn;
+  rtx l_unordered = gen_label_rtx ();
+  rtx l_eq = gen_label_rtx ();
+  rtx l_gt = gen_label_rtx ();
+  rtx l_end = gen_label_rtx ();
+
+  s390_emit_compare (VOIDmode, LTGT, op1, op2);
+  if (!flag_finite_math_only)
+    {
+      insn = s390_emit_jump (l_unordered, cond_unordered);
+      add_reg_br_prob_note (insn, profile_probability::very_unlikely ());
+    }
+  insn = s390_emit_jump (l_eq, cond_eq);
+  add_reg_br_prob_note (insn, profile_probability::unlikely ());
+  insn = s390_emit_jump (l_gt, cond_gt);
+  add_reg_br_prob_note (insn, profile_probability::even ());
+  emit_move_insn (op0, constm1_rtx);
+  emit_jump (l_end);
+  emit_label (l_eq);
+  emit_move_insn (op0, const0_rtx);
+  emit_jump (l_end);
+  emit_label (l_gt);
+  emit_move_insn (op0, const1_rtx);
+  if (!flag_finite_math_only)
+    {
+      emit_jump (l_end);
+      emit_label (l_unordered);
+      rtx unord_val = op3 == const0_rtx ? const2_rtx : op3;
+      emit_move_insn (op0, unord_val);
+    }
+  emit_label (l_end);
+}
+
 /* This is called from dwarf2out.cc via TARGET_ASM_OUTPUT_DWARF_DTPREL.
    We need to emit DTP-relative relocations.  */
 
@@ -16566,9 +16741,6 @@ s390_option_override_internal (struct gcc_options *opts,
   else
     SET_OPTION_IF_UNSET (opts, opts_set, param_vect_partial_vector_usage, 0);
 
-  /* Do not vectorize loops with a low trip count for now.  */
-  SET_OPTION_IF_UNSET (opts, opts_set, param_min_vect_loop_bound, 2);
-
   /* Set the default alignment.  */
   s390_default_align (opts);
 
@@ -16863,8 +17035,9 @@ s390_valid_target_attribute_inner_p (tree args,
 	      generate_option (opt, NULL, value, CL_TARGET, &decoded);
 	      s390_handle_option (opts, new_opts_set, &decoded, input_location);
 	      set_option (opts, new_opts_set, opt, value,
-			  p + opt_len, DK_UNSPECIFIED, input_location,
-			  global_dc);
+			  p + opt_len,
+			  static_cast<int> (diagnostics::kind::unspecified),
+			  input_location, global_dc);
 	    }
 	  else
 	    {
@@ -16881,8 +17054,9 @@ s390_valid_target_attribute_inner_p (tree args,
 	  arg_ok = opt_enum_arg_to_value (opt, p + opt_len, &value, CL_TARGET);
 	  if (arg_ok)
 	    set_option (opts, new_opts_set, opt, value,
-			p + opt_len, DK_UNSPECIFIED, input_location,
-			global_dc);
+			p + opt_len,
+			static_cast<int> (diagnostics::kind::unspecified),
+			input_location, global_dc);
 	  else
 	    {
 	      error ("attribute %<target%> argument %qs is unknown", orig_p);
@@ -17334,13 +17508,15 @@ static bool
 s390_support_vector_misalignment (machine_mode mode ATTRIBUTE_UNUSED,
 				  const_tree type ATTRIBUTE_UNUSED,
 				  int misalignment ATTRIBUTE_UNUSED,
-				  bool is_packed ATTRIBUTE_UNUSED)
+				  bool is_packed ATTRIBUTE_UNUSED,
+				  bool is_gather_scatter ATTRIBUTE_UNUSED)
 {
   if (TARGET_VX)
     return true;
 
   return default_builtin_support_vector_misalignment (mode, type, misalignment,
-						      is_packed);
+						      is_packed,
+						      is_gather_scatter);
 }
 
 /* The vector ABI requires vector types to be aligned on an 8 byte
@@ -17832,9 +18008,11 @@ f_constraint_p (const char *constraint)
   for (size_t i = 0, c_len = strlen (constraint); i < c_len;
        i += CONSTRAINT_LEN (constraint[i], constraint + i))
     {
-      if (constraint[i] == 'f')
+      if (constraint[i] == 'f'
+	  || (constraint[i] == '{' && constraint[i + 1] == 'f'))
 	seen_f_p = true;
-      if (constraint[i] == 'v')
+      if (constraint[i] == 'v'
+	  || (constraint[i] == '{' && constraint[i + 1] == 'v'))
 	seen_v_p = true;
     }
 
@@ -17924,7 +18102,8 @@ s390_md_asm_adjust (vec<rtx> &outputs, vec<rtx> &inputs,
 	continue;
       bool allows_mem, allows_reg, is_inout;
       bool ok = parse_output_constraint (&constraint, i, ninputs, noutputs,
-					 &allows_mem, &allows_reg, &is_inout);
+					 &allows_mem, &allows_reg, &is_inout,
+					 nullptr);
       gcc_assert (ok);
       if (!f_constraint_p (constraint))
 	/* Long double with a constraint other than "=f" - nothing to do.  */
@@ -17969,7 +18148,7 @@ s390_md_asm_adjust (vec<rtx> &outputs, vec<rtx> &inputs,
       bool allows_mem, allows_reg;
       bool ok = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
 					constraints.address (), &allows_mem,
-					&allows_reg);
+					&allows_reg, nullptr);
       gcc_assert (ok);
       if (!f_constraint_p (constraint))
 	/* Long double with a constraint other than "f" (or "=f" for inout
@@ -18041,9 +18220,34 @@ expand_perm_with_merge (const struct expand_vec_perm_d &d)
   static const unsigned char lo_perm_qi_swap[16]
     = {17, 1, 19, 3, 21, 5, 23, 7, 25, 9, 27, 11, 29, 13, 31, 15};
 
+  static const unsigned char hi_perm_qi_di[16]
+    = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
+  static const unsigned char hi_perm_qi_si[16]
+    = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23};
+  static const unsigned char hi_perm_qi_hi[16]
+    = {0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23};
+
+  static const unsigned char lo_perm_qi_di[16]
+    = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
+  static const unsigned char lo_perm_qi_si[16]
+    = {8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31};
+  static const unsigned char lo_perm_qi_hi[16]
+    = {8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31};
+
+  static const unsigned char hi_perm_hi_si[8] = {0, 1, 8, 9, 2, 3, 10, 11};
+  static const unsigned char hi_perm_hi_di[8] = {0, 1, 2, 3, 8, 9, 10, 11};
+
+  static const unsigned char lo_perm_hi_si[8] = {4, 5, 12, 13, 6, 7, 14, 15};
+  static const unsigned char lo_perm_hi_di[8] = {4, 5, 6, 7, 12, 13, 14, 15};
+
+  static const unsigned char hi_perm_si_di[4] = {0, 1, 4, 5};
+
+  static const unsigned char lo_perm_si_di[4] = {2, 3, 6, 7};
+
   bool merge_lo_p = false;
   bool merge_hi_p = false;
   bool swap_operands_p = false;
+  machine_mode mergemode = d.vmode;
 
   if ((d.nelt == 2 && memcmp (d.perm, hi_perm_di, 2) == 0)
       || (d.nelt == 4 && memcmp (d.perm, hi_perm_si, 4) == 0)
@@ -18075,6 +18279,75 @@ expand_perm_with_merge (const struct expand_vec_perm_d &d)
       merge_lo_p = true;
       swap_operands_p = true;
     }
+  else if (d.nelt == 16)
+    {
+      if (memcmp (d.perm, hi_perm_qi_di, 16) == 0)
+	{
+	  merge_hi_p = true;
+	  mergemode = E_V2DImode;
+	}
+      else if (memcmp (d.perm, hi_perm_qi_si, 16) == 0)
+	{
+	  merge_hi_p = true;
+	  mergemode = E_V4SImode;
+	}
+      else if (memcmp (d.perm, hi_perm_qi_hi, 16) == 0)
+	{
+	  merge_hi_p = true;
+	  mergemode = E_V8HImode;
+	}
+      else if (memcmp (d.perm, lo_perm_qi_di, 16) == 0)
+	{
+	  merge_lo_p = true;
+	  mergemode = E_V2DImode;
+	}
+      else if (memcmp (d.perm, lo_perm_qi_si, 16) == 0)
+	{
+	  merge_lo_p = true;
+	  mergemode = E_V4SImode;
+	}
+      else if (memcmp (d.perm, lo_perm_qi_hi, 16) == 0)
+	{
+	  merge_lo_p = true;
+	  mergemode = E_V8HImode;
+	}
+    }
+  else if (d.nelt == 8)
+    {
+      if (memcmp (d.perm, hi_perm_hi_di, 8) == 0)
+	{
+	  merge_hi_p = true;
+	  mergemode = E_V2DImode;
+	}
+      else if (memcmp (d.perm, hi_perm_hi_si, 8) == 0)
+	{
+	  merge_hi_p = true;
+	  mergemode = E_V4SImode;
+	}
+      else if (memcmp (d.perm, lo_perm_hi_di, 8) == 0)
+	{
+	  merge_lo_p = true;
+	  mergemode = E_V2DImode;
+	}
+      else if (memcmp (d.perm, lo_perm_hi_si, 8) == 0)
+	{
+	  merge_lo_p = true;
+	  mergemode = E_V4SImode;
+	}
+    }
+  else if (d.nelt == 4)
+    {
+      if (memcmp (d.perm, hi_perm_si_di, 4) == 0)
+	{
+	  merge_hi_p = true;
+	  mergemode = E_V2DImode;
+	}
+      else if (memcmp (d.perm, lo_perm_si_di, 4) == 0)
+	{
+	  merge_lo_p = true;
+	  mergemode = E_V2DImode;
+	}
+    }
 
   if (!merge_lo_p && !merge_hi_p)
     return false;
@@ -18082,7 +18355,7 @@ expand_perm_with_merge (const struct expand_vec_perm_d &d)
   if (d.testing_p)
     return merge_lo_p || merge_hi_p;
 
-  rtx op0, op1;
+  rtx op0, op1, target = d.target;
   if (swap_operands_p)
     {
       op0 = d.op1;
@@ -18093,12 +18366,80 @@ expand_perm_with_merge (const struct expand_vec_perm_d &d)
       op0 = d.op0;
       op1 = d.op1;
     }
+  if (mergemode != d.vmode)
+    {
+      target = simplify_gen_subreg (mergemode, target, d.vmode, 0);
+      op0 = simplify_gen_subreg (mergemode, op0, d.vmode, 0);
+      op1 = simplify_gen_subreg (mergemode, op1, d.vmode, 0);
+    }
 
-  s390_expand_merge (d.target, op0, op1, merge_hi_p);
+  s390_expand_merge (target, op0, op1, merge_hi_p);
 
   return true;
 }
 
+/* Try to expand the vector permute operation described by D using the vector
+   pack instruction vpk.  Return true if vector pack could be used.  */
+static bool
+expand_perm_with_pack (const struct expand_vec_perm_d &d)
+{
+  static const unsigned char qi_hi[16]
+    = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31};
+  static const unsigned char qi_si[16]
+    = {2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31};
+  static const unsigned char qi_di[16]
+    = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31};
+
+  static const unsigned char hi_si[8]
+    = {1, 3, 5, 7, 9, 11, 13, 15};
+  static const unsigned char hi_di[8]
+    = {2, 3, 6, 7, 10, 11, 14, 15};
+
+  static const unsigned char si_di[4]
+    = {1, 3, 5, 7};
+
+  machine_mode packmode, resmode;
+  enum insn_code code = CODE_FOR_nothing;
+
+  if (d.nelt == 16 && memcmp (d.perm, qi_hi, 16) == 0)
+    {
+      packmode = E_V8HImode;
+      resmode = E_V16QImode;
+      code = CODE_FOR_vec_pack_trunc_v8hi;
+    }
+  else if ((d.nelt == 16 && memcmp (d.perm, qi_si, 16) == 0)
+	   || (d.nelt == 8 && memcmp (d.perm, hi_si, 8) == 0))
+    {
+      packmode = E_V4SImode;
+      resmode = E_V8HImode;
+      code = CODE_FOR_vec_pack_trunc_v4si;
+    }
+  else if ((d.nelt == 16 && memcmp (d.perm, qi_di, 16) == 0)
+	   || (d.nelt == 8 && memcmp (d.perm, hi_di, 8) == 0)
+	   || (d.nelt == 4 && memcmp (d.perm, si_di, 4) == 0))
+    {
+      packmode = E_V2DImode;
+      resmode = E_V4SImode;
+      code = CODE_FOR_vec_pack_trunc_v2di;
+    }
+
+  if (code == CODE_FOR_nothing)
+    return false;
+
+  if (d.testing_p)
+    return true;
+  rtx target = simplify_gen_subreg (resmode, d.target, d.vmode, 0);
+  rtx op0 = simplify_gen_subreg (packmode,
+				 force_reg (GET_MODE (d.op0), d.op0),
+				 d.vmode, 0);
+  rtx op1 = simplify_gen_subreg (packmode,
+				 force_reg (GET_MODE (d.op1), d.op1),
+				 d.vmode, 0);
+  rtx pat = GEN_FCN (code) (target, op0, op1);
+  emit_insn (pat);
+  return true;
+}
+
 /* Try to expand the vector permute operation described by D using the
    vector permute doubleword immediate instruction vpdi.  Return true
    if vpdi could be used.
@@ -18322,6 +18663,9 @@ vectorize_vec_perm_const_1 (const struct expand_vec_perm_d &d)
   if (expand_perm_with_merge (d))
     return true;
 
+  if (expand_perm_with_pack (d))
+    return true;
+
   if (expand_perm_with_vpdi (d))
     return true;
 
diff --git a/gcc/config/s390/s390.md b/gcc/config/s390/s390.md
index 97a4bdf..8cc48b0 100644
--- a/gcc/config/s390/s390.md
+++ b/gcc/config/s390/s390.md
@@ -121,6 +121,7 @@
 
    ; Test Data Class (TDC)
    UNSPEC_TDC_INSN
+   UNSPEC_SIGNBIT
 
    ; Byte-wise Population Count
    UNSPEC_POPCNT
@@ -139,9 +140,6 @@
    UNSPEC_LCBB
 
    ; Vector
-   UNSPEC_VEC_SMULT_HI
-   UNSPEC_VEC_UMULT_HI
-   UNSPEC_VEC_SMULT_LO
    UNSPEC_VEC_SMULT_EVEN
    UNSPEC_VEC_UMULT_EVEN
    UNSPEC_VEC_SMULT_ODD
@@ -241,9 +239,6 @@
 
    UNSPEC_VEC_MSUM
 
-   UNSPEC_VEC_VFMIN
-   UNSPEC_VEC_VFMAX
-
    UNSPEC_VEC_VBLEND
    UNSPEC_VEC_VEVAL
    UNSPEC_VEC_VGEM
@@ -256,6 +251,9 @@
 
    UNSPEC_NNPA_VCFN_V8HI
    UNSPEC_NNPA_VCNF_V8HI
+
+   UNSPEC_FMAX
+   UNSPEC_FMIN
 ])
 
 ;;
@@ -311,6 +309,9 @@
    UNSPECV_SPLIT_STACK_CALL
 
    UNSPECV_OSC_BREAK
+
+   ; Stack Protector
+   UNSPECV_SP_GET_TP
   ])
 
 ;;
@@ -368,6 +369,9 @@
    (VR23_REGNUM                 45)
    (VR24_REGNUM                 46)
    (VR31_REGNUM                 53)
+   ; Access registers
+   (AR0_REGNUM                  36)
+   (AR1_REGNUM                  37)
   ])
 
 ; Rounding modes for binary floating point numbers
@@ -510,7 +514,7 @@
 				    S390_TDC_INFINITY
 				    S390_TDC_NORMAL_BFP])
 
-(define_int_attr tdc_insn [(S390_TDC_SIGNBIT_SET "signbit")
+(define_int_attr tdc_insn [(S390_TDC_SIGNBIT_SET "signbit_tdc")
 			   (S390_TDC_FINITE "isfinite")
 			   (S390_TDC_INFINITY "isinf")
 			   (S390_TDC_NORMAL_BFP "isnormal")
@@ -1523,6 +1527,27 @@
   operands[0] = SET_DEST (PATTERN (curr_insn));
 })
 
+; Restrict spaceship optab to z13 or later since there we have
+; LOAD HALFWORD IMMEDIATE ON CONDITION.
+
+(define_mode_iterator SPACESHIP_INT [(TI "TARGET_VX") DI SI HI QI])
+(define_expand "spaceship<mode>4"
+  [(match_operand:SI 0 "register_operand")
+   (match_operand:SPACESHIP_INT 1 "register_operand")
+   (match_operand:SPACESHIP_INT 2 "register_operand")
+   (match_operand:SI 3 "const_int_operand")]
+  "TARGET_Z13 && TARGET_64BIT"
+  "s390_expand_int_spaceship (operands[0], operands[1], operands[2], operands[3]); DONE;")
+
+(define_mode_iterator SPACESHIP_BFP [TF DF SF])
+(define_expand "spaceship<mode>4"
+  [(match_operand:SI 0 "register_operand")
+   (match_operand:SPACESHIP_BFP 1 "register_operand")
+   (match_operand:SPACESHIP_BFP 2 "register_operand")
+   (match_operand:SI 3 "const_int_operand")]
+  "TARGET_Z13 && TARGET_64BIT && TARGET_HARD_FLOAT"
+  "s390_expand_fp_spaceship (operands[0], operands[1], operands[2], operands[3]); DONE;")
+
 
 ; (TF|DF|SF|TD|DD|SD) instructions
 
@@ -3779,6 +3804,86 @@
 	(unspec:SI [(reg:CCZ CC_REGNUM)] UNSPEC_CC_TO_INT))]
   "TARGET_HARD_DFP")
 
+(define_mode_iterator SIGNBIT_SINGLE [(SF "TARGET_HARD_FLOAT")
+				      (SD "TARGET_HARD_DFP")])
+(define_expand "signbit<mode>2"
+  [(match_operand:SI 0 "register_operand")
+   (match_operand:SIGNBIT_SINGLE 1 "nonimmediate_operand")]
+  ""
+{
+  if (TARGET_VX && TARGET_64BIT)
+    {
+      emit_insn (gen_rtx_SET (operands[0], simplify_gen_subreg (SImode, operands[1], <MODE>mode, 0)));
+      emit_insn (gen_rtx_SET (operands[0], gen_rtx_LSHIFTRT (SImode, operands[0], GEN_INT (31))));
+    }
+  else if (TARGET_Z10 && TARGET_64BIT)
+    emit_insn (gen_signbit<mode>2_z10 (operands[0], operands[1]));
+  else
+    emit_insn (gen_signbit_tdc<mode>2 (operands[0], force_reg (<MODE>mode, operands[1])));
+  DONE;
+})
+
+(define_insn "signbit<mode>2_z10"
+  [(set (match_operand:SI 0 "register_operand" "=d")
+	(unspec:SI [(match_operand:SIGNBIT_SINGLE 1 "nonimmediate_operand" "fRT")]
+		   UNSPEC_SIGNBIT))]
+  "TARGET_Z10 && TARGET_64BIT"
+  "#")
+
+(define_split
+  [(set (match_operand:SI 0 "register_operand")
+	(unspec:SI [(match_operand:SIGNBIT_SINGLE 1 "register_operand")]
+		   UNSPEC_SIGNBIT))]
+  "TARGET_Z10 && TARGET_64BIT && reload_completed"
+  [(set (match_dup 0) (match_dup 1))
+   (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 63)))]
+{
+  operands[0] = gen_rtx_REG (DImode, REGNO (operands[0]));
+  operands[1] = gen_rtx_REG (DImode, REGNO (operands[1]));
+})
+
+(define_split
+  [(set (match_operand:SI 0 "register_operand")
+	(unspec:SI [(match_operand:SIGNBIT_SINGLE 1 "memory_operand")]
+		   UNSPEC_SIGNBIT))]
+  "TARGET_Z10 && TARGET_64BIT && reload_completed"
+  [(set (match_dup 0) (match_dup 1))
+   (set (match_dup 0) (lshiftrt:SI (match_dup 0) (const_int 31)))]
+{
+  operands[1] = change_address (operands[1], SImode, 0);
+})
+
+(define_mode_iterator SIGNBIT_DBL_TETRA [(DF "TARGET_HARD_FLOAT")
+					 (TF "TARGET_HARD_FLOAT")
+					 (DD "TARGET_HARD_DFP")
+					 (TD "TARGET_HARD_DFP")])
+(define_expand "signbit<mode>2"
+  [(match_operand:SI 0 "register_operand")
+   (match_operand:SIGNBIT_DBL_TETRA 1 "nonimmediate_operand")]
+  ""
+{
+  if (TARGET_Z10 && TARGET_64BIT)
+    {
+      rtx reg_di = gen_reg_rtx (DImode);
+      if (<MODE>mode == TFmode || <MODE>mode == TDmode)
+	{
+	  rtx reg_ti = gen_reg_rtx (TImode);
+	  emit_insn (gen_rtx_SET (reg_ti, simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0)));
+	  emit_insn (gen_rtx_SET (reg_di, simplify_gen_subreg (DImode, reg_ti, TImode, 0)));
+	}
+      else
+	emit_insn (gen_rtx_SET (reg_di, simplify_gen_subreg (DImode, operands[1], <MODE>mode, 0)));
+      emit_insn (gen_rtx_SET (reg_di, gen_rtx_LSHIFTRT (DImode, reg_di, GEN_INT (63))));
+      rtx subreg = gen_rtx_SUBREG (SImode, reg_di, 4);
+      SUBREG_PROMOTED_VAR_P (subreg) = 1;
+      SUBREG_PROMOTED_SET (subreg, SRP_SIGNED_AND_UNSIGNED);
+      emit_insn (gen_rtx_SET (operands[0], subreg));
+    }
+  else
+    emit_insn (gen_signbit_tdc<mode>2 (operands[0], force_reg (<MODE>mode, operands[1])));
+  DONE;
+})
+
 ; This extracts CC into a GPR properly shifted.  The actual IPM
 ; instruction will be issued by reload.  The constraint of operand 1
 ; forces reload to use a GPR.  So reload will issue a movcc insn for
@@ -11927,15 +12032,43 @@
 ; Stack Protector Patterns
 ;
 
+; Insns stack_protect_get_tp{si,di} are similar to *get_tp_{31,64} but still
+; distinct in the sense that they force recomputation of the thread pointer
+; instead of potentially reloading it from stack.
+
+(define_insn_and_split "stack_protect_get_tpsi"
+  [(set (match_operand:SI 0 "register_operand" "=d")
+	(unspec_volatile:SI [(const_int 0)] UNSPECV_SP_GET_TP))]
+  ""
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0) (reg:SI AR0_REGNUM))])
+
+(define_insn_and_split "stack_protect_get_tpdi"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+	(unspec_volatile:DI [(const_int 0)] UNSPECV_SP_GET_TP))]
+  ""
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 1) (reg:SI AR0_REGNUM))
+   (set (match_dup 0) (ashift:DI (match_dup 0) (const_int 32)))
+   (set (strict_low_part (match_dup 1)) (reg:SI AR1_REGNUM))]
+  "operands[1] = gen_rtx_REG (SImode, REGNO (operands[0]));")
+
 (define_expand "stack_protect_set"
   [(set (match_operand 0 "memory_operand" "")
 	(match_operand 1 "memory_operand" ""))]
   ""
 {
 #ifdef TARGET_THREAD_SSP_OFFSET
+  rtx tp = gen_reg_rtx (Pmode);
+  if (TARGET_64BIT)
+    emit_insn (gen_stack_protect_get_tpdi (tp));
+  else
+    emit_insn (gen_stack_protect_get_tpsi (tp));
   operands[1]
-    = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, s390_get_thread_pointer (),
-                                        GEN_INT (TARGET_THREAD_SSP_OFFSET)));
+    = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, tp,
+					GEN_INT (TARGET_THREAD_SSP_OFFSET)));
 #endif
   if (TARGET_64BIT)
     emit_insn (gen_stack_protect_setdi (operands[0], operands[1]));
@@ -11961,9 +12094,14 @@
 {
   rtx cc_reg, test;
 #ifdef TARGET_THREAD_SSP_OFFSET
+  rtx tp = gen_reg_rtx (Pmode);
+  if (TARGET_64BIT)
+    emit_insn (gen_stack_protect_get_tpdi (tp));
+  else
+    emit_insn (gen_stack_protect_get_tpsi (tp));
   operands[1]
-    = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, s390_get_thread_pointer (),
-                                        GEN_INT (TARGET_THREAD_SSP_OFFSET)));
+    = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, tp,
+					GEN_INT (TARGET_THREAD_SSP_OFFSET)));
 #endif
   if (TARGET_64BIT)
     emit_insn (gen_stack_protect_testdi (operands[0], operands[1]));
diff --git a/gcc/config/s390/vector.md b/gcc/config/s390/vector.md
index 7251a76..12bbeb6 100644
--- a/gcc/config/s390/vector.md
+++ b/gcc/config/s390/vector.md
@@ -89,6 +89,13 @@
 (define_mode_iterator VF_HW [(V4SF "TARGET_VXE") V2DF (V1TF "TARGET_VXE")
 			     (TF "TARGET_VXE")])
 
+; FP scalar and vector modes
+(define_mode_iterator VFT_BFP [SF DF
+			      (V1SF "TARGET_VXE") (V2SF "TARGET_VXE") (V4SF "TARGET_VXE")
+			      V1DF V2DF
+			      (V1TF "TARGET_VXE") (TF "TARGET_VXE")])
+
+
 (define_mode_iterator V_8   [V1QI])
 (define_mode_iterator V_16  [V2QI  V1HI])
 (define_mode_iterator V_32  [V4QI  V2HI V1SI V1SF])
@@ -142,13 +149,13 @@
 
 ; The instruction suffix for integer instructions and instructions
 ; which do not care about whether it is floating point or integer.
-(define_mode_attr bhfgq[(V1QI "b") (V2QI "b") (V4QI "b") (V8QI "b") (V16QI "b")
-			(V1HI "h") (V2HI "h") (V4HI "h") (V8HI "h")
-			(V1SI "f") (V2SI "f") (V4SI "f")
-			(V1DI "g") (V2DI "g")
+(define_mode_attr bhfgq[(V1QI "b") (V2QI "b") (V4QI "b") (V8QI "b") (V16QI "b") (QI "b")
+			(V1HI "h") (V2HI "h") (V4HI "h") (V8HI "h") (HI "h")
+			(V1SI "f") (V2SI "f") (V4SI "f") (SI "f")
+			(V1DI "g") (V2DI "g") (DI "g")
 			(V1TI "q") (TI "q")
-			(V1SF "f") (V2SF "f") (V4SF "f")
-			(V1DF "g") (V2DF "g")
+			(V1SF "f") (V2SF "f") (V4SF "f") (SF "f")
+			(V1DF "g") (V2DF "g") (DF "g")
 			(V1TF "q") (TF "q")])
 
 ; This is for vmalhw. It gets an 'w' attached to avoid confusion with
@@ -494,6 +501,54 @@
                         SIL,SIL,RI,RI,RRE,RRE,RIL,RR,RXY,RXY,RIL")])
 
 
+; Instructions vlgvb, vlgvh, vlgvf zero all remaining bits of a GPR, i.e.,
+; an implicit zero extend is done.
+
+(define_insn "*movdi<mode>_zero_extend_A"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+	(zero_extend:DI (match_operand:SINT 1 "register_operand" "v")))]
+  "TARGET_VX"
+  "vlgv<bhfgq>\t%0,%v1,0"
+  [(set_attr "op_type" "VRS")])
+
+(define_insn "*movsi<mode>_zero_extend_A"
+  [(set (match_operand:SI 0 "register_operand" "=d")
+	(zero_extend:SI (match_operand:HQI 1 "register_operand" "v")))]
+  "TARGET_VX"
+  "vlgv<bhfgq>\t%0,%v1,0"
+  [(set_attr "op_type" "VRS")])
+
+(define_mode_iterator VLGV_DI [V1QI V2QI V4QI V8QI V16QI
+			       V1HI V2HI V4HI V8HI
+			       V1SI V2SI V4SI])
+(define_insn "*movdi<mode>_zero_extend_B"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+	(zero_extend:DI (vec_select:<non_vec>
+			  (match_operand:VLGV_DI 1 "register_operand" "v")
+			  (parallel [(match_operand:SI 2 "const_int_operand" "n")]))))]
+  "TARGET_VX"
+{
+  operands[2] = GEN_INT (UINTVAL (operands[2]) & (GET_MODE_NUNITS (<MODE>mode) - 1));
+  return "vlgv<bhfgq>\t%0,%v1,%Y2";
+}
+  [(set_attr "op_type" "VRS")
+   (set_attr "mnemonic" "vlgv<bhfgq>")])
+
+(define_mode_iterator VLGV_SI [V1QI V2QI V4QI V8QI V16QI
+			       V1HI V2HI V4HI V8HI])
+(define_insn "*movsi<mode>_zero_extend_B"
+  [(set (match_operand:SI 0 "register_operand" "=d")
+	(zero_extend:SI (vec_select:<non_vec>
+			  (match_operand:VLGV_SI 1 "register_operand" "v")
+			  (parallel [(match_operand:SI 2 "const_int_operand" "n")]))))]
+  "TARGET_VX"
+{
+  operands[2] = GEN_INT (UINTVAL (operands[2]) & (GET_MODE_NUNITS (<MODE>mode) - 1));
+  return "vlgv<bhfgq>\t%0,%v1,%Y2";
+}
+  [(set_attr "op_type" "VRS")
+   (set_attr "mnemonic" "vlgv<bhfgq>")])
+
 ; vec_load_lanes?
 
 ; vec_store_lanes?
@@ -3565,14 +3620,341 @@
   "veval\t%v0,%v1,%v2,%v3,%b4"
   [(set_attr "op_type" "VRI")])
 
-; reduc_smin
-; reduc_smax
-; reduc_umin
-; reduc_umax
-
 ; vec_pack_sfix_trunc: convert + pack ?
 ; vec_pack_ufix_trunc
 ; vec_unpacks_float_hi
 ; vec_unpacks_float_lo
 ; vec_unpacku_float_hi
 ; vec_unpacku_float_lo
+
+(define_expand "avg<mode>3_ceil"
+  [(set (match_operand:VIT_HW_VXE3_T                        0 "register_operand")
+	(unspec:VIT_HW_VXE3_T [(match_operand:VIT_HW_VXE3_T 1 "register_operand")
+			       (match_operand:VIT_HW_VXE3_T 2 "register_operand")]
+			      UNSPEC_VEC_AVG))]
+  "TARGET_VX")
+
+(define_expand "uavg<mode>3_ceil"
+  [(set (match_operand:VIT_HW_VXE3_T                        0 "register_operand")
+	(unspec:VIT_HW_VXE3_T [(match_operand:VIT_HW_VXE3_T 1 "register_operand")
+			       (match_operand:VIT_HW_VXE3_T 2 "register_operand")]
+			      UNSPEC_VEC_AVGU))]
+  "TARGET_VX")
+
+(define_expand "smul<mode>3_highpart"
+  [(set (match_operand:VIT_HW_VXE3_DT 0 "register_operand")
+	(smul_highpart:VIT_HW_VXE3_DT (match_operand:VIT_HW_VXE3_DT 1 "register_operand")
+				      (match_operand:VIT_HW_VXE3_DT 2 "register_operand")))]
+  "TARGET_VX")
+
+(define_expand "umul<mode>3_highpart"
+  [(set (match_operand:VIT_HW_VXE3_DT 0 "register_operand")
+	(umul_highpart:VIT_HW_VXE3_DT (match_operand:VIT_HW_VXE3_DT 1 "register_operand")
+				      (match_operand:VIT_HW_VXE3_DT 2 "register_operand")))]
+  "TARGET_VX")
+
+; fmax
+(define_expand "fmax<mode>3"
+  [(set (match_operand:VFT_BFP                  0 "register_operand")
+	(unspec:VFT_BFP [(match_operand:VFT_BFP 1 "register_operand")
+	       (match_operand:VFT_BFP           2 "register_operand")
+	       (const_int 4)]
+	      UNSPEC_FMAX))]
+  "TARGET_VXE")
+
+; fmin
+(define_expand "fmin<mode>3"
+  [(set (match_operand:VFT_BFP                  0 "register_operand")
+	(unspec:VFT_BFP [(match_operand:VFT_BFP 1 "register_operand")
+	       (match_operand:VFT_BFP           2 "register_operand")
+	       (const_int 4)]
+	      UNSPEC_FMIN))]
+  "TARGET_VXE")
+
+; reduc_plus
+(define_expand "reduc_plus_scal_<mode>"
+  [(set (match_dup 4)
+	(unspec:V4SI [(match_operand:VI_HW_QH 1 "register_operand")
+		      (match_dup 2)]
+		      UNSPEC_VEC_VSUM))
+   (set (match_dup 5)
+	(unspec:V2DI [(match_dup 4) (match_dup 3)] UNSPEC_VEC_VSUMQ))
+   (set (match_operand:<non_vec> 0 "register_operand")
+	(vec_select:<non_vec> (match_dup 6)
+			      (parallel [(match_dup 7)])))]
+  "TARGET_VX"
+{
+  operands[2] = force_reg (<MODE>mode, CONST0_RTX (<MODE>mode));
+  operands[3] = simplify_gen_subreg (V4SImode, operands[2], <MODE>mode, 0);
+  operands[4] = gen_reg_rtx (V4SImode);
+  operands[5] = gen_reg_rtx (V2DImode);
+  operands[6] = simplify_gen_subreg (<MODE>mode, operands[5], V2DImode, 0);
+  operands[7] = GEN_INT (16 / GET_MODE_SIZE (<non_vec>mode) - 1);
+})
+
+(define_expand "reduc_plus_scal_<mode>"
+  [(set (match_dup 3)
+	(unspec:V2DI [(match_operand:VI_HW_SD 1 "register_operand")
+		      (match_dup 2)]
+		      UNSPEC_VEC_VSUMQ))
+   (set (match_operand:<non_vec> 0 "register_operand")
+	(vec_select:<non_vec> (match_dup 4)
+			      (parallel [(match_dup 5)])))]
+  "TARGET_VX"
+{
+  operands[2] = force_reg (<MODE>mode, CONST0_RTX (<MODE>mode));
+  operands[3] = gen_reg_rtx (V2DImode);
+  operands[4] = simplify_gen_subreg (<MODE>mode, operands[3], V2DImode, 0);
+  operands[5] = GEN_INT (16 / GET_MODE_SIZE (<non_vec>mode) - 1);
+})
+
+(define_expand "reduc_plus_scal_v2df"
+  [(set (match_dup 2)
+	(unspec:V2DF [(match_operand:V2DF 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3) (plus:V2DF (match_dup 1) (match_dup 2)))
+   (set (match_operand:DF 0 "register_operand")
+	(vec_select:DF (match_dup 3) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V2DFmode);
+  operands[3] = gen_reg_rtx (V2DFmode);
+})
+
+(define_expand "reduc_plus_scal_v4sf"
+  [(set (match_dup 2)
+	(unspec:V4SF [(match_operand:V4SF 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 4)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3) (plus:V4SF (match_dup 1) (match_dup 2)))
+   (set (match_dup 4)
+	(unspec:V4SF [(match_dup 3) (match_dup 3) (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 5) (plus:V4SF (match_dup 3) (match_dup 4)))
+   (set (match_operand:SF 0 "register_operand")
+	(vec_select:SF (match_dup 5) (parallel [(const_int 0)])))]
+  "TARGET_VXE"
+{
+  operands[2] = gen_reg_rtx (V4SFmode);
+  operands[3] = gen_reg_rtx (V4SFmode);
+  operands[4] = gen_reg_rtx (V4SFmode);
+  operands[5] = gen_reg_rtx (V4SFmode);
+})
+
+; reduc_fmin, reduc_fmax, reduc_smin, reduc_smax
+
+(define_int_iterator REDUC_FMINMAX [UNSPEC_FMAX UNSPEC_FMIN])
+(define_int_attr reduc_fminmax_name [(UNSPEC_FMAX "fmax") (UNSPEC_FMIN "fmin")])
+(define_code_iterator REDUC_MINMAX [smin smax])
+(define_code_attr reduc_minmax_name [(smin "smin") (smax "smax")])
+
+(define_expand "reduc_<reduc_fminmax_name>_scal_v2df"
+  [(set (match_dup 2)
+	(unspec:V2DF [(match_operand:V2DF 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(unspec:V2DF [(match_dup 1) (match_dup 2) (const_int 4)] REDUC_FMINMAX))
+   (set (match_operand:DF 0 "register_operand" "")
+	(vec_select:DF (match_dup 3) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V2DFmode);
+  operands[3] = gen_reg_rtx (V2DFmode);
+})
+
+(define_expand "reduc_<reduc_fminmax_name>_scal_v4sf"
+  [(set (match_dup 2)
+	(unspec:V4SF [(match_operand:V4SF 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 4)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(unspec:V4SF [(match_dup 1) (match_dup 2) (const_int 4)] REDUC_FMINMAX))
+   (set (match_dup 4)
+	(unspec:V4SF [(match_dup 3)
+		      (match_dup 3)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 5)
+	(unspec:V4SF [(match_dup 3) (match_dup 4) (const_int 4)] REDUC_FMINMAX))
+   (set (match_operand:SF 0 "register_operand")
+	(vec_select:SF (match_dup 5) (parallel [(const_int 0)])))]
+   "TARGET_VXE"
+{
+  operands[2] = gen_reg_rtx (V4SFmode);
+  operands[3] = gen_reg_rtx (V4SFmode);
+  operands[4] = gen_reg_rtx (V4SFmode);
+  operands[5] = gen_reg_rtx (V4SFmode);
+})
+
+(define_expand "reduc_<reduc_minmax_name>_scal_v2df"
+  [(set (match_dup 2)
+	(unspec:V2DF [(match_operand:V2DF 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(REDUC_MINMAX:V2DF (match_dup 1) (match_dup 2)))
+   (set (match_operand:DF 0 "register_operand" "")
+	(vec_select:DF (match_dup 3) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V2DFmode);
+  operands[3] = gen_reg_rtx (V2DFmode);
+})
+
+(define_expand "reduc_<reduc_minmax_name>_scal_v4sf"
+  [(set (match_dup 2)
+	(unspec:V4SF [(match_operand:V4SF 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 4)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(REDUC_MINMAX:V4SF (match_dup 1) (match_dup 2)))
+   (set (match_dup 4)
+	(unspec:V4SF [(match_dup 3)
+		      (match_dup 3)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 5)
+	(REDUC_MINMAX:V4SF (match_dup 3) (match_dup 4)))
+   (set (match_operand:SF 0 "register_operand" "")
+	(vec_select:SF (match_dup 5) (parallel [(const_int 0)])))]
+   "TARGET_VXE"
+{
+  operands[2] = gen_reg_rtx (V4SFmode);
+  operands[3] = gen_reg_rtx (V4SFmode);
+  operands[4] = gen_reg_rtx (V4SFmode);
+  operands[5] = gen_reg_rtx (V4SFmode);
+})
+
+; reduce_and, reduc_ior, reduc_xor
+; reduc_smin, reduc_smax, reduc_umin, reduc_umax
+
+(define_code_iterator REDUCBIN [and xor ior smin smax umin umax])
+(define_code_attr reduc_bin_insn [(and "and") (xor "xor") (ior "ior")
+				  (smin "smin") (smax "smax")
+				  (umin "umin") (umax "umax")])
+
+(define_expand "reduc_<reduc_bin_insn>_scal_v2di"
+  [(set (match_dup 2)
+	(unspec:V2DI [(match_operand:V2DI 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(REDUCBIN:V2DI (match_dup 1) (match_dup 2)))
+   (set (match_operand:DI 0 "register_operand" "")
+	(vec_select:DI (match_dup 3) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V2DImode);
+  operands[3] = gen_reg_rtx (V2DImode);
+})
+
+(define_expand "reduc_<reduc_bin_insn>_scal_v4si"
+  [(set (match_dup 2)
+	(unspec:V4SI [(match_operand:V4SI 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 4)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(REDUCBIN:V4SI (match_dup 1) (match_dup 2)))
+   (set (match_dup 4)
+	(unspec:V4SI [(match_dup 3)
+		      (match_dup 3)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 5)
+	(REDUCBIN:V4SI (match_dup 3) (match_dup 4)))
+   (set (match_operand:SI 0 "register_operand" "")
+	(vec_select:SI (match_dup 5) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V4SImode);
+  operands[3] = gen_reg_rtx (V4SImode);
+  operands[4] = gen_reg_rtx (V4SImode);
+  operands[5] = gen_reg_rtx (V4SImode);
+})
+
+(define_expand "reduc_<reduc_bin_insn>_scal_v8hi"
+  [(set (match_dup 2)
+	(unspec:V8HI [(match_operand:V8HI 1 "register_operand")
+		      (match_dup 1)
+		      (const_int 2)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(REDUCBIN:V8HI (match_dup 1) (match_dup 2)))
+   (set (match_dup 4)
+	(unspec:V8HI [(match_dup 3)
+		      (match_dup 3)
+		      (const_int 4)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 5)
+	(REDUCBIN:V8HI (match_dup 3) (match_dup 4)))
+   (set (match_dup 6)
+	(unspec:V8HI [(match_dup 5)
+		      (match_dup 5)
+		      (const_int 8)]
+		     UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 7)
+	(REDUCBIN:V8HI (match_dup 5) (match_dup 6)))
+   (set (match_operand:HI 0 "register_operand" "")
+	(vec_select:HI (match_dup 7) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V8HImode);
+  operands[3] = gen_reg_rtx (V8HImode);
+  operands[4] = gen_reg_rtx (V8HImode);
+  operands[5] = gen_reg_rtx (V8HImode);
+  operands[6] = gen_reg_rtx (V8HImode);
+  operands[7] = gen_reg_rtx (V8HImode);
+})
+
+(define_expand "reduc_<reduc_bin_insn>_scal_v16qi"
+  [(set (match_dup 2)
+	(unspec:V16QI [(match_operand:V16QI 1 "register_operand")
+		       (match_dup 1)
+		       (const_int 1)]
+		      UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 3)
+	(REDUCBIN:V16QI (match_dup 1) (match_dup 2)))
+   (set (match_dup 4)
+	(unspec:V16QI [(match_dup 3)
+		       (match_dup 3)
+		       (const_int 2)]
+		      UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 5)
+	(REDUCBIN:V16QI (match_dup 3) (match_dup 4)))
+   (set (match_dup 6)
+	(unspec:V16QI [(match_dup 5)
+		       (match_dup 5)
+		       (const_int 4)]
+		      UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 7)
+	(REDUCBIN:V16QI (match_dup 5) (match_dup 6)))
+   (set (match_dup 8)
+	(unspec:V16QI [(match_dup 7)
+		       (match_dup 7)
+		       (const_int 8)]
+		      UNSPEC_VEC_SLDBYTE))
+   (set (match_dup 9)
+	(REDUCBIN:V16QI (match_dup 7) (match_dup 8)))
+   (set (match_operand:QI 0 "register_operand" "")
+	(vec_select:QI (match_dup 9) (parallel [(const_int 0)])))]
+  "TARGET_VX"
+{
+  operands[2] = gen_reg_rtx (V16QImode);
+  operands[3] = gen_reg_rtx (V16QImode);
+  operands[4] = gen_reg_rtx (V16QImode);
+  operands[5] = gen_reg_rtx (V16QImode);
+  operands[6] = gen_reg_rtx (V16QImode);
+  operands[7] = gen_reg_rtx (V16QImode);
+  operands[8] = gen_reg_rtx (V16QImode);
+  operands[9] = gen_reg_rtx (V16QImode);
+})
diff --git a/gcc/config/s390/vx-builtins.md b/gcc/config/s390/vx-builtins.md
index a7bb7ff..9b89b13 100644
--- a/gcc/config/s390/vx-builtins.md
+++ b/gcc/config/s390/vx-builtins.md
@@ -982,20 +982,18 @@
 
 ; vmhb, vmhh, vmhf, vmhg, vmhq
 (define_insn "vec_smulh<mode>"
-  [(set (match_operand:VIT_HW_VXE3_DT 0 "register_operand"                       "=v")
-	(unspec:VIT_HW_VXE3_DT [(match_operand:VIT_HW_VXE3_DT 1 "register_operand" "v")
-				(match_operand:VIT_HW_VXE3_DT 2 "register_operand" "v")]
-			       UNSPEC_VEC_SMULT_HI))]
+  [(set (match_operand:VIT_HW_VXE3_DT 0 "register_operand"                              "=v")
+	(smul_highpart:VIT_HW_VXE3_DT (match_operand:VIT_HW_VXE3_DT 1 "register_operand" "v")
+				      (match_operand:VIT_HW_VXE3_DT 2 "register_operand" "v")))]
   "TARGET_VX"
   "vmh<bhfgq>\t%v0,%v1,%v2"
   [(set_attr "op_type" "VRR")])
 
 ; vmlhb, vmlhh, vmlhf, vmlhg, vmlhq
 (define_insn "vec_umulh<mode>"
-  [(set (match_operand:VIT_HW_VXE3_DT 0 "register_operand"                       "=v")
-	(unspec:VIT_HW_VXE3_DT [(match_operand:VIT_HW_VXE3_DT 1 "register_operand" "v")
-				(match_operand:VIT_HW_VXE3_DT 2 "register_operand" "v")]
-			       UNSPEC_VEC_UMULT_HI))]
+  [(set (match_operand:VIT_HW_VXE3_DT 0 "register_operand"                              "=v")
+	(umul_highpart:VIT_HW_VXE3_DT (match_operand:VIT_HW_VXE3_DT 1 "register_operand" "v")
+				      (match_operand:VIT_HW_VXE3_DT 2 "register_operand" "v")))]
   "TARGET_VX"
   "vmlh<bhfgq>\t%v0,%v1,%v2"
   [(set_attr "op_type" "VRR")])
@@ -2136,23 +2134,22 @@
   "<vw>fche<sdx>bs\t%v2,%v0,%v1"
   [(set_attr "op_type" "VRR")])
 
-
 (define_insn "vfmin<mode>"
-  [(set (match_operand:VF_HW                0 "register_operand"  "=v")
-	(unspec:VF_HW [(match_operand:VF_HW 1 "register_operand"   "v")
-		       (match_operand:VF_HW 2 "register_operand"   "v")
-		       (match_operand:QI    3 "const_mask_operand" "C")]
-		      UNSPEC_VEC_VFMIN))]
+  [(set (match_operand:VFT_BFP                  0 "register_operand"  "=v")
+	(unspec:VFT_BFP [(match_operand:VFT_BFP 1 "register_operand"   "v")
+		         (match_operand:VFT_BFP 2 "register_operand"   "v")
+		         (match_operand:QI      3 "const_mask_operand" "C")]
+		        UNSPEC_FMIN))]
   "TARGET_VXE"
   "<vw>fmin<sdx>b\t%v0,%v1,%v2,%b3"
   [(set_attr "op_type" "VRR")])
 
 (define_insn "vfmax<mode>"
-  [(set (match_operand:VF_HW                0 "register_operand"  "=v")
-	(unspec:VF_HW [(match_operand:VF_HW 1 "register_operand"   "v")
-		       (match_operand:VF_HW 2 "register_operand"   "v")
-		       (match_operand:QI    3 "const_mask_operand" "C")]
-		      UNSPEC_VEC_VFMAX))]
+  [(set (match_operand:VFT_BFP                  0 "register_operand"  "=v")
+	(unspec:VFT_BFP [(match_operand:VFT_BFP 1 "register_operand"   "v")
+		         (match_operand:VFT_BFP 2 "register_operand"   "v")
+		         (match_operand:QI      3 "const_mask_operand" "C")]
+		        UNSPEC_FMAX))]
   "TARGET_VXE"
   "<vw>fmax<sdx>b\t%v0,%v1,%v2,%b3"
   [(set_attr "op_type" "VRR")])
diff --git a/gcc/config/sh/predicates.md b/gcc/config/sh/predicates.md
index 7349c97..e67ec8a 100644
--- a/gcc/config/sh/predicates.md
+++ b/gcc/config/sh/predicates.md
@@ -630,9 +630,7 @@
 ;; Same as treg_set_expr but disallow constants 0 and 1 which can be loaded
 ;; into the T bit.
 (define_predicate "treg_set_expr_not_const01"
-  (and (match_test "op != const0_rtx")
-       (match_test "op != const1_rtx")
-       (match_operand 0 "treg_set_expr")))
+  (match_test "sh_recog_treg_set_expr_not_01 (op, mode)"))
 
 ;; A predicate describing the T bit register in any form.
 (define_predicate "t_reg_operand"
diff --git a/gcc/config/sh/sh-protos.h b/gcc/config/sh/sh-protos.h
index c8cc19f..e78b669 100644
--- a/gcc/config/sh/sh-protos.h
+++ b/gcc/config/sh/sh-protos.h
@@ -261,6 +261,7 @@ extern rtx_insn* sh_peephole_emit_move_insn (rtx dst, rtx src);
 
 extern bool sh_in_recog_treg_set_expr (void);
 extern bool sh_recog_treg_set_expr (rtx op, machine_mode mode);
+extern bool sh_recog_treg_set_expr_not_01 (rtx op, machine_mode mode);
 
 /* Result value of sh_split_treg_set_expr.  Contains the first insn emitted
    and the optional trailing nott insn.  */
diff --git a/gcc/config/sh/sh.cc b/gcc/config/sh/sh.cc
index 1bc34e0..09e4ff7 100644
--- a/gcc/config/sh/sh.cc
+++ b/gcc/config/sh/sh.cc
@@ -12348,6 +12348,23 @@ sh_recog_treg_set_expr (rtx op, machine_mode mode)
   return result >= 0;
 }
 
+/* Return TRUE if OP is an expression for which there is a pattern to
+   set the T bit unless the expression is trivially loadable into
+   the T bit, FALSE otherwise.  */
+bool
+sh_recog_treg_set_expr_not_01 (rtx op, machine_mode mode)
+{
+  if (op == const0_rtx || op == const1_rtx)
+    return false;
+
+  /* A right shift of 31 will return 0 or 1.  */
+  if ((GET_CODE (op) == LSHIFTRT || GET_CODE (op) == ASHIFTRT)
+      && INTVAL (XEXP (op, 1)) == 31)
+    return false;
+
+  return sh_recog_treg_set_expr (op, mode);
+}
+
 /* Returns true when recog of a 'treg_set_expr' is currently in progress.
    This can be used as a condition for insn/split patterns to allow certain
    T bit setting patters only to be matched as sub expressions of other
diff --git a/gcc/config/vxworks-dummy.h b/gcc/config/vxworks-dummy.h
index 494799d..516728c 100644
--- a/gcc/config/vxworks-dummy.h
+++ b/gcc/config/vxworks-dummy.h
@@ -40,9 +40,21 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 #define TARGET_VXWORKS_RTP false
 #endif
 
+/* True if offsets between different segments may vary, so we must avoid
+   cross-segment GOT- and PC-relative address computations.  */
+#ifndef TARGET_VXWORKS_VAROFF
+#define TARGET_VXWORKS_VAROFF false
+#endif
+
 /* The symbol that points to an RTP's table of GOTs.  */
 #define VXWORKS_GOTT_BASE (gcc_unreachable (), "")
 
 /* The symbol that holds the index of the current module's GOT in
    VXWORKS_GOTT_BASE.  */
 #define VXWORKS_GOTT_INDEX (gcc_unreachable (), "")
+
+/* True if PIC relies on the GOTT_* symbols above.  As of VxWorks7, they are no
+   longer used.  */
+#ifndef TARGET_VXWORKS_GOTTPIC
+#define TARGET_VXWORKS_GOTTPIC false
+#endif
diff --git a/gcc/config/vxworks.h b/gcc/config/vxworks.h
index 204a8e0..d2b6025 100644
--- a/gcc/config/vxworks.h
+++ b/gcc/config/vxworks.h
@@ -159,6 +159,18 @@ extern void vxworks_driver_init (unsigned int *, struct cl_decoded_option **);
    Earlier versions did not, not even for RTPS.  */
 #define VXWORKS_HAVE_TLS TARGET_VXWORKS7
 
+/* RTP segments could be loaded with varying offsets, so cross-segment offsets
+   could not be assumed to be constant.  This rules out some PC- and
+   GOT-relative addressing.  */
+#undef TARGET_VXWORKS_VAROFF
+#define TARGET_VXWORKS_VAROFF (!TARGET_VXWORKS7 && TARGET_VXWORKS_RTP)
+
+/* GOTT_BASE and GOTT_INDEX symbols are only used by some ports up to VxWorks6.
+   This macro is only used by i386 so far.  Other ports seem to keep on using
+   GOTTPIC from VxWorks7 on, but they don't test this macro.  */
+#undef TARGET_VXWORKS_GOTTPIC
+#define TARGET_VXWORKS_GOTTPIC (!TARGET_VXWORKS7)
+
 /* On Vx6 and previous, the libraries to pick up depends on the architecture,
    so cannot be defined for all archs at once.  On Vx7, a VSB is always needed
    and its structure is fixed and does not depend on the arch.  We can thus
diff --git a/gcc/config/xtensa/xtensa.cc b/gcc/config/xtensa/xtensa.cc
index 92a2364..d75cba4 100644
--- a/gcc/config/xtensa/xtensa.cc
+++ b/gcc/config/xtensa/xtensa.cc
@@ -159,6 +159,10 @@ static void xtensa_asm_trampoline_template (FILE *);
 static void xtensa_trampoline_init (rtx, tree, rtx);
 static bool xtensa_output_addr_const_extra (FILE *, rtx);
 static bool xtensa_cannot_force_const_mem (machine_mode, rtx);
+static machine_mode xtensa_promote_function_mode (const_tree,
+						  machine_mode,
+						  int *, const_tree,
+						  int);
 
 static reg_class_t xtensa_preferred_reload_class (rtx, reg_class_t);
 static reg_class_t xtensa_preferred_output_reload_class (rtx, reg_class_t);
@@ -235,9 +239,7 @@ static HARD_REG_SET xtensa_zero_call_used_regs (HARD_REG_SET);
 #define TARGET_EXPAND_BUILTIN_VA_START xtensa_va_start
 
 #undef TARGET_PROMOTE_FUNCTION_MODE
-#define TARGET_PROMOTE_FUNCTION_MODE default_promote_function_mode_always_promote
-#undef TARGET_PROMOTE_PROTOTYPES
-#define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
+#define TARGET_PROMOTE_FUNCTION_MODE xtensa_promote_function_mode
 
 #undef TARGET_RETURN_IN_MEMORY
 #define TARGET_RETURN_IN_MEMORY xtensa_return_in_memory
@@ -421,12 +423,13 @@ xtensa_uimm8x4 (HOST_WIDE_INT v)
 }
 
 
-static bool
-xtensa_b4const (HOST_WIDE_INT v)
+bool
+xtensa_b4const_or_zero (HOST_WIDE_INT v)
 {
   switch (v)
     {
     case -1:
+    case 0:
     case 1:
     case 2:
     case 3:
@@ -449,15 +452,6 @@ xtensa_b4const (HOST_WIDE_INT v)
 
 
 bool
-xtensa_b4const_or_zero (HOST_WIDE_INT v)
-{
-  if (v == 0)
-    return true;
-  return xtensa_b4const (v);
-}
-
-
-bool
 xtensa_b4constu (HOST_WIDE_INT v)
 {
   switch (v)
@@ -607,8 +601,8 @@ constantpool_address_p (const_rtx addr)
 
       /* Make sure the address is word aligned.  */
       offset = XEXP (addr, 1);
-      if ((!CONST_INT_P (offset))
-	  || ((INTVAL (offset) & 3) != 0))
+      if (! CONST_INT_P (offset)
+	  || (INTVAL (offset) & 3) != 0)
 	return false;
 
       sym = XEXP (addr, 0);
@@ -617,6 +611,7 @@ constantpool_address_p (const_rtx addr)
   if (SYMBOL_REF_P (sym)
       && CONSTANT_POOL_ADDRESS_P (sym))
     return true;
+
   return false;
 }
 
@@ -4510,7 +4505,8 @@ xtensa_rtx_costs (rtx x, machine_mode mode, int outer_code,
 	    }
 	  break;
 	case COMPARE:
-	  if ((INTVAL (x) == 0) || xtensa_b4const (INTVAL (x)))
+	  if (xtensa_b4const_or_zero (INTVAL (x))
+	      || xtensa_b4constu (INTVAL (x)))
 	    {
 	      *total = 0;
 	      return true;
@@ -4699,29 +4695,56 @@ xtensa_rtx_costs (rtx x, machine_mode mode, int outer_code,
     }
 }
 
+/* Return TRUE if the specified insn corresponds to one or more L32R machine
+   instructions.  */
+
 static bool
 xtensa_is_insn_L32R_p (const rtx_insn *insn)
 {
-  rtx x = PATTERN (insn);
+  rtx pat, dest, src;
+  machine_mode mode;
 
-  if (GET_CODE (x) != SET)
+  /* RTX insns that are not "(set (reg) ...)" cannot become L32R instructions:
+     - it is permitted to apply PATTERN() to the insn without validation.
+       See insn_cost() in gcc/rtlanal.cc.
+     - it is used register_operand() instead of REG() to identify things that
+       don't look like REGs but will eventually become so as well.  */
+  if (GET_CODE (pat = PATTERN (insn)) != SET
+      || ! register_operand (dest = SET_DEST (pat), VOIDmode))
     return false;
 
-  x = XEXP (x, 1);
-  if (MEM_P (x))
-    {
-      x = XEXP (x, 0);
-      return (SYMBOL_REF_P (x) || CONST_INT_P (x))
-	     && CONSTANT_POOL_ADDRESS_P (x);
-    }
-
-  /* relaxed MOVI instructions, that will be converted to L32R by the
-     assembler.  */
-  if (CONST_INT_P (x)
-      && ! xtensa_simm12b (INTVAL (x)))
+  /* If the source is a reference to a literal pool entry, then the insn
+     obviously corresponds to an L32R instruction.  */
+  if (constantpool_mem_p (src = SET_SRC (pat)))
     return true;
 
-  return false;
+  /* Similarly, an insn whose source is not a constant obviously does not
+     correspond to L32R.  */
+  if (! CONSTANT_P (src))
+    return false;
+
+  /* If the source is a CONST_INT whose value fits into signed 12 bits, then
+     the insn corresponds to a MOVI instruction (rather than an L32R one),
+     regardless of the configuration of TARGET_CONST16 or
+     TARGET_AUTOLITPOOLS.  Note that the destination register can be non-
+     SImode.  */
+  if (((mode = GET_MODE (dest)) == SImode
+       || mode == HImode || mode == SFmode)
+      && CONST_INT_P (src) && xtensa_simm12b (INTVAL (src)))
+    return false;
+
+  /* If TARGET_CONST16 is configured, constants of the remaining forms
+     correspond to pairs of CONST16 instructions, not L32R.  */
+  if (TARGET_CONST16)
+    return false;
+
+  /* The last remaining form of constant is one of the following:
+     - CONST_INTs with large values
+     - floating-point constants
+     - symbolic constants
+     and is all handled by a relaxed MOVI instruction, which is later
+     converted to an L32R instruction by the assembler.  */
+  return true;
 }
 
 /* Compute a relative costs of RTL insns.  This is necessary in order to
@@ -4730,7 +4753,7 @@ xtensa_is_insn_L32R_p (const rtx_insn *insn)
 static int
 xtensa_insn_cost (rtx_insn *insn, bool speed)
 {
-  if (!(recog_memoized (insn) < 0))
+  if (! (recog_memoized (insn) < 0))
     {
       int len = get_attr_length (insn);
 
@@ -4743,7 +4766,7 @@ xtensa_insn_cost (rtx_insn *insn, bool speed)
 
 	  /* "L32R" may be particular slow (implementation-dependent).  */
 	  if (xtensa_is_insn_L32R_p (insn))
-	    return COSTS_N_INSNS (1 + xtensa_extra_l32r_costs);
+	    return COSTS_N_INSNS ((1 + xtensa_extra_l32r_costs) * n);
 
 	  /* Cost based on the pipeline model.  */
 	  switch (get_attr_type (insn))
@@ -4788,7 +4811,7 @@ xtensa_insn_cost (rtx_insn *insn, bool speed)
 	    {
 	      /* "L32R" itself plus constant in litpool.  */
 	      if (xtensa_is_insn_L32R_p (insn))
-		len = 3 + 4;
+		len += (len / 3) * 4;
 
 	      /* Consider fractional instruction length (for example, ".n"
 		 short instructions or "L32R" litpool constants.  */
@@ -4801,6 +4824,19 @@ xtensa_insn_cost (rtx_insn *insn, bool speed)
   return pattern_cost (PATTERN (insn), speed);
 }
 
+/* Worker function for TARGET_PROMOTE_FUNCTION_MODE.  */
+
+static machine_mode
+xtensa_promote_function_mode (const_tree type, machine_mode mode,
+			      int *punsignedp, const_tree, int)
+{
+  if (GET_MODE_CLASS (mode) == MODE_INT
+      && GET_MODE_SIZE (mode) < GET_MODE_SIZE (SImode))
+    return SImode;
+
+  return promote_mode (type, mode, punsignedp);
+}
+
 /* Worker function for TARGET_RETURN_IN_MEMORY.  */
 
 static bool
diff --git a/gcc/config/xtensa/xtensa.md b/gcc/config/xtensa/xtensa.md
index 029be99..629dfdd 100644
--- a/gcc/config/xtensa/xtensa.md
+++ b/gcc/config/xtensa/xtensa.md
@@ -1297,7 +1297,10 @@
       std::swap (operands[0], operands[1]);
       std::swap (operands[2], operands[3]);
     }
-})
+}
+  [(set_attr "type"	"move,move,load,load,store")
+   (set_attr "mode"	"DI")
+   (set_attr "length"	"6,12,6,6,6")])
 
 (define_split
   [(set (match_operand:DI 0 "register_operand")
@@ -1344,7 +1347,7 @@
    %v0s32i\t%1, %0
    rsr\t%0, ACCLO
    wsr\t%1, ACCLO"
-  [(set_attr "type"	"move,move,move,load,store,store,move,move,move,move,move,load,load,store,rsr,wsr")
+  [(set_attr "type"	"move,move,move,load,store,store,move,move,move,load,move,load,load,store,rsr,wsr")
    (set_attr "mode"	"SI")
    (set_attr "length"	"2,2,2,2,2,2,3,3,3,3,6,3,3,3,3,3")])
 
@@ -1410,7 +1413,7 @@
    %v0s16i\t%1, %0
    rsr\t%0, ACCLO
    wsr\t%1, ACCLO"
-  [(set_attr "type"	"move,move,move,move,move,load,load,store,rsr,wsr")
+  [(set_attr "type"	"move,move,move,move,load,load,load,store,rsr,wsr")
    (set_attr "mode"	"HI")
    (set_attr "length"	"2,2,3,3,3,3,3,3,3,3")])
 
@@ -1519,7 +1522,7 @@
    const16\t%0, %t1\;const16\t%0, %b1
    %v1l32i\t%0, %1
    %v0s32i\t%1, %0"
-  [(set_attr "type"	"farith,fload,fstore,move,load,load,store,move,farith,farith,move,move,load,store")
+  [(set_attr "type"	"farith,fload,fstore,move,load,load,store,move,farith,farith,load,move,load,store")
    (set_attr "mode"	"SF")
    (set_attr "length"	"3,3,3,2,3,2,2,3,3,3,3,6,3,3")])
 
@@ -1643,7 +1646,10 @@
       std::swap (operands[0], operands[1]);
       std::swap (operands[2], operands[3]);
     }
-})
+}
+  [(set_attr "type"	"move,load,move,load,load,store")
+   (set_attr "mode"	"DF")
+   (set_attr "length"	"6,6,12,6,6,6")])
 
 ;; Block moves
 
diff --git a/gcc/configure b/gcc/configure
index f056cfe..bacdd29 100755
--- a/gcc/configure
+++ b/gcc/configure
@@ -1064,6 +1064,7 @@ enable_versioned_jit
 enable_default_pie
 enable_cet
 enable_s390_excess_float_precision
+enable_x86_64_mfentry
 '
       ac_precious_vars='build_alias
 host_alias
@@ -1842,6 +1843,7 @@ Optional Features:
   --enable-s390-excess-float-precision
                           on s390 targets, evaluate float with double
                           precision when in standards-conforming mode
+  --enable-x86-64-mfentry enable -mfentry by default on x86-64 targets
 
 Optional Packages:
   --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
@@ -21520,7 +21522,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 21523 "configure"
+#line 21525 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -21626,7 +21628,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 21629 "configure"
+#line 21631 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -35022,6 +35024,46 @@ $as_echo "#define ENABLE_S390_EXCESS_FLOAT_PRECISION 1" >>confdefs.h
   ;;
 esac
 
+# On x86-64, when profiling is enabled with shrink wrapping, the mcount
+# call may not be placed at the function entry after
+#	pushq %rbp
+#	movq %rsp,%rbp
+# As the result, the profile data may be skewed which makes PGO less
+# effective:
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=120881
+# Enable -mfentry by default on x86-64 to put the profiling counter call
+# before the prologue.
+# Check whether --enable-x86-64-mfentry was given.
+if test "${enable_x86_64_mfentry+set}" = set; then :
+  enableval=$enable_x86_64_mfentry; case "${enableval}" in
+   yes | no | auto)
+     enable_x86_64_mfentry=$enableval
+     ;;
+   *)
+     as_fn_error $? "'$enable_x86_64_mfentry' is an invalid value for --enable-x86-64-mfentry.  Valid choices are 'yes', 'no' and 'auto'." "$LINENO" 5
+     ;;
+   esac
+else
+  enable_x86_64_mfentry=auto
+fi
+
+
+if test x"$enable_x86_64_mfentry" = xauto; then
+  case "${target}" in
+  i?86-*-*gnu* | x86_64-*-*gnu*)
+    # Enable -mfentry by default with glibc on x86.
+    enable_x86_64_mfentry=yes
+    ;;
+  esac
+fi
+
+gif=`if test x$enable_x86_64_mfentry = xyes; then echo 1; else echo 0; fi`
+
+cat >>confdefs.h <<_ACEOF
+#define ENABLE_X86_64_MFENTRY $gif
+_ACEOF
+
+
 # Check if the linker supports '-z now'
 ld_now_support=no
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking linker -z now option" >&5
@@ -36456,7 +36498,7 @@ $as_echo "$as_me: executing $ac_file commands" >&6;}
     "depdir":C) $SHELL $ac_aux_dir/mkinstalldirs $DEPDIR ;;
     "gccdepdir":C)
   ${CONFIG_SHELL-/bin/sh} $ac_aux_dir/mkinstalldirs build/$DEPDIR
-  for lang in $subdirs c-family common analyzer text-art rtl-ssa sym-exec
+  for lang in $subdirs c-family common analyzer diagnostics text-art rtl-ssa sym-exec
   do
       ${CONFIG_SHELL-/bin/sh} $ac_aux_dir/mkinstalldirs $lang/$DEPDIR
   done ;;
diff --git a/gcc/configure.ac b/gcc/configure.ac
index 58bf63f..2c43b38 100644
--- a/gcc/configure.ac
+++ b/gcc/configure.ac
@@ -1397,7 +1397,7 @@ AC_CHECK_HEADERS(ext/hash_map)
 ZW_CREATE_DEPDIR
 AC_CONFIG_COMMANDS([gccdepdir],[
   ${CONFIG_SHELL-/bin/sh} $ac_aux_dir/mkinstalldirs build/$DEPDIR
-  for lang in $subdirs c-family common analyzer text-art rtl-ssa sym-exec
+  for lang in $subdirs c-family common analyzer diagnostics text-art rtl-ssa sym-exec
   do
       ${CONFIG_SHELL-/bin/sh} $ac_aux_dir/mkinstalldirs $lang/$DEPDIR
   done], [subdirs="$subdirs" ac_aux_dir=$ac_aux_dir DEPDIR=$DEPDIR])
@@ -7972,6 +7972,41 @@ standards-compatible mode on s390 targets.])
   ;;
 esac
 
+# On x86-64, when profiling is enabled with shrink wrapping, the mcount
+# call may not be placed at the function entry after
+#	pushq %rbp
+#	movq %rsp,%rbp
+# As the result, the profile data may be skewed which makes PGO less
+# effective:
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=120881
+# Enable -mfentry by default on x86-64 to put the profiling counter call
+# before the prologue.
+AC_ARG_ENABLE(x86-64-mfentry,
+  [AS_HELP_STRING([--enable-x86-64-mfentry],
+		  [enable -mfentry by default on x86-64 targets])],
+  [case "${enableval}" in
+   yes | no | auto)
+     enable_x86_64_mfentry=$enableval
+     ;;
+   *)
+     AC_MSG_ERROR(['$enable_x86_64_mfentry' is an invalid value for --enable-x86-64-mfentry.  Valid choices are 'yes', 'no' and 'auto'.])
+     ;;
+   esac],
+  [enable_x86_64_mfentry=auto])
+
+if test x"$enable_x86_64_mfentry" = xauto; then
+  case "${target}" in
+  i?86-*-*gnu* | x86_64-*-*gnu*)
+    # Enable -mfentry by default with glibc on x86.
+    enable_x86_64_mfentry=yes
+    ;;
+  esac
+fi
+
+gif=`if test x$enable_x86_64_mfentry = xyes; then echo 1; else echo 0; fi`
+AC_DEFINE_UNQUOTED(ENABLE_X86_64_MFENTRY, $gif,
+[Define to enable -mfentry by default on x86-64.])
+
 # Check if the linker supports '-z now'
 ld_now_support=no
 AC_MSG_CHECKING(linker -z now option)
diff --git a/gcc/coretypes.h b/gcc/coretypes.h
index a11ebd1..b6a922a 100644
--- a/gcc/coretypes.h
+++ b/gcc/coretypes.h
@@ -164,8 +164,10 @@ struct cl_option;
 struct cl_decoded_option;
 struct cl_option_handlers;
 class rich_location;
-class diagnostic_context;
-class diagnostic_text_output_format;
+namespace diagnostics {
+  class context;
+  class text_sink;
+} // namespace diagnostics
 class pretty_printer;
 class diagnostic_event_id_t;
 typedef const char * (*diagnostic_input_charset_callback)(const char *);
diff --git a/gcc/coverage.cc b/gcc/coverage.cc
index dd3ed2e..75a24c6 100644
--- a/gcc/coverage.cc
+++ b/gcc/coverage.cc
@@ -238,6 +238,7 @@ read_counts_file (void)
 	  gcov_profile_info = profile_info = XCNEW (gcov_summary);
 	  profile_info->runs = gcov_read_unsigned ();
 	  profile_info->sum_max = gcov_read_unsigned ();
+	  profile_info->cutoff = 1;
 	}
       else if (GCOV_TAG_IS_COUNTER (tag) && fn_ident)
 	{
diff --git a/gcc/cp/ChangeLog b/gcc/cp/ChangeLog
index ff7582d..3f76afd 100644
--- a/gcc/cp/ChangeLog
+++ b/gcc/cp/ChangeLog
@@ -1,3 +1,677 @@
+2025-08-01  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/108080
+	* module.cc (trees_out::core_vals): Warn when streaming
+	target/optimize node; adjust comments.
+	(trees_in::core_vals): Don't stream a target/optimize node.
+
+2025-08-01  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/121238
+	* module.cc (trees_in::fn_parms_fini): Merge properties for
+	definitions.
+
+2025-07-31  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120800
+	* constexpr.cc (cxx_eval_vec_init_1): Suppress access control.
+
+2025-07-31  Marek Polacek  <polacek@redhat.com>
+
+	PR c++/120775
+	* constexpr.cc (cxx_eval_outermost_constant_expr): Use
+	extract_call_expr.
+	* cp-tree.h (CONSTEVAL_BLOCK_P, LAMBDA_EXPR_CONSTEVAL_BLOCK_P): Define.
+	(finish_static_assert): Adjust declaration.
+	(current_nonlambda_function): Likewise.
+	* lambda.cc (current_nonlambda_function): New parameter.  Only keep
+	iterating if the function represents a consteval block.
+	* parser.cc (cp_parser_lambda_expression): New parameter for
+	consteval blocks.  Use it.  Set LAMBDA_EXPR_CONSTEVAL_BLOCK_P.
+	(cp_parser_lambda_declarator_opt): Likewise.
+	(build_empty_string): New.
+	(cp_parser_next_tokens_are_consteval_block_p): New.
+	(cp_parser_consteval_block): New.
+	(cp_parser_block_declaration): Handle consteval blocks.
+	(cp_parser_static_assert): Use build_empty_string.
+	(cp_parser_member_declaration): Handle consteval blocks.
+	* pt.cc (tsubst_stmt): Adjust a call to finish_static_assert.
+	* semantics.cc (finish_fname): Warn for consteval blocks.
+	(finish_static_assert): New parameter for consteval blocks.  Set
+	CONSTEVAL_BLOCK_P.  Evaluate consteval blocks specially.
+
+2025-07-30  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/121291
+	* constraint.cc (diagnose_trait_expr): Remove assumption about
+	failures returning error_mark_node.
+	* except.cc (explain_not_noexcept): Allow expr not being
+	noexcept.
+	* method.cc (build_invoke): Adjust comment.
+	(is_trivially_xible): Always note non-trivial components if expr
+	is not null or error_mark_node.
+	(is_nothrow_xible): Likewise for non-noexcept components.
+	(is_nothrow_convertible): Likewise.
+
+2025-07-30  Jason Merrill  <jason@redhat.com>
+
+	* pt.cc (convert_nontype_argument_function): Check
+	cxx_constant_value on failure.
+	(invalid_tparm_referent_p): Likewise.
+
+2025-07-30  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/121133
+	* parser.cc (cp_parser_unary_expression): Adjust
+	cp_parser_extension_opt caller and restore warn_long_long.
+	(cp_parser_declaration): Likewise.
+	(cp_parser_block_declaration): Likewise.
+	(cp_parser_member_declaration): Likewise.
+	(cp_parser_extension_opt): Add SAVED_LONG_LONG argument,
+	save previous warn_long_long state into it and clear it
+	for __extension__.
+
+2025-07-27  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	* cp-tree.h (struct lang_type): Add comment mentioning modules.
+	* module.cc (trees_out::lang_type_bools): Stream new flags, use
+	gcc_checking_assert.
+	(trees_in::lang_type_bools): Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* constexpr.cc: Update usage of "diagnostic_info" to explicitly
+	refer to "diagnostics::diagnostic_info".
+	* cp-tree.h: Likewise.
+	* error.cc: Likewise.
+	* module.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* call.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* constexpr.cc: Likewise.
+	* cp-tree.h: Likewise.
+	* decl.cc: Likewise.
+	* error.cc: Likewise.
+	* init.cc: Likewise.
+	* method.cc: Likewise.
+	* module.cc: Likewise.
+	* parser.cc: Likewise.
+	* pt.cc: Likewise.
+	* semantics.cc: Likewise.
+	* typeck.cc: Likewise.
+	* typeck2.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* cp-tree.h: Update for renaming of diagnostic_option_id to
+	diagnostics::option_id.
+	* decl.cc: Likewise.
+	* error.cc: Likewise.
+	* name-lookup.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* error.cc: Update for move of diagnostic-color.h to
+	diagnostics/color.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* cp-tree.h: Update for diagnostic_context becoming
+	diagnostics::context.
+	* error.cc: Likewise.
+	* module.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* constexpr.cc: Update to add "m_" prefix to fields of
+	diagnostic_info throughout.
+	* error.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* cp-tree.h: Update for move of diagnostics output formats into
+	namespace "diagnostics" as "sinks".
+	* error.cc: Likewise.
+
+2025-07-25  Patrick Palka  <ppalka@redhat.com>
+
+	* call.cc (build_new_op): If the selected candidate is
+	rewritten, communicate the LOOKUP_REWRITTEN/REVERSED flags to
+	the caller via the 'overload' out-parameter, and stop clearing
+	'*overload' in that case.
+	* tree.cc (build_min_non_dep_op_overload): Handle rebuilding all
+	C++20 rewritten comparison operator expressions.
+
+2025-07-25  Iain Sandoe  <iain@sandoe.co.uk>
+
+	PR c++/121219
+	* coroutines.cc
+	(cp_coroutine_transform::build_ramp_function): Reorder the return
+	expressions for the 'normal' and 'allocation failed' cases so that
+	NRV constraints are met.
+
+2025-07-24  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/117294
+	PR c++/113854
+	* call.cc (implicit_conversion_error): Hide label when printing
+	a stub object.
+	(convert_like_internal): Likewise, and nest candidate
+	diagnostics.
+	* constexpr.cc (diagnose_failing_condition): Nest diagnostics,
+	attempt to provide more helpful diagnostics for traits.
+	* constraint.cc (satisfy_atom): Pass result before constant
+	evaluation to diagnose_atomic_constraint.
+	(diagnose_trait_expr): Adjust diagnostics for clarity and
+	detail.
+	(maybe_diagnose_standard_trait): New function.
+	(diagnose_atomic_constraint): Attempt to provide more helpful
+	diagnostics for more traits.
+	* cp-tree.h (explain_not_noexcept): Declare new function.
+	(is_trivially_xible): Add parameter.
+	(is_nothrow_xible): Likewise.
+	(is_xible): Likewise.
+	(is_convertible): Likewise.
+	(is_nothrow_convertible): Likewise.
+	(diagnose_trait_expr): Declare new function.
+	(maybe_diagnose_standard_trait): Declare new function.
+	* error.cc (dump_type) <case TREE_VEC>: Handle trait types.
+	* except.cc (explain_not_noexcept): New function.
+	* method.cc (build_trait_object): Add complain parameter.
+	(build_invoke): Propagate complain parameter.
+	(assignable_expr): Add explain parameter to show diagnostics.
+	(constructible_expr): Likewise.
+	(destructible_expr): Likewise.
+	(is_xible_helper): Replace trivial flag with explain flag,
+	add diagnostics.
+	(is_trivially_xible): New explain flag.
+	(is_nothrow_xible): Likewise.
+	(is_xible): Likewise.
+	(is_convertible_helper): Add complain flag.
+	(is_convertible): New explain flag.
+	(is_nothrow_convertible): Likewise.
+	* typeck.cc (cp_build_function_call_vec): Add handling for stub
+	objects.
+	(convert_arguments): Always return -1 on error.
+	* typeck2.cc (cxx_readonly_error): Add handling for stub
+	objects.
+
+2025-07-24  Jason Merrill  <jason@redhat.com>
+
+	* pt.cc (tsubst_lambda_expr): Revert r9-5971 change.
+
+2025-07-24  Jason Merrill  <jason@redhat.com>
+
+	PR c++/114632
+	PR c++/101233
+	* lambda.cc (maybe_add_lambda_conv_op): Not for xobj lambda.
+	* pt.cc (tsubst_function_decl): Add cp_evaluated.
+	(alias_ctad_tweaks): Revert PR101233 fix.
+
+2025-07-24  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/120412
+	* module.cc (trees_out::core_vals): Write TU_LOCAL_ENTITY bits.
+	(trees_in::core_vals): Read it.
+	(trees_in::tree_node): Handle TU_LOCAL_ENTITY typedefs.
+
+2025-07-23  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/121179
+	* call.cc (build_new_op): Don't clear *overload for a simple
+	!= to == rewrite.
+	* tree.cc (build_min_non_dep_op_overload): Handle TRUTH_NOT_EXPR
+	appearing in a rewritten operator expression.
+
+2025-07-23  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/121055
+	* method.cc (build_invoke): Correct reference_wrapper handling.
+
+2025-07-22  Jason Merrill  <jason@redhat.com>
+
+	PR c++/121068
+	* constexpr.cc (cxx_eval_store_expression): Allow ARRAY_REFs
+	when activating an array member of a union.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* semantics.cc (finish_asm_stmt): Pass null pointer to
+	parse_{input,output}_constraint().
+
+2025-07-16  Kwok Cheung Yeung  <kcyeung@baylibre.com>
+
+	* pt.cc (tsubst_omp_clause_decl): Use OMP_ITERATOR_DECL_P.
+	* semantics.cc (handle_omp_array_sections): Likewise.
+	(finish_omp_clauses): Likewise.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* class.cc (add_method): Remove argument.
+	* cp-tree.h (maybe_version_functions): Ditto.
+	* decl.cc (decls_match): Ditto.
+	(maybe_version_functions): Ditto.
+
+2025-07-16  Jeremy Rifkin  <jeremy@rifkin.dev>
+
+	PR c/82134
+	* call.cc (build_call_a): Add suppress_warning
+	* cp-gimplify.cc (cp_gimplify_expr): Add suppress_warning
+
+2025-07-15  Jason Merrill  <jason@redhat.com>
+
+	PR c++/44677
+	* cp-gimplify.cc (cp_fold) [CLEANUP_POINT_EXPR]: Don't force rvalue.
+	[COMPOUND_EXPR]: Likewise.
+	* cvt.cc (convert_to_void): Call mark_exp_read later.
+	* expr.cc (mark_use): Turn off read_p for any void argument.
+	(mark_exp_read): Return early for void argument.
+
+2025-07-15  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120577
+	* constexpr.cc (cxx_eval_call_expression): Set
+	CONSTRUCTOR_NO_CLEARING on initial value for ctor.
+	(cxx_eval_component_reference): Make value-initialization
+	of an aggregate member explicit.
+
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* cp-gimplify.cc (cp_fold): Clear DECL_READ_P on lhs of MODIFY_EXPR
+	after cp_fold_rvalue if it wasn't set before.
+	* decl.cc (poplevel): Use OPT_Wunused_but_set_variable_
+	instead of OPT_Wunused_but_set_variable.
+	(finish_function): Use OPT_Wunused_but_set_parameter_
+	instead of OPT_Wunused_but_set_parameter.
+	* expr.cc (mark_use): Clear read_p for {PRE,POST}{IN,DE}CREMENT_EXPR
+	cast to void on {VAR,PARM}_DECL for
+	-Wunused-but-set-{parameter,variable}={2,3}.
+	(mark_exp_read): Handle {PRE,POST}{IN,DE}CREMENT_EXPR and don't handle
+	it when cast to void.
+	* module.cc (trees_in::fn_parms_fini): Remove unused but set variable
+	ix.
+	* semantics.cc (finish_unary_op_expr): Return early for
+	PRE{IN,DE}CREMENT_EXPR.
+	* typeck.cc (cp_build_unary_op): Clear DECL_READ_P
+	after mark_lvalue_use for -Wunused-but-set-{parameter,variable}={2,3}
+	on PRE{IN,DE}CREMENT_EXPR argument.
+	(cp_build_modify_expr): Clear DECL_READ_P after cp_build_binary_op
+	for -Wunused-but-set-{parameter,variable}=3.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/119064
+	* cp-tree.h: Implement C++26 P2786R13 - Trivial Relocatability.
+	(struct lang_type): Add trivially_relocatable,
+	trivially_relocatable_computed, replaceable and replaceable_computed
+	bitfields.  Change width of dummy from 2 to 30.
+	(CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT,
+	CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED, CLASSTYPE_REPLACEABLE_BIT,
+	CLASSTYPE_REPLACEABLE_COMPUTED): Define.
+	(enum virt_specifier): Add VIRT_SPEC_TRIVIALLY_RELOCATABLE_IF_ELIGIBLE
+	and VIRT_SPEC_REPLACEABLE_IF_ELIGIBLE enumerators.
+	(trivially_relocatable_type_p, replaceable_type_p): Declare.
+	* cp-trait.def (IS_NOTHROW_RELOCATABLE, IS_REPLACEABLE,
+	IS_TRIVIALLY_RELOCATABLE): New traits.
+	* parser.cc (cp_parser_class_property_specifier_seq_opt): Handle
+	trivially_relocatable_if_eligible,
+	__trivially_relocatable_if_eligible, replaceable_if_eligible and
+	__replaceable_if_eligible.
+	(cp_parser_class_head): Set CLASSTYPE_REPLACEABLE_BIT
+	and/or CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT if corresponding
+	conditional keywords were parsed and assert corresponding *_COMPUTED
+	macro is false.
+	* pt.cc (instantiate_class_template): Copy over also
+	CLASSTYPE_TRIVIALLY_RELOCATABLE_{BIT,COMPUTED} and
+	CLASSTYPE_REPLACEABLE_{BIT,COMPUTED} bits.
+	* semantics.cc (referenceable_type_p): Move definition earlier.
+	(trait_expr_value): Handle CPTK_IS_NOTHROW_RELOCATABLE,
+	CPTK_IS_REPLACEABLE and CPTK_IS_TRIVIALLY_RELOCATABLE.
+	(finish_trait_expr): Likewise.
+	* tree.cc (default_movable_type_p): New function.
+	(union_with_no_declared_special_member_fns): Likewise.
+	(trivially_relocatable_type_p): Likewise.
+	(replaceable_type_p): Likewise.
+	* constraint.cc (diagnose_trait_expr): Handle
+	CPTK_IS_NOTHROW_RELOCATABLE, CPTK_IS_REPLACEABLE and
+	CPTK_IS_TRIVIALLY_RELOCATABLE.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	* cp-tree.h (struct lang_type): Add comment before key_method.
+	Remove lambda_expr.
+	(CLASSTYPE_KEY_METHOD): Give NULL_TREE if not TYPE_POLYMORPHIC_P.
+	(SET_CLASSTYPE_KEY_METHOD): Define.
+	(CLASSTYPE_LAMBDA_EXPR): Give NULL_TREE if TYPE_POLYMORPHIC_P.
+	Use key_method member instead of lambda_expr.
+	(SET_CLASSTYPE_LAMBDA_EXPR): Define.
+	* class.cc (determine_key_method): Use SET_CLASSTYPE_KEY_METHOD
+	macro.
+	* decl.cc (xref_tag): Use SET_CLASSTYPE_LAMBDA_EXPR macro.
+	* lambda.cc (begin_lambda_type): Likewise.
+	* module.cc (trees_in::read_class_def): Use SET_CLASSTYPE_LAMBDA_EXPR
+	and SET_CLASSTYPE_KEY_METHOD macros, assert lambda is NULL if
+	TYPE_POLYMORPHIC_P and otherwise assert key_method is NULL.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120628
+	* parser.cc (cp_parser_elaborated_type_specifier): Use
+	cp_parser_nth_token_starts_class_definition_p with extra argument 1
+	instead of cp_parser_next_token_starts_class_definition_p.
+	(cp_parser_class_property_specifier_seq_opt): For final conditional
+	keyword in C++98 check if the token after it isn't
+	cp_parser_nth_token_starts_class_definition_p nor CPP_NAME and in
+	that case break without consuming it nor warning.
+	(cp_parser_class_head): Use
+	cp_parser_nth_token_starts_class_definition_p with extra argument 1
+	instead of cp_parser_next_token_starts_class_definition_p.
+	(cp_parser_next_token_starts_class_definition_p): Renamed to ...
+	(cp_parser_nth_token_starts_class_definition_p): ... this.  Add N
+	argument.  Use cp_lexer_peek_nth_token instead of cp_lexer_peek_token.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120569
+	* parser.cc (cp_parser_class_property_specifier_seq_opt): New
+	function.
+	(cp_parser_class_head): Use it instead of
+	cp_parser_property_specifier_seq_opt.  Don't diagnose
+	VIRT_SPEC_OVERRIDE here.  Formatting fix.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/117785
+	* constexpr.cc: Implement C++26 P3068R5 - constexpr exceptions.
+	(class constexpr_global_ctx): Add caught_exceptions and
+	uncaught_exceptions members.
+	(constexpr_global_ctx::constexpr_global_ctx): Initialize
+	uncaught_exceptions.
+	(returns, breaks, continues, switches): Move earlier.
+	(throws): New function.
+	(exception_what_str, diagnose_std_terminate,
+	diagnose_uncaught_exception): New functions.
+	(enum cxa_builtin): New type.
+	(cxx_cxa_builtin_fn_p, cxx_eval_cxa_builtin_fn): New functions.
+	(cxx_eval_builtin_function_call): Add jump_target argument.  Call
+	cxx_eval_cxa_builtin_fn for __builtin_eh_ptr_adjust_ref.  Adjust
+	cxx_eval_constant_expression calls, if it results in jmp_target,
+	set *jump_target to it and return.
+	(cxx_bind_parameters_in_call): Add jump_target argument.  Pass
+	it through to cxx_eval_constant_expression.  If it sets *jump_target,
+	break.
+	(fold_operand): Adjust cxx_eval_constant_expression caller.
+	(cxx_eval_assert): Likewise.  If it set jmp_target, return true.
+	(cxx_eval_internal_function): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression.  Return early if *jump_target
+	after recursing on args.
+	(cxx_eval_dynamic_cast_fn): Likewise.  Don't set reference_p for
+	C++26 with -fexceptions.
+	(cxx_eval_thunk_call): Add jump_target argument.  Pass it through
+	to cxx_eval_constant_expression.
+	(cxx_set_object_constness): Likewise.  Don't set TREE_READONLY if
+	throws (jump_target).
+	(cxx_eval_call_expression): Add jump_target argument.  Pass it
+	through to cxx_eval_internal_function, cxx_eval_builtin_function_call,
+	cxx_eval_thunk_call, cxx_eval_dynamic_cast_fn and
+	cxx_set_object_constness.  Pass it through also
+	cxx_eval_constant_expression on arguments, cxx_bind_parameters_in_call
+	and cxx_fold_indirect_ref and for those cases return early
+	if *jump_target.  Call cxx_eval_cxa_builtin_fn for cxx_cxa_builtin_fn_p
+	functions.  For cxx_eval_constant_expression on body, pass address of
+	cleared jmp_target automatic variable, if it throws propagate
+	to *jump_target and make it non-cacheable.  For C++26 don't diagnose
+	calls to non-constexpr functions before cxx_bind_parameters_in_call
+	could report some argument throwing an exception.
+	(cxx_eval_unary_expression): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression and return early
+	if *jump_target after the call.
+	(cxx_fold_pointer_plus_expression): Likewise.
+	(cxx_eval_binary_expression): Likewise and similarly for
+	cxx_fold_pointer_plus_expression call.
+	(cxx_eval_conditional_expression): Pass jump_target to
+	cxx_eval_constant_expression on first operand and return early
+	if *jump_target after the call.
+	(cxx_eval_vector_conditional_expression): Add jump_target argument.
+	Pass it through to cxx_eval_constant_expression for all 3 arguments
+	and return early if *jump_target after any of those calls.
+	(get_array_or_vector_nelts): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression.
+	(eval_and_check_array_index): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression calls and return early after
+	each of them if *jump_target.
+	(cxx_eval_array_reference): Likewise.
+	(cxx_eval_component_reference): Likewise.
+	(cxx_eval_bit_field_ref): Likewise.
+	(cxx_eval_bit_cast): Likewise.  Assert CHECKING_P call doesn't
+	throw or return.
+	(cxx_eval_logical_expression): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression calls and return early after
+	each of them if *jump_target.
+	(cxx_eval_bare_aggregate): Likewise.
+	(cxx_eval_vec_init_1): Add jump_target argument.  Pass it through
+	to cxx_eval_bare_aggregate and recursive call.  Pass it through
+	to get_array_or_vector_nelts and cxx_eval_constant_expression
+	and return early after it if *jump_target.
+	(cxx_eval_vec_init): Add jump_target argument.  Pass it through
+	to cxx_eval_constant_expression and cxx_eval_vec_init_1.
+	(cxx_union_active_member): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression and return early after it
+	if *jump_target.
+	(cxx_fold_indirect_ref_1): Add jump_target argument.  Pass it
+	through to cxx_union_active_member and recursive calls.
+	(cxx_eval_indirect_ref): Add jump_target argument.  Pass it through
+	to cxx_fold_indirect_ref_1 calls and to recursive call, in which
+	case return early after it if *jump_target.
+	(cxx_fold_indirect_ref): Add jump_target argument.  Pass it through
+	to cxx_fold_indirect_ref and cxx_eval_constant_expression calls and
+	return early after those if *jump_target.
+	(cxx_eval_trinary_expression): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression calls and return early after
+	those if *jump_target.
+	(cxx_eval_store_expression): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression and eval_and_check_array_index
+	calls and return early after those if *jump_target.
+	(cxx_eval_increment_expression): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression calls and return early after
+	those if *jump_target.
+	(label_matches): Handle VAR_DECL case.
+	(cxx_eval_statement_list): Remove local_target variable and
+	!jump_target handling.  Handle throws (jump_target) like returns or
+	breaks.
+	(cxx_eval_loop_expr): Remove local_target variable and !jump_target
+	handling.  Pass it through to cxx_eval_constant_expression.  Handle
+	throws (jump_target) like returns.
+	(cxx_eval_switch_expr): Pass jump_target through to
+	cxx_eval_constant_expression on cond, return early after it
+	if *jump_target.
+	(build_new_constexpr_heap_type): Add jump_target argument.  Pass it
+	through to cxx_eval_constant_expression calls, return early after
+	those if *jump_target.
+	(merge_jump_target): New function.
+	(cxx_eval_constant_expression): Make jump_target argument no longer
+	defaulted, don't test jump_target for NULL.  Pass jump_target
+	through to recursive calls, cxx_eval_call_expression,
+	cxx_eval_store_expression, cxx_eval_indirect_ref,
+	cxx_eval_unary_expression, cxx_eval_binary_expression,
+	cxx_eval_logical_expression, cxx_eval_array_reference,
+	cxx_eval_component_reference, cxx_eval_bit_field_ref,
+	cxx_eval_vector_conditional_expression, cxx_eval_bare_aggregate,
+	cxx_eval_vec_init, cxx_eval_trinary_expression, cxx_fold_indirect_ref,
+	build_new_constexpr_heap_type, cxx_eval_increment_expression,
+	cxx_eval_bit_cast and return earlyu after some of those
+	if *jump_target as needed.
+	(cxx_eval_constant_expression) <case TARGET_EXPR>: For C++26 push
+	also CLEANUP_EH_ONLY cleanups, with NULL_TREE marker after them.
+	(cxx_eval_constant_expression) <case RETURN_EXPR>: Don't
+	override *jump_target if throws (jump_target).
+	(cxx_eval_constant_expression) <case TRY_CATCH_EXPR, case TRY_BLOCK,
+	case MUST_NOT_THROW_EXPR, case TRY_FINALLY_EXPR, case CLEANUP_STMT>:
+	Handle C++26 constant expressions.
+	(cxx_eval_constant_expression) <case CLEANUP_POINT_EXPR>: For C++26
+	with throws (jump_target) evaluate the CLEANUP_EH_ONLY cleanups as
+	well, and if not throws (jump_target) skip those.  Set *jump_target
+	if some of the cleanups threw.
+	(cxx_eval_constant_expression) <case THROW_EXPR>: Recurse on operand
+	for C++26.
+	(cxx_eval_outermost_constant_expr): Diagnose uncaught exceptions both
+	from main expression and cleanups, diagnose also
+	break/continue/returns from the main expression.  Handle
+	CLEANUP_EH_ONLY cleanup markers.  Don't diagnose mutable poison stuff
+	if non_constant_p.  Use different diagnostics for non-deleted heap
+	allocations if they were allocated by __cxa_allocate_exception.
+	(callee_might_throw): New function.
+	(struct check_for_return_continue_data): Add could_throw field.
+	(check_for_return_continue): Handle AGGR_INIT_EXPR and CALL_EXPR and
+	set d->could_throw if they could throw.
+	(potential_constant_expression_1): For CALL_EXPR allow
+	cxx_dynamic_cast_fn_p calls.  For C++26 set *jump_target to void_node
+	for calls that could throw.  For C++26 if call to non-constexpr call
+	is seen, try to evaluate arguments first and if they could throw,
+	don't diagnose call to non-constexpr function nor return false.
+	Adjust check_for_return_continue_data initializers and
+	set *jump_target to void_node if data.could_throw_p.  For C++26
+	recurse on THROW_EXPR argument.  Add comment explaining TRY_BLOCK
+	handling with C++26 exceptions.  Handle throws like returns in some
+	cases.
+	* cp-tree.h (MUST_NOT_THROW_NOEXCEPT_P, MUST_NOT_THROW_THROW_P,
+	MUST_NOT_THROW_CATCH_P, DECL_EXCEPTION_REFCOUNT): Define.
+	(DECL_LOCAL_DECL_P): Fix comment typo, VARIABLE_DECL -> VAR_DECL.
+	(enum cp_built_in_function): Add CP_BUILT_IN_EH_PTR_ADJUST_REF,
+	(handler_match_for_exception_type): Declare.
+	* call.cc (handler_match_for_exception_type): New function.
+	* except.cc (initialize_handler_parm): Set MUST_NOT_THROW_CATCH_P
+	on newly created MUST_NOT_THROW_EXPR.
+	(begin_eh_spec_block): Set MUST_NOT_THROW_NOEXCEPT_P.
+	(wrap_cleanups_r): Set MUST_NOT_THROW_THROW_P.
+	(build_throw): Add another TARGET_EXPR whose scope spans
+	until after the __cxa_throw call and copy pointer value from ptr
+	to it and use it in __cxa_throw argument.
+	* tree.cc (builtin_valid_in_constant_expr_p): Handle
+	CP_BUILT_IN_EH_PTR_ADJUST_REF.
+	* decl.cc (cxx_init_decl_processing): Initialize
+	__builtin_eh_ptr_adjust_ref FE builtin.
+	* pt.cc (tsubst_stmt) <case MUST_NOT_THROW_EXPR>: Copy the
+	MUST_NOT_THROW_NOEXCEPT_P, MUST_NOT_THROW_THROW_P and
+	MUST_NOT_THROW_CATCH_P flags.
+	* cp-gimplify.cc (cp_gimplify_expr) <case CALL_EXPR>: Error on
+	non-folded CP_BUILT_IN_EH_PTR_ADJUST_REF calls.
+
+2025-07-09  Jason Merrill  <jason@redhat.com>
+
+	PR c++/121012
+	PR c++/120917
+	* parser.cc (cp_parser_lambda_expression): Clear
+	parser->in_template_argument_list_p.
+
+2025-07-09  Jason Merrill  <jason@redhat.com>
+
+	PR c++/121008
+	PR c++/113563
+	* semantics.cc (finish_this_expr): Do check current_class_ref for
+	non-lambda.
+
+2025-07-09  Marek Polacek  <polacek@redhat.com>
+
+	PR c++/119838
+	* parser.cc (cp_parser_nested_name_specifier_opt): New global_p
+	parameter.  Look for "template" when global_p is true.
+	(cp_parser_simple_type_specifier): Pass global_p to
+	cp_parser_nested_name_specifier_opt.
+
+2025-07-08  Marek Polacek  <polacek@redhat.com>
+	    Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR c++/83469
+	PR c++/93809
+	* cp-tree.h (UNION_TYPE_P): Define.
+	(TYPENAME_IS_UNION_P): Define.
+	* decl.cc (struct typename_info): Add union_p field.
+	(struct typename_hasher::equal): Compare union_p field.
+	(build_typename_type): Use ti.union_p for union_type.  Set
+	TYPENAME_IS_UNION_P.
+	* error.cc (dump_type) <case TYPENAME_TYPE>: Handle
+	TYPENAME_IS_UNION_P.
+	* module.cc (trees_out::type_node): Likewise.
+	* parser.cc (cp_parser_check_class_key): Allow typename key for union
+	types and allow union keyword for typename types.
+	* pt.cc (tsubst) <case TYPENAME_TYPE>: Don't conflate unions with
+	class_type.  For TYPENAME_IS_CLASS_P, check NON_UNION_CLASS_TYPE_P
+	rather than CLASS_TYPE_P.  Add TYPENAME_IS_UNION_P handling.
+
+2025-07-08  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/117784
+	* decl.cc: Implement part of C++26 P2686R4 - constexpr structured
+	bindings.
+	(cp_finish_decl): Pedwarn for C++23 and older on constinit on
+	structured bindings except for static/thread_local where it uses
+	earlier error.
+	(grokdeclarator): Pedwarn on constexpr structured bindings for
+	C++23 and older instead of emitting error always, don't clear
+	constexpr_p in that case.
+	* parser.cc (cp_parser_decomposition_declaration): Copy over
+	DECL_DECLARED_CONSTEXPR_P and DECL_DECLARED_CONSTINIT_P flags.
+
+2025-07-07  Alfie Richards  <alfie.richards@arm.com>
+
+	PR c++/119498
+	* decl.cc (duplicate_decls): Change logic to not always exclude FMV
+	annotated functions in cases of return type non-ambiguation.
+
+2025-07-07  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120917
+	* parser.cc (cp_parser_simple_type_specifier): Attach
+	auto in targ in parameter to -Wabbreviated-auto-in-template-arg.
+	(cp_parser_placeholder_type_specifier): Diagnose constrained auto in
+	template arg.
+
+2025-07-07  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/84009
+	* parser.cc (cp_parser_decomposition_declaration): Pedwarn
+	on thread_local, __thread or static in decl_specifiers for
+	for-range-declaration.
+	(cp_parser_init_declarator): Likewise, and also for extern
+	or register.
+
+2025-07-04  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120575
+	PR c++/116064
+	* parser.cc (cp_parser_abort_tentative_parse): Check seen_error
+	instead of errorcount.
+
+2025-07-03  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120716
+	* lambda.cc (finish_lambda_function): Pass cur_stmt_list to
+	prune_lambda_captures.
+
+2025-07-03  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120748
+	* lambda.cc (lambda_expr_this_capture): Don't return a FIELD_DECL.
+	* parser.cc (cp_parser_primary_expression): Ignore THIS_FORBIDDEN
+	if cp_unevaluated_operand.
+
+2025-07-03  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120940
+	* typeck.cc (cp_build_array_ref): Fix a pasto.
+
+2025-07-03  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120684
+	PR c++/118856
+	* constexpr.cc (cxx_eval_constant_expression) [TARGET_EXPR]: Clear
+	the value first if is_complex.
+
 2025-07-01  Jakub Jelinek  <jakub@redhat.com>
 
 	PR c++/120471
diff --git a/gcc/cp/call.cc b/gcc/cp/call.cc
index 2c3ef3d..9283d97 100644
--- a/gcc/cp/call.cc
+++ b/gcc/cp/call.cc
@@ -220,7 +220,8 @@ static void add_candidates (tree, tree, const vec<tree, va_gc> *, tree, tree,
 			    bool, tree, tree, int, struct z_candidate **,
 			    tsubst_flags_t);
 static conversion *merge_conversion_sequences (conversion *, conversion *);
-static tree build_temp (tree, tree, int, diagnostic_t *, tsubst_flags_t);
+static tree build_temp (tree, tree, int, enum diagnostics::kind *,
+			tsubst_flags_t);
 static conversion *build_identity_conv (tree, tree);
 static inline bool conv_binds_to_array_of_unknown_bound (conversion *);
 static bool conv_is_prvalue (conversion *);
@@ -412,6 +413,7 @@ build_call_a (tree function, int n, tree *argarray)
 	      /* We're disconnecting the initializer from its target,
 		 don't create a temporary.  */
 	      arg = TARGET_EXPR_INITIAL (arg);
+	    suppress_warning (arg, OPT_Wunused_result);
 	    tree t = build0 (EMPTY_CLASS_EXPR, TREE_TYPE (arg));
 	    arg = build2 (COMPOUND_EXPR, TREE_TYPE (t), arg, t);
 	    CALL_EXPR_ARG (function, i) = arg;
@@ -1723,6 +1725,56 @@ involves_qualification_conversion_p (tree to, tree from)
   return false;
 }
 
+/* Return true if HANDLER is a match for exception object with EXCEPT_TYPE as
+   per [except.handle]/3.  */
+
+bool
+handler_match_for_exception_type (tree handler, tree except_type)
+{
+  tree handler_type = HANDLER_TYPE (handler);
+  if (handler_type == NULL_TREE)
+    return true; /* ... */
+  if (same_type_ignoring_top_level_qualifiers_p (handler_type, except_type))
+    return true;
+  if (CLASS_TYPE_P (except_type) && CLASS_TYPE_P (handler_type))
+    {
+      base_kind b_kind;
+      tree binfo = lookup_base (except_type, handler_type, ba_check, &b_kind,
+				tf_none);
+      if (binfo && binfo != error_mark_node)
+	return true;
+    }
+  if (TYPE_PTR_P (handler_type) || TYPE_PTRDATAMEM_P (handler_type))
+    {
+      if (TREE_CODE (except_type) == NULLPTR_TYPE)
+	return true;
+      if ((TYPE_PTR_P (handler_type) && TYPE_PTR_P (except_type))
+	  || (TYPE_PTRDATAMEM_P (handler_type)
+	      && TYPE_PTRDATAMEM_P (except_type)))
+	{
+	  conversion *conv
+	    = standard_conversion (handler_type, except_type, NULL_TREE,
+				   /*c_cast_p=*/false, 0, tf_none);
+	  if (conv && !conv->bad_p)
+	    {
+	      for (conversion *t = conv; t; t = next_conversion (t))
+		switch (t->kind)
+		  {
+		  case ck_ptr:
+		  case ck_fnptr:
+		  case ck_qual:
+		  case ck_identity:
+		    break;
+		  default:
+		    return false;
+		  }
+	      return true;
+	    }
+	}
+    }
+  return false;
+}
+
 /* A reference of the indicated TYPE is being bound directly to the
    expression represented by the implicit conversion sequence CONV.
    Return a conversion sequence for this binding.  */
@@ -4876,6 +4928,11 @@ implicit_conversion_error (location_t loc, tree type, tree expr)
 	   && !CP_AGGREGATE_TYPE_P (type))
     error_at (loc, "designated initializers cannot be used with a "
 	      "non-aggregate type %qT", type);
+  else if (is_stub_object (expr))
+    /* The expression is generated by a trait check, we don't have
+       a useful location to highlight the label.  */
+    error_at (loc, "could not convert %qH to %qI",
+	      TREE_TYPE (expr), type);
   else
     {
       range_label_for_type_mismatch label (TREE_TYPE (expr), type);
@@ -6404,7 +6461,9 @@ build_conditional_expr (const op_location_t &loc,
 	       types when the enumeration is still being defined.  */;
 	  else if (complain & (cxx_dialect >= cxx26
 			       ? tf_warning_or_error : tf_warning))
-	    emit_diagnostic (cxx_dialect >= cxx26 ? DK_PEDWARN : DK_WARNING,
+	    emit_diagnostic ((cxx_dialect >= cxx26
+			      ? diagnostics::kind::pedwarn
+			      : diagnostics::kind::warning),
 			     loc, OPT_Wenum_compare, "enumerated mismatch "
 			     "in conditional expression: %qT vs %qT",
 			     arg2_type, arg3_type);
@@ -7435,7 +7494,16 @@ build_new_op (const op_location_t &loc, enum tree_code code, int flags,
       else if (TREE_CODE (cand->fn) == FUNCTION_DECL)
 	{
 	  if (overload)
-	    *overload = cand->fn;
+	    {
+	      if (cand->rewritten ())
+		/* build_min_non_dep_op_overload needs to know whether the
+		   candidate is rewritten/reversed.  */
+		*overload = build_tree_list (build_int_cst (integer_type_node,
+							    cand->flags),
+					     cand->fn);
+	      else
+		*overload = cand->fn;
+	    }
 
 	  if (resolve_args (arglist, complain) == NULL)
 	    result = error_mark_node;
@@ -7484,9 +7552,6 @@ build_new_op (const op_location_t &loc, enum tree_code code, int flags,
 	  /* If this was a C++20 rewritten comparison, adjust the result.  */
 	  if (cand->rewritten ())
 	    {
-	      /* FIXME build_min_non_dep_op_overload can't handle rewrites.  */
-	      if (overload)
-		*overload = NULL_TREE;
 	      switch (code)
 		{
 		case EQ_EXPR:
@@ -7585,8 +7650,9 @@ build_new_op (const op_location_t &loc, enum tree_code code, int flags,
 		      && (complain & tf_warning_or_error) == 0)
 		    result = error_mark_node;
 		  else if (cxx_dialect >= cxx26 || (complain & tf_warning))
-		    emit_diagnostic (cxx_dialect >= cxx26
-				     ? DK_PEDWARN : DK_WARNING,
+		    emit_diagnostic ((cxx_dialect >= cxx26
+				      ? diagnostics::kind::pedwarn
+				      : diagnostics::kind::warning),
 				     loc, OPT_Wenum_compare,
 				     "comparison between %q#T and %q#T",
 				     arg1_type, arg2_type);
@@ -8400,11 +8466,11 @@ complain_about_access (tree decl, tree diag_decl, tree diag_location,
 
 static tree
 build_temp (tree expr, tree type, int flags,
-	    diagnostic_t *diagnostic_kind, tsubst_flags_t complain)
+	    enum diagnostics::kind *diagnostic_kind, tsubst_flags_t complain)
 {
   int savew, savee;
 
-  *diagnostic_kind = DK_UNSPECIFIED;
+  *diagnostic_kind = diagnostics::kind::unspecified;
 
   /* If the source is a packed field, calling the copy constructor will require
      binding the field to the reference parameter to the copy constructor, and
@@ -8428,9 +8494,9 @@ build_temp (tree expr, tree type, int flags,
   expr = build_special_member_call (NULL_TREE, complete_ctor_identifier,
 				    &args, type, flags, complain);
   if (warningcount + werrorcount > savew)
-    *diagnostic_kind = DK_WARNING;
+    *diagnostic_kind = diagnostics::kind::warning;
   else if (errorcount > savee)
-    *diagnostic_kind = DK_ERROR;
+    *diagnostic_kind = diagnostics::kind::error;
   return expr;
 }
 
@@ -8642,9 +8708,10 @@ convert_like_internal (conversion *convs, tree expr, tree fn, int argnum,
 		       bool nested_p, tsubst_flags_t complain)
 {
   tree totype = convs->type;
-  diagnostic_t diag_kind;
+  enum diagnostics::kind diag_kind;
   int flags;
   location_t loc = cp_expr_loc_or_input_loc (expr);
+  const bool stub_object_p = is_stub_object (expr);
 
   if (convs->bad_p && !(complain & tf_error))
     return error_mark_node;
@@ -8721,7 +8788,10 @@ convert_like_internal (conversion *convs, tree expr, tree fn, int argnum,
 				      "from %qH to %qI", TREE_TYPE (expr),
 				      totype);
 	      if (complained)
-		print_z_candidate (loc, N_("candidate is:"), t->cand);
+		{
+		  auto_diagnostic_nesting_level sentinel;
+		  print_z_candidate (loc, N_("candidate is:"), t->cand);
+		}
 	      expr = convert_like (t, expr, fn, argnum,
 				   /*issue_conversion_warnings=*/false,
 				   /*c_cast_p=*/false, /*nested_p=*/true,
@@ -8746,7 +8816,14 @@ convert_like_internal (conversion *convs, tree expr, tree fn, int argnum,
 	  else if (t->kind == ck_identity)
 	    break;
 	}
-      if (!complained && expr != error_mark_node)
+      if (!complained && stub_object_p)
+	{
+	  /* An error diagnosed within a trait, don't give extra labels.  */
+	  error_at (loc, "invalid conversion from %qH to %qI",
+		    TREE_TYPE (expr), totype);
+	  complained = 1;
+	}
+      else if (!complained && expr != error_mark_node)
 	{
 	  range_label_for_type_mismatch label (TREE_TYPE (expr), totype);
 	  gcc_rich_location richloc (loc, &label, highlight_colors::percent_h);
@@ -9169,7 +9246,7 @@ convert_like_internal (conversion *convs, tree expr, tree fn, int argnum,
       if (convs->copy_init_p)
 	flags |= LOOKUP_ONLYCONVERTING;
       expr = build_temp (expr, totype, flags, &diag_kind, complain);
-      if (diag_kind && complain)
+      if (diag_kind != diagnostics::kind::unspecified && complain)
 	{
 	  auto_diagnostic_group d;
 	  maybe_print_user_conv_context (convs);
diff --git a/gcc/cp/class.cc b/gcc/cp/class.cc
index 9a41c00..f5d20e5 100644
--- a/gcc/cp/class.cc
+++ b/gcc/cp/class.cc
@@ -1407,7 +1407,7 @@ add_method (tree type, tree method, bool via_using)
       /* If these are versions of the same function, process and
 	 move on.  */
       if (TREE_CODE (fn) == FUNCTION_DECL
-	  && maybe_version_functions (method, fn, true))
+	  && maybe_version_functions (method, fn))
 	continue;
 
       if (DECL_INHERITED_CTOR (method))
@@ -7452,7 +7452,7 @@ determine_key_method (tree type)
 	&& ! DECL_DECLARED_INLINE_P (method)
 	&& ! DECL_PURE_VIRTUAL_P (method))
       {
-	CLASSTYPE_KEY_METHOD (type) = method;
+	SET_CLASSTYPE_KEY_METHOD (type, method);
 	break;
       }
 
diff --git a/gcc/cp/constexpr.cc b/gcc/cp/constexpr.cc
index f9066bc..be24ae2 100644
--- a/gcc/cp/constexpr.cc
+++ b/gcc/cp/constexpr.cc
@@ -153,7 +153,7 @@ static bool
 constexpr_error (location_t location, bool constexpr_fundef_p,
 		 const char *gmsgid, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   rich_location richloc (line_table, location);
   va_start (ap, gmsgid);
@@ -161,14 +161,17 @@ constexpr_error (location_t location, bool constexpr_fundef_p,
   if (!constexpr_fundef_p)
     {
       /* Report an error that cannot be suppressed.  */
-      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, DK_ERROR);
+      diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
+			   diagnostics::kind::error);
       ret = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   else if (warn_invalid_constexpr)
     {
       diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
-			   cxx_dialect < cxx23 ? DK_PEDWARN : DK_WARNING);
-      diagnostic.option_id = OPT_Winvalid_constexpr;
+			   (cxx_dialect < cxx23
+			    ? diagnostics::kind::pedwarn
+			    : diagnostics::kind::warning));
+      diagnostic.m_option_id = OPT_Winvalid_constexpr;
       ret = diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
   else
@@ -1184,6 +1187,10 @@ public:
   /* Heap VAR_DECLs created during the evaluation of the outermost constant
      expression.  */
   auto_vec<tree, 16> heap_vars;
+  /* Vector of caught exceptions, including exceptions still not active at
+     the start of a handler (those are immediately followed up by HANDLER_TYPE
+     until __cxa_begin_catch finishes).  */
+  auto_vec<tree, 2> caught_exceptions;
   /* Cleanups that need to be evaluated at the end of CLEANUP_POINT_EXPR.  */
   vec<tree> *cleanups;
   /* If non-null, only allow modification of existing values of the variables
@@ -1191,10 +1198,13 @@ public:
   hash_set<tree> *modifiable;
   /* Number of heap VAR_DECL deallocations.  */
   unsigned heap_dealloc_count;
+  /* Number of uncaught exceptions.  */
+  unsigned uncaught_exceptions;
+
   /* Constructor.  */
   constexpr_global_ctx ()
     : constexpr_ops_count (0), cleanups (NULL), modifiable (nullptr),
-      heap_dealloc_count (0) {}
+      heap_dealloc_count (0), uncaught_exceptions (0) {}
 
   bool is_outside_lifetime (tree t)
   {
@@ -1308,6 +1318,48 @@ struct constexpr_ctx {
   mce_value manifestly_const_eval;
 };
 
+/* Predicates for the meaning of *jump_target.  */
+
+static bool
+returns (tree *jump_target)
+{
+  return *jump_target && TREE_CODE (*jump_target) == RETURN_EXPR;
+}
+
+static bool
+breaks (tree *jump_target)
+{
+  return (*jump_target
+	  && ((TREE_CODE (*jump_target) == LABEL_DECL
+	       && LABEL_DECL_BREAK (*jump_target))
+	      || TREE_CODE (*jump_target) == BREAK_STMT
+	      || TREE_CODE (*jump_target) == EXIT_EXPR));
+}
+
+static bool
+continues (tree *jump_target)
+{
+  return (*jump_target
+	  && ((TREE_CODE (*jump_target) == LABEL_DECL
+	       && LABEL_DECL_CONTINUE (*jump_target))
+	      || TREE_CODE (*jump_target) == CONTINUE_STMT));
+}
+
+static bool
+switches (tree *jump_target)
+{
+  return *jump_target && TREE_CODE (*jump_target) == INTEGER_CST;
+}
+
+static bool
+throws (tree *jump_target)
+{
+  /* void_node is for use in potential_constant_expression_1, otherwise
+     it should an artificial VAR_DECL created by constant evaluation
+     of __cxa_allocate_exception ().  */
+  return (*jump_target && (VAR_P (*jump_target) || *jump_target == void_node));
+}
+
 /* True if the constexpr relaxations afforded by P2280R4 for unknown
    references and objects are in effect.  */
 
@@ -1543,13 +1595,672 @@ enum value_cat {
 };
 
 static tree cxx_eval_constant_expression (const constexpr_ctx *, tree,
-					  value_cat, bool *, bool *, tree * = NULL);
+					  value_cat, bool *, bool *, tree *);
 static tree cxx_eval_bare_aggregate (const constexpr_ctx *, tree,
-				     value_cat, bool *, bool *);
+				     value_cat, bool *, bool *, tree *);
 static tree cxx_fold_indirect_ref (const constexpr_ctx *, location_t, tree, tree,
-				   bool * = NULL);
+				   bool *, tree *);
 static tree find_heap_var_refs (tree *, int *, void *);
 
+/* For exception object EXC if it has class type and usable what () method
+   which returns cv char * return the xmalloced string literal which it returns
+   if possible, otherwise return NULL.  */
+
+static char *
+exception_what_str (const constexpr_ctx *ctx, tree exc)
+{
+  tree type = strip_array_types (TREE_TYPE (exc));
+  if (!CLASS_TYPE_P (type))
+    return NULL;
+  tree std_exception = lookup_qualified_name (std_node, "exception",
+					      LOOK_want::NORMAL, false);
+  if (TREE_CODE (std_exception) != TYPE_DECL)
+    return NULL;
+  if (!CLASS_TYPE_P (TREE_TYPE (std_exception)))
+    return NULL;
+  base_kind b_kind;
+  tree binfo = lookup_base (type, TREE_TYPE (std_exception), ba_check, &b_kind,
+			    tf_none);
+  if (binfo == NULL_TREE || binfo == error_mark_node)
+    return NULL;
+  if (type != TREE_TYPE (exc))
+    exc = build4 (ARRAY_REF, type, exc, size_zero_node, NULL, NULL);
+  tree call
+    = finish_class_member_access_expr (exc, get_identifier ("what"), false,
+				       tf_none);
+  if (call == error_mark_node)
+    return NULL;
+  releasing_vec what_args;
+  call = finish_call_expr (call, &what_args, false, false, tf_none);
+  if (call == error_mark_node)
+    return NULL;
+  if (TREE_CODE (TREE_TYPE (call)) != POINTER_TYPE
+      || !INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (call)))
+      || !COMPLETE_TYPE_P (TREE_TYPE (TREE_TYPE (call)))
+      || !tree_int_cst_equal (TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (call))),
+			      TYPE_SIZE_UNIT (char_type_node))
+      || TYPE_PRECISION (TREE_TYPE (TREE_TYPE (call))) != BITS_PER_UNIT)
+    return NULL;
+  if (!potential_constant_expression (call))
+    return NULL;
+  bool non_constant_p = false, overflow_p = false;
+  tree jmp_target = NULL;
+  tree ptr = cxx_eval_constant_expression (ctx, call, vc_prvalue,
+					   &non_constant_p, &overflow_p,
+					   &jmp_target);
+  if (throws (&jmp_target) || non_constant_p)
+    return NULL;
+  if (reduced_constant_expression_p (ptr))
+    if (const char *msg = c_getstr (ptr))
+      return xstrdup (msg);
+  auto_vec <char, 32> v;
+  for (unsigned i = 0; i < INT_MAX; ++i)
+    {
+      tree t = call;
+      if (i)
+	t = build2 (POINTER_PLUS_EXPR, TREE_TYPE (ptr), ptr, size_int (i));
+      t = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (t)), t);
+      non_constant_p = false;
+      overflow_p = false;
+      jmp_target = NULL;
+      tree t2 = cxx_eval_constant_expression (ctx, t, vc_prvalue,
+					      &non_constant_p, &overflow_p,
+					      &jmp_target);
+      if (throws (&jmp_target)
+	  || non_constant_p
+	  || !tree_fits_shwi_p (t2))
+	return NULL;
+      char c = tree_to_shwi (t2);
+      v.safe_push (c);
+      if (c == '\0')
+	break;
+    }
+  return xstrdup (v.address ());
+}
+
+/* Diagnose constant expression evaluation encountering call to
+   std::terminate due to exception EXC.  */
+
+static void
+diagnose_std_terminate (location_t loc, const constexpr_ctx *ctx, tree exc)
+{
+  tree type = strip_array_types (TREE_TYPE (exc));
+  if (char *str = exception_what_str (ctx, exc))
+    {
+      error_at (loc, "%qs called after throwing an exception of type %qT; "
+		     "%<what()%>: %qs", "std::terminate", type, str);
+      free (str);
+    }
+  else
+    {
+      if (type != TREE_TYPE (exc))
+	exc = build4 (ARRAY_REF, type, exc, size_zero_node, NULL, NULL);
+      bool non_constant_p = false, overflow_p = false;
+      tree jmp_target = NULL;
+      tree val = cxx_eval_constant_expression (ctx, exc, vc_prvalue,
+					       &non_constant_p, &overflow_p,
+					       &jmp_target);
+      gcc_assert (!throws (&jmp_target) && !non_constant_p);
+      if (reduced_constant_expression_p (val))
+	error_at (loc, "%qs called after throwing an exception %qE",
+		       "std::terminate", val);
+      else
+	error_at (loc, "%qs called after throwing an exception of type %qT",
+		       "std::terminate", type);
+    }
+}
+
+/* Diagnose constant expression evaluation encountering call to
+   uncaught exception EXC.  */
+
+static void
+diagnose_uncaught_exception (location_t loc, const constexpr_ctx *ctx, tree exc)
+{
+  tree type = strip_array_types (TREE_TYPE (exc));
+  if (char *str = exception_what_str (ctx, exc))
+    {
+      error_at (loc, "uncaught exception of type %qT; %<what()%>: %qs", type, str);
+      free (str);
+    }
+  else
+    {
+      if (type != TREE_TYPE (exc))
+	exc = build4 (ARRAY_REF, type, exc, size_zero_node, NULL, NULL);
+      bool non_constant_p = false, overflow_p = false;
+      tree jmp_target = NULL;
+      tree val = cxx_eval_constant_expression (ctx, exc, vc_prvalue,
+					       &non_constant_p, &overflow_p,
+					       &jmp_target);
+      gcc_assert (!throws (&jmp_target) && !non_constant_p);
+      if (reduced_constant_expression_p (val))
+	error_at (loc, "uncaught exception %qE", val);
+      else
+	error_at (loc, "uncaught exception of type %qT", type);
+    }
+}
+
+/* Kinds of __cxa_* functions (and a few other EH related ones) we handle as
+   magic constexpr functions for C++26.  */
+
+enum cxa_builtin {
+  CXA_NONE = 0,
+  CXA_ALLOCATE_EXCEPTION = 1,
+  CXA_FREE_EXCEPTION = 2,
+  CXA_THROW = 3,
+  CXA_BEGIN_CATCH = 4,
+  CXA_END_CATCH = 5,
+  CXA_RETHROW = 6,
+  CXA_GET_EXCEPTION_PTR = 7,
+  CXA_BAD_CAST = 8,
+  CXA_BAD_TYPEID = 9,
+  CXA_THROW_BAD_ARRAY_NEW_LENGTH = 10,
+  STD_UNCAUGHT_EXCEPTIONS = 11,
+  STD_CURRENT_EXCEPTION = 12,
+  STD_RETHROW_EXCEPTION = 13,
+  BUILTIN_EH_PTR_ADJUST_REF = 14
+};
+
+/* Return cxa_builtin if FNDECL is a __cxa_* function handled as
+   magic constexpr function for C++26.  Return CXA_NONE otherwise.  */
+
+static enum cxa_builtin
+cxx_cxa_builtin_fn_p (tree fndecl)
+{
+  if (cxx_dialect < cxx26)
+    return CXA_NONE;
+  if (DECL_LANGUAGE (fndecl) != lang_c)
+    {
+      if (!decl_in_std_namespace_p (fndecl))
+	return CXA_NONE;
+      if (id_equal (DECL_NAME (fndecl), "uncaught_exceptions"))
+	return STD_UNCAUGHT_EXCEPTIONS;
+      if (id_equal (DECL_NAME (fndecl), "current_exception"))
+	return STD_CURRENT_EXCEPTION;
+      if (id_equal (DECL_NAME (fndecl), "rethrow_exception"))
+	return STD_RETHROW_EXCEPTION;
+      return CXA_NONE;
+    }
+  if (!startswith (IDENTIFIER_POINTER (DECL_NAME (fndecl)), "__cxa_"))
+    return CXA_NONE;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_allocate_exception"))
+    return CXA_ALLOCATE_EXCEPTION;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_free_exception"))
+    return CXA_FREE_EXCEPTION;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_throw"))
+    return CXA_THROW;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_begin_catch"))
+    return CXA_BEGIN_CATCH;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_end_catch"))
+    return CXA_END_CATCH;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_rethrow"))
+    return CXA_RETHROW;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_get_exception_ptr"))
+    return CXA_GET_EXCEPTION_PTR;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_bad_cast"))
+    return CXA_BAD_CAST;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_bad_typeid"))
+    return CXA_BAD_TYPEID;
+  if (id_equal (DECL_NAME (fndecl), "__cxa_throw_bad_array_new_length"))
+    return CXA_THROW_BAD_ARRAY_NEW_LENGTH;
+  return CXA_NONE;
+}
+
+/* Helper function for cxx_eval_cxa_builtin_fn.
+   Check if ARG is a valid first argument of __cxa_throw or
+   __cxa_free_exception or __builtin_eh_ptr_adjust_ref.  Return NULL_TREE if
+   not, otherwise return the artificial __cxa_allocate_exception allocated
+   VAR_DECL.  FREE_EXC is true for __cxa_free_exception, false otherwise.  */
+
+static tree
+cxa_check_throw_arg (tree arg, bool free_exc)
+{
+  STRIP_NOPS (arg);
+  if (TREE_CODE (arg) != ADDR_EXPR)
+    return NULL_TREE;
+  arg = TREE_OPERAND (arg, 0);
+  if (!VAR_P (arg)
+      || !DECL_ARTIFICIAL (arg)
+      || ((!free_exc || DECL_NAME (arg) != heap_uninit_identifier)
+	  && DECL_NAME (arg) != heap_identifier)
+      || !DECL_LANG_SPECIFIC (arg))
+    return NULL_TREE;
+  return arg;
+}
+
+/* Helper function for cxx_eval_cxa_builtin_fn.
+   "Allocate" on the constexpr heap an exception object of TYPE
+   with REFCOUNT.  */
+
+static tree
+cxa_allocate_exception (location_t loc, const constexpr_ctx *ctx, tree type,
+			tree refcount)
+{
+  tree var = build_decl (loc, VAR_DECL, heap_uninit_identifier, type);
+  DECL_ARTIFICIAL (var) = 1;
+  retrofit_lang_decl (var);
+  DECL_EXCEPTION_REFCOUNT (var) = refcount;
+  ctx->global->heap_vars.safe_push (var);
+  return var;
+}
+
+/* Evaluate various __cxa_* calls as magic constexpr builtins for
+   C++26 constexpr exception support (P3068R5).  */
+
+static tree
+cxx_eval_cxa_builtin_fn (const constexpr_ctx *ctx, tree call,
+			 enum cxa_builtin kind, tree fndecl,
+			 bool *non_constant_p, bool *overflow_p,
+			 tree *jump_target)
+{
+  int nargs = call_expr_nargs (call);
+  location_t loc = cp_expr_loc_or_input_loc (call);
+  tree args[4], arg;
+  if (nargs > 4)
+    {
+    invalid_nargs:
+      if (!ctx->quiet)
+	error_at (loc, "call to %qD function with incorrect"
+		  "number of arguments", fndecl);
+      *non_constant_p = true;
+      return call;
+    }
+  if ((kind == CXA_BEGIN_CATCH || kind == CXA_GET_EXCEPTION_PTR)
+      && nargs == 1
+      && (arg = CALL_EXPR_ARG (call, 0))
+      && TREE_CODE (arg) == CALL_EXPR
+      && call_expr_nargs (arg) == 1
+      && integer_zerop (CALL_EXPR_ARG (arg, 0)))
+    if (tree fun = get_function_named_in_call (arg))
+      if (fndecl_built_in_p (fun, BUILT_IN_EH_POINTER))
+	{
+	  if (ctx->global->caught_exceptions.length () < 2)
+	    {
+	    no_caught_exceptions:
+	      if (!ctx->quiet)
+		error_at (loc, "%qD called with no caught exceptions pending",
+			  fndecl);
+	      *non_constant_p = true;
+	      return call;
+	    }
+	  /* Both __cxa_get_exception_ptr (__builtin_eh_pointer (0))
+	     and __cxa_begin_catch (__builtin_eh_pointer (0)) calls expect
+	     ctx->global->caught_exceptions vector to end with
+	     __cxa_allocate_exception created artificial VAR_DECL (the
+	     exception object) followed by handler type, pushed by TRY_BLOCK
+	     evaluation.  The only difference between the functions is that
+	     __cxa_begin_catch pops the handler type from the vector and keeps
+	     the VAR_DECL last and decreases uncaught_exceptions.  The
+	     VAR_DECL after __cxa_begin_catch serves as the current exception
+	     and is then popped in __cxa_end_catch evaluation.  */
+	  tree handler_type = ctx->global->caught_exceptions.last ();
+	  if (handler_type && VAR_P (handler_type))
+	    goto no_caught_exceptions;
+	  unsigned idx = ctx->global->caught_exceptions.length () - 2;
+	  arg = ctx->global->caught_exceptions[idx];
+	  gcc_assert (VAR_P (arg));
+	  if (kind == CXA_BEGIN_CATCH)
+	    {
+	      ctx->global->caught_exceptions.pop ();
+	      --ctx->global->uncaught_exceptions;
+	    }
+	  if (handler_type == NULL_TREE)
+	    /* Used for catch (...).  Just return void.  */
+	    return void_node;
+	  else if (POINTER_TYPE_P (handler_type))
+	    {
+	      /* Used for catch of a pointer.  */
+	      if (TREE_CODE (TREE_TYPE (arg)) == ARRAY_TYPE)
+		arg = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (arg)), arg,
+			      size_zero_node, NULL_TREE, NULL_TREE);
+	      arg = cp_convert (handler_type, arg,
+				ctx->quiet ? tf_none : tf_warning_or_error);
+	      if (arg == error_mark_node)
+		{
+		  *non_constant_p = true;
+		  return call;
+		}
+	    }
+	  else
+	    {
+	      /* Used for catch of a non-pointer type.  */
+	      tree exc_type = strip_array_types (TREE_TYPE (arg));
+	      tree exc_ptr_type = build_pointer_type (exc_type);
+	      arg = build_fold_addr_expr_with_type (arg, exc_ptr_type);
+	      if (CLASS_TYPE_P (handler_type))
+		{
+		  tree ptr_type = build_pointer_type (handler_type);
+		  arg = cp_convert (ptr_type, arg,
+				    ctx->quiet ? tf_none
+				    : tf_warning_or_error);
+		  if (arg == error_mark_node)
+		    {
+		      *non_constant_p = true;
+		      return call;
+		    }
+		}
+	    }
+	  return cxx_eval_constant_expression (ctx, arg, vc_prvalue,
+					       non_constant_p, overflow_p,
+					       jump_target);
+	}
+  for (int i = 0; i < nargs; ++i)
+    {
+      args[i] = cxx_eval_constant_expression (ctx, CALL_EXPR_ARG (call, i),
+					      vc_prvalue, non_constant_p,
+					      overflow_p, jump_target);
+      if (*non_constant_p)
+	return call;
+      if (*jump_target)
+	return NULL_TREE;
+    }
+  switch (kind)
+    {
+    case CXA_ALLOCATE_EXCEPTION:
+      if (nargs != 1)
+	goto invalid_nargs;
+      if (!tree_fits_uhwi_p (args[0]))
+	{
+	  if (!ctx->quiet)
+	    error_at (loc, "cannot allocate exception: size not constant");
+	  *non_constant_p = true;
+	  return call;
+	}
+      else
+	{
+	  tree type = build_array_type_nelts (char_type_node,
+					      tree_to_uhwi (args[0]));
+	  tree var = cxa_allocate_exception (loc, ctx, type, size_zero_node);
+	  ctx->global->put_value (var, NULL_TREE);
+	  return fold_convert (ptr_type_node, build_address (var));
+	}
+    case CXA_FREE_EXCEPTION:
+      if (nargs != 1)
+	goto invalid_nargs;
+      arg = cxa_check_throw_arg (args[0], true);
+      if (arg == NULL_TREE)
+	{
+	invalid_ptr:
+	  if (!ctx->quiet)
+	    error_at (loc, "first argument to %qD function not result of "
+		      "%<__cxa_allocate_exception%>", fndecl);
+	  *non_constant_p = true;
+	  return call;
+	}
+      DECL_NAME (arg) = heap_deleted_identifier;
+      ctx->global->destroy_value (arg);
+      ctx->global->heap_dealloc_count++;
+      return void_node;
+    case CXA_THROW:
+      if (nargs != 3)
+	goto invalid_nargs;
+      arg = cxa_check_throw_arg (args[0], false);
+      if (arg == NULL_TREE)
+	goto invalid_ptr;
+      DECL_EXCEPTION_REFCOUNT (arg)
+	= size_binop (PLUS_EXPR, DECL_EXCEPTION_REFCOUNT (arg),
+		      size_one_node);
+      ++ctx->global->uncaught_exceptions;
+      *jump_target = arg;
+      return void_node;
+    case CXA_BEGIN_CATCH:
+    case CXA_GET_EXCEPTION_PTR:
+      goto invalid_nargs;
+    case CXA_END_CATCH:
+      if (nargs != 0)
+	goto invalid_nargs;
+      if (ctx->global->caught_exceptions.is_empty ())
+	{
+	no_active_exc:
+	  if (!ctx->quiet)
+	    error_at (loc, "%qD called with no caught exceptions active",
+		      fndecl);
+	  *non_constant_p = true;
+	  return call;
+	}
+      else
+	{
+	  arg = ctx->global->caught_exceptions.pop ();
+	  if (arg == NULL_TREE || !VAR_P (arg))
+	    goto no_active_exc;
+	free_except:
+	  DECL_EXCEPTION_REFCOUNT (arg)
+	    = size_binop (MINUS_EXPR, DECL_EXCEPTION_REFCOUNT (arg),
+			  size_one_node);
+	  if (integer_zerop (DECL_EXCEPTION_REFCOUNT (arg)))
+	    {
+	      if (type_build_dtor_call (TREE_TYPE (arg)))
+		{
+		  tree cleanup
+		    = cxx_maybe_build_cleanup (arg, (ctx->quiet ? tf_none
+						     : tf_warning_or_error));
+		  if (cleanup == error_mark_node)
+		    *non_constant_p = true;
+		  tree jmp_target = NULL_TREE;
+		  cxx_eval_constant_expression (ctx, cleanup, vc_discard,
+						non_constant_p, overflow_p,
+						&jmp_target);
+		  if (throws (&jmp_target))
+		    *jump_target = jmp_target;
+		}
+	      DECL_NAME (arg) = heap_deleted_identifier;
+	      ctx->global->destroy_value (arg);
+	      ctx->global->heap_dealloc_count++;
+	    }
+	}
+      return void_node;
+    case CXA_RETHROW:
+      if (nargs != 0)
+	goto invalid_nargs;
+      unsigned idx;
+      FOR_EACH_VEC_ELT_REVERSE (ctx->global->caught_exceptions, idx, arg)
+	if (arg == NULL_TREE || !VAR_P (arg))
+	  --idx;
+	else
+	  break;
+      if (arg == NULL_TREE)
+	{
+	  if (!ctx->quiet)
+	    error_at (loc, "%qD called with no caught exceptions active",
+		      fndecl);
+	  *non_constant_p = true;
+	  return call;
+	}
+      DECL_EXCEPTION_REFCOUNT (arg)
+	= size_binop (PLUS_EXPR, DECL_EXCEPTION_REFCOUNT (arg), size_one_node);
+      ++ctx->global->uncaught_exceptions;
+      *jump_target = arg;
+      return void_node;
+    case CXA_BAD_CAST:
+    case CXA_BAD_TYPEID:
+    case CXA_THROW_BAD_ARRAY_NEW_LENGTH:
+      if (nargs != 0)
+	goto invalid_nargs;
+      else
+	{
+	  tree name;
+	  switch (kind)
+	    {
+	    case CXA_BAD_CAST:
+	      name = get_identifier ("bad_cast");
+	      break;
+	    case CXA_BAD_TYPEID:
+	      name = get_identifier ("bad_typeid");
+	      break;
+	    case CXA_THROW_BAD_ARRAY_NEW_LENGTH:
+	      name = get_identifier ("bad_array_new_length");
+	      break;
+	    default:
+	      gcc_unreachable ();
+	    }
+	  tree decl = lookup_qualified_name (std_node, name);
+	  if (TREE_CODE (decl) != TYPE_DECL
+	      || !CLASS_TYPE_P (TREE_TYPE (decl))
+	      || !type_build_ctor_call (TREE_TYPE (decl)))
+	    {
+	      if (!ctx->quiet)
+		error_at (loc, "%qD called without %<std::%D%> being defined",
+			  fndecl, name);
+	      *non_constant_p = true;
+	      return call;
+	    }
+	  tree type = TREE_TYPE (decl);
+	  tree var = cxa_allocate_exception (loc, ctx, type, size_one_node);
+	  tree ctor
+	    = build_special_member_call (var, complete_ctor_identifier,
+					 NULL, type, LOOKUP_NORMAL,
+					 ctx->quiet ? tf_none
+					 : tf_warning_or_error);
+	  if (ctor == error_mark_node)
+	    {
+	      *non_constant_p = true;
+	      return call;
+	    }
+	  if (TREE_CONSTANT (ctor))
+	    ctx->global->put_value (var, ctor);
+	  else
+	    {
+	      ctx->global->put_value (var, NULL_TREE);
+	      cxx_eval_constant_expression (ctx, ctor, vc_discard,
+					    non_constant_p, overflow_p,
+					    jump_target);
+	      if (*non_constant_p)
+		return call;
+	      if (throws (jump_target))
+		return NULL_TREE;
+	    }
+	  ++ctx->global->uncaught_exceptions;
+	  *jump_target = var;
+	}
+      return void_node;
+    case STD_UNCAUGHT_EXCEPTIONS:
+      if (nargs != 0)
+	goto invalid_nargs;
+      /* Similarly to __builtin_is_constant_evaluated (), we don't
+	 want to give a definite answer during mce_unknown evaluation,
+	 because that might prevent evaluation later on when some
+	 exceptions might be uncaught.  But unlike that, we don't
+	 want to constant fold it even during cp_fold, because at runtime
+	 std::uncaught_exceptions () might still be non-zero.  */
+      if (ctx->manifestly_const_eval != mce_true)
+	{
+	  *non_constant_p = true;
+	  return call;
+	}
+      return build_int_cst (integer_type_node,
+			    ctx->global->uncaught_exceptions);
+    case STD_CURRENT_EXCEPTION:
+      if (nargs != 0)
+	goto invalid_nargs;
+      else
+	{
+	  tree name = get_identifier ("exception_ptr");
+	  tree decl = lookup_qualified_name (std_node, name);
+	  tree fld;
+	  if (TREE_CODE (decl) != TYPE_DECL
+	      || !CLASS_TYPE_P (TREE_TYPE (decl))
+	      || !COMPLETE_TYPE_P (TREE_TYPE (decl))
+	      || !(fld = next_aggregate_field (TYPE_FIELDS (TREE_TYPE (decl))))
+	      || DECL_ARTIFICIAL (fld)
+	      || TREE_CODE (TREE_TYPE (fld)) != POINTER_TYPE
+	      || next_aggregate_field (DECL_CHAIN (fld))
+	      || !tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (decl)),
+				      TYPE_SIZE (TREE_TYPE (fld))))
+	    {
+	      if (!ctx->quiet)
+		error_at (loc, "%qD called without supportable %qs",
+			  fndecl, "std::exception_ptr");
+	      *non_constant_p = true;
+	      return call;
+	    }
+	  FOR_EACH_VEC_ELT_REVERSE (ctx->global->caught_exceptions, idx, arg)
+	    if (arg == NULL_TREE || !VAR_P (arg))
+	      --idx;
+	    else
+	      break;
+	  /* Similarly to __builtin_is_constant_evaluated (), we don't
+	     want to give a definite answer during mce_unknown evaluation,
+	     because that might prevent evaluation later on when some
+	     exceptions might be current.  But unlike that, we don't
+	     want to constant fold it to null even during cp_fold, because
+	     at runtime std::current_exception () might still be non-null.  */
+	  if (ctx->manifestly_const_eval != mce_true && arg == NULL_TREE)
+	    {
+	      *non_constant_p = true;
+	      return call;
+	    }
+	  if (arg == NULL_TREE)
+	    arg = build_zero_cst (TREE_TYPE (fld));
+	  else
+	    {
+	      DECL_EXCEPTION_REFCOUNT (arg)
+		= size_binop (PLUS_EXPR, DECL_EXCEPTION_REFCOUNT (arg),
+			      size_one_node);
+	      arg = fold_convert (ptr_type_node, build_address (arg));
+	    }
+	  return build_constructor_single (TREE_TYPE (decl), fld, arg);
+	}
+    case STD_RETHROW_EXCEPTION:
+      if (nargs != 1)
+	goto invalid_nargs;
+      if (TYPE_REF_P (TREE_TYPE (args[0])))
+	{
+	  arg = args[0];
+	  STRIP_NOPS (arg);
+	  if (TREE_CODE (arg) == ADDR_EXPR)
+	    {
+	      args[0]
+		= cxx_eval_constant_expression (ctx, TREE_OPERAND (arg, 0),
+						vc_prvalue, non_constant_p,
+						overflow_p, jump_target);
+	      if (*non_constant_p)
+		return call;
+	      if (*jump_target)
+		return NULL_TREE;
+	    }
+	}
+      if (TREE_CODE (args[0]) != CONSTRUCTOR
+	  || CONSTRUCTOR_NELTS (args[0]) != 1)
+	{
+	invalid_std_rethrow:
+	  if (!ctx->quiet)
+	    error_at (loc, "%qD called with unexpected %qs argument",
+		      fndecl, "std::exception_ptr");
+	  *non_constant_p = true;
+	  return void_node;
+	}
+      arg = cxa_check_throw_arg (CONSTRUCTOR_ELT (args[0], 0)->value, false);
+      if (arg == NULL_TREE)
+	goto invalid_std_rethrow;
+      DECL_EXCEPTION_REFCOUNT (arg)
+	= size_binop (PLUS_EXPR, DECL_EXCEPTION_REFCOUNT (arg), size_one_node);
+      ++ctx->global->uncaught_exceptions;
+      *jump_target = arg;
+      return void_node;
+    case BUILTIN_EH_PTR_ADJUST_REF:
+      if (nargs != 2)
+	goto invalid_nargs;
+      arg = cxa_check_throw_arg (args[0], false);
+      if (arg == NULL_TREE)
+	goto invalid_ptr;
+      if (integer_onep (args[1]))
+	DECL_EXCEPTION_REFCOUNT (arg)
+	  = size_binop (PLUS_EXPR, DECL_EXCEPTION_REFCOUNT (arg),
+			size_one_node);
+      else if (integer_minus_onep (args[1]))
+	goto free_except;
+      else
+	{
+	  if (!ctx->quiet)
+	    error_at (loc, "%qD called with second argument "
+		      "other than 1 or -1", fndecl);
+	  *non_constant_p = true;
+	}
+      return void_node;
+    default:
+      gcc_unreachable ();
+    }
+}
+
 /* Attempt to evaluate T which represents a call to a builtin function.
    We assume here that all builtin functions evaluate to scalar types
    represented by _CST nodes.  */
@@ -1557,7 +2268,8 @@ static tree find_heap_var_refs (tree *, int *, void *);
 static tree
 cxx_eval_builtin_function_call (const constexpr_ctx *ctx, tree t, tree fun,
 				value_cat lval,
-				bool *non_constant_p, bool *overflow_p)
+				bool *non_constant_p, bool *overflow_p,
+				tree *jump_target)
 {
   const int nargs = call_expr_nargs (t);
   tree *args = (tree *) alloca (nargs * sizeof (tree));
@@ -1603,6 +2315,12 @@ cxx_eval_builtin_function_call (const constexpr_ctx *ctx, tree t, tree fun,
       return fold_builtin_source_location (t);
     }
 
+  if (fndecl_built_in_p (fun, CP_BUILT_IN_EH_PTR_ADJUST_REF,
+			 BUILT_IN_FRONTEND))
+    return cxx_eval_cxa_builtin_fn (ctx, t, BUILTIN_EH_PTR_ADJUST_REF,
+				    fun, non_constant_p, overflow_p,
+				    jump_target);
+
   int strops = 0;
   int strret = 0;
   if (fndecl_built_in_p (fun, BUILT_IN_NORMAL))
@@ -1677,8 +2395,14 @@ cxx_eval_builtin_function_call (const constexpr_ctx *ctx, tree t, tree fun,
 	  || potential_constant_expression (arg))
 	{
 	  bool dummy1 = false, dummy2 = false;
+	  tree jmp_target = NULL_TREE;
 	  arg = cxx_eval_constant_expression (&new_ctx, arg, vc_prvalue,
-					      &dummy1, &dummy2);
+					      &dummy1, &dummy2, &jmp_target);
+	  if (jmp_target)
+	    {
+	      *jump_target = jmp_target;
+	      return NULL_TREE;
+	    }
 	}
 
       if (bi_const_p)
@@ -1767,7 +2491,8 @@ cxx_eval_builtin_function_call (const constexpr_ctx *ctx, tree t, tree fun,
     }
 
   return cxx_eval_constant_expression (&new_ctx, new_call, lval,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
 }
 
 /* TEMP is the constant value of a temporary object of type TYPE.  Adjust
@@ -1882,7 +2607,8 @@ addr_of_non_const_var (tree *tp, int *walk_subtrees, void *data)
 static tree
 cxx_bind_parameters_in_call (const constexpr_ctx *ctx, tree t, tree fun,
 			     tree orig_fun, bool *non_constant_p,
-			     bool *overflow_p, bool *non_constant_args)
+			     bool *overflow_p, bool *non_constant_args,
+			     tree *jump_target)
 {
   int nargs = call_expr_nargs (t);
   tree parms = DECL_ARGUMENTS (fun);
@@ -1958,14 +2684,16 @@ cxx_bind_parameters_in_call (const constexpr_ctx *ctx, tree t, tree fun,
 	  /* Undo convert_for_arg_passing work here.  */
 	  x = convert_from_reference (x);
 	  arg = cxx_eval_constant_expression (ctx, x, vc_glvalue,
-					      non_constant_p, overflow_p);
+					      non_constant_p, overflow_p,
+					      jump_target);
 	}
       else
 	/* Normally we would strip a TARGET_EXPR in an initialization context
 	   such as this, but here we do the elision differently: we keep the
 	   TARGET_EXPR, and use its CONSTRUCTOR as the value of the parm.  */
 	arg = cxx_eval_constant_expression (ctx, x, vc_prvalue,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
       /* Check we aren't dereferencing a null pointer when calling a non-static
 	 member function, which is undefined behaviour.  */
       if (i == 0 && DECL_OBJECT_MEMBER_FUNCTION_P (fun)
@@ -1983,6 +2711,8 @@ cxx_bind_parameters_in_call (const constexpr_ctx *ctx, tree t, tree fun,
       /* Don't VERIFY_CONSTANT here.  */
       if (*non_constant_p && ctx->quiet)
 	break;
+      if (*jump_target)
+	break;
       /* Just discard ellipsis args after checking their constantitude.  */
       if (!parms)
 	continue;
@@ -2094,9 +2824,10 @@ fold_operand (tree e, const constexpr_ctx *ctx)
   if (ctx)
     {
       bool new_non_constant_p = false, new_overflow_p = false;
+      tree jmp_target = NULL_TREE;
       e = cxx_eval_constant_expression (ctx, e, vc_prvalue,
 					&new_non_constant_p,
-					&new_overflow_p);
+					&new_overflow_p, &jmp_target);
     }
   else
     e = fold_non_dependent_expr (e, tf_none, /*manifestly_const_eval=*/true);
@@ -2155,10 +2886,15 @@ diagnose_failing_condition (tree bad, location_t cloc, bool show_expr_p,
   if (TREE_CODE (bad) == CLEANUP_POINT_EXPR)
     bad = TREE_OPERAND (bad, 0);
 
+  auto_diagnostic_nesting_level sentinel;
+
   /* Actually explain the failure if this is a concept check or a
      requires-expression.  */
   if (concept_check_p (bad) || TREE_CODE (bad) == REQUIRES_EXPR)
     diagnose_constraints (cloc, bad, NULL_TREE);
+  /* Similarly if this is a standard trait.  */
+  else if (maybe_diagnose_standard_trait (cloc, bad))
+    ;
   else if (COMPARISON_CLASS_P (bad)
 	   && ARITHMETIC_TYPE_P (TREE_TYPE (TREE_OPERAND (bad, 0))))
     {
@@ -2183,7 +2919,7 @@ cxx_eval_assert (const constexpr_ctx *ctx, tree arg, const char *msg,
   if (*non_constant_p)
     return true;
 
-  tree eval;
+  tree eval, jmp_target = NULL_TREE;
   if (!evaluated)
     {
       if (!potential_rvalue_constant_expression (arg))
@@ -2196,12 +2932,15 @@ cxx_eval_assert (const constexpr_ctx *ctx, tree arg, const char *msg,
       modifiable_tracker ms (new_ctx.global);
       eval = cxx_eval_constant_expression (&new_ctx, arg, vc_prvalue,
 					   &new_non_constant_p,
-					   &new_overflow_p);
+					   &new_overflow_p, &jmp_target);
     }
   else
     eval = cxx_eval_constant_expression (ctx, arg, vc_prvalue,
 					 non_constant_p,
-					 overflow_p);
+					 overflow_p, &jmp_target);
+  if (jmp_target)
+    return true;
+
   if (!*non_constant_p && integer_zerop (eval))
     {
       if (!ctx->quiet)
@@ -2233,7 +2972,8 @@ cxx_eval_assert (const constexpr_ctx *ctx, tree arg, const char *msg,
 static tree
 cxx_eval_internal_function (const constexpr_ctx *ctx, tree t,
 			    value_cat lval,
-			    bool *non_constant_p, bool *overflow_p)
+			    bool *non_constant_p, bool *overflow_p,
+			    tree *jump_target)
 {
   enum tree_code opcode = ERROR_MARK;
 
@@ -2266,13 +3006,15 @@ cxx_eval_internal_function (const constexpr_ctx *ctx, tree t,
     case IFN_LAUNDER:
       return cxx_eval_constant_expression (ctx, CALL_EXPR_ARG (t, 0),
 					   vc_prvalue, non_constant_p,
-					   overflow_p);
+					   overflow_p, jump_target);
 
     case IFN_VEC_CONVERT:
       {
 	tree arg = cxx_eval_constant_expression (ctx, CALL_EXPR_ARG (t, 0),
 						 vc_prvalue, non_constant_p,
-						 overflow_p);
+						 overflow_p, jump_target);
+	if (*jump_target)
+	  return NULL_TREE;
 	if (TREE_CODE (arg) == VECTOR_CST)
 	  if (tree r = fold_const_call (CFN_VEC_CONVERT, TREE_TYPE (t), arg))
 	    return r;
@@ -2290,10 +3032,13 @@ cxx_eval_internal_function (const constexpr_ctx *ctx, tree t,
   /* Evaluate constant arguments using OPCODE and return a complex
      number containing the result and the overflow bit.  */
   tree arg0 = cxx_eval_constant_expression (ctx, CALL_EXPR_ARG (t, 0), lval,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
   tree arg1 = cxx_eval_constant_expression (ctx, CALL_EXPR_ARG (t, 1), lval,
-					    non_constant_p, overflow_p);
-
+					    non_constant_p, overflow_p,
+					    jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   if (TREE_CODE (arg0) == INTEGER_CST && TREE_CODE (arg1) == INTEGER_CST)
     {
       location_t loc = cp_expr_loc_or_input_loc (t);
@@ -2566,7 +3311,8 @@ get_component_with_type (tree path, tree type, tree stop)
 
 static tree
 cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
-			  bool *non_constant_p, bool *overflow_p)
+			  bool *non_constant_p, bool *overflow_p,
+			  tree *jump_target)
 {
   /* T will be something like
       __dynamic_cast ((B*) b, &_ZTI1B, &_ZTI1D, 8)
@@ -2585,19 +3331,26 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
 
   /* TYPE can only be either T* or T&.  We can't know which of these it
      is by looking at TYPE, but OBJ will be "(T*) x" in the first case,
-     and something like "(T*)(T&)(T*) x" in the second case.  */
-  bool reference_p = false;
+     and something like "(T*)(T&)(T*) x" in the second case.
+     This is true for the reference cases in C++ < 26 or when exceptions
+     aren't enabled, in that case we should diagnose errors.  For C++26
+     with exceptions we should silently evaluate to null pointer and
+     let the callers call __cxa_bad_cast () later to throw an exception.  */
+  bool fail_for_non_constant_p = false;
   while (CONVERT_EXPR_P (obj) || TREE_CODE (obj) == SAVE_EXPR)
     {
-      reference_p |= TYPE_REF_P (TREE_TYPE (obj));
+      if (cxx_dialect < cxx26 || !flag_exceptions)
+	fail_for_non_constant_p |= TYPE_REF_P (TREE_TYPE (obj));
       obj = TREE_OPERAND (obj, 0);
     }
 
   /* Evaluate the object so that we know its dynamic type.  */
   obj = cxx_eval_constant_expression (ctx, obj, vc_prvalue, non_constant_p,
-				      overflow_p);
+				      overflow_p, jump_target);
   if (*non_constant_p)
     return call;
+  if (*jump_target)
+    return NULL_TREE;
 
   /* For dynamic_cast from classes with virtual bases we can get something
      like (virt_base *)(&d + 16) as OBJ.  Try to convert that into
@@ -2609,7 +3362,7 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
       if (TREE_CODE (objo) == POINTER_PLUS_EXPR)
 	{
 	  objo = cxx_fold_indirect_ref (ctx, loc, TREE_TYPE (TREE_TYPE (obj)),
-					obj);
+					obj, NULL, jump_target);
 	  if (objo)
 	    obj = build_fold_addr_expr (objo);
 	}
@@ -2625,7 +3378,7 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
 		? TREE_OPERAND (obj, 1) : obj))
     if (TREE_CODE (t) != FIELD_DECL || !DECL_FIELD_IS_BASE (t))
       {
-	if (reference_p)
+	if (fail_for_non_constant_p)
 	  {
 	    if (!ctx->quiet)
 	      {
@@ -2647,9 +3400,12 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
      constructor or destructor's class.  */
   tree vtable = build_vfield_ref (obj, objtype);
   vtable = cxx_eval_constant_expression (ctx, vtable, vc_prvalue,
-					 non_constant_p, overflow_p);
+					 non_constant_p, overflow_p,
+					 jump_target);
   if (*non_constant_p)
     return call;
+  if (*jump_target)
+    return NULL_TREE;
   /* With -fsanitize=vptr, we initialize all vtable pointers to null,
      so it's possible that we got a null pointer now.  */
   if (integer_zerop (vtable))
@@ -2681,7 +3437,7 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
       /* If not accessible, give an error.  */
       if (t == error_mark_node)
 	{
-	  if (reference_p)
+	  if (fail_for_non_constant_p)
 	    {
 	      if (!ctx->quiet)
 		{
@@ -2714,7 +3470,7 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
   obj = get_component_with_type (obj, mdtype, NULL_TREE);
   if (obj == error_mark_node)
     {
-      if (reference_p)
+      if (fail_for_non_constant_p)
 	{
 	  if (!ctx->quiet)
 	    {
@@ -2736,7 +3492,7 @@ cxx_eval_dynamic_cast_fn (const constexpr_ctx *ctx, tree call,
   tree binfo = lookup_base (mdtype, type, ba_check, &b_kind, tf_none);
   if (!binfo || binfo == error_mark_node)
     {
-      if (reference_p)
+      if (fail_for_non_constant_p)
 	{
 	  if (!ctx->quiet)
 	    {
@@ -2832,7 +3588,7 @@ replace_decl (tree *tp, tree decl, tree replacement)
 static tree
 cxx_eval_thunk_call (const constexpr_ctx *ctx, tree t, tree thunk_fndecl,
 		     value_cat lval,
-		     bool *non_constant_p, bool *overflow_p)
+		     bool *non_constant_p, bool *overflow_p, tree *jump_target)
 {
   tree function = THUNK_TARGET (thunk_fndecl);
 
@@ -2875,7 +3631,8 @@ cxx_eval_thunk_call (const constexpr_ctx *ctx, tree t, tree thunk_fndecl,
 		       new_call, offset);
 
   return cxx_eval_constant_expression (ctx, new_call, lval,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
 }
 
 /* If OBJECT is of const class type, evaluate it to a CONSTRUCTOR and set
@@ -2885,7 +3642,7 @@ cxx_eval_thunk_call (const constexpr_ctx *ctx, tree t, tree thunk_fndecl,
 static void
 cxx_set_object_constness (const constexpr_ctx *ctx, tree object,
 			  bool readonly_p, bool *non_constant_p,
-			  bool *overflow_p)
+			  bool *overflow_p, tree *jump_target)
 {
   if (CLASS_TYPE_P (TREE_TYPE (object))
       && CP_TYPE_CONST_P (TREE_TYPE (object)))
@@ -2893,8 +3650,11 @@ cxx_set_object_constness (const constexpr_ctx *ctx, tree object,
       /* Subobjects might not be stored in ctx->global->values but we
 	 can get its CONSTRUCTOR by evaluating *this.  */
       tree e = cxx_eval_constant_expression (ctx, object, vc_prvalue,
-					     non_constant_p, overflow_p);
-      if (TREE_CODE (e) == CONSTRUCTOR && !*non_constant_p)
+					     non_constant_p, overflow_p,
+					     jump_target);
+      if (!*non_constant_p
+	  && !throws (jump_target)
+	  && TREE_CODE (e) == CONSTRUCTOR)
 	TREE_READONLY (e) = readonly_p;
     }
 }
@@ -2906,20 +3666,25 @@ cxx_set_object_constness (const constexpr_ctx *ctx, tree object,
 static tree
 cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 			  value_cat lval,
-			  bool *non_constant_p, bool *overflow_p)
+			  bool *non_constant_p, bool *overflow_p,
+			  tree *jump_target)
 {
   location_t loc = cp_expr_loc_or_input_loc (t);
   tree fun = get_function_named_in_call (t);
 
   if (fun == NULL_TREE)
     return cxx_eval_internal_function (ctx, t, lval,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
 
   if (TREE_CODE (fun) != FUNCTION_DECL)
     {
       /* Might be a constexpr function pointer.  */
       fun = cxx_eval_constant_expression (ctx, fun, vc_prvalue,
-					  non_constant_p, overflow_p);
+					  non_constant_p, overflow_p,
+					  jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       STRIP_NOPS (fun);
       if (TREE_CODE (fun) == ADDR_EXPR)
 	fun = TREE_OPERAND (fun, 0);
@@ -2971,9 +3736,12 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 
   if (fndecl_built_in_p (fun))
     return cxx_eval_builtin_function_call (ctx, t, fun,
-					   lval, non_constant_p, overflow_p);
+					   lval, non_constant_p, overflow_p,
+					   jump_target);
   if (DECL_THUNK_P (fun))
-    return cxx_eval_thunk_call (ctx, t, fun, lval, non_constant_p, overflow_p);
+    return cxx_eval_thunk_call (ctx, t, fun, lval, non_constant_p, overflow_p,
+				jump_target);
+  bool non_constexpr_call = false;
   if (!maybe_constexpr_fn (fun))
     {
       if (TREE_CODE (t) == CALL_EXPR
@@ -2988,7 +3756,10 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 	    {
 	      tree arg = CALL_EXPR_ARG (t, i);
 	      arg = cxx_eval_constant_expression (ctx, arg, vc_prvalue,
-						  non_constant_p, overflow_p);
+						  non_constant_p, overflow_p,
+						  jump_target);
+	      if (*jump_target)
+		return NULL_TREE;
 	      /* Deleting a non-constant pointer has a better error message
 		 below.  */
 	      if (new_op_p || i != 0)
@@ -3103,7 +3874,10 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 	    {
 	      tree arg = CALL_EXPR_ARG (t, i);
 	      arg = cxx_eval_constant_expression (ctx, arg, vc_prvalue,
-						  non_constant_p, overflow_p);
+						  non_constant_p, overflow_p,
+						  jump_target);
+	      if (*jump_target)
+		return NULL_TREE;
 	      if (i == 1)
 		arg1 = arg;
 	      else
@@ -3113,16 +3887,31 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 	  return arg1;
 	}
       else if (cxx_dynamic_cast_fn_p (fun))
-	return cxx_eval_dynamic_cast_fn (ctx, t, non_constant_p, overflow_p);
+	return cxx_eval_dynamic_cast_fn (ctx, t, non_constant_p, overflow_p,
+					 jump_target);
+      else if (enum cxa_builtin kind = cxx_cxa_builtin_fn_p (fun))
+	return cxx_eval_cxa_builtin_fn (ctx, t, kind, fun,
+					non_constant_p, overflow_p,
+					jump_target);
 
-      if (!ctx->quiet)
+      /* Calls to non-constexpr functions can be diagnosed right away
+	 before C++26, though in C++26 evaluation of the arguments might
+	 throw and if caught it could be still constant expression.
+	 So for C++26 this is diagnosed only after
+	 cxx_bind_parameters_in_call.  */
+      if (cxx_dialect >= cxx26)
+	non_constexpr_call = true;
+      else
 	{
-	  if (!lambda_static_thunk_p (fun))
-	    error_at (loc, "call to non-%<constexpr%> function %qD", fun);
-	  explain_invalid_constexpr_fn (fun);
+	  if (!ctx->quiet)
+	    {
+	      if (!lambda_static_thunk_p (fun))
+		error_at (loc, "call to non-%<constexpr%> function %qD", fun);
+	      explain_invalid_constexpr_fn (fun);
+	    }
+	  *non_constant_p = true;
+	  return t;
 	}
-      *non_constant_p = true;
-      return t;
     }
 
   constexpr_ctx new_ctx = *ctx;
@@ -3158,7 +3947,8 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
   constexpr_call new_call;
   new_call.bindings
     = cxx_bind_parameters_in_call (ctx, t, fun, orig_fun, non_constant_p,
-				   overflow_p, &non_constant_args);
+				   overflow_p, &non_constant_args,
+				   jump_target);
 
   /* We build up the bindings list before we know whether we already have this
      call cached.  If we don't end up saving these bindings, ggc_free them when
@@ -3172,8 +3962,21 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
     void preserve () { bindings = NULL; }
   } fb (new_call.bindings);
 
+  if (*jump_target)
+    return NULL_TREE;
   if (*non_constant_p)
     return t;
+  if (non_constexpr_call)
+    {
+      if (!ctx->quiet)
+	{
+	  if (!lambda_static_thunk_p (fun))
+	    error_at (loc, "call to non-%<constexpr%> function %qD", fun);
+	  explain_invalid_constexpr_fn (fun);
+	}
+      *non_constant_p = true;
+      return t;
+    }
 
   /* We can't defer instantiating the function any longer.  */
   if (!DECL_INITIAL (fun)
@@ -3246,7 +4049,9 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
       new_obj = TREE_VEC_ELT (new_call.bindings, 0);
       bool empty_base = false;
       new_obj = cxx_fold_indirect_ref (ctx, loc, DECL_CONTEXT (fun), new_obj,
-				       &empty_base);
+				       &empty_base, jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       /* If we're initializing an empty class, don't set constness, because
 	 cxx_fold_indirect_ref will return the wrong object to set constness
 	 of.  */
@@ -3395,7 +4200,7 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 	     semantics are no longer in effect; see [class.dtor]p5.  */
 	  if (new_obj && DECL_DESTRUCTOR_P (fun))
 	    cxx_set_object_constness (ctx, new_obj, /*readonly_p=*/false,
-				      non_constant_p, overflow_p);
+				      non_constant_p, overflow_p, jump_target);
 
 	  /* If this is a constructor, we are beginning the lifetime of the
 	     object we are initializing.  */
@@ -3404,21 +4209,30 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 	      && TREE_CODE (new_obj) == COMPONENT_REF
 	      && TREE_CODE (TREE_TYPE (TREE_OPERAND (new_obj, 0))) == UNION_TYPE)
 	    {
+	      tree ctor = build_constructor (TREE_TYPE (new_obj), NULL);
+	      CONSTRUCTOR_NO_CLEARING (ctor) = true;
 	      tree activate = build2 (INIT_EXPR, TREE_TYPE (new_obj),
-				      new_obj,
-				      build_constructor (TREE_TYPE (new_obj),
-							 NULL));
+				      new_obj, ctor);
 	      cxx_eval_constant_expression (ctx, activate,
-					    lval, non_constant_p, overflow_p);
+					    lval, non_constant_p, overflow_p,
+					    jump_target);
 	      ggc_free (activate);
+	      if (*jump_target)
+		return NULL_TREE;
 	    }
 
-	  tree jump_target = NULL_TREE;
+	  tree jmp_target = NULL_TREE;
 	  cxx_eval_constant_expression (&call_ctx, body,
 					vc_discard, non_constant_p, overflow_p,
-					&jump_target);
+					&jmp_target);
 
-	  if (DECL_CONSTRUCTOR_P (fun))
+	  if (!*non_constant_p && throws (&jmp_target))
+	    {
+	      result = NULL_TREE;
+	      cacheable = false;
+	      *jump_target = jmp_target;
+	    }
+	  else if (DECL_CONSTRUCTOR_P (fun))
 	    /* This can be null for a subobject constructor call, in
 	       which case what we care about is the initialization
 	       side-effects rather than the value.  We could get at the
@@ -3446,7 +4260,7 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 	     marking the CONSTRUCTOR TREE_READONLY.  */
 	  if (new_obj && DECL_CONSTRUCTOR_P (fun))
 	    cxx_set_object_constness (ctx, new_obj, /*readonly_p=*/true,
-				      non_constant_p, overflow_p);
+				      non_constant_p, overflow_p, jump_target);
 
 	  /* Remove the parms/result from the values map.  */
 	  destroy_value_checked (ctx, res, non_constant_p);
@@ -3506,7 +4320,13 @@ cxx_eval_call_expression (const constexpr_ctx *ctx, tree t,
 		    cacheable = false;
 		    result = cxx_eval_constant_expression (ctx, result, lval,
 							   non_constant_p,
-							   overflow_p);
+							   overflow_p,
+							   jump_target);
+		    if (*jump_target)
+		      {
+			cacheable = false;
+			result = NULL_TREE;
+		      }
 		  }
 	      }
 
@@ -3864,12 +4684,16 @@ cxx_eval_check_shift_p (location_t loc, const constexpr_ctx *ctx,
 static tree
 cxx_eval_unary_expression (const constexpr_ctx *ctx, tree t,
 			   bool /*lval*/,
-			   bool *non_constant_p, bool *overflow_p)
+			   bool *non_constant_p, bool *overflow_p,
+			   tree *jump_target)
 {
   tree r;
   tree orig_arg = TREE_OPERAND (t, 0);
   tree arg = cxx_eval_constant_expression (ctx, orig_arg, vc_prvalue,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (arg);
   location_t loc = EXPR_LOCATION (t);
   enum tree_code code = TREE_CODE (t);
@@ -3893,7 +4717,7 @@ cxx_eval_unary_expression (const constexpr_ctx *ctx, tree t,
 static tree
 cxx_fold_pointer_plus_expression (const constexpr_ctx *ctx, tree t,
 				  tree lhs, tree rhs, bool *non_constant_p,
-				  bool *overflow_p)
+				  bool *overflow_p, tree *jump_target)
 {
   STRIP_NOPS (lhs);
   if (TREE_CODE (lhs) != ADDR_EXPR)
@@ -3915,9 +4739,12 @@ cxx_fold_pointer_plus_expression (const constexpr_ctx *ctx, tree t,
       t = fold_convert_loc (loc, ssizetype, TREE_OPERAND (lhs, 1));
       tree nelts = array_type_nelts_top (TREE_TYPE (TREE_OPERAND (lhs, 0)));
       nelts = cxx_eval_constant_expression (ctx, nelts, vc_prvalue,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
       if (*non_constant_p)
 	return NULL_TREE;
+      if (*jump_target)
+	return NULL_TREE;
       /* Don't fold an out-of-bound access.  */
       if (!tree_int_cst_le (t, nelts))
 	return NULL_TREE;
@@ -3937,7 +4764,8 @@ cxx_fold_pointer_plus_expression (const constexpr_ctx *ctx, tree t,
       t = cp_build_addr_expr (t, tf_warning_or_error);
       t = cp_fold_convert (orig_type, t);
       return cxx_eval_constant_expression (ctx, t, vc_prvalue,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
     }
 
   return NULL_TREE;
@@ -3981,22 +4809,29 @@ cxx_maybe_fold_addr_pointer_plus (tree t)
 static tree
 cxx_eval_binary_expression (const constexpr_ctx *ctx, tree t,
 			    value_cat lval,
-			    bool *non_constant_p, bool *overflow_p)
+			    bool *non_constant_p, bool *overflow_p,
+			    tree *jump_target)
 {
   tree r = NULL_TREE;
   tree orig_lhs = TREE_OPERAND (t, 0);
   tree orig_rhs = TREE_OPERAND (t, 1);
   tree lhs, rhs;
   lhs = cxx_eval_constant_expression (ctx, orig_lhs, vc_prvalue,
-				      non_constant_p, overflow_p);
+				      non_constant_p, overflow_p,
+				      jump_target);
   /* Don't VERIFY_CONSTANT here, it's unnecessary and will break pointer
      subtraction.  */
   if (*non_constant_p)
     return t;
+  if (*jump_target)
+    return NULL_TREE;
   rhs = cxx_eval_constant_expression (ctx, orig_rhs, vc_prvalue,
-				      non_constant_p, overflow_p);
+				      non_constant_p, overflow_p,
+				      jump_target);
   if (*non_constant_p)
     return t;
+  if (*jump_target)
+    return NULL_TREE;
 
   location_t loc = EXPR_LOCATION (t);
   enum tree_code code = TREE_CODE (t);
@@ -4052,13 +4887,17 @@ cxx_eval_binary_expression (const constexpr_ctx *ctx, tree t,
       return t;
     }
   else if (code == POINTER_PLUS_EXPR)
-    r = cxx_fold_pointer_plus_expression (ctx, t, lhs, rhs, non_constant_p,
-					  overflow_p);
+    {
+      r = cxx_fold_pointer_plus_expression (ctx, t, lhs, rhs, non_constant_p,
+					    overflow_p, jump_target);
+      if (*jump_target)
+	return NULL_TREE;
+    }
   else if (code == SPACESHIP_EXPR)
     {
       r = genericize_spaceship (loc, type, lhs, rhs);
       return cxx_eval_constant_expression (ctx, r, lval, non_constant_p,
-					   overflow_p);
+					   overflow_p, jump_target);
     }
 
   if (r == NULL_TREE)
@@ -4117,7 +4956,10 @@ cxx_eval_conditional_expression (const constexpr_ctx *ctx, tree t,
 {
   tree val = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
 					   vc_prvalue,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (val);
   if (TREE_CODE (t) == IF_STMT && IF_STMT_CONSTEVAL_P (t))
     {
@@ -4178,19 +5020,29 @@ cxx_eval_conditional_expression (const constexpr_ctx *ctx, tree t,
 
 static tree
 cxx_eval_vector_conditional_expression (const constexpr_ctx *ctx, tree t,
-					bool *non_constant_p, bool *overflow_p)
+					bool *non_constant_p, bool *overflow_p,
+					tree *jump_target)
 {
   tree arg1 = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
 					    vc_prvalue,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (arg1);
   tree arg2 = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1),
 					    vc_prvalue,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (arg2);
   tree arg3 = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 2),
 					    vc_prvalue,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (arg3);
   location_t loc = EXPR_LOCATION (t);
   tree type = TREE_TYPE (t);
@@ -4578,7 +5430,8 @@ diag_array_subscript (location_t loc, const constexpr_ctx *ctx, tree array, tree
 
 static tree
 get_array_or_vector_nelts (const constexpr_ctx *ctx, tree type,
-			   bool *non_constant_p, bool *overflow_p)
+			   bool *non_constant_p, bool *overflow_p,
+			   tree *jump_target)
 {
   tree nelts;
   if (TREE_CODE (type) == ARRAY_TYPE)
@@ -4595,7 +5448,8 @@ get_array_or_vector_nelts (const constexpr_ctx *ctx, tree type,
 
   /* For VLAs, the number of elements won't be an integer constant.  */
   nelts = cxx_eval_constant_expression (ctx, nelts, vc_prvalue,
-					non_constant_p, overflow_p);
+					non_constant_p, overflow_p,
+					jump_target);
   return nelts;
 }
 
@@ -4626,13 +5480,17 @@ extract_string_elt (tree string, unsigned chars_per_elt, unsigned index)
 static tree
 eval_and_check_array_index (const constexpr_ctx *ctx,
 			    tree t, bool allow_one_past,
-			    bool *non_constant_p, bool *overflow_p)
+			    bool *non_constant_p, bool *overflow_p,
+			    tree *jump_target)
 {
   location_t loc = cp_expr_loc_or_input_loc (t);
   tree ary = TREE_OPERAND (t, 0);
   t = TREE_OPERAND (t, 1);
   tree index = cxx_eval_constant_expression (ctx, t, vc_prvalue,
-					     non_constant_p, overflow_p);
+					     non_constant_p, overflow_p,
+					     jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (index);
 
   if (!tree_fits_shwi_p (index)
@@ -4644,7 +5502,9 @@ eval_and_check_array_index (const constexpr_ctx *ctx,
     }
 
   tree nelts = get_array_or_vector_nelts (ctx, TREE_TYPE (ary), non_constant_p,
-					  overflow_p);
+					  overflow_p, jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (nelts);
   if (allow_one_past
       ? !tree_int_cst_le (index, nelts)
@@ -4664,14 +5524,18 @@ eval_and_check_array_index (const constexpr_ctx *ctx,
 static tree
 cxx_eval_array_reference (const constexpr_ctx *ctx, tree t,
 			  value_cat lval,
-			  bool *non_constant_p, bool *overflow_p)
+			  bool *non_constant_p, bool *overflow_p,
+			  tree *jump_target)
 {
   tree oldary = TREE_OPERAND (t, 0);
   tree ary = cxx_eval_constant_expression (ctx, oldary,
 					   lval,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
   if (*non_constant_p)
     return t;
+  if (*jump_target)
+    return NULL_TREE;
   if (!lval
       && TREE_CODE (ary) == VIEW_CONVERT_EXPR
       && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (ary, 0)))
@@ -4681,9 +5545,12 @@ cxx_eval_array_reference (const constexpr_ctx *ctx, tree t,
 
   tree oldidx = TREE_OPERAND (t, 1);
   tree index = eval_and_check_array_index (ctx, t, lval,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
   if (*non_constant_p)
     return t;
+  if (*jump_target)
+    return NULL_TREE;
 
   if (lval && ary == oldary && index == oldidx)
     return t;
@@ -4801,7 +5668,7 @@ cxx_eval_array_reference (const constexpr_ctx *ctx, tree t,
       ctx = &new_ctx;
     }
   t = cxx_eval_constant_expression (ctx, val, lval, non_constant_p,
-				    overflow_p);
+				    overflow_p, jump_target);
   if (new_ctor && t != ctx->ctor)
     free_constructor (ctx->ctor);
   return t;
@@ -4813,7 +5680,8 @@ cxx_eval_array_reference (const constexpr_ctx *ctx, tree t,
 static tree
 cxx_eval_component_reference (const constexpr_ctx *ctx, tree t,
 			      value_cat lval,
-			      bool *non_constant_p, bool *overflow_p)
+			      bool *non_constant_p, bool *overflow_p,
+			      tree *jump_target)
 {
   unsigned HOST_WIDE_INT i;
   tree field;
@@ -4822,9 +5690,12 @@ cxx_eval_component_reference (const constexpr_ctx *ctx, tree t,
   tree orig_whole = TREE_OPERAND (t, 0);
   tree whole = cxx_eval_constant_expression (ctx, orig_whole,
 					     lval,
-					     non_constant_p, overflow_p);
+					     non_constant_p, overflow_p,
+					     jump_target);
   if (*non_constant_p)
     return t;
+  if (*jump_target)
+    return NULL_TREE;
   if (INDIRECT_REF_P (whole)
       && integer_zerop (TREE_OPERAND (whole, 0)))
     {
@@ -4930,10 +5801,23 @@ cxx_eval_component_reference (const constexpr_ctx *ctx, tree t,
     }
 
   /* If there's no explicit init for this field, it's value-initialized.  */
+
+  if (AGGREGATE_TYPE_P (TREE_TYPE (t)))
+    {
+      /* As in cxx_eval_store_expression, insert an empty CONSTRUCTOR
+	 and copy the flags.  */
+      constructor_elt *e = get_or_insert_ctor_field (whole, part);
+      e->value = value = build_constructor (TREE_TYPE (part), NULL);
+      CONSTRUCTOR_ZERO_PADDING_BITS (value)
+	= CONSTRUCTOR_ZERO_PADDING_BITS (whole);
+      return value;
+    }
+
   value = build_value_init (TREE_TYPE (t), tf_warning_or_error);
   return cxx_eval_constant_expression (ctx, value,
 				       lval,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
 }
 
 /* Subroutine of cxx_eval_constant_expression.
@@ -4943,7 +5827,8 @@ cxx_eval_component_reference (const constexpr_ctx *ctx, tree t,
 static tree
 cxx_eval_bit_field_ref (const constexpr_ctx *ctx, tree t,
 			value_cat lval,
-			bool *non_constant_p, bool *overflow_p)
+			bool *non_constant_p, bool *overflow_p,
+			tree *jump_target)
 {
   tree orig_whole = TREE_OPERAND (t, 0);
   tree retval, fldval, utype, mask;
@@ -4951,10 +5836,13 @@ cxx_eval_bit_field_ref (const constexpr_ctx *ctx, tree t,
   HOST_WIDE_INT istart, isize;
   tree whole = cxx_eval_constant_expression (ctx, orig_whole,
 					     lval,
-					     non_constant_p, overflow_p);
+					     non_constant_p, overflow_p,
+					     jump_target);
   tree start, field, value;
   unsigned HOST_WIDE_INT i;
 
+  if (*jump_target)
+    return NULL_TREE;
   if (whole == orig_whole)
     return t;
   /* Don't VERIFY_CONSTANT here; we only want to check that we got a
@@ -5235,7 +6123,7 @@ clear_uchar_or_std_byte_in_mask (location_t loc, tree t, unsigned char *mask)
 
 static tree
 cxx_eval_bit_cast (const constexpr_ctx *ctx, tree t, bool *non_constant_p,
-		   bool *overflow_p)
+		   bool *overflow_p, tree *jump_target)
 {
   if (check_bit_cast_type (ctx, EXPR_LOCATION (t), TREE_TYPE (t),
 			   TREE_TYPE (t))
@@ -5249,9 +6137,12 @@ cxx_eval_bit_cast (const constexpr_ctx *ctx, tree t, bool *non_constant_p,
     }
 
   tree op = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), vc_prvalue,
-					  non_constant_p, overflow_p);
+					  non_constant_p, overflow_p,
+					  jump_target);
   if (*non_constant_p)
     return t;
+  if (*jump_target)
+    return NULL_TREE;
 
   location_t loc = EXPR_LOCATION (t);
   if (BITS_PER_UNIT != 8 || CHAR_BIT != 8)
@@ -5329,8 +6220,9 @@ cxx_eval_bit_cast (const constexpr_ctx *ctx, tree t, bool *non_constant_p,
 	  if (CHECKING_P)
 	    {
 	      tree e = cxx_eval_bare_aggregate (ctx, r, vc_prvalue,
-						non_constant_p, overflow_p);
-	      gcc_checking_assert (e == r);
+						non_constant_p, overflow_p,
+						jump_target);
+	      gcc_checking_assert (e == r && !*jump_target);
 	      r = e;
 	    }
 	}
@@ -5371,19 +6263,24 @@ cxx_eval_bit_cast (const constexpr_ctx *ctx, tree t, bool *non_constant_p,
 static tree
 cxx_eval_logical_expression (const constexpr_ctx *ctx, tree t,
                              tree bailout_value, tree continue_value,
-			     bool *non_constant_p, bool *overflow_p)
+			     bool *non_constant_p, bool *overflow_p,
+			     tree *jump_target)
 {
   tree r;
   tree lhs = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
 					   vc_prvalue, non_constant_p,
-					   overflow_p);
+					   overflow_p, jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (lhs);
   if (tree_int_cst_equal (lhs, bailout_value))
     return lhs;
   gcc_assert (tree_int_cst_equal (lhs, continue_value));
   r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1),
 				    vc_prvalue, non_constant_p,
-				    overflow_p);
+				    overflow_p, jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (r);
   return r;
 }
@@ -5540,7 +6437,8 @@ verify_ctor_sanity (const constexpr_ctx *ctx, tree type)
 static tree
 cxx_eval_bare_aggregate (const constexpr_ctx *ctx, tree t,
 			 value_cat lval,
-			 bool *non_constant_p, bool *overflow_p)
+			 bool *non_constant_p, bool *overflow_p,
+			 tree *jump_target)
 {
   vec<constructor_elt, va_gc> *v = CONSTRUCTOR_ELTS (t);
   bool changed = false;
@@ -5593,7 +6491,10 @@ cxx_eval_bare_aggregate (const constexpr_ctx *ctx, tree t,
 	get_or_insert_ctor_field (ctx->ctor, index);
       tree elt = cxx_eval_constant_expression (&new_ctx, value,
 					       lval,
-					       non_constant_p, overflow_p);
+					       non_constant_p, overflow_p,
+					       jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       /* Don't VERIFY_CONSTANT here.  */
       if (ctx->quiet && *non_constant_p)
 	break;
@@ -5683,7 +6584,8 @@ cxx_eval_bare_aggregate (const constexpr_ctx *ctx, tree t,
 static tree
 cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
 		     bool value_init, value_cat lval,
-		     bool *non_constant_p, bool *overflow_p)
+		     bool *non_constant_p, bool *overflow_p,
+		     tree *jump_target)
 {
   tree elttype = TREE_TYPE (atype);
   verify_ctor_sanity (ctx, atype);
@@ -5694,7 +6596,10 @@ cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
 
   if (init && TREE_CODE (init) == CONSTRUCTOR)
     return cxx_eval_bare_aggregate (ctx, init, lval,
-				    non_constant_p, overflow_p);
+				    non_constant_p, overflow_p, jump_target);
+
+  /* We already checked access when building the VEC_INIT_EXPR.  */
+  deferring_access_check_sentinel acs (dk_deferred);
 
   /* For the default constructor, build up a call to the default
      constructor of the element type.  We only need to handle class types
@@ -5731,7 +6636,9 @@ cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
     }
 
   tree nelts = get_array_or_vector_nelts (ctx, atype, non_constant_p,
-					  overflow_p);
+					  overflow_p, jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   unsigned HOST_WIDE_INT max = tree_to_uhwi (nelts);
   for (i = 0; i < max; ++i)
     {
@@ -5757,9 +6664,9 @@ cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
 	    }
 	  else
 	    eltinit = cp_build_array_ref (input_location, init, idx, complain);
-	  eltinit = cxx_eval_vec_init_1 (&new_ctx, elttype, eltinit, value_init,
-					 lval,
-					 non_constant_p, overflow_p);
+	  eltinit = cxx_eval_vec_init_1 (&new_ctx, elttype, eltinit,
+					 value_init, lval, non_constant_p,
+					 overflow_p, jump_target);
 	}
       else if (pre_init)
 	{
@@ -5773,7 +6680,8 @@ cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
 	    /* Clarify what object is being initialized (118285).  */
 	    eltinit = build2 (INIT_EXPR, elttype, new_ctx.object, eltinit);
 	  eltinit = cxx_eval_constant_expression (&new_ctx, eltinit, lval,
-						  non_constant_p, overflow_p);
+						  non_constant_p, overflow_p,
+						  jump_target);
 	  reuse = i == 0;
 	}
       else
@@ -5789,8 +6697,11 @@ cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
 	    /* Clarify what object is being initialized (118285).  */
 	    eltinit = build2 (INIT_EXPR, elttype, new_ctx.object, eltinit);
 	  eltinit = cxx_eval_constant_expression (&new_ctx, eltinit, lval,
-						  non_constant_p, overflow_p);
+						  non_constant_p, overflow_p,
+						  jump_target);
 	}
+      if (*jump_target)
+	return NULL_TREE;
       if (*non_constant_p)
 	break;
       if (no_slot)
@@ -5840,7 +6751,7 @@ cxx_eval_vec_init_1 (const constexpr_ctx *ctx, tree atype, tree init,
 static tree
 cxx_eval_vec_init (const constexpr_ctx *ctx, tree t,
 		   value_cat lval,
-		   bool *non_constant_p, bool *overflow_p)
+		   bool *non_constant_p, bool *overflow_p, tree *jump_target)
 {
   tree atype = TREE_TYPE (t);
   tree init = VEC_INIT_EXPR_INIT (t);
@@ -5872,10 +6783,10 @@ cxx_eval_vec_init (const constexpr_ctx *ctx, tree t,
 	}
       init = expand_vec_init_expr (ctx->object, t, complain);
       return cxx_eval_constant_expression (ctx, init, lval, non_constant_p,
-					   overflow_p);
+					   overflow_p, jump_target);
     }
   tree r = cxx_eval_vec_init_1 (ctx, atype, init, value_init,
-				lval, non_constant_p, overflow_p);
+				lval, non_constant_p, overflow_p, jump_target);
   if (*non_constant_p)
     return t;
   else
@@ -5904,14 +6815,16 @@ same_type_ignoring_tlq_and_bounds_p (tree type1, tree type2)
    otherwise return NULL_TREE.  */
 
 static tree
-cxx_union_active_member (const constexpr_ctx *ctx, tree t)
+cxx_union_active_member (const constexpr_ctx *ctx, tree t, tree *jump_target)
 {
   constexpr_ctx new_ctx = *ctx;
   new_ctx.quiet = true;
   bool non_constant_p = false, overflow_p = false;
   tree ctor = cxx_eval_constant_expression (&new_ctx, t, vc_prvalue,
 					    &non_constant_p,
-					    &overflow_p);
+					    &overflow_p, jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   if (TREE_CODE (ctor) == CONSTRUCTOR
       && CONSTRUCTOR_NELTS (ctor) == 1
       && CONSTRUCTOR_ELT (ctor, 0)->index
@@ -5924,7 +6837,8 @@ cxx_union_active_member (const constexpr_ctx *ctx, tree t)
 
 static tree
 cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
-			 tree op, unsigned HOST_WIDE_INT off, bool *empty_base)
+			 tree op, unsigned HOST_WIDE_INT off, bool *empty_base,
+			 tree *jump_target)
 {
   tree optype = TREE_TYPE (op);
   unsigned HOST_WIDE_INT const_nunits;
@@ -5941,7 +6855,8 @@ cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
 	 than pointer type.  */
       if (tree ret = cxx_fold_indirect_ref_1 (ctx, loc,
 					      strip_array_types (optype),
-					      op, off, empty_base))
+					      op, off, empty_base,
+					      jump_target))
 	return fold_convert (type, ret);
     }
   else if (TREE_CODE (optype) == COMPLEX_TYPE
@@ -5987,7 +6902,7 @@ cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
 	  op = build4_loc (loc, ARRAY_REF, TREE_TYPE (optype), op, index,
 			   NULL_TREE, NULL_TREE);
 	  return cxx_fold_indirect_ref_1 (ctx, loc, type, op, rem,
-					  empty_base);
+					  empty_base, jump_target);
 	}
     }
   /* ((foo *)&struct_with_foo_field)[x] => COMPONENT_REF */
@@ -5996,7 +6911,7 @@ cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
     {
       if (TREE_CODE (optype) == UNION_TYPE)
 	/* For unions prefer the currently active member.  */
-	if (tree field = cxx_union_active_member (ctx, op))
+	if (tree field = cxx_union_active_member (ctx, op, jump_target))
 	  {
 	    unsigned HOST_WIDE_INT el_sz
 	      = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (field)));
@@ -6005,7 +6920,8 @@ cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
 		tree cop = build3 (COMPONENT_REF, TREE_TYPE (field),
 				   op, field, NULL_TREE);
 		if (tree ret = cxx_fold_indirect_ref_1 (ctx, loc, type, cop,
-							off, empty_base))
+							off, empty_base,
+							jump_target))
 		  return ret;
 	      }
 	  }
@@ -6050,7 +6966,8 @@ cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
 				   op, field, NULL_TREE);
 		if (tree ret = cxx_fold_indirect_ref_1 (ctx, loc, type, cop,
 							off - upos,
-							empty_base))
+							empty_base,
+							jump_target))
 		  return ret;
 	      }
 	  }
@@ -6070,7 +6987,7 @@ cxx_fold_indirect_ref_1 (const constexpr_ctx *ctx, location_t loc, tree type,
 
 static tree
 cxx_fold_indirect_ref (const constexpr_ctx *ctx, location_t loc, tree type,
-		       tree op0, bool *empty_base /* = NULL*/)
+		       tree op0, bool *empty_base, tree *jump_target)
 {
   tree sub = op0;
   tree subtype;
@@ -6152,7 +7069,8 @@ cxx_fold_indirect_ref (const constexpr_ctx *ctx, location_t loc, tree type,
 	  tree off = integer_zero_node;
 	  canonicalize_obj_off (op, off);
 	  return cxx_fold_indirect_ref_1 (ctx, loc, type, op,
-					  tree_to_uhwi (off), empty_base);
+					  tree_to_uhwi (off), empty_base,
+					  jump_target);
 	}
     }
   else if (TREE_CODE (sub) == POINTER_PLUS_EXPR
@@ -6167,7 +7085,8 @@ cxx_fold_indirect_ref (const constexpr_ctx *ctx, location_t loc, tree type,
 	  tree obj = TREE_OPERAND (op00, 0);
 	  canonicalize_obj_off (obj, off);
 	  return cxx_fold_indirect_ref_1 (ctx, loc, type, obj,
-					  tree_to_uhwi (off), empty_base);
+					  tree_to_uhwi (off), empty_base,
+					  jump_target);
 	}
     }
   /* *(foo *)fooarrptr => (*fooarrptr)[0] */
@@ -6177,7 +7096,10 @@ cxx_fold_indirect_ref (const constexpr_ctx *ctx, location_t loc, tree type,
       tree type_domain;
       tree min_val = size_zero_node;
       tree newsub
-	= cxx_fold_indirect_ref (ctx, loc, TREE_TYPE (subtype), sub, NULL);
+	= cxx_fold_indirect_ref (ctx, loc, TREE_TYPE (subtype), sub, NULL,
+				 jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       if (newsub)
 	sub = newsub;
       else
@@ -6195,7 +7117,8 @@ cxx_fold_indirect_ref (const constexpr_ctx *ctx, location_t loc, tree type,
 static tree
 cxx_eval_indirect_ref (const constexpr_ctx *ctx, tree t,
 		       value_cat lval,
-		       bool *non_constant_p, bool *overflow_p)
+		       bool *non_constant_p, bool *overflow_p,
+		       tree *jump_target)
 {
   tree orig_op0 = TREE_OPERAND (t, 0);
   bool empty_base = false;
@@ -6213,13 +7136,17 @@ cxx_eval_indirect_ref (const constexpr_ctx *ctx, tree t,
 
   /* First try to simplify it directly.  */
   tree r = cxx_fold_indirect_ref (ctx, EXPR_LOCATION (t), TREE_TYPE (t),
-				  orig_op0, &empty_base);
+				  orig_op0, &empty_base, jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   if (!r)
     {
       /* If that didn't work, evaluate the operand first.  */
       tree op0 = cxx_eval_constant_expression (ctx, orig_op0,
 					       vc_prvalue, non_constant_p,
-					       overflow_p);
+					       overflow_p, jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       /* Don't VERIFY_CONSTANT here.  */
       if (*non_constant_p)
 	return t;
@@ -6233,7 +7160,9 @@ cxx_eval_indirect_ref (const constexpr_ctx *ctx, tree t,
 	}
 
       r = cxx_fold_indirect_ref (ctx, EXPR_LOCATION (t), TREE_TYPE (t), op0,
-				 &empty_base);
+				 &empty_base, jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       if (r == NULL_TREE)
 	{
 	  /* We couldn't fold to a constant value.  Make sure it's not
@@ -6263,7 +7192,10 @@ cxx_eval_indirect_ref (const constexpr_ctx *ctx, tree t,
     }
 
   r = cxx_eval_constant_expression (ctx, r,
-				    lval, non_constant_p, overflow_p);
+				    lval, non_constant_p, overflow_p,
+				    jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   if (*non_constant_p)
     return t;
 
@@ -6373,7 +7305,8 @@ non_const_var_error (location_t loc, tree r, bool fundef_p)
 static tree
 cxx_eval_trinary_expression (const constexpr_ctx *ctx, tree t,
 			     value_cat lval,
-			     bool *non_constant_p, bool *overflow_p)
+			     bool *non_constant_p, bool *overflow_p,
+			     tree *jump_target)
 {
   int i;
   tree args[3];
@@ -6383,7 +7316,10 @@ cxx_eval_trinary_expression (const constexpr_ctx *ctx, tree t,
     {
       args[i] = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, i),
 					      lval,
-					      non_constant_p, overflow_p);
+					      non_constant_p, overflow_p,
+					      jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       VERIFY_CONSTANT (args[i]);
     }
 
@@ -6505,7 +7441,8 @@ modifying_const_object_p (tree_code code, tree obj, bool mutable_p)
 static tree
 cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 			   value_cat lval,
-			   bool *non_constant_p, bool *overflow_p)
+			   bool *non_constant_p, bool *overflow_p,
+			   tree *jump_target)
 {
   constexpr_ctx new_ctx = *ctx;
 
@@ -6531,7 +7468,10 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
       if (!SCALAR_TYPE_P (type))
 	new_ctx.ctor = new_ctx.object = NULL_TREE;
       init = cxx_eval_constant_expression (&new_ctx, init, vc_prvalue,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       if (*non_constant_p)
 	return t;
     }
@@ -6543,8 +7483,11 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 	 as a whole; otherwise, only evaluate the innermost piece to avoid
 	 building up unnecessary *_REFs.  */
       target = cxx_eval_constant_expression (ctx, target, lval,
-					     non_constant_p, overflow_p);
+					     non_constant_p, overflow_p,
+					     jump_target);
       evaluated = true;
+      if (*jump_target)
+	return NULL_TREE;
       if (*non_constant_p)
 	return t;
     }
@@ -6570,7 +7513,10 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 	    if (TREE_CODE (probe) == ARRAY_REF)
 	      {
 		elt = eval_and_check_array_index (ctx, probe, false,
-						  non_constant_p, overflow_p);
+						  non_constant_p, overflow_p,
+						  jump_target);
+		if (*jump_target)
+		  return NULL_TREE;
 		if (*non_constant_p)
 		  return t;
 	      }
@@ -6627,8 +7573,11 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 		  break;
 		}
 	      probe = cxx_eval_constant_expression (ctx, probe, vc_glvalue,
-						    non_constant_p, overflow_p);
+						    non_constant_p, overflow_p,
+						    jump_target);
 	      evaluated = true;
+	      if (*jump_target)
+		return NULL_TREE;
 	      if (*non_constant_p)
 		return t;
 	    }
@@ -6798,13 +7747,24 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 	  CONSTRUCTOR_APPEND_ELT (CONSTRUCTOR_ELTS (*valp), first, NULL_TREE);
 
       /* Check for implicit change of active member for a union.  */
+
+      /* LWG3436, CWG2675, c++/121068: The array object model is confused.  For
+	 now allow initializing an array element to activate the array.  */
+      auto only_array_refs = [](const releasing_vec &refs)
+      {
+	for (unsigned i = 1; i < refs->length(); i += 3)
+	  if (TREE_CODE ((*refs)[i]) != INTEGER_CST)
+	    return false;
+	return true;
+      };
+
       if (code == UNION_TYPE
 	  && (CONSTRUCTOR_NELTS (*valp) == 0
 	      || CONSTRUCTOR_ELT (*valp, 0)->index != index)
 	  /* An INIT_EXPR of the last member in an access chain is always OK,
 	     but still check implicit change of members earlier on; see
 	     cpp2a/constexpr-union6.C.  */
-	  && !(TREE_CODE (t) == INIT_EXPR && refs->is_empty ()))
+	  && !(TREE_CODE (t) == INIT_EXPR && only_array_refs (refs)))
 	{
 	  bool has_active_member = CONSTRUCTOR_NELTS (*valp) != 0;
 	  tree inner = strip_array_types (reftype);
@@ -6972,7 +7932,10 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 	if (tree tinit = TARGET_EXPR_INITIAL (init))
 	  init = tinit;
       init = cxx_eval_constant_expression (&new_ctx, init, vc_prvalue,
-					   non_constant_p, overflow_p);
+					   non_constant_p, overflow_p,
+					   jump_target);
+      if (*jump_target)
+	return NULL_TREE;
       /* The hash table might have moved since the get earlier, and the
 	 initializer might have mutated the underlying CONSTRUCTORs, so we must
 	 recompute VALP. */
@@ -7098,7 +8061,8 @@ cxx_eval_store_expression (const constexpr_ctx *ctx, tree t,
 static tree
 cxx_eval_increment_expression (const constexpr_ctx *ctx, tree t,
 			      value_cat lval,
-			      bool *non_constant_p, bool *overflow_p)
+			      bool *non_constant_p, bool *overflow_p,
+			      tree *jump_target)
 {
   enum tree_code code = TREE_CODE (t);
   tree type = TREE_TYPE (t);
@@ -7112,12 +8076,18 @@ cxx_eval_increment_expression (const constexpr_ctx *ctx, tree t,
 
   /* The operand as an lvalue.  */
   op = cxx_eval_constant_expression (ctx, op, vc_glvalue,
-				     non_constant_p, overflow_p);
+				     non_constant_p, overflow_p,
+				     jump_target);
+  if (*jump_target)
+    return NULL_TREE;
 
   /* The operand as an rvalue.  */
   tree val
     = cxx_eval_constant_expression (ctx, op, vc_prvalue,
-				    non_constant_p, overflow_p);
+				    non_constant_p, overflow_p,
+				    jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   /* Don't VERIFY_CONSTANT if this might be dealing with a pointer to
      a local array in a constexpr function.  */
   bool ptr = INDIRECT_TYPE_P (TREE_TYPE (val));
@@ -7156,8 +8126,11 @@ cxx_eval_increment_expression (const constexpr_ctx *ctx, tree t,
   tree store = build2_loc (cp_expr_loc_or_loc (t, input_location),
 			   MODIFY_EXPR, type, op, mod);
   mod = cxx_eval_constant_expression (ctx, store, lval,
-				      non_constant_p, overflow_p);
+				      non_constant_p, overflow_p,
+				      jump_target);
   ggc_free (store);
+  if (*jump_target)
+    return NULL_TREE;
   if (*non_constant_p)
     return t;
 
@@ -7171,42 +8144,6 @@ cxx_eval_increment_expression (const constexpr_ctx *ctx, tree t,
     return val;
 }
 
-/* Predicates for the meaning of *jump_target.  */
-
-static bool
-returns (tree *jump_target)
-{
-  return *jump_target
-    && TREE_CODE (*jump_target) == RETURN_EXPR;
-}
-
-static bool
-breaks (tree *jump_target)
-{
-  return *jump_target
-    && ((TREE_CODE (*jump_target) == LABEL_DECL
-	 && LABEL_DECL_BREAK (*jump_target))
-	|| TREE_CODE (*jump_target) == BREAK_STMT
-	|| TREE_CODE (*jump_target) == EXIT_EXPR);
-}
-
-static bool
-continues (tree *jump_target)
-{
-  return *jump_target
-    && ((TREE_CODE (*jump_target) == LABEL_DECL
-	 && LABEL_DECL_CONTINUE (*jump_target))
-	|| TREE_CODE (*jump_target) == CONTINUE_STMT);
-
-}
-
-static bool
-switches (tree *jump_target)
-{
-  return *jump_target
-    && TREE_CODE (*jump_target) == INTEGER_CST;
-}
-
 /* Subroutine of cxx_eval_statement_list.  Determine whether the statement
    STMT matches *jump_target.  If we're looking for a case label and we see
    the default label, note it in ctx->css_state.  */
@@ -7254,6 +8191,11 @@ label_matches (const constexpr_ctx *ctx, tree *jump_target, tree stmt)
 	 breaks (jump_target) or continues (jump_target).  */
       break;
 
+    case VAR_DECL:
+      /* Uncaught exception.  This is handled by TRY_BLOCK evaluation
+	 and other places by testing throws (jump_target).  */
+      break;
+
     default:
       gcc_unreachable ();
     }
@@ -7268,15 +8210,9 @@ cxx_eval_statement_list (const constexpr_ctx *ctx, tree t,
 			 bool *non_constant_p, bool *overflow_p,
 			 tree *jump_target)
 {
-  tree local_target;
   /* In a statement-expression we want to return the last value.
      For empty statement expression return void_node.  */
   tree r = void_node;
-  if (!jump_target)
-    {
-      local_target = NULL_TREE;
-      jump_target = &local_target;
-    }
   for (tree_stmt_iterator i = tsi_start (t); !tsi_end_p (i); ++i)
     {
       tree stmt = *i;
@@ -7304,18 +8240,11 @@ cxx_eval_statement_list (const constexpr_ctx *ctx, tree t,
 					jump_target);
       if (*non_constant_p)
 	break;
-      if (returns (jump_target) || breaks (jump_target))
+      if (returns (jump_target)
+	  || breaks (jump_target)
+	  || throws (jump_target))
 	break;
     }
-  if (*jump_target && jump_target == &local_target)
-    {
-      /* We aren't communicating the jump to our caller, so give up.  We don't
-	 need to support evaluation of jumps out of statement-exprs.  */
-      if (!ctx->quiet)
-	error_at (cp_expr_loc_or_input_loc (r),
-		  "statement is not a constant expression");
-      *non_constant_p = true;
-    }
   return r;
 }
 
@@ -7327,13 +8256,6 @@ cxx_eval_loop_expr (const constexpr_ctx *ctx, tree t,
 		    bool *non_constant_p, bool *overflow_p,
 		    tree *jump_target)
 {
-  tree local_target;
-  if (!jump_target)
-    {
-      local_target = NULL_TREE;
-      jump_target = &local_target;
-    }
-
   tree body, cond = NULL_TREE, expr = NULL_TREE;
   tree cond_prep = NULL_TREE, cond_cleanup = NULL_TREE;
   unsigned cond_cleanup_depth = 0;
@@ -7389,7 +8311,7 @@ cxx_eval_loop_expr (const constexpr_ctx *ctx, tree t,
 	tree c;
 	FOR_EACH_VEC_ELT_REVERSE (cleanups, i, c)
 	  cxx_eval_constant_expression (ctx, c, vc_discard, non_constant_p,
-					overflow_p);
+					overflow_p, jump_target);
       }
     if (cond_prep)
       for (tree decl = BIND_EXPR_VARS (cond_prep);
@@ -7484,7 +8406,8 @@ cxx_eval_loop_expr (const constexpr_ctx *ctx, tree t,
 		      if (*non_constant_p
 			  || returns (jump_target)
 			  || breaks (jump_target)
-			  || continues (jump_target))
+			  || continues (jump_target)
+			  || throws (jump_target))
 			{
 			  depth = 1;
 			  break;
@@ -7531,6 +8454,7 @@ cxx_eval_loop_expr (const constexpr_ctx *ctx, tree t,
 	 && !breaks (jump_target)
 	 && !continues (jump_target)
 	 && (!switches (jump_target) || count == 0)
+	 && !throws (jump_target)
 	 && !*non_constant_p);
 
   cleanup_cond ();
@@ -7549,7 +8473,10 @@ cxx_eval_switch_expr (const constexpr_ctx *ctx, tree t,
   tree cond
     = TREE_CODE (t) == SWITCH_STMT ? SWITCH_STMT_COND (t) : SWITCH_COND (t);
   cond = cxx_eval_constant_expression (ctx, cond, vc_prvalue,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
+  if (*jump_target)
+    return NULL_TREE;
   VERIFY_CONSTANT (cond);
   if (TREE_CODE (cond) != INTEGER_CST)
     {
@@ -7682,7 +8609,8 @@ maybe_warn_about_constant_value (location_t loc, tree decl)
 static tree
 build_new_constexpr_heap_type (const constexpr_ctx *ctx, tree elt_type,
 			       tree cookie_size, tree full_size, tree arg_size,
-			       bool *non_constant_p, bool *overflow_p)
+			       bool *non_constant_p, bool *overflow_p,
+			       tree *jump_target)
 {
   gcc_assert (cookie_size == NULL_TREE || tree_fits_uhwi_p (cookie_size));
   gcc_assert (tree_fits_uhwi_p (full_size));
@@ -7718,13 +8646,17 @@ build_new_constexpr_heap_type (const constexpr_ctx *ctx, tree elt_type,
 	  if (integer_zerop (op0))
 	    arg_size
 	      = cxx_eval_constant_expression (ctx, op1, vc_prvalue,
-					      non_constant_p, overflow_p);
+					      non_constant_p, overflow_p,
+					      jump_target);
 	  else if (integer_zerop (op1))
 	    arg_size
 	      = cxx_eval_constant_expression (ctx, op0, vc_prvalue,
-					      non_constant_p, overflow_p);
+					      non_constant_p, overflow_p,
+					      jump_target);
 	  else
 	    arg_size = NULL_TREE;
+	  if (*jump_target)
+	    return NULL_TREE;
 	}
       else
 	arg_size = NULL_TREE;
@@ -7745,6 +8677,38 @@ build_new_constexpr_heap_type (const constexpr_ctx *ctx, tree elt_type,
   return build_new_constexpr_heap_type (elt_type, cookie_size, itype2);
 }
 
+/* Handle the case when a cleanup of some expression throws.  JMP_TARGET
+   indicates whether the cleanup threw or not, *JUMP_TARGET indicates whether
+   the expression which needed the cleanup threw.  If both threw, diagnose
+   it and return NULL, otherwise return R.  If only the cleanup threw, set
+   *JUMP_TARGET to the exception object from the cleanup.  */
+
+static tree
+merge_jump_target (location_t loc, const constexpr_ctx *ctx, tree r,
+		   bool *non_constant_p, tree *jump_target, tree jmp_target)
+{
+  if (!throws (&jmp_target))
+    return r;
+  if (throws (jump_target))
+    {
+      /* [except.throw]/9 - If the exception handling mechanism
+	 handling an uncaught exception directly invokes a function
+	 that exits via an exception, the function std::terminate is
+	 invoked.  */
+      if (!ctx->quiet)
+	{
+	  auto_diagnostic_group d;
+	  diagnose_std_terminate (loc, ctx, *jump_target);
+	  inform (loc, "destructor exited with an exception");
+	}
+      *non_constant_p = true;
+      *jump_target = NULL_TREE;
+      return NULL_TREE;
+    }
+  *jump_target = jmp_target;
+  return r;
+}
+
 /* Attempt to reduce the expression T to a constant value.
    On failure, issue diagnostic and return error_mark_node.  */
 /* FIXME unify with c_fully_fold */
@@ -7754,9 +8718,9 @@ static tree
 cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 			      value_cat lval,
 			      bool *non_constant_p, bool *overflow_p,
-			      tree *jump_target /* = NULL */)
+			      tree *jump_target)
 {
-  if (jump_target && *jump_target)
+  if (*jump_target)
     {
       /* If we are jumping, ignore all statements/expressions except those
 	 that could have LABEL_EXPR or CASE_LABEL_EXPR in their bodies.  */
@@ -7880,7 +8844,7 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 		r = convert_from_reference (r);
 	    }
 	  return cxx_eval_constant_expression (ctx, r, lval, non_constant_p,
-					       overflow_p);
+					       overflow_p, jump_target);
 	}
       /* fall through */
     case CONST_DECL:
@@ -7958,7 +8922,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	  r = v;
 	  if (TREE_ADDRESSABLE (TREE_TYPE (t)))
 	    r = cxx_eval_constant_expression (ctx, r, vc_prvalue,
-					      non_constant_p, overflow_p);
+					      non_constant_p, overflow_p,
+					      jump_target);
+	  if (*jump_target)
+	    return NULL_TREE;
 	}
       else if (lval)
 	/* Defer in case this is only used for its type.  */;
@@ -7991,7 +8958,7 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case CALL_EXPR:
     case AGGR_INIT_EXPR:
       r = cxx_eval_call_expression (ctx, t, lval,
-				    non_constant_p, overflow_p);
+				    non_constant_p, overflow_p, jump_target);
       break;
 
     case DECL_EXPR:
@@ -8055,7 +9022,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	if (tree init = DECL_INITIAL (r))
 	  {
 	    init = cxx_eval_constant_expression (ctx, init, vc_prvalue,
-						 non_constant_p, overflow_p);
+						 non_constant_p, overflow_p,
+						 jump_target);
+	    if (*jump_target)
+	      return NULL_TREE;
 	    /* Don't share a CONSTRUCTOR that might be changed.  */
 	    init = unshare_constructor (init);
 	    /* Remember that a constant object's constructor has already
@@ -8125,9 +9095,12 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	/* Pass vc_prvalue because this indicates
 	   initialization of a temporary.  */
 	r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1), vc_prvalue,
-					  non_constant_p, overflow_p);
+					  non_constant_p, overflow_p,
+					  jump_target);
 	if (*non_constant_p)
 	  break;
+	if (*jump_target)
+	  return NULL_TREE;
 	if (!is_complex)
 	  {
 	    r = unshare_constructor (r);
@@ -8135,8 +9108,15 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	    r = adjust_temp_type (type, r);
 	    ctx->global->put_value (slot, r);
 	  }
-	if (TARGET_EXPR_CLEANUP (t) && !CLEANUP_EH_ONLY (t))
-	  ctx->global->cleanups->safe_push (TARGET_EXPR_CLEANUP (t));
+	if (TARGET_EXPR_CLEANUP (t)
+	    && (!CLEANUP_EH_ONLY (t) || cxx_dialect >= cxx26))
+	  {
+	    ctx->global->cleanups->safe_push (TARGET_EXPR_CLEANUP (t));
+	    /* Mark CLEANUP_EH_ONLY cleanups by pushing NULL_TREE after
+	       them.  */
+	    if (CLEANUP_EH_ONLY (t))
+	      ctx->global->cleanups->safe_push (NULL_TREE);
+	  }
 	if (ctx->save_exprs)
 	  ctx->save_exprs->safe_push (slot);
 	if (lval)
@@ -8150,33 +9130,28 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case MODIFY_EXPR:
       gcc_assert (jump_target == NULL || *jump_target == NULL_TREE);
       r = cxx_eval_store_expression (ctx, t, lval,
-				     non_constant_p, overflow_p);
+				     non_constant_p, overflow_p, jump_target);
       break;
 
     case SCOPE_REF:
       r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1),
 					lval,
-					non_constant_p, overflow_p);
+					non_constant_p, overflow_p,
+					jump_target);
       break;
 
     case RETURN_EXPR:
       if (TREE_OPERAND (t, 0) != NULL_TREE)
 	r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
 					  lval,
-					  non_constant_p, overflow_p);
-      /* FALLTHRU */
+					  non_constant_p, overflow_p,
+					  jump_target);
+      if (!throws (jump_target))
+	*jump_target = t;
+      break;
     case BREAK_STMT:
     case CONTINUE_STMT:
-      if (jump_target)
-	*jump_target = t;
-      else
-	{
-	  /* Can happen with ({ return true; }) && false; passed to
-	     maybe_constant_value.  There is nothing to jump over in this
-	     case, and the bug will be diagnosed later.  */
-	  gcc_assert (ctx->quiet);
-	  *non_constant_p = true;
-	}
+      *jump_target = t;
       break;
 
     case SAVE_EXPR:
@@ -8185,9 +9160,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	r = v;
       else
 	{
-	  r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), vc_prvalue,
-					    non_constant_p, overflow_p);
-	  if (*non_constant_p)
+	  r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
+					    vc_prvalue, non_constant_p,
+					    overflow_p, jump_target);
+	  if (*non_constant_p || *jump_target)
 	    break;
 	  ctx->global->put_value (t, r);
 	  if (ctx->save_exprs)
@@ -8195,16 +9171,7 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	}
       break;
 
-    case TRY_CATCH_EXPR:
-      if (TREE_OPERAND (t, 0) == NULL_TREE)
-	{
-	  r = void_node;
-	  break;
-	}
-      /* FALLTHRU */
     case NON_LVALUE_EXPR:
-    case TRY_BLOCK:
-    case MUST_NOT_THROW_EXPR:
     case EXPR_STMT:
     case EH_SPEC_BLOCK:
       r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
@@ -8213,6 +9180,42 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 					jump_target);
       break;
 
+    case TRY_BLOCK:
+      r = cxx_eval_constant_expression (ctx, TRY_STMTS (t), lval,
+					non_constant_p, overflow_p,
+					jump_target);
+      if (!*non_constant_p && throws (jump_target))
+	if (tree h = TRY_HANDLERS (t))
+	  {
+	    tree type = strip_array_types (TREE_TYPE (*jump_target));
+	    if (TREE_CODE (h) == STATEMENT_LIST)
+	      {
+		for (tree stmt : tsi_range (h))
+		  if (TREE_CODE (stmt) == HANDLER
+		      && handler_match_for_exception_type (stmt, type))
+		    {
+		      h = stmt;
+		      break;
+		    }
+		if (TREE_CODE (h) == STATEMENT_LIST)
+		  h = NULL_TREE;
+	      }
+	    else if (TREE_CODE (h) != HANDLER
+		     || !handler_match_for_exception_type (h, type))
+	      h = NULL_TREE;
+	    if (h)
+	      {
+		gcc_assert (VAR_P (*jump_target));
+		ctx->global->caught_exceptions.safe_push (*jump_target);
+		ctx->global->caught_exceptions.safe_push (HANDLER_TYPE (h));
+		*jump_target = NULL_TREE;
+		r = cxx_eval_constant_expression (ctx, HANDLER_BODY (h),
+						  vc_discard, non_constant_p,
+						  overflow_p, jump_target);
+	      }
+	  }
+      break;
+
     case CLEANUP_POINT_EXPR:
       {
 	auto_vec<tree, 2> cleanups;
@@ -8230,47 +9233,132 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 
 	ctx->global->cleanups = prev_cleanups;
 	unsigned int i;
-	tree cleanup;
+	tree cleanup, jmp_target = NULL_TREE;
+	bool eh = throws (jump_target);
 	/* Evaluate the cleanups.  */
 	FOR_EACH_VEC_ELT_REVERSE (cleanups, i, cleanup)
-	  cxx_eval_constant_expression (&new_ctx, cleanup, vc_discard,
-					non_constant_p, overflow_p);
+	  if (cleanup == NULL_TREE)
+	    {
+	      /* NULL_TREE cleanup is a marker that before it is
+		 CLEANUP_EH_ONLY cleanup.  Skip the cleanup before it
+		 if the body didn't throw.  */
+	      if (!eh)
+		--i;
+	    }
+	  else
+	    cxx_eval_constant_expression (&new_ctx, cleanup, vc_discard,
+					  non_constant_p, overflow_p,
+					  &jmp_target);
 
 	/* Forget SAVE_EXPRs and TARGET_EXPRs created by this
 	   full-expression.  */
 	for (tree save_expr : save_exprs)
 	  destroy_value_checked (ctx, save_expr, non_constant_p);
+	if (throws (&jmp_target))
+	  *jump_target = jmp_target;
       }
       break;
 
+    case MUST_NOT_THROW_EXPR:
+      r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0),
+					lval,
+					non_constant_p, overflow_p,
+					jump_target);
+      if (throws (jump_target))
+	{
+	  /* [except.handle]/7 - If the search for a handler exits the
+	     function body of a function with a non-throwing exception
+	     specification, the function std::terminate is invoked.  */
+	  if (!ctx->quiet)
+	    {
+	      auto_diagnostic_group d;
+	      diagnose_std_terminate (loc, ctx, *jump_target);
+	      if (MUST_NOT_THROW_NOEXCEPT_P (t)
+		  && ctx->call
+		  && ctx->call->fundef)
+		inform (loc, "uncaught exception exited from %<noexcept%> "
+			     "function %qD",
+			ctx->call->fundef->decl);
+	      else if (MUST_NOT_THROW_THROW_P (t))
+		inform (loc, "destructor exited with an exception after "
+			     "initializing the exception object");
+	      else if (MUST_NOT_THROW_CATCH_P (t))
+		inform (loc, "constructor exited with another exception while "
+			     "entering handler");
+	    }
+	  *non_constant_p = true;
+	  *jump_target = NULL_TREE;
+	  r = NULL_TREE;
+	}
+      break;
+
+    case TRY_CATCH_EXPR:
+      if (TREE_OPERAND (t, 0) == NULL_TREE)
+	{
+	  r = void_node;
+	  break;
+	}
+      r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), lval,
+					non_constant_p, overflow_p,
+					jump_target);
+      if (!*non_constant_p && throws (jump_target))
+	{
+	  tree jmp_target = NULL_TREE;
+	  cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1), vc_discard,
+					non_constant_p, overflow_p,
+					&jmp_target);
+	  r = merge_jump_target (loc, ctx, r, non_constant_p, jump_target,
+				 jmp_target);
+	}
+      break;
+
     case TRY_FINALLY_EXPR:
       r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), lval,
 					non_constant_p, overflow_p,
 					jump_target);
       if (!*non_constant_p)
-	/* Also evaluate the cleanup.  */
-	cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1), vc_discard,
-				      non_constant_p, overflow_p);
+	{
+	  tree jmp_target = NULL_TREE;
+	  /* Also evaluate the cleanup.  */
+	  if (TREE_CODE (TREE_OPERAND (t, 1)) == EH_ELSE_EXPR
+	      && throws (jump_target))
+	    cxx_eval_constant_expression (ctx,
+					  TREE_OPERAND (TREE_OPERAND (t, 1),
+							1), vc_discard,
+					  non_constant_p, overflow_p,
+					  &jmp_target);
+	  else
+	    cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 1), vc_discard,
+					  non_constant_p, overflow_p,
+					  &jmp_target);
+	  r = merge_jump_target (loc, ctx, r, non_constant_p, jump_target,
+				 jmp_target);
+	}
       break;
 
     case EH_ELSE_EXPR:
       /* Evaluate any cleanup that applies to non-EH exits.  */
       cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), vc_discard,
-				    non_constant_p, overflow_p);
+				    non_constant_p, overflow_p,
+				    jump_target);
 
-      /* We do not have constexpr exceptions yet, so skip the EH path.  */
+      /* The EH path is handled in TRY_FINALLY_EXPR handling above.  */
       break;
 
     case CLEANUP_STMT:
       r = cxx_eval_constant_expression (ctx, CLEANUP_BODY (t), lval,
 					non_constant_p, overflow_p,
 					jump_target);
-      if (!CLEANUP_EH_ONLY (t) && !*non_constant_p)
+      if ((!CLEANUP_EH_ONLY (t) || throws (jump_target)) && !*non_constant_p)
 	{
 	  iloc_sentinel ils (loc);
+	  tree jmp_target = NULL_TREE;
 	  /* Also evaluate the cleanup.  */
 	  cxx_eval_constant_expression (ctx, CLEANUP_EXPR (t), vc_discard,
-					non_constant_p, overflow_p);
+					non_constant_p, overflow_p,
+					&jmp_target);
+	  r = merge_jump_target (loc, ctx, r, non_constant_p, jump_target,
+				 jmp_target);
 	}
       break;
 
@@ -8280,14 +9368,18 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case MEM_REF:
     case INDIRECT_REF:
       r = cxx_eval_indirect_ref (ctx, t, lval,
-				 non_constant_p, overflow_p);
+				 non_constant_p, overflow_p,
+				 jump_target);
       break;
 
     case ADDR_EXPR:
       {
 	tree oldop = TREE_OPERAND (t, 0);
 	tree op = cxx_eval_constant_expression (ctx, oldop, vc_glvalue,
-						non_constant_p, overflow_p);
+						non_constant_p, overflow_p,
+						jump_target);
+	if (*jump_target)
+	  return NULL_TREE;
 	/* Don't VERIFY_CONSTANT here.  */
 	if (*non_constant_p)
 	  return t;
@@ -8307,7 +9399,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
       if (lval)
 	{
 	  r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), lval,
-					    non_constant_p, overflow_p);
+					    non_constant_p, overflow_p,
+					    jump_target);
+	  if (*jump_target)
+	    return NULL_TREE;
 	  if (r == error_mark_node)
 	    ;
 	  else if (r == TREE_OPERAND (t, 0) || lval == vc_discard)
@@ -8328,7 +9423,8 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case FIXED_CONVERT_EXPR:
     case VEC_DUPLICATE_EXPR:
       r = cxx_eval_unary_expression (ctx, t, lval,
-				     non_constant_p, overflow_p);
+				     non_constant_p, overflow_p,
+				     jump_target);
       break;
 
     case SIZEOF_EXPR:
@@ -8366,6 +9462,8 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	    cxx_eval_constant_expression (ctx, op0, vc_discard,
 					  non_constant_p, overflow_p,
 					  jump_target);
+	    if (*jump_target)
+	      return NULL_TREE;
 	    if (*non_constant_p)
 	      return t;
 	    op1 = TREE_OPERAND (t, 1);
@@ -8418,7 +9516,8 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case RANGE_EXPR:
     case COMPLEX_EXPR:
       r = cxx_eval_binary_expression (ctx, t, lval,
-				      non_constant_p, overflow_p);
+				      non_constant_p, overflow_p,
+				      jump_target);
       break;
 
       /* fold can introduce non-IF versions of these; still treat them as
@@ -8427,19 +9526,22 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case TRUTH_ANDIF_EXPR:
       r = cxx_eval_logical_expression (ctx, t, boolean_false_node,
 				       boolean_true_node,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
       break;
 
     case TRUTH_OR_EXPR:
     case TRUTH_ORIF_EXPR:
       r = cxx_eval_logical_expression (ctx, t, boolean_true_node,
 				       boolean_false_node,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
       break;
 
     case ARRAY_REF:
       r = cxx_eval_array_reference (ctx, t, lval,
-				    non_constant_p, overflow_p);
+				    non_constant_p, overflow_p,
+				    jump_target);
       break;
 
     case COMPONENT_REF:
@@ -8454,17 +9556,19 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	  return t;
 	}
       r = cxx_eval_component_reference (ctx, t, lval,
-					non_constant_p, overflow_p);
+					non_constant_p, overflow_p,
+					jump_target);
       break;
 
     case BIT_FIELD_REF:
       r = cxx_eval_bit_field_ref (ctx, t, lval,
-				  non_constant_p, overflow_p);
+				  non_constant_p, overflow_p,
+				  jump_target);
       break;
 
     case COND_EXPR:
     case IF_STMT:
-      if (jump_target && *jump_target)
+      if (*jump_target)
 	{
 	  tree orig_jump = *jump_target;
 	  tree arg = ((TREE_CODE (t) != IF_STMT || TREE_OPERAND (t, 1))
@@ -8502,7 +9606,7 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
       break;
     case VEC_COND_EXPR:
       r = cxx_eval_vector_conditional_expression (ctx, t, non_constant_p,
-						  overflow_p);
+						  overflow_p, jump_target);
       break;
 
     case CONSTRUCTOR:
@@ -8514,7 +9618,7 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	    return t;
 	}
       r = cxx_eval_bare_aggregate (ctx, t, lval,
-				   non_constant_p, overflow_p);
+				   non_constant_p, overflow_p, jump_target);
       break;
 
     case VEC_INIT_EXPR:
@@ -8524,12 +9628,13 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	 or xvalue of the same type, meaning direct-initialization from the
 	 corresponding member.  */
       r = cxx_eval_vec_init (ctx, t, lval,
-			     non_constant_p, overflow_p);
+			     non_constant_p, overflow_p, jump_target);
       break;
 
     case VEC_PERM_EXPR:
       r = cxx_eval_trinary_expression (ctx, t, lval,
-				       non_constant_p, overflow_p);
+				       non_constant_p, overflow_p,
+				       jump_target);
       break;
 
     case PAREN_EXPR:
@@ -8537,7 +9642,8 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
       /* A PAREN_EXPR resulting from __builtin_assoc_barrier has no effect in
          constant expressions since it's unaffected by -fassociative-math.  */
       r = cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), lval,
-					non_constant_p, overflow_p);
+					non_constant_p, overflow_p,
+					jump_target);
       break;
 
     case NOP_EXPR:
@@ -8561,7 +9667,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 						? vc_discard
 						: tcode == VIEW_CONVERT_EXPR
 						? lval : vc_prvalue,
-						non_constant_p, overflow_p);
+						non_constant_p, overflow_p,
+						jump_target);
+	if (*jump_target)
+	  return NULL_TREE;
 	if (*non_constant_p)
 	  return t;
 	tree type = TREE_TYPE (t);
@@ -8618,7 +9727,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	      {
 		if (integer_zerop (sop))
 		  return build_int_cst (type, 0);
-		r = cxx_fold_indirect_ref (ctx, loc, TREE_TYPE (type), sop);
+		r = cxx_fold_indirect_ref (ctx, loc, TREE_TYPE (type), sop,
+					   NULL, jump_target);
+		if (*jump_target)
+		  return NULL_TREE;
 		if (r)
 		  {
 		    r = build1 (ADDR_EXPR, type, r);
@@ -8745,10 +9857,14 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 		if (cxx_replaceable_global_alloc_fn (fun)
 		    && IDENTIFIER_NEW_OP_P (DECL_NAME (fun)))
 		  arg_size = CALL_EXPR_ARG (oldop, 0);
-	    TREE_TYPE (var)
+	    tree new_type
 	      = build_new_constexpr_heap_type (ctx, elt_type, cookie_size,
 					       var_size, arg_size,
-					       non_constant_p, overflow_p);
+					       non_constant_p, overflow_p,
+					       jump_target);
+	    if (*jump_target)
+	      return NULL_TREE;
+	    TREE_TYPE (var) = new_type;
 	    TREE_TYPE (TREE_OPERAND (op, 0))
 	      = build_pointer_type (TREE_TYPE (var));
 	  }
@@ -8787,7 +9903,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 
 	tree op = cxx_eval_constant_expression (ctx, oldop,
 						lval,
-						non_constant_p, overflow_p);
+						non_constant_p, overflow_p,
+						jump_target);
+	if (*jump_target)
+	  return NULL_TREE;
 	if (*non_constant_p)
 	  return t;
 	r = fold_convert (TREE_TYPE (t), op);
@@ -8824,14 +9943,20 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case PREDECREMENT_EXPR:
     case POSTDECREMENT_EXPR:
       return cxx_eval_increment_expression (ctx, t,
-					    lval, non_constant_p, overflow_p);
+					    lval, non_constant_p, overflow_p,
+					    jump_target);
 
+    case THROW_EXPR:
+      if (cxx_dialect >= cxx26)
+	return cxx_eval_constant_expression (ctx, TREE_OPERAND (t, 0), lval,
+					     non_constant_p, overflow_p,
+					     jump_target);
+      /* FALLTHROUGH */
     case LAMBDA_EXPR:
     case NEW_EXPR:
     case VEC_NEW_EXPR:
     case DELETE_EXPR:
     case VEC_DELETE_EXPR:
-    case THROW_EXPR:
     case MODOP_EXPR:
       /* GCC internal stuff.  */
     case VA_ARG_EXPR:
@@ -8845,7 +9970,8 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
     case OBJ_TYPE_REF:
       /* Virtual function lookup.  We don't need to do anything fancy.  */
       return cxx_eval_constant_expression (ctx, OBJ_TYPE_REF_EXPR (t),
-					   lval, non_constant_p, overflow_p);
+					   lval, non_constant_p, overflow_p,
+					   jump_target);
 
     case PLACEHOLDER_EXPR:
       /* Use of the value or address of the current object.  */
@@ -8855,7 +9981,8 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	    return ctor;
 	  else
 	    return cxx_eval_constant_expression (ctx, ctor, lval,
-						 non_constant_p, overflow_p);
+						 non_constant_p, overflow_p,
+						 jump_target);
 	}
       /* A placeholder without a referent.  We can get here when
 	 checking whether NSDMIs are noexcept, or in massage_init_elt;
@@ -8868,7 +9995,10 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
       {
 	tree cond = TREE_OPERAND (t, 0);
 	cond = cxx_eval_constant_expression (ctx, cond, vc_prvalue,
-					     non_constant_p, overflow_p);
+					     non_constant_p, overflow_p,
+					     jump_target);
+	if (*jump_target)
+	  return NULL_TREE;
 	VERIFY_CONSTANT (cond);
 	if (integer_nonzerop (cond))
 	  *jump_target = t;
@@ -8980,7 +10110,7 @@ cxx_eval_constant_expression (const constexpr_ctx *ctx, tree t,
 	  *non_constant_p = true;
 	  return t;
 	}
-      r = cxx_eval_bit_cast (ctx, t, non_constant_p, overflow_p);
+      r = cxx_eval_bit_cast (ctx, t, non_constant_p, overflow_p, jump_target);
       break;
 
     case OMP_PARALLEL:
@@ -9202,11 +10332,14 @@ cxx_eval_outermost_constant_expr (tree t, bool allow_non_constant,
 	{
 	  if (cxx_dialect < cxx20)
 	    return t;
-	  if (TREE_CODE (t) != CALL_EXPR && TREE_CODE (t) != AGGR_INIT_EXPR)
+	  /* We could have a COMPOUND_EXPR here coming from
+	     keep_unused_object_arg.  */
+	  tree x = extract_call_expr (t);
+	  if (x == NULL_TREE || x == error_mark_node)
 	    return t;
 	  /* Calls to immediate functions returning void need to be
 	     evaluated.  */
-	  tree fndecl = cp_get_callee_fndecl_nofold (t);
+	  tree fndecl = cp_get_callee_fndecl_nofold (x);
 	  if (fndecl == NULL_TREE || !DECL_IMMEDIATE_FUNCTION_P (fndecl))
 	    return t;
 	  else
@@ -9299,8 +10432,34 @@ cxx_eval_outermost_constant_expr (tree t, bool allow_non_constant,
 
   if (manifestly_const_eval == mce_true)
     instantiate_constexpr_fns (r);
+  tree jmp_target = NULL_TREE;
   r = cxx_eval_constant_expression (&ctx, r, vc_prvalue,
-				    &non_constant_p, &overflow_p);
+				    &non_constant_p, &overflow_p,
+				    &jmp_target);
+  if (throws (&jmp_target) && !non_constant_p)
+    {
+      if (!ctx.quiet)
+	diagnose_uncaught_exception (input_location, &ctx, jmp_target);
+      non_constant_p = true;
+      jmp_target = NULL_TREE;
+      r = t;
+    }
+  else if (!non_constant_p && jmp_target)
+    {
+      non_constant_p = true;
+      if (!ctx.quiet)
+	{
+	  if (breaks (&jmp_target))
+	    error ("%<break%> outside of a loop or %<switch%>");
+	  else if (continues (&jmp_target))
+	    error ("%<continue%> outside of a loop");
+	  else if (returns (&jmp_target))
+	    error ("%<return%> in a statement expression");
+	  else
+	    gcc_unreachable ();
+	}
+      r = t;
+    }
 
   /* If we got a non-simple TARGET_EXPR, the initializer was a sequence
      of statements, and the result ought to be stored in ctx.ctor.  */
@@ -9309,15 +10468,31 @@ cxx_eval_outermost_constant_expr (tree t, bool allow_non_constant,
 
   unsigned int i;
   tree cleanup;
+  jmp_target = NULL_TREE;
   /* Evaluate the cleanups.  */
   FOR_EACH_VEC_ELT_REVERSE (cleanups, i, cleanup)
-    cxx_eval_constant_expression (&ctx, cleanup, vc_discard,
-				  &non_constant_p, &overflow_p);
+    if (cleanup == NULL_TREE)
+      /* NULL_TREE cleanup is a marker that before it is
+	 CLEANUP_EH_ONLY cleanup.  Skip the cleanup before it.  */
+      --i;
+    else
+      cxx_eval_constant_expression (&ctx, cleanup, vc_discard,
+				    &non_constant_p, &overflow_p,
+				    &jmp_target);
+  if (throws (&jmp_target) && !non_constant_p)
+    {
+      if (!ctx.quiet)
+	diagnose_uncaught_exception (input_location, &ctx, jmp_target);
+      non_constant_p = true;
+      r = t;
+    }
 
   /* Mutable logic is a bit tricky: we want to allow initialization of
      constexpr variables with mutable members, but we can't copy those
      members to another constexpr variable.  */
-  if (TREE_CODE (r) == CONSTRUCTOR && CONSTRUCTOR_MUTABLE_POISON (r))
+  if (!non_constant_p
+      && TREE_CODE (r) == CONSTRUCTOR
+      && CONSTRUCTOR_MUTABLE_POISON (r))
     {
       if (!allow_non_constant)
 	error ("%qE is not a constant expression because it refers to "
@@ -9335,8 +10510,13 @@ cxx_eval_outermost_constant_expr (tree t, bool allow_non_constant,
 	{
 	  if (!allow_non_constant && !non_constant_p)
 	    {
-	      error ("%qE is not a constant expression because it refers to "
-		     "a result of %<operator new%>", t);
+	      if (DECL_LANG_SPECIFIC (heap_var))
+		error ("%qE is not a constant expression because it refers to "
+		       "exception object allocated with "
+		       "%<__cxa_allocate_exception%>", t);
+	      else
+		error ("%qE is not a constant expression because it refers to "
+		       "a result of %<operator new%>", t);
 	      inform (DECL_SOURCE_LOCATION (heap_var), "allocated here");
 	    }
 	  r = t;
@@ -9917,6 +11097,24 @@ cxx_constant_init (tree t, tree decl)
   return maybe_constant_init_1 (t, decl, false, mce_true);
 }
 
+/* Return true if CALL_EXPR T might throw during constant evaluation.  */
+
+static bool
+callee_might_throw (tree t)
+{
+  if (cxx_dialect < cxx26 || !flag_exceptions)
+    return false;
+  tree callee = cp_get_callee (t);
+  if (callee == NULL_TREE)
+    return false;
+  tree callee_fn = cp_get_fndecl_from_callee (callee, false);
+  return (!flag_enforce_eh_specs
+	  || type_dependent_expression_p (callee)
+	  || !POINTER_TYPE_P (TREE_TYPE (callee))
+	  || (!type_noexcept_p (TREE_TYPE (TREE_TYPE (callee)))
+	      && (callee_fn == NULL_TREE || !TREE_NOTHROW (callee_fn))));
+}
+
 #if 0
 /* FIXME see ADDR_EXPR section in potential_constant_expression_1.  */
 /* Return true if the object referred to by REF has automatic or thread
@@ -9949,11 +11147,13 @@ struct check_for_return_continue_data {
   hash_set<tree> *pset;
   tree continue_stmt;
   tree break_stmt;
+  bool could_throw;
 };
 
 /* Helper function for potential_constant_expression_1 SWITCH_STMT handling,
    called through cp_walk_tree.  Return the first RETURN_EXPR found, or note
-   the first CONTINUE_STMT and/or BREAK_STMT if RETURN_EXPR is not found.  */
+   the first CONTINUE_STMT and/or BREAK_STMT if RETURN_EXPR is not found.
+   For C++26 also note presence of possibly throwing calls.  */
 static tree
 check_for_return_continue (tree *tp, int *walk_subtrees, void *data)
 {
@@ -10038,6 +11238,13 @@ check_for_return_continue (tree *tp, int *walk_subtrees, void *data)
     case CONSTRUCTOR:
       break;
 
+    case AGGR_INIT_EXPR:
+    case CALL_EXPR:
+      /* In C++26 a function could throw.  */
+      if (callee_might_throw (t))
+	d->could_throw = true;
+      break;
+
     default:
       if (!EXPR_P (t))
 	*walk_subtrees = 0;
@@ -10243,8 +11450,27 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 			|| TREE_CODE (t) != CALL_EXPR
 			|| current_function_decl == NULL_TREE
 			|| !is_std_construct_at (current_function_decl))
-		    && !cxx_dynamic_cast_fn_p (fun))
+		    && !cxx_dynamic_cast_fn_p (fun)
+		    && !cxx_cxa_builtin_fn_p (fun))
 		  {
+		    /* In C++26 evaluation of the function arguments might
+		       throw and in that case it is irrelevant whether
+		       fun is constexpr or not.  */
+		    if (cxx_dialect >= cxx26)
+		      for (; i < nargs; ++i)
+			{
+			  tree x = get_nth_callarg (t, i);
+			  bool rv = processing_template_decl ? any : rval;
+			  bool sub_now = false;
+			  if (!potential_constant_expression_1 (x, rv, strict,
+								sub_now,
+								fundef_p,
+								flags,
+								jump_target))
+			    return false;
+			  if (throws (jump_target))
+			    return true;
+			}
 		    if ((flags & tf_error)
 			&& constexpr_error (loc, fundef_p,
 					    "call to non-%<constexpr%> "
@@ -10289,7 +11515,12 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 						  sub_now, fundef_p, flags,
 						  jump_target))
 	      return false;
+	    if (throws (jump_target))
+	      return true;
           }
+	/* In C++26 a function could throw.  */
+	if (*jump_target == NULL_TREE && callee_might_throw (t))
+	  *jump_target = void_node;
         return true;
       }
 
@@ -10512,11 +11743,13 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 		 a return.  */
 	      hash_set<tree> pset;
 	      check_for_return_continue_data data = { &pset, NULL_TREE,
-						      NULL_TREE };
+						      NULL_TREE, false };
 	      if (tree ret_expr
 		  = cp_walk_tree (&FOR_BODY (t), check_for_return_continue,
 				  &data, &pset))
 		*jump_target = ret_expr;
+	      if (data.could_throw)
+		*jump_target = void_node;
 	      return true;
 	    }
 	}
@@ -10556,11 +11789,13 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 	     a return.  */
 	  hash_set<tree> pset;
 	  check_for_return_continue_data data = { &pset, NULL_TREE,
-						  NULL_TREE  };
+						  NULL_TREE, false };
 	  if (tree ret_expr
 	      = cp_walk_tree (&WHILE_BODY (t), check_for_return_continue,
 			      &data, &pset))
 	    *jump_target = ret_expr;
+	  if (data.could_throw)
+	    *jump_target = void_node;
 	  return true;
 	}
       if (!RECUR (WHILE_BODY (t), any))
@@ -10584,7 +11819,7 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 	{
 	  hash_set<tree> pset;
 	  check_for_return_continue_data data = { &pset, NULL_TREE,
-						  NULL_TREE };
+						  NULL_TREE, false };
 	  if (tree ret_expr
 	      = cp_walk_tree (&SWITCH_STMT_BODY (t), check_for_return_continue,
 			      &data, &pset))
@@ -10593,6 +11828,8 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 	  else if (data.continue_stmt)
 	    /* The switch can't return, but might continue.  */
 	    *jump_target = data.continue_stmt;
+	  if (data.could_throw)
+	    *jump_target = void_node;
 	}
       return true;
 
@@ -10622,7 +11859,6 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 
     case DYNAMIC_CAST_EXPR:
     case PSEUDO_DTOR_EXPR:
-    case THROW_EXPR:
     case OMP_PARALLEL:
     case OMP_TASK:
     case OMP_FOR:
@@ -10678,6 +11914,11 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 	 constant.  */
       return true;
 
+    case THROW_EXPR:
+      if (cxx_dialect < cxx26)
+	goto fail;
+      return RECUR (TREE_OPERAND (t, 0), rval);
+
     case ASM_EXPR:
       if (flags & tf_error)
 	inline_asm_in_constexpr_error (loc, fundef_p);
@@ -10806,6 +12047,22 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
     case CLEANUP_POINT_EXPR:
     case MUST_NOT_THROW_EXPR:
     case TRY_CATCH_EXPR:
+      /* Even for C++26 handle TRY_BLOCK conservatively, if we detect the
+	 body could throw, even with catch (...) among handlers we'd need
+	 to analyze them in detail if they couldn't rethrow it.  More
+	 importantly though, throws (jump_target) is just conservative,
+	 and there could be e.g.
+	   try
+	     {
+	       possibly_throwing_fn (args);
+	       break;
+	     }
+	   catch (...)
+	     {
+	     }
+	 or continue or return instead of break.  So, clearing *jump_target
+	 because we see catch (...) handler might mean we missed break
+	 etc.  */
     case TRY_BLOCK:
     case EH_SPEC_BLOCK:
     case EXPR_STMT:
@@ -11047,9 +12304,9 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 					       want_rval, strict, now, fundef_p,
 					       tf_none, &this_jump_target))
 	    {
-	      if (returns (&this_jump_target))
+	      if (returns (&this_jump_target) || throws (&this_jump_target))
 		*jump_target = this_jump_target;
-	      else if (!returns (jump_target))
+	      else if (!returns (jump_target) && !throws (jump_target))
 		{
 		  if (breaks (&this_jump_target)
 		      || continues (&this_jump_target))
@@ -11061,7 +12318,8 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 			 couldn't return, break or continue.  */
 		      hash_set<tree> pset;
 		      check_for_return_continue_data data = { &pset, NULL_TREE,
-							      NULL_TREE };
+							      NULL_TREE,
+							      false };
 		      if (tree ret_expr
 			= cp_walk_tree (&TREE_OPERAND (t, 2),
 					check_for_return_continue, &data,
@@ -11074,6 +12332,8 @@ potential_constant_expression_1 (tree t, bool want_rval, bool strict, bool now,
 			  else if (data.break_stmt)
 			    *jump_target = data.break_stmt;
 			}
+		      if (data.could_throw)
+			*jump_target = void_node;
 		    }
 		}
 	      return true;
diff --git a/gcc/cp/constraint.cc b/gcc/cp/constraint.cc
index c8eef24..cbdfafc 100644
--- a/gcc/cp/constraint.cc
+++ b/gcc/cp/constraint.cc
@@ -2490,10 +2490,11 @@ satisfy_atom (tree t, tree args, sat_info info)
   result = force_rvalue (result, info.complain);
   if (result == error_mark_node)
     return cache.save (inst_cache.save (error_mark_node));
+  tree substituted = result;
   if (!same_type_p (TREE_TYPE (result), boolean_type_node))
     {
       if (info.noisy ())
-	diagnose_atomic_constraint (t, args, result, info);
+	diagnose_atomic_constraint (t, args, substituted, info);
       return cache.save (inst_cache.save (error_mark_node));
     }
 
@@ -2511,7 +2512,7 @@ satisfy_atom (tree t, tree args, sat_info info)
     }
   result = satisfaction_value (result);
   if (result == boolean_false_node && info.diagnose_unsatisfaction_p ())
-    diagnose_atomic_constraint (t, args, result, info);
+    diagnose_atomic_constraint (t, args, substituted, info);
 
   return cache.save (inst_cache.save (result));
 }
@@ -3063,11 +3064,9 @@ get_constraint_error_location (tree t)
 
 /* Emit a diagnostic for a failed trait.  */
 
-static void
-diagnose_trait_expr (tree expr, tree args)
+void
+diagnose_trait_expr (location_t loc, tree expr, tree args)
 {
-  location_t loc = cp_expr_location (expr);
-
   /* Build a "fake" version of the instantiated trait, so we can
      get the instantiated types from result.  */
   ++processing_template_decl;
@@ -3076,209 +3075,245 @@ diagnose_trait_expr (tree expr, tree args)
 
   tree t1 = TRAIT_EXPR_TYPE1 (expr);
   tree t2 = TRAIT_EXPR_TYPE2 (expr);
-  if (t2 && TREE_CODE (t2) == TREE_VEC)
-    {
-      /* Convert the TREE_VEC of arguments into a TREE_LIST, since we can't
-	 directly print a TREE_VEC but we can a TREE_LIST via the E format
-	 specifier.  */
-      tree list = NULL_TREE;
-      for (tree t : tree_vec_range (t2))
-	list = tree_cons (NULL_TREE, t, list);
-      t2 = nreverse (list);
-    }
+
+  /* For traits intrinsically about the properties of user-defined types,
+     decl_loc will point to the declaration of that type.  */
+  location_t decl_loc = location_of (t1);
+  if (decl_loc == input_location)
+    decl_loc = loc;
+
   switch (TRAIT_EXPR_KIND (expr))
     {
     case CPTK_HAS_NOTHROW_ASSIGN:
-      inform (loc, "  %qT is not nothrow copy assignable", t1);
+      inform (decl_loc, "%qT is not nothrow copy assignable", t1);
       break;
     case CPTK_HAS_NOTHROW_CONSTRUCTOR:
-      inform (loc, "  %qT is not nothrow default constructible", t1);
+      inform (decl_loc, "%qT is not nothrow default constructible", t1);
       break;
     case CPTK_HAS_NOTHROW_COPY:
-      inform (loc, "  %qT is not nothrow copy constructible", t1);
+      inform (decl_loc, "%qT is not nothrow copy constructible", t1);
       break;
     case CPTK_HAS_TRIVIAL_ASSIGN:
-      inform (loc, "  %qT is not trivially copy assignable", t1);
+      inform (decl_loc, "%qT is not trivially copy assignable", t1);
       break;
     case CPTK_HAS_TRIVIAL_CONSTRUCTOR:
-      inform (loc, "  %qT is not trivially default constructible", t1);
+      inform (decl_loc, "%qT is not trivially default constructible", t1);
       break;
     case CPTK_HAS_TRIVIAL_COPY:
-      inform (loc, "  %qT is not trivially copy constructible", t1);
+      inform (decl_loc, "%qT is not trivially copy constructible", t1);
       break;
     case CPTK_HAS_TRIVIAL_DESTRUCTOR:
-      inform (loc, "  %qT is not trivially destructible", t1);
+      inform (decl_loc, "%qT is not trivially destructible", t1);
       break;
     case CPTK_HAS_UNIQUE_OBJ_REPRESENTATIONS:
-      inform (loc, "  %qT does not have unique object representations", t1);
+      inform (decl_loc, "%qT does not have unique object representations", t1);
       break;
     case CPTK_HAS_VIRTUAL_DESTRUCTOR:
-      inform (loc, "  %qT does not have a virtual destructor", t1);
+      {
+	location_t dtor_loc = decl_loc;
+	if (NON_UNION_CLASS_TYPE_P (t1))
+	  if (tree dtor = CLASSTYPE_DESTRUCTOR (t1))
+	    dtor_loc = DECL_SOURCE_LOCATION (dtor);
+	inform (dtor_loc, "%qT does not have a virtual destructor", t1);
+      }
       break;
     case CPTK_IS_ABSTRACT:
-      inform (loc, "  %qT is not an abstract class", t1);
+      inform (decl_loc, "%qT is not an abstract class", t1);
       break;
     case CPTK_IS_AGGREGATE:
-      inform (loc, "  %qT is not an aggregate", t1);
+      inform (decl_loc, "%qT is not an aggregate", t1);
       break;
     case CPTK_IS_ARRAY:
-      inform (loc, "  %qT is not an array", t1);
+      inform (loc, "%qT is not an array", t1);
       break;
     case CPTK_IS_ASSIGNABLE:
-      inform (loc, "  %qT is not assignable from %qT", t1, t2);
+      inform (loc, "%qT is not assignable from %qT, because", t1, t2);
+      is_xible (MODIFY_EXPR, t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_BASE_OF:
-      inform (loc, "  %qT is not a base of %qT", t1, t2);
+      inform (decl_loc, "%qT is not a base of %qT", t1, t2);
       break;
     case CPTK_IS_BOUNDED_ARRAY:
-      inform (loc, "  %qT is not a bounded array", t1);
+      inform (loc, "%qT is not a bounded array", t1);
       break;
     case CPTK_IS_CLASS:
-      inform (loc, "  %qT is not a class", t1);
+      inform (decl_loc, "%qT is not a class", t1);
       break;
     case CPTK_IS_CONST:
-      inform (loc, "  %qT is not a const type", t1);
+      inform (loc, "%qT is not a const type", t1);
       break;
     case CPTK_IS_CONSTRUCTIBLE:
-      if (!t2)
-	inform (loc, "  %qT is not default constructible", t1);
+      if (!TREE_VEC_LENGTH (t2))
+	inform (loc, "%qT is not default constructible, because", t1);
       else
-	inform (loc, "  %qT is not constructible from %qE", t1, t2);
+	inform (loc, "%qT is not constructible from %qT, because", t1, t2);
+      is_xible (INIT_EXPR, t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_CONVERTIBLE:
-      inform (loc, "  %qT is not convertible from %qE", t2, t1);
+      /* The errors produced here all seem to mention "convertible" in the
+	 diagnostic, so an extra inform here appears redundant.  */
+      is_convertible (t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_DESTRUCTIBLE:
-      inform (loc, "  %qT is not destructible", t1);
+      inform (loc, "%qT is not destructible, because", t1);
+      is_xible (BIT_NOT_EXPR, t1, NULL_TREE, /*explain=*/true);
       break;
     case CPTK_IS_EMPTY:
-      inform (loc, "  %qT is not an empty class", t1);
+      inform (decl_loc, "%qT is not an empty class", t1);
       break;
     case CPTK_IS_ENUM:
-      inform (loc, "  %qT is not an enum", t1);
+      inform (decl_loc, "%qT is not an enum", t1);
       break;
     case CPTK_IS_FINAL:
-      inform (loc, "  %qT is not a final class", t1);
+      inform (decl_loc, "%qT is not a final class", t1);
       break;
     case CPTK_IS_FUNCTION:
-      inform (loc, "  %qT is not a function", t1);
+      inform (loc, "%qT is not a function", t1);
       break;
     case CPTK_IS_INVOCABLE:
-      if (!t2)
-	inform (loc, "  %qT is not invocable", t1);
-      else
-	inform (loc, "  %qT is not invocable by %qE", t1, t2);
+      {
+	if (!TREE_VEC_LENGTH (t2))
+	  inform (loc, "%qT is not invocable, because", t1);
+	else
+	  inform (loc, "%qT is not invocable by %qT, because", t1, t2);
+	build_invoke (t1, t2, tf_error);
+      }
       break;
     case CPTK_IS_LAYOUT_COMPATIBLE:
-      inform (loc, "  %qT is not layout compatible with %qT", t1, t2);
+      inform (loc, "%qT is not layout compatible with %qT", t1, t2);
       break;
     case CPTK_IS_LITERAL_TYPE:
-      inform (loc, "  %qT is not a literal type", t1);
+      inform (decl_loc, "%qT is not a literal type", t1);
       break;
     case CPTK_IS_MEMBER_FUNCTION_POINTER:
-      inform (loc, "  %qT is not a member function pointer", t1);
+      inform (loc, "%qT is not a member function pointer", t1);
       break;
     case CPTK_IS_MEMBER_OBJECT_POINTER:
-      inform (loc, "  %qT is not a member object pointer", t1);
+      inform (loc, "%qT is not a member object pointer", t1);
       break;
     case CPTK_IS_MEMBER_POINTER:
-      inform (loc, "  %qT is not a member pointer", t1);
+      inform (loc, "%qT is not a member pointer", t1);
       break;
     case CPTK_IS_NOTHROW_ASSIGNABLE:
-      inform (loc, "  %qT is not nothrow assignable from %qT", t1, t2);
+      inform (loc, "%qT is not nothrow assignable from %qT, because", t1, t2);
+      is_nothrow_xible (MODIFY_EXPR, t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_NOTHROW_CONSTRUCTIBLE:
-      if (!t2)
-	inform (loc, "  %qT is not nothrow default constructible", t1);
+      if (!TREE_VEC_LENGTH (t2))
+	inform (loc, "%qT is not nothrow default constructible, because", t1);
       else
-	inform (loc, "  %qT is not nothrow constructible from %qE", t1, t2);
+	inform (loc, "%qT is not nothrow constructible from %qT, because",
+		t1, t2);
+      is_nothrow_xible (INIT_EXPR, t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_NOTHROW_CONVERTIBLE:
-      inform (loc, "  %qT is not nothrow convertible from %qE", t2, t1);
+      inform (loc, "%qT is not nothrow convertible from %qT, because", t1, t2);
+      is_nothrow_convertible (t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_NOTHROW_DESTRUCTIBLE:
-      inform (loc, "  %qT is not nothrow destructible", t1);
+      inform (loc, "%qT is not nothrow destructible, because", t1);
+      is_nothrow_xible (BIT_NOT_EXPR, t1, NULL_TREE, /*explain=*/true);
       break;
     case CPTK_IS_NOTHROW_INVOCABLE:
-      if (!t2)
-	inform (loc, "  %qT is not nothrow invocable", t1);
-      else
-	inform (loc, "  %qT is not nothrow invocable by %qE", t1, t2);
+      {
+	if (!TREE_VEC_LENGTH (t2))
+	  inform (loc, "%qT is not nothrow invocable, because", t1);
+	else
+	  inform (loc, "%qT is not nothrow invocable by %qT, because", t1, t2);
+	tree call = build_invoke (t1, t2, tf_error);
+	if (call != error_mark_node)
+	  explain_not_noexcept (call);
+      }
+      break;
+    case CPTK_IS_NOTHROW_RELOCATABLE:
+      inform (loc, "%qT is not nothrow relocatable", t1);
       break;
     case CPTK_IS_OBJECT:
-      inform (loc, "  %qT is not an object type", t1);
+      inform (loc, "%qT is not an object type", t1);
       break;
     case CPTK_IS_POINTER_INTERCONVERTIBLE_BASE_OF:
-      inform (loc, "  %qT is not pointer-interconvertible base of %qT",
+      inform (decl_loc, "%qT is not a pointer-interconvertible base of %qT",
 	      t1, t2);
       break;
     case CPTK_IS_POD:
-      inform (loc, "  %qT is not a POD type", t1);
+      inform (loc, "%qT is not a POD type", t1);
       break;
     case CPTK_IS_POINTER:
-      inform (loc, "  %qT is not a pointer", t1);
+      inform (loc, "%qT is not a pointer", t1);
       break;
     case CPTK_IS_POLYMORPHIC:
-      inform (loc, "  %qT is not a polymorphic type", t1);
+      inform (decl_loc, "%qT is not a polymorphic type", t1);
       break;
     case CPTK_IS_REFERENCE:
-      inform (loc, "  %qT is not a reference", t1);
+      inform (loc, "%qT is not a reference", t1);
+      break;
+    case CPTK_IS_REPLACEABLE:
+      inform (loc, "%qT is not replaceable", t1);
       break;
     case CPTK_IS_SAME:
-      inform (loc, "  %qT is not the same as %qT", t1, t2);
+      inform (loc, "%q#T is not the same as %q#T", t1, t2);
       break;
     case CPTK_IS_SCOPED_ENUM:
-      inform (loc, "  %qT is not a scoped enum", t1);
+      inform (decl_loc, "%qT is not a scoped enum", t1);
       break;
     case CPTK_IS_STD_LAYOUT:
-      inform (loc, "  %qT is not an standard layout type", t1);
+      inform (decl_loc, "%qT is not a standard layout type", t1);
       break;
     case CPTK_IS_TRIVIAL:
-      inform (loc, "  %qT is not a trivial type", t1);
+      inform (decl_loc, "%qT is not a trivial type", t1);
       break;
     case CPTK_IS_TRIVIALLY_ASSIGNABLE:
-      inform (loc, "  %qT is not trivially assignable from %qT", t1, t2);
+      inform (loc, "%qT is not trivially assignable from %qT, because", t1, t2);
+      is_trivially_xible (MODIFY_EXPR, t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_TRIVIALLY_CONSTRUCTIBLE:
-      if (!t2)
-	inform (loc, "  %qT is not trivially default constructible", t1);
+      if (!TREE_VEC_LENGTH (t2))
+	inform (loc, "%qT is not trivially default constructible, because", t1);
       else
-	inform (loc, "  %qT is not trivially constructible from %qE", t1, t2);
+	inform (loc, "%qT is not trivially constructible from %qT, because",
+		t1, t2);
+      is_trivially_xible (INIT_EXPR, t1, t2, /*explain=*/true);
       break;
     case CPTK_IS_TRIVIALLY_COPYABLE:
-      inform (loc, "  %qT is not trivially copyable", t1);
+      inform (decl_loc, "%qT is not trivially copyable", t1);
       break;
     case CPTK_IS_TRIVIALLY_DESTRUCTIBLE:
-      inform (loc, "  %qT is not trivially destructible", t1);
+      inform (loc, "%qT is not trivially destructible, because", t1);
+      is_trivially_xible (BIT_NOT_EXPR, t1, NULL_TREE, /*explain=*/true);
+      break;
+    case CPTK_IS_TRIVIALLY_RELOCATABLE:
+      inform (loc, "%qT is not trivially relocatable", t1);
       break;
     case CPTK_IS_UNBOUNDED_ARRAY:
-      inform (loc, "  %qT is not an unbounded array", t1);
+      inform (loc, "%qT is not an unbounded array", t1);
       break;
     case CPTK_IS_UNION:
-      inform (loc, "  %qT is not a union", t1);
+      inform (decl_loc, "%qT is not a union", t1);
       break;
     case CPTK_IS_VIRTUAL_BASE_OF:
-      inform (loc, "  %qT is not a virtual base of %qT", t1, t2);
+      inform (decl_loc, "%qT is not a virtual base of %qT", t1, t2);
+      if (CLASS_TYPE_P (t2))
+	inform (location_of (t2), "%qT declared here", t2);
       break;
     case CPTK_IS_VOLATILE:
-      inform (loc, "  %qT is not a volatile type", t1);
+      inform (loc, "%qT is not a volatile type", t1);
       break;
     case CPTK_RANK:
-      inform (loc, "  %qT cannot yield a rank", t1);
+      inform (loc, "%qT cannot yield a rank", t1);
       break;
     case CPTK_TYPE_ORDER:
-      inform (loc, "  %qT and %qT cannot be ordered", t1, t2);
+      inform (loc, "%qT and %qT cannot be ordered", t1, t2);
       break;
     case CPTK_REF_CONSTRUCTS_FROM_TEMPORARY:
-      inform (loc, "  %qT is not a reference that binds to a temporary "
+      inform (loc, "%qT is not a reference that binds to a temporary "
 	      "object of type %qT (direct-initialization)", t1, t2);
       break;
     case CPTK_REF_CONVERTS_FROM_TEMPORARY:
-      inform (loc, "  %qT is not a reference that binds to a temporary "
+      inform (loc, "%qT is not a reference that binds to a temporary "
 	      "object of type %qT (copy-initialization)", t1, t2);
       break;
     case CPTK_IS_DEDUCIBLE:
-      inform (loc, "  %qD is not deducible from %qT", t1, t2);
+      inform (loc, "%qD is not deducible from %qT", t1, t2);
       break;
 #define DEFTRAIT_TYPE(CODE, NAME, ARITY) \
     case CPTK_##CODE:
@@ -3291,10 +3326,50 @@ diagnose_trait_expr (tree expr, tree args)
     }
 }
 
+/* Attempt to detect if this is a standard type trait, defined in terms
+   of a compiler builtin (above).  If so, this will allow us to provide
+   more helpful diagnostics.   */
+
+bool
+maybe_diagnose_standard_trait (location_t loc, tree expr)
+{
+  gcc_assert (TREE_CODE (expr) != TRAIT_EXPR);
+  expr = tree_strip_nop_conversions (expr);
+
+  /* TODO: in some cases it would be possible to provide more helpful
+     diagnostics for negations of traits, e.g. '!is_same_v<T1, T2>'.  */
+
+  tree args = NULL_TREE;
+  if (VAR_P (expr) && DECL_LANG_SPECIFIC (expr) && DECL_USE_TEMPLATE (expr))
+    {
+      tree tinfo = DECL_TEMPLATE_INFO (expr);
+      if (PRIMARY_TEMPLATE_P (TI_TEMPLATE (tinfo)) && TI_PARTIAL_INFO (tinfo))
+	tinfo = TI_PARTIAL_INFO (tinfo);
+      else if (DECL_TEMPLATE_SPECIALIZATION (expr))
+	/* In an explicit specialisation we no longer know what the original
+	   initializer looked like.  */
+	tinfo = NULL_TREE;
+
+      if (tinfo)
+	{
+	  expr = DECL_INITIAL (DECL_TEMPLATE_RESULT (TI_TEMPLATE (tinfo)));
+	  args = TI_ARGS (tinfo);
+	}
+    }
+
+  if (TREE_CODE (expr) == TRAIT_EXPR)
+    {
+      diagnose_trait_expr (loc, expr, args);
+      return true;
+    }
+
+  return false;
+}
+
 /* Diagnose a substitution failure in the atomic constraint T using ARGS.  */
 
 static void
-diagnose_atomic_constraint (tree t, tree args, tree result, sat_info info)
+diagnose_atomic_constraint (tree t, tree args, tree substituted, sat_info info)
 {
   /* If the constraint is already ill-formed, we've previously diagnosed
      the reason.  We should still say why the constraints aren't satisfied.  */
@@ -3315,25 +3390,26 @@ diagnose_atomic_constraint (tree t, tree args, tree result, sat_info info)
   /* Generate better diagnostics for certain kinds of expressions.  */
   tree expr = ATOMIC_CONSTR_EXPR (t);
   STRIP_ANY_LOCATION_WRAPPER (expr);
-  switch (TREE_CODE (expr))
+
+  if (TREE_CODE (expr) == REQUIRES_EXPR)
     {
-    case TRAIT_EXPR:
-      diagnose_trait_expr (expr, args);
-      break;
-    case REQUIRES_EXPR:
       gcc_checking_assert (info.diagnose_unsatisfaction_p ());
       /* Clear in_decl before replaying the substitution to avoid emitting
 	 seemingly unhelpful "in declaration ..." notes that follow some
 	 substitution failure error messages.  */
       info.in_decl = NULL_TREE;
       tsubst_requires_expr (expr, args, info);
-      break;
-    default:
-      if (!same_type_p (TREE_TYPE (result), boolean_type_node))
-	error_at (loc, "constraint %qE has type %qT, not %<bool%>",
-		  t, TREE_TYPE (result));
+    }
+  else if (!same_type_p (TREE_TYPE (substituted), boolean_type_node))
+    error_at (loc, "constraint %qE has type %qT, not %<bool%>",
+	      t, TREE_TYPE (substituted));
+  else
+    {
+      inform (loc, "the expression %qE evaluated to %<false%>", t);
+      if (TREE_CODE (expr) == TRAIT_EXPR)
+	diagnose_trait_expr (loc, expr, args);
       else
-	inform (loc, "the expression %qE evaluated to %<false%>", t);
+	maybe_diagnose_standard_trait (loc, substituted);
     }
 }
 
diff --git a/gcc/cp/coroutines.cc b/gcc/cp/coroutines.cc
index 52cc186..690e510 100644
--- a/gcc/cp/coroutines.cc
+++ b/gcc/cp/coroutines.cc
@@ -5050,6 +5050,8 @@ cp_coroutine_transform::build_ramp_function ()
      check the returned pointer and call the func if it's null.
      Otherwise, no check, and we fail for noexcept/fno-exceptions cases.  */
 
+  tree grooaf_if_stmt = NULL_TREE;
+  tree alloc_ok_scope = NULL_TREE;
   if (grooaf)
     {
       /* [dcl.fct.def.coroutine] / 10 (part 3)
@@ -5057,20 +5059,11 @@ cp_coroutine_transform::build_ramp_function ()
 	 control to the caller of the coroutine and the return value is
 	 obtained by a call to T::get_return_object_on_allocation_failure(),
 	 where T is the promise type.  */
-      tree if_stmt = begin_if_stmt ();
       tree cond = build1 (CONVERT_EXPR, frame_ptr_type, nullptr_node);
-      cond = build2 (EQ_EXPR, boolean_type_node, coro_fp, cond);
-      finish_if_stmt_cond (cond, if_stmt);
-      r = NULL_TREE;
-      if (void_ramp_p)
-	/* Execute the get-return-object-on-alloc-fail call...  */
-	finish_expr_stmt (grooaf);
-      else
-	/* Get the fallback return object.  */
-	r = grooaf;
-      finish_return_stmt (r);
-      finish_then_clause (if_stmt);
-      finish_if_stmt (if_stmt);
+      cond = build2 (NE_EXPR, boolean_type_node, coro_fp, cond);
+      grooaf_if_stmt = begin_if_stmt ();
+      finish_if_stmt_cond (cond, grooaf_if_stmt);
+      alloc_ok_scope = begin_compound_stmt (BCS_NORMAL);
     }
 
   /* Dereference the frame pointer, to use in member access code.  */
@@ -5295,7 +5288,6 @@ cp_coroutine_transform::build_ramp_function ()
 	 a temp which is then used to intialize the return object, including
 	 NVRO.  */
 
-      /* Temporary var to hold the g_r_o across the function body.  */
       coro_gro
 	= coro_build_and_push_artificial_var (loc, "_Coro_gro", gro_type,
 					      orig_fn_decl, NULL_TREE);
@@ -5328,9 +5320,28 @@ cp_coroutine_transform::build_ramp_function ()
   /* The ramp is done, we just need the return statement, which we build from
      the return object we constructed before we called the actor.  */
 
+  /* This is our 'normal' exit.  */
   r = void_ramp_p ? NULL_TREE : convert_from_reference (coro_gro);
   finish_return_stmt (r);
 
+  if (grooaf)
+    {
+      finish_compound_stmt (alloc_ok_scope);
+      finish_then_clause (grooaf_if_stmt);
+
+      begin_else_clause (grooaf_if_stmt);
+      /* We come here if the frame allocation failed.  */
+      r = NULL_TREE;
+      if (void_ramp_p)
+	/* Execute the get-return-object-on-alloc-fail call...  */
+	finish_expr_stmt (grooaf);
+      else
+	/* Get the fallback return object.  */
+	r = grooaf;
+      finish_return_stmt (r);
+      finish_if_stmt (grooaf_if_stmt);
+    }
+
   finish_compound_stmt (ramp_fnbody);
   return true;
 }
diff --git a/gcc/cp/cp-gimplify.cc b/gcc/cp/cp-gimplify.cc
index ce69bd6..4ff8f36a 100644
--- a/gcc/cp/cp-gimplify.cc
+++ b/gcc/cp/cp-gimplify.cc
@@ -690,6 +690,7 @@ cp_gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
 		    && (REFERENCE_CLASS_P (op1) || DECL_P (op1)))
 		  op1 = build_fold_addr_expr (op1);
 
+		suppress_warning (op1, OPT_Wunused_result);
 		gimplify_and_add (op1, pre_p);
 	      }
 	    gimplify_expr (&TREE_OPERAND (*expr_p, 0), pre_p, post_p,
@@ -889,6 +890,12 @@ cp_gimplify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
 			(EXPR_LOCATION (*expr_p), call_expr_nargs (*expr_p),
 			 &CALL_EXPR_ARG (*expr_p, 0));
 		break;
+	      case CP_BUILT_IN_EH_PTR_ADJUST_REF:
+		error_at (EXPR_LOCATION (*expr_p),
+			  "%qs used outside of constant expressions",
+			  "__builtin_eh_ptr_adjust_ref");
+		*expr_p = void_node;
+		break;
 	      default:
 		break;
 	      }
@@ -3022,7 +3029,7 @@ cp_fold (tree x, fold_flags_t flags)
     case CLEANUP_POINT_EXPR:
       /* Strip CLEANUP_POINT_EXPR if the expression doesn't have side
 	 effects.  */
-      r = cp_fold_rvalue (TREE_OPERAND (x, 0), flags);
+      r = cp_fold (TREE_OPERAND (x, 0), flags);
       if (!TREE_SIDE_EFFECTS (r))
 	x = r;
       break;
@@ -3211,7 +3218,16 @@ cp_fold (tree x, fold_flags_t flags)
 
       loc = EXPR_LOCATION (x);
       op0 = cp_fold_maybe_rvalue (TREE_OPERAND (x, 0), rval_ops, flags);
-      op1 = cp_fold_rvalue (TREE_OPERAND (x, 1), flags);
+      bool clear_decl_read;
+      clear_decl_read = false;
+      if (code == MODIFY_EXPR
+	  && (VAR_P (op0) || TREE_CODE (op0) == PARM_DECL)
+	  && !DECL_READ_P (op0))
+	clear_decl_read = true;
+      op1 = cp_fold_maybe_rvalue (TREE_OPERAND (x, 1),
+				  code != COMPOUND_EXPR, flags);
+      if (clear_decl_read)
+	DECL_READ_P (op0) = 0;
 
       /* decltype(nullptr) has only one value, so optimize away all comparisons
 	 with that type right away, keeping them in the IL causes troubles for
diff --git a/gcc/cp/cp-trait.def b/gcc/cp/cp-trait.def
index e71b28c..9fedfd7 100644
--- a/gcc/cp/cp-trait.def
+++ b/gcc/cp/cp-trait.def
@@ -87,12 +87,14 @@ DEFTRAIT_EXPR (IS_NOTHROW_CONSTRUCTIBLE, "__is_nothrow_constructible", -1)
 DEFTRAIT_EXPR (IS_NOTHROW_CONVERTIBLE, "__is_nothrow_convertible", 2)
 DEFTRAIT_EXPR (IS_NOTHROW_DESTRUCTIBLE, "__is_nothrow_destructible", 1)
 DEFTRAIT_EXPR (IS_NOTHROW_INVOCABLE, "__is_nothrow_invocable", -1)
+DEFTRAIT_EXPR (IS_NOTHROW_RELOCATABLE, "__builtin_is_nothrow_relocatable", 1)
 DEFTRAIT_EXPR (IS_OBJECT, "__is_object", 1)
 DEFTRAIT_EXPR (IS_POINTER_INTERCONVERTIBLE_BASE_OF, "__is_pointer_interconvertible_base_of", 2)
 DEFTRAIT_EXPR (IS_POD, "__is_pod", 1)
 DEFTRAIT_EXPR (IS_POINTER, "__is_pointer", 1)
 DEFTRAIT_EXPR (IS_POLYMORPHIC, "__is_polymorphic", 1)
 DEFTRAIT_EXPR (IS_REFERENCE, "__is_reference", 1)
+DEFTRAIT_EXPR (IS_REPLACEABLE, "__builtin_is_replaceable", 1)
 DEFTRAIT_EXPR (IS_SAME, "__is_same", 2)
 DEFTRAIT_EXPR (IS_SCOPED_ENUM, "__is_scoped_enum", 1)
 DEFTRAIT_EXPR (IS_STD_LAYOUT, "__is_standard_layout", 1)
@@ -101,6 +103,7 @@ DEFTRAIT_EXPR (IS_TRIVIALLY_ASSIGNABLE, "__is_trivially_assignable", 2)
 DEFTRAIT_EXPR (IS_TRIVIALLY_CONSTRUCTIBLE, "__is_trivially_constructible", -1)
 DEFTRAIT_EXPR (IS_TRIVIALLY_COPYABLE, "__is_trivially_copyable", 1)
 DEFTRAIT_EXPR (IS_TRIVIALLY_DESTRUCTIBLE, "__is_trivially_destructible", 1)
+DEFTRAIT_EXPR (IS_TRIVIALLY_RELOCATABLE, "__builtin_is_trivially_relocatable", 1)
 DEFTRAIT_EXPR (IS_UNBOUNDED_ARRAY, "__is_unbounded_array", 1)
 DEFTRAIT_EXPR (IS_UNION, "__is_union", 1)
 DEFTRAIT_EXPR (IS_VIRTUAL_BASE_OF, "__builtin_is_virtual_base_of", 2)
diff --git a/gcc/cp/cp-tree.h b/gcc/cp/cp-tree.h
index 1b893e2..fb8e0d8 100644
--- a/gcc/cp/cp-tree.h
+++ b/gcc/cp/cp-tree.h
@@ -452,6 +452,9 @@ extern GTY(()) tree cp_global_trees[CPTI_MAX];
       contract_semantic (in ASSERTION_, PRECONDITION_, POSTCONDITION_STMT)
       RETURN_EXPR_LOCAL_ADDR_P (in RETURN_EXPR)
       PACK_INDEX_PARENTHESIZED_P (in PACK_INDEX_*)
+      MUST_NOT_THROW_NOEXCEPT_P (in MUST_NOT_THROW_EXPR)
+      CONSTEVAL_BLOCK_P (in STATIC_ASSERT)
+      LAMBDA_EXPR_CONSTEVAL_BLOCK_P (in LAMBDA_EXPR)
    1: IDENTIFIER_KIND_BIT_1 (in IDENTIFIER_NODE)
       TI_PENDING_TEMPLATE_FLAG.
       TEMPLATE_PARMS_FOR_INLINE.
@@ -472,6 +475,7 @@ extern GTY(()) tree cp_global_trees[CPTI_MAX];
       BIND_EXPR_VEC_DTOR (in BIND_EXPR)
       ATOMIC_CONSTR_EXPR_FROM_CONCEPT_P (in ATOMIC_CONSTR)
       STATIC_INIT_DECOMP_BASE_P (in the TREE_LIST for {static,tls}_aggregates)
+      MUST_NOT_THROW_THROW_P (in MUST_NOT_THROW_EXPR)
    2: IDENTIFIER_KIND_BIT_2 (in IDENTIFIER_NODE)
       ICS_THIS_FLAG (in _CONV)
       DECL_INITIALIZED_BY_CONSTANT_EXPRESSION_P (in VAR_DECL)
@@ -493,6 +497,7 @@ extern GTY(()) tree cp_global_trees[CPTI_MAX];
       contract_semantic (in ASSERTION_, PRECONDITION_, POSTCONDITION_STMT)
       STATIC_INIT_DECOMP_NONBASE_P (in the TREE_LIST
 				    for {static,tls}_aggregates)
+      MUST_NOT_THROW_CATCH_P (in MUST_NOT_THROW_EXPR)
    3: IMPLICIT_RVALUE_P (in NON_LVALUE_EXPR or STATIC_CAST_EXPR)
       ICS_BAD_FLAG (in _CONV)
       FN_TRY_BLOCK_P (in TRY_BLOCK)
@@ -506,6 +511,7 @@ extern GTY(()) tree cp_global_trees[CPTI_MAX];
       LAMBDA_EXPR_STATIC_P (in LAMBDA_EXPR)
       TARGET_EXPR_ELIDING_P (in TARGET_EXPR)
       contract_semantic (in ASSERTION_, PRECONDITION_, POSTCONDITION_STMT)
+      TYPENAME_IS_UNION_P (in TYPENAME_TYPE)
    4: IDENTIFIER_MARKED (IDENTIFIER_NODEs)
       TREE_HAS_CONSTRUCTOR (in INDIRECT_REF, SAVE_EXPR, CONSTRUCTOR,
 	  CALL_EXPR, or FIELD_DECL).
@@ -1429,6 +1435,10 @@ struct GTY (()) tree_deferred_noexcept {
 #define STATIC_ASSERT_SOURCE_LOCATION(NODE) \
   (((struct tree_static_assert *)STATIC_ASSERT_CHECK (NODE))->location)
 
+/* True if this static assert represents a C++26 consteval block.  */
+#define CONSTEVAL_BLOCK_P(NODE) \
+  TREE_LANG_FLAG_0 (STATIC_ASSERT_CHECK (NODE))
+
 struct GTY (()) tree_static_assert {
   struct tree_base base;
   tree condition;
@@ -1543,6 +1553,10 @@ enum cp_lambda_default_capture_mode_type {
 #define LAMBDA_EXPR_THIS_CAPTURE(NODE) \
   (((struct tree_lambda_expr *)LAMBDA_EXPR_CHECK (NODE))->this_capture)
 
+/* True iff this lambda was created for a consteval block.  */
+#define LAMBDA_EXPR_CONSTEVAL_BLOCK_P(NODE) \
+  TREE_LANG_FLAG_0 (LAMBDA_EXPR_CHECK (NODE))
+
 /* True iff uses of a const variable capture were optimized away.  */
 #define LAMBDA_EXPR_CAPTURE_OPTIMIZED(NODE) \
   TREE_LANG_FLAG_2 (LAMBDA_EXPR_CHECK (NODE))
@@ -2354,6 +2368,10 @@ enum languages { lang_c, lang_cplusplus };
 #define NON_UNION_CLASS_TYPE_P(T) \
   (TREE_CODE (T) == RECORD_TYPE && TYPE_LANG_FLAG_5 (T))
 
+/* Nonzero if T is a class type and is a union.  */
+#define UNION_TYPE_P(T) \
+  (TREE_CODE (T) == UNION_TYPE && TYPE_LANG_FLAG_5 (T))
+
 /* Keep these checks in ascending code order.  */
 #define RECORD_OR_UNION_CODE_P(T)	\
   ((T) == RECORD_TYPE || (T) == UNION_TYPE)
@@ -2492,15 +2510,22 @@ struct GTY(()) lang_type {
   bool erroneous : 1;
   bool non_pod_aggregate : 1;
   bool non_aggregate_pod : 1;
+  bool trivially_relocatable : 1;
+  bool trivially_relocatable_computed : 1;
+
+  bool replaceable : 1;
+  bool replaceable_computed : 1;
 
   /* When adding a flag here, consider whether or not it ought to
      apply to a template instance if it applies to the template.  If
-     so, make sure to copy it in instantiate_class_template!  */
+     so, make sure to copy it in instantiate_class_template!
+
+     Also make sure new flags here are streamed in module.cc.  */
 
   /* There are some bits left to fill out a 32-bit word.  Keep track
      of this by updating the size of this bitfield whenever you add or
      remove a flag.  */
-  unsigned dummy : 2;
+  unsigned dummy : 30;
 
   tree primary_base;
   vec<tree_pair_s, va_gc> *vcall_indices;
@@ -2511,11 +2536,11 @@ struct GTY(()) lang_type {
   vec<tree, va_gc> *pure_virtuals;
   tree friend_classes;
   vec<tree, va_gc> * GTY((reorder ("resort_type_member_vec"))) members;
+  /* CLASSTYPE_KEY_METHOD for TYPE_POLYMORPHIC_P types, CLASSTYPE_LAMBDA_EXPR
+     otherwise.  */
   tree key_method;
   tree decl_list;
   tree befriending_classes;
-  /* FIXME reuse another field?  */
-  tree lambda_expr;
   union maybe_objc_info {
     /* If not c_dialect_objc, this part is not even allocated.  */
     char GTY((tag ("0"))) non_objc;
@@ -2638,7 +2663,13 @@ struct GTY(()) lang_type {
 /* The member function with which the vtable will be emitted:
    the first noninline non-pure-virtual member function.  NULL_TREE
    if there is no key function or if this is a class template */
-#define CLASSTYPE_KEY_METHOD(NODE) (LANG_TYPE_CLASS_CHECK (NODE)->key_method)
+#define CLASSTYPE_KEY_METHOD(NODE) \
+  (TYPE_POLYMORPHIC_P (NODE)					\
+   ? LANG_TYPE_CLASS_CHECK (NODE)->key_method			\
+   : NULL_TREE)
+#define SET_CLASSTYPE_KEY_METHOD(NODE, VALUE) \
+  (gcc_checking_assert (TYPE_POLYMORPHIC_P (NODE)),		\
+   LANG_TYPE_CLASS_CHECK (NODE)->key_method = (VALUE))
 
 /* Vector of members.  During definition, it is unordered and only
    member functions are present.  After completion it is sorted and
@@ -2770,7 +2801,12 @@ struct GTY(()) lang_type {
 
 /* The associated LAMBDA_EXPR that made this class.  */
 #define CLASSTYPE_LAMBDA_EXPR(NODE) \
-  (LANG_TYPE_CLASS_CHECK (NODE)->lambda_expr)
+  (TYPE_POLYMORPHIC_P (NODE)				\
+   ? NULL_TREE						\
+   : LANG_TYPE_CLASS_CHECK (NODE)->key_method)
+#define SET_CLASSTYPE_LAMBDA_EXPR(NODE, VALUE) \
+  (gcc_checking_assert (!TYPE_POLYMORPHIC_P (NODE)),	\
+   LANG_TYPE_CLASS_CHECK (NODE)->key_method = (VALUE))
 /* The extra mangling scope for this closure type.  */
 #define LAMBDA_TYPE_EXTRA_SCOPE(NODE) \
   (LAMBDA_EXPR_EXTRA_SCOPE (CLASSTYPE_LAMBDA_EXPR (NODE)))
@@ -2836,6 +2872,29 @@ struct GTY(()) lang_type {
    above (c++/120012).  This could also be a hash_set.  */
 #define CLASSTYPE_NON_AGGREGATE_POD(NODE) \
   (LANG_TYPE_CLASS_CHECK (NODE)->non_aggregate_pod)
+
+/* If CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED, true if this class is
+   trivially relocatable.
+   If !CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED, true if this class
+   is marked with trivially_relocatable_if_eligible conditional keyword.  */
+#define CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT(NODE) \
+  (LANG_TYPE_CLASS_CHECK (NODE)->trivially_relocatable)
+
+/* True if whether this class is trivially relocatable or not
+   has been computed already.  */
+#define CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED(NODE) \
+  (LANG_TYPE_CLASS_CHECK (NODE)->trivially_relocatable_computed)
+
+/* If CLASSTYPE_REPLACEABLE_COMPUTED, true if this class is replaceable.
+   If !CLASSTYPE_REPLACEABLE_COMPUTED, true if this class is marked with
+   replaceable_if_eligible conditional keyword.  */
+#define CLASSTYPE_REPLACEABLE_BIT(NODE) \
+  (LANG_TYPE_CLASS_CHECK (NODE)->replaceable)
+
+/* True if whether this class is replaceable or not has been computed
+   already.  */
+#define CLASSTYPE_REPLACEABLE_COMPUTED(NODE) \
+  (LANG_TYPE_CLASS_CHECK (NODE)->replaceable_computed)
 
 /* Additional macros for inheritance information.  */
 
@@ -3011,6 +3070,8 @@ struct GTY(()) lang_decl_min {
      In a lambda-capture proxy VAR_DECL, this is DECL_CAPTURED_VARIABLE.
      In a function-scope TREE_STATIC VAR_DECL or IMPLICIT_TYPEDEF_P TYPE_DECL,
      this is DECL_DISCRIMINATOR.
+     In constexpr exception artificial VAR_DECL, this is
+     DECL_EXCEPTION_REFCOUNT.
      In a DECL_LOCAL_DECL_P decl, this is the namespace decl it aliases.
      Otherwise, in a class-scope DECL, this is DECL_ACCESS.   */
   tree access;
@@ -4465,6 +4526,23 @@ get_vec_init_expr (tree t)
 #define MUST_NOT_THROW_COND(NODE) \
   TREE_OPERAND (MUST_NOT_THROW_EXPR_CHECK (NODE), 1)
 
+/* Reasons why MUST_NOT_THROW_EXPR has been created.  */
+
+/* Indicates MUST_NOT_THROW_EXPR has been created to wrap body of
+   a noexcept function.  */
+#define MUST_NOT_THROW_NOEXCEPT_P(NODE) \
+  TREE_LANG_FLAG_0 (MUST_NOT_THROW_EXPR_CHECK (NODE))
+
+/* Indicates MUST_NOT_THROW_EXPR has been created to wrap construction of
+   exception object during throw.  */
+#define MUST_NOT_THROW_THROW_P(NODE) \
+  TREE_LANG_FLAG_1 (MUST_NOT_THROW_EXPR_CHECK (NODE))
+
+/* Indicates MUST_NOT_THROW_EXPR has been created to wrap construction of
+   handler parameter during catch.  */
+#define MUST_NOT_THROW_CATCH_P(NODE) \
+  TREE_LANG_FLAG_2 (MUST_NOT_THROW_EXPR_CHECK (NODE))
+
 /* The TYPE_MAIN_DECL for a class template type is a TYPE_DECL, not a
    TEMPLATE_DECL.  This macro determines whether or not a given class
    type is really a template type, as opposed to an instantiation or
@@ -4485,11 +4563,14 @@ get_vec_init_expr (tree t)
 #define TYPENAME_IS_ENUM_P(NODE) \
   (TREE_LANG_FLAG_0 (TYPENAME_TYPE_CHECK (NODE)))
 
-/* True if a TYPENAME_TYPE was declared as a "class", "struct", or
-   "union".  */
+/* True if a TYPENAME_TYPE was declared as a "class" or "struct".  */
 #define TYPENAME_IS_CLASS_P(NODE) \
   (TREE_LANG_FLAG_1 (TYPENAME_TYPE_CHECK (NODE)))
 
+/* True if a TYPENAME_TYPE was declared as a "union".  */
+#define TYPENAME_IS_UNION_P(NODE) \
+  (TREE_LANG_FLAG_3 (TYPENAME_TYPE_CHECK (NODE)))
+
 /* True if a TYPENAME_TYPE is in the process of being resolved.  */
 #define TYPENAME_IS_RESOLVING_P(NODE) \
   (TREE_LANG_FLAG_2 (TYPENAME_TYPE_CHECK (NODE)))
@@ -4504,7 +4585,7 @@ get_vec_init_expr (tree t)
 #define TYPE_CONTAINS_VPTR_P(NODE)		\
   (TYPE_POLYMORPHIC_P (NODE) || CLASSTYPE_VBASECLASSES (NODE))
 
-/* Nonzero if NODE is a FUNCTION_DECL or VARIABLE_DECL (for a decl
+/* Nonzero if NODE is a FUNCTION_DECL or VAR_DECL (for a decl
    with namespace scope) declared in a local scope.  */
 #define DECL_LOCAL_DECL_P(NODE) \
   DECL_LANG_FLAG_0 (VAR_OR_FUNCTION_DECL_CHECK (NODE))
@@ -5145,6 +5226,10 @@ get_vec_init_expr (tree t)
    protected_access_node will appear in the DECL_ACCESS for the node.  */
 #define DECL_ACCESS(NODE) (LANG_DECL_MIN_CHECK (NODE)->access)
 
+/* In artificial VAR_DECL created by cxa_allocate_exception
+   this is reference count.  */
+#define DECL_EXCEPTION_REFCOUNT(NODE) (LANG_DECL_MIN_CHECK (NODE)->access)
+
 /* Nonzero if the FUNCTION_DECL is a global constructor.  */
 #define DECL_GLOBAL_CTOR_P(NODE) \
   (LANG_DECL_FN_CHECK (NODE)->global_ctor_p)
@@ -6463,7 +6548,9 @@ enum virt_specifier
   {
     VIRT_SPEC_UNSPECIFIED = 0x0,
     VIRT_SPEC_FINAL       = 0x1,
-    VIRT_SPEC_OVERRIDE    = 0x2
+    VIRT_SPEC_OVERRIDE    = 0x2,
+    VIRT_SPEC_TRIVIALLY_RELOCATABLE_IF_ELIGIBLE = 0x4,
+    VIRT_SPEC_REPLACEABLE_IF_ELIGIBLE = 0x8
   };
 
 /* A type-qualifier, or bitmask therefore, using the VIRT_SPEC
@@ -6805,6 +6892,7 @@ enum cp_built_in_function {
   CP_BUILT_IN_IS_CORRESPONDING_MEMBER,
   CP_BUILT_IN_IS_POINTER_INTERCONVERTIBLE_WITH_CLASS,
   CP_BUILT_IN_SOURCE_LOCATION,
+  CP_BUILT_IN_EH_PTR_ADJUST_REF,
   CP_BUILT_IN_LAST
 };
 
@@ -6985,6 +7073,7 @@ extern bool type_has_extended_temps		(tree);
 extern tree strip_top_quals			(tree);
 extern bool reference_related_p			(tree, tree);
 extern bool reference_compatible_p		(tree, tree);
+extern bool handler_match_for_exception_type	(tree, tree);
 extern int remaining_arguments			(tree);
 extern tree build_implicit_conv_flags		(tree, tree, int);
 extern tree perform_implicit_conversion		(tree, tree, tsubst_flags_t);
@@ -7155,7 +7244,7 @@ extern void determine_local_discriminator	(tree, tree = NULL_TREE);
 extern bool member_like_constrained_friend_p	(tree);
 extern bool fns_correspond			(tree, tree);
 extern int decls_match				(tree, tree, bool = true);
-extern bool maybe_version_functions		(tree, tree, bool);
+extern bool maybe_version_functions		(tree, tree);
 extern bool validate_constexpr_redeclaration	(tree, tree);
 extern bool merge_default_template_args		(tree, tree, bool);
 extern tree duplicate_decls			(tree, tree,
@@ -7378,7 +7467,7 @@ extern void maybe_warn_variadic_templates       (void);
 extern void maybe_warn_cpp0x			(cpp0x_warn_str str,
 						 location_t = input_location);
 extern bool pedwarn_cxx98                       (location_t,
-						 diagnostic_option_id option_id,
+						 diagnostics::option_id option_id,
 						 const char *, ...) ATTRIBUTE_GCC_DIAG(3,4);
 extern location_t location_of                   (tree);
 extern void qualified_name_lookup_error		(tree, tree, tree,
@@ -7404,6 +7493,7 @@ extern int nothrow_libfn_p			(const_tree);
 extern void check_handlers			(tree);
 extern tree finish_noexcept_expr		(tree, tsubst_flags_t);
 extern bool expr_noexcept_p			(tree, tsubst_flags_t);
+extern void explain_not_noexcept		(tree);
 extern void perform_deferred_noexcept_checks	(void);
 extern bool nothrow_spec_p			(const_tree);
 extern bool type_noexcept_p			(const_tree);
@@ -7525,11 +7615,14 @@ extern void finish_thunk			(tree);
 extern void use_thunk				(tree, bool);
 extern bool trivial_fn_p			(tree);
 extern tree forward_parm			(tree);
-extern bool is_trivially_xible			(enum tree_code, tree, tree);
-extern bool is_nothrow_xible			(enum tree_code, tree, tree);
-extern bool is_xible				(enum tree_code, tree, tree);
-extern bool is_convertible			(tree, tree);
-extern bool is_nothrow_convertible		(tree, tree);
+extern bool is_trivially_xible			(enum tree_code, tree, tree,
+						 bool = false);
+extern bool is_nothrow_xible			(enum tree_code, tree, tree,
+						 bool = false);
+extern bool is_xible				(enum tree_code, tree, tree,
+						 bool = false);
+extern bool is_convertible			(tree, tree, bool = false);
+extern bool is_nothrow_convertible		(tree, tree, bool = false);
 extern bool ref_xes_from_temporary		(tree, tree, bool);
 extern tree get_defaulted_eh_spec		(tree, tsubst_flags_t = tf_warning_or_error);
 extern bool maybe_explain_implicit_delete	(tree);
@@ -8160,7 +8253,7 @@ extern bool cxx_omp_create_clause_info		(tree, tree, bool, bool,
 						 bool, bool);
 extern tree baselink_for_fns                    (tree);
 extern void finish_static_assert                (tree, tree, location_t,
-						 bool, bool);
+						 bool, bool, bool = false);
 extern tree finish_decltype_type                (tree, bool, tsubst_flags_t);
 extern tree fold_builtin_is_corresponding_member (location_t, int, tree *);
 extern tree fold_builtin_is_pointer_inverconvertible_with_class (location_t, int, tree *);
@@ -8185,7 +8278,7 @@ extern void register_capture_members		(tree);
 extern tree lambda_expr_this_capture            (tree, int);
 extern void maybe_generic_this_capture		(tree, tree);
 extern tree maybe_resolve_dummy			(tree, bool);
-extern tree current_nonlambda_function		(void);
+extern tree current_nonlambda_function		(bool = false);
 extern tree nonlambda_method_basetype		(void);
 extern tree current_nonlambda_scope		(bool = false);
 extern tree current_lambda_expr			(void);
@@ -8235,6 +8328,8 @@ extern bool pod_type_p				(const_tree);
 extern bool layout_pod_type_p			(const_tree);
 extern bool std_layout_type_p			(const_tree);
 extern bool trivial_type_p			(const_tree);
+extern bool trivially_relocatable_type_p	(tree);
+extern bool replaceable_type_p			(tree);
 extern bool trivially_copyable_p		(const_tree);
 extern bool type_has_unique_obj_representations (const_tree);
 extern bool scalarish_type_p			(const_tree);
@@ -8380,9 +8475,9 @@ extern void cxx_print_xnode			(FILE *, tree, int);
 extern void cxx_print_decl			(FILE *, tree, int);
 extern void cxx_print_type			(FILE *, tree, int);
 extern void cxx_print_identifier		(FILE *, tree, int);
-extern void cxx_print_error_function		(diagnostic_text_output_format &,
+extern void cxx_print_error_function		(diagnostics::text_sink &,
 						 const char *,
-						 const diagnostic_info *);
+						 const diagnostics::diagnostic_info *);
 
 /* in typeck.cc */
 /* Says how we should behave when comparing two arrays one of which
@@ -8569,7 +8664,7 @@ extern void maybe_warn_pessimizing_move	     (tree, tree, bool);
 /* in typeck2.cc */
 extern void require_complete_eh_spec_types	(tree, tree);
 extern bool cxx_incomplete_type_diagnostic	(location_t, const_tree,
-						 const_tree, diagnostic_t);
+						 const_tree, enum diagnostics::kind);
 inline location_t
 loc_or_input_loc (location_t loc)
 {
@@ -8617,7 +8712,7 @@ cp_expr_loc_or_input_loc (const_tree t)
 
 inline bool
 cxx_incomplete_type_diagnostic (const_tree value, const_tree type,
-				diagnostic_t diag_kind)
+				enum diagnostics::kind diag_kind)
 {
   return cxx_incomplete_type_diagnostic (cp_expr_loc_or_input_loc (value),
 					 value, type, diag_kind);
@@ -8628,7 +8723,7 @@ extern void cxx_incomplete_type_error		(location_t, const_tree,
 inline void
 cxx_incomplete_type_error (const_tree value, const_tree type)
 {
-  cxx_incomplete_type_diagnostic (value, type, DK_ERROR);
+  cxx_incomplete_type_diagnostic (value, type, diagnostics::kind::error);
 }
 
 extern void cxx_incomplete_type_inform 	        (const_tree);
@@ -8699,7 +8794,7 @@ extern alias_set_type cxx_get_alias_set		(tree);
 extern bool cxx_warn_unused_global_decl		(const_tree);
 extern size_t cp_tree_size			(enum tree_code);
 extern bool cp_var_mod_type_p			(tree, tree);
-extern void cxx_initialize_diagnostics		(diagnostic_context *);
+extern void cxx_initialize_diagnostics		(diagnostics::context *);
 extern int cxx_types_compatible_p		(tree, tree);
 extern bool cxx_block_may_fallthru		(const_tree);
 
@@ -8844,6 +8939,8 @@ extern bool constraints_equivalent_p            (tree, tree);
 extern bool atomic_constraints_identical_p	(tree, tree);
 extern hashval_t iterative_hash_constraint      (tree, hashval_t);
 extern hashval_t hash_atomic_constraint         (tree);
+extern void diagnose_trait_expr			(location_t, tree, tree);
+extern bool maybe_diagnose_standard_trait	(location_t, tree);
 extern void diagnose_constraints                (location_t, tree, tree);
 
 extern void note_failed_type_completion		(tree, tsubst_flags_t);
diff --git a/gcc/cp/cvt.cc b/gcc/cp/cvt.cc
index f663a6d..55be12d 100644
--- a/gcc/cp/cvt.cc
+++ b/gcc/cp/cvt.cc
@@ -1186,13 +1186,6 @@ convert_to_void (tree expr, impl_conv_void implicit, tsubst_flags_t complain)
 
   expr = maybe_undo_parenthesized_ref (expr);
 
-  expr = mark_discarded_use (expr);
-  if (implicit == ICV_CAST)
-    /* An explicit cast to void avoids all -Wunused-but-set* warnings.  */
-    mark_exp_read (expr);
-
-  if (!TREE_TYPE (expr))
-    return expr;
   if (invalid_nonstatic_memfn_p (loc, expr, complain))
     return error_mark_node;
   if (TREE_CODE (expr) == PSEUDO_DTOR_EXPR)
@@ -1209,6 +1202,12 @@ convert_to_void (tree expr, impl_conv_void implicit, tsubst_flags_t complain)
 
   if (VOID_TYPE_P (TREE_TYPE (expr)))
     return expr;
+
+  expr = mark_discarded_use (expr);
+  if (implicit == ICV_CAST)
+    /* An explicit cast to void avoids all -Wunused-but-set* warnings.  */
+    mark_exp_read (expr);
+
   switch (TREE_CODE (expr))
     {
     case COND_EXPR:
diff --git a/gcc/cp/decl.cc b/gcc/cp/decl.cc
index 83c8e28..cb3ebff 100644
--- a/gcc/cp/decl.cc
+++ b/gcc/cp/decl.cc
@@ -110,7 +110,7 @@ static void initialize_local_var (tree, tree, bool);
 static void expand_static_init (tree, tree);
 static location_t smallest_type_location (const cp_decl_specifier_seq*);
 static bool identify_goto (tree, location_t, const location_t *,
-			   diagnostic_t, bool);
+			   enum diagnostics::kind, bool);
 
 /* The following symbols are subsumed in the cp_global_trees array, and
    listed here individually for documentation purposes.
@@ -747,11 +747,11 @@ poplevel (int keep, int reverse, int functionbody)
 	      {
 		if (!DECL_NAME (decl) && DECL_DECOMPOSITION_P (decl))
 		  warning_at (DECL_SOURCE_LOCATION (decl),
-			      OPT_Wunused_but_set_variable, "structured "
+			      OPT_Wunused_but_set_variable_, "structured "
 			      "binding declaration set but not used");
 		else
 		  warning_at (DECL_SOURCE_LOCATION (decl),
-			      OPT_Wunused_but_set_variable,
+			      OPT_Wunused_but_set_variable_,
 			      "variable %qD set but not used", decl);
 		unused_but_set_errorcount = errorcount;
 	      }
@@ -1214,9 +1214,7 @@ decls_match (tree newdecl, tree olddecl, bool record_versions /* = true */)
 	  && targetm.target_option.function_versions (newdecl, olddecl))
 	{
 	  if (record_versions)
-	    maybe_version_functions (newdecl, olddecl,
-				     (!DECL_FUNCTION_VERSIONED (newdecl)
-				      || !DECL_FUNCTION_VERSIONED (olddecl)));
+	    maybe_version_functions (newdecl, olddecl);
 	  return 0;
 	}
     }
@@ -1283,11 +1281,11 @@ maybe_mark_function_versioned (tree decl)
 }
 
 /* NEWDECL and OLDDECL have identical signatures.  If they are
-   different versions adjust them and return true.
-   If RECORD is set to true, record function versions.  */
+   different versions adjust them, record function versions, and return
+   true.  */
 
 bool
-maybe_version_functions (tree newdecl, tree olddecl, bool record)
+maybe_version_functions (tree newdecl, tree olddecl)
 {
   if (!targetm.target_option.function_versions (newdecl, olddecl))
     return false;
@@ -1310,16 +1308,13 @@ maybe_version_functions (tree newdecl, tree olddecl, bool record)
       maybe_mark_function_versioned (newdecl);
     }
 
-  if (record)
-    {
-      /* Add the new version to the function version structure.  */
-      cgraph_node *fn_node = cgraph_node::get_create (olddecl);
-      cgraph_function_version_info *fn_v = fn_node->function_version ();
-      if (!fn_v)
-	fn_v = fn_node->insert_new_function_version ();
+  /* Add the new version to the function version structure.  */
+  cgraph_node *fn_node = cgraph_node::get_create (olddecl);
+  cgraph_function_version_info *fn_v = fn_node->function_version ();
+  if (!fn_v)
+    fn_v = fn_node->insert_new_function_version ();
 
-      cgraph_node::add_function_version (fn_v, newdecl);
-    }
+  cgraph_node::add_function_version (fn_v, newdecl);
 
   return true;
 }
@@ -2014,8 +2009,10 @@ duplicate_decls (tree newdecl, tree olddecl, bool hiding, bool was_hidden)
 	    }
 	  /* For function versions, params and types match, but they
 	     are not ambiguous.  */
-	  else if ((!DECL_FUNCTION_VERSIONED (newdecl)
-		    && !DECL_FUNCTION_VERSIONED (olddecl))
+	  else if (((!DECL_FUNCTION_VERSIONED (newdecl)
+		     && !DECL_FUNCTION_VERSIONED (olddecl))
+		    || !same_type_p (fndecl_declared_return_type (newdecl),
+				     fndecl_declared_return_type (olddecl)))
 		   /* Let constrained hidden friends coexist for now, we'll
 		      check satisfaction later.  */
 		   && !member_like_constrained_friend_p (newdecl)
@@ -3740,10 +3737,10 @@ decl_jump_unsafe (tree decl)
 
 static bool
 identify_goto (tree decl, location_t loc, const location_t *locus,
-	       diagnostic_t diag_kind, bool computed)
+	       enum diagnostics::kind diag_kind, bool computed)
 {
   if (computed)
-    diag_kind = DK_WARNING;
+    diag_kind = diagnostics::kind::warning;
   bool complained
     = emit_diagnostic (diag_kind, loc, 0,
 		       decl ? G_("jump to label %qD")
@@ -3779,7 +3776,8 @@ check_previous_goto_1 (tree decl, cp_binding_level* level, tree names,
 
   if (exited_omp)
     {
-      complained = identify_goto (decl, input_location, locus, DK_ERROR,
+      complained = identify_goto (decl, input_location, locus,
+				  diagnostics::kind::error,
 				  computed);
       if (complained)
 	inform (input_location, "  exits OpenMP structured block");
@@ -3801,7 +3799,8 @@ check_previous_goto_1 (tree decl, cp_binding_level* level, tree names,
 
 	  if (!identified)
 	    {
-	      complained = identify_goto (decl, input_location, locus, DK_ERROR,
+	      complained = identify_goto (decl, input_location, locus,
+					  diagnostics::kind::error,
 					  computed);
 	      identified = 2;
 	    }
@@ -3869,7 +3868,8 @@ check_previous_goto_1 (tree decl, cp_binding_level* level, tree names,
       if (inf)
 	{
 	  if (identified < 2)
-	    complained = identify_goto (decl, input_location, locus, DK_ERROR,
+	    complained = identify_goto (decl, input_location, locus,
+					diagnostics::kind::error,
 					computed);
 	  identified = 2;
 	  if (complained)
@@ -3880,7 +3880,8 @@ check_previous_goto_1 (tree decl, cp_binding_level* level, tree names,
   if (!vec_safe_is_empty (computed))
     {
       if (!identified)
-	complained = identify_goto (decl, input_location, locus, DK_ERROR,
+	complained = identify_goto (decl, input_location, locus,
+				    diagnostics::kind::error,
 				    computed);
       identified = 2;
       if (complained)
@@ -3952,14 +3953,14 @@ check_goto_1 (named_label_entry *ent, bool computed)
       || ent->in_omp_scope || ent->in_stmt_expr
       || !vec_safe_is_empty (ent->bad_decls))
     {
-      diagnostic_t diag_kind = DK_PERMERROR;
+      enum diagnostics::kind diag_kind = diagnostics::kind::permerror;
       if (ent->in_try_scope || ent->in_catch_scope || ent->in_constexpr_if
 	  || ent->in_consteval_if || ent->in_transaction_scope
 	  || ent->in_omp_scope || ent->in_stmt_expr)
-	diag_kind = DK_ERROR;
+	diag_kind = diagnostics::kind::error;
       complained = identify_goto (decl, DECL_SOURCE_LOCATION (decl),
 				  &input_location, diag_kind, computed);
-      identified = 1 + (diag_kind == DK_ERROR);
+      identified = 1 + (diag_kind == diagnostics::kind::error);
     }
 
   FOR_EACH_VEC_SAFE_ELT (ent->bad_decls, ix, bad)
@@ -3972,7 +3973,9 @@ check_goto_1 (named_label_entry *ent, bool computed)
 	  if (identified == 1)
 	    {
 	      complained = identify_goto (decl, DECL_SOURCE_LOCATION (decl),
-					  &input_location, DK_ERROR, computed);
+					  &input_location,
+					  diagnostics::kind::error,
+					  computed);
 	      identified = 2;
 	    }
 	  if (complained)
@@ -4016,7 +4019,8 @@ check_goto_1 (named_label_entry *ent, bool computed)
 	      {
 		complained = identify_goto (decl,
 					    DECL_SOURCE_LOCATION (decl),
-					    &input_location, DK_ERROR,
+					    &input_location,
+					    diagnostics::kind::error,
 					    computed);
 		identified = 2;
 	      }
@@ -4040,7 +4044,8 @@ check_goto_1 (named_label_entry *ent, bool computed)
 		{
 		  complained
 		    = identify_goto (decl, DECL_SOURCE_LOCATION (decl),
-				     &input_location, DK_ERROR, computed);
+				     &input_location, diagnostics::kind::error,
+				     computed);
 		  identified = 2;
 		}
 	      if (complained)
@@ -4056,7 +4061,8 @@ check_goto_1 (named_label_entry *ent, bool computed)
 		    {
 		      complained
 			= identify_goto (decl, DECL_SOURCE_LOCATION (decl),
-					 &input_location, DK_ERROR, computed);
+					 &input_location, diagnostics::kind::error,
+					 computed);
 		      identified = 2;
 		    }
 		  if (complained)
@@ -4368,6 +4374,7 @@ struct typename_info {
   tree template_id;
   bool enum_p;
   bool class_p;
+  bool union_p;
 };
 
 struct typename_hasher : ggc_ptr_hash<tree_node>
@@ -4406,7 +4413,8 @@ struct typename_hasher : ggc_ptr_hash<tree_node>
 	    && TYPE_CONTEXT (t1) == t2->scope
 	    && TYPENAME_TYPE_FULLNAME (t1) == t2->template_id
 	    && TYPENAME_IS_ENUM_P (t1) == t2->enum_p
-	    && TYPENAME_IS_CLASS_P (t1) == t2->class_p);
+	    && TYPENAME_IS_CLASS_P (t1) == t2->class_p
+	    && TYPENAME_IS_UNION_P (t1) == t2->union_p);
   }
 };
 
@@ -4430,9 +4438,8 @@ build_typename_type (tree context, tree name, tree fullname,
   ti.name = name;
   ti.template_id = fullname;
   ti.enum_p = tag_type == enum_type;
-  ti.class_p = (tag_type == class_type
-		|| tag_type == record_type
-		|| tag_type == union_type);
+  ti.class_p = (tag_type == class_type || tag_type == record_type);
+  ti.union_p = tag_type == union_type;
   hashval_t hash = typename_hasher::hash (&ti);
 
   /* See if we already have this type.  */
@@ -4448,6 +4455,7 @@ build_typename_type (tree context, tree name, tree fullname,
       TYPENAME_TYPE_FULLNAME (t) = ti.template_id;
       TYPENAME_IS_ENUM_P (t) = ti.enum_p;
       TYPENAME_IS_CLASS_P (t) = ti.class_p;
+      TYPENAME_IS_UNION_P (t) = ti.union_p;
 
       /* Build the corresponding TYPE_DECL.  */
       tree d = build_decl (input_location, TYPE_DECL, name, t);
@@ -5078,6 +5086,18 @@ cxx_init_decl_processing (void)
 			    BUILT_IN_FRONTEND, NULL, NULL_TREE);
   set_call_expr_flags (decl, ECF_CONST | ECF_NOTHROW | ECF_LEAF);
 
+  if (cxx_dialect >= cxx26)
+    {
+      tree void_ptrintftype
+	= build_function_type_list (void_type_node, ptr_type_node,
+				    integer_type_node, NULL_TREE);
+      decl = add_builtin_function ("__builtin_eh_ptr_adjust_ref",
+				   void_ptrintftype,
+				   CP_BUILT_IN_EH_PTR_ADJUST_REF,
+				   BUILT_IN_FRONTEND, NULL, NULL_TREE);
+      set_call_expr_flags (decl, ECF_NOTHROW | ECF_LEAF);
+    }
+
   integer_two_node = build_int_cst (NULL_TREE, 2);
 
   /* Guess at the initial static decls size.  */
@@ -9172,6 +9192,10 @@ cp_finish_decl (tree decl, tree init, bool init_const_expr_p,
 
       if (decomp)
 	{
+	  if (DECL_DECLARED_CONSTINIT_P (decl) && cxx_dialect < cxx26)
+	    pedwarn (DECL_SOURCE_LOCATION (decl), OPT_Wc__26_extensions,
+		     "%<constinit%> can be applied to structured binding "
+		     "only with %<-std=c++2c%> or %<-std=gnu++2c%>");
 	  cp_maybe_mangle_decomp (decl, decomp);
 	  if (TREE_STATIC (decl) && !DECL_FUNCTION_SCOPE_P (decl))
 	    {
@@ -11260,12 +11284,12 @@ grokfndecl (tree ctype,
 	   t && t != void_list_node; t = TREE_CHAIN (t))
 	if (TREE_PURPOSE (t))
 	  {
-	    diagnostic_t diag_kind = DK_PERMERROR;
+	    enum diagnostics::kind diag_kind = diagnostics::kind::permerror;
 	    /* For templates, mark the default argument as erroneous and give a
 	       hard error.  */
 	    if (processing_template_decl)
 	      {
-		diag_kind = DK_ERROR;
+		diag_kind = diagnostics::kind::error;
 		TREE_PURPOSE (t) = error_mark_node;
 	      }
 	    if (!has_errored)
@@ -11273,7 +11297,7 @@ grokfndecl (tree ctype,
 		has_errored = true;
 		emit_diagnostic (diag_kind,
 				 DECL_SOURCE_LOCATION (decl),
-				 /*diagnostic_option_id=*/0,
+				 /*diagnostics::option_id=*/0,
 				 "friend declaration of %qD specifies default "
 				 "arguments and isn%'t a definition", decl);
 	      }
@@ -13619,9 +13643,10 @@ grokdeclarator (const cp_declarator *declarator,
       if (typedef_p)
 	error_at (declspecs->locations[ds_typedef],
 		  "structured binding declaration cannot be %qs", "typedef");
-      if (constexpr_p && !concept_p)
-	error_at (declspecs->locations[ds_constexpr], "structured "
-		  "binding declaration cannot be %qs", "constexpr");
+      if (constexpr_p && !concept_p && cxx_dialect < cxx26)
+	pedwarn (declspecs->locations[ds_constexpr], OPT_Wc__26_extensions,
+		 "structured binding declaration can be %qs only with "
+		 "%<-std=c++2c%> or %<-std=gnu++2c%>", "constexpr");
       if (consteval_p)
 	error_at (declspecs->locations[ds_consteval], "structured "
 		  "binding declaration cannot be %qs", "consteval");
@@ -13632,8 +13657,11 @@ grokdeclarator (const cp_declarator *declarator,
 		 declspecs->gnu_thread_keyword_p
 		 ? "__thread" : "thread_local");
       if (concept_p)
-	error_at (declspecs->locations[ds_concept],
-		  "structured binding declaration cannot be %qs", "concept");
+	{
+	  error_at (declspecs->locations[ds_concept],
+		    "structured binding declaration cannot be %qs", "concept");
+	  constexpr_p = 0;
+	}
       /* [dcl.struct.bind] "A cv that includes volatile is deprecated."  */
       if (type_quals & TYPE_QUAL_VOLATILE)
 	warning_at (declspecs->locations[ds_volatile], OPT_Wvolatile,
@@ -13688,7 +13716,6 @@ grokdeclarator (const cp_declarator *declarator,
 		 "%<auto%> type %qT", type);
       inlinep = 0;
       typedef_p = 0;
-      constexpr_p = 0;
       consteval_p = 0;
       concept_p = 0;
       if (storage_class != sc_static)
@@ -17266,7 +17293,7 @@ xref_tag (enum tag_types tag_code, tree name,
       if (IDENTIFIER_LAMBDA_P (name))
 	/* Mark it as a lambda type right now.  Our caller will
 	   correct the value.  */
-	CLASSTYPE_LAMBDA_EXPR (t) = error_mark_node;
+	SET_CLASSTYPE_LAMBDA_EXPR (t, error_mark_node);
       t = pushtag (name, t, how);
     }
   else
@@ -17379,7 +17406,8 @@ xref_basetypes (tree ref, tree base_list)
 	 compatibility.  */
       if (processing_template_decl
 	  && CLASS_TYPE_P (basetype) && TYPE_BEING_DEFINED (basetype))
-	cxx_incomplete_type_diagnostic (NULL_TREE, basetype, DK_PEDWARN);
+	cxx_incomplete_type_diagnostic (NULL_TREE, basetype,
+					diagnostics::kind::pedwarn);
       if (!dependent_type_p (basetype)
 	  && !complete_type_or_else (basetype, NULL))
 	/* An incomplete type.  Remove it from the list.  */
@@ -19473,14 +19501,14 @@ finish_function (bool inline_p)
 	    && !DECL_READ_P (decl)
 	    && DECL_NAME (decl)
 	    && !DECL_ARTIFICIAL (decl)
-	    && !warning_suppressed_p (decl,OPT_Wunused_but_set_parameter)
+	    && !warning_suppressed_p (decl, OPT_Wunused_but_set_parameter_)
 	    && !DECL_IN_SYSTEM_HEADER (decl)
 	    && TREE_TYPE (decl) != error_mark_node
 	    && !TYPE_REF_P (TREE_TYPE (decl))
 	    && (!CLASS_TYPE_P (TREE_TYPE (decl))
 	        || !TYPE_HAS_NONTRIVIAL_DESTRUCTOR (TREE_TYPE (decl))))
 	  warning_at (DECL_SOURCE_LOCATION (decl),
-		      OPT_Wunused_but_set_parameter,
+		      OPT_Wunused_but_set_parameter_,
 		      "parameter %qD set but not used", decl);
       unused_but_set_errorcount = errorcount;
     }
diff --git a/gcc/cp/error.cc b/gcc/cp/error.cc
index abeb028..c427163 100644
--- a/gcc/cp/error.cc
+++ b/gcc/cp/error.cc
@@ -24,7 +24,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "cp-tree.h"
 #include "stringpool.h"
 #include "tree-diagnostic.h"
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
 #include "langhooks-def.h"
 #include "intl.h"
 #include "cxx-pretty-print.h"
@@ -38,7 +38,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "cp-name-hint.h"
 #include "attribs.h"
 #include "pretty-print-format-impl.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 
 #define pp_separate_with_comma(PP) pp_cxx_separate_with (PP, ',')
 #define pp_separate_with_semicolon(PP) pp_cxx_separate_with (PP, ';')
@@ -96,17 +96,17 @@ static void dump_scope (cxx_pretty_printer *, tree, int);
 static void dump_template_parms (cxx_pretty_printer *, tree, int, int);
 static int get_non_default_template_args_count (tree, int);
 static const char *function_category (tree);
-static void maybe_print_constexpr_context (diagnostic_text_output_format &);
-static void maybe_print_instantiation_context (diagnostic_text_output_format &);
-static void print_instantiation_full_context (diagnostic_text_output_format &);
-static void print_instantiation_partial_context (diagnostic_text_output_format &,
+static void maybe_print_constexpr_context (diagnostics::text_sink &);
+static void maybe_print_instantiation_context (diagnostics::text_sink &);
+static void print_instantiation_full_context (diagnostics::text_sink &);
+static void print_instantiation_partial_context (diagnostics::text_sink &,
 						 struct tinst_level *,
 						 location_t);
-static void maybe_print_constraint_context (diagnostic_text_output_format &);
-static void cp_diagnostic_text_starter (diagnostic_text_output_format &,
-					const diagnostic_info *);
-static void cp_print_error_function (diagnostic_text_output_format &,
-				     const diagnostic_info *);
+static void maybe_print_constraint_context (diagnostics::text_sink &);
+static void cp_diagnostic_text_starter (diagnostics::text_sink &,
+					const diagnostics::diagnostic_info *);
+static void cp_print_error_function (diagnostics::text_sink &,
+				     const diagnostics::diagnostic_info *);
 
 static bool cp_printer (pretty_printer *, text_info *, const char *,
 			int, bool, bool, bool, bool *, pp_token_list &);
@@ -242,7 +242,7 @@ get_current_template ()
 
 erroneous_templates_t *erroneous_templates;
 
-/* Callback function diagnostic_context::m_adjust_diagnostic_info.
+/* Callback function diagnostics::context::m_adjust_diagnostic_info.
 
    Errors issued when parsing a template are automatically treated like
    permerrors associated with the -Wtemplate-body flag and can be
@@ -250,16 +250,16 @@ erroneous_templates_t *erroneous_templates;
    issue an error if we later need to instantiate the template.  */
 
 static void
-cp_adjust_diagnostic_info (diagnostic_context *context,
-			   diagnostic_info *diagnostic)
+cp_adjust_diagnostic_info (diagnostics::context *context,
+			   diagnostics::diagnostic_info *diagnostic)
 {
-  if (diagnostic->kind == DK_ERROR)
+  if (diagnostic->m_kind == diagnostics::kind::error)
     if (tree tmpl = get_current_template ())
       {
-	diagnostic->option_id = OPT_Wtemplate_body;
+	diagnostic->m_option_id = OPT_Wtemplate_body;
 
 	if (context->m_permissive)
-	  diagnostic->kind = DK_WARNING;
+	  diagnostic->m_kind = diagnostics::kind::warning;
 
 	bool existed;
 	location_t &error_loc
@@ -269,7 +269,7 @@ cp_adjust_diagnostic_info (diagnostic_context *context,
 	  /* Remember that this template had a parse-time error so
 	     that we'll ensure a hard error has been issued upon
 	     its instantiation.  */
-	  error_loc = diagnostic->richloc->get_loc ();
+	  error_loc = diagnostic->m_richloc->get_loc ();
       }
 }
 
@@ -298,14 +298,14 @@ cp_seen_error ()
    capacities.  */
 
 void
-cxx_initialize_diagnostics (diagnostic_context *context)
+cxx_initialize_diagnostics (diagnostics::context *context)
 {
   cxx_pretty_printer *pp = new cxx_pretty_printer ();
   pp->set_format_postprocessor (std::make_unique<cxx_format_postprocessor> ());
   context->set_pretty_printer (std::unique_ptr<pretty_printer> (pp));
 
   c_common_diagnostics_set_defaults (context);
-  diagnostic_text_starter (context) = cp_diagnostic_text_starter;
+  diagnostics::text_starter (context) = cp_diagnostic_text_starter;
   /* diagnostic_finalizer is already c_diagnostic_text_finalizer.  */
   context->set_format_decoder (cp_printer);
   context->set_adjust_diagnostic_info_callback (cp_adjust_diagnostic_info);
@@ -704,6 +704,20 @@ dump_type (cxx_pretty_printer *pp, tree t, int flags)
 	}
       break;
 
+    case TREE_VEC:
+      {
+	/* A list of types used for a trait.  */
+	bool need_comma = false;
+	for (tree arg : tree_vec_range (t))
+	  {
+	    if (need_comma)
+	      pp_separate_with_comma (pp);
+	    dump_type (pp, arg, flags);
+	    need_comma = true;
+	  }
+      }
+      break;
+
     case TREE_LIST:
       /* A list of function parms.  */
       dump_parameters (pp, t, flags);
@@ -810,6 +824,7 @@ dump_type (cxx_pretty_printer *pp, tree t, int flags)
       pp_cxx_ws_string (pp,
 			 TYPENAME_IS_ENUM_P (t) ? "enum"
 			 : TYPENAME_IS_CLASS_P (t) ? "class"
+			 : TYPENAME_IS_UNION_P (t) ? "union"
 			 : "typename");
       dump_typename (pp, t, flags);
       break;
@@ -3746,9 +3761,9 @@ eh_spec_to_string (tree p, int /*v*/)
 
 /* Langhook for print_error_function.  */
 void
-cxx_print_error_function (diagnostic_text_output_format &text_output,
+cxx_print_error_function (diagnostics::text_sink &text_output,
 			  const char *file,
-			  const diagnostic_info *diagnostic)
+			  const diagnostics::diagnostic_info *diagnostic)
 {
   char *prefix;
   if (file)
@@ -3762,8 +3777,8 @@ cxx_print_error_function (diagnostic_text_output_format &text_output,
 }
 
 static void
-cp_diagnostic_text_starter (diagnostic_text_output_format &text_output,
-			    const diagnostic_info *diagnostic)
+cp_diagnostic_text_starter (diagnostics::text_sink &text_output,
+			    const diagnostics::diagnostic_info *diagnostic)
 {
   pp_set_prefix (text_output.get_printer (),
 		 text_output.build_indent_prefix (true));
@@ -3779,8 +3794,8 @@ cp_diagnostic_text_starter (diagnostic_text_output_format &text_output,
 /* Print current function onto BUFFER, in the process of reporting
    a diagnostic message.  Called from cp_diagnostic_starter.  */
 static void
-cp_print_error_function (diagnostic_text_output_format &text_output,
-			 const diagnostic_info *diagnostic)
+cp_print_error_function (diagnostics::text_sink &text_output,
+			 const diagnostics::diagnostic_info *diagnostic)
 {
   /* If we are in an instantiation context, current_function_decl is likely
      to be wrong, so just rely on print_instantiation_full_context.  */
@@ -3789,7 +3804,7 @@ cp_print_error_function (diagnostic_text_output_format &text_output,
   /* The above is true for constraint satisfaction also.  */
   if (current_failed_constraint)
     return;
-  diagnostic_context *const context = &text_output.get_context ();
+  diagnostics::context *const context = &text_output.get_context ();
   if (diagnostic_last_function_changed (context, diagnostic))
     {
       pretty_printer *const pp = text_output.get_printer ();
@@ -3927,7 +3942,7 @@ function_category (tree fn)
 /* Report the full context of a current template instantiation,
    onto BUFFER.  */
 static void
-print_instantiation_full_context (diagnostic_text_output_format &text_output)
+print_instantiation_full_context (diagnostics::text_sink &text_output)
 {
   struct tinst_level *p = current_instantiation ();
   location_t location = input_location;
@@ -3955,7 +3970,7 @@ print_instantiation_full_context (diagnostic_text_output_format &text_output)
 }
 
 static void
-print_location (diagnostic_text_output_format &text_output,
+print_location (diagnostics::text_sink &text_output,
 		location_t loc)
 {
   expanded_location xloc = expand_location (loc);
@@ -3969,7 +3984,7 @@ print_location (diagnostic_text_output_format &text_output,
 }
 
 /* A RAII class for use when emitting a line of contextual information
-   via pp_verbatim to a diagnostic_text_output_format to add before/after
+   via pp_verbatim to a diagnostics::text_sink to add before/after
    behaviors to the pp_verbatim calls.
 
    If the text output has show_nesting_p (), then the ctor prints
@@ -3984,7 +3999,7 @@ print_location (diagnostic_text_output_format &text_output,
 class auto_context_line
 {
 public:
-  auto_context_line (diagnostic_text_output_format &text_output,
+  auto_context_line (diagnostics::text_sink &text_output,
 		     location_t loc,
 		     bool show_locus = false)
   : m_text_output (text_output),
@@ -4015,7 +4030,7 @@ public:
 	    diagnostic_show_locus (&m_text_output.get_context (),
 				   m_text_output.get_source_printing_options (),
 				   &rich_loc,
-				   DK_NOTE, pp);
+				   diagnostics::kind::note, pp);
 	    pp_set_prefix (pp, saved_prefix);
 	  }
       }
@@ -4027,12 +4042,12 @@ public:
 	diagnostic_show_locus (&m_text_output.get_context (),
 			       m_text_output.get_source_printing_options (),
 			       &rich_loc,
-			       DK_NOTE, pp);
+			       diagnostics::kind::note, pp);
 	pp_set_prefix (pp, saved_prefix);
       }
   }
 private:
-  diagnostic_text_output_format &m_text_output;
+  diagnostics::text_sink &m_text_output;
   location_t m_loc;
   bool m_show_locus;
 };
@@ -4041,7 +4056,7 @@ private:
    prints a single line of instantiation context.  */
 
 static void
-print_instantiation_partial_context_line (diagnostic_text_output_format &text_output,
+print_instantiation_partial_context_line (diagnostics::text_sink &text_output,
 					  struct tinst_level *t,
 					  location_t loc, bool recursive_p)
 {
@@ -4079,7 +4094,7 @@ print_instantiation_partial_context_line (diagnostic_text_output_format &text_ou
 /* Same as print_instantiation_full_context but less verbose.  */
 
 static void
-print_instantiation_partial_context (diagnostic_text_output_format &text_output,
+print_instantiation_partial_context (diagnostics::text_sink &text_output,
 				     struct tinst_level *t0, location_t loc)
 {
   struct tinst_level *t;
@@ -4150,7 +4165,7 @@ print_instantiation_partial_context (diagnostic_text_output_format &text_output,
 
 /* Called from cp_thing to print the template context for an error.  */
 static void
-maybe_print_instantiation_context (diagnostic_text_output_format &text_output)
+maybe_print_instantiation_context (diagnostics::text_sink &text_output)
 {
   if (!problematic_instantiation_changed () || current_instantiation () == 0)
     return;
@@ -4162,7 +4177,7 @@ maybe_print_instantiation_context (diagnostic_text_output_format &text_output)
 /* Report what constexpr call(s) we're trying to expand, if any.  */
 
 void
-maybe_print_constexpr_context (diagnostic_text_output_format &text_output)
+maybe_print_constexpr_context (diagnostics::text_sink &text_output)
 {
   vec<tree> call_stack = cx_error_context ();
   unsigned ix;
@@ -4182,7 +4197,7 @@ maybe_print_constexpr_context (diagnostic_text_output_format &text_output)
 
 
 static void
-print_constrained_decl_info (diagnostic_text_output_format &text_output,
+print_constrained_decl_info (diagnostics::text_sink &text_output,
 			     tree decl)
 {
   auto_context_line sentinel (text_output, DECL_SOURCE_LOCATION (decl));
@@ -4191,7 +4206,7 @@ print_constrained_decl_info (diagnostic_text_output_format &text_output,
 }
 
 static void
-print_concept_check_info (diagnostic_text_output_format &text_output,
+print_concept_check_info (diagnostics::text_sink &text_output,
 			  tree expr, tree map, tree args)
 {
   gcc_assert (concept_check_p (expr));
@@ -4216,7 +4231,7 @@ print_concept_check_info (diagnostic_text_output_format &text_output,
    context, if any.  */
 
 static tree
-print_constraint_context_head (diagnostic_text_output_format &text_output,
+print_constraint_context_head (diagnostics::text_sink &text_output,
 			       tree cxt, tree args)
 {
   tree src = TREE_VALUE (cxt);
@@ -4240,7 +4255,7 @@ print_constraint_context_head (diagnostic_text_output_format &text_output,
 }
 
 static void
-print_requires_expression_info (diagnostic_text_output_format &text_output,
+print_requires_expression_info (diagnostics::text_sink &text_output,
 				tree constr, tree args)
 {
 
@@ -4270,7 +4285,7 @@ print_requires_expression_info (diagnostic_text_output_format &text_output,
 }
 
 void
-maybe_print_single_constraint_context (diagnostic_text_output_format &text_output,
+maybe_print_single_constraint_context (diagnostics::text_sink &text_output,
 				       tree failed)
 {
   if (!failed)
@@ -4301,7 +4316,7 @@ maybe_print_single_constraint_context (diagnostic_text_output_format &text_outpu
 }
 
 void
-maybe_print_constraint_context (diagnostic_text_output_format &text_output)
+maybe_print_constraint_context (diagnostics::text_sink &text_output)
 {
   if (!current_failed_constraint)
     return;
@@ -4914,18 +4929,20 @@ maybe_warn_variadic_templates (void)
    C++0x.  */
 bool
 pedwarn_cxx98 (location_t location,
-	       diagnostic_option_id option_id,
+	       diagnostics::option_id option_id,
 	       const char *gmsgid, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   bool ret;
   rich_location richloc (line_table, location);
 
   va_start (ap, gmsgid);
   diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc,
-		       (cxx_dialect == cxx98) ? DK_PEDWARN : DK_WARNING);
-  diagnostic.option_id = option_id;
+		       (cxx_dialect == cxx98
+			? diagnostics::kind::pedwarn
+			: diagnostics::kind::warning));
+  diagnostic.m_option_id = option_id;
   ret = diagnostic_report_diagnostic (global_dc, &diagnostic);
   va_end (ap);
   return ret;
diff --git a/gcc/cp/except.cc b/gcc/cp/except.cc
index a9d8e2f..204769f 100644
--- a/gcc/cp/except.cc
+++ b/gcc/cp/except.cc
@@ -367,6 +367,8 @@ initialize_handler_parm (tree decl, tree exp)
 	 MUST_NOT_THROW_EXPR.  */
       init = fold_build_cleanup_point_expr (TREE_TYPE (init), init);
       init = build_must_not_throw_expr (init, NULL_TREE);
+      if (init && TREE_CODE (init) == MUST_NOT_THROW_EXPR)
+	MUST_NOT_THROW_CATCH_P (init) = 1;
     }
 
   decl = pushdecl (decl);
@@ -523,6 +525,7 @@ begin_eh_spec_block (void)
       r = build_stmt (spec_location, MUST_NOT_THROW_EXPR,
 		      NULL_TREE, NULL_TREE);
       TREE_SIDE_EFFECTS (r) = 1;
+      MUST_NOT_THROW_NOEXCEPT_P (r) = 1;
     }
   else
     r = build_stmt (spec_location, EH_SPEC_BLOCK, NULL_TREE, NULL_TREE);
@@ -614,6 +617,7 @@ wrap_cleanups_r (tree *tp, int *walk_subtrees, void * /*data*/)
     {
       cleanup = build2 (MUST_NOT_THROW_EXPR, void_type_node, cleanup,
 			NULL_TREE);
+      MUST_NOT_THROW_THROW_P (cleanup) = 1;
       TARGET_EXPR_CLEANUP (exp) = cleanup;
     }
 
@@ -712,6 +716,11 @@ build_throw (location_t loc, tree exp, tsubst_flags_t complain)
       allocate_expr = do_allocate_exception (temp_type);
       if (allocate_expr == error_mark_node)
 	return error_mark_node;
+      /* Copy ptr inside of the CLEANUP_POINT_EXPR
+	 added below to a TARGET_EXPR slot added outside of it,
+	 otherwise during constant evaluation of throw expression
+	 we'd diagnose accessing ptr outside of its lifetime.  */
+      tree ptr_copy = get_internal_target_expr (null_pointer_node);
       allocate_expr = get_internal_target_expr (allocate_expr);
       ptr = TARGET_EXPR_SLOT (allocate_expr);
       TARGET_EXPR_CLEANUP (allocate_expr) = do_free_exception (ptr);
@@ -763,10 +772,17 @@ build_throw (location_t loc, tree exp, tsubst_flags_t complain)
       /* Prepend the allocation.  */
       exp = build2 (COMPOUND_EXPR, TREE_TYPE (exp), allocate_expr, exp);
 
+      exp = build2 (COMPOUND_EXPR, void_type_node, exp,
+		    build2 (MODIFY_EXPR, void_type_node,
+			    TARGET_EXPR_SLOT (ptr_copy), ptr));
+      ptr = TARGET_EXPR_SLOT (ptr_copy);
+
       /* Force all the cleanups to be evaluated here so that we don't have
 	 to do them during unwinding.  */
       exp = build1 (CLEANUP_POINT_EXPR, void_type_node, exp);
 
+      exp = build2 (COMPOUND_EXPR, TREE_TYPE (exp), ptr_copy, exp);
+
       throw_type = build_eh_type_type (prepare_eh_type (TREE_TYPE (object)));
 
       cleanup = NULL_TREE;
@@ -1202,6 +1218,20 @@ expr_noexcept_p (tree expr, tsubst_flags_t complain)
     return true;
 }
 
+/* If EXPR is not noexcept, explain why.  */
+
+void
+explain_not_noexcept (tree expr)
+{
+  tree fn = cp_walk_tree_without_duplicates (&expr, check_noexcept_r, 0);
+  if (!fn)
+    /* The call was noexcept, nothing to do.  */;
+  else if (DECL_P (fn))
+    inform (DECL_SOURCE_LOCATION (fn), "%qD is not %<noexcept%>", fn);
+  else
+    inform (location_of (fn), "%qT is not %<noexcept%>", TREE_TYPE (fn));
+}
+
 /* Return true iff SPEC is throw() or noexcept(true).  */
 
 bool
diff --git a/gcc/cp/expr.cc b/gcc/cp/expr.cc
index 2157cfb..32dc3ee 100644
--- a/gcc/cp/expr.cc
+++ b/gcc/cp/expr.cc
@@ -102,6 +102,9 @@ mark_use (tree expr, bool rvalue_p, bool read_p,
   if (reject_builtin && reject_gcc_builtin (expr, loc))
     return error_mark_node;
 
+  if (TREE_TYPE (expr) && VOID_TYPE_P (TREE_TYPE (expr)))
+    read_p = false;
+
   if (read_p)
     mark_exp_read (expr);
 
@@ -211,7 +214,7 @@ mark_use (tree expr, bool rvalue_p, bool read_p,
 	    }
 	  return expr;
 	}
-      gcc_fallthrough();
+      gcc_fallthrough ();
     CASE_CONVERT:
       recurse_op[0] = true;
       break;
@@ -352,6 +355,9 @@ mark_exp_read (tree exp)
   if (exp == NULL)
     return;
 
+  if (TREE_TYPE (exp) && VOID_TYPE_P (TREE_TYPE (exp)))
+    return;
+
   switch (TREE_CODE (exp))
     {
     case VAR_DECL:
@@ -361,16 +367,20 @@ mark_exp_read (tree exp)
     case PARM_DECL:
       DECL_READ_P (exp) = 1;
       break;
+    CASE_CONVERT:
     case ARRAY_REF:
     case COMPONENT_REF:
     case MODIFY_EXPR:
     case REALPART_EXPR:
     case IMAGPART_EXPR:
-    CASE_CONVERT:
     case ADDR_EXPR:
     case INDIRECT_REF:
     case FLOAT_EXPR:
     case VIEW_CONVERT_EXPR:
+    case PREINCREMENT_EXPR:
+    case PREDECREMENT_EXPR:
+    case POSTINCREMENT_EXPR:
+    case POSTDECREMENT_EXPR:
       mark_exp_read (TREE_OPERAND (exp, 0));
       break;
     case COMPOUND_EXPR:
diff --git a/gcc/cp/init.cc b/gcc/cp/init.cc
index 0a389fb..09fb4f3 100644
--- a/gcc/cp/init.cc
+++ b/gcc/cp/init.cc
@@ -4181,7 +4181,8 @@ build_vec_delete_1 (location_t loc, tree base, tree maxindex, tree type,
 			  "possible problem detected in invocation of "
 			  "operator %<delete []%>"))
 	    {
-	      cxx_incomplete_type_diagnostic (base, type, DK_WARNING);
+	      cxx_incomplete_type_diagnostic (base, type,
+					      diagnostics::kind::warning);
 	      inform (loc, "neither the destructor nor the "
 		      "class-specific operator %<delete []%> will be called, "
 		      "even if they are declared when the class is defined");
@@ -5286,7 +5287,8 @@ build_delete (location_t loc, tree otype, tree addr,
 				  "possible problem detected in invocation of "
 				  "%<operator delete%>"))
 		    {
-		      cxx_incomplete_type_diagnostic (addr, type, DK_WARNING);
+		      cxx_incomplete_type_diagnostic (addr, type,
+						      diagnostics::kind::warning);
 		      inform (loc,
 			      "neither the destructor nor the class-specific "
 			      "%<operator delete%> will be called, even if "
diff --git a/gcc/cp/lambda.cc b/gcc/cp/lambda.cc
index 2a9061a..c798967 100644
--- a/gcc/cp/lambda.cc
+++ b/gcc/cp/lambda.cc
@@ -150,7 +150,7 @@ begin_lambda_type (tree lambda)
 
   /* Cross-reference the expression and the type.  */
   LAMBDA_EXPR_CLOSURE (lambda) = type;
-  CLASSTYPE_LAMBDA_EXPR (type) = lambda;
+  SET_CLASSTYPE_LAMBDA_EXPR (type, lambda);
 
   /* In C++17, assume the closure is literal; we'll clear the flag later if
      necessary.  */
@@ -823,6 +823,14 @@ lambda_expr_this_capture (tree lambda, int add_capture_p)
   if (cp_unevaluated_operand)
     add_capture_p = false;
 
+  /* If we captured 'this' but don't have a capture proxy yet, look up the
+     captured 'this' again.  */
+  if (this_capture && TREE_CODE (this_capture) == FIELD_DECL)
+    {
+      gcc_assert (!add_capture_p);
+      this_capture = NULL_TREE;
+    }
+
   /* Try to default capture 'this' if we can.  */
   if (!this_capture)
     {
@@ -940,6 +948,9 @@ lambda_expr_this_capture (tree lambda, int add_capture_p)
       result = rvalue (result);
     }
 
+  gcc_checking_assert (!result || result == error_mark_node
+		       || TYPE_PTR_P (TREE_TYPE (result)));
+
   return result;
 }
 
@@ -1027,13 +1038,19 @@ maybe_generic_this_capture (tree object, tree fns)
       }
 }
 
-/* Returns the innermost non-lambda function.  */
+/* Returns the innermost non-lambda function.  If ONLY_SKIP_CONSTEVAL_BLOCK_P,
+   we only skip lambda functions that represent consteval blocks.  */
 
 tree
-current_nonlambda_function (void)
+current_nonlambda_function (bool only_skip_consteval_block_p/*=false*/)
 {
   tree fn = current_function_decl;
-  while (fn && LAMBDA_FUNCTION_P (fn))
+  tree lam;
+  while (fn && LAMBDA_FUNCTION_P (fn)
+	 && (!only_skip_consteval_block_p
+	     /* Only keep going if FN represents a consteval block.  */
+	     || ((lam = CLASSTYPE_LAMBDA_EXPR (CP_DECL_CONTEXT (fn)))
+		 && LAMBDA_EXPR_CONSTEVAL_BLOCK_P (lam))))
     fn = decl_function_context (fn);
   return fn;
 }
@@ -1132,7 +1149,9 @@ maybe_add_lambda_conv_op (tree type)
   tree lam = CLASSTYPE_LAMBDA_EXPR (type);
 
   if (LAMBDA_EXPR_CAPTURE_LIST (lam) != NULL_TREE
-      || LAMBDA_EXPR_DEFAULT_CAPTURE_MODE (lam) != CPLD_NONE)
+      || LAMBDA_EXPR_DEFAULT_CAPTURE_MODE (lam) != CPLD_NONE
+      /* CWG2561 ...and no explicit object parameter.  */
+      || DECL_XOBJ_MEMBER_FUNCTION_P (callop))
     return;
 
   if (processing_template_decl)
@@ -1939,7 +1958,7 @@ finish_lambda_function (tree body)
 {
   finish_function_body (body);
 
-  prune_lambda_captures (body);
+  prune_lambda_captures (cur_stmt_list);
 
   /* Finish the function and generate code for it if necessary.  */
   tree fn = finish_function (/*inline_p=*/true);
diff --git a/gcc/cp/method.cc b/gcc/cp/method.cc
index a4089c5..397e496 100644
--- a/gcc/cp/method.cc
+++ b/gcc/cp/method.cc
@@ -1187,15 +1187,15 @@ early_check_defaulted_comparison (tree fn)
   if (!DECL_OVERLOADED_OPERATOR_IS (fn, SPACESHIP_EXPR)
       && !same_type_p (TREE_TYPE (TREE_TYPE (fn)), boolean_type_node))
     {
-      diagnostic_t kind = DK_UNSPECIFIED;
+      enum diagnostics::kind kind = diagnostics::kind::unspecified;
       int opt = 0;
       if (is_auto (TREE_TYPE (fn)))
-	kind = DK_PEDWARN;
+	kind = diagnostics::kind::pedwarn;
       else
-	kind = DK_ERROR;
+	kind = diagnostics::kind::error;
       emit_diagnostic (kind, loc, opt,
 		       "defaulted %qD must return %<bool%>", fn);
-      if (kind == DK_ERROR)
+      if (kind == diagnostics::kind::error)
 	ok = false;
     }
 
@@ -1928,8 +1928,8 @@ is_stub_object (tree expr)
 
 /* Build a std::declval<TYPE>() expression and return it.  */
 
-tree
-build_trait_object (tree type)
+static tree
+build_trait_object (tree type, tsubst_flags_t complain)
 {
   /* TYPE can't be a function with cv-/ref-qualifiers: std::declval is
      defined as
@@ -1942,13 +1942,18 @@ build_trait_object (tree type)
   if (FUNC_OR_METHOD_TYPE_P (type)
       && (type_memfn_quals (type) != TYPE_UNQUALIFIED
 	  || type_memfn_rqual (type) != REF_QUAL_NONE))
-    return error_mark_node;
+    {
+      if (complain & tf_error)
+	error ("object cannot have qualified function type %qT", type);
+      return error_mark_node;
+    }
 
   return build_stub_object (type);
 }
 
 /* [func.require] Build an expression of INVOKE(FN_TYPE, ARG_TYPES...).  If the
-   given is not invocable, returns error_mark_node.  */
+   given is not invocable, returns error_mark_node, unless COMPLAIN includes
+   tf_error.  */
 
 tree
 build_invoke (tree fn_type, const_tree arg_types, tsubst_flags_t complain)
@@ -2036,22 +2041,26 @@ build_invoke (tree fn_type, const_tree arg_types, tsubst_flags_t complain)
 	      const_tree name = DECL_NAME (datum_decl);
 	      if (name && (id_equal (name, "reference_wrapper")))
 		{
-		  /* 1.2 & 1.5: Retrieve T from std::reference_wrapper<T>,
+		  /* 1.2 & 1.5: Retrieve T& from std::reference_wrapper<T>,
 		     i.e., decltype(datum.get()).  */
 		  datum_type =
 		    TREE_VEC_ELT (TYPE_TI_ARGS (non_ref_datum_type), 0);
+		  datum_type = cp_build_reference_type (datum_type, false);
 		  datum_is_refwrap = true;
 		}
 	    }
 	}
 
-      tree datum_expr = build_trait_object (datum_type);
+      tree datum_expr = build_trait_object (datum_type, complain);
       if (!ptrmem_is_same_or_base_of_datum && !datum_is_refwrap)
 	/* 1.3 & 1.6: Try to dereference datum_expr.  */
 	datum_expr = build_x_indirect_ref (UNKNOWN_LOCATION, datum_expr,
 					   RO_UNARY_STAR, NULL_TREE, complain);
 
-      tree fn_expr = build_trait_object (fn_type);
+      if (error_operand_p (datum_expr))
+	return error_mark_node;
+
+      tree fn_expr = build_trait_object (fn_type, complain);
       ptrmem_expr = build_m_component_ref (datum_expr, fn_expr, complain);
 
       if (error_operand_p (ptrmem_expr))
@@ -2068,7 +2077,9 @@ build_invoke (tree fn_type, const_tree arg_types, tsubst_flags_t complain)
   for (int i = is_ptrmemfunc ? 1 : 0; i < TREE_VEC_LENGTH (arg_types); ++i)
     {
       tree arg_type = TREE_VEC_ELT (arg_types, i);
-      tree arg = build_trait_object (arg_type);
+      tree arg = build_trait_object (arg_type, complain);
+      if (error_operand_p (arg))
+	return error_mark_node;
       vec_safe_push (args, arg);
     }
 
@@ -2077,8 +2088,8 @@ build_invoke (tree fn_type, const_tree arg_types, tsubst_flags_t complain)
     invoke_expr = build_offset_ref_call_from_tree (ptrmem_expr, &args,
 						   complain);
   else  /* 1.7.  */
-    invoke_expr = finish_call_expr (build_trait_object (fn_type), &args, false,
-				    false, complain);
+    invoke_expr = finish_call_expr (build_trait_object (fn_type, complain),
+				    &args, false, false, complain);
   return invoke_expr;
 }
 
@@ -2227,12 +2238,20 @@ check_nontriv (tree *tp, int *, void *)
 /* Return declval<T>() = declval<U>() treated as an unevaluated operand.  */
 
 static tree
-assignable_expr (tree to, tree from)
+assignable_expr (tree to, tree from, bool explain)
 {
   cp_unevaluated cp_uneval_guard;
-  to = build_trait_object (to);
-  from = build_trait_object (from);
-  tree r = cp_build_modify_expr (input_location, to, NOP_EXPR, from, tf_none);
+  tsubst_flags_t complain = explain ? tf_error : tf_none;
+
+  to = build_trait_object (to, complain);
+  if (to == error_mark_node)
+    return error_mark_node;
+
+  from = build_trait_object (from, complain);
+  if (from == error_mark_node)
+    return error_mark_node;
+
+  tree r = cp_build_modify_expr (input_location, to, NOP_EXPR, from, complain);
   return r;
 }
 
@@ -2244,10 +2263,11 @@ assignable_expr (tree to, tree from)
    Return something equivalent in well-formedness and triviality.  */
 
 static tree
-constructible_expr (tree to, tree from)
+constructible_expr (tree to, tree from, bool explain)
 {
   tree expr;
   cp_unevaluated cp_uneval_guard;
+  tsubst_flags_t complain = explain ? tf_error : tf_none;
   const int len = TREE_VEC_LENGTH (from);
   if (CLASS_TYPE_P (to))
     {
@@ -2259,14 +2279,14 @@ constructible_expr (tree to, tree from)
 	to = cp_build_reference_type (to, /*rval*/false);
       tree ob = build_stub_object (to);
       if (len == 0)
-	expr = build_value_init (ctype, tf_none);
+	expr = build_value_init (ctype, complain);
       else
 	{
 	  vec_alloc (args, len);
 	  for (tree arg : tree_vec_range (from))
 	    args->quick_push (build_stub_object (arg));
 	  expr = build_special_member_call (ob, complete_ctor_identifier, &args,
-					    ctype, LOOKUP_NORMAL, tf_none);
+					    ctype, LOOKUP_NORMAL, complain);
 	}
       if (expr == error_mark_node)
 	return error_mark_node;
@@ -2276,7 +2296,7 @@ constructible_expr (tree to, tree from)
 	{
 	  tree dtor = build_special_member_call (ob, complete_dtor_identifier,
 						 NULL, ctype, LOOKUP_NORMAL,
-						 tf_none);
+						 complain);
 	  if (dtor == error_mark_node)
 	    return error_mark_node;
 	  if (!TYPE_HAS_TRIVIAL_DESTRUCTOR (ctype))
@@ -2286,12 +2306,15 @@ constructible_expr (tree to, tree from)
   else
     {
       if (len == 0)
-	return build_value_init (strip_array_types (to), tf_none);
+	return build_value_init (strip_array_types (to), complain);
       if (len > 1)
 	{
 	  if (cxx_dialect < cxx20)
-	    /* Too many initializers.  */
-	    return error_mark_node;
+	    {
+	      if (explain)
+		error ("too many initializers for non-class type %qT", to);
+	      return error_mark_node;
+	    }
 
 	  /* In C++20 this is well-formed:
 	       using T = int[2];
@@ -2312,9 +2335,11 @@ constructible_expr (tree to, tree from)
 	}
       else
 	from = build_stub_object (TREE_VEC_ELT (from, 0));
+
+      tree orig_from = from;
       expr = perform_direct_initialization_if_possible (to, from,
 							/*cast*/false,
-							tf_none);
+							complain);
       /* If t(e) didn't work, maybe t{e} will.  */
       if (expr == NULL_TREE
 	  && len == 1
@@ -2326,7 +2351,24 @@ constructible_expr (tree to, tree from)
 	  CONSTRUCTOR_IS_PAREN_INIT (from) = true;
 	  expr = perform_direct_initialization_if_possible (to, from,
 							    /*cast*/false,
-							    tf_none);
+							    complain);
+	}
+
+      if (expr == NULL_TREE && explain)
+	{
+	  if (len > 1)
+	    error ("too many initializers for non-class type %qT", to);
+	  else
+	    {
+	      /* Redo the implicit conversion for diagnostics.  */
+	      int count = errorcount + warningcount;
+	      perform_implicit_conversion_flags (to, orig_from, complain,
+						 LOOKUP_NORMAL);
+	      if (count == errorcount + warningcount)
+		/* The message may have been suppressed due to -w + -fpermissive,
+		   emit a generic response instead.  */
+		error ("the conversion is invalid");
+	    }
 	}
     }
   return expr;
@@ -2340,20 +2382,25 @@ constructible_expr (tree to, tree from)
    valid or error_mark_node if not.  */
 
 static tree
-destructible_expr (tree to)
+destructible_expr (tree to, bool explain)
 {
   cp_unevaluated cp_uneval_guard;
+  tsubst_flags_t complain = explain ? tf_error : tf_none;
   int flags = LOOKUP_NORMAL|LOOKUP_DESTRUCTOR;
   if (TYPE_REF_P (to))
     return void_node;
   if (!COMPLETE_TYPE_P (complete_type (to)))
-    return error_mark_node;
+    {
+      if (explain)
+	error_at (location_of (to), "%qT is incomplete", to);
+      return error_mark_node;
+    }
   to = strip_array_types (to);
   if (CLASS_TYPE_P (to))
     {
-      to = build_trait_object (to);
+      to = build_trait_object (to, complain);
       return build_delete (input_location, TREE_TYPE (to), to,
-			     sfk_complete_destructor, flags, 0, tf_none);
+			   sfk_complete_destructor, flags, 0, complain);
     }
   /* [expr.prim.id.dtor] If the id-expression names a pseudo-destructor, T
      shall be a scalar type.... */
@@ -2365,70 +2412,95 @@ destructible_expr (tree to)
 
 /* Returns a tree iff TO is assignable (if CODE is MODIFY_EXPR) or
    constructible (otherwise) from FROM, which is a single type for
-   assignment or a list of types for construction.  */
+   assignment or a list of types for construction.  If EXPLAIN is
+   set, emit a diagnostic explaining why the operation failed.  */
 
 static tree
-is_xible_helper (enum tree_code code, tree to, tree from, bool trivial)
+is_xible_helper (enum tree_code code, tree to, tree from, bool explain)
 {
   to = complete_type (to);
   deferring_access_check_sentinel acs (dk_no_deferred);
-  if (VOID_TYPE_P (to)
-      || (from && FUNC_OR_METHOD_TYPE_P (from)
-	  && (TYPE_READONLY (from) || FUNCTION_REF_QUALIFIED (from))))
-    return error_mark_node;
+
+  if (VOID_TYPE_P (to))
+    {
+      if (explain)
+	error_at (location_of (to), "%qT is incomplete", to);
+      return error_mark_node;
+    }
+  if (from
+      && FUNC_OR_METHOD_TYPE_P (from)
+      && (TYPE_READONLY (from) || FUNCTION_REF_QUALIFIED (from)))
+    {
+      if (explain)
+	error ("%qT is a qualified function type", from);
+      return error_mark_node;
+    }
+
   tree expr;
   if (code == MODIFY_EXPR)
-    expr = assignable_expr (to, from);
+    expr = assignable_expr (to, from, explain);
   else if (code == BIT_NOT_EXPR)
-    expr = destructible_expr (to);
-  else if (trivial && TREE_VEC_LENGTH (from) > 1
-	   && cxx_dialect < cxx20)
-    return error_mark_node; // only 0- and 1-argument ctors can be trivial
-			    // before C++20 aggregate paren init
+    expr = destructible_expr (to, explain);
   else if (TREE_CODE (to) == ARRAY_TYPE && !TYPE_DOMAIN (to))
-    return error_mark_node; // can't construct an array of unknown bound
+    {
+      if (explain)
+	error ("cannot construct an array of unknown bound");
+      return error_mark_node;
+    }
   else
-    expr = constructible_expr (to, from);
+    expr = constructible_expr (to, from, explain);
   return expr;
 }
 
 /* Returns true iff TO is trivially assignable (if CODE is MODIFY_EXPR) or
    constructible (otherwise) from FROM, which is a single type for
-   assignment or a list of types for construction.  */
+   assignment or a list of types for construction.  If EXPLAIN, diagnose
+   why we returned false.  */
 
 bool
-is_trivially_xible (enum tree_code code, tree to, tree from)
+is_trivially_xible (enum tree_code code, tree to, tree from,
+		    bool explain/*=false*/)
 {
-  tree expr = is_xible_helper (code, to, from, /*trivial*/true);
+  tree expr = is_xible_helper (code, to, from, explain);
   if (expr == NULL_TREE || expr == error_mark_node)
     return false;
+
   tree nt = cp_walk_tree_without_duplicates (&expr, check_nontriv, NULL);
+  if (explain && nt)
+    inform (location_of (nt), "%qE is non-trivial", nt);
   return !nt;
 }
 
 /* Returns true iff TO is nothrow assignable (if CODE is MODIFY_EXPR) or
    constructible (otherwise) from FROM, which is a single type for
-   assignment or a list of types for construction.  */
+   assignment or a list of types for construction.  If EXPLAIN, diagnose
+   why we returned false.  */
 
 bool
-is_nothrow_xible (enum tree_code code, tree to, tree from)
+is_nothrow_xible (enum tree_code code, tree to, tree from,
+		  bool explain/*=false*/)
 {
   ++cp_noexcept_operand;
-  tree expr = is_xible_helper (code, to, from, /*trivial*/false);
+  tree expr = is_xible_helper (code, to, from, explain);
   --cp_noexcept_operand;
   if (expr == NULL_TREE || expr == error_mark_node)
     return false;
-  return expr_noexcept_p (expr, tf_none);
+
+  bool is_noexcept = expr_noexcept_p (expr, tf_none);
+  if (explain && !is_noexcept)
+    explain_not_noexcept (expr);
+  return is_noexcept;
 }
 
 /* Returns true iff TO is assignable (if CODE is MODIFY_EXPR) or
    constructible (otherwise) from FROM, which is a single type for
-   assignment or a list of types for construction.  */
+   assignment or a list of types for construction.  If EXPLAIN, diagnose
+   why we returned false.  */
 
 bool
-is_xible (enum tree_code code, tree to, tree from)
+is_xible (enum tree_code code, tree to, tree from, bool explain/*=false*/)
 {
-  tree expr = is_xible_helper (code, to, from, /*trivial*/false);
+  tree expr = is_xible_helper (code, to, from, explain);
   if (expr == error_mark_node)
     return false;
   return !!expr;
@@ -2453,7 +2525,7 @@ ref_xes_from_temporary (tree to, tree from, bool direct_init_p)
     return false;
   /* We don't check is_constructible<T, U>: if T isn't constructible
      from U, we won't be able to create a conversion.  */
-  tree val = build_trait_object (from);
+  tree val = build_trait_object (from, tf_none);
   if (val == error_mark_node)
     return false;
   if (!TYPE_REF_P (from) && TREE_CODE (from) != FUNCTION_TYPE)
@@ -2462,25 +2534,36 @@ ref_xes_from_temporary (tree to, tree from, bool direct_init_p)
 }
 
 /* Worker for is_{,nothrow_}convertible.  Attempt to perform an implicit
-   conversion from FROM to TO and return the result.  */
+   conversion from FROM to TO and return the result.  If EXPLAIN, emit a
+   diagnostic about why the conversion failed.  */
 
 static tree
-is_convertible_helper (tree from, tree to)
+is_convertible_helper (tree from, tree to, bool explain)
 {
   if (VOID_TYPE_P (from) && VOID_TYPE_P (to))
     return integer_one_node;
   cp_unevaluated u;
-  tree expr = build_trait_object (from);
+  tsubst_flags_t complain = explain ? tf_error : tf_none;
+
   /* std::is_{,nothrow_}convertible test whether the imaginary function
      definition
 
        To test() { return std::declval<From>(); }
 
      is well-formed.  A function can't return a function.  */
-  if (FUNC_OR_METHOD_TYPE_P (to) || expr == error_mark_node)
+  if (FUNC_OR_METHOD_TYPE_P (to))
+    {
+      if (explain)
+	error ("%qT is a function type", to);
+      return error_mark_node;
+    }
+
+  tree expr = build_trait_object (from, complain);
+  if (expr == error_mark_node)
     return error_mark_node;
+
   deferring_access_check_sentinel acs (dk_no_deferred);
-  return perform_implicit_conversion (to, expr, tf_none);
+  return perform_implicit_conversion (to, expr, complain);
 }
 
 /* Return true if FROM can be converted to TO using implicit conversions,
@@ -2489,9 +2572,9 @@ is_convertible_helper (tree from, tree to)
    to either type" restriction.  */
 
 bool
-is_convertible (tree from, tree to)
+is_convertible (tree from, tree to, bool explain/*=false*/)
 {
-  tree expr = is_convertible_helper (from, to);
+  tree expr = is_convertible_helper (from, to, explain);
   if (expr == error_mark_node)
     return false;
   return !!expr;
@@ -2500,12 +2583,16 @@ is_convertible (tree from, tree to)
 /* Like is_convertible, but the conversion is also noexcept.  */
 
 bool
-is_nothrow_convertible (tree from, tree to)
+is_nothrow_convertible (tree from, tree to, bool explain/*=false*/)
 {
-  tree expr = is_convertible_helper (from, to);
+  tree expr = is_convertible_helper (from, to, explain);
   if (expr == NULL_TREE || expr == error_mark_node)
     return false;
-  return expr_noexcept_p (expr, tf_none);
+
+  bool is_noexcept = expr_noexcept_p (expr, tf_none);
+  if (explain && !is_noexcept)
+    explain_not_noexcept (expr);
+  return is_noexcept;
 }
 
 /* Categorize various special_function_kinds.  */
@@ -3586,21 +3673,24 @@ maybe_delete_defaulted_fn (tree fn, tree implicit_fn)
 	      the program is ill-formed"  */
 	   || !TYPE_REF_P (parmtype)));
   /* Decide if we want to emit a pedwarn, error, or a warning.  */
-  diagnostic_t diag_kind;
+  enum diagnostics::kind diag_kind;
   int opt;
   if (illformed_p)
     {
-      diag_kind = DK_ERROR;
+      diag_kind = diagnostics::kind::error;
       opt = 0;
     }
   else
     {
-      diag_kind = cxx_dialect >= cxx20 ? DK_WARNING : DK_PEDWARN;
+      diag_kind = (cxx_dialect >= cxx20
+		   ? diagnostics::kind::warning
+		   : diagnostics::kind::pedwarn);
       opt = OPT_Wdefaulted_function_deleted;
     }
 
   /* Don't warn for template instantiations.  */
-  if (DECL_TEMPLATE_INSTANTIATION (fn) && diag_kind == DK_WARNING)
+  if (DECL_TEMPLATE_INSTANTIATION (fn)
+      && diag_kind == diagnostics::kind::warning)
     return;
 
   const char *wmsg;
diff --git a/gcc/cp/module.cc b/gcc/cp/module.cc
index c8e79f3..9412f78 100644
--- a/gcc/cp/module.cc
+++ b/gcc/cp/module.cc
@@ -3883,9 +3883,9 @@ class GTY((chain_next ("%h.parent"), for_user)) module_state {
   void write_macro_maps (elf_out *to, range_t &, unsigned *crc_ptr);
   bool read_macro_maps (line_map_uint_t);
 
-  void write_diagnostic_classification (elf_out *, diagnostic_context *,
+  void write_diagnostic_classification (elf_out *, diagnostics::context *,
 					unsigned *);
-  bool read_diagnostic_classification (diagnostic_context *);
+  bool read_diagnostic_classification (diagnostics::context *);
 
  private:
   void write_define (bytes_out &, const cpp_macro *);
@@ -4822,7 +4822,8 @@ noisy_p ()
     return false;
 
   pp_needs_newline (global_dc->get_reference_printer ()) = true;
-  diagnostic_set_last_function (global_dc, (diagnostic_info *) NULL);
+  diagnostic_set_last_function (global_dc,
+				(diagnostics::diagnostic_info *) nullptr);
 
   return true;
 }
@@ -6157,7 +6158,7 @@ trees_out::lang_type_bools (tree t, bits_out& bits)
   WB (lang->declared_class);
   WB (lang->diamond_shaped);
   WB (lang->repeated_base);
-  gcc_assert (!lang->being_defined);
+  gcc_checking_assert (!lang->being_defined);
   // lang->debug_requested
   WB (lang->fields_readonly);
   WB (lang->ptrmemfunc_flag);
@@ -6183,6 +6184,14 @@ trees_out::lang_type_bools (tree t, bits_out& bits)
   WB (lang->has_constexpr_ctor);
   WB (lang->unique_obj_representations);
   WB (lang->unique_obj_representations_set);
+  gcc_checking_assert (!lang->erroneous);
+  WB (lang->non_pod_aggregate);
+  WB (lang->non_aggregate_pod);
+  WB (lang->trivially_relocatable);
+  WB (lang->trivially_relocatable_computed);
+
+  WB (lang->replaceable);
+  WB (lang->replaceable_computed);
 #undef WB
 }
 
@@ -6227,8 +6236,8 @@ trees_in::lang_type_bools (tree t, bits_in& bits)
   RB (lang->declared_class);
   RB (lang->diamond_shaped);
   RB (lang->repeated_base);
-  gcc_assert (!lang->being_defined);
-  gcc_assert (!lang->debug_requested);
+  gcc_checking_assert (!lang->being_defined);
+  gcc_checking_assert (!lang->debug_requested);
   RB (lang->fields_readonly);
   RB (lang->ptrmemfunc_flag);
 
@@ -6253,6 +6262,14 @@ trees_in::lang_type_bools (tree t, bits_in& bits)
   RB (lang->has_constexpr_ctor);
   RB (lang->unique_obj_representations);
   RB (lang->unique_obj_representations_set);
+  gcc_checking_assert (!lang->erroneous);
+  RB (lang->non_pod_aggregate);
+  RB (lang->non_aggregate_pod);
+  RB (lang->trivially_relocatable);
+  RB (lang->trivially_relocatable_computed);
+
+  RB (lang->replaceable);
+  RB (lang->replaceable_computed);
 #undef RB
   return !get_overrun ();
 }
@@ -6543,8 +6560,14 @@ trees_out::core_vals (tree t)
 	}
 
       WT (t->function_decl.personality);
-      WT (t->function_decl.function_specific_target);
-      WT (t->function_decl.function_specific_optimization);
+      /* Rather than streaming target/optimize nodes, we should reconstruct
+	 them on stream-in from any attributes applied to the function.  */
+      if (streaming_p () && t->function_decl.function_specific_target)
+	warning_at (DECL_SOURCE_LOCATION (t), 0,
+		    "%<target%> attribute currently unsupported in modules");
+      if (streaming_p () && t->function_decl.function_specific_optimization)
+	warning_at (DECL_SOURCE_LOCATION (t), 0,
+		    "%<optimize%> attribute currently unsupported in modules");
       WT (t->function_decl.vindex);
 
       if (DECL_HAS_DEPENDENT_EXPLICIT_SPEC_P (t))
@@ -6634,11 +6657,12 @@ trees_out::core_vals (tree t)
     case TARGET_OPTION_NODE:
       // FIXME: Our representation for these two nodes is a cache of
       // the resulting set of options.  Not a record of the options
-      // that got changed by a particular attribute or pragma.  Should
-      // we record that, or should we record the diff from the command
-      // line options?  The latter seems the right behaviour, but is
-      // (a) harder, and I guess could introduce strangeness if the
-      // importer has set some incompatible set of optimization flags?
+      // that got changed by a particular attribute or pragma.  Instead
+      // of recording that, we probably should just rebuild the options
+      // on stream-in from the function attributes.  This could introduce
+      // strangeness if the importer has some incompatible set of flags
+      // but we currently assume users "know what they're doing" in such
+      // a case anyway.
       gcc_unreachable ();
       break;
 
@@ -6785,6 +6809,13 @@ trees_out::core_vals (tree t)
       if (streaming_p ())
 	WU (((lang_tree_node *)t)->trait_expression.kind);
       break;
+
+    case TU_LOCAL_ENTITY:
+      WT (((lang_tree_node *)t)->tu_local_entity.name);
+      if (state)
+	state->write_location
+	  (*this, ((lang_tree_node *)t)->tu_local_entity.loc);
+      break;
     }
 
   if (CODE_CONTAINS_STRUCT (code, TS_TYPED))
@@ -7090,8 +7121,10 @@ trees_in::core_vals (tree t)
 	  }
 
 	RT (t->function_decl.personality);
-	RT (t->function_decl.function_specific_target);
-	RT (t->function_decl.function_specific_optimization);
+	/* These properties are not streamed, and should be reconstructed
+	   from any function attributes.  */
+	// t->function_decl.function_specific_target);
+	// t->function_decl.function_specific_optimization);
 	RT (t->function_decl.vindex);
 
 	if (DECL_HAS_DEPENDENT_EXPLICIT_SPEC_P (t))
@@ -7197,7 +7230,7 @@ trees_in::core_vals (tree t)
 
     case OPTIMIZATION_NODE:
     case TARGET_OPTION_NODE:
-      /* Not yet implemented, see trees_out::core_vals.  */
+      /* Not implemented, see trees_out::core_vals.  */
       gcc_unreachable ();
       break;
 
@@ -7328,6 +7361,11 @@ trees_in::core_vals (tree t)
       RT (((lang_tree_node *)t)->trait_expression.type2);
       RUC (cp_trait_kind, ((lang_tree_node *)t)->trait_expression.kind);
       break;
+
+    case TU_LOCAL_ENTITY:
+      RT (((lang_tree_node *)t)->tu_local_entity.name);
+      ((lang_tree_node *)t)->tu_local_entity.loc
+	= state->read_location (*this);
     }
 
   if (CODE_CONTAINS_STRUCT (code, TS_TYPED))
@@ -9619,6 +9657,8 @@ trees_out::type_node (tree type)
 	      tag_type = enum_type;
 	    else if (TYPENAME_IS_CLASS_P (type))
 	      tag_type = class_type;
+	    else if (TYPENAME_IS_UNION_P (type))
+	      tag_type = union_type;
 	    u (int (tag_type));
 	  }
 	}
@@ -10266,7 +10306,8 @@ trees_in::tree_node (bool is_use)
 	    && dump ("Read %stypedef %C:%N",
 		     DECL_IMPLICIT_TYPEDEF_P (res) ? "implicit " : "",
 		     TREE_CODE (res), res);
-	  res = TREE_TYPE (res);
+	  if (TREE_CODE (res) != TU_LOCAL_ENTITY)
+	    res = TREE_TYPE (res);
 	}
       break;
 
@@ -11122,8 +11163,7 @@ trees_in::fn_parms_fini (int tag, tree fn, tree existing, bool is_defn)
 {
   tree existing_parm = existing ? DECL_ARGUMENTS (existing) : NULL_TREE;
   tree parms = DECL_ARGUMENTS (fn);
-  unsigned ix = 0;
-  for (tree parm = parms; parm; parm = DECL_CHAIN (parm), ix++)
+  for (tree parm = parms; parm; parm = DECL_CHAIN (parm))
     {
       if (existing_parm)
 	{
@@ -11133,6 +11173,20 @@ trees_in::fn_parms_fini (int tag, tree fn, tree existing, bool is_defn)
 		 names of the parms from us.  */
 	      DECL_NAME (existing_parm) = DECL_NAME (parm);
 	      DECL_SOURCE_LOCATION (existing_parm) = DECL_SOURCE_LOCATION (parm);
+
+	      /* And some other flags important for codegen are only set
+		 by the definition.  */
+	      TREE_ADDRESSABLE (existing_parm) = TREE_ADDRESSABLE (parm);
+	      DECL_BY_REFERENCE (existing_parm) = DECL_BY_REFERENCE (parm);
+	      DECL_NONLOCAL (existing_parm) = DECL_NONLOCAL (parm);
+	      DECL_ARG_TYPE (existing_parm) = DECL_ARG_TYPE (parm);
+
+	      /* Invisiref parms had their types adjusted by cp_genericize. */
+	      if (DECL_BY_REFERENCE (parm))
+		{
+		  TREE_TYPE (existing_parm) = TREE_TYPE (parm);
+		  relayout_decl (existing_parm);
+		}
 	    }
 
 	  back_refs[~tag] = existing_parm;
@@ -13391,13 +13445,19 @@ trees_in::read_class_def (tree defn, tree maybe_template)
 
       if (TYPE_LANG_SPECIFIC (type))
 	{
-	  CLASSTYPE_LAMBDA_EXPR (type) = lambda;
+	  if (!TYPE_POLYMORPHIC_P (type))
+	    SET_CLASSTYPE_LAMBDA_EXPR (type, lambda);
+	  else
+	    gcc_checking_assert (lambda == NULL_TREE);
 
 	  CLASSTYPE_MEMBER_VEC (type) = member_vec;
 	  CLASSTYPE_PURE_VIRTUALS (type) = pure_virts;
 	  CLASSTYPE_VCALL_INDICES (type) = vcall_indices;
 
-	  CLASSTYPE_KEY_METHOD (type) = key_method;
+	  if (TYPE_POLYMORPHIC_P (type))
+	    SET_CLASSTYPE_KEY_METHOD (type, key_method);
+	  else
+	    gcc_checking_assert (key_method == NULL_TREE);
 
 	  CLASSTYPE_VBASECLASSES (type) = vbase_vec;
 
@@ -18318,22 +18378,23 @@ module_state::write_ordinary_maps (elf_out *to, range_t &info,
 /* Return the prefix to use for dumping a #pragma diagnostic change to DK.  */
 
 static const char *
-dk_string (diagnostic_t dk)
+dk_string (enum diagnostics::kind dk)
 {
-  gcc_assert (dk > DK_UNSPECIFIED && dk < DK_LAST_DIAGNOSTIC_KIND);
-  if (dk == DK_IGNORED)
-    /* diagnostic.def has an empty string for ignored.  */
+  gcc_assert (dk > diagnostics::kind::unspecified
+	      && dk < diagnostics::kind::last_diagnostic_kind);
+  if (dk == diagnostics::kind::ignored)
+    /* diagnostics/kinds.def has an empty string for ignored.  */
     return "ignored: ";
   else
-    return get_diagnostic_kind_text (dk);
+    return diagnostics::get_text_for_kind (dk);
 }
 
 /* Dump one #pragma GCC diagnostic entry.  */
 
 static bool
-dump_dc_change (unsigned index, unsigned opt, diagnostic_t dk)
+dump_dc_change (unsigned index, unsigned opt, enum diagnostics::kind dk)
 {
-  if (dk == DK_POP)
+  if (dk == diagnostics::kind::pop)
     return dump (" Index %u: pop from %d", index, opt);
   else
     return dump (" Index %u: %s%s", index, dk_string (dk),
@@ -18344,7 +18405,7 @@ dump_dc_change (unsigned index, unsigned opt, diagnostic_t dk)
 
 void
 module_state::write_diagnostic_classification (elf_out *to,
-					       diagnostic_context *dc,
+					       diagnostics::context *dc,
 					       unsigned *crc_p)
 {
   auto &changes = dc->get_classification_history ();
@@ -18360,9 +18421,10 @@ module_state::write_diagnostic_classification (elf_out *to,
   unsigned len = changes.length ();
 
   /* We don't want to write out any entries that came from one of our imports.
-     But then we need to adjust the total, and change DK_POP targets to match
-     the index in our actual output.  So remember how many lines we had skipped
-     at each step, where -1 means this line itself is skipped.  */
+     But then we need to adjust the total, and change diagnostics::kind::pop
+     targets to match the index in our actual output.  So remember how many
+     lines we had skipped at each step, where -1 means this line itself
+     is skipped.  */
   int skips = 0;
   auto_vec<int> skips_at (len);
   skips_at.safe_grow (len);
@@ -18395,10 +18457,10 @@ module_state::write_diagnostic_classification (elf_out *to,
       if (sec.streaming_p ())
 	{
 	  unsigned opt = c.option;
-	  if (c.kind == DK_POP)
+	  if (c.kind == diagnostics::kind::pop)
 	    opt -= skips_at[opt];
 	  sec.u (opt);
-	  sec.u (c.kind);
+	  sec.u (static_cast<unsigned> (c.kind));
 	  dump () && dump_dc_change (i - skips_at[i], opt, c.kind);
 	}
     }
@@ -18413,7 +18475,7 @@ module_state::write_diagnostic_classification (elf_out *to,
 /* Read any #pragma GCC diagnostic info from the .dgc section.  */
 
 bool
-module_state::read_diagnostic_classification (diagnostic_context *dc)
+module_state::read_diagnostic_classification (diagnostics::context *dc)
 {
   bytes_in sec;
 
@@ -18433,8 +18495,8 @@ module_state::read_diagnostic_classification (diagnostic_context *dc)
     {
       location_t loc = read_location (sec);
       int opt = sec.u ();
-      diagnostic_t kind = (diagnostic_t) sec.u ();
-      if (kind == DK_POP)
+      enum diagnostics::kind kind = (enum diagnostics::kind) sec.u ();
+      if (kind == diagnostics::kind::pop)
 	/* For a pop, opt is the 'changes' index to return to.  */
 	opt += offset;
       changes.quick_push ({ loc, opt, kind });
@@ -18449,7 +18511,7 @@ module_state::read_diagnostic_classification (diagnostic_context *dc)
 	gcc_checking_assert (i >= offset);
 
 	const auto &c = changes[i];
-	if (c.kind != DK_POP)
+	if (c.kind != diagnostics::kind::pop)
 	  break;
 	else if (c.option == offset)
 	  {
@@ -18466,7 +18528,7 @@ module_state::read_diagnostic_classification (diagnostic_context *dc)
       /* It didn't, so add a pop at its last location to avoid affecting later
 	 imports.  */
       location_t last_loc = ordinary_locs.first + ordinary_locs.second - 1;
-      changes.quick_push ({ last_loc, offset, DK_POP });
+      changes.quick_push ({ last_loc, offset, diagnostics::kind::pop });
       dump () && dump (" Adding final pop from index %d", offset);
     }
 
diff --git a/gcc/cp/name-lookup.cc b/gcc/cp/name-lookup.cc
index 9aa7c16..f5b36c9 100644
--- a/gcc/cp/name-lookup.cc
+++ b/gcc/cp/name-lookup.cc
@@ -7195,7 +7195,7 @@ suggest_alternatives_for_1 (location_t location, tree name,
   /* Look for exact matches for builtin defines that would have been
      defined if the user had passed a command-line option (e.g. -fopenmp
      for "_OPENMP").  */
-  diagnostic_option_id option_id
+  diagnostics::option_id option_id
     = get_option_for_builtin_define (IDENTIFIER_POINTER (name));
   if (option_id.m_idx > 0)
     return name_hint
diff --git a/gcc/cp/parser.cc b/gcc/cp/parser.cc
index b1626ac..860f3f0 100644
--- a/gcc/cp/parser.cc
+++ b/gcc/cp/parser.cc
@@ -2519,7 +2519,7 @@ static cp_expr cp_parser_id_expression
 static cp_expr cp_parser_unqualified_id
   (cp_parser *, bool, bool, bool, bool);
 static tree cp_parser_nested_name_specifier_opt
-  (cp_parser *, bool, bool, bool, bool, bool = false);
+  (cp_parser *, bool, bool, bool, bool, bool = false, bool = false);
 static tree cp_parser_nested_name_specifier
   (cp_parser *, bool, bool, bool, bool);
 static tree cp_parser_qualifying_entity
@@ -2576,11 +2576,11 @@ static cp_expr cp_parser_constant_expression
 static cp_expr cp_parser_builtin_offsetof
   (cp_parser *);
 static cp_expr cp_parser_lambda_expression
-  (cp_parser *);
+  (cp_parser *, bool = false);
 static void cp_parser_lambda_introducer
   (cp_parser *, tree);
 static bool cp_parser_lambda_declarator_opt
-  (cp_parser *, tree);
+  (cp_parser *, tree, bool = false);
 static void cp_parser_lambda_body
   (cp_parser *, tree);
 
@@ -2921,7 +2921,7 @@ static size_t cp_parser_skip_std_attribute_spec_seq
 static size_t cp_parser_skip_attributes_opt
   (cp_parser *, size_t);
 static bool cp_parser_extension_opt
-  (cp_parser *, int *);
+  (cp_parser *, int *, int *);
 static void cp_parser_label_declaration
   (cp_parser *);
 
@@ -3091,8 +3091,8 @@ static cp_token *cp_parser_require_keyword
   (cp_parser *, enum rid, required_token);
 static bool cp_parser_token_starts_function_definition_p
   (cp_token *);
-static bool cp_parser_next_token_starts_class_definition_p
-  (cp_parser *);
+static bool cp_parser_nth_token_starts_class_definition_p
+  (cp_parser *, size_t);
 static bool cp_parser_next_token_ends_template_argument_p
   (cp_parser *);
 static bool cp_parser_nth_token_starts_template_argument_list_p
@@ -5266,7 +5266,7 @@ cp_parser_userdef_numeric_literal (cp_parser *parser)
 	      && (id_equal (suffix_id, "i")
 		  || id_equal (suffix_id, "if")
 		  || id_equal (suffix_id, "il")));
-  diagnostic_t kind = DK_ERROR;
+  enum diagnostics::kind kind = diagnostics::kind::error;
   int opt = 0;
 
   if (i14 && ext)
@@ -5276,7 +5276,7 @@ cp_parser_userdef_numeric_literal (cp_parser *parser)
       if (cxlit == error_mark_node)
 	{
 	  /* No <complex>, so pedwarn and use GNU semantics.  */
-	  kind = DK_PEDWARN;
+	  kind = diagnostics::kind::pedwarn;
 	  opt = OPT_Wpedantic;
 	}
     }
@@ -5303,7 +5303,7 @@ cp_parser_userdef_numeric_literal (cp_parser *parser)
 	      "to enable more built-in suffixes");
   }
 
-  if (kind == DK_ERROR)
+  if (kind == diagnostics::kind::error)
     value = error_mark_node;
   else
     {
@@ -6307,7 +6307,10 @@ cp_parser_primary_expression (cp_parser *parser,
 	  /* Recognize the `this' keyword.  */
 	case RID_THIS:
 	  cp_lexer_consume_token (parser->lexer);
-	  if (parser->local_variables_forbidden_p & THIS_FORBIDDEN)
+	  if ((parser->local_variables_forbidden_p & THIS_FORBIDDEN)
+	      /* It's OK to refer to 'this' in an unevaluated operand in a
+		 lambda default argument (lambda-targ16.C).  */
+	      && !cp_unevaluated_operand)
 	    {
 	      error_at (token->location,
 			"%<this%> may not be used in this context");
@@ -6644,7 +6647,8 @@ cp_parser_primary_expression (cp_parser *parser,
    member template.  */
 
 static void
-missing_template_diag (location_t loc, diagnostic_t diag_kind = DK_WARNING)
+missing_template_diag (location_t loc,
+		       enum diagnostics::kind diag_kind = diagnostics::kind::warning)
 {
   if (warning_suppressed_at (loc, OPT_Wmissing_template_keyword))
     return;
@@ -7239,18 +7243,22 @@ check_template_keyword_in_nested_name_spec (tree name)
      nested-name-specifier template [opt] simple-template-id ::
 
    PARSER->SCOPE should be set appropriately before this function is
-   called.  TYPENAME_KEYWORD_P is TRUE if the `typename' keyword is in
-   effect.  TYPE_P is TRUE if we non-type bindings should be ignored
-   in name lookups.
+   called.  TYPENAME_KEYWORD_P is true if the `typename' keyword is in
+   effect.  TYPE_P is true if we non-type bindings should be ignored
+   in name lookups.  TEMPLATE_KEYWORD_P is true if the `template' keyword
+   was seen.  GLOBAL_P is true if `::' has already been parsed.
+   TODO: This function doesn't handle the C++14 change to make `::'
+   a nested-name-specifier by itself.  If it did, GLOBAL_P could probably
+   go.
 
    Sets PARSER->SCOPE to the class (TYPE) or namespace
    (NAMESPACE_DECL) specified by the nested-name-specifier, or leaves
    it unchanged if there is no nested-name-specifier.  Returns the new
    scope iff there is a nested-name-specifier, or NULL_TREE otherwise.
 
-   If CHECK_DEPENDENCY_P is FALSE, names are looked up in dependent scopes.
+   If CHECK_DEPENDENCY_P is false, names are looked up in dependent scopes.
 
-   If IS_DECLARATION is TRUE, the nested-name-specifier is known to be
+   If IS_DECLARATION is true, the nested-name-specifier is known to be
    part of a declaration and/or decl-specifier.  */
 
 static tree
@@ -7259,7 +7267,8 @@ cp_parser_nested_name_specifier_opt (cp_parser *parser,
 				     bool check_dependency_p,
 				     bool type_p,
 				     bool is_declaration,
-				     bool template_keyword_p /* = false */)
+				     bool template_keyword_p /* = false */,
+				     bool global_p /* = false */)
 {
   bool success = false;
   cp_token_position start = 0;
@@ -7307,8 +7316,9 @@ cp_parser_nested_name_specifier_opt (cp_parser *parser,
 
       /* Spot cases that cannot be the beginning of a
 	 nested-name-specifier.  On the second and subsequent times
-	 through the loop, we look for the `template' keyword.  */
-      if (success && token->keyword == RID_TEMPLATE)
+	 (or the first, if '::' has already been parsed) through the
+	 loop, we look for the `template' keyword.  */
+      if ((success || global_p) && token->keyword == RID_TEMPLATE)
 	;
       /* A template-id can start a nested-name-specifier.  */
       else if (token->type == CPP_TEMPLATE_ID)
@@ -7356,8 +7366,11 @@ cp_parser_nested_name_specifier_opt (cp_parser *parser,
       cp_parser_parse_tentatively (parser);
 
       /* Look for the optional `template' keyword, if this isn't the
-	 first time through the loop.  */
-      if (success)
+	 first time through the loop, or if we've already parsed '::';
+	 this is then the
+	   nested-name-specifier template [opt] simple-template-id ::
+	 production.  */
+      if (success || global_p)
 	{
 	  template_keyword_p = cp_parser_optional_template_keyword (parser);
 	  /* DR1710: "In a qualified-id used as the name in
@@ -8842,8 +8855,10 @@ cp_parser_dot_deref_incomplete (tree *scope, cp_expr *postfix_expression,
 {
   /* In a template, be permissive by treating an object expression
      of incomplete type as dependent (after a pedwarn).  */
-  diagnostic_t kind = (processing_template_decl
-		       && MAYBE_CLASS_TYPE_P (*scope) ? DK_PEDWARN : DK_ERROR);
+  enum diagnostics::kind kind = ((processing_template_decl
+				  && MAYBE_CLASS_TYPE_P (*scope))
+				 ? diagnostics::kind::pedwarn
+				 : diagnostics::kind::error);
 
   switch (TREE_CODE (*postfix_expression))
     {
@@ -8855,27 +8870,27 @@ cp_parser_dot_deref_incomplete (tree *scope, cp_expr *postfix_expression,
     case IMPLICIT_CONV_EXPR:
     case VIEW_CONVERT_EXPR:
     case NON_LVALUE_EXPR:
-      kind = DK_ERROR;
+      kind = diagnostics::kind::error;
       break;
     case OVERLOAD:
       /* Don't emit any diagnostic for OVERLOADs.  */
-      kind = DK_IGNORED;
+      kind = diagnostics::kind::ignored;
       break;
     default:
       /* Avoid clobbering e.g. DECLs.  */
       if (!EXPR_P (*postfix_expression))
-	kind = DK_ERROR;
+	kind = diagnostics::kind::error;
       break;
     }
 
-  if (kind == DK_IGNORED)
+  if (kind == diagnostics::kind::ignored)
     return false;
 
   location_t exploc = location_of (*postfix_expression);
   cxx_incomplete_type_diagnostic (exploc, *postfix_expression, *scope, kind);
   if (!MAYBE_CLASS_TYPE_P (*scope))
     return true;
-  if (kind == DK_ERROR)
+  if (kind == diagnostics::kind::error)
     *scope = *postfix_expression = error_mark_node;
   else if (processing_template_decl)
     {
@@ -9489,21 +9504,23 @@ cp_parser_unary_expression (cp_parser *parser, cp_id_kind * pidk,
 	case RID_EXTENSION:
 	  {
 	    /* The saved value of the PEDANTIC flag.  */
-	    int saved_pedantic;
+	    int saved_pedantic, saved_long_long;
 	    tree expr;
 
 	    /* Save away the PEDANTIC flag.  */
-	    cp_parser_extension_opt (parser, &saved_pedantic);
+	    cp_parser_extension_opt (parser, &saved_pedantic,
+				     &saved_long_long);
 	    /* Also suppress -Wconditionally-supported.  */
 	    diagnostic_push_diagnostics (global_dc, input_location);
 	    diagnostic_classify_diagnostic
 	      (global_dc, OPT_Wconditionally_supported,
-	       DK_IGNORED, input_location);
+	       diagnostics::kind::ignored, input_location);
 	    /* Parse the cast-expression.  */
 	    expr = cp_parser_simple_cast_expression (parser);
 	    /* Restore the PEDANTIC flag.  */
 	    diagnostic_pop_diagnostics (global_dc, input_location);
 	    pedantic = saved_pedantic;
+	    warn_long_long = saved_long_long;
 
 	    return expr;
 	  }
@@ -11727,10 +11744,14 @@ cp_parser_trait (cp_parser* parser, const cp_trait* trait)
      lambda-introducer < template-parameter-list > requires-clause [opt]
        lambda-declarator [opt] compound-statement
 
+   If CONSTEVAL_BLOCK_P is true, we are parsing a consteval block, which
+   is syntactic sugar for a consteval lambda.
+
    Returns a representation of the expression.  */
 
 static cp_expr
-cp_parser_lambda_expression (cp_parser* parser)
+cp_parser_lambda_expression (cp_parser* parser,
+			     bool consteval_block_p/*=false*/)
 {
   tree lambda_expr = build_lambda_expr ();
   tree type;
@@ -11739,6 +11760,7 @@ cp_parser_lambda_expression (cp_parser* parser)
   cp_token_position start = 0;
 
   LAMBDA_EXPR_LOCATION (lambda_expr) = token->location;
+  LAMBDA_EXPR_CONSTEVAL_BLOCK_P (lambda_expr) = consteval_block_p;
 
   if (cxx_dialect >= cxx20)
     {
@@ -11782,9 +11804,12 @@ cp_parser_lambda_expression (cp_parser* parser)
      it now.  */
   push_deferring_access_checks (dk_no_deferred);
 
-  cp_parser_lambda_introducer (parser, lambda_expr);
-  if (cp_parser_error_occurred (parser))
-    return error_mark_node;
+  if (!consteval_block_p)
+    {
+      cp_parser_lambda_introducer (parser, lambda_expr);
+      if (cp_parser_error_occurred (parser))
+	return error_mark_node;
+    }
 
   {
     /* OK, this is a bit tricksy.  cp_parser_requires_expression sets
@@ -11826,6 +11851,7 @@ cp_parser_lambda_expression (cp_parser* parser)
     bool auto_is_implicit_function_template_parm_p
         = parser->auto_is_implicit_function_template_parm_p;
     bool saved_omp_array_section_p = parser->omp_array_section_p;
+    bool saved_in_targ = parser->in_template_argument_list_p;
 
     parser->num_template_parameter_lists = 0;
     parser->in_statement = 0;
@@ -11835,6 +11861,7 @@ cp_parser_lambda_expression (cp_parser* parser)
     parser->implicit_template_scope = 0;
     parser->auto_is_implicit_function_template_parm_p = false;
     parser->omp_array_section_p = false;
+    parser->in_template_argument_list_p = false;
 
     /* Inside the lambda, outside unevaluated context do not apply.  */
     cp_evaluated ev;
@@ -11854,7 +11881,8 @@ cp_parser_lambda_expression (cp_parser* parser)
     if (cp_parser_start_tentative_firewall (parser))
       start = token;
 
-    ok &= cp_parser_lambda_declarator_opt (parser, lambda_expr);
+    ok &= cp_parser_lambda_declarator_opt (parser, lambda_expr,
+					   consteval_block_p);
 
     if (ok && cp_parser_error_occurred (parser))
       ok = false;
@@ -11889,6 +11917,7 @@ cp_parser_lambda_expression (cp_parser* parser)
     parser->auto_is_implicit_function_template_parm_p
 	= auto_is_implicit_function_template_parm_p;
     parser->omp_array_section_p = saved_omp_array_section_p;
+    parser->in_template_argument_list_p = saved_in_targ;
   }
 
   /* This lambda shouldn't have any proxies left at this point.  */
@@ -12236,10 +12265,13 @@ cp_parser_lambda_introducer (cp_parser* parser, tree lambda_expr)
      decl-specifier-seq [opt] noexcept-specifier [opt]
        attribute-specifier-seq [opt] trailing-return-type [opt]
 
-   LAMBDA_EXPR is the current representation of the lambda expression.  */
+   LAMBDA_EXPR is the current representation of the lambda expression.
+   If CONSTEVAL_BLOCK_P is true, we are parsing a consteval block, which
+   is syntactic sugar for a consteval lambda.  */
 
 static bool
-cp_parser_lambda_declarator_opt (cp_parser* parser, tree lambda_expr)
+cp_parser_lambda_declarator_opt (cp_parser* parser, tree lambda_expr,
+				 bool consteval_block_p/*=false*/)
 {
   /* 5.1.1.4 of the standard says:
        If a lambda-expression does not include a lambda-declarator, it is as if
@@ -12342,6 +12374,18 @@ cp_parser_lambda_declarator_opt (cp_parser* parser, tree lambda_expr)
 				  CP_PARSER_FLAGS_ONLY_MUTABLE_OR_CONSTEXPR,
 				  &lambda_specs, &declares_class_or_enum);
 
+  /* [dcl.pre] For a consteval-block-declaration D, the expression E
+     corresponding to D is:
+       [] -> void static consteval compound-statement ()
+     Make it so.  */
+  if (consteval_block_p)
+    {
+      return_type = void_type_node;
+      lambda_specs.storage_class = sc_static;
+      set_and_check_decl_spec_loc (&lambda_specs, ds_consteval,
+				   cp_lexer_peek_token (parser->lexer));
+    }
+
   if (omitted_parms_loc && lambda_specs.any_specifiers_p)
     {
       pedwarn (omitted_parms_loc, OPT_Wc__23_extensions,
@@ -16029,15 +16073,16 @@ cp_parser_declaration_seq_opt (cp_parser* parser)
 static void
 cp_parser_declaration (cp_parser* parser, tree prefix_attrs)
 {
-  int saved_pedantic;
+  int saved_pedantic, saved_long_long;
 
   /* Check for the `__extension__' keyword.  */
-  if (cp_parser_extension_opt (parser, &saved_pedantic))
+  if (cp_parser_extension_opt (parser, &saved_pedantic, &saved_long_long))
     {
       /* Parse the qualified declaration.  */
       cp_parser_declaration (parser, prefix_attrs);
       /* Restore the PEDANTIC flag.  */
       pedantic = saved_pedantic;
+      warn_long_long = saved_long_long;
 
       return;
     }
@@ -16279,6 +16324,56 @@ cp_parser_toplevel_declaration (cp_parser* parser)
     cp_parser_declaration (parser, NULL_TREE);
 }
 
+/* Build an empty string for static_assert.  */
+
+static tree
+build_empty_string ()
+{
+  tree message = build_string (1, "");
+  TREE_TYPE (message) = char_array_type_node;
+  fix_string_type (message);
+  return message;
+}
+
+/* Return true iff the next tokens start a C++26 consteval block.  */
+
+static bool
+cp_parser_next_tokens_are_consteval_block_p (cp_parser *parser)
+{
+  return (cxx_dialect >= cxx26
+	  && cp_lexer_next_token_is_keyword (parser->lexer, RID_CONSTEVAL)
+	  && cp_lexer_nth_token_is (parser->lexer, 2, CPP_OPEN_BRACE));
+}
+
+/* Parse a consteval-block-declaration.
+
+   consteval-block-declaration:
+     consteval compound-statement
+
+   If MEMBER_P, this consteval block is a member declaration.  */
+
+static void
+cp_parser_consteval_block (cp_parser *parser, bool member_p)
+{
+  const location_t loc = cp_lexer_peek_token (parser->lexer)->location;
+  /* Consume the 'consteval'.  */
+  cp_lexer_consume_token (parser->lexer);
+
+  /* We know the next token is '{'.  Let cp_parser_lambda_body handle it.  */
+  cp_expr lam = cp_parser_lambda_expression (parser,
+					     /*consteval_block_p=*/true);
+  if (!cp_parser_error_occurred (parser))
+    {
+      releasing_vec args;
+      tree call = finish_call_expr (lam, &args,
+				    /*disallow_virtual=*/false,
+				    /*koenig_p=*/false,
+				    tf_warning_or_error);
+      finish_static_assert (call, build_empty_string (), loc, member_p,
+			    /*show_expr_p=*/false, /*consteval_block_p=*/true);
+    }
+}
+
 /* Parse a block-declaration.
 
    block-declaration:
@@ -16286,18 +16381,18 @@ cp_parser_toplevel_declaration (cp_parser* parser)
      asm-definition
      namespace-alias-definition
      using-declaration
+     using-enum-declaration
      using-directive
+     static_assert-declaration
+     consteval-block-declaration
+     alias-declaration
+     opaque-enum-declaration
 
    GNU Extension:
 
    block-declaration:
      __extension__ block-declaration
 
-   C++0x Extension:
-
-   block-declaration:
-     static_assert-declaration
-
    If STATEMENT_P is TRUE, then this block-declaration is occurring as
    part of a declaration-statement.  */
 
@@ -16305,15 +16400,16 @@ static void
 cp_parser_block_declaration (cp_parser *parser,
 			     bool      statement_p)
 {
-  int saved_pedantic;
+  int saved_pedantic, saved_long_long;
 
   /* Check for the `__extension__' keyword.  */
-  if (cp_parser_extension_opt (parser, &saved_pedantic))
+  if (cp_parser_extension_opt (parser, &saved_pedantic, &saved_long_long))
     {
       /* Parse the qualified declaration.  */
       cp_parser_block_declaration (parser, statement_p);
       /* Restore the PEDANTIC flag.  */
       pedantic = saved_pedantic;
+      warn_long_long = saved_long_long;
 
       return;
     }
@@ -16374,6 +16470,8 @@ cp_parser_block_declaration (cp_parser *parser,
   /* If the next token is `static_assert' we have a static assertion.  */
   else if (token1->keyword == RID_STATIC_ASSERT)
     cp_parser_static_assert (parser, /*member_p=*/false);
+  else if (cp_parser_next_tokens_are_consteval_block_p (parser))
+    cp_parser_consteval_block (parser, /*member_p=*/false);
   else
     {
       size_t attr_idx = cp_parser_skip_std_attribute_spec_seq (parser, 1);
@@ -16867,7 +16965,11 @@ cp_parser_decomposition_declaration (cp_parser *parser,
 	  decl = error_mark_node;
 	}
       else
-	prev = decl2;
+	{
+	  prev = decl2;
+	  DECL_DECLARED_CONSTEXPR_P (decl2) = DECL_DECLARED_CONSTEXPR_P (decl);
+	  DECL_DECLARED_CONSTINIT_P (decl2) = DECL_DECLARED_CONSTINIT_P (decl);
+	}
       if (elt_pushed_scope)
 	pop_scope (elt_pushed_scope);
     }
@@ -16916,6 +17018,15 @@ cp_parser_decomposition_declaration (cp_parser *parser,
       /* Ensure DECL_VALUE_EXPR is created for all the decls but
 	 the underlying DECL.  */
       cp_finish_decomp (decl, &decomp);
+      if (decl_spec_seq_has_spec_p (decl_specifiers, ds_thread))
+	pedwarn (decl_specifiers->locations[ds_thread],
+		 0, "for-range-declaration cannot be %qs",
+		 decl_specifiers->gnu_thread_keyword_p
+		 ? "__thread" : "thread_local");
+      else if (decl_specifiers->storage_class == sc_static)
+	pedwarn (decl_specifiers->locations[ds_storage_class],
+		 0, "for-range-declaration cannot be %qs",
+		 "static");
     }
 
   if (pushed_scope)
@@ -17664,9 +17775,7 @@ cp_parser_static_assert (cp_parser *parser, bool member_p)
 		 "only available with %<-std=c++17%> or %<-std=gnu++17%>");
       /* Eat the ')'  */
       cp_lexer_consume_token (parser->lexer);
-      message = build_string (1, "");
-      TREE_TYPE (message) = char_array_type_node;
-      fix_string_type (message);
+      message = build_empty_string ();
     }
   else
     {
@@ -21009,9 +21118,6 @@ cp_parser_simple_type_specifier (cp_parser* parser,
 			 "only available with "
 			 "%<-std=c++14%> or %<-std=gnu++14%>");
 	    }
-	  else if (parser->in_template_argument_list_p)
-	    error_at (token->location,
-		     "use of %<auto%> in template argument");
 	  else if (!flag_concepts)
 	    pedwarn (token->location, OPT_Wc__20_extensions,
 		     "use of %<auto%> in parameter declaration "
@@ -21021,6 +21127,11 @@ cp_parser_simple_type_specifier (cp_parser* parser,
 		     "use of %<auto%> in parameter declaration "
 		     "only available with "
 		     "%<-std=c++14%> or %<-std=gnu++14%>");
+
+	  if (parser->in_template_argument_list_p)
+	    permerror_opt (token->location,
+			   OPT_Wabbreviated_auto_in_template_arg,
+			   "use of %<auto%> in template argument");
 	}
       else
 	type = make_auto ();
@@ -21149,7 +21260,9 @@ cp_parser_simple_type_specifier (cp_parser* parser,
 						/*typename_keyword_p=*/false,
 						/*check_dependency_p=*/true,
 						/*type_p=*/false,
-						/*is_declaration=*/false)
+						/*is_declaration=*/false,
+						/*template_keyword_p=*/false,
+						global_p)
 	   != NULL_TREE);
       /* If we have seen a nested-name-specifier, and the next token
 	 is `template', then we are using the template-id production.  */
@@ -21467,6 +21580,10 @@ cp_parser_placeholder_type_specifier (cp_parser *parser, location_t loc,
 	  error_at (loc, "cannot declare a parameter with %<decltype(auto)%>");
 	  return error_mark_node;
 	}
+      if (parser->in_template_argument_list_p)
+	permerror_opt (placeholder->location,
+		       OPT_Wabbreviated_auto_in_template_arg,
+		       "use of %<auto%> in template argument");
       tree parm = build_constrained_parameter (con, proto, args);
       return synthesize_implicit_template_parm (parser, parm);
     }
@@ -21995,7 +22112,7 @@ cp_parser_elaborated_type_specifier (cp_parser* parser,
 
 	  bool template_p =
 	    (template_parm_lists_apply
-	     && (cp_parser_next_token_starts_class_definition_p (parser)
+	     && (cp_parser_nth_token_starts_class_definition_p (parser, 1)
 		 || cp_lexer_next_token_is (parser->lexer, CPP_SEMICOLON)));
 	  /* An unqualified name was used to reference this type, so
 	     there were no qualifying templates.  */
@@ -24159,7 +24276,26 @@ cp_parser_init_declarator (cp_parser* parser,
 	  && token->type != CPP_SEMICOLON)
 	{
 	  if (maybe_range_for_decl && *maybe_range_for_decl != error_mark_node)
-	    range_for_decl_p = true;
+	    {
+	      range_for_decl_p = true;
+	      if (decl_spec_seq_has_spec_p (decl_specifiers, ds_thread))
+		pedwarn (decl_specifiers->locations[ds_thread],
+			 0, "for-range-declaration cannot be %qs",
+			 decl_specifiers->gnu_thread_keyword_p
+			 ? "__thread" : "thread_local");
+	      else if (decl_specifiers->storage_class == sc_static)
+		pedwarn (decl_specifiers->locations[ds_storage_class],
+			 0, "for-range-declaration cannot be %qs",
+			 "static");
+	      else if (decl_specifiers->storage_class == sc_extern)
+		pedwarn (decl_specifiers->locations[ds_storage_class],
+			 0, "for-range-declaration cannot be %qs",
+			 "extern");
+	      else if (decl_specifiers->storage_class == sc_register)
+		pedwarn (decl_specifiers->locations[ds_storage_class],
+			 0, "for-range-declaration cannot be %qs",
+			 "register");
+	    }
 	  else
 	    {
 	      if (!maybe_range_for_decl)
@@ -28013,6 +28149,98 @@ cp_parser_class_specifier (cp_parser* parser)
   return type;
 }
 
+/* Parse an (optional) class-property-specifier-seq.
+
+   class-property-specifier-seq:
+     class-property-specifier class-property-specifier-seq [opt]
+
+   class-property-specifier:
+     final
+     trivially_relocatable_if_eligible (C++26)
+     replaceable_if_eligible (C++26)
+
+   Returns a bitmask representing the class-property-specifiers.  */
+
+static cp_virt_specifiers
+cp_parser_class_property_specifier_seq_opt (cp_parser *parser)
+{
+  cp_virt_specifiers virt_specifiers = VIRT_SPEC_UNSPECIFIED;
+
+  while (true)
+    {
+      cp_token *token;
+      cp_virt_specifiers virt_specifier;
+
+      /* Peek at the next token.  */
+      token = cp_lexer_peek_token (parser->lexer);
+      /* See if it's a class-property-specifier.  */
+      if (token->type != CPP_NAME)
+	break;
+      if (id_equal (token->u.value, "final"))
+	{
+	  /* For C++98, quietly ignore final in e.g.
+	     struct S final = 24;  */
+	  if (cxx_dialect == cxx98
+	      && virt_specifiers == VIRT_SPEC_UNSPECIFIED
+	      && !cp_parser_nth_token_starts_class_definition_p (parser, 2)
+	      && !cp_lexer_nth_token_is (parser->lexer, 2, CPP_NAME))
+	    break;
+	  maybe_warn_cpp0x (CPP0X_OVERRIDE_CONTROLS);
+	  virt_specifier = VIRT_SPEC_FINAL;
+	}
+      else if (id_equal (token->u.value, "__final"))
+	virt_specifier = VIRT_SPEC_FINAL;
+      else if (id_equal (token->u.value, "trivially_relocatable_if_eligible"))
+	{
+	  if (cxx_dialect < cxx26)
+	    {
+	      /* Warn about the C++26 conditional keyword (but don't parse
+		 it).  */
+	      warning_at (token->location, OPT_Wc__26_compat,
+			  "identifier %qE is a conditional keyword in C++26",
+			  token->u.value);
+	      break;
+	    }
+	  virt_specifier = VIRT_SPEC_TRIVIALLY_RELOCATABLE_IF_ELIGIBLE;
+	}
+      else if (id_equal (token->u.value,
+			 "__trivially_relocatable_if_eligible"))
+	virt_specifier = VIRT_SPEC_TRIVIALLY_RELOCATABLE_IF_ELIGIBLE;
+      else if (id_equal (token->u.value, "replaceable_if_eligible"))
+	{
+	  if (cxx_dialect < cxx26)
+	    {
+	      /* Warn about the C++26 conditional keyword (but don't parse
+		 it).  */
+	      warning_at (token->location, OPT_Wc__26_compat,
+			  "identifier %qE is a conditional keyword in C++26",
+			  token->u.value);
+	      break;
+	    }
+	  virt_specifier = VIRT_SPEC_REPLACEABLE_IF_ELIGIBLE;
+	}
+      else if (id_equal (token->u.value,
+			 "__replaceable_if_eligible"))
+	virt_specifier = VIRT_SPEC_REPLACEABLE_IF_ELIGIBLE;
+      else
+	break;
+
+      if (virt_specifiers & virt_specifier)
+	{
+	  gcc_rich_location richloc (token->location);
+	  richloc.add_fixit_remove ();
+	  error_at (&richloc, "duplicate %qD specifier", token->u.value);
+	  cp_lexer_purge_token (parser->lexer);
+	}
+      else
+	{
+	  cp_lexer_consume_token (parser->lexer);
+	  virt_specifiers |= virt_specifier;
+	}
+    }
+  return virt_specifiers;
+}
+
 /* Parse a class-head.
 
    class-head:
@@ -28203,18 +28431,16 @@ cp_parser_class_head (cp_parser* parser,
   pop_deferring_access_checks ();
 
   if (id)
-    {
-      cp_parser_check_for_invalid_template_id (parser, id,
-					       class_key,
-                                               type_start_token->location);
-    }
-  virt_specifiers = cp_parser_virt_specifier_seq_opt (parser);
+    cp_parser_check_for_invalid_template_id (parser, id,
+					     class_key,
+					     type_start_token->location);
+  virt_specifiers = cp_parser_class_property_specifier_seq_opt (parser);
 
   /* If it's not a `:' or a `{' then we can't really be looking at a
      class-head, since a class-head only appears as part of a
      class-specifier.  We have to detect this situation before calling
      xref_tag, since that has irreversible side-effects.  */
-  if (!cp_parser_next_token_starts_class_definition_p (parser))
+  if (!cp_parser_nth_token_starts_class_definition_p (parser, 1))
     {
       cp_parser_error (parser, "expected %<{%> or %<:%>");
       type = error_mark_node;
@@ -28224,13 +28450,6 @@ cp_parser_class_head (cp_parser* parser,
   /* At this point, we're going ahead with the class-specifier, even
      if some other problem occurs.  */
   cp_parser_commit_to_tentative_parse (parser);
-  if (virt_specifiers & VIRT_SPEC_OVERRIDE)
-    {
-      cp_parser_error (parser,
-                       "cannot specify %<override%> for a class");
-      type = error_mark_node;
-      goto out;
-    }
   /* Issue the error about the overly-qualified name now.  */
   if (qualified_p)
     {
@@ -28558,6 +28777,16 @@ cp_parser_class_head (cp_parser* parser,
     DECL_SOURCE_LOCATION (TYPE_NAME (type)) = type_start_token->location;
   if (type && (virt_specifiers & VIRT_SPEC_FINAL))
     CLASSTYPE_FINAL (type) = 1;
+  if (type && (virt_specifiers & VIRT_SPEC_TRIVIALLY_RELOCATABLE_IF_ELIGIBLE))
+    {
+      gcc_assert (!CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED (type));
+      CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (type) = 1;
+    }
+  if (type && (virt_specifiers & VIRT_SPEC_REPLACEABLE_IF_ELIGIBLE))
+    {
+      gcc_assert (!CLASSTYPE_REPLACEABLE_COMPUTED (type));
+      CLASSTYPE_REPLACEABLE_BIT (type) = 1;
+    }
  out:
   parser->colon_corrects_to_scope_p = saved_colon_corrects_to_scope_p;
   return type;
@@ -28676,12 +28905,20 @@ cp_parser_member_specification_opt (cp_parser* parser)
 /* Parse a member-declaration.
 
    member-declaration:
-     decl-specifier-seq [opt] member-declarator-list [opt] ;
-     function-definition ; [opt]
-     :: [opt] nested-name-specifier template [opt] unqualified-id ;
+     attribute-specifier-seq [opt] decl-specifier-seq [opt]
+       member-declarator-list [opt] ;
+     function-definition
+     friend-type-declaration
      using-declaration
+     using-enum-declaration
+     static_assert-declaration
+     consteval-block-declaration
      template-declaration
+     explicit-specialization
+     deduction-guide
      alias-declaration
+     opaque-enum-declaration
+     empty-declaration
 
    member-declarator-list:
      member-declarator
@@ -28700,12 +28937,7 @@ cp_parser_member_specification_opt (cp_parser* parser)
    member-declarator:
      declarator attributes [opt] pure-specifier [opt]
      declarator attributes [opt] constant-initializer [opt]
-     identifier [opt] attributes [opt] : constant-expression
-
-   C++0x Extensions:
-
-   member-declaration:
-     static_assert-declaration  */
+     identifier [opt] attributes [opt] : constant-expression  */
 
 static void
 cp_parser_member_declaration (cp_parser* parser)
@@ -28718,16 +28950,17 @@ cp_parser_member_declaration (cp_parser* parser)
   cp_token *token = NULL;
   cp_token *decl_spec_token_start = NULL;
   cp_token *initializer_token_start = NULL;
-  int saved_pedantic;
+  int saved_pedantic, saved_long_long;
   bool saved_colon_corrects_to_scope_p = parser->colon_corrects_to_scope_p;
 
   /* Check for the `__extension__' keyword.  */
-  if (cp_parser_extension_opt (parser, &saved_pedantic))
+  if (cp_parser_extension_opt (parser, &saved_pedantic, &saved_long_long))
     {
       /* Recurse.  */
       cp_parser_member_declaration (parser);
       /* Restore the old value of the PEDANTIC flag.  */
       pedantic = saved_pedantic;
+      warn_long_long = saved_long_long;
 
       return;
     }
@@ -28804,6 +29037,12 @@ cp_parser_member_declaration (cp_parser* parser)
       return;
     }
 
+  if (cp_parser_next_tokens_are_consteval_block_p (parser))
+    {
+      cp_parser_consteval_block (parser, /*member_p=*/true);
+      return;
+    }
+
   parser->colon_corrects_to_scope_p = false;
 
   cp_omp_declare_simd_data odsd;
@@ -31869,13 +32108,16 @@ cp_parser_skip_attributes_opt (cp_parser *parser, size_t n)
    present, and FALSE otherwise.  *SAVED_PEDANTIC is set to the
    current value of the PEDANTIC flag, regardless of whether or not
    the `__extension__' keyword is present.  The caller is responsible
-   for restoring the value of the PEDANTIC flag.  */
+   for restoring the value of the PEDANTIC flag.  Similarly *SAVED_LONG_LONG
+   for warn_long_long flag.  */
 
 static bool
-cp_parser_extension_opt (cp_parser* parser, int* saved_pedantic)
+cp_parser_extension_opt (cp_parser *parser, int *saved_pedantic,
+			 int *saved_long_long)
 {
   /* Save the old value of the PEDANTIC flag.  */
   *saved_pedantic = pedantic;
+  *saved_long_long = warn_long_long;
 
   if (cp_lexer_next_token_is_keyword (parser->lexer, RID_EXTENSION))
     {
@@ -31884,6 +32126,8 @@ cp_parser_extension_opt (cp_parser* parser, int* saved_pedantic)
       /* We're not being pedantic while the `__extension__' keyword is
 	 in effect.  */
       pedantic = 0;
+      /* And we don't want -Wlong-long warning.  */
+      warn_long_long = 0;
 
       return true;
     }
@@ -33558,7 +33802,9 @@ cp_parser_constructor_declarator_p (cp_parser *parser, cp_parser_flags flags,
       && cp_lexer_peek_token (parser->lexer)->type == CPP_TEMPLATE_ID)
     {
       auto_diagnostic_group d;
-      if (emit_diagnostic (cxx_dialect >= cxx20 ? DK_PEDWARN : DK_WARNING,
+      if (emit_diagnostic ((cxx_dialect >= cxx20
+			    ? diagnostics::kind::pedwarn
+			    : diagnostics::kind::warning),
 			   input_location, OPT_Wtemplate_id_cdtor,
 			   "template-id not allowed for constructor in C++20"))
 	inform (input_location, "remove the %qs", "< >");
@@ -35599,15 +35845,15 @@ cp_parser_token_starts_function_definition_p (cp_token* token)
 	  || token->keyword == RID_RETURN);
 }
 
-/* Returns TRUE iff the next token is the ":" or "{" beginning a class
+/* Returns TRUE iff the Nth token is the ":" or "{" beginning a class
    definition.  */
 
 static bool
-cp_parser_next_token_starts_class_definition_p (cp_parser *parser)
+cp_parser_nth_token_starts_class_definition_p (cp_parser *parser, size_t n)
 {
   cp_token *token;
 
-  token = cp_lexer_peek_token (parser->lexer);
+  token = cp_lexer_peek_nth_token (parser->lexer, n);
   return (token->type == CPP_OPEN_BRACE
 	  || (token->type == CPP_COLON
 	      && !parser->colon_doesnt_start_class_def_p));
@@ -35887,7 +36133,9 @@ cp_parser_check_class_key (cp_parser *parser, location_t key_loc,
     return;
 
   bool seen_as_union = TREE_CODE (type) == UNION_TYPE;
-  if (seen_as_union != (class_key == union_type))
+  if (class_key != typename_type
+      && TREE_CODE (type) != TYPENAME_TYPE
+      && seen_as_union != (class_key == union_type))
     {
       auto_diagnostic_group d;
       if (permerror (input_location, "%qs tag used in naming %q#T",
@@ -36661,7 +36909,7 @@ static void
 cp_parser_abort_tentative_parse (cp_parser* parser)
 {
   gcc_assert (parser->context->status != CP_PARSER_STATUS_KIND_COMMITTED
-	      || errorcount > 0);
+	      || seen_error ());
   cp_parser_simulate_error (parser);
   /* Now, pretend that we want to see if the construct was
      successfully parsed.  */
@@ -52898,11 +53146,14 @@ cp_parser_omp_error (cp_parser *parser, cp_token *pragma_tok,
       if (msg == NULL)
 	msg = _("<message unknown at compile time>");
     }
+  const enum diagnostics::kind diag_kind = (severity_fatal
+					    ? diagnostics::kind::error
+					    : diagnostics::kind::warning);
   if (msg)
-    emit_diagnostic (severity_fatal ? DK_ERROR : DK_WARNING, loc, 0,
+    emit_diagnostic (diag_kind, loc, 0,
 		     "%<pragma omp error%> encountered: %s", msg);
   else
-    emit_diagnostic (severity_fatal ? DK_ERROR : DK_WARNING, loc, 0,
+    emit_diagnostic (diag_kind, loc, 0,
 		     "%<pragma omp error%> encountered");
   return false;
 }
diff --git a/gcc/cp/pt.cc b/gcc/cp/pt.cc
index 3362a6f..acfeb81 100644
--- a/gcc/cp/pt.cc
+++ b/gcc/cp/pt.cc
@@ -6952,14 +6952,22 @@ convert_nontype_argument_function (tree type, tree expr,
 	{
 	  auto_diagnostic_group d;
 	  location_t loc = cp_expr_loc_or_input_loc (expr);
-	  error_at (loc, "%qE is not a valid template argument for type %qT",
-		    expr, type);
-	  if (TYPE_PTR_P (type))
-	    inform (loc, "it must be the address of a function "
-		    "with external linkage");
+	  tree c;
+	  if (cxx_dialect >= cxx17
+	      && (c = cxx_constant_value (fn),
+		  c == error_mark_node))
+	    ;
 	  else
-	    inform (loc, "it must be the name of a function with "
-		    "external linkage");
+	    {
+	      error_at (loc, "%qE is not a valid template argument for "
+			"type %qT", expr, type);
+	      if (TYPE_PTR_P (type))
+		inform (loc, "it must be the address of a function "
+			"with external linkage");
+	      else
+		inform (loc, "it must be the name of a function with "
+			"external linkage");
+	    }
 	}
       return NULL_TREE;
     }
@@ -7402,22 +7410,22 @@ invalid_tparm_referent_p (tree type, tree expr, tsubst_flags_t complain)
 	/* Null pointer values are OK in C++11.  */;
       else
 	{
-	  if (VAR_P (expr))
-	    {
-	      if (complain & tf_error)
-		error ("%qD is not a valid template argument "
-		       "because %qD is a variable, not the address of "
-		       "a variable", expr, expr);
-	      return true;
-	    }
+	  tree c;
+	  if (!(complain & tf_error))
+	    ;
+	  else if (cxx_dialect >= cxx17
+		   && (c = cxx_constant_value (expr),
+		       c == error_mark_node))
+	    ;
+	  else if (VAR_P (expr))
+	    error ("%qD is not a valid template argument "
+		   "because %qD is a variable, not the address of "
+		   "a variable", expr, expr);
 	  else
-	    {
-	      if (complain & tf_error)
-		error ("%qE is not a valid template argument for %qT "
-		       "because it is not the address of a variable",
-		       expr, type);
-	      return true;
-	    }
+	    error ("%qE is not a valid template argument for %qT "
+		   "because it is not the address of a variable",
+		   expr, type);
+	  return true;
 	}
     }
   return false;
@@ -12699,7 +12707,17 @@ instantiate_class_template (tree type)
       determine_visibility (TYPE_MAIN_DECL (type));
     }
   if (CLASS_TYPE_P (type))
-    CLASSTYPE_FINAL (type) = CLASSTYPE_FINAL (pattern);
+    {
+      CLASSTYPE_FINAL (type) = CLASSTYPE_FINAL (pattern);
+      CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (type)
+	= CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (pattern);
+      CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED (type)
+	= CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED (pattern);
+      CLASSTYPE_REPLACEABLE_BIT (type)
+	= CLASSTYPE_REPLACEABLE_BIT (pattern);
+      CLASSTYPE_REPLACEABLE_COMPUTED (type)
+	= CLASSTYPE_REPLACEABLE_COMPUTED (pattern);
+    }
 
   pbinfo = TYPE_BINFO (pattern);
 
@@ -14906,6 +14924,9 @@ tsubst_function_decl (tree t, tree args, tsubst_flags_t complain,
       argvec = NULL_TREE;
     }
 
+  /* Make sure tsubst_decl substitutes all the parameters.  */
+  cp_evaluated ev;
+
   tree closure = (lambda_fntype ? TYPE_METHOD_BASETYPE (lambda_fntype)
 		  : NULL_TREE);
   tree ctx = closure ? closure : DECL_CONTEXT (t);
@@ -17250,13 +17271,14 @@ tsubst (tree t, tree args, tsubst_flags_t complain, tree in_decl)
 	      return error_mark_node;
 	  }
 
-	/* FIXME: TYPENAME_IS_CLASS_P conflates 'class' vs 'struct' vs 'union'
-	   tags.  TYPENAME_TYPE should probably remember the exact tag that
-	   was written.  */
+	/* FIXME: TYPENAME_IS_CLASS_P conflates 'class' vs 'struct' tags.
+	   TYPENAME_TYPE should probably remember the exact tag that
+	   was written for -Wmismatched-tags.  */
 	enum tag_types tag_type
-	  = TYPENAME_IS_CLASS_P (t) ? class_type
-	  : TYPENAME_IS_ENUM_P (t) ? enum_type
-	  : typename_type;
+	  = (TYPENAME_IS_CLASS_P (t) ? class_type
+	     : TYPENAME_IS_UNION_P (t) ? union_type
+	     : TYPENAME_IS_ENUM_P (t) ? enum_type
+	     : typename_type);
 	tsubst_flags_t tcomplain = complain | tf_keep_type_decl;
 	tcomplain |= tst_ok_flag | qualifying_scope_flag;
 	f = make_typename_type (ctx, f, tag_type, tcomplain);
@@ -17278,10 +17300,18 @@ tsubst (tree t, tree args, tsubst_flags_t complain, tree in_decl)
 		else
 		  return error_mark_node;
 	      }
-	    else if (TYPENAME_IS_CLASS_P (t) && !CLASS_TYPE_P (f))
+	    else if (TYPENAME_IS_CLASS_P (t) && !NON_UNION_CLASS_TYPE_P (f))
 	      {
 		if (complain & tf_error)
-		  error ("%qT resolves to %qT, which is not a class type",
+		  error ("%qT resolves to %qT, which is not a non-union "
+			 "class type", t, f);
+		else
+		  return error_mark_node;
+	      }
+	    else if (TYPENAME_IS_UNION_P (t) && !UNION_TYPE_P (f))
+	      {
+		if (complain & tf_error)
+		  error ("%qT resolves to %qT, which is not a union type",
 			 t, f);
 		else
 		  return error_mark_node;
@@ -17967,9 +17997,7 @@ tsubst_omp_clause_decl (tree decl, tree args, tsubst_flags_t complain,
     return decl;
 
   /* Handle OpenMP iterators.  */
-  if (TREE_CODE (decl) == TREE_LIST
-      && TREE_PURPOSE (decl)
-      && TREE_CODE (TREE_PURPOSE (decl)) == TREE_VEC)
+  if (OMP_ITERATOR_DECL_P (decl))
     {
       tree ret;
       if (iterator_cache[0] == TREE_PURPOSE (decl))
@@ -19573,7 +19601,8 @@ tsubst_stmt (tree t, tree args, tsubst_flags_t complain, tree in_decl)
 
 	finish_static_assert (condition, message,
                               STATIC_ASSERT_SOURCE_LOCATION (t),
-			      /*member_p=*/false, /*show_expr_p=*/true);
+			      /*member_p=*/false, /*show_expr_p=*/true,
+			      CONSTEVAL_BLOCK_P (t));
       }
       break;
 
@@ -20138,7 +20167,14 @@ tsubst_stmt (tree t, tree args, tsubst_flags_t complain, tree in_decl)
       {
 	tree op0 = RECUR (TREE_OPERAND (t, 0));
 	tree cond = RECUR (MUST_NOT_THROW_COND (t));
-	RETURN (build_must_not_throw_expr (op0, cond));
+	stmt = build_must_not_throw_expr (op0, cond);
+	if (stmt && TREE_CODE (stmt) == MUST_NOT_THROW_EXPR)
+	  {
+	    MUST_NOT_THROW_NOEXCEPT_P (stmt) = MUST_NOT_THROW_NOEXCEPT_P (t);
+	    MUST_NOT_THROW_THROW_P (stmt) = MUST_NOT_THROW_THROW_P (t);
+	    MUST_NOT_THROW_CATCH_P (stmt) = MUST_NOT_THROW_CATCH_P (t);
+	  }
+	RETURN (stmt);
       }
 
     case EXPR_PACK_EXPANSION:
@@ -20470,11 +20506,6 @@ tsubst_lambda_expr (tree t, tree args, tsubst_flags_t complain, tree in_decl)
     r = error_mark_node;
   else
     {
-      /* The body of a lambda-expression is not a subexpression of the
-	 enclosing expression.  Parms are to have DECL_CHAIN tsubsted,
-	 which would be skipped if cp_unevaluated_operand.  */
-      cp_evaluated ev;
-
       /* Fix the type of 'this'.
 	 For static and xobj member functions we use this to transport the
 	 lambda's closure type.  It appears that in the regular case the
@@ -20500,6 +20531,10 @@ tsubst_lambda_expr (tree t, tree args, tsubst_flags_t complain, tree in_decl)
       /* Let finish_function set this.  */
       DECL_DECLARED_CONSTEXPR_P (fn) = false;
 
+      /* The body of a lambda-expression is not a subexpression of the
+	 enclosing expression.  */
+      cp_evaluated ev;
+
       bool nested = cfun;
       if (nested)
 	push_function_context ();
@@ -31190,14 +31225,8 @@ alias_ctad_tweaks (tree tmpl, tree uguides)
 	  /* Substitute the deduced arguments plus the rewritten template
 	     parameters into f to get g.  This covers the type, copyness,
 	     guideness, and explicit-specifier.  */
-	  tree g;
-	    {
-	      /* Parms are to have DECL_CHAIN tsubsted, which would be skipped
-		 if cp_unevaluated_operand.  */
-	      cp_evaluated ev;
-	      g = tsubst_decl (DECL_TEMPLATE_RESULT (f), targs, complain,
+	  tree g = tsubst_decl (DECL_TEMPLATE_RESULT (f), targs, complain,
 			       /*use_spec_table=*/false);
-	    }
 	  if (g == error_mark_node)
 	    continue;
 	  DECL_NAME (g) = name;
@@ -31627,7 +31656,9 @@ do_class_deduction (tree ptype, tree tmpl, tree init, tree outer_targs,
 	  /* Be permissive with equivalent alias templates.  */
 	  tree u = get_underlying_template (tmpl);
 	  auto_diagnostic_group d;
-	  diagnostic_t dk = (u == tmpl) ? DK_ERROR : DK_PEDWARN;
+	  const enum diagnostics::kind dk = ((u == tmpl)
+					     ? diagnostics::kind::error
+					     : diagnostics::kind::pedwarn);
 	  bool complained
 	    = emit_diagnostic (dk, input_location, 0,
 			       "alias template deduction only available "
diff --git a/gcc/cp/semantics.cc b/gcc/cp/semantics.cc
index 28baf7b..be79b50 100644
--- a/gcc/cp/semantics.cc
+++ b/gcc/cp/semantics.cc
@@ -2338,7 +2338,8 @@ finish_asm_stmt (location_t loc, int volatile_p, tree string,
 	  oconstraints[i] = constraint;
 
 	  if (parse_output_constraint (&constraint, i, ninputs, noutputs,
-				       &allows_mem, &allows_reg, &is_inout))
+				       &allows_mem, &allows_reg, &is_inout,
+				       nullptr))
 	    {
 	      /* If the operand is going to end up in memory,
 		 mark it addressable.  */
@@ -2397,7 +2398,8 @@ finish_asm_stmt (location_t loc, int volatile_p, tree string,
 	  constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
 	  bool constraint_parsed
 	    = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
-				      oconstraints, &allows_mem, &allows_reg);
+				      oconstraints, &allows_mem, &allows_reg,
+				      nullptr);
 	  /* If the operand is going to end up in memory, don't call
 	     decay_conversion.  */
 	  if (constraint_parsed && !allows_reg && allows_mem)
@@ -3605,11 +3607,13 @@ finish_this_expr (void)
 {
   tree result = NULL_TREE;
 
-  if (current_class_type && LAMBDA_TYPE_P (current_class_type))
+  if (current_class_ref && !LAMBDA_TYPE_P (TREE_TYPE (current_class_ref)))
+    result = current_class_ptr;
+  else if (current_class_type && LAMBDA_TYPE_P (current_class_type))
     result = (lambda_expr_this_capture
 	      (CLASSTYPE_LAMBDA_EXPR (current_class_type), /*add*/true));
-  else if (current_class_ptr)
-    result = current_class_ptr;
+  else
+    gcc_checking_assert (!current_class_ptr);
 
   if (result)
     /* The keyword 'this' is a prvalue expression.  */
@@ -3739,6 +3743,11 @@ finish_unary_op_expr (location_t op_loc, enum tree_code code, cp_expr expr,
   if (!(complain & tf_warning))
     return result;
 
+  /* These will never fold into a constant, so no need to check for
+     overflow for them.  */
+  if (code == PREINCREMENT_EXPR || code == PREDECREMENT_EXPR)
+    return result;
+
   tree result_ovl = result;
   tree expr_ovl = expr;
 
@@ -3983,9 +3992,15 @@ finish_compound_literal (tree type, tree compound_literal,
 tree
 finish_fname (tree id)
 {
-  tree decl;
-
-  decl = fname_decl (input_location, C_RID_CODE (id), id);
+  tree decl = fname_decl (input_location, C_RID_CODE (id), id);
+  /* [expr.prim.lambda.closure]/16 "Unless the compound-statement is that
+     of a consteval-block-declaration, a variable __func__ is implicitly
+     defined...".  We could be in a consteval block in a function, though,
+     and then we shouldn't warn.  */
+  if (current_function_decl
+      && !current_nonlambda_function (/*only_skip_consteval_block_p=*/true))
+    pedwarn (input_location, 0, "%qD is not defined outside of function scope",
+	     decl);
   if (processing_template_decl && current_function_decl
       && decl != error_mark_node)
     decl = DECL_NAME (decl);
@@ -6295,9 +6310,7 @@ handle_omp_array_sections (tree &c, enum c_omp_region_type ort)
   tree *tp = &OMP_CLAUSE_DECL (c);
   if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
        || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_AFFINITY)
-      && TREE_CODE (*tp) == TREE_LIST
-      && TREE_PURPOSE (*tp)
-      && TREE_CODE (TREE_PURPOSE (*tp)) == TREE_VEC)
+      && OMP_ITERATOR_DECL_P (*tp))
     tp = &TREE_VALUE (*tp);
   tree first = handle_omp_array_sections_1 (c, *tp, types,
 					    maybe_zero_len, first_non_one,
@@ -8817,9 +8830,7 @@ finish_omp_clauses (tree clauses, enum c_omp_region_type ort)
 	  /* FALLTHRU */
 	case OMP_CLAUSE_AFFINITY:
 	  t = OMP_CLAUSE_DECL (c);
-	  if (TREE_CODE (t) == TREE_LIST
-	      && TREE_PURPOSE (t)
-	      && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	  if (OMP_ITERATOR_DECL_P (t))
 	    {
 	      if (TREE_PURPOSE (t) != last_iterators)
 		last_iterators_remove
@@ -12593,11 +12604,14 @@ cexpr_str::extract (location_t location, const char * & msg, int &len)
    CONDITION and the message text MESSAGE.  LOCATION is the location
    of the static assertion in the source code.  When MEMBER_P, this
    static assertion is a member of a class.  If SHOW_EXPR_P is true,
-   print the condition (because it was instantiation-dependent).  */
+   print the condition (because it was instantiation-dependent).
+   If CONSTEVAL_BLOCK_P is true, this static assertion represents
+   a consteval block.  */
 
 void
 finish_static_assert (tree condition, tree message, location_t location,
-		      bool member_p, bool show_expr_p)
+		      bool member_p, bool show_expr_p,
+		      bool consteval_block_p/*=false*/)
 {
   tsubst_flags_t complain = tf_warning_or_error;
 
@@ -12625,6 +12639,7 @@ finish_static_assert (tree condition, tree message, location_t location,
       STATIC_ASSERT_CONDITION (assertion) = orig_condition;
       STATIC_ASSERT_MESSAGE (assertion) = cstr.message;
       STATIC_ASSERT_SOURCE_LOCATION (assertion) = location;
+      CONSTEVAL_BLOCK_P (assertion) = consteval_block_p;
 
       if (member_p)
         maybe_add_class_template_decl_list (current_class_type,
@@ -12636,6 +12651,13 @@ finish_static_assert (tree condition, tree message, location_t location,
       return;
     }
 
+  /* Evaluate the consteval { }.  This must be done only once.  */
+  if (consteval_block_p)
+    {
+      cxx_constant_value (condition);
+      return;
+    }
+
   /* Fold the expression and convert it to a boolean value. */
   condition = contextual_conv_bool (condition, complain);
   condition = fold_non_dependent_expr (condition, complain,
@@ -13359,6 +13381,18 @@ object_type_p (const_tree type)
           && !VOID_TYPE_P (type));
 }
 
+/* [defns.referenceable] True iff TYPE is a referenceable type.  */
+
+static bool
+referenceable_type_p (const_tree type)
+{
+  return (TYPE_REF_P (type)
+	  || object_type_p (type)
+	  || (FUNC_OR_METHOD_TYPE_P (type)
+	      && type_memfn_quals (type) == TYPE_UNQUALIFIED
+	      && type_memfn_rqual (type) == REF_QUAL_NONE));
+}
+
 /* Actually evaluates the trait.  */
 
 static bool
@@ -13528,6 +13562,21 @@ trait_expr_value (cp_trait_kind kind, tree type1, tree type2)
     case CPTK_IS_NOTHROW_INVOCABLE:
       return expr_noexcept_p (build_invoke (type1, type2, tf_none), tf_none);
 
+    case CPTK_IS_NOTHROW_RELOCATABLE:
+      if (trivially_relocatable_type_p (type1))
+	return true;
+      else
+	{
+	  type1 = strip_array_types (type1);
+	  if (!referenceable_type_p (type1))
+	    return false;
+	  tree arg = make_tree_vec (1);
+	  TREE_VEC_ELT (arg, 0)
+	    = cp_build_reference_type (type1, /*rval=*/true);
+	  return (is_nothrow_xible (INIT_EXPR, type1, arg)
+		  && is_nothrow_xible (BIT_NOT_EXPR, type1, NULL_TREE));
+	}
+
     case CPTK_IS_OBJECT:
       return object_type_p (type1);
 
@@ -13546,6 +13595,9 @@ trait_expr_value (cp_trait_kind kind, tree type1, tree type2)
     case CPTK_IS_REFERENCE:
       return type_code1 == REFERENCE_TYPE;
 
+    case CPTK_IS_REPLACEABLE:
+      return replaceable_type_p (type1);
+
     case CPTK_IS_SAME:
       return same_type_p (type1, type2);
 
@@ -13570,6 +13622,9 @@ trait_expr_value (cp_trait_kind kind, tree type1, tree type2)
     case CPTK_IS_TRIVIALLY_DESTRUCTIBLE:
       return is_trivially_xible (BIT_NOT_EXPR, type1, NULL_TREE);
 
+    case CPTK_IS_TRIVIALLY_RELOCATABLE:
+      return trivially_relocatable_type_p (type1);
+
     case CPTK_IS_UNBOUNDED_ARRAY:
       return array_of_unknown_bound_p (type1);
 
@@ -13653,7 +13708,8 @@ check_trait_type (tree type, int kind = 1)
 
   type = complete_type (strip_array_types (type));
   if (!COMPLETE_TYPE_P (type)
-      && cxx_incomplete_type_diagnostic (NULL_TREE, type, DK_PERMERROR)
+      && cxx_incomplete_type_diagnostic (NULL_TREE, type,
+					 diagnostics::kind::permerror)
       && !flag_permissive)
     return false;
   return true;
@@ -13700,18 +13756,6 @@ same_type_ref_bind_p (cp_trait_kind kind, tree type1, tree type2)
 	      (non_reference (to), non_reference (from))));
 }
 
-/* [defns.referenceable] True iff TYPE is a referenceable type.  */
-
-static bool
-referenceable_type_p (const_tree type)
-{
-  return (TYPE_REF_P (type)
-	  || object_type_p (type)
-	  || (FUNC_OR_METHOD_TYPE_P (type)
-	      && (type_memfn_quals (type) == TYPE_UNQUALIFIED
-		  && type_memfn_rqual (type) == REF_QUAL_NONE)));
-}
-
 /* Process a trait expression.  */
 
 tree
@@ -13760,8 +13804,11 @@ finish_trait_expr (location_t loc, cp_trait_kind kind, tree type1, tree type2)
     case CPTK_IS_LITERAL_TYPE:
     case CPTK_IS_POD:
     case CPTK_IS_STD_LAYOUT:
+    case CPTK_IS_REPLACEABLE:
+    case CPTK_IS_NOTHROW_RELOCATABLE:
     case CPTK_IS_TRIVIAL:
     case CPTK_IS_TRIVIALLY_COPYABLE:
+    case CPTK_IS_TRIVIALLY_RELOCATABLE:
     case CPTK_HAS_UNIQUE_OBJ_REPRESENTATIONS:
       if (!check_trait_type (type1, /* kind = */ 2))
 	return error_mark_node;
diff --git a/gcc/cp/tree.cc b/gcc/cp/tree.cc
index 5863b68..d56d73f 100644
--- a/gcc/cp/tree.cc
+++ b/gcc/cp/tree.cc
@@ -488,6 +488,7 @@ builtin_valid_in_constant_expr_p (const_tree decl)
 	  case CP_BUILT_IN_SOURCE_LOCATION:
 	  case CP_BUILT_IN_IS_CORRESPONDING_MEMBER:
 	  case CP_BUILT_IN_IS_POINTER_INTERCONVERTIBLE_WITH_CLASS:
+	  case CP_BUILT_IN_EH_PTR_ADJUST_REF:
 	    return true;
 	  default:
 	    break;
@@ -3695,6 +3696,58 @@ build_min_non_dep_op_overload (enum tree_code op,
   int nargs, expected_nargs;
   tree fn, call, obj = NULL_TREE;
 
+  releasing_vec args;
+  va_start (p, overload);
+
+  bool negated = false, rewritten = false, reversed = false;
+  if (cxx_dialect >= cxx20 && TREE_CODE (overload) == TREE_LIST)
+    {
+      /* Handle rebuilding a C++20 rewritten comparison operator expression,
+	 e.g. !(x == y), y <=> x, (x <=> y) @ 0 etc, that resolved to a call
+	 to a user-defined operator<=>/==.  */
+      gcc_checking_assert (TREE_CODE_CLASS (op) == tcc_comparison
+			   || op == SPACESHIP_EXPR);
+      int flags = TREE_INT_CST_LOW (TREE_PURPOSE (overload));
+      if (TREE_CODE (non_dep) == TRUTH_NOT_EXPR)
+	{
+	  negated = true;
+	  non_dep = TREE_OPERAND (non_dep, 0);
+	}
+      if (flags & LOOKUP_REWRITTEN)
+	rewritten = true;
+      if (flags & LOOKUP_REVERSED)
+	reversed = true;
+      if (rewritten
+	  && DECL_OVERLOADED_OPERATOR_IS (TREE_VALUE (overload),
+					  SPACESHIP_EXPR))
+	{
+	  /* Handle (x <=> y) @ 0 and 0 @ (y <=> x) by recursing to first
+	     rebuild the <=>.  Note that both OVERLOAD and the provided arguments
+	     in this case already correspond to the selected operator<=>.  */
+
+	  tree spaceship_non_dep = CALL_EXPR_ARG (non_dep, reversed ? 1 : 0);
+	  gcc_checking_assert (TREE_CODE (spaceship_non_dep) == CALL_EXPR);
+	  tree spaceship_op0 = va_arg (p, tree);
+	  tree spaceship_op1 = va_arg (p, tree);
+	  if (reversed)
+	    std::swap (spaceship_op0, spaceship_op1);
+
+	  /* Push the correct arguments for the operator OP expression, and
+	     set OVERLOAD appropriately.  */
+	  tree op0 = build_min_non_dep_op_overload (SPACESHIP_EXPR,
+						    spaceship_non_dep,
+						    TREE_VALUE (overload),
+						    spaceship_op0,
+						    spaceship_op1);
+	  tree op1 = CALL_EXPR_ARG (non_dep, reversed ? 0 : 1);
+	  gcc_checking_assert (integer_zerop (op1));
+	  vec_safe_push (args, op0);
+	  vec_safe_push (args, op1);
+	  overload = CALL_EXPR_FN (non_dep);
+	}
+      else
+	overload = TREE_VALUE (overload);
+    }
   non_dep = extract_call_expr (non_dep);
 
   nargs = call_expr_nargs (non_dep);
@@ -3715,32 +3768,40 @@ build_min_non_dep_op_overload (enum tree_code op,
     expected_nargs += 1;
   gcc_assert (nargs == expected_nargs);
 
-  releasing_vec args;
-  va_start (p, overload);
-
   if (!DECL_OBJECT_MEMBER_FUNCTION_P (overload))
     {
       fn = overload;
-      if (op == ARRAY_REF)
-	obj = va_arg (p, tree);
-      for (int i = 0; i < nargs; i++)
+      if (vec_safe_length (args) != 0)
+	/* The correct arguments were already pushed above.  */
+	gcc_checking_assert (rewritten);
+      else
 	{
-	  tree arg = va_arg (p, tree);
-	  vec_safe_push (args, arg);
+	  if (op == ARRAY_REF)
+	    obj = va_arg (p, tree);
+	  for (int i = 0; i < nargs; i++)
+	    {
+	      tree arg = va_arg (p, tree);
+	      vec_safe_push (args, arg);
+	    }
 	}
+      if (reversed)
+	std::swap ((*args)[0], (*args)[1]);
     }
   else
     {
+      gcc_checking_assert (vec_safe_length (args) == 0);
       tree object = va_arg (p, tree);
-      tree binfo = TYPE_BINFO (TREE_TYPE (object));
-      tree method = build_baselink (binfo, binfo, overload, NULL_TREE);
-      fn = build_min (COMPONENT_REF, TREE_TYPE (overload),
-		      object, method, NULL_TREE);
       for (int i = 0; i < nargs; i++)
 	{
 	  tree arg = va_arg (p, tree);
 	  vec_safe_push (args, arg);
 	}
+      if (reversed)
+	std::swap (object, (*args)[0]);
+      tree binfo = TYPE_BINFO (TREE_TYPE (object));
+      tree method = build_baselink (binfo, binfo, overload, NULL_TREE);
+      fn = build_min (COMPONENT_REF, TREE_TYPE (overload),
+		      object, method, NULL_TREE);
     }
 
   va_end (p);
@@ -3752,6 +3813,8 @@ build_min_non_dep_op_overload (enum tree_code op,
   CALL_EXPR_ORDERED_ARGS (call_expr) = CALL_EXPR_ORDERED_ARGS (non_dep);
   CALL_EXPR_REVERSE_ARGS (call_expr) = CALL_EXPR_REVERSE_ARGS (non_dep);
 
+  if (negated)
+    call = build_min (TRUTH_NOT_EXPR, boolean_type_node, call);
   if (obj)
     return keep_unused_object_arg (call, obj, overload);
   return call;
@@ -4715,6 +4778,293 @@ trivial_type_p (const_tree t)
     return scalarish_type_p (t);
 }
 
+/* Returns 1 iff type T is a default-movable type, as defined in
+   [class.prop].  */
+
+static bool
+default_movable_type_p (tree t)
+{
+  if (!CLASS_TYPE_P (t) || !COMPLETE_TYPE_P (t))
+    return false;
+  if (CLASSTYPE_LAZY_DESTRUCTOR (t))
+    lazily_declare_fn (sfk_destructor, t);
+  if (tree dtor = CLASSTYPE_DESTRUCTOR (t))
+    if (user_provided_p (dtor) || DECL_DELETED_FN (dtor))
+      return false;
+
+  tree copy_ctor = NULL_TREE, move_ctor = NULL_TREE;
+  tree copy_assign = NULL_TREE, move_assign = NULL_TREE;
+  if (CLASSTYPE_LAZY_MOVE_CTOR (t))
+    move_ctor = lazily_declare_fn (sfk_move_constructor, t);
+  if (CLASSTYPE_LAZY_MOVE_ASSIGN (t))
+    move_assign = lazily_declare_fn (sfk_move_assignment, t);
+  if (!move_ctor)
+    for (ovl_iterator iter (CLASSTYPE_CONSTRUCTORS (t)); iter; ++iter)
+      if (TREE_CODE (*iter) == FUNCTION_DECL)
+	{
+	  if (copy_fn_p (*iter))
+	    copy_ctor = *iter;
+	  else if (move_fn_p (*iter))
+	    {
+	      move_ctor = *iter;
+	      break;
+	    }
+	}
+  if (!move_assign)
+    for (ovl_iterator iter (get_class_binding_direct (t,
+						      assign_op_identifier));
+	 iter; ++iter)
+      if (TREE_CODE (*iter) == FUNCTION_DECL)
+	{
+	  if (copy_fn_p (*iter))
+	    copy_assign = *iter;
+	  else if (move_fn_p (*iter))
+	    {
+	      move_assign = *iter;
+	      break;
+	    }
+	}
+  if (!move_ctor)
+    {
+      if (CLASSTYPE_LAZY_COPY_CTOR (t))
+	copy_ctor = lazily_declare_fn (sfk_copy_constructor, t);
+      if (!copy_ctor)
+	return false;
+      if (user_provided_p (copy_ctor)
+	  || DECL_DELETED_FN (copy_ctor)
+	  || DECL_CONTEXT (copy_ctor) != t
+	  || DECL_INHERITED_CTOR (copy_ctor))
+	return false;
+    }
+  else if (user_provided_p (move_ctor)
+	   || DECL_DELETED_FN (move_ctor)
+	   || DECL_CONTEXT (move_ctor) != t
+	   || DECL_INHERITED_CTOR (move_ctor))
+    return false;
+  if (!move_assign)
+    {
+      if (CLASSTYPE_LAZY_COPY_ASSIGN (t))
+	copy_assign = lazily_declare_fn (sfk_copy_assignment, t);
+      if (!copy_assign)
+	return false;
+      if (user_provided_p (copy_assign)
+	  || DECL_DELETED_FN (copy_assign)
+	  || DECL_CONTEXT (copy_assign) != t)
+	return false;
+    }
+  else if (user_provided_p (move_assign)
+	   || DECL_DELETED_FN (move_assign)
+	   || DECL_CONTEXT (move_assign) != t)
+    return false;
+  return true;
+}
+
+/* Returns 1 iff type T is a union with no user declared special member
+   functions.  */
+
+static bool
+union_with_no_declared_special_member_fns (tree t)
+{
+  if (TREE_CODE (t) != UNION_TYPE)
+    return false;
+
+  for (ovl_iterator iter (CLASSTYPE_CONSTRUCTORS (t)); iter; ++iter)
+    if (TREE_CODE (*iter) == FUNCTION_DECL
+	&& !DECL_ARTIFICIAL (*iter)
+	&& (default_ctor_p (*iter) || copy_fn_p (*iter) || move_fn_p (*iter)))
+      return false;
+
+  for (ovl_iterator iter (get_class_binding_direct (t, assign_op_identifier));
+       iter; ++iter)
+    if (TREE_CODE (*iter) == FUNCTION_DECL
+	&& !DECL_ARTIFICIAL (*iter)
+	&& (copy_fn_p (*iter) || move_fn_p (*iter)))
+      return false;
+
+  if (tree dtor = CLASSTYPE_DESTRUCTOR (t))
+    if (!DECL_ARTIFICIAL (dtor))
+      return false;
+
+  return true;
+}
+
+/* Returns 1 iff type T is a trivially relocatable type, as defined in
+   [basic.types.general] and [class.prop].  */
+
+bool
+trivially_relocatable_type_p (tree t)
+{
+  t = strip_array_types (t);
+
+  if (!CLASS_TYPE_P (t))
+    return scalarish_type_p (t);
+
+  t = TYPE_MAIN_VARIANT (t);
+  if (CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED (t))
+    return CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (t);
+  if (!COMPLETE_TYPE_P (t))
+    return false;
+
+  if (!CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (t)
+      && !union_with_no_declared_special_member_fns (t)
+      && !default_movable_type_p (t))
+    {
+    nontriv:
+      CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (t) = 0;
+      CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED (t) = 1;
+      return false;
+    }
+
+  if (CLASSTYPE_VBASECLASSES (t))
+    goto nontriv;
+
+  if (CLASSTYPE_LAZY_DESTRUCTOR (t))
+    lazily_declare_fn (sfk_destructor, t);
+  if (tree dtor = CLASSTYPE_DESTRUCTOR (t))
+    if (DECL_DELETED_FN (dtor))
+      goto nontriv;
+
+  tree binfo, base_binfo;
+  unsigned int i;
+  for (binfo = TYPE_BINFO (t), i = 0;
+       BINFO_BASE_ITERATE (binfo, i, base_binfo); i++)
+    {
+      tree basetype = TREE_TYPE (base_binfo);
+      if (!trivially_relocatable_type_p (basetype))
+	goto nontriv;
+    }
+
+  for (tree field = TYPE_FIELDS (t); field; field = DECL_CHAIN (field))
+    if (TREE_CODE (field) == FIELD_DECL
+	&& !DECL_ARTIFICIAL (field)
+	&& !DECL_UNNAMED_BIT_FIELD (field))
+      {
+	tree type = TREE_TYPE (field);
+	if (type == error_mark_node)
+	  goto nontriv;
+	if (!TYPE_REF_P (type) && !trivially_relocatable_type_p (type))
+	  goto nontriv;
+      }
+
+  CLASSTYPE_TRIVIALLY_RELOCATABLE_BIT (t) = 1;
+  CLASSTYPE_TRIVIALLY_RELOCATABLE_COMPUTED (t) = 1;
+  return true;
+}
+
+/* Returns 1 iff type T is a replaceable type, as defined in [basic.types]
+   and [class].  */
+
+bool
+replaceable_type_p (tree t)
+{
+  t = strip_array_types (t);
+
+  if (cv_qualified_p (t))
+    return false;
+
+  if (!CLASS_TYPE_P (t))
+    return scalarish_type_p (t);
+
+  t = TYPE_MAIN_VARIANT (t);
+  if (CLASSTYPE_REPLACEABLE_COMPUTED (t))
+    return CLASSTYPE_REPLACEABLE_BIT (t);
+  if (!COMPLETE_TYPE_P (t))
+    return false;
+
+  if (!CLASSTYPE_REPLACEABLE_BIT (t)
+      && !union_with_no_declared_special_member_fns (t)
+      && !default_movable_type_p (t))
+    {
+    nonrepl:
+      CLASSTYPE_REPLACEABLE_BIT (t) = 0;
+      CLASSTYPE_REPLACEABLE_COMPUTED (t) = 1;
+      return false;
+    }
+
+  if (CLASSTYPE_LAZY_DESTRUCTOR (t))
+    lazily_declare_fn (sfk_destructor, t);
+  if (tree dtor = CLASSTYPE_DESTRUCTOR (t))
+    if (DECL_DELETED_FN (dtor))
+      goto nonrepl;
+
+  tree copy_ctor = NULL_TREE, move_ctor = NULL_TREE;
+  tree copy_assign = NULL_TREE, move_assign = NULL_TREE;
+  if (CLASSTYPE_LAZY_MOVE_CTOR (t))
+    move_ctor = lazily_declare_fn (sfk_move_constructor, t);
+  if (CLASSTYPE_LAZY_MOVE_ASSIGN (t))
+    move_assign = lazily_declare_fn (sfk_move_assignment, t);
+  if (!move_ctor)
+    for (ovl_iterator iter (CLASSTYPE_CONSTRUCTORS (t)); iter; ++iter)
+      if (TREE_CODE (*iter) == FUNCTION_DECL)
+	{
+	  if (copy_fn_p (*iter))
+	    copy_ctor = *iter;
+	  else if (move_fn_p (*iter))
+	    {
+	      move_ctor = *iter;
+	      break;
+	    }
+	}
+  if (!move_assign)
+    for (ovl_iterator iter (get_class_binding_direct (t,
+						      assign_op_identifier));
+	 iter; ++iter)
+      if (TREE_CODE (*iter) == FUNCTION_DECL)
+	{
+	  if (copy_fn_p (*iter))
+	    copy_assign = *iter;
+	  else if (move_fn_p (*iter))
+	    {
+	      move_assign = *iter;
+	      break;
+	    }
+	}
+  if (!move_ctor)
+    {
+      if (CLASSTYPE_LAZY_COPY_CTOR (t))
+	copy_ctor = lazily_declare_fn (sfk_copy_constructor, t);
+      if (!copy_ctor || DECL_DELETED_FN (copy_ctor))
+	goto nonrepl;
+    }
+  else if (DECL_DELETED_FN (move_ctor))
+    goto nonrepl;
+  if (!move_assign)
+    {
+      if (CLASSTYPE_LAZY_COPY_ASSIGN (t))
+	copy_assign = lazily_declare_fn (sfk_copy_assignment, t);
+      if (!copy_assign || DECL_DELETED_FN (copy_assign))
+	goto nonrepl;
+    }
+  else if (DECL_DELETED_FN (move_assign))
+    goto nonrepl;
+
+  tree binfo, base_binfo;
+  unsigned int i;
+  for (binfo = TYPE_BINFO (t), i = 0;
+       BINFO_BASE_ITERATE (binfo, i, base_binfo); i++)
+    {
+      tree basetype = TREE_TYPE (base_binfo);
+      if (!replaceable_type_p (basetype))
+	goto nonrepl;
+    }
+
+  for (tree field = TYPE_FIELDS (t); field; field = DECL_CHAIN (field))
+    if (TREE_CODE (field) == FIELD_DECL
+	&& !DECL_ARTIFICIAL (field)
+	&& !DECL_UNNAMED_BIT_FIELD (field))
+      {
+	tree type = TREE_TYPE (field);
+	if (type == error_mark_node)
+	  goto nonrepl;
+	if (!replaceable_type_p (type))
+	  goto nonrepl;
+      }
+
+  CLASSTYPE_REPLACEABLE_BIT (t) = 1;
+  CLASSTYPE_REPLACEABLE_COMPUTED (t) = 1;
+  return true;
+}
+
 /* Returns 1 iff type T is a POD type, as defined in [basic.types].  */
 
 bool
diff --git a/gcc/cp/typeck.cc b/gcc/cp/typeck.cc
index 447fe81..f592894 100644
--- a/gcc/cp/typeck.cc
+++ b/gcc/cp/typeck.cc
@@ -155,7 +155,7 @@ complete_type_or_maybe_complain (tree type, tree value, tsubst_flags_t complain)
   else if (!COMPLETE_TYPE_P (type))
     {
       if (complain & tf_error)
-	cxx_incomplete_type_diagnostic (value, type, DK_ERROR);
+	cxx_incomplete_type_diagnostic (value, type, diagnostics::kind::error);
       note_failed_type_completion (type, complain);
       return NULL_TREE;
     }
@@ -618,7 +618,7 @@ type_after_usual_arithmetic_conversions (tree t1, tree t2)
 
 static void
 composite_pointer_error (const op_location_t &location,
-			 diagnostic_t kind, tree t1, tree t2,
+			 enum diagnostics::kind kind, tree t1, tree t2,
 			 composite_pointer_operation operation)
 {
   switch (operation)
@@ -690,7 +690,7 @@ composite_pointer_type_r (const op_location_t &location,
   else
     {
       if (complain & tf_error)
-	composite_pointer_error (location, DK_PERMERROR,
+	composite_pointer_error (location, diagnostics::kind::permerror,
 				 t1, t2, operation);
       else
 	return error_mark_node;
@@ -716,7 +716,7 @@ composite_pointer_type_r (const op_location_t &location,
 			TYPE_PTRMEM_CLASS_TYPE (t2)))
 	{
 	  if (complain & tf_error)
-	    composite_pointer_error (location, DK_PERMERROR,
+	    composite_pointer_error (location, diagnostics::kind::permerror,
 				     t1, t2, operation);
 	  else
 	    return error_mark_node;
@@ -851,7 +851,8 @@ composite_pointer_type (const op_location_t &location,
       else
         {
           if (complain & tf_error)
-	    composite_pointer_error (location, DK_ERROR, t1, t2, operation);
+	    composite_pointer_error (location, diagnostics::kind::error,
+				     t1, t2, operation);
           return error_mark_node;
         }
     }
@@ -4004,7 +4005,7 @@ cp_build_array_ref (location_t loc, tree array, tree idx,
       tree op0, op1, op2;
       op0 = TREE_OPERAND (array, 0);
       op1 = TREE_OPERAND (array, 1);
-      op2 = TREE_OPERAND (array, 1);
+      op2 = TREE_OPERAND (array, 2);
       if (TREE_SIDE_EFFECTS (idx) || !tree_invariant_p (idx))
 	{
 	  /* If idx could possibly have some SAVE_EXPRs, turning
@@ -4530,7 +4531,11 @@ cp_build_function_call_vec (tree function, vec<tree, va_gc> **params,
     {
       if (complain & tf_error)
 	{
-	  if (!flag_diagnostics_show_caret)
+	  if (is_stub_object (original))
+	    error_at (input_location,
+		      "%qT cannot be used as a function",
+		      TREE_TYPE (original));
+	  else if (!flag_diagnostics_show_caret)
 	    error_at (input_location,
 		      "%qE cannot be used as a function", original);
 	  else if (DECL_P (original))
@@ -4672,12 +4677,8 @@ convert_arguments (tree typelist, vec<tree, va_gc> **values, tree fndecl,
       if (type == void_type_node)
 	{
           if (complain & tf_error)
-            {
-	      error_args_num (input_location, fndecl, /*too_many_p=*/true);
-              return i;
-            }
-          else
-            return -1;
+	    error_args_num (input_location, fndecl, /*too_many_p=*/true);
+	  return -1;
 	}
 
       /* build_c_cast puts on a NOP_EXPR to make the result not an lvalue.
@@ -7680,7 +7681,18 @@ cp_build_unary_op (enum tree_code code, tree xarg, bool noconvert,
       if (val != 0)
 	goto return_build_unary_op;
 
-      arg = mark_lvalue_use (arg);
+      tree stripped_arg;
+      stripped_arg = tree_strip_any_location_wrapper (arg);
+      if ((VAR_P (stripped_arg) || TREE_CODE (stripped_arg) == PARM_DECL)
+	  && !DECL_READ_P (stripped_arg)
+	  && (VAR_P (stripped_arg) ? warn_unused_but_set_variable
+				   : warn_unused_but_set_parameter) > 1)
+	{
+	  arg = mark_lvalue_use (arg);
+	  DECL_READ_P (stripped_arg) = 0;
+	}
+      else
+	arg = mark_lvalue_use (arg);
 
       /* Increment or decrement the real part of the value,
 	 and don't change the imaginary part.  */
@@ -9796,7 +9808,22 @@ cp_build_modify_expr (location_t loc, tree lhs, enum tree_code modifycode,
 	  {
 	    auto_diagnostic_group d;
 	    rhs = stabilize_expr (rhs, &init);
+	    bool clear_decl_read = false;
+	    tree stripped_lhs = tree_strip_any_location_wrapper (lhs);
+	    if ((VAR_P (stripped_lhs) || TREE_CODE (stripped_lhs) == PARM_DECL)
+		&& !DECL_READ_P (stripped_lhs)
+		&& (VAR_P (stripped_lhs) ? warn_unused_but_set_variable
+					 : warn_unused_but_set_parameter) > 2
+		&& !CLASS_TYPE_P (TREE_TYPE (lhs))
+		&& !CLASS_TYPE_P (TREE_TYPE (rhs)))
+	      {
+		mark_exp_read (rhs);
+		if (!DECL_READ_P (stripped_lhs))
+		  clear_decl_read = true;
+	      }
 	    newrhs = cp_build_binary_op (loc, modifycode, lhs, rhs, complain);
+	    if (clear_decl_read)
+	      DECL_READ_P (stripped_lhs) = 0;
 	    if (newrhs == error_mark_node)
 	      {
 		if (complain & tf_error)
@@ -10779,7 +10806,9 @@ maybe_warn_about_returning_address_of_local (tree retval, location_t loc)
     {
       if (TYPE_REF_P (valtype))
 	/* P2748 made this an error in C++26.  */
-	emit_diagnostic (cxx_dialect >= cxx26 ? DK_PERMERROR : DK_WARNING,
+	emit_diagnostic ((cxx_dialect >= cxx26
+			  ? diagnostics::kind::permerror
+			  : diagnostics::kind::warning),
 			 loc, OPT_Wreturn_local_addr,
 			 "returning reference to temporary");
       else if (TYPE_PTR_P (valtype))
diff --git a/gcc/cp/typeck2.cc b/gcc/cp/typeck2.cc
index 45edd18..faaf1df 100644
--- a/gcc/cp/typeck2.cc
+++ b/gcc/cp/typeck2.cc
@@ -119,6 +119,11 @@ cxx_readonly_error (location_t loc, tree arg, enum lvalue_use errstring)
 			  G_("increment of read-only reference %qD"),
 			  G_("decrement of read-only reference %qD"),
 			  TREE_OPERAND (arg, 0));
+  else if (is_stub_object (arg))
+    {
+      gcc_assert (errstring == lv_assign);
+      error_at (loc, "assignment to read-only type %qT", TREE_TYPE (arg));
+    }
   else
     readonly_error (loc, arg, errstring);
 }
@@ -290,11 +295,12 @@ cxx_incomplete_type_inform (const_tree type)
 /* Print an error message for invalid use of an incomplete type.
    VALUE is the expression that was used (or 0 if that isn't known)
    and TYPE is the type that was invalid.  DIAG_KIND indicates the
-   type of diagnostic (see diagnostic.def).  */
+   type of diagnostic (see diagnostics/kinds.def).  */
 
 bool
 cxx_incomplete_type_diagnostic (location_t loc, const_tree value,
-				const_tree type, diagnostic_t diag_kind)
+				const_tree type,
+				enum diagnostics::kind diag_kind)
 {
   bool is_decl = false, complained = false;
 
@@ -440,7 +446,7 @@ cxx_incomplete_type_diagnostic (location_t loc, const_tree value,
 void
 cxx_incomplete_type_error (location_t loc, const_tree value, const_tree type)
 {
-  cxx_incomplete_type_diagnostic (loc, value, type, DK_ERROR);
+  cxx_incomplete_type_diagnostic (loc, value, type, diagnostics::kind::error);
 }
 
 
@@ -2627,7 +2633,7 @@ add_exception_specifier (tree list, tree spec, tsubst_flags_t complain)
   bool ok;
   tree core = spec;
   bool is_ptr;
-  diagnostic_t diag_type = DK_UNSPECIFIED; /* none */
+  enum diagnostics::kind diag_type = diagnostics::kind::unspecified; /* none */
 
   if (spec == error_mark_node)
     return list;
@@ -2659,7 +2665,7 @@ add_exception_specifier (tree list, tree spec, tsubst_flags_t complain)
 	 and calls.  So just give a pedwarn at this point; we will give an
 	 error later if we hit one of those two cases.  */
       if (!COMPLETE_TYPE_P (complete_type (core)))
-	diag_type = DK_PEDWARN; /* pedwarn */
+	diag_type = diagnostics::kind::pedwarn; /* pedwarn */
     }
 
   if (ok)
@@ -2673,9 +2679,9 @@ add_exception_specifier (tree list, tree spec, tsubst_flags_t complain)
 	list = tree_cons (NULL_TREE, spec, list);
     }
   else
-    diag_type = DK_ERROR; /* error */
+    diag_type = diagnostics::kind::error; /* error */
 
-  if (diag_type != DK_UNSPECIFIED
+  if (diag_type != diagnostics::kind::unspecified
       && (complain & tf_warning_or_error))
     cxx_incomplete_type_diagnostic (NULL_TREE, core, diag_type);
 
diff --git a/gcc/cprop.cc b/gcc/cprop.cc
index bc72e64..dfe3462 100644
--- a/gcc/cprop.cc
+++ b/gcc/cprop.cc
@@ -1525,6 +1525,7 @@ static bool
 bypass_block (basic_block bb, rtx_insn *setcc, rtx_insn *jump)
 {
   rtx_insn *insn;
+  rtx setcc_src, setcc_dest;
   rtx note;
   edge e, edest;
   bool change;
@@ -1533,7 +1534,19 @@ bypass_block (basic_block bb, rtx_insn *setcc, rtx_insn *jump)
   unsigned i;
   edge_iterator ei;
 
-  insn = (setcc != NULL) ? setcc : jump;
+  if (setcc != NULL)
+    {
+      rtx set = single_set (setcc);
+      setcc_dest = SET_DEST (set);
+      setcc_src = SET_SRC (set);
+      insn = setcc;
+    }
+  else
+    {
+      setcc_dest = NULL;
+      setcc_src = NULL;
+      insn = jump;
+    }
 
   /* Determine set of register uses in INSN.  */
   reg_use_count = 0;
@@ -1608,9 +1621,7 @@ bypass_block (basic_block bb, rtx_insn *setcc, rtx_insn *jump)
 	  src = SET_SRC (pc_set (jump));
 
 	  if (setcc != NULL)
-	    src = simplify_replace_rtx (src,
-					SET_DEST (PATTERN (setcc)),
-					SET_SRC (PATTERN (setcc)));
+	    src = simplify_replace_rtx (src, setcc_dest, setcc_src);
 
 	  new_rtx = simplify_replace_rtx (src, reg_used, set->src);
 
@@ -1716,10 +1727,11 @@ bypass_conditional_jumps (void)
 	      {
 		if (setcc)
 		  break;
-		if (GET_CODE (PATTERN (insn)) != SET)
+		rtx singleset = single_set (insn);
+		if (singleset == NULL_RTX)
 		  break;
 
-		dest = SET_DEST (PATTERN (insn));
+		dest = SET_DEST (singleset);
 		if (REG_P (dest))
 		  setcc = insn;
 		else
diff --git a/gcc/d/ChangeLog b/gcc/d/ChangeLog
index b6815d3..4fe6418 100644
--- a/gcc/d/ChangeLog
+++ b/gcc/d/ChangeLog
@@ -1,3 +1,22 @@
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* d-diagnostic.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* d-diagnostic.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* d-diagnostic.cc (d_diagnostic_report_diagnostic): Update to add
+	"m_" prefix to fields of diagnostic_info throughout.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* toir.cc: Pass null pointer to
+	parse_{input,output}_constraint().
+
 2025-04-29  Iain Buclaw  <ibuclaw@gdcproject.org>
 
 	PR d/103044
diff --git a/gcc/d/d-diagnostic.cc b/gcc/d/d-diagnostic.cc
index 55cb42e..e666ddf 100644
--- a/gcc/d/d-diagnostic.cc
+++ b/gcc/d/d-diagnostic.cc
@@ -185,7 +185,7 @@ escape_d_format (const char *format)
 static void ATTRIBUTE_GCC_DIAG(3,0)
 d_diagnostic_report_diagnostic (const SourceLoc &loc, int opt,
 				const char *format, va_list ap,
-				diagnostic_t kind, bool verbatim)
+				enum diagnostics::kind kind, bool verbatim)
 {
   va_list argp;
   va_copy (argp, ap);
@@ -193,13 +193,13 @@ d_diagnostic_report_diagnostic (const SourceLoc &loc, int opt,
   if (loc.filename.length != 0 || !verbatim)
     {
       rich_location rich_loc (line_table, make_location_t (loc));
-      diagnostic_info diagnostic;
+      diagnostics::diagnostic_info diagnostic;
       char *xformat = expand_d_format (format);
 
       diagnostic_set_info_translated (&diagnostic, xformat, &argp,
 				      &rich_loc, kind);
       if (opt != 0)
-	diagnostic.option_id = opt;
+	diagnostic.m_option_id = opt;
 
       diagnostic_report_diagnostic (global_dc, &diagnostic);
     }
@@ -224,7 +224,7 @@ void D_ATTRIBUTE_FORMAT(2,0) ATTRIBUTE_GCC_DIAG(2,0)
 verrorReport (const SourceLoc loc, const char *format, va_list ap,
 	      ErrorKind kind, const char *prefix1, const char *prefix2)
 {
-  diagnostic_t diag_kind = DK_UNSPECIFIED;
+  enum diagnostics::kind diag_kind = diagnostics::kind::unspecified;
   int opt = 0;
   bool verbatim = false;
   char *xformat;
@@ -238,7 +238,9 @@ verrorReport (const SourceLoc loc, const char *format, va_list ap,
       if (global.gag && !global.params.v.showGaggedErrors)
 	return;
 
-      diag_kind = global.gag ? DK_ANACHRONISM : DK_ERROR;
+      diag_kind = (global.gag
+		   ? diagnostics::kind::anachronism
+		   : diagnostics::kind::error);
     }
   else if (kind == ErrorKind::warning)
     {
@@ -254,7 +256,7 @@ verrorReport (const SourceLoc loc, const char *format, va_list ap,
       if (global.params.useWarnings == DIAGNOSTICerror)
 	global.warnings++;
 
-      diag_kind = DK_WARNING;
+      diag_kind = diagnostics::kind::warning;
     }
   else if (kind == ErrorKind::deprecation)
     {
@@ -270,11 +272,11 @@ verrorReport (const SourceLoc loc, const char *format, va_list ap,
 	}
 
       opt = OPT_Wdeprecated;
-      diag_kind = DK_WARNING;
+      diag_kind = diagnostics::kind::warning;
     }
   else if (kind == ErrorKind::message)
     {
-      diag_kind = DK_NOTE;
+      diag_kind = diagnostics::kind::note;
       verbatim = true;
     }
   else if (kind == ErrorKind::tip)
@@ -282,7 +284,7 @@ verrorReport (const SourceLoc loc, const char *format, va_list ap,
       if (global.gag)
 	return;
 
-      diag_kind = DK_DEBUG;
+      diag_kind = diagnostics::kind::debug;
       verbatim = true;
     }
   else
@@ -328,7 +330,8 @@ verrorReportSupplemental (const SourceLoc loc, const char* format, va_list ap,
   else
     gcc_unreachable ();
 
-  d_diagnostic_report_diagnostic (loc, 0, format, ap, DK_NOTE, false);
+  d_diagnostic_report_diagnostic (loc, 0, format, ap, diagnostics::kind::note,
+				  false);
 }
 
 /* Call this after printing out fatal error messages to clean up and
diff --git a/gcc/d/toir.cc b/gcc/d/toir.cc
index b70db7a..554399b 100644
--- a/gcc/d/toir.cc
+++ b/gcc/d/toir.cc
@@ -1450,7 +1450,8 @@ public:
 	    oconstraints[i] = constraint;
 
 	    if (parse_output_constraint (&constraint, i, ninputs, noutputs,
-					 &allows_mem, &allows_reg, &is_inout))
+					 &allows_mem, &allows_reg, &is_inout,
+					 nullptr))
 	      {
 		/* If the output argument is going to end up in memory.  */
 		if (!allows_reg)
@@ -1469,7 +1470,8 @@ public:
 	      = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
 
 	    if (parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
-					oconstraints, &allows_mem, &allows_reg))
+					oconstraints, &allows_mem, &allows_reg,
+					nullptr))
 	      {
 		/* If the input argument is going to end up in memory.  */
 		if (!allows_reg && allows_mem)
diff --git a/gcc/diagnostic-core.h b/gcc/diagnostic-core.h
index 1e89c9f..fc32f48 100644
--- a/gcc/diagnostic-core.h
+++ b/gcc/diagnostic-core.h
@@ -23,21 +23,8 @@ along with GCC; see the file COPYING3.  If not see
 #define GCC_DIAGNOSTIC_CORE_H
 
 #include "bversion.h"
-
-/* Constants used to discriminate diagnostics.  */
-typedef enum
-{
-#define DEFINE_DIAGNOSTIC_KIND(K, msgid, C) K,
-#include "diagnostic.def"
-#undef DEFINE_DIAGNOSTIC_KIND
-  DK_LAST_DIAGNOSTIC_KIND,
-  /* This is used for tagging pragma pops in the diagnostic
-     classification history chain.  */
-  DK_POP,
-  /* This is used internally to note that a diagnostic is enabled
-     without mandating any specific type.  */
-  DK_ANY,
-} diagnostic_t;
+#include "diagnostics/kinds.h"
+#include "diagnostics/option-id.h"
 
 /* RAII-style class for grouping related diagnostics within global_dc.  */
 
@@ -61,28 +48,9 @@ class auto_diagnostic_nesting_level
 };
 
 /* Forward decl.  */
-class diagnostic_metadata; /* See diagnostic-metadata.h.  */
-
-/* A class to use for the ID of an option that controls
-   a particular diagnostic.
-   This is just a wrapper around "int", but better documents
-   the intent of the code.  */
-
-struct diagnostic_option_id
-{
-  diagnostic_option_id () : m_idx (0) {}
-
-  diagnostic_option_id (int idx) : m_idx (idx) {}
-  /* Ideally we'd take an enum opt_code here, but we don't
-     want to depend on its decl.  */
-
-  bool operator== (diagnostic_option_id other) const
-  {
-    return m_idx == other.m_idx;
-  }
-
-  int m_idx;
-};
+namespace diagnostics {
+   class metadata; /* See diagnostics/metadata.h.  */
+} // namespace diagnostics
 
 extern const char *progname;
 
@@ -109,29 +77,29 @@ extern void internal_error (const char *, ...) ATTRIBUTE_GCC_DIAG(1,2)
 extern void internal_error_no_backtrace (const char *, ...)
      ATTRIBUTE_GCC_DIAG(1,2) ATTRIBUTE_NORETURN;
 /* Pass one of the OPT_W* from options.h as the first parameter.  */
-extern bool warning (diagnostic_option_id,
+extern bool warning (diagnostics::option_id,
 		     const char *, ...) ATTRIBUTE_GCC_DIAG(2,3);
 extern bool warning_n (location_t,
-		       diagnostic_option_id,
+		       diagnostics::option_id,
 		       unsigned HOST_WIDE_INT,
 		       const char *, const char *, ...)
     ATTRIBUTE_GCC_DIAG(4,6) ATTRIBUTE_GCC_DIAG(5,6);
 extern bool warning_n (rich_location *,
-		       diagnostic_option_id,
+		       diagnostics::option_id,
 		       unsigned HOST_WIDE_INT,
 		       const char *, const char *, ...)
     ATTRIBUTE_GCC_DIAG(4, 6) ATTRIBUTE_GCC_DIAG(5, 6);
 extern bool warning_at (location_t,
-			diagnostic_option_id,
+			diagnostics::option_id,
 			const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
 extern bool warning_at (rich_location *,
-			diagnostic_option_id,
+			diagnostics::option_id,
 			const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,4);
 extern bool warning_meta (rich_location *,
-			  const diagnostic_metadata &,
-			  diagnostic_option_id,
+			  const diagnostics::metadata &,
+			  diagnostics::option_id,
 			  const char *, ...)
     ATTRIBUTE_GCC_DIAG(4,5);
 extern void error (const char *, ...) ATTRIBUTE_GCC_DIAG(1,2);
@@ -141,29 +109,29 @@ extern void error_n (location_t, unsigned HOST_WIDE_INT, const char *,
 extern void error_at (location_t, const char *, ...) ATTRIBUTE_GCC_DIAG(2,3);
 extern void error_at (rich_location *, const char *, ...)
   ATTRIBUTE_GCC_DIAG(2,3);
-extern void error_meta (rich_location *, const diagnostic_metadata &,
+extern void error_meta (rich_location *, const diagnostics::metadata &,
 			const char *, ...)
   ATTRIBUTE_GCC_DIAG(3,4);
 extern void fatal_error (location_t, const char *, ...) ATTRIBUTE_GCC_DIAG(2,3)
      ATTRIBUTE_NORETURN;
 /* Pass one of the OPT_W* from options.h as the second parameter.  */
 extern bool pedwarn (location_t,
-		     diagnostic_option_id,
+		     diagnostics::option_id,
 		     const char *, ...)
      ATTRIBUTE_GCC_DIAG(3,4);
 extern bool pedwarn (rich_location *,
-		     diagnostic_option_id,
+		     diagnostics::option_id,
 		     const char *, ...)
      ATTRIBUTE_GCC_DIAG(3,4);
 extern bool permerror (location_t, const char *, ...) ATTRIBUTE_GCC_DIAG(2,3);
 extern bool permerror (rich_location *, const char *,
 				   ...) ATTRIBUTE_GCC_DIAG(2,3);
 extern bool permerror_opt (location_t,
-			   diagnostic_option_id,
+			   diagnostics::option_id,
 			   const char *, ...)
   ATTRIBUTE_GCC_DIAG(3,4);
 extern bool permerror_opt (rich_location *,
-			   diagnostic_option_id,
+			   diagnostics::option_id,
 			   const char *, ...)
   ATTRIBUTE_GCC_DIAG(3,4);
 extern void sorry (const char *, ...) ATTRIBUTE_GCC_DIAG(1,2);
@@ -174,23 +142,23 @@ extern void inform_n (location_t, unsigned HOST_WIDE_INT, const char *,
 		      const char *, ...)
     ATTRIBUTE_GCC_DIAG(3,5) ATTRIBUTE_GCC_DIAG(4,5);
 extern void verbatim (const char *, ...) ATTRIBUTE_GCC_DIAG(1,2);
-extern bool emit_diagnostic (diagnostic_t,
+extern bool emit_diagnostic (enum diagnostics::kind,
 			     location_t,
-			     diagnostic_option_id,
+			     diagnostics::option_id,
 			     const char *, ...) ATTRIBUTE_GCC_DIAG(4,5);
-extern bool emit_diagnostic (diagnostic_t,
+extern bool emit_diagnostic (enum diagnostics::kind,
 			     rich_location *,
-			     diagnostic_option_id,
+			     diagnostics::option_id,
 			     const char *, ...) ATTRIBUTE_GCC_DIAG(4,5);
-extern bool emit_diagnostic_valist (diagnostic_t,
+extern bool emit_diagnostic_valist (enum diagnostics::kind,
 				    location_t,
-				    diagnostic_option_id,
+				    diagnostics::option_id,
 				    const char *, va_list *)
   ATTRIBUTE_GCC_DIAG (4,0);
-extern bool emit_diagnostic_valist_meta (diagnostic_t,
+extern bool emit_diagnostic_valist_meta (enum diagnostics::kind,
 					 rich_location *,
-					 const diagnostic_metadata *,
-					 diagnostic_option_id,
+					 const diagnostics::metadata *,
+					 diagnostics::option_id,
 					 const char *,
 					 va_list *) ATTRIBUTE_GCC_DIAG (5,0);
 extern bool seen_error (void);
diff --git a/gcc/diagnostic-global-context.cc b/gcc/diagnostic-global-context.cc
index 1165915..500f19c 100644
--- a/gcc/diagnostic-global-context.cc
+++ b/gcc/diagnostic-global-context.cc
@@ -27,11 +27,11 @@ along with GCC; see the file COPYING3.  If not see
 #include "coretypes.h"
 #include "intl.h"
 #include "diagnostic.h"
-#include "diagnostic-format.h"
+#include "diagnostics/sink.h"
 
-/* A diagnostic_context surrogate for stderr.  */
-static diagnostic_context global_diagnostic_context;
-diagnostic_context *global_dc = &global_diagnostic_context;
+/* A diagnostics::context surrogate for stderr.  */
+static diagnostics::context global_diagnostic_context;
+diagnostics::context *global_dc = &global_diagnostic_context;
 
 /* Standard error reporting routines in increasing order of severity.  */
 
@@ -49,13 +49,13 @@ verbatim (const char *gmsgid, ...)
   va_end (ap);
 }
 
-/* Wrapper around diagnostic_context::diagnostic_impl
+/* Wrapper around diagnostics::context::diagnostic_impl
    implying global_dc and taking a variable argument list.  */
 
 bool
-emit_diagnostic (diagnostic_t kind,
+emit_diagnostic (enum diagnostics::kind kind,
 		 location_t location,
-		 diagnostic_option_id option_id,
+		 diagnostics::option_id option_id,
 		 const char *gmsgid, ...)
 {
   auto_diagnostic_group d;
@@ -71,9 +71,9 @@ emit_diagnostic (diagnostic_t kind,
 /* As above, but for rich_location *.  */
 
 bool
-emit_diagnostic (diagnostic_t kind,
+emit_diagnostic (enum diagnostics::kind kind,
 		 rich_location *richloc,
-		 diagnostic_option_id option_id,
+		 diagnostics::option_id option_id,
 		 const char *gmsgid, ...)
 {
   auto_diagnostic_group d;
@@ -88,9 +88,9 @@ emit_diagnostic (diagnostic_t kind,
 /* As above, but taking a variable argument list.  */
 
 bool
-emit_diagnostic_valist (diagnostic_t kind,
+emit_diagnostic_valist (enum diagnostics::kind kind,
 			location_t location,
-			diagnostic_option_id option_id,
+			diagnostics::option_id option_id,
 			const char *gmsgid, va_list *ap)
 {
   rich_location richloc (line_table, location);
@@ -101,10 +101,10 @@ emit_diagnostic_valist (diagnostic_t kind,
 /* As above, but with rich_location and metadata.  */
 
 bool
-emit_diagnostic_valist_meta (diagnostic_t kind,
+emit_diagnostic_valist_meta (enum diagnostics::kind kind,
 			     rich_location *richloc,
-			     const diagnostic_metadata *metadata,
-			     diagnostic_option_id option_id,
+			     const diagnostics::metadata *metadata,
+			     diagnostics::option_id option_id,
 			     const char *gmsgid, va_list *ap)
 {
   return global_dc->diagnostic_impl (richloc, metadata, option_id,
@@ -120,7 +120,8 @@ inform (location_t location, const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, location);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_NOTE);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::note);
   va_end (ap);
 }
 
@@ -133,7 +134,8 @@ inform (rich_location *richloc, const char *gmsgid, ...)
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
-  global_dc->diagnostic_impl (richloc, nullptr, -1, gmsgid, &ap, DK_NOTE);
+  global_dc->diagnostic_impl (richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::note);
   va_end (ap);
 }
 
@@ -149,7 +151,7 @@ inform_n (location_t location, unsigned HOST_WIDE_INT n,
   rich_location richloc (line_table, location);
   global_dc->diagnostic_n_impl (&richloc, nullptr, -1, n,
 				singular_gmsgid, plural_gmsgid,
-				&ap, DK_NOTE);
+				&ap, diagnostics::kind::note);
   va_end (ap);
 }
 
@@ -157,14 +159,15 @@ inform_n (location_t location, unsigned HOST_WIDE_INT n,
    to the relevant language specification but is likely to be buggy anyway.
    Returns true if the warning was printed, false if it was inhibited.  */
 bool
-warning (diagnostic_option_id option_id, const char *gmsgid, ...)
+warning (diagnostics::option_id option_id, const char *gmsgid, ...)
 {
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, input_location);
   bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_WARNING);
+					 gmsgid, &ap,
+					 diagnostics::kind::warning);
   va_end (ap);
   return ret;
 }
@@ -175,7 +178,7 @@ warning (diagnostic_option_id option_id, const char *gmsgid, ...)
 
 bool
 warning_at (location_t location,
-	    diagnostic_option_id option_id,
+	    diagnostics::option_id option_id,
 	    const char *gmsgid, ...)
 {
   auto_diagnostic_group d;
@@ -183,7 +186,8 @@ warning_at (location_t location,
   va_start (ap, gmsgid);
   rich_location richloc (line_table, location);
   bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_WARNING);
+					 gmsgid, &ap,
+					 diagnostics::kind::warning);
   va_end (ap);
   return ret;
 }
@@ -192,7 +196,7 @@ warning_at (location_t location,
 
 bool
 warning_at (rich_location *richloc,
-	    diagnostic_option_id option_id,
+	    diagnostics::option_id option_id,
 	    const char *gmsgid, ...)
 {
   gcc_assert (richloc);
@@ -201,7 +205,8 @@ warning_at (rich_location *richloc,
   va_list ap;
   va_start (ap, gmsgid);
   bool ret = global_dc->diagnostic_impl (richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_WARNING);
+					 gmsgid, &ap,
+					 diagnostics::kind::warning);
   va_end (ap);
   return ret;
 }
@@ -210,8 +215,8 @@ warning_at (rich_location *richloc,
 
 bool
 warning_meta (rich_location *richloc,
-	      const diagnostic_metadata &metadata,
-	      diagnostic_option_id option_id,
+	      const diagnostics::metadata &metadata,
+	      diagnostics::option_id option_id,
 	      const char *gmsgid, ...)
 {
   gcc_assert (richloc);
@@ -220,7 +225,8 @@ warning_meta (rich_location *richloc,
   va_list ap;
   va_start (ap, gmsgid);
   bool ret = global_dc->diagnostic_impl (richloc, &metadata, option_id,
-					 gmsgid, &ap, DK_WARNING);
+					 gmsgid, &ap,
+					 diagnostics::kind::warning);
   va_end (ap);
   return ret;
 }
@@ -229,7 +235,7 @@ warning_meta (rich_location *richloc,
 
 bool
 warning_n (rich_location *richloc,
-	   diagnostic_option_id option_id,
+	   diagnostics::option_id option_id,
 	   unsigned HOST_WIDE_INT n,
 	   const char *singular_gmsgid, const char *plural_gmsgid, ...)
 {
@@ -240,7 +246,7 @@ warning_n (rich_location *richloc,
   va_start (ap, plural_gmsgid);
   bool ret = global_dc->diagnostic_n_impl (richloc, nullptr, option_id, n,
 					   singular_gmsgid, plural_gmsgid,
-					   &ap, DK_WARNING);
+					   &ap, diagnostics::kind::warning);
   va_end (ap);
   return ret;
 }
@@ -251,7 +257,7 @@ warning_n (rich_location *richloc,
 
 bool
 warning_n (location_t location,
-	   diagnostic_option_id option_id,
+	   diagnostics::option_id option_id,
 	   unsigned HOST_WIDE_INT n,
 	   const char *singular_gmsgid, const char *plural_gmsgid, ...)
 {
@@ -261,7 +267,7 @@ warning_n (location_t location,
   rich_location richloc (line_table, location);
   bool ret = global_dc->diagnostic_n_impl (&richloc, nullptr, option_id, n,
 					   singular_gmsgid, plural_gmsgid,
-					   &ap, DK_WARNING);
+					   &ap, diagnostics::kind::warning);
   va_end (ap);
   return ret;
 }
@@ -281,7 +287,7 @@ warning_n (location_t location,
 
 bool
 pedwarn (location_t location,
-	 diagnostic_option_id option_id,
+	 diagnostics::option_id option_id,
 	 const char *gmsgid, ...)
 {
   auto_diagnostic_group d;
@@ -289,7 +295,8 @@ pedwarn (location_t location,
   va_start (ap, gmsgid);
   rich_location richloc (line_table, location);
   bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_PEDWARN);
+					 gmsgid, &ap,
+					 diagnostics::kind::pedwarn);
   va_end (ap);
   return ret;
 }
@@ -298,7 +305,7 @@ pedwarn (location_t location,
 
 bool
 pedwarn (rich_location *richloc,
-	 diagnostic_option_id option_id,
+	 diagnostics::option_id option_id,
 	 const char *gmsgid, ...)
 {
   gcc_assert (richloc);
@@ -307,7 +314,8 @@ pedwarn (rich_location *richloc,
   va_list ap;
   va_start (ap, gmsgid);
   bool ret = global_dc->diagnostic_impl (richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_PEDWARN);
+					 gmsgid, &ap,
+					 diagnostics::kind::pedwarn);
   va_end (ap);
   return ret;
 }
@@ -327,7 +335,7 @@ permerror (location_t location, const char *gmsgid, ...)
   va_start (ap, gmsgid);
   rich_location richloc (line_table, location);
   bool ret = global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
-					 DK_PERMERROR);
+					 diagnostics::kind::permerror);
   va_end (ap);
   return ret;
 }
@@ -343,7 +351,7 @@ permerror (rich_location *richloc, const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   bool ret = global_dc->diagnostic_impl (richloc, nullptr, -1, gmsgid, &ap,
-					 DK_PERMERROR);
+					 diagnostics::kind::permerror);
   va_end (ap);
   return ret;
 }
@@ -354,7 +362,7 @@ permerror (rich_location *richloc, const char *gmsgid, ...)
 
 bool
 permerror_opt (location_t location,
-	       diagnostic_option_id option_id,
+	       diagnostics::option_id option_id,
 	       const char *gmsgid, ...)
 {
   auto_diagnostic_group d;
@@ -362,7 +370,8 @@ permerror_opt (location_t location,
   va_start (ap, gmsgid);
   rich_location richloc (line_table, location);
   bool ret = global_dc->diagnostic_impl (&richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_PERMERROR);
+					 gmsgid, &ap,
+					 diagnostics::kind::permerror);
   va_end (ap);
   return ret;
 }
@@ -371,7 +380,7 @@ permerror_opt (location_t location,
 
 bool
 permerror_opt (rich_location *richloc,
-	       diagnostic_option_id option_id,
+	       diagnostics::option_id option_id,
 	       const char *gmsgid, ...)
 {
   gcc_assert (richloc);
@@ -380,7 +389,8 @@ permerror_opt (rich_location *richloc,
   va_list ap;
   va_start (ap, gmsgid);
   bool ret = global_dc->diagnostic_impl (richloc, nullptr, option_id,
-					 gmsgid, &ap, DK_PERMERROR);
+					 gmsgid, &ap,
+					 diagnostics::kind::permerror);
   va_end (ap);
   return ret;
 }
@@ -394,7 +404,8 @@ error (const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, input_location);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_ERROR);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::error);
   va_end (ap);
 }
 
@@ -410,7 +421,7 @@ error_n (location_t location, unsigned HOST_WIDE_INT n,
   rich_location richloc (line_table, location);
   global_dc->diagnostic_n_impl (&richloc, nullptr, -1, n,
 				singular_gmsgid, plural_gmsgid,
-				&ap, DK_ERROR);
+				&ap, diagnostics::kind::error);
   va_end (ap);
 }
 
@@ -422,7 +433,8 @@ error_at (location_t loc, const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, loc);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_ERROR);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::error);
   va_end (ap);
 }
 
@@ -436,14 +448,15 @@ error_at (rich_location *richloc, const char *gmsgid, ...)
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
-  global_dc->diagnostic_impl (richloc, nullptr, -1, gmsgid, &ap, DK_ERROR);
+  global_dc->diagnostic_impl (richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::error);
   va_end (ap);
 }
 
 /* Same as above, but with metadata.  */
 
 void
-error_meta (rich_location *richloc, const diagnostic_metadata &metadata,
+error_meta (rich_location *richloc, const diagnostics::metadata &metadata,
 	    const char *gmsgid, ...)
 {
   gcc_assert (richloc);
@@ -451,7 +464,8 @@ error_meta (rich_location *richloc, const diagnostic_metadata &metadata,
   auto_diagnostic_group d;
   va_list ap;
   va_start (ap, gmsgid);
-  global_dc->diagnostic_impl (richloc, &metadata, -1, gmsgid, &ap, DK_ERROR);
+  global_dc->diagnostic_impl (richloc, &metadata, -1, gmsgid, &ap,
+			      diagnostics::kind::error);
   va_end (ap);
 }
 
@@ -465,7 +479,8 @@ sorry (const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, input_location);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_SORRY);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::sorry);
   va_end (ap);
 }
 
@@ -477,7 +492,8 @@ sorry_at (location_t loc, const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, loc);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_SORRY);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::sorry);
   va_end (ap);
 }
 
@@ -499,7 +515,8 @@ fatal_error (location_t loc, const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, loc);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_FATAL);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::fatal);
   va_end (ap);
 
   gcc_unreachable ();
@@ -514,7 +531,8 @@ internal_error (const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, input_location);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_ICE);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::ice);
   va_end (ap);
 
   gcc_unreachable ();
@@ -530,7 +548,8 @@ internal_error_no_backtrace (const char *gmsgid, ...)
   va_list ap;
   va_start (ap, gmsgid);
   rich_location richloc (line_table, input_location);
-  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap, DK_ICE_NOBT);
+  global_dc->diagnostic_impl (&richloc, nullptr, -1, gmsgid, &ap,
+			      diagnostics::kind::ice_nobt);
   va_end (ap);
 
   gcc_unreachable ();
diff --git a/gcc/diagnostic-state-to-dot.cc b/gcc/diagnostic-state-to-dot.cc
deleted file mode 100644
index ddae83b..0000000
--- a/gcc/diagnostic-state-to-dot.cc
+++ /dev/null
@@ -1,537 +0,0 @@
-/* Creating GraphViz .dot files from XML state documents.
-   Copyright (C) 2025 Free Software Foundation, Inc.
-   Contributed by David Malcolm <dmalcolm@redhat.com>.
-
-This file is part of GCC.
-
-GCC is free software; you can redistribute it and/or modify it
-under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 3, or (at your option)
-any later version.
-
-GCC is distributed in the hope that it will be useful, but
-WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with GCC; see the file COPYING3.  If not see
-<http://www.gnu.org/licenses/>.  */
-
-#define INCLUDE_ALGORITHM
-#define INCLUDE_MAP
-#define INCLUDE_SET
-#define INCLUDE_STRING
-#define INCLUDE_VECTOR
-#include "config.h"
-#include "system.h"
-#include "coretypes.h"
-
-#include "xml.h"
-#include "xml-printer.h"
-#include "graphviz.h"
-
-static int
-get_depth (const xml::element &e)
-{
-  int deepest_child = 0;
-  for (auto &iter : e.m_children)
-    if (xml::element *child_element = iter->dyn_cast_element ())
-      deepest_child = std::max (deepest_child,
-				get_depth (*child_element));
-  return deepest_child + 1;
-}
-
-enum class dynalloc_state
-{
-  unknown,
-  nonnull,
-  unchecked,
-  freed
-};
-
-static const char *
-get_color_for_dynalloc_state (enum dynalloc_state dynalloc_st)
-{
-  switch (dynalloc_st)
-    {
-    default:
-      gcc_unreachable ();
-      break;
-    case dynalloc_state::unknown:
-    case dynalloc_state::nonnull:
-      return nullptr;
-
-    case dynalloc_state::unchecked:
-      return "#ec7a08"; // pf-orange-400
-
-    case dynalloc_state::freed:
-      return "#cc0000"; // pf-red-100
-    }
-}
-
-static void
-set_color_for_dynalloc_state (dot::attr_list &attrs,
-			      enum dynalloc_state dynalloc_state)
-{
-  if (const char *color = get_color_for_dynalloc_state (dynalloc_state))
-    attrs.add (dot::id ("color"), dot::id (color));
-}
-
-static enum dynalloc_state
-get_dynalloc_state (const xml::element &input_element)
-{
-  const char *dyn_alloc_state = input_element.get_attr ("dynamic-alloc-state");
-  if (!dyn_alloc_state)
-    return dynalloc_state::unknown;
-
-  if (dyn_alloc_state == std::string ("unchecked"))
-    return dynalloc_state::unchecked;
-
-  if (dyn_alloc_state == std::string ("nonnull"))
-    return dynalloc_state::nonnull;
-
-  if (dyn_alloc_state == std::string ("freed"))
-    return dynalloc_state::freed;
-
-  return dynalloc_state::unknown;
-}
-
-class state_diagram : public dot::graph
-{
-public:
-  state_diagram (const xml::document &input_state_doc)
-  : m_next_id (0),
-    m_show_tags (false)
-  {
-    // "node [shape=plaintext]\n"
-    {
-      auto attr_stmt
-	= std::make_unique<dot::attr_stmt> (dot::attr_stmt::kind::node);
-      attr_stmt->m_attrs.add (dot::id ("shape"), dot::id ("plaintext"));
-      add_stmt (std::move (attr_stmt));
-    }
-
-    /* Recurse down the XML state diagram, creating subgraphs
-       and then eventually creating nodes, and recursively
-       creating XML tables, adding ports for the endpoints of edges,
-       and recording edges we'll want to create (into m_pending_edges).  */
-    xml::element *input_elmt_state_diagram
-      = input_state_doc.find_child_element ("state-diagram");
-    gcc_assert (input_elmt_state_diagram);
-    xml::element *input_elmt_mem_regions
-      = input_elmt_state_diagram->find_child_element ("memory-regions");
-    if (!input_elmt_mem_regions)
-      return;
-    auto root_cluster
-      = std::make_unique<dot::subgraph> (dot::id ("cluster_memory_regions"));
-    for (auto &iter : input_elmt_mem_regions->m_children)
-      on_input_xml_node (*root_cluster, *iter);
-    add_stmt (std::move (root_cluster));
-
-    /* We should now have ports for edge endpoints for all region ids.
-       Use them now to create edges.  */
-    for (auto &pe : m_pending_edges)
-      {
-	auto search = m_region_id_to_dst_node_id.find (pe.m_dst_region_id);
-	if (search != m_region_id_to_dst_node_id.end ())
-	  {
-	    auto &dst_node_id = search->second;
-	    auto e = std::make_unique<dot::edge_stmt> (pe.m_src_node_id,
-						       dst_node_id);
-
-	    auto dynalloc_state = m_region_id_to_dynalloc_state.find (pe.m_dst_region_id);
-	    if (dynalloc_state != m_region_id_to_dynalloc_state.end ())
-	      set_color_for_dynalloc_state (e->m_attrs,
-					    dynalloc_state->second);
-
-	    add_stmt (std::move (e));
-	  }
-      }
-  }
-
-private:
-  struct pending_edge
-  {
-    dot::node_id m_src_node_id;
-    std::string m_dst_region_id;
-  };
-
-  dot::id
-  get_id_for_region (const char *region_id)
-  {
-    gcc_assert (region_id);
-    return std::string ("cluster_region_") + region_id;
-  }
-
-  dot::id
-  make_id (bool cluster = false)
-  {
-    if (cluster)
-      return std::string ("cluster_") + std::to_string (m_next_id++);
-    else
-      return std::string ("id_") + std::to_string (m_next_id++);
-  }
-
-  bool
-  starts_node_p (const xml::element &e)
-  {
-    if (e.m_kind == "stack"
-	|| e.m_kind == "heap-buffer"
-	|| e.m_kind == "variable") // e.g. within globals
-      return true;
-    return false;
-  }
-
-  void
-  on_input_xml_node (dot::subgraph &parent_subgraph,
-		     xml::node &input_node)
-  {
-    xml::element *input_element = input_node.dyn_cast_element ();
-    if (!input_element)
-      return;
-
-    dot::id sg_id = make_id (true);
-
-    if (starts_node_p (*input_element))
-      {
-	// Create node with table
-	xml::element table ("table", false);
-	xml::printer xp (table);
-	xp.set_attr ("border", "0");
-	xp.set_attr ("cellborder", "1");
-	xp.set_attr ("cellspacing", "0");
-
-	const int max_depth = get_depth (*input_element);
-	const int num_columns = max_depth + 2;
-
-	dot::id id_of_node = make_id ();
-	on_xml_node (id_of_node, xp, *input_element,
-		     max_depth, 0, num_columns);
-
-	auto node = std::make_unique<dot::node_stmt> (std::move (id_of_node));
-	node->m_attrs.add (dot::id ("shape"),
-			   dot::id ("plaintext"));
-
-	// xml must be done by now
-
-	node->m_attrs.add (dot::id ("label"),
-			   dot::id (table));
-
-	parent_subgraph.m_stmt_list.add_stmt (std::move (node));
-      }
-    else
-      {
-	auto child_subgraph = std::make_unique<dot::subgraph> (std::move (sg_id));
-
-	if (const char *label = input_element->get_attr ("label"))
-	  child_subgraph->add_attr (dot::id ("label"), dot::id (label));
-
-	// recurse:
-	for (auto &iter : input_element->m_children)
-	  on_input_xml_node (*child_subgraph, *iter);
-	parent_subgraph.m_stmt_list.add_stmt (std::move (child_subgraph));
-      }
-  }
-
-  enum class style { h1, h2 };
-
-  void
-  add_title_tr (const dot::id &id_of_node,
-		xml::printer &xp,
-		int num_columns,
-		const xml::element &input_element,
-		std::string heading,
-		enum style styl,
-		enum dynalloc_state dynalloc_state)
-  {
-    xp.push_tag ("tr", true);
-    xp.push_tag ("td", false);
-    xp.set_attr ("colspan", std::to_string (num_columns));
-    xp.set_attr ("cellpadding", "5");
-
-    const char *bgcolor;
-    const char *color;
-    if (const char *c = get_color_for_dynalloc_state (dynalloc_state))
-      {
-	bgcolor = c;
-	color = "white";
-      }
-    else
-      switch (styl)
-	{
-	default:
-	  gcc_unreachable ();
-	case style::h1:
-	  // from diagnostic-format-html.cc: HTML_STYLE .linenum
-	  bgcolor = "#0088ce";
-	  color = "white";
-	  break;
-	case style::h2:
-	  // from diagnostic-format-html.cc: HTML_STYLE .events-hdr
-	  bgcolor = "#393f44"; // pf-black-800
-	  color = "white";
-	  break;
-	}
-
-    xp.set_attr ("bgcolor", bgcolor);
-    xp.push_tag ("font", false);
-    xp.set_attr ("color", color);
-    if (heading == "")
-      heading = " ";
-    xp.add_text (std::move (heading));
-    xp.pop_tag ("font");
-
-    maybe_add_dst_port (id_of_node, xp, input_element);
-
-    xp.pop_tag ("td");
-    xp.pop_tag ("tr");
-  }
-
-  /* Recursively add <TR> to XP for INPUT_NODE and its descendents.  */
-  void
-  on_xml_node (const dot::id &id_of_node,
-	       xml::printer &xp,
-	       xml::node &input_node,
-	       int max_depth,
-	       int depth,
-	       int num_columns)
-  {
-    bool recurse = true;
-
-    xml::element *input_element = input_node.dyn_cast_element ();
-    if (!input_element)
-      return;
-
-    if (input_element->m_kind == "concrete-bindings")
-      return;
-    if (input_element->m_kind == "padding")
-      return;
-
-    if (input_element->m_kind == "stack")
-      {
-	add_title_tr (id_of_node, xp, num_columns, *input_element, "Stack",
-		      style::h1, dynalloc_state::unknown);
-      }
-    else if (input_element->m_kind == "stack-frame")
-      {
-	if (const char *function = input_element->get_attr ("function"))
-	  add_title_tr (id_of_node, xp, num_columns, *input_element,
-			std::string ("Frame: ") + function,
-			style::h2, dynalloc_state::unknown);
-      }
-    else if (input_element->m_kind == "heap-buffer")
-      {
-	const char *extents = input_element->get_attr ("dynamic-extents");
-	enum dynalloc_state dynalloc_st = get_dynalloc_state (*input_element);
-	if (auto region_id = input_element->get_attr ("region_id"))
-	    m_region_id_to_dynalloc_state[region_id] = dynalloc_st;
-	const char *type = input_element->get_attr ("type");
-	pretty_printer pp;
-	switch (dynalloc_st)
-	  {
-	  default:
-	    gcc_unreachable ();
-
-	  case dynalloc_state::unknown:
-	  case dynalloc_state::nonnull:
-	    if (type)
-	      {
-		if (extents)
-		  pp_printf (&pp, "%s (%s byte allocation)",
-			     type, extents);
-		else
-		  pp_printf (&pp, "%s", type);
-	      }
-	    else
-	      {
-		if (extents)
-		  pp_printf (&pp, "%s byte allocation",
-			     extents);
-	      }
-	    break;
-
-	  case dynalloc_state::unchecked:
-	    if (type)
-	      {
-		if (extents)
-		  pp_printf (&pp, "%s (unchecked %s byte allocation)",
-			     type, extents);
-	      }
-	    else
-	      {
-		if (extents)
-		  pp_printf (&pp, "Unchecked %s byte allocation",
-			     extents);
-	      }
-	    break;
-
-	  case dynalloc_state::freed:
-	    // TODO: show deallocator
-	    // TODO: show deallocation event
-	    pp_printf (&pp, "Freed buffer");
-	    break;
-	  }
-	add_title_tr (id_of_node, xp, num_columns, *input_element,
-		      pp_formatted_text (&pp),
-		      style::h2,
-		      dynalloc_st);
-      }
-    else
-      {
-	xp.push_tag ("tr", true);
-	if (depth > 0)
-	  {
-	    /* Indent, by create a <td> spanning "depth" columns.  */
-	    xp.push_tag ("td", false);
-	    xp.set_attr ("colspan", std::to_string (depth));
-	    xp.add_text (" "); // graphviz doesn't like <td/>
-	    xp.pop_tag ("td");
-	  }
-	if (m_show_tags)
-	  {
-	    // Debug: show XML tag
-	    xp.push_tag ("td", false);
-	    xp.add_text ("<");
-	    xp.add_text (input_element->m_kind);
-	    xp.add_text (">");
-	    xp.pop_tag ("td");
-	  }
-	if (input_element->m_kind == "variable")
-	  {
-	    const char *name = input_element->get_attr ("name");
-	    gcc_assert (name);
-	    xp.push_tag ("td", false);
-	    maybe_add_dst_port (id_of_node, xp, *input_element);
-	    push_src_text (xp);
-	    xp.add_text (name);
-	    pop_src_text (xp);
-	    xp.pop_tag ("td");
-	  }
-	else if (input_element->m_kind == "element")
-	  {
-	    const char *index = input_element->get_attr ("index");
-	    gcc_assert (index);
-	    xp.push_tag ("td", false);
-	    maybe_add_dst_port (id_of_node, xp, *input_element);
-	    push_src_text (xp);
-	    xp.add_text ("[");
-	    xp.add_text (index);
-	    xp.add_text ("]");
-	    pop_src_text (xp);
-	    xp.pop_tag ("td");
-	  }
-	else if (input_element->m_kind == "field")
-	  {
-	    const char *name = input_element->get_attr ("name");
-	    gcc_assert (name);
-	    xp.push_tag ("td", false);
-	    maybe_add_dst_port (id_of_node, xp, *input_element);
-	    push_src_text (xp);
-	    xp.add_text (".");
-	    xp.add_text (name);
-	    pop_src_text (xp);
-	    xp.pop_tag ("td");
-	  }
-	if (const char *type = input_element->get_attr ("type"))
-	  {
-	    xp.push_tag ("td", false);
-	    if (max_depth > depth)
-	      xp.set_attr ("colspan", std::to_string (max_depth - depth));
-	    xp.set_attr ("align", "right");
-	    push_src_text (xp);
-	    xp.add_text (type);
-	    pop_src_text (xp);
-	    xp.pop_tag ("td");
-	  }
-	if (auto value = input_element->find_child_element ("value-of-region"))
-	  {
-	    xp.push_tag ("td", false);
-	    for (auto &iter : value->m_children)
-	      if (auto child_element = iter->dyn_cast_element ())
-		print_value (id_of_node, xp, *child_element);
-	    xp.pop_tag ("td");
-	    recurse = false;
-	  }
-	xp.pop_tag ("tr");
-      }
-
-    if (recurse)
-      for (auto &iter : input_element->m_children)
-	on_xml_node (id_of_node, xp, *iter, max_depth, depth + 1, num_columns);
-  }
-
-  void
-  push_src_text (xml::printer &xp)
-  {
-    xp.push_tag ("font");
-    xp.set_attr ("color", "blue");
-  }
-
-  void
-  pop_src_text (xml::printer &xp)
-  {
-    xp.pop_tag ("font");
-  }
-
-  void
-  print_value (const dot::id &id_of_node,
-	       xml::printer &xp,
-	       xml::element &input_element)
-  {
-    if (input_element.m_kind == "pointer-to-region")
-      if (const char *dst_region_id = input_element.get_attr ("region_id"))
-	{
-	  dot::id src_port_id = make_id ();
-	  xp.set_attr ("port", src_port_id.m_str);
-	  m_pending_edges.push_back
-	    ({dot::node_id (id_of_node,
-			    dot::port (src_port_id,
-				       dot::compass_pt::e)),
-	       dst_region_id});
-      }
-
-    if (input_element.m_kind == "uninitialized")
-      {
-	xp.add_text ("(uninitialized)");
-	return;
-      }
-
-    if (auto dump_text = input_element.get_attr ("dump-text"))
-      xp.add_text (dump_text);
-  }
-
-  /* If INPUT_ELEMENT has a "region_id", add a port to XP for possible
-     incoming edges to use.  */
-
-  void
-  maybe_add_dst_port (const dot::id &id_of_node,
-		      xml::printer &xp,
-		      const xml::element &input_element)
-  {
-    if (const char *region_id = input_element.get_attr ("region_id"))
-      {
-	dot::id dst_id = make_id ();
-	dot::node_id node_id (id_of_node,
-			      dot::port (dst_id/*,
-						 dot::compass_pt::w*/));
-	xp.set_attr ("port", dst_id.m_str);
-	m_region_id_to_dst_node_id.emplace (std::string (region_id),
-					    std::move (node_id));
-      }
-  }
-
-
-private:
-  int m_next_id;
-  std::vector<pending_edge> m_pending_edges;
-  std::map<std::string, dot::node_id> m_region_id_to_dst_node_id;
-  std::map<std::string, enum dynalloc_state> m_region_id_to_dynalloc_state;
-  bool m_show_tags;
-};
-
-std::unique_ptr<dot::graph>
-make_dot_graph_from_xml_state (const xml::document &xml_state)
-{
-  return std::make_unique<state_diagram> (xml_state);
-}
diff --git a/gcc/diagnostic-state.h b/gcc/diagnostic-state.h
deleted file mode 100644
index 68c3e00..0000000
--- a/gcc/diagnostic-state.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/* Capturing changing state in diagnostic paths.
-   Copyright (C) 2025 Free Software Foundation, Inc.
-   Contributed by David Malcolm <dmalcolm@redhat.com>.
-
-This file is part of GCC.
-
-GCC is free software; you can redistribute it and/or modify it
-under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 3, or (at your option)
-any later version.
-
-GCC is distributed in the hope that it will be useful, but
-WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with GCC; see the file COPYING3.  If not see
-<http://www.gnu.org/licenses/>.  */
-
-#ifndef GCC_DIAGNOSTIC_STATE_H
-#define GCC_DIAGNOSTIC_STATE_H
-
-/* We want to be able to express changing program states in diagnostic paths,
-   so that we can emit this in HTML and SARIF output, and to keep this
-   separate from implementation details of -fanalyzer.
-
-   For now, we use xml::document as the type in the diagnostic subsystem
-   for (optionally) tracking the state at a diagnostic_event.  */
-
-namespace xml { class document; }
-namespace dot { class graph; }
-
-extern std::unique_ptr<dot::graph>
-make_dot_graph_from_xml_state (const xml::document &xml_state);
-
-#endif /* GCC_DIAGNOSTIC_STATE_H */
diff --git a/gcc/diagnostic.h b/gcc/diagnostic.h
index 00f6e35..7572e04 100644
--- a/gcc/diagnostic.h
+++ b/gcc/diagnostic.h
@@ -21,1135 +21,31 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_DIAGNOSTIC_H
 #define GCC_DIAGNOSTIC_H
 
-#include "unique-argv.h"
 #include "rich-location.h"
 #include "pretty-print.h"
 #include "diagnostic-core.h"
 
-namespace text_art
-{
-  class theme;
-} // namespace text_art
-
-namespace xml
-{
-  class printer;
-} // namespace xml
-
-/* An enum for controlling what units to use for the column number
-   when diagnostics are output, used by the -fdiagnostics-column-unit option.
-   Tabs will be expanded or not according to the value of -ftabstop.  The origin
-   (default 1) is controlled by -fdiagnostics-column-origin.  */
-
-enum diagnostics_column_unit
-{
-  /* The default from GCC 11 onwards: display columns.  */
-  DIAGNOSTICS_COLUMN_UNIT_DISPLAY,
-
-  /* The behavior in GCC 10 and earlier: simple bytes.  */
-  DIAGNOSTICS_COLUMN_UNIT_BYTE
-};
-
-/* An enum for controlling how to print non-ASCII characters/bytes when
-   a diagnostic suggests escaping the source code on output.  */
-
-enum diagnostics_escape_format
-{
-  /* Escape non-ASCII Unicode characters in the form <U+XXXX> and
-     non-UTF-8 bytes in the form <XX>.  */
-  DIAGNOSTICS_ESCAPE_FORMAT_UNICODE,
-
-  /* Escape non-ASCII bytes in the form <XX> (thus showing the underlying
-     encoding of non-ASCII Unicode characters).  */
-  DIAGNOSTICS_ESCAPE_FORMAT_BYTES
-};
-
-/* Enum for overriding the standard output format.  */
-
-enum diagnostics_output_format
-{
-  /* The default: textual output.  */
-  DIAGNOSTICS_OUTPUT_FORMAT_TEXT,
-
-  /* SARIF-based output, as JSON to stderr.  */
-  DIAGNOSTICS_OUTPUT_FORMAT_SARIF_STDERR,
-
-  /* SARIF-based output, to a JSON file.  */
-  DIAGNOSTICS_OUTPUT_FORMAT_SARIF_FILE
-};
-
-/* An enum for controlling how diagnostic_paths should be printed.  */
-enum diagnostic_path_format
-{
-  /* Don't print diagnostic_paths.  */
-  DPF_NONE,
-
-  /* Print diagnostic_paths by emitting a separate "note" for every event
-     in the path.  */
-  DPF_SEPARATE_EVENTS,
-
-  /* Print diagnostic_paths by consolidating events together where they
-     are close enough, and printing such runs of events with multiple
-     calls to diagnostic_show_locus, showing the individual events in
-     each run via labels in the source.  */
-  DPF_INLINE_EVENTS
-};
-
-/* An enum for capturing values of GCC_EXTRA_DIAGNOSTIC_OUTPUT,
-   and for -fdiagnostics-parseable-fixits.  */
-
-enum diagnostics_extra_output_kind
-{
-  /* No extra output, or an unrecognized value.  */
-  EXTRA_DIAGNOSTIC_OUTPUT_none,
-
-  /* Emit fix-it hints using the "fixits-v1" format, equivalent to
-     -fdiagnostics-parseable-fixits.  */
-  EXTRA_DIAGNOSTIC_OUTPUT_fixits_v1,
-
-  /* Emit fix-it hints using the "fixits-v2" format.  */
-  EXTRA_DIAGNOSTIC_OUTPUT_fixits_v2
-};
-
-/* Values for -fdiagnostics-text-art-charset=.  */
-
-enum diagnostic_text_art_charset
-{
-  /* No text art diagrams shall be emitted.  */
-  DIAGNOSTICS_TEXT_ART_CHARSET_NONE,
-
-  /* Use pure ASCII for text art diagrams.  */
-  DIAGNOSTICS_TEXT_ART_CHARSET_ASCII,
-
-  /* Use ASCII + conservative use of other unicode characters
-     in text art diagrams.  */
-  DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE,
-
-  /* Use Emoji.  */
-  DIAGNOSTICS_TEXT_ART_CHARSET_EMOJI
-};
-
-/* A diagnostic is described by the MESSAGE to send, the FILE and LINE of
-   its context and its KIND (ice, error, warning, note, ...)  See complete
-   list in diagnostic.def.  */
-struct diagnostic_info
-{
-  diagnostic_info ()
-    : message (), richloc (), metadata (), x_data (), kind (), option_id (),
-      m_iinfo ()
-  { }
-
-  /* Text to be formatted.  */
-  text_info message;
-
-  /* The location at which the diagnostic is to be reported.  */
-  rich_location *richloc;
-
-  /* An optional bundle of metadata associated with the diagnostic
-     (or NULL).  */
-  const diagnostic_metadata *metadata;
-
-  /* Auxiliary data for client.  */
-  void *x_data;
-  /* The kind of diagnostic it is about.  */
-  diagnostic_t kind;
-  /* Which OPT_* directly controls this diagnostic.  */
-  diagnostic_option_id option_id;
-
-  /* Inlining context containing locations for each call site along
-     the inlining stack.  */
-  struct inlining_info
-  {
-    /* Locations along the inlining stack.  */
-    auto_vec<location_t, 8> m_ilocs;
-    /* The abstract origin of the location.  */
-    void *m_ao;
-    /* Set if every M_ILOCS element is in a system header.  */
-    bool m_allsyslocs;
-  } m_iinfo;
-};
-
-/*  Forward declarations.  */
-class diagnostic_location_print_policy;
-class diagnostic_source_print_policy;
-
-typedef void (*diagnostic_text_starter_fn) (diagnostic_text_output_format &,
-					    const diagnostic_info *);
-
-struct to_text;
-struct to_html;
-
-extern pretty_printer *get_printer (to_text &);
-
-template <typename Sink>
-using diagnostic_start_span_fn = void (*) (const diagnostic_location_print_policy &,
-					   Sink &sink,
-					   expanded_location);
-
-typedef void (*diagnostic_text_finalizer_fn) (diagnostic_text_output_format &,
-					      const diagnostic_info *,
-					      diagnostic_t);
-
-/* Abstract base class for the diagnostic subsystem to make queries
-   about command-line options.  */
-
-class diagnostic_option_manager
-{
-public:
-  virtual ~diagnostic_option_manager () {}
-
-  /* Return 1 if option OPTION_ID is enabled, 0 if it is disabled,
-     or -1 if it isn't a simple on-off switch
-     (or if the value is unknown, typically set later in target).  */
-  virtual int option_enabled_p (diagnostic_option_id option_id) const = 0;
-
-  /* Return malloced memory for the name of the option OPTION_ID
-     which enabled a diagnostic, originally of type ORIG_DIAG_KIND but
-     possibly converted to DIAG_KIND by options such as -Werror.
-     May return NULL if no name is to be printed.
-     May be passed 0 as well as the index of a particular option.  */
-  virtual char *make_option_name (diagnostic_option_id option_id,
-				  diagnostic_t orig_diag_kind,
-				  diagnostic_t diag_kind) const = 0;
-
-  /* Return malloced memory for a URL describing the option that controls
-     a diagnostic.
-     May return NULL if no URL is available.
-     May be passed 0 as well as the index of a particular option.  */
-  virtual char *make_option_url (diagnostic_option_id option_id) const = 0;
-};
-
-class edit_context;
-class diagnostic_client_data_hooks;
-class logical_location_manager;
-class diagnostic_diagram;
-class diagnostic_source_effect_info;
-class diagnostic_output_format;
-  class diagnostic_text_output_format;
-class diagnostic_buffer;
-
-/* A stack of sets of classifications: each entry in the stack is
-   a mapping from option index to diagnostic severity that can be changed
-   via pragmas.  The stack can be pushed and popped.  */
-
-class diagnostic_option_classifier
-{
-public:
-  void init (int n_opts);
-  void fini ();
-
-  /* Save all diagnostic classifications in a stack.  */
-  void push ();
-
-  /* Restore the topmost classification set off the stack.  If the stack
-     is empty, revert to the state based on command line parameters.  */
-  void pop (location_t where);
-
-  bool option_unspecified_p (diagnostic_option_id option_id) const
-  {
-    return get_current_override (option_id) == DK_UNSPECIFIED;
-  }
-
-  diagnostic_t get_current_override (diagnostic_option_id option_id) const
-  {
-    gcc_assert (option_id.m_idx < m_n_opts);
-    return m_classify_diagnostic[option_id.m_idx];
-  }
-
-  diagnostic_t
-  classify_diagnostic (const diagnostic_context *context,
-		       diagnostic_option_id option_id,
-		       diagnostic_t new_kind,
-		       location_t where);
-
-  diagnostic_t
-  update_effective_level_from_pragmas (diagnostic_info *diagnostic) const;
-
-  int pch_save (FILE *);
-  int pch_restore (FILE *);
-
-private:
-  /* Each time a diagnostic's classification is changed with a pragma,
-     we record the change and the location of the change in an array of
-     these structs.  */
-  struct diagnostic_classification_change_t
-  {
-    location_t location;
-
-    /* For DK_POP, this is the index of the corresponding push (as stored
-       in m_push_list).
-       Otherwise, this is an option index.  */
-    int option;
-
-    diagnostic_t kind;
-  };
-
-  int m_n_opts;
-
-  /* For each option index that can be passed to warning() et al
-     (OPT_* from options.h when using this code with the core GCC
-     options), this array may contain a new kind that the diagnostic
-     should be changed to before reporting, or DK_UNSPECIFIED to leave
-     it as the reported kind, or DK_IGNORED to not report it at
-     all.  */
-  diagnostic_t *m_classify_diagnostic;
-
-  /* History of all changes to the classifications above.  This list
-     is stored in location-order, so we can search it, either
-     binary-wise or end-to-front, to find the most recent
-     classification for a given diagnostic, given the location of the
-     diagnostic.  */
-  vec<diagnostic_classification_change_t> m_classification_history;
-
-  /* For diagnostic_context::get_classification_history, declared later.  */
-  friend class diagnostic_context;
-
-  /* For pragma push/pop.  */
-  vec<int> m_push_list;
-};
-
-/* A bundle of options relating to printing the user's source code
-   (potentially with a margin, underlining, labels, etc).  */
-
-struct diagnostic_source_printing_options
-{
-  /* True if we should print the source line with a caret indicating
-     the location.
-     Corresponds to -fdiagnostics-show-caret.  */
-  bool enabled;
-
-  /* Maximum width of the source line printed.  */
-  int max_width;
-
-  /* Character used at the caret when printing source locations.  */
-  char caret_chars[rich_location::STATICALLY_ALLOCATED_RANGES];
-
-  /* When printing source code, should the characters at carets and ranges
-     be colorized? (assuming colorization is on at all).
-     This should be true for frontends that generate range information
-     (so that the ranges of code are colorized),
-     and false for frontends that merely specify points within the
-     source code (to avoid e.g. colorizing just the first character in
-     a token, which would look strange).  */
-  bool colorize_source_p;
-
-  /* When printing source code, should labelled ranges be printed?
-     Corresponds to -fdiagnostics-show-labels.  */
-  bool show_labels_p;
-
-  /* When printing source code, should there be a left-hand margin
-     showing line numbers?
-     Corresponds to -fdiagnostics-show-line-numbers.  */
-  bool show_line_numbers_p;
-
-  /* If printing source code, what should the minimum width of the margin
-     be?  Line numbers will be right-aligned, and padded to this width.
-     Corresponds to -fdiagnostics-minimum-margin-width=VALUE.  */
-  int min_margin_width;
-
-  /* Usable by plugins; if true, print a debugging ruler above the
-     source output.  */
-  bool show_ruler_p;
-
-  /* When printing events in an inline path, should we print lines
-     visualizing links between related events (e.g. for CFG paths)?
-     Corresponds to -fdiagnostics-show-event-links.  */
-  bool show_event_links_p;
-};
-
-/* A bundle of state for determining column numbers in diagnostics
-   (tab stops, whether to start at 0 or 1, etc).
-   Uses a file_cache to handle tabs.  */
-
-class diagnostic_column_policy
-{
-public:
-  diagnostic_column_policy (const diagnostic_context &dc);
-
-  int converted_column (expanded_location s) const;
-
-  label_text get_location_text (const expanded_location &s,
-				bool show_column,
-				bool colorize) const;
-
-  int get_tabstop () const { return m_tabstop; }
-
-private:
-  file_cache &m_file_cache;
-  enum diagnostics_column_unit m_column_unit;
-  int m_column_origin;
-  int m_tabstop;
-};
-
-/* A bundle of state for printing locations within diagnostics
-   (e.g. "FILENAME:LINE:COLUMN"), to isolate the interactions between
-   diagnostic_context and the start_span callbacks.  */
-
-class diagnostic_location_print_policy
-{
-public:
-  diagnostic_location_print_policy (const diagnostic_context &dc);
-  diagnostic_location_print_policy (const diagnostic_text_output_format &);
-
-  bool show_column_p () const { return m_show_column; }
-
-  const diagnostic_column_policy &
-  get_column_policy () const { return m_column_policy; }
-
-  void
-  print_text_span_start (const diagnostic_context &dc,
-			 pretty_printer &pp,
-			 const expanded_location &exploc);
-
-  void
-  print_html_span_start (const diagnostic_context &dc,
-			 xml::printer &xp,
-			 const expanded_location &exploc);
-
-private:
-  diagnostic_column_policy m_column_policy;
-  bool m_show_column;
-};
-
-/* Abstract base class for optionally supplying extra tags when writing
-   out annotation labels in HTML output.  */
-
-class html_label_writer
-{
-public:
-  virtual ~html_label_writer () {}
-  virtual void begin_label () = 0;
-  virtual void end_label () = 0;
-};
-
-/* A bundle of state for printing source within a diagnostic,
-   to isolate the interactions between diagnostic_context and the
-   implementation of diagnostic_show_locus.  */
-
-class diagnostic_source_print_policy
-{
-public:
-  diagnostic_source_print_policy (const diagnostic_context &);
-  diagnostic_source_print_policy (const diagnostic_context &,
-				  const diagnostic_source_printing_options &);
-
-  void
-  print (pretty_printer &pp,
-	 const rich_location &richloc,
-	 diagnostic_t diagnostic_kind,
-	 diagnostic_source_effect_info *effect_info) const;
-
-  void
-  print_as_html (xml::printer &xp,
-		 const rich_location &richloc,
-		 diagnostic_t diagnostic_kind,
-		 diagnostic_source_effect_info *effect_info,
-		 html_label_writer *label_writer) const;
-
-  const diagnostic_source_printing_options &
-  get_options () const { return m_options; }
-
-  diagnostic_start_span_fn<to_text>
-  get_text_start_span_fn () const { return m_text_start_span_cb; }
-
-  diagnostic_start_span_fn<to_html>
-  get_html_start_span_fn () const { return m_html_start_span_cb; }
-
-  file_cache &
-  get_file_cache () const { return m_file_cache; }
-
-  enum diagnostics_escape_format
-  get_escape_format () const
-  {
-    return m_escape_format;
-  }
-
-  text_art::theme *
-  get_diagram_theme () const { return m_diagram_theme; }
-
-  const diagnostic_column_policy &get_column_policy () const
-  {
-    return m_location_policy.get_column_policy ();
-  }
-
-  const diagnostic_location_print_policy &get_location_policy () const
-  {
-    return m_location_policy;
-  }
-
-private:
-  const diagnostic_source_printing_options &m_options;
-  class diagnostic_location_print_policy m_location_policy;
-  diagnostic_start_span_fn<to_text> m_text_start_span_cb;
-  diagnostic_start_span_fn<to_html> m_html_start_span_cb;
-  file_cache &m_file_cache;
-
-  /* Other data copied from diagnostic_context.  */
-  text_art::theme *m_diagram_theme;
-  enum diagnostics_escape_format m_escape_format;
-};
-
-/* A collection of counters of diagnostics, per-kind
-   (e.g. "3 errors and 1 warning"), for use by both diagnostic_context
-   and by diagnostic_buffer.  */
-
-struct diagnostic_counters
-{
-  diagnostic_counters ();
-
-  void dump (FILE *out, int indent) const;
-  void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
-
-  int get_count (diagnostic_t kind) const { return m_count_for_kind[kind]; }
-
-  void move_to (diagnostic_counters &dest);
-  void clear ();
-
-  int m_count_for_kind[DK_LAST_DIAGNOSTIC_KIND];
-};
-
-/* This class encapsulates the state of the diagnostics subsystem
-   as a whole (either directly, or via owned objects of other classes, to
-   avoid global variables).
-
-   It has responsibility for:
-   - being a central place for clients to report diagnostics
-   - reporting those diagnostics to zero or more output sinks
-     (e.g. text vs SARIF)
-   - providing a "dump" member function for a debug dump of the state of
-     the diagnostics subsytem
-   - direct vs buffered diagnostics (see class diagnostic_buffer)
-   - tracking the original argv of the program (for SARIF output)
-   - crash-handling
-
-   It delegates responsibility to various other classes:
-   - the various output sinks (instances of diagnostic_output_format
-     subclasses)
-   - formatting of messages (class pretty_printer)
-   - an optional urlifier to inject URLs into formatted messages
-   - counting the number of diagnostics reported of each kind
-     (class diagnostic_counters)
-   - calling out to a diagnostic_option_manager to determine if
-     a particular warning is enabled or disabled
-   - tracking pragmas that enable/disable warnings in a range of
-     source code
-   - a cache for use when quoting the user's source code (class file_cache)
-   - a text_art::theme
-   - an edit_context for generating patches from fix-it hints
-   - diagnostic_client_data_hooks for metadata.
-
-   Try to avoid adding new responsibilities to this class itself, to avoid
-   the "blob" anti-pattern.  */
-
-class diagnostic_context
-{
-public:
-  /* Give access to m_text_callbacks.  */
-  friend diagnostic_text_starter_fn &
-  diagnostic_text_starter (diagnostic_context *context);
-  friend diagnostic_start_span_fn<to_text> &
-  diagnostic_start_span (diagnostic_context *context);
-  friend diagnostic_text_finalizer_fn &
-  diagnostic_text_finalizer (diagnostic_context *context);
-
-  friend class diagnostic_source_print_policy;
-  friend class diagnostic_text_output_format;
-  friend class diagnostic_buffer;
-
-  typedef void (*set_locations_callback_t) (diagnostic_context *,
-					    diagnostic_info *);
-
-  void initialize (int n_opts);
-  void color_init (int value);
-  void urls_init (int value);
-  void set_pretty_printer (std::unique_ptr<pretty_printer> pp);
-  void refresh_output_sinks ();
-
-  void finish ();
-
-  void dump (FILE *out) const;
-  void DEBUG_FUNCTION dump () const { dump (stderr); }
-
-  bool execution_failed_p () const;
-
-  void set_original_argv (unique_argv original_argv);
-  const char * const *get_original_argv ()
-  {
-    return const_cast<const char * const *> (m_original_argv);
-  }
-
-  void set_set_locations_callback (set_locations_callback_t cb)
-  {
-    m_set_locations_cb = cb;
-  }
-
-  void
-  initialize_input_context (diagnostic_input_charset_callback ccb,
-			    bool should_skip_bom);
-
-  void begin_group ();
-  void end_group ();
-
-  void push_nesting_level ();
-  void pop_nesting_level ();
-
-  bool warning_enabled_at (location_t loc, diagnostic_option_id option_id);
-
-  bool option_unspecified_p (diagnostic_option_id option_id) const
-  {
-    return m_option_classifier.option_unspecified_p (option_id);
-  }
-
-  bool emit_diagnostic_with_group (diagnostic_t kind,
-				   rich_location &richloc,
-				   const diagnostic_metadata *metadata,
-				   diagnostic_option_id option_id,
-				   const char *gmsgid, ...)
-    ATTRIBUTE_GCC_DIAG(6,7);
-  bool emit_diagnostic_with_group_va (diagnostic_t kind,
-				      rich_location &richloc,
-				      const diagnostic_metadata *metadata,
-				      diagnostic_option_id option_id,
-				      const char *gmsgid, va_list *ap)
-    ATTRIBUTE_GCC_DIAG(6,0);
-
-  bool report_diagnostic (diagnostic_info *);
-  void report_verbatim (text_info &);
-
-  diagnostic_t
-  classify_diagnostic (diagnostic_option_id option_id,
-		       diagnostic_t new_kind,
-		       location_t where)
-  {
-    return m_option_classifier.classify_diagnostic (this,
-						    option_id,
-						    new_kind,
-						    where);
-  }
-
-  void push_diagnostics (location_t where ATTRIBUTE_UNUSED)
-  {
-    m_option_classifier.push ();
-  }
-  void pop_diagnostics (location_t where)
-  {
-    m_option_classifier.pop (where);
-  }
-
-  void maybe_show_locus (const rich_location &richloc,
-			 const diagnostic_source_printing_options &opts,
-			 diagnostic_t diagnostic_kind,
-			 pretty_printer &pp,
-			 diagnostic_source_effect_info *effect_info);
-  void maybe_show_locus_as_html (const rich_location &richloc,
-				 const diagnostic_source_printing_options &opts,
-				 diagnostic_t diagnostic_kind,
-				 xml::printer &xp,
-				 diagnostic_source_effect_info *effect_info,
-				 html_label_writer *label_writer);
-
-  void emit_diagram (const diagnostic_diagram &diagram);
-
-  /* Various setters for use by option-handling logic.  */
-  void set_output_format (std::unique_ptr<diagnostic_output_format> output_format);
-  void set_text_art_charset (enum diagnostic_text_art_charset charset);
-  void set_client_data_hooks (std::unique_ptr<diagnostic_client_data_hooks> hooks);
-
-  void push_owned_urlifier (std::unique_ptr<urlifier>);
-  void push_borrowed_urlifier (const urlifier &);
-  void pop_urlifier ();
-
-  void create_edit_context ();
-  void set_warning_as_error_requested (bool val)
-  {
-    m_warning_as_error_requested = val;
-  }
-  void set_report_bug (bool val) { m_report_bug = val; }
-  void set_extra_output_kind (enum diagnostics_extra_output_kind kind)
-  {
-    m_extra_output_kind = kind;
-  }
-  void set_show_cwe (bool val) { m_show_cwe = val;  }
-  void set_show_rules (bool val) { m_show_rules = val; }
-  void set_show_highlight_colors (bool val);
-  void set_path_format (enum diagnostic_path_format val)
-  {
-    m_path_format = val;
-  }
-  void set_show_path_depths (bool val) { m_show_path_depths = val; }
-  void set_show_option_requested (bool val) { m_show_option_requested = val; }
-  void set_max_errors (int val) { m_max_errors = val; }
-  void set_escape_format (enum diagnostics_escape_format val)
-  {
-    m_escape_format = val;
-  }
-
-  void set_format_decoder (printer_fn format_decoder);
-  void set_prefixing_rule (diagnostic_prefixing_rule_t rule);
-
-  /* Various accessors.  */
-  bool warning_as_error_requested_p () const
-  {
-    return m_warning_as_error_requested;
-  }
-  bool show_path_depths_p () const { return m_show_path_depths; }
-  diagnostic_output_format &get_output_format (size_t idx) const;
-  enum diagnostic_path_format get_path_format () const { return m_path_format; }
-  enum diagnostics_escape_format get_escape_format () const
-  {
-    return m_escape_format;
-  }
-
-  file_cache &
-  get_file_cache () const
-  {
-    gcc_assert (m_file_cache);
-    return *m_file_cache;
-  }
-
-  edit_context *get_edit_context () const
-  {
-    return m_edit_context_ptr;
-  }
-  const diagnostic_client_data_hooks *get_client_data_hooks () const
-  {
-    return m_client_data_hooks;
-  }
-
-  const logical_location_manager *
-  get_logical_location_manager () const;
-
-  const urlifier *get_urlifier () const;
-
-  text_art::theme *get_diagram_theme () const { return m_diagrams.m_theme; }
-
-  int &diagnostic_count (diagnostic_t kind)
-  {
-    return m_diagnostic_counters.m_count_for_kind[kind];
-  }
-  int diagnostic_count (diagnostic_t kind) const
-  {
-    return m_diagnostic_counters.get_count (kind);
-  }
-
-  /* Option-related member functions.  */
-  inline bool option_enabled_p (diagnostic_option_id option_id) const
-  {
-    if (!m_option_mgr)
-      return true;
-    return m_option_mgr->option_enabled_p (option_id);
-  }
-
-  inline char *make_option_name (diagnostic_option_id option_id,
-				 diagnostic_t orig_diag_kind,
-				 diagnostic_t diag_kind) const
-  {
-    if (!m_option_mgr)
-      return nullptr;
-    return m_option_mgr->make_option_name (option_id,
-					   orig_diag_kind,
-					   diag_kind);
-  }
-
-  inline char *make_option_url (diagnostic_option_id option_id) const
-  {
-    if (!m_option_mgr)
-      return nullptr;
-    return m_option_mgr->make_option_url (option_id);
-  }
-
-  void
-  set_option_manager (std::unique_ptr<diagnostic_option_manager> mgr,
-		      unsigned lang_mask);
-
-  unsigned get_lang_mask () const
-  {
-    return m_lang_mask;
-  }
-
-  bool diagnostic_impl (rich_location *, const diagnostic_metadata *,
-			diagnostic_option_id, const char *,
-			va_list *, diagnostic_t) ATTRIBUTE_GCC_DIAG(5,0);
-  bool diagnostic_n_impl (rich_location *, const diagnostic_metadata *,
-			  diagnostic_option_id, unsigned HOST_WIDE_INT,
-			  const char *, const char *, va_list *,
-			  diagnostic_t) ATTRIBUTE_GCC_DIAG(7,0);
-
-  int get_diagnostic_nesting_level () const
-  {
-    return m_diagnostic_groups.m_diagnostic_nesting_level;
-  }
-
-  char *build_indent_prefix () const;
-
-  int
-  pch_save (FILE *f)
-  {
-    return m_option_classifier.pch_save (f);
-  }
-
-  int
-  pch_restore (FILE *f)
-  {
-    return m_option_classifier.pch_restore (f);
-  }
-
-
-  void set_diagnostic_buffer (diagnostic_buffer *);
-  diagnostic_buffer *get_diagnostic_buffer () const
-  {
-    return m_diagnostic_buffer;
-  }
-  void clear_diagnostic_buffer (diagnostic_buffer &);
-  void flush_diagnostic_buffer (diagnostic_buffer &);
-
-  std::unique_ptr<pretty_printer> clone_printer () const
-  {
-    return m_reference_printer->clone ();
-  }
-
-  pretty_printer *get_reference_printer () const
-  {
-    return m_reference_printer;
-  }
-
-  void
-  add_sink (std::unique_ptr<diagnostic_output_format>);
-
-  void remove_all_output_sinks ();
-
-  bool supports_fnotice_on_stderr_p () const;
-
-  /* Raise SIGABRT on any diagnostic of severity DK_ERROR or higher.  */
-  void
-  set_abort_on_error (bool val)
-  {
-    m_abort_on_error = val;
-  }
-
-  /* Accessor for use in serialization, e.g. by C++ modules.  */
-  auto &
-  get_classification_history ()
-  {
-    return m_option_classifier.m_classification_history;
-  }
-
-  void set_main_input_filename (const char *filename);
-
-  void
-  set_permissive_option (diagnostic_option_id opt_permissive)
-  {
-    m_opt_permissive = opt_permissive;
-  }
-
-  void
-  set_fatal_errors (bool fatal_errors)
-  {
-    m_fatal_errors = fatal_errors;
-  }
-
-  void
-  set_internal_error_callback (void (*cb) (diagnostic_context *,
-					   const char *,
-					   va_list *))
-  {
-    m_internal_error = cb;
-  }
-
-  void
-  set_adjust_diagnostic_info_callback (void (*cb) (diagnostic_context *,
-						   diagnostic_info *))
-  {
-    m_adjust_diagnostic_info = cb;
-  }
-
-  void
-  inhibit_notes () { m_inhibit_notes_p = true; }
-
-private:
-  void error_recursion () ATTRIBUTE_NORETURN;
-
-  bool diagnostic_enabled (diagnostic_info *diagnostic);
-
-  void get_any_inlining_info (diagnostic_info *diagnostic);
-
-  void check_max_errors (bool flush);
-  void action_after_output (diagnostic_t diag_kind);
-
-  /* Data members.
-     Ideally, all of these would be private.  */
-
-private:
-  /* A reference instance of pretty_printer created by the client
-     and owned by the context.  Used for cloning when creating/adding
-     output formats.
-     Owned by the context; this would be a std::unique_ptr if
-     diagnostic_context had a proper ctor.  */
-  pretty_printer *m_reference_printer;
-
-  /* Cache of source code.
-     Owned by the context; this would be a std::unique_ptr if
-     diagnostic_context had a proper ctor.  */
-  file_cache *m_file_cache;
-
-  /* The number of times we have issued diagnostics.  */
-  diagnostic_counters m_diagnostic_counters;
-
-  /* True if it has been requested that warnings be treated as errors.  */
-  bool m_warning_as_error_requested;
-
-  /* The number of option indexes that can be passed to warning() et
-     al.  */
-  int m_n_opts;
-
-  /* The stack of sets of overridden diagnostic option severities.  */
-  diagnostic_option_classifier m_option_classifier;
-
-  /* True if we should print any CWE identifiers associated with
-     diagnostics.  */
-  bool m_show_cwe;
-
-  /* True if we should print any rules associated with diagnostics.  */
-  bool m_show_rules;
-
-  /* How should diagnostic_path objects be printed.  */
-  enum diagnostic_path_format m_path_format;
-
-  /* True if we should print stack depths when printing diagnostic paths.  */
-  bool m_show_path_depths;
-
-  /* True if we should print the command line option which controls
-     each diagnostic, if known.  */
-  bool m_show_option_requested;
-
-  /* True if we should raise a SIGABRT on errors.  */
-  bool m_abort_on_error;
-
-public:
-  /* True if we should show the column number on diagnostics.  */
-  bool m_show_column;
-
-  /* True if pedwarns are errors.  */
-  bool m_pedantic_errors;
-
-  /* True if permerrors are warnings.  */
-  bool m_permissive;
-
-private:
-  /* The option to associate with turning permerrors into warnings,
-     if any.  */
-  diagnostic_option_id m_opt_permissive;
-
-  /* True if errors are fatal.  */
-  bool m_fatal_errors;
-
-public:
-  /* True if all warnings should be disabled.  */
-  bool m_inhibit_warnings;
-
-  /* True if warnings should be given in system headers.  */
-  bool m_warn_system_headers;
-
-private:
-  /* Maximum number of errors to report.  */
-  int m_max_errors;
-
-  /* Client-supplied callbacks for use in text output.  */
-  struct {
-    /* This function is called before any message is printed out.  It is
-       responsible for preparing message prefix and such.  For example, it
-       might say:
-       In file included from "/usr/local/include/curses.h:5:
-       from "/home/gdr/src/nifty_printer.h:56:
-       ...
-    */
-    diagnostic_text_starter_fn m_begin_diagnostic;
-
-    /* This function is called by diagnostic_show_locus in between
-       disjoint spans of source code, so that the context can print
-       something to indicate that a new span of source code has begun.  */
-    diagnostic_start_span_fn<to_text> m_text_start_span;
-    diagnostic_start_span_fn<to_html> m_html_start_span;
-
-    /* This function is called after the diagnostic message is printed.  */
-    diagnostic_text_finalizer_fn m_end_diagnostic;
-  } m_text_callbacks;
-
-  /* Client hook to report an internal error.  */
-  void (*m_internal_error) (diagnostic_context *, const char *, va_list *);
-
-  /* Client hook to adjust properties of the given diagnostic that we're
-     about to issue, such as its kind.  */
-  void (*m_adjust_diagnostic_info)(diagnostic_context *, diagnostic_info *);
-
-  /* Owned by the context; this would be a std::unique_ptr if
-     diagnostic_context had a proper ctor.  */
-  diagnostic_option_manager *m_option_mgr;
-  unsigned m_lang_mask;
-
-  /* A stack of optional hooks for adding URLs to quoted text strings in
-     diagnostics.  Only used for the main diagnostic message.
-     Typically a single one owner by the context, but can be temporarily
-     overridden by a borrowed urlifier (e.g. on-stack).  */
-  struct urlifier_stack_node
-  {
-    urlifier *m_urlifier;
-    bool m_owned;
-  };
-  auto_vec<urlifier_stack_node> *m_urlifier_stack;
-
-public:
-  /* Auxiliary data for client.  */
-  void *m_client_aux_data;
-
-  /* Used to detect that the last caret was printed at the same location.  */
-  location_t m_last_location;
-
-private:
-  int m_lock;
-
-  bool m_inhibit_notes_p;
-
-public:
-  diagnostic_source_printing_options m_source_printing;
-
-private:
-  /* True if -freport-bug option is used.  */
-  bool m_report_bug;
-
-  /* Used to specify additional diagnostic output to be emitted after the
-     rest of the diagnostic.  This is for implementing
-     -fdiagnostics-parseable-fixits and GCC_EXTRA_DIAGNOSTIC_OUTPUT.  */
-  enum diagnostics_extra_output_kind m_extra_output_kind;
-
-public:
-  /* What units to use when outputting the column number.  */
-  enum diagnostics_column_unit m_column_unit;
-
-  /* The origin for the column number (1-based or 0-based typically).  */
-  int m_column_origin;
-
-  /* The size of the tabstop for tab expansion.  */
-  int m_tabstop;
-
-private:
-  /* How should non-ASCII/non-printable bytes be escaped when
-     a diagnostic suggests escaping the source code on output.  */
-  enum diagnostics_escape_format m_escape_format;
-
-  /* If non-NULL, an edit_context to which fix-it hints should be
-     applied, for generating patches.
-     Owned by the context; this would be a std::unique_ptr if
-     diagnostic_context had a proper ctor.  */
-  edit_context *m_edit_context_ptr;
-
-  /* Fields relating to diagnostic groups.  */
-  struct {
-    /* How many diagnostic_group instances are currently alive.  */
-    int m_group_nesting_depth;
-
-    /* How many nesting levels have been pushed within this group.  */
-    int m_diagnostic_nesting_level;
-
-    /* How many diagnostics have been emitted since the bottommost
-       diagnostic_group was pushed.  */
-    int m_emission_count;
-
-    /* The "group+diagnostic" nesting depth from which to inhibit notes.  */
-    int m_inhibiting_notes_from;
-  } m_diagnostic_groups;
-
-  void inhibit_notes_in_group (bool inhibit = true);
-  bool notes_inhibited_in_group () const;
-
-  /* The various sinks to which diagnostics are to be outputted
-     (text vs structured formats such as SARIF).
-     The sinks are owned by the context; this would be a
-     std::vector<std::unique_ptr> if diagnostic_context had a
-     proper ctor.  */
-  auto_vec<diagnostic_output_format *> m_output_sinks;
-
-  /* Callback to set the locations of call sites along the inlining
-     stack corresponding to a diagnostic location.  Needed to traverse
-     the BLOCK_SUPERCONTEXT() chain hanging off the LOCATION_BLOCK()
-     of a diagnostic's location.  */
-  set_locations_callback_t m_set_locations_cb;
-
-  /* A bundle of hooks for providing data to the context about its client
-     e.g. version information, plugins, etc.
-     Used by SARIF output to give metadata about the client that's
-     producing diagnostics.
-     Owned by the context; this would be a std::unique_ptr if
-     diagnostic_context had a proper ctor.  */
-  diagnostic_client_data_hooks *m_client_data_hooks;
-
-  /* Support for diagrams.  */
-  struct
-  {
-    /* Theme to use when generating diagrams.
-       Can be NULL (if text art is disabled).
-       Owned by the context; this would be a std::unique_ptr if
-       diagnostic_context had a proper ctor.  */
-    text_art::theme *m_theme;
-
-  } m_diagrams;
-
-  /* Owned by the context.  */
-  char **m_original_argv;
-
-  /* Borrowed pointer to the active diagnostic_buffer, if any.
-     If null (the default), then diagnostics that are reported to the
-     context are immediately issued to the output format.
-     If non-null, then diagnostics that are reported to the context
-     are buffered in the buffer, and may be issued to the output format
-     later (if the buffer is flushed), moved to other buffers, or
-     discarded (if the buffer is cleared).  */
-  diagnostic_buffer *m_diagnostic_buffer;
-};
-
-/* Client supplied function to announce a diagnostic
-   (for text-based diagnostic output).  */
-inline diagnostic_text_starter_fn &
-diagnostic_text_starter (diagnostic_context *context)
-{
-  return context->m_text_callbacks.m_begin_diagnostic;
-}
-
-/* Client supplied function called between disjoint spans of source code,
-   so that the context can print
-   something to indicate that a new span of source code has begun.  */
-inline diagnostic_start_span_fn<to_text> &
-diagnostic_start_span (diagnostic_context *context)
-{
-  return context->m_text_callbacks.m_text_start_span;
-}
-
-/* Client supplied function called after a diagnostic message is
-   displayed (for text-based diagnostic output).  */
-inline diagnostic_text_finalizer_fn &
-diagnostic_text_finalizer (diagnostic_context *context)
-{
-  return context->m_text_callbacks.m_end_diagnostic;
-}
+#include "diagnostics/diagnostic-info.h"
+#include "diagnostics/context.h"
 
 /* Extension hooks for client.  */
 #define diagnostic_context_auxiliary_data(DC) (DC)->m_client_aux_data
-#define diagnostic_info_auxiliary_data(DI) (DI)->x_data
+#define diagnostic_info_auxiliary_data(DI) (DI)->m_x_data
 
-/* This diagnostic_context is used by front-ends that directly output
+/* This diagnostics::context is used by front-ends that directly output
    diagnostic messages without going through `error', `warning',
    and similar functions.  */
-extern diagnostic_context *global_dc;
+extern diagnostics::context *global_dc;
 
 /* The number of errors that have been issued so far.  Ideally, these
-   would take a diagnostic_context as an argument.  */
-#define errorcount global_dc->diagnostic_count (DK_ERROR)
+   would take a diagnostics::context as an argument.  */
+#define errorcount global_dc->diagnostic_count (diagnostics::kind::error)
 /* Similarly, but for warnings.  */
-#define warningcount global_dc->diagnostic_count (DK_WARNING)
+#define warningcount global_dc->diagnostic_count (diagnostics::kind::warning)
 /* Similarly, but for warnings promoted to errors.  */
-#define werrorcount global_dc->diagnostic_count (DK_WERROR)
+#define werrorcount global_dc->diagnostic_count (diagnostics::kind::werror)
 /* Similarly, but for sorrys.  */
-#define sorrycount global_dc->diagnostic_count (DK_SORRY)
+#define sorrycount global_dc->diagnostic_count (diagnostics::kind::sorry)
 
 /* Returns nonzero if warnings should be emitted.  */
 #define diagnostic_report_warnings_p(DC, LOC)				\
@@ -1160,45 +56,45 @@ extern diagnostic_context *global_dc;
    diagnostic.  */
 
 inline void
-diagnostic_set_option_id (diagnostic_info *info,
-			  diagnostic_option_id option_id)
+diagnostic_set_option_id (diagnostics::diagnostic_info *info,
+			  diagnostics::option_id opt_id)
 {
-  info->option_id = option_id;
+  info->m_option_id = opt_id;
 }
 
 /* Diagnostic related functions.  */
 
 inline void
-diagnostic_initialize (diagnostic_context *context, int n_opts)
+diagnostic_initialize (diagnostics::context *context, int n_opts)
 {
   context->initialize (n_opts);
 }
 
 inline void
-diagnostic_color_init (diagnostic_context *context, int value = -1)
+diagnostic_color_init (diagnostics::context *context, int value = -1)
 {
   context->color_init (value);
 }
 
 inline void
-diagnostic_urls_init (diagnostic_context *context, int value = -1)
+diagnostic_urls_init (diagnostics::context *context, int value = -1)
 {
   context->urls_init (value);
 }
 
 inline void
-diagnostic_finish (diagnostic_context *context)
+diagnostic_finish (diagnostics::context *context)
 {
   context->finish ();
 }
 
 inline void
-diagnostic_show_locus (diagnostic_context *context,
-		       const diagnostic_source_printing_options &opts,
+diagnostic_show_locus (diagnostics::context *context,
+		       const diagnostics::source_printing_options &opts,
 		       rich_location *richloc,
-		       diagnostic_t diagnostic_kind,
+		       enum diagnostics::kind diagnostic_kind,
 		       pretty_printer *pp,
-		       diagnostic_source_effect_info *effect_info = nullptr)
+		       diagnostics::source_effect_info *effect_info = nullptr)
 {
   gcc_assert (context);
   gcc_assert (richloc);
@@ -1207,13 +103,13 @@ diagnostic_show_locus (diagnostic_context *context,
 }
 
 inline void
-diagnostic_show_locus_as_html (diagnostic_context *context,
-			       const diagnostic_source_printing_options &opts,
+diagnostic_show_locus_as_html (diagnostics::context *context,
+			       const diagnostics::source_printing_options &opts,
 			       rich_location *richloc,
-			       diagnostic_t diagnostic_kind,
+			       enum diagnostics::kind diagnostic_kind,
 			       xml::printer &xp,
-			       diagnostic_source_effect_info *effect_info = nullptr,
-			       html_label_writer *label_writer = nullptr)
+			       diagnostics::source_effect_info *effect_info = nullptr,
+			       diagnostics::html_label_writer *label_writer = nullptr)
 {
   gcc_assert (context);
   gcc_assert (richloc);
@@ -1237,7 +133,7 @@ diagnostic_show_locus_as_html (diagnostic_context *context,
    rather skipped as part of the conversion process.)  */
 
 inline void
-diagnostic_initialize_input_context (diagnostic_context *context,
+diagnostic_initialize_input_context (diagnostics::context *context,
 				     diagnostic_input_charset_callback ccb,
 				     bool should_skip_bom)
 {
@@ -1245,23 +141,23 @@ diagnostic_initialize_input_context (diagnostic_context *context,
 }
 
 /* Force diagnostics controlled by OPTIDX to be kind KIND.  */
-inline diagnostic_t
-diagnostic_classify_diagnostic (diagnostic_context *context,
-				diagnostic_option_id option_id,
-				diagnostic_t kind,
+inline diagnostics::kind
+diagnostic_classify_diagnostic (diagnostics::context *context,
+				diagnostics::option_id opt_id,
+				enum diagnostics::kind kind,
 				location_t where)
 {
-  return context->classify_diagnostic (option_id, kind, where);
+  return context->classify_diagnostic (opt_id, kind, where);
 }
 
 inline void
-diagnostic_push_diagnostics (diagnostic_context *context,
+diagnostic_push_diagnostics (diagnostics::context *context,
 			     location_t where)
 {
   context->push_diagnostics (where);
 }
 inline void
-diagnostic_pop_diagnostics (diagnostic_context *context,
+diagnostic_pop_diagnostics (diagnostics::context *context,
 			    location_t where)
 {
   context->pop_diagnostics (where);
@@ -1275,8 +171,8 @@ diagnostic_pop_diagnostics (diagnostic_context *context,
    Return true if a diagnostic was printed, false otherwise.  */
 
 inline bool
-diagnostic_report_diagnostic (diagnostic_context *context,
-			      diagnostic_info *diagnostic)
+diagnostic_report_diagnostic (diagnostics::context *context,
+			      diagnostics::diagnostic_info *diagnostic)
 {
   context->begin_group ();
   bool warned = context->report_diagnostic (diagnostic);
@@ -1285,23 +181,30 @@ diagnostic_report_diagnostic (diagnostic_context *context,
 }
 
 #ifdef ATTRIBUTE_GCC_DIAG
-extern void diagnostic_set_info (diagnostic_info *, const char *, va_list *,
-				 rich_location *, diagnostic_t) ATTRIBUTE_GCC_DIAG(2,0);
-extern void diagnostic_set_info_translated (diagnostic_info *, const char *,
-					    va_list *, rich_location *,
-					    diagnostic_t)
+extern void diagnostic_set_info (diagnostics::diagnostic_info *,
+				 const char *, va_list *,
+				 rich_location *,
+				 enum diagnostics::kind)
+  ATTRIBUTE_GCC_DIAG(2,0);
+extern void diagnostic_set_info_translated (diagnostics::diagnostic_info *,
+					    const char *, va_list *,
+					    rich_location *,
+					    enum diagnostics::kind)
      ATTRIBUTE_GCC_DIAG(2,0);
 #endif
-void default_diagnostic_text_starter (diagnostic_text_output_format &,
-				      const diagnostic_info *);
-template <typename Sink>
-void default_diagnostic_start_span_fn (const diagnostic_location_print_policy &,
-				       Sink &sink,
-				       expanded_location);
-void default_diagnostic_text_finalizer (diagnostic_text_output_format &,
-					const diagnostic_info *,
-					diagnostic_t);
-void diagnostic_set_caret_max_width (diagnostic_context *context, int value);
+
+namespace diagnostics {
+
+void default_text_starter (diagnostics::text_sink &,
+			   const diagnostics::diagnostic_info *);
+template <typename TextOrHtml>
+void default_start_span_fn (const diagnostics::location_print_policy &,
+			    TextOrHtml &text_or_html,
+			    expanded_location);
+void default_text_finalizer (diagnostics::text_sink &,
+			     const diagnostics::diagnostic_info *,
+			     enum diagnostics::kind);
+} // namespace diagnostics
 
 int get_terminal_width (void);
 
@@ -1309,17 +212,18 @@ int get_terminal_width (void);
    specifies which location. By default, expand the first one.  */
 
 inline location_t
-diagnostic_location (const diagnostic_info * diagnostic, int which = 0)
+diagnostic_location (const diagnostics::diagnostic_info *diagnostic,
+		     int which = 0)
 {
-  return diagnostic->message.get_location (which);
+  return diagnostic->m_message.get_location (which);
 }
 
 /* Return the number of locations to be printed in DIAGNOSTIC.  */
 
 inline unsigned int
-diagnostic_num_locations (const diagnostic_info * diagnostic)
+diagnostic_num_locations (const diagnostics::diagnostic_info *diagnostic)
 {
-  return diagnostic->message.m_richloc->get_num_locations ();
+  return diagnostic->m_message.m_richloc->get_num_locations ();
 }
 
 /* Expand the location of this diagnostic. Use this function for
@@ -1327,9 +231,10 @@ diagnostic_num_locations (const diagnostic_info * diagnostic)
    expand the first one.  */
 
 inline expanded_location
-diagnostic_expand_location (const diagnostic_info * diagnostic, int which = 0)
+diagnostic_expand_location (const diagnostics::diagnostic_info *diagnostic,
+			    int which = 0)
 {
-  return diagnostic->richloc->get_expanded_location (which);
+  return diagnostic->m_richloc->get_expanded_location (which);
 }
 
 /* This is somehow the right-side margin of a caret line, that is, we
@@ -1342,39 +247,39 @@ const int CARET_LINE_MARGIN = 10;
    whether to print one or two caret lines.  */
 
 inline bool
-diagnostic_same_line (const diagnostic_context *context,
-		       expanded_location s1, expanded_location s2)
+diagnostic_same_line (const diagnostics::context *context,
+		      expanded_location s1, expanded_location s2)
 {
   return (s2.column && s1.line == s2.line
-	  && (context->m_source_printing.max_width - CARET_LINE_MARGIN
+	  && ((context->get_source_printing_options ().max_width
+	       - CARET_LINE_MARGIN)
 	      > abs (s1.column - s2.column)));
 }
 
-extern const char *diagnostic_get_color_for_kind (diagnostic_t kind);
-
 /* Pure text formatting support functions.  */
 
 extern char *build_message_string (const char *, ...) ATTRIBUTE_PRINTF_1;
 
-/* Compute the number of digits in the decimal representation of an integer.  */
-extern int num_digits (int);
-
 inline bool
-warning_enabled_at (location_t loc, diagnostic_option_id option_id)
+warning_enabled_at (location_t loc, diagnostics::option_id opt_id)
 {
-  return global_dc->warning_enabled_at (loc, option_id);
+  return global_dc->warning_enabled_at (loc, opt_id);
 }
 
 inline bool
-option_unspecified_p (diagnostic_option_id option_id)
+option_unspecified_p (diagnostics::option_id opt_id)
 {
-  return global_dc->option_unspecified_p (option_id);
+  return global_dc->option_unspecified_p (opt_id);
 }
 
-extern char *get_cwe_url (int cwe);
+namespace diagnostics {
 
-extern const char *get_diagnostic_kind_text (diagnostic_t kind);
+/* Compute the number of digits in the decimal representation of an integer.  */
+extern int num_digits (int);
+
+extern char *get_cwe_url (int cwe);
+extern const char *maybe_line_and_column (int line, int col);
 
-const char *maybe_line_and_column (int line, int col);
+} // namespace diagnostics
 
 #endif /* ! GCC_DIAGNOSTIC_H */
diff --git a/gcc/diagnostics/buffering.cc b/gcc/diagnostics/buffering.cc
new file mode 100644
index 0000000..29f039f
--- /dev/null
+++ b/gcc/diagnostics/buffering.cc
@@ -0,0 +1,199 @@
+/* Support for buffering diagnostics before flushing them to output sinks.
+   Copyright (C) 2024-2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "diagnostic.h"
+#include "diagnostics/buffering.h"
+#include "diagnostics/sink.h"
+
+namespace diagnostics {
+
+/* Methods fns of diagnostics::context relating to buffering.  */
+
+/* If BUFFER_ is non-null, use BUFFER as the active diagnostics::buffer on
+   this context.  BUFFER is borrowed.
+
+   If BUFFER_ is null, stop any buffering on this context until the next call
+   to this function.  */
+
+void
+context::set_diagnostic_buffer (buffer *buffer_)
+{
+  /* We don't allow changing buffering within a diagnostic group
+     (to simplify handling of buffered diagnostics within the
+     diagnostic_format implementations).  */
+  gcc_assert (m_diagnostic_groups.m_group_nesting_depth == 0);
+
+  /* Likewise, for simplicity, we only allow changing buffers
+     at nesting level 0.  */
+  gcc_assert (m_diagnostic_groups.m_diagnostic_nesting_level == 0);
+
+  m_diagnostic_buffer = buffer_;
+
+  if (buffer_)
+    {
+      buffer_->ensure_per_sink_buffers ();
+      gcc_assert (buffer_->m_per_sink_buffers);
+      gcc_assert (buffer_->m_per_sink_buffers->length ()
+		  == m_sinks.length ());
+      for (unsigned idx = 0; idx < m_sinks.length (); ++idx)
+	{
+	  auto sink_ = m_sinks[idx];
+	  auto per_sink_buffer = (*buffer_->m_per_sink_buffers)[idx];
+	  sink_->set_buffer (per_sink_buffer);
+	}
+    }
+  else
+    for (auto sink_ : m_sinks)
+      sink_->set_buffer (nullptr);
+}
+
+/* Clear BUFFER_ without flushing it.  */
+
+void
+context::clear_diagnostic_buffer (buffer &buffer_)
+{
+  if (buffer_.m_per_sink_buffers)
+    for (auto per_sink_buffer_ : *buffer_.m_per_sink_buffers)
+      per_sink_buffer_->clear ();
+
+  buffer_.m_diagnostic_counters.clear ();
+
+  /* We need to reset last_location, otherwise we may skip caret lines
+     when we actually give a diagnostic.  */
+  m_last_location = UNKNOWN_LOCATION;
+}
+
+/* Flush the diagnostics in BUFFER_ to this context, clearing BUFFER_.  */
+
+void
+context::flush_diagnostic_buffer (buffer &buffer_)
+{
+  bool had_errors
+    = (buffer_.diagnostic_count (kind::error) > 0
+       || buffer_.diagnostic_count (kind::werror) > 0);
+  if (buffer_.m_per_sink_buffers)
+    for (auto per_sink_buffer_ : *buffer_.m_per_sink_buffers)
+      per_sink_buffer_->flush ();
+  buffer_.m_diagnostic_counters.move_to (m_diagnostic_counters);
+
+  action_after_output (had_errors ? kind::error : kind::warning);
+  check_max_errors (true);
+}
+
+/* class diagnostics::buffer.  */
+
+buffer::buffer (context &ctxt)
+: m_ctxt (ctxt),
+  m_per_sink_buffers (nullptr)
+{
+}
+
+buffer::~buffer ()
+{
+  if (m_per_sink_buffers)
+    {
+      for (auto iter : *m_per_sink_buffers)
+	delete iter;
+      delete m_per_sink_buffers;
+    }
+}
+
+void
+buffer::dump (FILE *out, int indent) const
+{
+  m_diagnostic_counters.dump (out, indent + 2);
+  fprintf (out, "%*sm_per_sink_buffers:\n", indent, "");
+  if (m_per_sink_buffers)
+    for (auto per_sink_buffer_ : *m_per_sink_buffers)
+      per_sink_buffer_->dump (out, indent + 2);
+  else
+    fprintf (out, "%*s(none)\n", indent + 2, "");
+}
+
+bool
+buffer::empty_p () const
+{
+  if (m_per_sink_buffers)
+    for (auto per_sink_buffer_ : *m_per_sink_buffers)
+      /* Query initial buffer.  */
+      return per_sink_buffer_->empty_p ();
+  return true;
+}
+
+void
+buffer::move_to (buffer &dest)
+{
+  /* Bail if there's nothing to move.  */
+  if (!m_per_sink_buffers)
+    return;
+
+  m_diagnostic_counters.move_to (dest.m_diagnostic_counters);
+
+  if (!dest.m_per_sink_buffers)
+    {
+      /* Optimization for the "move to empty" case:
+	 simply move the vec to the dest.  */
+      dest.m_per_sink_buffers = m_per_sink_buffers;
+      m_per_sink_buffers = nullptr;
+      return;
+    }
+
+  dest.ensure_per_sink_buffers ();
+  gcc_assert (m_per_sink_buffers);
+  gcc_assert (m_per_sink_buffers->length ()
+	      == m_ctxt.m_sinks.length ());
+  gcc_assert (dest.m_per_sink_buffers);
+  gcc_assert (dest.m_per_sink_buffers->length ()
+	      == m_ctxt.m_sinks.length ());
+  for (unsigned idx = 0; idx < m_ctxt.m_sinks.length (); ++idx)
+    {
+      auto per_sink_buffer_src = (*m_per_sink_buffers)[idx];
+      auto per_sink_buffer_dest = (*dest.m_per_sink_buffers)[idx];
+      per_sink_buffer_src->move_to (*per_sink_buffer_dest);
+    }
+}
+
+/* Lazily get the output formats to create their own kind of buffers.
+   We can't change the output sinks on a context once this has been called
+   on any diagnostics::buffer instances for that context, since there's no
+   way to update all diagnostics::buffer instances for that context.  */
+
+void
+buffer::ensure_per_sink_buffers ()
+{
+  if (!m_per_sink_buffers)
+    {
+      m_per_sink_buffers = new auto_vec<per_sink_buffer *> ();
+      for (unsigned idx = 0; idx < m_ctxt.m_sinks.length (); ++idx)
+	{
+	  auto sink_ = m_ctxt.m_sinks[idx];
+	  auto per_sink_buffer = sink_->make_per_sink_buffer ();
+	  m_per_sink_buffers->safe_push (per_sink_buffer.release ());
+	}
+    }
+  gcc_assert (m_per_sink_buffers);
+  gcc_assert (m_per_sink_buffers->length ()
+	      == m_ctxt.m_sinks.length ());
+}
+
+} // namespace diagnostics
diff --git a/gcc/diagnostic-buffer.h b/gcc/diagnostics/buffering.h
index e766a3b..c3ac070 100644
--- a/gcc/diagnostic-buffer.h
+++ b/gcc/diagnostics/buffering.h
@@ -1,4 +1,4 @@
-/* Support for buffering diagnostics before flushing them to output format.
+/* Support for buffering diagnostics before flushing them to output sinks.
    Copyright (C) 2024-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -18,26 +18,28 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_BUFFER_H
-#define GCC_DIAGNOSTIC_BUFFER_H
+#ifndef GCC_DIAGNOSTICS_BUFFERING_H
+#define GCC_DIAGNOSTICS_BUFFERING_H
 
 #include "diagnostic.h"
 
-class diagnostic_per_format_buffer;
-class diagnostic_output_format;
-  class diagnostic_text_output_format;
+namespace diagnostics {
+
+class per_sink_buffer;
+class sink;
+  class text_sink;
 
 /* Class representing a buffer of zero or more diagnostics that
-   have been reported to a diagnostic_context, but which haven't
+   have been reported to a diagnostics::context, but which haven't
    yet been flushed.
 
-   A diagnostic_buffer can be:
+   A diagnostics::buffer can be:
 
-   * flushed to the diagnostic_context, which issues
+   * flushed to the diagnostics::context, which issues
    the diagnostics within the buffer to the output format
    and checks for limits such as -fmax-errors=, or
 
-   * moved to another diagnostic_buffer, which moves the diagnostics
+   * moved to another diagnostics::buffer, which moves the diagnostics
    within the first buffer to the other buffer, appending them after any
    existing diagnostics within the destination buffer, emptying the
    source buffer, or
@@ -47,63 +49,65 @@ class diagnostic_output_format;
 
    Since a buffer needs to contain output-format-specific data,
    it's not possible to change the output format of the
-   diagnostic_context once any buffers are non-empty.
+   diagnostics::context once any buffers are non-empty.
 
    To simplify implementing output formats, it's not possible
-   to change buffering on a diagnostic_context whilst within a
+   to change buffering on a diagnostics::context whilst within a
    diagnostic group.  */
 
-class diagnostic_buffer
+class buffer
 {
  public:
-  friend class diagnostic_context;
+  friend class context;
 
-  diagnostic_buffer (diagnostic_context &ctxt);
-  ~diagnostic_buffer ();
+  buffer (context &ctxt);
+  ~buffer ();
 
   void dump (FILE *out, int indent) const;
   void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
 
-  int diagnostic_count (diagnostic_t kind) const
+  int diagnostic_count (enum kind kind) const
   {
     return m_diagnostic_counters.get_count (kind);
   }
 
   bool empty_p () const;
 
-  void move_to (diagnostic_buffer &dest);
+  void move_to (buffer &dest);
 
  private:
-  void ensure_per_format_buffers ();
+  void ensure_per_sink_buffers ();
 
-  diagnostic_context &m_ctxt;
-  auto_vec<diagnostic_per_format_buffer *> *m_per_format_buffers;
+  context &m_ctxt;
+  auto_vec<per_sink_buffer *> *m_per_sink_buffers;
 
   /* The number of buffered diagnostics of each kind.  */
-  diagnostic_counters m_diagnostic_counters;
+  counters m_diagnostic_counters;
 };
 
-/* Implementation detail of diagnostic_buffer.
+/* Implementation detail of diagnostics::buffer.
 
    Abstract base class describing how to represent zero of more
-   buffered diagnostics for a particular diagnostic_output_format
+   buffered diagnostics for a particular diagnostics::sink
    (e.g. text vs SARIF).
 
-   Each diagnostic_output_format subclass should implement its own
-   subclass for handling diagnostic_buffer.  */
+   Each diagnostics::sink subclass should implement its own
+   subclass for handling diagnostics::buffer.  */
 
-class diagnostic_per_format_buffer
+class per_sink_buffer
 {
 public:
-  virtual ~diagnostic_per_format_buffer () {}
+  virtual ~per_sink_buffer () {}
 
   virtual void dump (FILE *out, int indent) const = 0;
   void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
 
   virtual bool empty_p () const = 0;
-  virtual void move_to (diagnostic_per_format_buffer &dest) = 0;
+  virtual void move_to (per_sink_buffer &dest) = 0;
   virtual void clear () = 0;
   virtual void flush () = 0;
 };
 
-#endif /* ! GCC_DIAGNOSTIC_BUFFER_H */
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_BUFFERING_H */
diff --git a/gcc/edit-context.cc b/gcc/diagnostics/changes.cc
index c767394..e1caab0 100644
--- a/gcc/edit-context.cc
+++ b/gcc/diagnostics/changes.cc
@@ -21,14 +21,18 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "line-map.h"
-#include "edit-context.h"
+#include "diagnostics/changes.h"
 #include "pretty-print.h"
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
+#include "diagnostics/file-cache.h"
 #include "selftest.h"
 
+namespace diagnostics {
+namespace changes {
+
 /* This file implements a way to track the effect of fix-its,
-   via a class edit_context; the other classes are support classes for
-   edit_context.
+   via a class change_set; the other classes are support classes for
+   change_set.
 
    A complication here is that fix-its are expressed relative to coordinates
    in the file when it was parsed, before any changes have been made, and
@@ -36,14 +40,14 @@ along with GCC; see the file COPYING3.  If not see
    later fix-its to allow for the changes made by earlier ones.  This
    is done by the various "get_effective_column" methods.
 
-   The "filename" params are required to outlive the edit_context (no
+   The "filename" params are required to outlive the change_set (no
    copy of the underlying str is taken, just the ptr).  */
 
-/* Forward decls.  class edit_context is declared within edit-context.h.
+/* Forward decls.  class change_set is declared within changes.h.
    The other types are declared here.  */
-class edit_context;
-class edited_file;
-class edited_line;
+class change_set;
+class changed_file;
+class changed_line;
 class line_event;
 
 /* A struct to hold the params of a print_diff call.  */
@@ -58,14 +62,14 @@ public:
   bool m_show_filenames;
 };
 
-/* The state of one named file within an edit_context: the filename,
+/* The state of one named file within an change_set: the filename,
    and the lines that have been edited so far.  */
 
-class edited_file
+class changed_file
 {
  public:
-  edited_file (edit_context &ec, const char *filename);
-  static void delete_cb (edited_file *file);
+  changed_file (change_set &ec, const char *filename);
+  static void delete_cb (changed_file *file);
 
   const char *get_filename () const { return m_filename; }
   char *get_content ();
@@ -76,7 +80,7 @@ class edited_file
 		    int replacement_len);
   int get_effective_column (int line, int column);
 
-  static int call_print_diff (const char *, edited_file *file,
+  static int call_print_diff (const char *, changed_file *file,
 			      void *user_data)
   {
     diff *d = (diff *)user_data;
@@ -86,7 +90,7 @@ class edited_file
 
   file_cache &get_file_cache () const
   {
-    return m_edit_context.get_file_cache ();
+    return m_change_set.get_file_cache ();
   }
 
  private:
@@ -94,8 +98,8 @@ class edited_file
   void print_diff (pretty_printer *pp, bool show_filenames);
   int print_diff_hunk (pretty_printer *pp, int old_start_of_hunk,
 		       int old_end_of_hunk, int new_start_of_hunk);
-  edited_line *get_line (int line);
-  edited_line *get_or_insert_line (int line);
+  changed_line *get_line (int line);
+  changed_line *get_or_insert_line (int line);
   int get_num_lines (bool *missing_trailing_newline);
 
   int get_effective_line_count (int old_start_of_hunk,
@@ -105,13 +109,13 @@ class edited_file
 				   int start_of_run,
 				   int end_of_run);
 
-  edit_context &m_edit_context;
+  change_set &m_change_set;
   const char *m_filename;
-  typed_splay_tree<int, edited_line *> m_edited_lines;
+  typed_splay_tree<int, changed_line *> m_changed_lines;
   int m_num_lines;
 };
 
-/* A line added before an edited_line.  */
+/* A line added before an changed_line.  */
 
 class added_line
 {
@@ -154,24 +158,24 @@ class line_event
   int m_delta;
 };
 
-/* The state of one edited line within an edited_file.
+/* The state of one edited line within an changed_file.
    As well as the current content of the line, it contains a record of
    the changes, so that further changes can be applied in the correct
    place.
 
    When handling fix-it hints containing newlines, new lines are added
-   as added_line predecessors to an edited_line.  Hence it's possible
-   for an "edited_line" to not actually have been changed, but to merely
+   as added_line predecessors to an changed_line.  Hence it's possible
+   for an "changed_line" to not actually have been changed, but to merely
    be a placeholder for the lines added before it.  This can be tested
    for with actuall_edited_p, and has a slight effect on how diff hunks
    are generated.  */
 
-class edited_line
+class changed_line
 {
  public:
-  edited_line (file_cache &fc, const char *filename, int line_num);
-  ~edited_line ();
-  static void delete_cb (edited_line *el);
+  changed_line (file_cache &fc, const char *filename, int line_num);
+  ~changed_line ();
+  static void delete_cb (changed_line *el);
 
   int get_line_num () const { return m_line_num; }
   const char *get_content () const { return m_content; }
@@ -210,21 +214,21 @@ static void
 print_diff_line (pretty_printer *pp, char prefix_char,
 		 const char *line, int line_size);
 
-/* Implementation of class edit_context.  */
+/* Implementation of class change_set.  */
 
-/* edit_context's ctor.  */
+/* change_set's ctor.  */
 
-edit_context::edit_context (file_cache &fc)
+change_set::change_set (file_cache &fc)
 : m_file_cache (fc),
   m_valid (true),
-  m_files (strcmp, NULL, edited_file::delete_cb)
+  m_files (strcmp, NULL, changed_file::delete_cb)
 {}
 
 /* Add any fixits within RICHLOC to this context, recording the
    changes that they make.  */
 
 void
-edit_context::add_fixits (rich_location *richloc)
+change_set::add_fixits (rich_location *richloc)
 {
   if (!m_valid)
     return;
@@ -242,15 +246,15 @@ edit_context::add_fixits (rich_location *richloc)
 }
 
 /* Get the content of the given file, with fix-its applied.
-   If any errors occurred in this edit_context, return NULL.
+   If any errors occurred in this change_set, return NULL.
    The ptr should be freed by the caller.  */
 
 char *
-edit_context::get_content (const char *filename)
+change_set::get_content (const char *filename)
 {
   if (!m_valid)
     return NULL;
-  edited_file &file = get_or_insert_file (filename);
+  changed_file &file = get_or_insert_file (filename);
   return file.get_content ();
 }
 
@@ -258,10 +262,10 @@ edit_context::get_content (const char *filename)
    This method is for the selftests.  */
 
 int
-edit_context::get_effective_column (const char *filename, int line,
+change_set::get_effective_column (const char *filename, int line,
 				    int column)
 {
-  edited_file *file = get_file (filename);
+  changed_file *file = get_file (filename);
   if (!file)
     return column;
   return file->get_effective_column (line, column);
@@ -269,10 +273,10 @@ edit_context::get_effective_column (const char *filename, int line,
 
 /* Generate a unified diff.  The resulting string should be freed by the
    caller.  Primarily for selftests.
-   If any errors occurred in this edit_context, return NULL.  */
+   If any errors occurred in this change_set, return NULL.  */
 
 char *
-edit_context::generate_diff (bool show_filenames)
+change_set::generate_diff (bool show_filenames)
 {
   if (!m_valid)
     return NULL;
@@ -286,19 +290,19 @@ edit_context::generate_diff (bool show_filenames)
    context.  */
 
 void
-edit_context::print_diff (pretty_printer *pp, bool show_filenames)
+change_set::print_diff (pretty_printer *pp, bool show_filenames)
 {
   if (!m_valid)
     return;
   diff d (pp, show_filenames);
-  m_files.foreach (edited_file::call_print_diff, &d);
+  m_files.foreach (changed_file::call_print_diff, &d);
 }
 
 /* Attempt to apply the given fixit.  Return true if it can be
    applied, or false otherwise.  */
 
 bool
-edit_context::apply_fixit (const fixit_hint *hint)
+change_set::apply_fixit (const fixit_hint *hint)
 {
   expanded_location start = expand_location (hint->get_start_loc ());
   expanded_location next_loc = expand_location (hint->get_next_loc ());
@@ -311,7 +315,7 @@ edit_context::apply_fixit (const fixit_hint *hint)
   if (next_loc.column == 0)
     return false;
 
-  edited_file &file = get_or_insert_file (start.file);
+  changed_file &file = get_or_insert_file (start.file);
   if (!m_valid)
     return false;
   return file.apply_fixit (start.line, start.column, next_loc.column,
@@ -319,57 +323,57 @@ edit_context::apply_fixit (const fixit_hint *hint)
 			   hint->get_length ());
 }
 
-/* Locate the edited_file * for FILENAME, if any
+/* Locate the changed_file * for FILENAME, if any
    Return NULL if there isn't one.  */
 
-edited_file *
-edit_context::get_file (const char *filename)
+changed_file *
+change_set::get_file (const char *filename)
 {
   gcc_assert (filename);
   return m_files.lookup (filename);
 }
 
-/* Locate the edited_file for FILENAME, adding one if there isn't one.  */
+/* Locate the changed_file for FILENAME, adding one if there isn't one.  */
 
-edited_file &
-edit_context::get_or_insert_file (const char *filename)
+changed_file &
+change_set::get_or_insert_file (const char *filename)
 {
   gcc_assert (filename);
 
-  edited_file *file = get_file (filename);
+  changed_file *file = get_file (filename);
   if (file)
     return *file;
 
   /* Not found.  */
-  file = new edited_file (*this, filename);
+  file = new changed_file (*this, filename);
   m_files.insert (filename, file);
   return *file;
 }
 
-/* Implementation of class edited_file.  */
+/* Implementation of class changed_file.  */
 
-/* Callback for m_edited_lines, for comparing line numbers.  */
+/* Callback for m_changed_lines, for comparing line numbers.  */
 
 static int line_comparator (int a, int b)
 {
   return a - b;
 }
 
-/* edited_file's constructor.  */
+/* changed_file's constructor.  */
 
-edited_file::edited_file (edit_context &ec, const char *filename)
-: m_edit_context (ec),
+changed_file::changed_file (change_set &ec, const char *filename)
+: m_change_set (ec),
   m_filename (filename),
-  m_edited_lines (line_comparator, NULL, edited_line::delete_cb),
+  m_changed_lines (line_comparator, NULL, changed_line::delete_cb),
   m_num_lines (-1)
 {
 }
 
-/* A callback for deleting edited_file *, for use as a
-   delete_value_fn for edit_context::m_files.  */
+/* A callback for deleting changed_file *, for use as a
+   delete_value_fn for change_set::m_files.  */
 
 void
-edited_file::delete_cb (edited_file *file)
+changed_file::delete_cb (changed_file *file)
 {
   delete file;
 }
@@ -378,7 +382,7 @@ edited_file::delete_cb (edited_file *file)
    The ptr should be freed by the caller.  */
 
 char *
-edited_file::get_content ()
+changed_file::get_content ()
 {
   pretty_printer pp;
   if (!print_content (&pp))
@@ -392,11 +396,11 @@ edited_file::get_content ()
    the line.  */
 
 bool
-edited_file::apply_fixit (int line, int start_column, int next_column,
+changed_file::apply_fixit (int line, int start_column, int next_column,
 			  const char *replacement_str,
 			  int replacement_len)
 {
-  edited_line *el = get_or_insert_line (line);
+  changed_line *el = get_or_insert_line (line);
   if (!el)
     return false;
   return el->apply_fixit (start_column, next_column, replacement_str,
@@ -407,9 +411,9 @@ edited_file::apply_fixit (int line, int start_column, int next_column,
    column after edits have been applied.  */
 
 int
-edited_file::get_effective_column (int line, int column)
+changed_file::get_effective_column (int line, int column)
 {
-  const edited_line *el = get_line (line);
+  const changed_line *el = get_line (line);
   if (!el)
     return column;
   return el->get_effective_column (column);
@@ -419,13 +423,13 @@ edited_file::get_effective_column (int line, int column)
    Return true if successful, false otherwise.  */
 
 bool
-edited_file::print_content (pretty_printer *pp)
+changed_file::print_content (pretty_printer *pp)
 {
   bool missing_trailing_newline;
   int line_count = get_num_lines (&missing_trailing_newline);
   for (int line_num = 1; line_num <= line_count; line_num++)
     {
-      edited_line *el = get_line (line_num);
+      changed_line *el = get_line (line_num);
       if (el)
 	el->print_content (pp);
       else
@@ -451,7 +455,7 @@ edited_file::print_content (pretty_printer *pp)
    to this file.  */
 
 void
-edited_file::print_diff (pretty_printer *pp, bool show_filenames)
+changed_file::print_diff (pretty_printer *pp, bool show_filenames)
 {
   if (show_filenames)
     {
@@ -466,7 +470,7 @@ edited_file::print_diff (pretty_printer *pp, bool show_filenames)
       pp_string (pp, colorize_stop (pp_show_color (pp)));
     }
 
-  edited_line *el = m_edited_lines.min ();
+  changed_line *el = m_changed_lines.min ();
 
   bool missing_trailing_newline;
   int line_count = get_num_lines (&missing_trailing_newline);
@@ -488,8 +492,8 @@ edited_file::print_diff (pretty_printer *pp, bool show_filenames)
 	 that are sufficiently close.  */
       while (true)
 	{
-	  edited_line *next_el
-	    = m_edited_lines.successor (el->get_line_num ());
+	  changed_line *next_el
+	    = m_changed_lines.successor (el->get_line_num ());
 	  if (!next_el)
 	    break;
 
@@ -514,7 +518,7 @@ edited_file::print_diff (pretty_printer *pp, bool show_filenames)
       int new_start_of_hunk = start_of_hunk + line_delta;
       line_delta += print_diff_hunk (pp, start_of_hunk, end_of_hunk,
 				     new_start_of_hunk);
-      el = m_edited_lines.successor (el->get_line_num ());
+      el = m_changed_lines.successor (el->get_line_num ());
     }
 }
 
@@ -525,7 +529,7 @@ edited_file::print_diff (pretty_printer *pp, bool show_filenames)
    Return the change in the line count within the hunk.  */
 
 int
-edited_file::print_diff_hunk (pretty_printer *pp, int old_start_of_hunk,
+changed_file::print_diff_hunk (pretty_printer *pp, int old_start_of_hunk,
 			      int old_end_of_hunk, int new_start_of_hunk)
 {
   int old_num_lines = old_end_of_hunk - old_start_of_hunk + 1;
@@ -541,7 +545,7 @@ edited_file::print_diff_hunk (pretty_printer *pp, int old_start_of_hunk,
   int line_num = old_start_of_hunk;
   while (line_num <= old_end_of_hunk)
     {
-      edited_line *el = get_line (line_num);
+      changed_line *el = get_line (line_num);
       if (el)
 	{
 	  /* We have an edited line.
@@ -566,12 +570,12 @@ edited_file::print_diff_hunk (pretty_printer *pp, int old_start_of_hunk,
   return new_num_lines - old_num_lines;
 }
 
-/* Subroutine of edited_file::print_diff_hunk: given a run of lines
-   from START_OF_RUN to END_OF_RUN that all have edited_line instances,
+/* Subroutine of changed_file::print_diff_hunk: given a run of lines
+   from START_OF_RUN to END_OF_RUN that all have changed_line instances,
    print the diff to PP.  */
 
 void
-edited_file::print_run_of_changed_lines (pretty_printer *pp,
+changed_file::print_run_of_changed_lines (pretty_printer *pp,
 					 int start_of_run,
 					 int end_of_run)
 {
@@ -582,7 +586,7 @@ edited_file::print_run_of_changed_lines (pretty_printer *pp,
        line_num <= end_of_run;
        line_num++)
     {
-      edited_line *el_in_run = get_line (line_num);
+      changed_line *el_in_run = get_line (line_num);
       gcc_assert (el_in_run);
       if (el_in_run->actually_edited_p ())
 	{
@@ -601,7 +605,7 @@ edited_file::print_run_of_changed_lines (pretty_printer *pp,
        line_num <= end_of_run;
        line_num++)
     {
-      edited_line *el_in_run = get_line (line_num);
+      changed_line *el_in_run = get_line (line_num);
       gcc_assert (el_in_run);
       el_in_run->print_diff_lines (pp);
     }
@@ -627,14 +631,14 @@ print_diff_line (pretty_printer *pp, char prefix_char,
    OLD_END_OF_HUNK inclusive.  */
 
 int
-edited_file::get_effective_line_count (int old_start_of_hunk,
+changed_file::get_effective_line_count (int old_start_of_hunk,
 				       int old_end_of_hunk)
 {
   int line_count = 0;
   for (int old_line_num = old_start_of_hunk; old_line_num <= old_end_of_hunk;
        old_line_num++)
     {
-      edited_line *el = get_line (old_line_num);
+      changed_line *el = get_line (old_line_num);
       if (el)
 	line_count += el->get_effective_line_count ();
       else
@@ -645,28 +649,28 @@ edited_file::get_effective_line_count (int old_start_of_hunk,
 
 /* Get the state of LINE within the file, or NULL if it is untouched.  */
 
-edited_line *
-edited_file::get_line (int line)
+changed_line *
+changed_file::get_line (int line)
 {
-  return m_edited_lines.lookup (line);
+  return m_changed_lines.lookup (line);
 }
 
 /* Get the state of LINE within the file, creating a state for it
    if necessary.  Return NULL if an error occurs.  */
 
-edited_line *
-edited_file::get_or_insert_line (int line)
+changed_line *
+changed_file::get_or_insert_line (int line)
 {
-  edited_line *el = get_line (line);
+  changed_line *el = get_line (line);
   if (el)
     return el;
-  el = new edited_line (get_file_cache (), m_filename, line);
+  el = new changed_line (get_file_cache (), m_filename, line);
   if (el->get_content () == NULL)
     {
       delete el;
       return NULL;
     }
-  m_edited_lines.insert (line, el);
+  m_changed_lines.insert (line, el);
   return el;
 }
 
@@ -675,7 +679,7 @@ edited_file::get_or_insert_line (int line)
    if missing a newline, false otherwise.  */
 
 int
-edited_file::get_num_lines (bool *missing_trailing_newline)
+changed_file::get_num_lines (bool *missing_trailing_newline)
 {
   gcc_assert (missing_trailing_newline);
   if (m_num_lines == -1)
@@ -696,11 +700,11 @@ edited_file::get_num_lines (bool *missing_trailing_newline)
   return m_num_lines;
 }
 
-/* Implementation of class edited_line.  */
+/* Implementation of class changed_line.  */
 
-/* edited_line's ctor.  */
+/* changed_line's ctor.  */
 
-edited_line::edited_line (file_cache &fc, const char *filename, int line_num)
+changed_line::changed_line (file_cache &fc, const char *filename, int line_num)
 : m_line_num (line_num),
   m_content (NULL), m_len (0), m_alloc_sz (0),
   m_line_events (),
@@ -715,9 +719,9 @@ edited_line::edited_line (file_cache &fc, const char *filename, int line_num)
   ensure_terminated ();
 }
 
-/* edited_line's dtor.  */
+/* changed_line's dtor.  */
 
-edited_line::~edited_line ()
+changed_line::~changed_line ()
 {
   unsigned i;
   added_line *pred;
@@ -727,11 +731,11 @@ edited_line::~edited_line ()
     delete pred;
 }
 
-/* A callback for deleting edited_line *, for use as a
-   delete_value_fn for edited_file::m_edited_lines.  */
+/* A callback for deleting changed_line *, for use as a
+   delete_value_fn for changed_file::m_changed_lines.  */
 
 void
-edited_line::delete_cb (edited_line *el)
+changed_line::delete_cb (changed_line *el)
 {
   delete el;
 }
@@ -740,7 +744,7 @@ edited_line::delete_cb (edited_line *el)
    within a specific line.  */
 
 int
-edited_line::get_effective_column (int orig_column) const
+changed_line::get_effective_column (int orig_column) const
 {
   int i;
   line_event *event;
@@ -756,7 +760,7 @@ edited_line::get_effective_column (int orig_column) const
    Return true if successful; false if an error occurred.  */
 
 bool
-edited_line::apply_fixit (int start_column,
+changed_line::apply_fixit (int start_column,
 			  int next_column,
 			  const char *replacement_str,
 			  int replacement_len)
@@ -824,16 +828,16 @@ edited_line::apply_fixit (int start_column,
    also be counted.  */
 
 int
-edited_line::get_effective_line_count () const
+changed_line::get_effective_line_count () const
 {
   return m_predecessors.length () + 1;
 }
 
-/* Subroutine of edited_file::print_content.
+/* Subroutine of changed_file::print_content.
    Print this line and any new lines added before it, to PP.  */
 
 void
-edited_line::print_content (pretty_printer *pp) const
+changed_line::print_content (pretty_printer *pp) const
 {
   unsigned i;
   added_line *pred;
@@ -845,18 +849,18 @@ edited_line::print_content (pretty_printer *pp) const
   pp_string (pp, m_content);
 }
 
-/* Subroutine of edited_file::print_run_of_changed_lines for
+/* Subroutine of changed_file::print_run_of_changed_lines for
    printing diff hunks to PP.
    Print the '+' line for this line, and any newlines added
    before it.
-   Note that if this edited_line was actually edited, the '-'
+   Note that if this changed_line was actually edited, the '-'
    line has already been printed.  If it wasn't, then we merely
-   have a placeholder edited_line for adding newlines to, and
-   we need to print a ' ' line for the edited_line as we haven't
+   have a placeholder changed_line for adding newlines to, and
+   we need to print a ' ' line for the changed_line as we haven't
    printed it yet.  */
 
 void
-edited_line::print_diff_lines (pretty_printer *pp) const
+changed_line::print_diff_lines (pretty_printer *pp) const
 {
   unsigned i;
   added_line *pred;
@@ -874,7 +878,7 @@ edited_line::print_diff_lines (pretty_printer *pp) const
    allocations.  */
 
 void
-edited_line::ensure_capacity (int len)
+changed_line::ensure_capacity (int len)
 {
   /* Allow 1 extra byte for 0-termination.  */
   if (m_alloc_sz < (len + 1))
@@ -888,7 +892,7 @@ edited_line::ensure_capacity (int len)
 /* Ensure that m_content is 0-terminated.  */
 
 void
-edited_line::ensure_terminated ()
+changed_line::ensure_terminated ()
 {
   /* 0-terminate the buffer.  */
   gcc_assert (m_len < m_alloc_sz);
@@ -901,6 +905,11 @@ edited_line::ensure_terminated ()
 
 namespace selftest {
 
+using line_table_case = ::selftest::line_table_case;
+using line_table_test = ::selftest::line_table_test;
+using temp_source_file = ::selftest::temp_source_file;
+using named_temp_file = ::selftest::named_temp_file;
+
 /* A wrapper class for ensuring that the underlying pointer is freed.  */
 
 template <typename POINTER_T>
@@ -916,7 +925,7 @@ class auto_free
   POINTER_T m_ptr;
 };
 
-/* Verify that edit_context::get_content works for unedited files.  */
+/* Verify that change_set::get_content works for unedited files.  */
 
 static void
 test_get_content ()
@@ -926,7 +935,7 @@ test_get_content ()
     const char *content = ("");
     temp_source_file tmp (SELFTEST_LOCATION, ".c", content);
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     auto_free <char *> result = edit.get_content (tmp.get_filename ());
     ASSERT_STREQ ("", result);
   }
@@ -938,7 +947,7 @@ test_get_content ()
 			   "/* after */\n");
     temp_source_file tmp (SELFTEST_LOCATION, ".c", content);
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     auto_free <char *> result = edit.get_content (tmp.get_filename ());
     ASSERT_STREQ ("/* before */\n"
 		  "foo = bar.field;\n"
@@ -952,7 +961,7 @@ test_get_content ()
 			   "/* after */");
     temp_source_file tmp (SELFTEST_LOCATION, ".c", content);
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     auto_free <char *> result = edit.get_content (tmp.get_filename ());
     /* We should respect the omitted trailing newline.  */
     ASSERT_STREQ ("/* before */\n"
@@ -986,7 +995,7 @@ test_applying_fixits_insert_before (const line_table_case &case_)
     return;
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   if (start <= LINE_MAP_MAX_LOCATION_WITH_COLS)
@@ -1045,7 +1054,7 @@ test_applying_fixits_insert_after (const line_table_case &case_)
 
   /* Verify that the text was inserted after the end of "field". */
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   ASSERT_STREQ ("/* before */\n"
@@ -1088,7 +1097,7 @@ test_applying_fixits_insert_after_at_line_end (const line_table_case &case_)
     return;
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   ASSERT_STREQ ("/* before */\n"
@@ -1147,7 +1156,7 @@ test_applying_fixits_insert_after_failure (const line_table_case &case_)
   ASSERT_TRUE (richloc.seen_impossible_fixit_p ());
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   ASSERT_FALSE (edit.valid_p ());
   ASSERT_EQ (NULL, edit.get_content (filename));
@@ -1185,7 +1194,7 @@ test_applying_fixits_insert_containing_newline (const line_table_case &case_)
     return;
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   ASSERT_STREQ (("    case 'a':\n"
@@ -1230,7 +1239,7 @@ test_applying_fixits_growing_replace (const line_table_case &case_)
   richloc.add_fixit_replace ("m_field");
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   if (finish <= LINE_MAP_MAX_LOCATION_WITH_COLS)
@@ -1276,7 +1285,7 @@ test_applying_fixits_shrinking_replace (const line_table_case &case_)
   richloc.add_fixit_replace ("field");
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   if (finish <= LINE_MAP_MAX_LOCATION_WITH_COLS)
@@ -1331,7 +1340,7 @@ test_applying_fixits_replace_containing_newline (const line_table_case &case_)
     return;
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   //ASSERT_STREQ ("foo\n  = bar ();\n", new_content);
@@ -1363,7 +1372,7 @@ test_applying_fixits_remove (const line_table_case &case_)
   richloc.add_fixit_remove (range);
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&richloc);
   auto_free <char *> new_content = edit.get_content (filename);
   if (finish <= LINE_MAP_MAX_LOCATION_WITH_COLS)
@@ -1429,7 +1438,7 @@ test_applying_fixits_multiple (const line_table_case &case_)
 			       "meadow");
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   edit.add_fixits (&insert_a);
   ASSERT_EQ (100, edit.get_effective_column (filename, 1, 100));
   ASSERT_EQ (1, edit.get_effective_column (filename, 2, 1));
@@ -1464,7 +1473,7 @@ test_applying_fixits_multiple (const line_table_case &case_)
    Add the text "CHANGED: " to the front of the given line.  */
 
 static location_t
-change_line (edit_context &edit, int line_num)
+change_line (change_set &edit, int line_num)
 {
   const line_map_ordinary *ord_map
     = LINEMAPS_LAST_ORDINARY_MAP (line_table);
@@ -1490,7 +1499,7 @@ change_line (edit_context &edit, int line_num)
    Add the text "INSERTED\n" in front of the given line.  */
 
 static location_t
-insert_line (edit_context &edit, int line_num)
+insert_line (change_set &edit, int line_num)
 {
   const line_map_ordinary *ord_map
     = LINEMAPS_LAST_ORDINARY_MAP (line_table);
@@ -1532,7 +1541,7 @@ test_applying_fixits_multiple_lines (const line_table_case &case_)
   linemap_position_for_column (line_table, 127);
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
 
   /* A run of consecutive lines.  */
   change_line (edit, 2);
@@ -1621,7 +1630,7 @@ test_applying_fixits_modernize_named_init (const line_table_case &case_)
   /* The order should not matter.  Do r1 then r2. */
   {
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     edit.add_fixits (&r1);
 
     /* Verify state after first replacement.  */
@@ -1655,7 +1664,7 @@ test_applying_fixits_modernize_named_init (const line_table_case &case_)
   /* Try again, doing r2 then r1; the new_content should be the same.  */
   {
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     edit.add_fixits (&r2);
     edit.add_fixits (&r1);
     auto_free <char *> new_content = edit.get_content (tmp.get_filename ());
@@ -1684,7 +1693,7 @@ test_applying_fixits_unreadable_file ()
   insert.add_fixit_insert_before ("change 2");
 
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   /* Attempting to add the fixits affecting the unreadable file
      should transition the edit from valid to invalid.  */
   ASSERT_TRUE (edit.valid_p ());
@@ -1715,10 +1724,10 @@ test_applying_fixits_line_out_of_range ()
   rich_location insert (line_table, loc);
   insert.add_fixit_insert_before ("change");
 
-  /* Verify that attempting the insertion puts an edit_context
+  /* Verify that attempting the insertion puts an change_set
      into an invalid state.  */
   file_cache fc;
-  edit_context edit (fc);
+  change_set edit (fc);
   ASSERT_TRUE (edit.valid_p ());
   edit.add_fixits (&insert);
   ASSERT_FALSE (edit.valid_p ());
@@ -1727,7 +1736,7 @@ test_applying_fixits_line_out_of_range ()
 }
 
 /* Verify the boundary conditions of column values in fix-it
-   hints applied to edit_context instances.  */
+   hints applied to change_set instances.  */
 
 static void
 test_applying_fixits_column_validation (const line_table_case &case_)
@@ -1756,7 +1765,7 @@ test_applying_fixits_column_validation (const line_table_case &case_)
     /* Col 15 is at the end of the line, so the insertion
        should succeed.  */
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     edit.add_fixits (&richloc);
     auto_free <char *> new_content = edit.get_content (tmp.get_filename ());
     if (c15 <= LINE_MAP_MAX_LOCATION_WITH_COLS)
@@ -1773,7 +1782,7 @@ test_applying_fixits_column_validation (const line_table_case &case_)
     /* Col 16 is beyond the end of the line, so the insertion
        should fail gracefully.  */
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     ASSERT_TRUE (edit.valid_p ());
     edit.add_fixits (&richloc);
     ASSERT_FALSE (edit.valid_p ());
@@ -1792,7 +1801,7 @@ test_applying_fixits_column_validation (const line_table_case &case_)
     /* Col 14 is at the end of the line, so the replacement
        should succeed.  */
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     edit.add_fixits (&richloc);
     auto_free <char *> new_content = edit.get_content (tmp.get_filename ());
     if (c14 <= LINE_MAP_MAX_LOCATION_WITH_COLS)
@@ -1810,7 +1819,7 @@ test_applying_fixits_column_validation (const line_table_case &case_)
     /* Col 15 is after the end of the line, so the replacement
        should fail; verify that the attempt fails gracefully.  */
     file_cache fc;
-    edit_context edit (fc);
+    change_set edit (fc);
     ASSERT_TRUE (edit.valid_p ());
     edit.add_fixits (&richloc);
     ASSERT_FALSE (edit.valid_p ());
@@ -1819,10 +1828,8 @@ test_applying_fixits_column_validation (const line_table_case &case_)
   }
 }
 
-/* Run all of the selftests within this file.  */
-
-void
-edit_context_cc_tests ()
+static void
+run_all_tests ()
 {
   test_get_content ();
   for_each_line_table_case (test_applying_fixits_insert_before);
@@ -1842,6 +1849,26 @@ edit_context_cc_tests ()
   for_each_line_table_case (test_applying_fixits_column_validation);
 }
 
+} // namespace diagnostics::changes::selftest
+
+#endif /* CHECKING_P */
+
+} // namespace diagnostics::changes
+
+#if CHECKING_P
+
+namespace selftest { // diagnostics::selftest
+
+/* Run all of the selftests within this file.  */
+
+void
+changes_cc_tests ()
+{
+  diagnostics::changes::selftest::run_all_tests ();
+}
+
 } // namespace selftest
 
 #endif /* CHECKING_P */
+
+} // namespace diagnostics
diff --git a/gcc/edit-context.h b/gcc/diagnostics/changes.h
index 9bd2aba..a5fe4b0 100644
--- a/gcc/edit-context.h
+++ b/gcc/diagnostics/changes.h
@@ -17,21 +17,25 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_EDIT_CONTEXT_H
-#define GCC_EDIT_CONTEXT_H
+#ifndef GCC_DIAGNOSTICS_CHANGES_H
+#define GCC_DIAGNOSTICS_CHANGES_H
 
 #include "typed-splay-tree.h"
 
 class fixit_hint;
-class edit_context;
-class edited_file;
+
+namespace diagnostics {
+namespace changes {
+
+class change_set;
+class changed_file;
 
 /* A set of changes to the source code.
 
    The changes are "atomic" - if any changes can't be applied,
    none of them can be (tracked by the m_valid flag).
    Similarly, attempts to add the changes from a rich_location flagged
-   as containing invalid changes mean that the whole of the edit_context
+   as containing invalid changes mean that the whole of the change_set
    is flagged as invalid.
 
    A complication here is that fix-its are expressed relative to coordinates
@@ -40,10 +44,10 @@ class edited_file;
    later fix-its to allow for the changes made by earlier ones.  This
    is done by the various "get_effective_column" methods.  */
 
-class edit_context
+class change_set
 {
  public:
-  edit_context (file_cache &);
+  change_set (file_cache &);
 
   bool valid_p () const { return m_valid; }
 
@@ -60,12 +64,15 @@ class edit_context
 
  private:
   bool apply_fixit (const fixit_hint *hint);
-  edited_file *get_file (const char *filename);
-  edited_file &get_or_insert_file (const char *filename);
+  changed_file *get_file (const char *filename);
+  changed_file &get_or_insert_file (const char *filename);
 
   file_cache &m_file_cache;
   bool m_valid;
-  typed_splay_tree<const char *, edited_file *> m_files;
+  typed_splay_tree<const char *, changed_file *> m_files;
 };
 
-#endif /* GCC_EDIT_CONTEXT_H.  */
+} // namespace diagnostics::changes
+} // namespace diagnostics
+
+#endif /* GCC_DIAGNOSTICS_CHANGES_H.  */
diff --git a/gcc/diagnostic-client-data-hooks.h b/gcc/diagnostics/client-data-hooks.h
index 9909172..94c51b2 100644
--- a/gcc/diagnostic-client-data-hooks.h
+++ b/gcc/diagnostics/client-data-hooks.h
@@ -18,32 +18,36 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_CLIENT_DATA_HOOKS_H
-#define GCC_DIAGNOSTIC_CLIENT_DATA_HOOKS_H
+#ifndef GCC_DIAGNOSTICS_CLIENT_DATA_HOOKS_H
+#define GCC_DIAGNOSTICS_CLIENT_DATA_HOOKS_H
 
-#include "logical-location.h"
+#include "diagnostics/logical-locations.h"
+
+namespace diagnostics {
 
 class sarif_object;
 class client_version_info;
 
-/* A bundle of additional metadata, owned by the diagnostic_context,
+/* A bundle of additional metadata, owned by the diagnostics::context,
    for querying things about the client, like version data.  */
 
-class diagnostic_client_data_hooks
+class client_data_hooks
 {
  public:
-  virtual ~diagnostic_client_data_hooks () {}
+  virtual ~client_data_hooks () {}
 
   /* Get version info for this client, or NULL.  */
   virtual const client_version_info *get_any_version_info () const = 0;
 
-  /* Get the current logical_location_manager for this client, or NULL.  */
-  virtual const logical_location_manager *get_logical_location_manager () const = 0;
+  /* Get the current logical_locations::manager for this client, or null.  */
+  virtual const logical_locations::manager *
+  get_logical_location_manager () const = 0;
 
-  /* Get the current logical_location, or null.
-     If this returns a non-null logical_location, then
-     get_logical_location_manager must return non-NULL.  */
-  virtual logical_location get_current_logical_location () const = 0;
+  /* Get the current logical location, or null.
+     If this returns a non-null logical location, then
+     get_logical_location_manager must return non-null.  */
+  virtual logical_locations::key
+  get_current_logical_location () const = 0;
 
   /* Get a sourceLanguage value for FILENAME, or return NULL.
      See SARIF v2.1.0 Appendix J for suggested values.  */
@@ -56,15 +60,9 @@ class diagnostic_client_data_hooks
   add_sarif_invocation_properties (sarif_object &invocation_obj) const = 0;
 };
 
-/* Factory function for making an instance of diagnostic_client_data_hooks
-   for use in the compiler (i.e. with knowledge of "tree", access to
-   langhooks, etc).  */
-
-extern std::unique_ptr<diagnostic_client_data_hooks> make_compiler_data_hooks ();
-
-class diagnostic_client_plugin_info;
+class client_plugin_info;
 
-/* Abstract base class for a diagnostic_context to get at
+/* Abstract base class for a diagnostics::context to get at
    version information about the client.  */
 
 class client_version_info
@@ -73,7 +71,7 @@ public:
   class plugin_visitor
   {
   public:
-    virtual void on_plugin (const diagnostic_client_plugin_info &) = 0;
+    virtual void on_plugin (const client_plugin_info &) = 0;
   };
 
   virtual ~client_version_info () {}
@@ -97,10 +95,10 @@ public:
   virtual void for_each_plugin (plugin_visitor &v) const = 0;
 };
 
-/* Abstract base class for a diagnostic_context to get at
+/* Abstract base class for a diagnostics::context to get at
    information about a specific plugin within a client.  */
 
-class diagnostic_client_plugin_info
+class client_plugin_info
 {
 public:
   /* For use e.g. by SARIF "name" property (SARIF v2.1.0 section 3.19.8).  */
@@ -115,4 +113,13 @@ public:
   virtual const char *get_version () const = 0;
 };
 
-#endif /* ! GCC_DIAGNOSTIC_CLIENT_DATA_HOOKS_H */
+} // namespace diagnostics
+
+/* Factory function for making an instance of client_data_hooks
+   for use in the compiler (i.e. with knowledge of "tree", access to
+   langhooks, etc).  */
+
+extern std::unique_ptr<diagnostics::client_data_hooks>
+make_compiler_data_hooks ();
+
+#endif /* ! GCC_DIAGNOSTICS_CLIENT_DATA_HOOKS_H */
diff --git a/gcc/diagnostic-color.cc b/gcc/diagnostics/color.cc
index e95aaeb..7b499fe 100644
--- a/gcc/diagnostic-color.cc
+++ b/gcc/diagnostics/color.cc
@@ -19,8 +19,8 @@
 #include "config.h"
 #define INCLUDE_VECTOR
 #include "system.h"
-#include "diagnostic-color.h"
-#include "diagnostic-url.h"
+#include "diagnostics/color.h"
+#include "diagnostics/url.h"
 #include "label-text.h"
 
 #ifdef __MINGW32__
@@ -475,6 +475,7 @@ determine_url_format (diagnostic_url_rule_t rule)
 
 #if CHECKING_P
 
+namespace diagnostics {
 namespace selftest {
 
 /* Test of an empty diagnostic_color_dict.  */
@@ -522,7 +523,7 @@ test_color_dict_envvar_parsing ()
 /* Run all of the selftests within this file.  */
 
 void
-diagnostic_color_cc_tests ()
+color_cc_tests ()
 {
   test_empty_color_dict ();
   test_default_color_dict ();
@@ -530,5 +531,6 @@ diagnostic_color_cc_tests ()
 }
 
 } // namespace selftest
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
diff --git a/gcc/diagnostic-color.h b/gcc/diagnostics/color.h
index 8aeaa57..42b67eb 100644
--- a/gcc/diagnostic-color.h
+++ b/gcc/diagnostics/color.h
@@ -38,8 +38,8 @@ along with GCC; see the file COPYING3.  If not see
 
    Written July 1992 by Mike Haertel.  */
 
-#ifndef GCC_DIAGNOSTIC_COLOR_H
-#define GCC_DIAGNOSTIC_COLOR_H
+#ifndef GCC_DIAGNOSTICS_COLOR_H
+#define GCC_DIAGNOSTICS_COLOR_H
 
 /* Whether to add color to diagnostics:
    o DIAGNOSTICS_COLOR_NO: never
@@ -62,4 +62,4 @@ colorize_start (bool show_color, const char *name)
   return colorize_start (show_color, name, strlen (name));
 }
 
-#endif /* ! GCC_DIAGNOSTIC_COLOR_H */
+#endif /* ! GCC_DIAGNOSTICS_COLOR_H */
diff --git a/gcc/diagnostics/context-options.h b/gcc/diagnostics/context-options.h
new file mode 100644
index 0000000..c4e92eb
--- /dev/null
+++ b/gcc/diagnostics/context-options.h
@@ -0,0 +1,116 @@
+/* Declare enums for diagnostics::context and related types.
+   Copyright (C) 2000-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_CONTEXT_OPTIONS_H
+#define GCC_DIAGNOSTICS_CONTEXT_OPTIONS_H
+
+/* An enum for controlling what units to use for the column number
+   when diagnostics are output, used by the -fdiagnostics-column-unit option.
+   Tabs will be expanded or not according to the value of -ftabstop.  The origin
+   (default 1) is controlled by -fdiagnostics-column-origin.  */
+
+enum diagnostics_column_unit
+{
+  /* The default from GCC 11 onwards: display columns.  */
+  DIAGNOSTICS_COLUMN_UNIT_DISPLAY,
+
+  /* The behavior in GCC 10 and earlier: simple bytes.  */
+  DIAGNOSTICS_COLUMN_UNIT_BYTE
+};
+
+/* An enum for controlling how to print non-ASCII characters/bytes when
+   a diagnostic suggests escaping the source code on output.  */
+
+enum diagnostics_escape_format
+{
+  /* Escape non-ASCII Unicode characters in the form <U+XXXX> and
+     non-UTF-8 bytes in the form <XX>.  */
+  DIAGNOSTICS_ESCAPE_FORMAT_UNICODE,
+
+  /* Escape non-ASCII bytes in the form <XX> (thus showing the underlying
+     encoding of non-ASCII Unicode characters).  */
+  DIAGNOSTICS_ESCAPE_FORMAT_BYTES
+};
+
+/* Enum for overriding the standard output format.  */
+
+enum diagnostics_output_format
+{
+  /* The default: textual output.  */
+  DIAGNOSTICS_OUTPUT_FORMAT_TEXT,
+
+  /* SARIF-based output, as JSON to stderr.  */
+  DIAGNOSTICS_OUTPUT_FORMAT_SARIF_STDERR,
+
+  /* SARIF-based output, to a JSON file.  */
+  DIAGNOSTICS_OUTPUT_FORMAT_SARIF_FILE
+};
+
+/* An enum for controlling how diagnostic paths should be printed.  */
+enum diagnostic_path_format
+{
+  /* Don't print diagnostic paths.  */
+  DPF_NONE,
+
+  /* Print diagnostic paths by emitting a separate "note" for every event
+     in the path.  */
+  DPF_SEPARATE_EVENTS,
+
+  /* Print diagnostic paths by consolidating events together where they
+     are close enough, and printing such runs of events with multiple
+     calls to diagnostic_show_locus, showing the individual events in
+     each run via labels in the source.  */
+  DPF_INLINE_EVENTS
+};
+
+/* An enum for capturing values of GCC_EXTRA_DIAGNOSTIC_OUTPUT,
+   and for -fdiagnostics-parseable-fixits.  */
+
+enum diagnostics_extra_output_kind
+{
+  /* No extra output, or an unrecognized value.  */
+  EXTRA_DIAGNOSTIC_OUTPUT_none,
+
+  /* Emit fix-it hints using the "fixits-v1" format, equivalent to
+     -fdiagnostics-parseable-fixits.  */
+  EXTRA_DIAGNOSTIC_OUTPUT_fixits_v1,
+
+  /* Emit fix-it hints using the "fixits-v2" format.  */
+  EXTRA_DIAGNOSTIC_OUTPUT_fixits_v2
+};
+
+/* Values for -fdiagnostics-text-art-charset=.  */
+
+enum diagnostic_text_art_charset
+{
+  /* No text art diagrams shall be emitted.  */
+  DIAGNOSTICS_TEXT_ART_CHARSET_NONE,
+
+  /* Use pure ASCII for text art diagrams.  */
+  DIAGNOSTICS_TEXT_ART_CHARSET_ASCII,
+
+  /* Use ASCII + conservative use of other unicode characters
+     in text art diagrams.  */
+  DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE,
+
+  /* Use Emoji.  */
+  DIAGNOSTICS_TEXT_ART_CHARSET_EMOJI
+};
+
+#endif /* ! GCC_DIAGNOSTICS_CONTEXT_OPTIONS_H */
diff --git a/gcc/diagnostic.cc b/gcc/diagnostics/context.cc
index 088390b..85f7d2a 100644
--- a/gcc/diagnostic.cc
+++ b/gcc/diagnostics/context.cc
@@ -31,24 +31,25 @@ along with GCC; see the file COPYING3.  If not see
 #include "intl.h"
 #include "backtrace.h"
 #include "diagnostic.h"
-#include "diagnostic-color.h"
-#include "diagnostic-url.h"
-#include "diagnostic-metadata.h"
-#include "diagnostic-path.h"
-#include "diagnostic-client-data-hooks.h"
-#include "diagnostic-diagram.h"
-#include "diagnostic-format.h"
-#include "diagnostic-format-sarif.h"
-#include "diagnostic-format-text.h"
-#include "edit-context.h"
+#include "diagnostics/color.h"
+#include "diagnostics/url.h"
+#include "diagnostics/metadata.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/client-data-hooks.h"
+#include "diagnostics/diagram.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/sarif-sink.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/changes.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
+#include "diagnostics/selftest-context.h"
 #include "opts.h"
 #include "cpplib.h"
 #include "text-art/theme.h"
 #include "pretty-print-urlifier.h"
-#include "logical-location.h"
-#include "diagnostic-buffer.h"
+#include "diagnostics/logical-locations.h"
+#include "diagnostics/buffering.h"
+#include "diagnostics/file-cache.h"
 
 #ifdef HAVE_TERMIOS_H
 # include <termios.h>
@@ -111,114 +112,28 @@ get_terminal_width (void)
   return INT_MAX;
 }
 
+namespace diagnostics {
+
 /* Set caret_max_width to value.  */
+
 void
-diagnostic_set_caret_max_width (diagnostic_context *context, int value)
+context::set_caret_max_width (int value)
 {
   /* One minus to account for the leading empty space.  */
   value = value ? value - 1
-    : (isatty (fileno (pp_buffer (context->get_reference_printer ())->m_stream))
+    : (isatty (fileno (pp_buffer (get_reference_printer ())->m_stream))
        ? get_terminal_width () - 1 : INT_MAX);
 
   if (value <= 0)
     value = INT_MAX;
 
-  context->m_source_printing.max_width = value;
-}
-
-void
-diagnostic_option_classifier::init (int n_opts)
-{
-  m_n_opts = n_opts;
-  m_classify_diagnostic = XNEWVEC (diagnostic_t, n_opts);
-  for (int i = 0; i < n_opts; i++)
-    m_classify_diagnostic[i] = DK_UNSPECIFIED;
-  m_push_list = vNULL;
-  m_classification_history = vNULL;
-}
-
-void
-diagnostic_option_classifier::fini ()
-{
-  XDELETEVEC (m_classify_diagnostic);
-  m_classify_diagnostic = nullptr;
-  m_classification_history.release ();
-  m_push_list.release ();
-}
-
-/* Save the diagnostic_option_classifier state to F for PCH
-   output.  Returns 0 on success, -1 on error.  */
-
-int
-diagnostic_option_classifier::pch_save (FILE *f)
-{
-  unsigned int lengths[2] = { m_classification_history.length (),
-			      m_push_list.length () };
-  if (fwrite (lengths, sizeof (lengths), 1, f) != 1
-      || (lengths[0]
-	  && fwrite (m_classification_history.address (),
-		     sizeof (diagnostic_classification_change_t),
-		     lengths[0], f) != lengths[0])
-      || (lengths[1]
-	  && fwrite (m_push_list.address (), sizeof (int),
-		     lengths[1], f) != lengths[1]))
-    return -1;
-  return 0;
-}
-
-/* Read the diagnostic_option_classifier state from F for PCH
-   read.  Returns 0 on success, -1 on error.  */
-
-int
-diagnostic_option_classifier::pch_restore (FILE *f)
-{
-  unsigned int lengths[2];
-  if (fread (lengths, sizeof (lengths), 1, f) != 1)
-    return -1;
-  gcc_checking_assert (m_classification_history.is_empty ());
-  gcc_checking_assert (m_push_list.is_empty ());
-  m_classification_history.safe_grow (lengths[0]);
-  m_push_list.safe_grow (lengths[1]);
-  if ((lengths[0]
-       && fread (m_classification_history.address (),
-		 sizeof (diagnostic_classification_change_t),
-		 lengths[0], f) != lengths[0])
-      || (lengths[1]
-	  && fread (m_push_list.address (), sizeof (int),
-		    lengths[1], f) != lengths[1]))
-    return -1;
-  return 0;
-}
-
-/* Save all diagnostic classifications in a stack.  */
-
-void
-diagnostic_option_classifier::push ()
-{
-  m_push_list.safe_push (m_classification_history.length ());
-}
-
-/* Restore the topmost classification set off the stack.  If the stack
-   is empty, revert to the state based on command line parameters.  */
-
-void
-diagnostic_option_classifier::pop (location_t where)
-{
-  int jump_to;
-
-  if (!m_push_list.is_empty ())
-    jump_to = m_push_list.pop ();
-  else
-    jump_to = 0;
-
-  diagnostic_classification_change_t v = { where, jump_to, DK_POP };
-  m_classification_history.safe_push (v);
+  m_source_printing.max_width = value;
 }
 
 /* Initialize the diagnostic message outputting machinery.  */
 
 void
-diagnostic_context::initialize (int n_opts)
+context::initialize (int n_opts)
 {
   /* Allocate a basic pretty-printer.  Clients will replace this a
      much more elaborated pretty-printer if they wish.  */
@@ -230,7 +145,7 @@ diagnostic_context::initialize (int n_opts)
   m_n_opts = n_opts;
   m_option_classifier.init (n_opts);
   m_source_printing.enabled = false;
-  diagnostic_set_caret_max_width (this, pp_line_cutoff (get_reference_printer ()));
+  set_caret_max_width (pp_line_cutoff (get_reference_printer ()));
   for (int i = 0; i < rich_location::STATICALLY_ALLOCATED_RANGES; i++)
     m_source_printing.caret_chars[i] = '^';
   m_show_cwe = false;
@@ -249,12 +164,12 @@ diagnostic_context::initialize (int n_opts)
   m_max_errors = 0;
   m_internal_error = nullptr;
   m_adjust_diagnostic_info = nullptr;
-  m_text_callbacks.m_begin_diagnostic = default_diagnostic_text_starter;
+  m_text_callbacks.m_begin_diagnostic = default_text_starter;
   m_text_callbacks.m_text_start_span
-    = default_diagnostic_start_span_fn<to_text>;
+    = default_start_span_fn<to_text>;
   m_text_callbacks.m_html_start_span
-    = default_diagnostic_start_span_fn<to_html>;
-  m_text_callbacks.m_end_diagnostic = default_diagnostic_text_finalizer;
+    = default_start_span_fn<to_html>;
+  m_text_callbacks.m_end_diagnostic = default_text_finalizer;
   m_option_mgr = nullptr;
   m_urlifier_stack = new auto_vec<urlifier_stack_node> ();
   m_last_location = UNKNOWN_LOCATION;
@@ -281,13 +196,12 @@ diagnostic_context::initialize (int n_opts)
   m_column_origin = 1;
   m_tabstop = 8;
   m_escape_format = DIAGNOSTICS_ESCAPE_FORMAT_UNICODE;
-  m_edit_context_ptr = nullptr;
+  m_fixits_change_set = nullptr;
   m_diagnostic_groups.m_group_nesting_depth = 0;
   m_diagnostic_groups.m_diagnostic_nesting_level = 0;
   m_diagnostic_groups.m_emission_count = 0;
   m_diagnostic_groups.m_inhibiting_notes_from = 0;
-  m_output_sinks.safe_push
-    (new diagnostic_text_output_format (*this, nullptr, true));
+  m_sinks.safe_push (new text_sink (*this, nullptr, true));
   m_set_locations_cb = nullptr;
   m_client_data_hooks = nullptr;
   m_diagrams.m_theme = nullptr;
@@ -311,7 +225,7 @@ diagnostic_context::initialize (int n_opts)
    without a VALUE, it initializes with DIAGNOSTICS_COLOR_DEFAULT.  */
 
 void
-diagnostic_context::color_init (int value)
+context::color_init (int value)
 {
   /* value == -1 is the default value.  */
   if (value < 0)
@@ -332,9 +246,9 @@ diagnostic_context::color_init (int value)
     }
   pp_show_color (m_reference_printer)
     = colorize_init ((diagnostic_color_rule_t) value);
-  for (auto sink : m_output_sinks)
-    if (sink->follows_reference_printer_p ())
-      pp_show_color (sink->get_printer ())
+  for (auto sink_ : m_sinks)
+    if (sink_->follows_reference_printer_p ())
+      pp_show_color (sink_->get_printer ())
 	= pp_show_color (m_reference_printer);
 }
 
@@ -342,7 +256,7 @@ diagnostic_context::color_init (int value)
    handling "auto".  */
 
 void
-diagnostic_context::urls_init (int value)
+context::urls_init (int value)
 {
   /* value == -1 is the default value.  */
   if (value < 0)
@@ -364,18 +278,17 @@ diagnostic_context::urls_init (int value)
 
   m_reference_printer->set_url_format
     (determine_url_format ((diagnostic_url_rule_t) value));
-  for (auto sink : m_output_sinks)
-    if (sink->follows_reference_printer_p ())
-      sink->get_printer ()->set_url_format
+  for (auto sink_ : m_sinks)
+    if (sink_->follows_reference_printer_p ())
+      sink_->get_printer ()->set_url_format
 	(m_reference_printer->get_url_format ());
 }
 
 /* Create the file_cache, if not already created, and tell it how to
    translate files on input.  */
 void
-diagnostic_context::
-initialize_input_context (diagnostic_input_charset_callback ccb,
-			  bool should_skip_bom)
+context::initialize_input_context (diagnostic_input_charset_callback ccb,
+				   bool should_skip_bom)
 {
   m_file_cache->initialize_input_context (ccb, should_skip_bom);
 }
@@ -383,7 +296,7 @@ initialize_input_context (diagnostic_input_charset_callback ccb,
 /* Do any cleaning up required after the last diagnostic is emitted.  */
 
 void
-diagnostic_context::finish ()
+context::finish ()
 {
   /* We might be handling a fatal error.
      Close any active diagnostic groups, which may trigger flushing
@@ -395,8 +308,8 @@ diagnostic_context::finish ()
 
   /* Clean ups.  */
 
-  while (!m_output_sinks.is_empty ())
-    delete m_output_sinks.pop ();
+  while (!m_sinks.is_empty ())
+    delete m_sinks.pop ();
 
   if (m_diagrams.m_theme)
     {
@@ -412,10 +325,10 @@ diagnostic_context::finish ()
   delete m_reference_printer;
   m_reference_printer = nullptr;
 
-  if (m_edit_context_ptr)
+  if (m_fixits_change_set)
     {
-      delete m_edit_context_ptr;
-      m_edit_context_ptr = nullptr;
+      delete m_fixits_change_set;
+      m_fixits_change_set = nullptr;
     }
 
   if (m_client_data_hooks)
@@ -439,22 +352,22 @@ diagnostic_context::finish ()
   m_original_argv = nullptr;
 }
 
-/* Dump state of this diagnostic_context to OUT, for debugging.  */
+/* Dump state of this diagnostics::context to OUT, for debugging.  */
 
 void
-diagnostic_context::dump (FILE *out) const
+context::dump (FILE *out) const
 {
-  fprintf (out, "diagnostic_context:\n");
+  fprintf (out, "diagnostics::context:\n");
   m_diagnostic_counters.dump (out, 2);
   fprintf (out, "  reference printer:\n");
   m_reference_printer->dump (out, 4);
   fprintf (out, "  output sinks:\n");
-  if (m_output_sinks.length () > 0)
+  if (m_sinks.length () > 0)
     {
-      for (unsigned i = 0; i < m_output_sinks.length (); ++i)
+      for (unsigned i = 0; i < m_sinks.length (); ++i)
 	{
 	  fprintf (out, "  sink %i:\n", i);
-	  m_output_sinks[i]->dump (out, 4);
+	  m_sinks[i]->dump (out, 4);
 	}
     }
   else
@@ -475,65 +388,63 @@ diagnostic_context::dump (FILE *out) const
    we ought to exit with a non-zero exit code.  */
 
 bool
-diagnostic_context::execution_failed_p () const
+context::execution_failed_p () const
 {
   /* Equivalent to (seen_error () || werrorcount), but on
      this context, rather than global_dc.  */
-  return (diagnostic_count (DK_ERROR)
-	  || diagnostic_count (DK_SORRY)
-	  || diagnostic_count (DK_WERROR));
+  return (diagnostic_count (kind::error)
+	  || diagnostic_count (kind::sorry)
+	  || diagnostic_count (kind::werror));
 }
 
 void
-diagnostic_context::remove_all_output_sinks ()
+context::remove_all_output_sinks ()
 {
-  while (!m_output_sinks.is_empty ())
-    delete m_output_sinks.pop ();
+  while (!m_sinks.is_empty ())
+    delete m_sinks.pop ();
 }
 
 void
-diagnostic_context::
-set_output_format (std::unique_ptr<diagnostic_output_format> output_format)
+context::set_sink (std::unique_ptr<sink> sink_)
 {
   remove_all_output_sinks ();
-  m_output_sinks.safe_push (output_format.release ());
+  m_sinks.safe_push (sink_.release ());
 }
 
-diagnostic_output_format &
-diagnostic_context::get_output_format (size_t idx) const
+sink &
+context::get_sink (size_t idx) const
 {
-  gcc_assert (idx < m_output_sinks.length ());
-  gcc_assert (m_output_sinks[idx]);
-  return *m_output_sinks[idx];
+  gcc_assert (idx < m_sinks.length ());
+  gcc_assert (m_sinks[idx]);
+  return *m_sinks[idx];
 }
 
 void
-diagnostic_context::add_sink (std::unique_ptr<diagnostic_output_format> sink)
+context::add_sink (std::unique_ptr<sink> sink_)
 {
-  m_output_sinks.safe_push (sink.release ());
+  m_sinks.safe_push (sink_.release ());
 }
 
 /* Return true if there are no machine-readable formats writing to stderr.  */
 
 bool
-diagnostic_context::supports_fnotice_on_stderr_p () const
+context::supports_fnotice_on_stderr_p () const
 {
-  for (auto sink : m_output_sinks)
-    if (sink->machine_readable_stderr_p ())
+  for (auto sink_ : m_sinks)
+    if (sink_->machine_readable_stderr_p ())
       return false;
   return true;
 }
 
 void
-diagnostic_context::set_main_input_filename (const char *filename)
+context::set_main_input_filename (const char *filename)
 {
-  for (auto sink : m_output_sinks)
-    sink->set_main_input_filename (filename);
+  for (auto sink_ : m_sinks)
+    sink_->set_main_input_filename (filename);
 }
 
 void
-diagnostic_context::
-set_client_data_hooks (std::unique_ptr<diagnostic_client_data_hooks> hooks)
+context::set_client_data_hooks (std::unique_ptr<client_data_hooks> hooks)
 {
   delete m_client_data_hooks;
   /* Ideally the field would be a std::unique_ptr here.  */
@@ -541,10 +452,10 @@ set_client_data_hooks (std::unique_ptr<diagnostic_client_data_hooks> hooks)
 }
 
 void
-diagnostic_context::set_original_argv (unique_argv original_argv)
+context::set_original_argv (unique_argv original_argv)
 {
   /* Ideally we'd use a unique_argv for m_original_argv, but
-     diagnostic_context doesn't yet have a ctor/dtor pair.  */
+     diagnostics::context doesn't yet have a ctor/dtor pair.  */
 
   // Ensure any old value is freed
   freeargv (m_original_argv);
@@ -554,9 +465,8 @@ diagnostic_context::set_original_argv (unique_argv original_argv)
 }
 
 void
-diagnostic_context::
-set_option_manager (std::unique_ptr<diagnostic_option_manager> mgr,
-		    unsigned lang_mask)
+context::set_option_manager (std::unique_ptr<option_manager> mgr,
+			     unsigned lang_mask)
 {
   delete m_option_mgr;
   m_option_mgr = mgr.release ();
@@ -564,7 +474,7 @@ set_option_manager (std::unique_ptr<diagnostic_option_manager> mgr,
 }
 
 void
-diagnostic_context::push_owned_urlifier (std::unique_ptr<urlifier> ptr)
+context::push_owned_urlifier (std::unique_ptr<urlifier> ptr)
 {
   gcc_assert (m_urlifier_stack);
   const urlifier_stack_node node = { ptr.release (), true };
@@ -572,7 +482,7 @@ diagnostic_context::push_owned_urlifier (std::unique_ptr<urlifier> ptr)
 }
 
 void
-diagnostic_context::push_borrowed_urlifier (const urlifier &loan)
+context::push_borrowed_urlifier (const urlifier &loan)
 {
   gcc_assert (m_urlifier_stack);
   const urlifier_stack_node node = { const_cast <urlifier *> (&loan), false };
@@ -580,7 +490,7 @@ diagnostic_context::push_borrowed_urlifier (const urlifier &loan)
 }
 
 void
-diagnostic_context::pop_urlifier ()
+context::pop_urlifier ()
 {
   gcc_assert (m_urlifier_stack);
   gcc_assert (m_urlifier_stack->length () > 0);
@@ -590,8 +500,8 @@ diagnostic_context::pop_urlifier ()
     delete node.m_urlifier;
 }
 
-const logical_location_manager *
-diagnostic_context::get_logical_location_manager () const
+const logical_locations::manager *
+context::get_logical_location_manager () const
 {
   if (!m_client_data_hooks)
     return nullptr;
@@ -599,7 +509,7 @@ diagnostic_context::get_logical_location_manager () const
 }
 
 const urlifier *
-diagnostic_context::get_urlifier () const
+context::get_urlifier () const
 {
   if (!m_urlifier_stack)
     return nullptr;
@@ -613,7 +523,7 @@ diagnostic_context::get_urlifier () const
    Refresh all output sinks.  */
 
 void
-diagnostic_context::set_pretty_printer (std::unique_ptr<pretty_printer> pp)
+context::set_pretty_printer (std::unique_ptr<pretty_printer> pp)
 {
   delete m_reference_printer;
   m_reference_printer = pp.release ();
@@ -623,81 +533,102 @@ diagnostic_context::set_pretty_printer (std::unique_ptr<pretty_printer> pp)
 /* Give all output sinks a chance to rebuild their pretty_printer.  */
 
 void
-diagnostic_context::refresh_output_sinks ()
+context::refresh_output_sinks ()
 {
-  for (auto sink : m_output_sinks)
-    sink->update_printer ();
+  for (auto sink_ : m_sinks)
+    sink_->update_printer ();
 }
 
 /* Set FORMAT_DECODER on the reference printer and on the pretty_printer
    of all output sinks.  */
 
 void
-diagnostic_context::set_format_decoder (printer_fn format_decoder)
+context::set_format_decoder (printer_fn format_decoder)
 {
   pp_format_decoder (m_reference_printer) = format_decoder;
-  for (auto sink : m_output_sinks)
-    pp_format_decoder (sink->get_printer ()) = format_decoder;
+  for (auto sink_ : m_sinks)
+    pp_format_decoder (sink_->get_printer ()) = format_decoder;
 }
 
 void
-diagnostic_context::set_show_highlight_colors (bool val)
+context::set_show_highlight_colors (bool val)
 {
   pp_show_highlight_colors (m_reference_printer) = val;
-  for (auto sink : m_output_sinks)
-    if (sink->follows_reference_printer_p ())
-      pp_show_highlight_colors (sink->get_printer ()) = val;
+  for (auto sink_ : m_sinks)
+    if (sink_->follows_reference_printer_p ())
+      pp_show_highlight_colors (sink_->get_printer ()) = val;
 }
 
 void
-diagnostic_context::set_prefixing_rule (diagnostic_prefixing_rule_t rule)
+context::set_prefixing_rule (diagnostic_prefixing_rule_t rule)
 {
   pp_prefixing_rule (m_reference_printer) = rule;
-  for (auto sink : m_output_sinks)
-    if (sink->follows_reference_printer_p ())
-      pp_prefixing_rule (sink->get_printer ()) = rule;
+  for (auto sink_ : m_sinks)
+    if (sink_->follows_reference_printer_p ())
+      pp_prefixing_rule (sink_->get_printer ()) = rule;
 }
 
 void
-diagnostic_context::create_edit_context ()
+context::initialize_fixits_change_set ()
 {
-  delete m_edit_context_ptr;
+  delete m_fixits_change_set;
   gcc_assert (m_file_cache);
-  m_edit_context_ptr = new edit_context (*m_file_cache);
+  m_fixits_change_set = new changes::change_set (*m_file_cache);
 }
 
+} // namespace diagnostics
+
 /* Initialize DIAGNOSTIC, where the message MSG has already been
    translated.  */
 void
-diagnostic_set_info_translated (diagnostic_info *diagnostic, const char *msg,
-				va_list *args, rich_location *richloc,
-				diagnostic_t kind)
+diagnostic_set_info_translated (diagnostics::diagnostic_info *diagnostic,
+				const char *msg, va_list *args,
+				rich_location *richloc,
+				enum diagnostics::kind kind)
 {
   gcc_assert (richloc);
-  diagnostic->message.m_err_no = errno;
-  diagnostic->message.m_args_ptr = args;
-  diagnostic->message.m_format_spec = msg;
-  diagnostic->message.m_richloc = richloc;
-  diagnostic->richloc = richloc;
-  diagnostic->metadata = nullptr;
-  diagnostic->kind = kind;
-  diagnostic->option_id = 0;
+  diagnostic->m_message.m_err_no = errno;
+  diagnostic->m_message.m_args_ptr = args;
+  diagnostic->m_message.m_format_spec = msg;
+  diagnostic->m_message.m_richloc = richloc;
+  diagnostic->m_richloc = richloc;
+  diagnostic->m_metadata = nullptr;
+  diagnostic->m_kind = kind;
+  diagnostic->m_option_id = 0;
 }
 
 /* Initialize DIAGNOSTIC, where the message GMSGID has not yet been
    translated.  */
 void
-diagnostic_set_info (diagnostic_info *diagnostic, const char *gmsgid,
-		     va_list *args, rich_location *richloc,
-		     diagnostic_t kind)
+diagnostic_set_info (diagnostics::diagnostic_info *diagnostic,
+		     const char *gmsgid, va_list *args,
+		     rich_location *richloc,
+		     enum diagnostics::kind kind)
 {
   gcc_assert (richloc);
   diagnostic_set_info_translated (diagnostic, _(gmsgid), args, richloc, kind);
 }
 
+namespace diagnostics {
+
+static const char *const diagnostic_kind_text[] = {
+#define DEFINE_DIAGNOSTIC_KIND(K, T, C) (T),
+#include "diagnostics/kinds.def"
+#undef DEFINE_DIAGNOSTIC_KIND
+  "must-not-happen"
+};
+
+/* Get unlocalized string describing KIND.  */
+
+const char *
+get_text_for_kind (enum kind kind)
+{
+  return diagnostic_kind_text[static_cast<int> (kind)];
+}
+
 static const char *const diagnostic_kind_color[] = {
 #define DEFINE_DIAGNOSTIC_KIND(K, T, C) (C),
-#include "diagnostic.def"
+#include "diagnostics/kinds.def"
 #undef DEFINE_DIAGNOSTIC_KIND
   nullptr
 };
@@ -706,9 +637,9 @@ static const char *const diagnostic_kind_color[] = {
    Result could be nullptr.  */
 
 const char *
-diagnostic_get_color_for_kind (diagnostic_t kind)
+get_color_for_kind (enum kind kind)
 {
-  return diagnostic_kind_color[kind];
+  return diagnostic_kind_color[static_cast<int> (kind)];
 }
 
 /* Given an expanded_location, convert the column (which is in 1-based bytes)
@@ -740,8 +671,7 @@ convert_column_unit (file_cache &fc,
     }
 }
 
-diagnostic_column_policy::
-diagnostic_column_policy (const diagnostic_context &dc)
+column_policy::column_policy (const context &dc)
 : m_file_cache (dc.get_file_cache ()),
   m_column_unit (dc.m_column_unit),
   m_column_origin (dc.m_column_origin),
@@ -753,7 +683,7 @@ diagnostic_column_policy (const diagnostic_context &dc)
    to the requested units and origin.  Return -1 if the column is
    invalid (<= 0).  */
 int
-diagnostic_column_policy::converted_column (expanded_location s) const
+column_policy::converted_column (expanded_location s) const
 {
   int one_based_col = convert_column_unit (m_file_cache,
 					   m_column_unit, m_tabstop, s);
@@ -765,9 +695,9 @@ diagnostic_column_policy::converted_column (expanded_location s) const
 /* Return a string describing a location e.g. "foo.c:42:10".  */
 
 label_text
-diagnostic_column_policy::get_location_text (const expanded_location &s,
-					     bool show_column,
-					     bool colorize) const
+column_policy::get_location_text (const expanded_location &s,
+				  bool show_column,
+				  bool colorize) const
 {
   const char *locus_cs = colorize_start (colorize, "locus");
   const char *locus_ce = colorize_stop (colorize);
@@ -786,35 +716,22 @@ diagnostic_column_policy::get_location_text (const expanded_location &s,
 						 line_col, locus_ce));
 }
 
-diagnostic_location_print_policy::
-diagnostic_location_print_policy (const diagnostic_context &dc)
+location_print_policy::
+location_print_policy (const context &dc)
 : m_column_policy (dc),
   m_show_column (dc.m_show_column)
 {
 }
 
-diagnostic_location_print_policy::
-diagnostic_location_print_policy (const diagnostic_text_output_format &text_output)
+location_print_policy::
+location_print_policy (const text_sink &text_output)
 :
   m_column_policy (text_output.get_context ()),
   m_show_column (text_output.get_context ().m_show_column)
 {
 }
 
-static const char *const diagnostic_kind_text[] = {
-#define DEFINE_DIAGNOSTIC_KIND(K, T, C) (T),
-#include "diagnostic.def"
-#undef DEFINE_DIAGNOSTIC_KIND
-  "must-not-happen"
-};
-
-/* Get unlocalized string describing KIND.  */
-
-const char *
-get_diagnostic_kind_text (diagnostic_t kind)
-{
-  return diagnostic_kind_text[kind];
-}
+} // namespace diagnostics
 
 /* Functions at which to stop the backtrace print.  It's not
    particularly helpful to print the callers of these functions.  */
@@ -840,10 +757,10 @@ bt_callback (void *data, uintptr_t pc, const char *filename, int lineno,
   if (filename == nullptr && function == nullptr)
     return 0;
 
-  /* Skip functions in diagnostic.cc.  */
+  /* Skip functions in context.cc.  */
   if (*pcount == 0
       && filename != nullptr
-      && strcmp (lbasename (filename), "diagnostic.cc") == 0)
+      && strcmp (lbasename (filename), "context.cc") == 0)
     return 0;
 
   /* Print up to 20 functions.  We could make this a --param, but
@@ -909,19 +826,21 @@ bt_err_callback (void *data ATTRIBUTE_UNUSED, const char *msg, int errnum)
 	   errnum == 0 ? "" : xstrerror (errnum));
 }
 
+namespace diagnostics {
+
 /* Check if we've met the maximum error limit, and if so fatally exit
    with a message.
-   FLUSH indicates whether a diagnostic_context::finish call is needed.  */
+   FLUSH indicates whether a diagnostics::context::finish call is needed.  */
 
 void
-diagnostic_context::check_max_errors (bool flush)
+context::check_max_errors (bool flush)
 {
   if (!m_max_errors)
     return;
 
-  int count = (diagnostic_count (DK_ERROR)
-	       + diagnostic_count (DK_SORRY)
-	       + diagnostic_count (DK_WERROR));
+  int count = (diagnostic_count (kind::error)
+	       + diagnostic_count (kind::sorry)
+	       + diagnostic_count (kind::werror));
 
   if (count >= m_max_errors)
     {
@@ -938,18 +857,18 @@ diagnostic_context::check_max_errors (bool flush)
    is written out.  This function does not always return.  */
 
 void
-diagnostic_context::action_after_output (diagnostic_t diag_kind)
+context::action_after_output (enum kind diag_kind)
 {
   switch (diag_kind)
     {
-    case DK_DEBUG:
-    case DK_NOTE:
-    case DK_ANACHRONISM:
-    case DK_WARNING:
+    case kind::debug:
+    case kind::note:
+    case kind::anachronism:
+    case kind::warning:
       break;
 
-    case DK_ERROR:
-    case DK_SORRY:
+    case kind::error:
+    case kind::sorry:
       if (m_abort_on_error)
 	real_abort ();
       if (m_fatal_errors)
@@ -960,8 +879,8 @@ diagnostic_context::action_after_output (diagnostic_t diag_kind)
 	}
       break;
 
-    case DK_ICE:
-    case DK_ICE_NOBT:
+    case kind::ice:
+    case kind::ice_nobt:
       {
 	/* Attempt to ensure that any outputs are flushed e.g. that .sarif
 	   files are written out.
@@ -974,7 +893,7 @@ diagnostic_context::action_after_output (diagnostic_t diag_kind)
 	  }
 
 	struct backtrace_state *state = nullptr;
-	if (diag_kind == DK_ICE)
+	if (diag_kind == kind::ice)
 	  state = backtrace_create_state (nullptr, 0, bt_err_callback, nullptr);
 	int count = 0;
 	if (state != nullptr)
@@ -999,7 +918,7 @@ diagnostic_context::action_after_output (diagnostic_t diag_kind)
 	exit (ICE_EXIT_CODE);
       }
 
-    case DK_FATAL:
+    case kind::fatal:
       if (m_abort_on_error)
 	real_abort ();
       fnotice (stderr, "compilation terminated.\n");
@@ -1015,7 +934,7 @@ diagnostic_context::action_after_output (diagnostic_t diag_kind)
    its future children if any.  */
 
 void
-diagnostic_context::inhibit_notes_in_group (bool inhibit)
+context::inhibit_notes_in_group (bool inhibit)
 {
   int curr_depth = (m_diagnostic_groups.m_group_nesting_depth
 		    + m_diagnostic_groups.m_diagnostic_nesting_level);
@@ -1045,7 +964,7 @@ diagnostic_context::inhibit_notes_in_group (bool inhibit)
 /* Return whether notes must be inhibited in the current diagnostic_group.  */
 
 bool
-diagnostic_context::notes_inhibited_in_group () const
+context::notes_inhibited_in_group () const
 {
   if (m_diagnostic_groups.m_inhibiting_notes_from
       && (m_diagnostic_groups.m_group_nesting_depth
@@ -1055,84 +974,32 @@ diagnostic_context::notes_inhibited_in_group () const
   return false;
 }
 
-/* class logical_location_manager.  */
+/* class diagnostics::logical_locations::manager.  */
 
 /* Return true iff this is a function or method.  */
 
 bool
-logical_location_manager::function_p (key k) const
+logical_locations::manager::function_p (key k) const
 {
   switch (get_kind (k))
     {
     default:
       gcc_unreachable ();
-    case logical_location_kind::unknown:
-    case logical_location_kind::module_:
-    case logical_location_kind::namespace_:
-    case logical_location_kind::type:
-    case logical_location_kind::return_type:
-    case logical_location_kind::parameter:
-    case logical_location_kind::variable:
+    case kind::unknown:
+    case kind::module_:
+    case kind::namespace_:
+    case kind::type:
+    case kind::return_type:
+    case kind::parameter:
+    case kind::variable:
       return false;
 
-    case logical_location_kind::function:
-    case logical_location_kind::member:
+    case kind::function:
+    case kind::member:
       return true;
     }
 }
 
-/* Interface to specify diagnostic kind overrides.  Returns the
-   previous setting, or DK_UNSPECIFIED if the parameters are out of
-   range.  If OPTION_ID is zero, the new setting is for all the
-   diagnostics.  */
-diagnostic_t
-diagnostic_option_classifier::
-classify_diagnostic (const diagnostic_context *context,
-		     diagnostic_option_id option_id,
-		     diagnostic_t new_kind,
-		     location_t where)
-{
-  diagnostic_t old_kind;
-
-  if (option_id.m_idx < 0
-      || option_id.m_idx >= m_n_opts
-      || new_kind >= DK_LAST_DIAGNOSTIC_KIND)
-    return DK_UNSPECIFIED;
-
-  old_kind = m_classify_diagnostic[option_id.m_idx];
-
-  /* Handle pragmas separately, since we need to keep track of *where*
-     the pragmas were.  */
-  if (where != UNKNOWN_LOCATION)
-    {
-      unsigned i;
-
-      /* Record the command-line status, so we can reset it back on DK_POP. */
-      if (old_kind == DK_UNSPECIFIED)
-	{
-	  old_kind = (!context->option_enabled_p (option_id)
-		      ? DK_IGNORED : DK_ANY);
-	  m_classify_diagnostic[option_id.m_idx] = old_kind;
-	}
-
-      diagnostic_classification_change_t *p;
-      FOR_EACH_VEC_ELT_REVERSE (m_classification_history, i, p)
-	if (p->option == option_id.m_idx)
-	  {
-	    old_kind = p->kind;
-	    break;
-	  }
-
-      diagnostic_classification_change_t v
-	= { where, option_id.m_idx, new_kind };
-      m_classification_history.safe_push (v);
-    }
-  else
-    m_classify_diagnostic[option_id.m_idx] = new_kind;
-
-  return old_kind;
-}
-
 /* Helper function for print_parseable_fixits.  Print TEXT to PP, obeying the
    escaping rules for -fdiagnostics-parseable-fixits.  */
 
@@ -1223,7 +1090,7 @@ print_parseable_fixits (file_cache &fc,
 /* Update the inlining info in this context for a DIAGNOSTIC.  */
 
 void
-diagnostic_context::get_any_inlining_info (diagnostic_info *diagnostic)
+context::get_any_inlining_info (diagnostic_info *diagnostic)
 {
   auto &ilocs = diagnostic->m_iinfo.m_ilocs;
 
@@ -1242,58 +1109,6 @@ diagnostic_context::get_any_inlining_info (diagnostic_info *diagnostic)
     }
 }
 
-/* Update the kind of DIAGNOSTIC based on its location(s), including
-   any of those in its inlining stack, relative to any
-     #pragma GCC diagnostic
-   directives recorded within this object.
-
-   Return the new kind of DIAGNOSTIC if it was updated, or DK_UNSPECIFIED
-   otherwise.  */
-
-diagnostic_t
-diagnostic_option_classifier::
-update_effective_level_from_pragmas (diagnostic_info *diagnostic) const
-{
-  if (m_classification_history.is_empty ())
-    return DK_UNSPECIFIED;
-
-  /* Iterate over the locations, checking the diagnostic disposition
-     for the diagnostic at each.  If it's explicitly set as opposed
-     to unspecified, update the disposition for this instance of
-     the diagnostic and return it.  */
-  for (location_t loc: diagnostic->m_iinfo.m_ilocs)
-    {
-      /* FIXME: Stupid search.  Optimize later. */
-      unsigned int i;
-      diagnostic_classification_change_t *p;
-      FOR_EACH_VEC_ELT_REVERSE (m_classification_history, i, p)
-	{
-	  location_t pragloc = p->location;
-	  if (!linemap_location_before_p (line_table, pragloc, loc))
-	    continue;
-
-	  if (p->kind == (int) DK_POP)
-	    {
-	      /* Move on to the next region.  */
-	      i = p->option;
-	      continue;
-	    }
-
-	  diagnostic_option_id option = p->option;
-	  /* The option 0 is for all the diagnostics.  */
-	  if (option == 0 || option == diagnostic->option_id)
-	    {
-	      diagnostic_t kind = p->kind;
-	      if (kind != DK_UNSPECIFIED)
-		diagnostic->kind = kind;
-	      return kind;
-	    }
-	}
-    }
-
-  return DK_UNSPECIFIED;
-}
-
 /* Generate a URL string describing CWE.  The caller is responsible for
    freeing the string.  */
 
@@ -1307,80 +1122,79 @@ get_cwe_url (int cwe)
    as appropriate for #pragma GCC diagnostic and -Werror=foo.  */
 
 bool
-diagnostic_context::diagnostic_enabled (diagnostic_info *diagnostic)
+context::diagnostic_enabled (diagnostic_info *diagnostic)
 {
   /* Update the inlining stack for this diagnostic.  */
   get_any_inlining_info (diagnostic);
 
   /* Diagnostics with no option or -fpermissive are always enabled.  */
-  if (!diagnostic->option_id.m_idx
-      || diagnostic->option_id == m_opt_permissive)
+  if (!diagnostic->m_option_id.m_idx
+      || diagnostic->m_option_id == m_opt_permissive)
     return true;
 
   /* This tests if the user provided the appropriate -Wfoo or
      -Wno-foo option.  */
-  if (!option_enabled_p (diagnostic->option_id))
+  if (!option_enabled_p (diagnostic->m_option_id))
     return false;
 
   /* This tests for #pragma diagnostic changes.  */
-  diagnostic_t diag_class
+  enum kind diag_class
     = m_option_classifier.update_effective_level_from_pragmas (diagnostic);
 
   /* This tests if the user provided the appropriate -Werror=foo
      option.  */
-  if (diag_class == DK_UNSPECIFIED
-      && !option_unspecified_p (diagnostic->option_id))
+  if (diag_class == kind::unspecified
+      && !option_unspecified_p (diagnostic->m_option_id))
     {
-      const diagnostic_t new_kind
-	= m_option_classifier.get_current_override (diagnostic->option_id);
-      if (new_kind != DK_ANY)
-	/* DK_ANY means the diagnostic is not to be ignored, but we don't want
-	   to change it specifically to DK_ERROR or DK_WARNING; we want to
+      const enum kind new_kind
+	= m_option_classifier.get_current_override (diagnostic->m_option_id);
+      if (new_kind != kind::any)
+	/* kind::any means the diagnostic is not to be ignored, but we don't want
+	   to change it specifically to kind::error or kind::warning; we want to
 	   preserve whatever the caller has specified.  */
-	diagnostic->kind = new_kind;
+	diagnostic->m_kind = new_kind;
     }
 
   /* This allows for future extensions, like temporarily disabling
      warnings for ranges of source code.  */
-  if (diagnostic->kind == DK_IGNORED)
+  if (diagnostic->m_kind == kind::ignored)
     return false;
 
   return true;
 }
 
-/* Returns whether warning OPTION_ID is enabled at LOC.  */
+/* Returns whether warning OPT_ID is enabled at LOC.  */
 
 bool
-diagnostic_context::warning_enabled_at (location_t loc,
-					diagnostic_option_id option_id)
+context::warning_enabled_at (location_t loc,
+			     option_id opt_id)
 {
   if (!diagnostic_report_warnings_p (this, loc))
     return false;
 
   rich_location richloc (line_table, loc);
   diagnostic_info diagnostic = {};
-  diagnostic.option_id = option_id;
-  diagnostic.richloc = &richloc;
-  diagnostic.message.m_richloc = &richloc;
-  diagnostic.kind = DK_WARNING;
+  diagnostic.m_option_id = opt_id;
+  diagnostic.m_richloc = &richloc;
+  diagnostic.m_message.m_richloc = &richloc;
+  diagnostic.m_kind = kind::warning;
   return diagnostic_enabled (&diagnostic);
 }
 
 /* Emit a diagnostic within a diagnostic group on this context.  */
 
 bool
-diagnostic_context::
-emit_diagnostic_with_group (diagnostic_t kind,
-			    rich_location &richloc,
-			    const diagnostic_metadata *metadata,
-			    diagnostic_option_id option_id,
-			    const char *gmsgid, ...)
+context::emit_diagnostic_with_group (enum kind kind,
+				     rich_location &richloc,
+				     const metadata *metadata,
+				     option_id opt_id,
+				     const char *gmsgid, ...)
 {
   begin_group ();
 
   va_list ap;
   va_start (ap, gmsgid);
-  bool ret = emit_diagnostic_with_group_va (kind, richloc, metadata, option_id,
+  bool ret = emit_diagnostic_with_group_va (kind, richloc, metadata, opt_id,
 					    gmsgid, &ap);
   va_end (ap);
 
@@ -1392,16 +1206,15 @@ emit_diagnostic_with_group (diagnostic_t kind,
 /* As above, but taking a va_list *.  */
 
 bool
-diagnostic_context::
-emit_diagnostic_with_group_va (diagnostic_t kind,
-			       rich_location &richloc,
-			       const diagnostic_metadata *metadata,
-			       diagnostic_option_id option_id,
-			       const char *gmsgid, va_list *ap)
+context::emit_diagnostic_with_group_va (enum kind kind,
+					rich_location &richloc,
+					const metadata *metadata,
+					option_id opt_id,
+					const char *gmsgid, va_list *ap)
 {
   begin_group ();
 
-  bool ret = diagnostic_impl (&richloc, metadata, option_id,
+  bool ret = diagnostic_impl (&richloc, metadata, opt_id,
 			      gmsgid, ap, kind);
 
   end_group ();
@@ -1410,15 +1223,15 @@ emit_diagnostic_with_group_va (diagnostic_t kind,
 }
 
 /* Report a diagnostic message (an error or a warning) as specified by
-   this diagnostic_context.
+   this diagnostics::context.
    front-end independent format specifiers are exactly those described
    in the documentation of output_format.
    Return true if a diagnostic was printed, false otherwise.  */
 
 bool
-diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
+context::report_diagnostic (diagnostic_info *diagnostic)
 {
-  diagnostic_t orig_diag_kind = diagnostic->kind;
+  enum kind orig_diag_kind = diagnostic->m_kind;
 
   /* Every call to report_diagnostic should be within a
      begin_group/end_group pair so that output formats can reliably
@@ -1427,8 +1240,8 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
 
   /* Give preference to being able to inhibit warnings, before they
      get reclassified to something else.  */
-  bool was_warning = (diagnostic->kind == DK_WARNING
-		      || diagnostic->kind == DK_PEDWARN);
+  bool was_warning = (diagnostic->m_kind == kind::warning
+		      || diagnostic->m_kind == kind::pedwarn);
   if (was_warning && m_inhibit_warnings)
     {
       inhibit_notes_in_group ();
@@ -1438,15 +1251,15 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
   if (m_adjust_diagnostic_info)
     m_adjust_diagnostic_info (this, diagnostic);
 
-  if (diagnostic->kind == DK_PEDWARN)
+  if (diagnostic->m_kind == kind::pedwarn)
     {
-      diagnostic->kind = m_pedantic_errors ? DK_ERROR : DK_WARNING;
+      diagnostic->m_kind = m_pedantic_errors ? kind::error : kind::warning;
 
       /* We do this to avoid giving the message for -pedantic-errors.  */
-      orig_diag_kind = diagnostic->kind;
+      orig_diag_kind = diagnostic->m_kind;
     }
 
-  if (diagnostic->kind == DK_NOTE && m_inhibit_notes_p)
+  if (diagnostic->m_kind == kind::note && m_inhibit_notes_p)
     return false;
 
   /* If the user requested that warnings be treated as errors, so be
@@ -1454,10 +1267,10 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
      individual warnings can be overridden back to warnings with
      -Wno-error=*.  */
   if (m_warning_as_error_requested
-      && diagnostic->kind == DK_WARNING)
-    diagnostic->kind = DK_ERROR;
+      && diagnostic->m_kind == kind::warning)
+    diagnostic->m_kind = kind::error;
 
-  diagnostic->message.m_data = &diagnostic->x_data;
+  diagnostic->m_message.m_data = &diagnostic->m_x_data;
 
   /* Check to see if the diagnostic is enabled at the location and
      not disabled by #pragma GCC diagnostic anywhere along the inlining
@@ -1468,7 +1281,7 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
       return false;
     }
 
-  if ((was_warning || diagnostic->kind == DK_WARNING)
+  if ((was_warning || diagnostic->m_kind == kind::warning)
       && ((!m_warn_system_headers
 	   && diagnostic->m_iinfo.m_allsyslocs)
 	  || m_inhibit_warnings))
@@ -1476,11 +1289,11 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
        inlining stack (if there is one) are in system headers.  */
     return false;
 
-  if (diagnostic->kind == DK_NOTE && notes_inhibited_in_group ())
+  if (diagnostic->m_kind == kind::note && notes_inhibited_in_group ())
     /* Bail for all the notes in the diagnostic_group that started to inhibit notes.  */
     return false;
 
-  if (diagnostic->kind != DK_NOTE && diagnostic->kind != DK_ICE)
+  if (diagnostic->m_kind != kind::note && diagnostic->m_kind != kind::ice)
     check_max_errors (false);
 
   if (m_lock > 0)
@@ -1488,7 +1301,8 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
       /* If we're reporting an ICE in the middle of some other error,
 	 try to flush out the previous error, then let this one
 	 through.  Don't do this more than once.  */
-      if ((diagnostic->kind == DK_ICE || diagnostic->kind == DK_ICE_NOBT)
+      if ((diagnostic->m_kind == kind::ice
+	   || diagnostic->m_kind == kind::ice_nobt)
 	  && m_lock == 1)
 	pp_newline_and_flush (m_reference_printer);
       else
@@ -1500,14 +1314,14 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
 
   m_lock++;
 
-  if (diagnostic->kind == DK_ICE || diagnostic->kind == DK_ICE_NOBT)
+  if (diagnostic->m_kind == kind::ice || diagnostic->m_kind == kind::ice_nobt)
     {
       /* When not checking, ICEs are converted to fatal errors when an
 	 error has already occurred.  This is counteracted by
 	 abort_on_error.  */
       if (!CHECKING_P
-	  && (diagnostic_count (DK_ERROR) > 0
-	      || diagnostic_count (DK_SORRY) > 0)
+	  && (diagnostic_count (kind::error) > 0
+	      || diagnostic_count (kind::sorry) > 0)
 	  && !m_abort_on_error)
 	{
 	  expanded_location s
@@ -1518,32 +1332,32 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
 	}
       if (m_internal_error)
 	(*m_internal_error) (this,
-			     diagnostic->message.m_format_spec,
-			     diagnostic->message.m_args_ptr);
+			     diagnostic->m_message.m_format_spec,
+			     diagnostic->m_message.m_args_ptr);
     }
 
   /* Increment the counter for the appropriate diagnostic kind, either
      within this context, or within the diagnostic_buffer.  */
   {
-    const diagnostic_t kind_for_count =
-      ((diagnostic->kind == DK_ERROR && orig_diag_kind == DK_WARNING)
-       ? DK_WERROR
-       : diagnostic->kind);
-    diagnostic_counters &counters
+    const enum kind kind_for_count =
+      ((diagnostic->m_kind == kind::error && orig_diag_kind == kind::warning)
+       ? kind::werror
+       : diagnostic->m_kind);
+    counters &cs
       = (m_diagnostic_buffer
 	 ? m_diagnostic_buffer->m_diagnostic_counters
 	 : m_diagnostic_counters);
-    ++counters.m_count_for_kind[kind_for_count];
+    ++cs.m_count_for_kind[static_cast<size_t> (kind_for_count)];
   }
 
   /* Is this the initial diagnostic within the stack of groups?  */
   if (m_diagnostic_groups.m_emission_count == 0)
-    for (auto sink : m_output_sinks)
-      sink->on_begin_group ();
+    for (auto sink_ : m_sinks)
+      sink_->on_begin_group ();
   m_diagnostic_groups.m_emission_count++;
 
-  va_list *orig_args = diagnostic->message.m_args_ptr;
-  for (auto sink : m_output_sinks)
+  va_list *orig_args = diagnostic->m_message.m_args_ptr;
+  for (auto sink_ : m_sinks)
     {
       /* Formatting the message is done per-output-format,
 	 so that each output format gets its own set of pp_token_lists
@@ -1562,13 +1376,13 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
 	 so that each has its own set to consume.  */
       va_list copied_args;
       va_copy (copied_args, *orig_args);
-      diagnostic->message.m_args_ptr = &copied_args;
-      pp_format (sink->get_printer (), &diagnostic->message);
+      diagnostic->m_message.m_args_ptr = &copied_args;
+      pp_format (sink_->get_printer (), &diagnostic->m_message);
       va_end (copied_args);
 
       /* Call vfunc in the output format.  This is responsible for
 	 phase 3 of formatting, and for printing the result.  */
-      sink->on_report_diagnostic (*diagnostic, orig_diag_kind);
+      sink_->on_report_diagnostic (*diagnostic, orig_diag_kind);
     }
 
   switch (m_extra_output_kind)
@@ -1577,53 +1391,60 @@ diagnostic_context::report_diagnostic (diagnostic_info *diagnostic)
       break;
     case EXTRA_DIAGNOSTIC_OUTPUT_fixits_v1:
       print_parseable_fixits (get_file_cache (),
-			      m_reference_printer, diagnostic->richloc,
+			      m_reference_printer, diagnostic->m_richloc,
 			      DIAGNOSTICS_COLUMN_UNIT_BYTE,
 			      m_tabstop);
       pp_flush (m_reference_printer);
       break;
     case EXTRA_DIAGNOSTIC_OUTPUT_fixits_v2:
       print_parseable_fixits (get_file_cache (),
-			      m_reference_printer, diagnostic->richloc,
+			      m_reference_printer, diagnostic->m_richloc,
 			      DIAGNOSTICS_COLUMN_UNIT_DISPLAY,
 			      m_tabstop);
       pp_flush (m_reference_printer);
       break;
     }
   if (m_diagnostic_buffer == nullptr
-      || diagnostic->kind == DK_ICE
-      || diagnostic->kind == DK_ICE_NOBT)
-    action_after_output (diagnostic->kind);
-  diagnostic->x_data = nullptr;
+      || diagnostic->m_kind == kind::ice
+      || diagnostic->m_kind == kind::ice_nobt)
+    action_after_output (diagnostic->m_kind);
+  diagnostic->m_x_data = nullptr;
 
-  if (m_edit_context_ptr)
-    if (diagnostic->richloc->fixits_can_be_auto_applied_p ())
+  if (m_fixits_change_set)
+    if (diagnostic->m_richloc->fixits_can_be_auto_applied_p ())
       if (!m_diagnostic_buffer)
-	m_edit_context_ptr->add_fixits (diagnostic->richloc);
+	m_fixits_change_set->add_fixits (diagnostic->m_richloc);
 
   m_lock--;
 
   if (!m_diagnostic_buffer)
-    for (auto sink : m_output_sinks)
-      sink->after_diagnostic (*diagnostic);
+    for (auto sink_ : m_sinks)
+      sink_->after_diagnostic (*diagnostic);
 
   return true;
 }
 
 void
-diagnostic_context::report_verbatim (text_info &text)
+context::report_verbatim (text_info &text)
 {
   va_list *orig_args = text.m_args_ptr;
-  for (auto sink : m_output_sinks)
+  for (auto sink_ : m_sinks)
     {
       va_list copied_args;
       va_copy (copied_args, *orig_args);
       text.m_args_ptr = &copied_args;
-      sink->on_report_verbatim (text);
+      sink_->on_report_verbatim (text);
       va_end (copied_args);
     }
 }
 
+void
+context::report_global_digraph (const lazily_created<digraphs::digraph> &ldg)
+{
+  for (auto sink_ : m_sinks)
+    sink_->report_global_digraph (ldg);
+}
+
 /* Get the number of digits in the decimal representation of VALUE.  */
 
 int
@@ -1645,6 +1466,8 @@ num_digits (int value)
   return digits;
 }
 
+} // namespace diagnostics
+
 /* Given a partial pathname as input, return another pathname that
    shares no directory elements with the pathname of __FILE__.  This
    is used by fancy_abort() to print `internal compiler error in expr.cc'
@@ -1675,43 +1498,48 @@ trim_filename (const char *name)
   return p;
 }
 
+namespace diagnostics {
+
 /* Implement emit_diagnostic, inform, warning, warning_at, pedwarn,
    permerror, error, error_at, error_at, sorry, fatal_error, internal_error,
    and internal_error_no_backtrace, as documented and defined below.  */
 bool
-diagnostic_context::diagnostic_impl (rich_location *richloc,
-				     const diagnostic_metadata *metadata,
-				     diagnostic_option_id option_id,
-				     const char *gmsgid,
-				     va_list *ap, diagnostic_t kind)
+context::diagnostic_impl (rich_location *richloc,
+			  const metadata *metadata,
+			  option_id opt_id,
+			  const char *gmsgid,
+			  va_list *ap, enum kind kind)
 {
   diagnostic_info diagnostic;
-  if (kind == DK_PERMERROR)
+  if (kind == diagnostics::kind::permerror)
     {
       diagnostic_set_info (&diagnostic, gmsgid, ap, richloc,
-			   m_permissive ? DK_WARNING : DK_ERROR);
-      diagnostic.option_id = (option_id.m_idx != -1 ? option_id : m_opt_permissive);
+			   (m_permissive
+			    ? diagnostics::kind::warning
+			    : diagnostics::kind::error));
+      diagnostic.m_option_id = (opt_id.m_idx != -1 ? opt_id : m_opt_permissive);
     }
   else
     {
       diagnostic_set_info (&diagnostic, gmsgid, ap, richloc, kind);
-      if (kind == DK_WARNING || kind == DK_PEDWARN)
-	diagnostic.option_id = option_id;
+      if (kind == diagnostics::kind::warning
+	  || kind == diagnostics::kind::pedwarn)
+	diagnostic.m_option_id = opt_id;
     }
-  diagnostic.metadata = metadata;
+  diagnostic.m_metadata = metadata;
   return report_diagnostic (&diagnostic);
 }
 
 /* Implement inform_n, warning_n, and error_n, as documented and
    defined below.  */
 bool
-diagnostic_context::diagnostic_n_impl (rich_location *richloc,
-				       const diagnostic_metadata *metadata,
-				       diagnostic_option_id option_id,
-				       unsigned HOST_WIDE_INT n,
-				       const char *singular_gmsgid,
-				       const char *plural_gmsgid,
-				       va_list *ap, diagnostic_t kind)
+context::diagnostic_n_impl (rich_location *richloc,
+			    const metadata *metadata,
+			    option_id opt_id,
+			    unsigned HOST_WIDE_INT n,
+			    const char *singular_gmsgid,
+			    const char *plural_gmsgid,
+			    va_list *ap, enum kind kind)
 {
   diagnostic_info diagnostic;
   unsigned long gtn;
@@ -1726,9 +1554,9 @@ diagnostic_context::diagnostic_n_impl (rich_location *richloc,
 
   const char *text = ngettext (singular_gmsgid, plural_gmsgid, gtn);
   diagnostic_set_info_translated (&diagnostic, text, ap, richloc, kind);
-  if (kind == DK_WARNING)
-    diagnostic.option_id = option_id;
-  diagnostic.metadata = metadata;
+  if (kind == diagnostics::kind::warning)
+    diagnostic.m_option_id = opt_id;
+  diagnostic.m_metadata = metadata;
   return report_diagnostic (&diagnostic);
 }
 
@@ -1736,13 +1564,13 @@ diagnostic_context::diagnostic_n_impl (rich_location *richloc,
 /* Emit DIAGRAM to this context, respecting the output format.  */
 
 void
-diagnostic_context::emit_diagram (const diagnostic_diagram &diagram)
+context::emit_diagram (const diagram &diag)
 {
   if (m_diagrams.m_theme == nullptr)
     return;
 
-  for (auto sink : m_output_sinks)
-    sink->on_diagram (diagram);
+  for (auto sink_ : m_sinks)
+    sink_->on_diagram (diag);
 }
 
 /* Inform the user that an error occurred while trying to report some
@@ -1751,7 +1579,7 @@ diagnostic_context::emit_diagram (const diagnostic_diagram &diagram)
    This mustn't use internal_error, that will cause infinite recursion.  */
 
 void
-diagnostic_context::error_recursion ()
+context::error_recursion ()
 {
   if (m_lock < 3)
     pp_newline_and_flush (m_reference_printer);
@@ -1761,13 +1589,15 @@ diagnostic_context::error_recursion ()
 
   /* Call action_after_output to get the "please submit a bug report"
      message.  */
-  action_after_output (DK_ICE);
+  action_after_output (kind::ice);
 
   /* Do not use gcc_unreachable here; that goes through internal_error
      and therefore would cause infinite recursion.  */
   real_abort ();
 }
 
+} // namespace diagnostics
+
 /* Report an internal compiler error in a friendly manner.  This is
    the function that gets called upon use of abort() in the source
    code generally, thanks to a special macro.  */
@@ -1787,7 +1617,7 @@ fancy_abort (const char *file, int line, const char *function)
   if (global_dc->get_reference_printer () == nullptr)
     {
       /* Print the error message.  */
-      fnotice (stderr, diagnostic_kind_text[DK_ICE]);
+      fnotice (stderr, diagnostics::get_text_for_kind (diagnostics::kind::ice));
       fnotice (stderr, "in %s, at %s:%d", function, trim_filename (file), line);
       fputc ('\n', stderr);
 
@@ -1810,16 +1640,18 @@ fancy_abort (const char *file, int line, const char *function)
   internal_error ("in %s, at %s:%d", function, trim_filename (file), line);
 }
 
-/* class diagnostic_context.  */
+namespace diagnostics {
+
+/* class diagnostics::context.  */
 
 void
-diagnostic_context::begin_group ()
+context::begin_group ()
 {
   m_diagnostic_groups.m_group_nesting_depth++;
 }
 
 void
-diagnostic_context::end_group ()
+context::end_group ()
 {
   if (--m_diagnostic_groups.m_group_nesting_depth == 0)
     {
@@ -1827,8 +1659,8 @@ diagnostic_context::end_group ()
 	 If any diagnostics were emitted, give the context a chance
 	 to do something.  */
       if (m_diagnostic_groups.m_emission_count > 0)
-	for (auto sink : m_output_sinks)
-	  sink->on_end_group ();
+	for (auto sink_ : m_sinks)
+	  sink_->on_end_group ();
       m_diagnostic_groups.m_emission_count = 0;
     }
   /* We're popping one level, so might need to stop inhibiting notes.  */
@@ -1836,13 +1668,13 @@ diagnostic_context::end_group ()
 }
 
 void
-diagnostic_context::push_nesting_level ()
+context::push_nesting_level ()
 {
   ++m_diagnostic_groups.m_diagnostic_nesting_level;
 }
 
 void
-diagnostic_context::pop_nesting_level ()
+context::pop_nesting_level ()
 {
   --m_diagnostic_groups.m_diagnostic_nesting_level;
   /* We're popping one level, so might need to stop inhibiting notes.  */
@@ -1850,29 +1682,29 @@ diagnostic_context::pop_nesting_level ()
 }
 
 void
-diagnostic_output_format::dump (FILE *out, int indent) const
+sink::dump (FILE *out, int indent) const
 {
   fprintf (out, "%*sprinter:\n", indent, "");
   m_printer->dump (out, indent + 2);
 }
 
 void
-diagnostic_output_format::on_report_verbatim (text_info &)
+sink::on_report_verbatim (text_info &)
 {
   /* No-op.  */
 }
 
-/* Set the output format for CONTEXT to FORMAT, using BASE_FILE_NAME for
+/* Set the output format for DC to FORMAT, using BASE_FILE_NAME for
    file-based output formats.  */
 
 void
-diagnostic_output_format_init (diagnostic_context &context,
-			       const char *main_input_filename_,
-			       const char *base_file_name,
-			       enum diagnostics_output_format format,
-			       bool json_formatting)
+output_format_init (context &dc,
+		    const char *main_input_filename_,
+		    const char *base_file_name,
+		    enum diagnostics_output_format format,
+		    bool json_formatting)
 {
-  diagnostic_output_format *new_sink = nullptr;
+  sink *new_sink = nullptr;
   switch (format)
     {
     default:
@@ -1882,16 +1714,16 @@ diagnostic_output_format_init (diagnostic_context &context,
       break;
 
     case DIAGNOSTICS_OUTPUT_FORMAT_SARIF_STDERR:
-      new_sink = &diagnostic_output_format_init_sarif_stderr (context,
-							      line_table,
-							      json_formatting);
+      new_sink = &init_sarif_stderr (dc,
+				     line_table,
+				     json_formatting);
       break;
 
     case DIAGNOSTICS_OUTPUT_FORMAT_SARIF_FILE:
-      new_sink = &diagnostic_output_format_init_sarif_file (context,
-							line_table,
-							    json_formatting,
-							    base_file_name);
+      new_sink = &init_sarif_file (dc,
+				   line_table,
+				   json_formatting,
+				   base_file_name);
       break;
     }
   if (new_sink)
@@ -1903,8 +1735,7 @@ diagnostic_output_format_init (diagnostic_context &context,
    (or nullptr for "no diagrams").  */
 
 void
-diagnostic_context::
-set_text_art_charset (enum diagnostic_text_art_charset charset)
+context::set_text_art_charset (enum diagnostic_text_art_charset charset)
 {
   delete m_diagrams.m_theme;
   switch (charset)
@@ -1930,95 +1761,24 @@ set_text_art_charset (enum diagnostic_text_art_charset charset)
     }
 }
 
-/* If BUFFER is non-null, use BUFFER as the active diagnostic_buffer on
-   this context.  BUFFER is borrowed.
-
-   If BUFFER is null, stop any buffering on this context until the next call
-   to this function.  */
-
-void
-diagnostic_context::set_diagnostic_buffer (diagnostic_buffer *buffer)
-{
-  /* We don't allow changing buffering within a diagnostic group
-     (to simplify handling of buffered diagnostics within the
-     diagnostic_format implementations).  */
-  gcc_assert (m_diagnostic_groups.m_group_nesting_depth == 0);
-
-  /* Likewise, for simplicity, we only allow changing buffers
-     at nesting level 0.  */
-  gcc_assert (m_diagnostic_groups.m_diagnostic_nesting_level == 0);
-
-  m_diagnostic_buffer = buffer;
-
-  if (buffer)
-    {
-      buffer->ensure_per_format_buffers ();
-      gcc_assert (buffer->m_per_format_buffers);
-      gcc_assert (buffer->m_per_format_buffers->length ()
-		  == m_output_sinks.length ());
-      for (unsigned idx = 0; idx < m_output_sinks.length (); ++idx)
-	{
-	  auto sink = m_output_sinks[idx];
-	  auto per_format_buffer = (*buffer->m_per_format_buffers)[idx];
-	  sink->set_buffer (per_format_buffer);
-	}
-    }
-  else
-    for (auto sink : m_output_sinks)
-      sink->set_buffer (nullptr);
-}
-
-/* Clear BUFFER without flushing it.  */
-
-void
-diagnostic_context::clear_diagnostic_buffer (diagnostic_buffer &buffer)
-{
-  if (buffer.m_per_format_buffers)
-    for (auto per_format_buffer : *buffer.m_per_format_buffers)
-      per_format_buffer->clear ();
-
-  buffer.m_diagnostic_counters.clear ();
-
-  /* We need to reset last_location, otherwise we may skip caret lines
-     when we actually give a diagnostic.  */
-  m_last_location = UNKNOWN_LOCATION;
-}
-
-/* Flush the diagnostics in BUFFER to this context, clearing BUFFER.  */
+/* struct diagnostics::counters.  */
 
-void
-diagnostic_context::flush_diagnostic_buffer (diagnostic_buffer &buffer)
-{
-  bool had_errors
-    = (buffer.m_diagnostic_counters.m_count_for_kind[DK_ERROR] > 0
-       || buffer.m_diagnostic_counters.m_count_for_kind[DK_WERROR] > 0);
-  if (buffer.m_per_format_buffers)
-    for (auto per_format_buffer : *buffer.m_per_format_buffers)
-      per_format_buffer->flush ();
-  buffer.m_diagnostic_counters.move_to (m_diagnostic_counters);
-
-  action_after_output (had_errors ? DK_ERROR : DK_WARNING);
-  check_max_errors (true);
-}
-
-/* struct diagnostic_counters.  */
-
-diagnostic_counters::diagnostic_counters ()
+counters::counters ()
 {
   clear ();
 }
 
 void
-diagnostic_counters::dump (FILE *out, int indent) const
+counters::dump (FILE *out, int indent) const
 {
   fprintf (out, "%*scounts:\n", indent, "");
   bool none = true;
-  for (int i = 0; i < DK_LAST_DIAGNOSTIC_KIND; i++)
+  for (int i = 0; i < static_cast<int> (kind::last_diagnostic_kind); i++)
     if (m_count_for_kind[i] > 0)
       {
 	fprintf (out, "%*s%s%i\n",
 		 indent + 2, "",
-		 get_diagnostic_kind_text (static_cast<diagnostic_t> (i)),
+		 get_text_for_kind (static_cast<enum kind> (i)),
 		 m_count_for_kind[i]);
 	none = false;
       }
@@ -2027,133 +1787,31 @@ diagnostic_counters::dump (FILE *out, int indent) const
 }
 
 void
-diagnostic_counters::move_to (diagnostic_counters &dest)
+counters::move_to (counters &dest)
 {
-  for (int i = 0; i < DK_LAST_DIAGNOSTIC_KIND; i++)
+  for (int i = 0; i < static_cast<int> (kind::last_diagnostic_kind); i++)
     dest.m_count_for_kind[i] += m_count_for_kind[i];
   clear ();
 }
 
 void
-diagnostic_counters::clear ()
+counters::clear ()
 {
   memset (&m_count_for_kind, 0, sizeof m_count_for_kind);
 }
 
-/* class diagnostic_buffer.  */
-
-diagnostic_buffer::diagnostic_buffer (diagnostic_context &ctxt)
-: m_ctxt (ctxt),
-  m_per_format_buffers (nullptr)
-{
-}
-
-diagnostic_buffer::~diagnostic_buffer ()
-{
-  if (m_per_format_buffers)
-    {
-      for (auto iter : *m_per_format_buffers)
-	delete iter;
-      delete m_per_format_buffers;
-    }
-}
-
-void
-diagnostic_buffer::dump (FILE *out, int indent) const
-{
-  m_diagnostic_counters.dump (out, indent + 2);
-  fprintf (out, "%*sm_per_format_buffers:\n", indent, "");
-  if (m_per_format_buffers)
-    for (auto per_format_buffer : *m_per_format_buffers)
-      per_format_buffer->dump (out, indent + 2);
-  else
-    fprintf (out, "%*s(none)\n", indent + 2, "");
-}
-
-bool
-diagnostic_buffer::empty_p () const
-{
-  if (m_per_format_buffers)
-    for (auto per_format_buffer : *m_per_format_buffers)
-      /* Query initial buffer.  */
-      return per_format_buffer->empty_p ();
-  return true;
-}
-
-void
-diagnostic_buffer::move_to (diagnostic_buffer &dest)
-{
-  /* Bail if there's nothing to move.  */
-  if (!m_per_format_buffers)
-    return;
-
-  m_diagnostic_counters.move_to (dest.m_diagnostic_counters);
-
-  if (!dest.m_per_format_buffers)
-    {
-      /* Optimization for the "move to empty" case:
-	 simply move the vec to the dest.  */
-      dest.m_per_format_buffers = m_per_format_buffers;
-      m_per_format_buffers = nullptr;
-      return;
-    }
-
-  dest.ensure_per_format_buffers ();
-  gcc_assert (m_per_format_buffers);
-  gcc_assert (m_per_format_buffers->length ()
-	      == m_ctxt.m_output_sinks.length ());
-  gcc_assert (dest.m_per_format_buffers);
-  gcc_assert (dest.m_per_format_buffers->length ()
-	      == m_ctxt.m_output_sinks.length ());
-  for (unsigned idx = 0; idx < m_ctxt.m_output_sinks.length (); ++idx)
-    {
-      auto per_format_buffer_src = (*m_per_format_buffers)[idx];
-      auto per_format_buffer_dest = (*dest.m_per_format_buffers)[idx];
-      per_format_buffer_src->move_to (*per_format_buffer_dest);
-    }
-}
-
-/* Lazily get the output formats to create their own kind of buffers.
-   We can't change the output sinks on a context once this has been called
-   on any diagnostic_buffer instances for that context, since there's no
-   way to update all diagnostic_buffer instances for that context.  */
-
-void
-diagnostic_buffer::ensure_per_format_buffers ()
-{
-  if (!m_per_format_buffers)
-    {
-      m_per_format_buffers = new auto_vec<diagnostic_per_format_buffer *> ();
-      for (unsigned idx = 0; idx < m_ctxt.m_output_sinks.length (); ++idx)
-	{
-	  auto sink = m_ctxt.m_output_sinks[idx];
-	  auto per_format_buffer = sink->make_per_format_buffer ();
-	  m_per_format_buffers->safe_push (per_format_buffer.release ());
-	}
-    }
-  gcc_assert (m_per_format_buffers);
-  gcc_assert (m_per_format_buffers->length ()
-	      == m_ctxt.m_output_sinks.length ());
-}
-
-/* Really call the system 'abort'.  This has to go right at the end of
-   this file, so that there are no functions after it that call abort
-   and get the system abort instead of our macro.  */
-#undef abort
-static void
-real_abort (void)
-{
-  abort ();
-}
-
 #if CHECKING_P
 
 namespace selftest {
 
+using line_table_test = ::selftest::line_table_test;
+using temp_source_file = ::selftest::temp_source_file;
+
 /* Helper function for test_print_escaped_string.  */
 
 static void
-assert_print_escaped_string (const location &loc, const char *expected_output,
+assert_print_escaped_string (const ::selftest::location &loc,
+			     const char *expected_output,
 			     const char *input)
 {
   pretty_printer pp;
@@ -2338,7 +1996,7 @@ test_print_parseable_fixits_bytes_vs_display_columns ()
 }
 
 /* Verify that
-     diagnostic_column_policy::get_location_text (..., SHOW_COLUMN, ...)
+     diagnostics::column_policy::get_location_text (..., SHOW_COLUMN, ...)
    generates EXPECTED_LOC_TEXT, given FILENAME, LINE, COLUMN, with
    colorization disabled.  */
 
@@ -2350,7 +2008,7 @@ assert_location_text (const char *expected_loc_text,
 		      enum diagnostics_column_unit column_unit
 			= DIAGNOSTICS_COLUMN_UNIT_BYTE)
 {
-  test_diagnostic_context dc;
+  diagnostics::selftest::test_context dc;
   dc.m_column_unit = column_unit;
   dc.m_column_origin = origin;
 
@@ -2361,7 +2019,7 @@ assert_location_text (const char *expected_loc_text,
   xloc.data = nullptr;
   xloc.sysp = false;
 
-  diagnostic_column_policy column_policy (dc);
+  diagnostics::column_policy column_policy (dc);
   label_text actual_loc_text
     = column_policy.get_location_text (xloc, show_column, false);
   ASSERT_STREQ (expected_loc_text, actual_loc_text.get ());
@@ -2388,8 +2046,8 @@ test_get_location_text ()
   assert_location_text ("foo.c:42:", "foo.c", 42, 10, false);
   assert_location_text ("foo.c:", "foo.c", 0, 10, false);
 
-  maybe_line_and_column (INT_MAX, INT_MAX);
-  maybe_line_and_column (INT_MIN, INT_MIN);
+  diagnostics::maybe_line_and_column (INT_MAX, INT_MAX);
+  diagnostics::maybe_line_and_column (INT_MIN, INT_MIN);
 
   {
     /* In order to test display columns vs byte columns, we need to create a
@@ -2452,10 +2110,14 @@ test_num_digits ()
   ASSERT_EQ (8, num_digits (99999999));
 }
 
-/* Run all of the selftests within this file.  */
+/* Run all of the selftests within this file.
+
+   According to https://gcc.gnu.org/pipermail/gcc/2021-November/237703.html
+   there are some language-specific assumptions within these tests, so only
+   run them from C/C++.  */
 
 void
-c_diagnostic_cc_tests ()
+context_cc_tests ()
 {
   test_print_escaped_string ();
   test_print_parseable_fixits_none ();
@@ -2467,10 +2129,24 @@ c_diagnostic_cc_tests ()
   test_num_digits ();
 }
 
-} // namespace selftest
+} // namespace diagnostics::selftest
 
 #endif /* #if CHECKING_P */
 
+} // namespace diagnostics
+
 #if __GNUC__ >= 10
 #  pragma GCC diagnostic pop
 #endif
+
+static void real_abort (void) ATTRIBUTE_NORETURN;
+
+/* Really call the system 'abort'.  This has to go right at the end of
+   this file, so that there are no functions after it that call abort
+   and get the system abort instead of our macro.  */
+#undef abort
+static void
+real_abort (void)
+{
+  abort ();
+}
diff --git a/gcc/diagnostics/context.h b/gcc/diagnostics/context.h
new file mode 100644
index 0000000..f47370b
--- /dev/null
+++ b/gcc/diagnostics/context.h
@@ -0,0 +1,959 @@
+/* Declare diagnostics::context and related types.
+   Copyright (C) 2000-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_CONTEXT_H
+#define GCC_DIAGNOSTICS_CONTEXT_H
+
+#include "lazily-created.h"
+#include "unique-argv.h"
+#include "diagnostics/option-classifier.h"
+#include "diagnostics/context-options.h"
+
+namespace diagnostics {
+
+  namespace changes {
+    class change_set;
+  }
+
+  namespace digraphs { class digraph; }
+
+  namespace logical_locations {
+    class manager;
+  }
+
+  class buffer;
+  class client_data_hooks;
+  class diagram;
+  class sink;
+    class text_sink;
+
+  class source_effect_info;
+
+} // namespace diagnostics
+
+namespace text_art
+{
+  class theme;
+} // namespace text_art
+
+namespace xml
+{
+  class printer;
+} // namespace xml
+
+namespace diagnostics {
+
+/*  Forward declarations.  */
+class context;
+class location_print_policy;
+class source_print_policy;
+
+typedef void (*text_starter_fn) (text_sink &,
+				 const diagnostic_info *);
+
+struct to_text;
+struct to_html;
+
+extern pretty_printer *get_printer (to_text &);
+
+template <typename TextOrHtml>
+using start_span_fn = void (*) (const location_print_policy &,
+				TextOrHtml &text_or_html,
+				expanded_location);
+
+typedef void (*text_finalizer_fn) (text_sink &,
+				   const diagnostic_info *,
+				   enum kind);
+
+/* Abstract base class for the diagnostic subsystem to make queries
+   about command-line options.  */
+
+class option_manager
+{
+public:
+  virtual ~option_manager () {}
+
+  /* Return 1 if option OPT_ID is enabled, 0 if it is disabled,
+     or -1 if it isn't a simple on-off switch
+     (or if the value is unknown, typically set later in target).  */
+  virtual int option_enabled_p (option_id opt_id) const = 0;
+
+  /* Return malloced memory for the name of the option OPT_ID
+     which enabled a diagnostic, originally of type ORIG_DIAG_KIND but
+     possibly converted to DIAG_KIND by options such as -Werror.
+     May return NULL if no name is to be printed.
+     May be passed 0 as well as the index of a particular option.  */
+  virtual char *make_option_name (option_id opt_id,
+				  enum kind orig_diag_kind,
+				  enum kind diag_kind) const = 0;
+
+  /* Return malloced memory for a URL describing the option that controls
+     a diagnostic.
+     May return NULL if no URL is available.
+     May be passed 0 as well as the index of a particular option.  */
+  virtual char *make_option_url (option_id opt_id) const = 0;
+};
+
+/* A bundle of options relating to printing the user's source code
+   (potentially with a margin, underlining, labels, etc).  */
+
+struct source_printing_options
+{
+  /* True if we should print the source line with a caret indicating
+     the location.
+     Corresponds to -fdiagnostics-show-caret.  */
+  bool enabled;
+
+  /* Maximum width of the source line printed.  */
+  int max_width;
+
+  /* Character used at the caret when printing source locations.  */
+  char caret_chars[rich_location::STATICALLY_ALLOCATED_RANGES];
+
+  /* When printing source code, should the characters at carets and ranges
+     be colorized? (assuming colorization is on at all).
+     This should be true for frontends that generate range information
+     (so that the ranges of code are colorized),
+     and false for frontends that merely specify points within the
+     source code (to avoid e.g. colorizing just the first character in
+     a token, which would look strange).  */
+  bool colorize_source_p;
+
+  /* When printing source code, should labelled ranges be printed?
+     Corresponds to -fdiagnostics-show-labels.  */
+  bool show_labels_p;
+
+  /* When printing source code, should there be a left-hand margin
+     showing line numbers?
+     Corresponds to -fdiagnostics-show-line-numbers.  */
+  bool show_line_numbers_p;
+
+  /* If printing source code, what should the minimum width of the margin
+     be?  Line numbers will be right-aligned, and padded to this width.
+     Corresponds to -fdiagnostics-minimum-margin-width=VALUE.  */
+  int min_margin_width;
+
+  /* Usable by plugins; if true, print a debugging ruler above the
+     source output.  */
+  bool show_ruler_p;
+
+  /* When printing events in an inline path, should we print lines
+     visualizing links between related events (e.g. for CFG paths)?
+     Corresponds to -fdiagnostics-show-event-links.  */
+  bool show_event_links_p;
+};
+
+/* A bundle of state for determining column numbers in diagnostics
+   (tab stops, whether to start at 0 or 1, etc).
+   Uses a file_cache to handle tabs.  */
+
+class column_policy
+{
+public:
+  column_policy (const context &dc);
+
+  int converted_column (expanded_location s) const;
+
+  label_text get_location_text (const expanded_location &s,
+				bool show_column,
+				bool colorize) const;
+
+  int get_tabstop () const { return m_tabstop; }
+
+private:
+  file_cache &m_file_cache;
+  enum diagnostics_column_unit m_column_unit;
+  int m_column_origin;
+  int m_tabstop;
+};
+
+/* A bundle of state for printing locations within diagnostics
+   (e.g. "FILENAME:LINE:COLUMN"), to isolate the interactions between
+   context and the start_span callbacks.  */
+
+class location_print_policy
+{
+public:
+  location_print_policy (const context &dc);
+  location_print_policy (const text_sink &);
+
+  bool show_column_p () const { return m_show_column; }
+
+  const column_policy &
+  get_column_policy () const { return m_column_policy; }
+
+  void
+  print_text_span_start (const context &dc,
+			 pretty_printer &pp,
+			 const expanded_location &exploc);
+
+  void
+  print_html_span_start (const context &dc,
+			 xml::printer &xp,
+			 const expanded_location &exploc);
+
+private:
+  column_policy m_column_policy;
+  bool m_show_column;
+};
+
+/* Abstract base class for optionally supplying extra tags when writing
+   out annotation labels in HTML output.  */
+
+class html_label_writer
+{
+public:
+  virtual ~html_label_writer () {}
+  virtual void begin_label () = 0;
+  virtual void end_label () = 0;
+};
+
+/* A bundle of state for printing source within a diagnostic,
+   to isolate the interactions between context and the
+   implementation of diagnostic_show_locus.  */
+
+class source_print_policy
+{
+public:
+  source_print_policy (const context &);
+  source_print_policy (const context &,
+		       const source_printing_options &);
+
+  void
+  print (pretty_printer &pp,
+	 const rich_location &richloc,
+	 enum kind diagnostic_kind,
+	 source_effect_info *effect_info) const;
+
+  void
+  print_as_html (xml::printer &xp,
+		 const rich_location &richloc,
+		 enum kind diagnostic_kind,
+		 source_effect_info *effect_info,
+		 html_label_writer *label_writer) const;
+
+  const source_printing_options &
+  get_options () const { return m_options; }
+
+  start_span_fn<to_text>
+  get_text_start_span_fn () const { return m_text_start_span_cb; }
+
+  start_span_fn<to_html>
+  get_html_start_span_fn () const { return m_html_start_span_cb; }
+
+  file_cache &
+  get_file_cache () const { return m_file_cache; }
+
+  enum diagnostics_escape_format
+  get_escape_format () const
+  {
+    return m_escape_format;
+  }
+
+  text_art::theme *
+  get_diagram_theme () const { return m_diagram_theme; }
+
+  const column_policy &get_column_policy () const
+  {
+    return m_location_policy.get_column_policy ();
+  }
+
+  const location_print_policy &get_location_policy () const
+  {
+    return m_location_policy;
+  }
+
+private:
+  const source_printing_options &m_options;
+  location_print_policy m_location_policy;
+  start_span_fn<to_text> m_text_start_span_cb;
+  start_span_fn<to_html> m_html_start_span_cb;
+  file_cache &m_file_cache;
+
+  /* Other data copied from context.  */
+  text_art::theme *m_diagram_theme;
+  enum diagnostics_escape_format m_escape_format;
+};
+
+/* A collection of counters of diagnostics, per-kind
+   (e.g. "3 errors and 1 warning"), for use by both context
+   and by diagnostics::buffer.  */
+
+struct counters
+{
+  counters ();
+
+  void dump (FILE *out, int indent) const;
+  void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
+
+  int get_count (enum kind kind) const
+  {
+    return m_count_for_kind[static_cast<size_t> (kind)];
+  }
+
+  void move_to (counters &dest);
+  void clear ();
+
+  int m_count_for_kind[static_cast<size_t> (kind::last_diagnostic_kind)];
+};
+
+/* This class encapsulates the state of the diagnostics subsystem
+   as a whole (either directly, or via owned objects of other classes, to
+   avoid global variables).
+
+   It has responsibility for:
+   - being a central place for clients to report diagnostics
+   - reporting those diagnostics to zero or more output sinks
+     (e.g. text vs SARIF)
+   - providing a "dump" member function for a debug dump of the state of
+     the diagnostics subsytem
+   - direct vs buffered diagnostics (see class diagnostics::buffer)
+   - tracking the original argv of the program (for SARIF output)
+   - crash-handling
+
+   It delegates responsibility to various other classes:
+   - the various output sinks (instances of diagnostics::sink
+     subclasses)
+   - formatting of messages (class pretty_printer)
+   - an optional urlifier to inject URLs into formatted messages
+   - counting the number of diagnostics reported of each kind
+     (class diagnostics::counters)
+   - calling out to a option_manager to determine if
+     a particular warning is enabled or disabled
+   - tracking pragmas that enable/disable warnings in a range of
+     source code
+   - a cache for use when quoting the user's source code (class file_cache)
+   - a text_art::theme
+   - a diagnostics::changes::change_set for generating patches from fix-it hints
+   - diagnostics::client_data_hooks for metadata.
+
+   Try to avoid adding new responsibilities to this class itself, to avoid
+   the "blob" anti-pattern.  */
+
+class context
+{
+public:
+  /* Give access to m_text_callbacks.  */
+  // FIXME: these need updating
+  friend text_starter_fn &
+  text_starter (context *dc);
+  friend start_span_fn<to_text> &
+  start_span (context *dc);
+  friend text_finalizer_fn &
+  text_finalizer (context *dc);
+
+  friend class source_print_policy;
+  friend class text_sink;
+  friend class buffer;
+
+  typedef void (*set_locations_callback_t) (context *,
+					    diagnostic_info *);
+
+  void initialize (int n_opts);
+  void color_init (int value);
+  void urls_init (int value);
+  void set_pretty_printer (std::unique_ptr<pretty_printer> pp);
+  void refresh_output_sinks ();
+
+  void finish ();
+
+  void dump (FILE *out) const;
+  void DEBUG_FUNCTION dump () const { dump (stderr); }
+
+  bool execution_failed_p () const;
+
+  void set_original_argv (unique_argv original_argv);
+  const char * const *get_original_argv ()
+  {
+    return const_cast<const char * const *> (m_original_argv);
+  }
+
+  void set_set_locations_callback (set_locations_callback_t cb)
+  {
+    m_set_locations_cb = cb;
+  }
+
+  void
+  initialize_input_context (diagnostic_input_charset_callback ccb,
+			    bool should_skip_bom);
+
+  void begin_group ();
+  void end_group ();
+
+  void push_nesting_level ();
+  void pop_nesting_level ();
+
+  bool warning_enabled_at (location_t loc, option_id opt_id);
+
+  bool option_unspecified_p (option_id opt_id) const
+  {
+    return m_option_classifier.option_unspecified_p (opt_id);
+  }
+
+  bool emit_diagnostic_with_group (enum kind kind,
+				   rich_location &richloc,
+				   const metadata *metadata,
+				   option_id opt_id,
+				   const char *gmsgid, ...)
+    ATTRIBUTE_GCC_DIAG(6,7);
+  bool emit_diagnostic_with_group_va (enum kind kind,
+				      rich_location &richloc,
+				      const metadata *metadata,
+				      option_id opt_id,
+				      const char *gmsgid, va_list *ap)
+    ATTRIBUTE_GCC_DIAG(6,0);
+
+  bool report_diagnostic (diagnostic_info *);
+  void report_verbatim (text_info &);
+
+  /* Report a directed graph associated with the run as a whole
+     to any sinks that support directed graphs.  */
+  void
+  report_global_digraph (const lazily_created<digraphs::digraph> &);
+
+  enum kind
+  classify_diagnostic (option_id opt_id,
+		       enum kind new_kind,
+		       location_t where)
+  {
+    return m_option_classifier.classify_diagnostic (this,
+						    opt_id,
+						    new_kind,
+						    where);
+  }
+
+  void push_diagnostics (location_t where ATTRIBUTE_UNUSED)
+  {
+    m_option_classifier.push ();
+  }
+  void pop_diagnostics (location_t where)
+  {
+    m_option_classifier.pop (where);
+  }
+
+  void maybe_show_locus (const rich_location &richloc,
+			 const source_printing_options &opts,
+			 enum kind diagnostic_kind,
+			 pretty_printer &pp,
+			 source_effect_info *effect_info);
+  void maybe_show_locus_as_html (const rich_location &richloc,
+				 const source_printing_options &opts,
+				 enum kind diagnostic_kind,
+				 xml::printer &xp,
+				 source_effect_info *effect_info,
+				 html_label_writer *label_writer);
+
+  void emit_diagram (const diagram &diag);
+
+  /* Various setters for use by option-handling logic.  */
+  void set_sink (std::unique_ptr<sink> sink_);
+  void set_text_art_charset (enum diagnostic_text_art_charset charset);
+  void set_client_data_hooks (std::unique_ptr<client_data_hooks> hooks);
+
+  void push_owned_urlifier (std::unique_ptr<urlifier>);
+  void push_borrowed_urlifier (const urlifier &);
+  void pop_urlifier ();
+
+  void initialize_fixits_change_set ();
+  void set_warning_as_error_requested (bool val)
+  {
+    m_warning_as_error_requested = val;
+  }
+  void set_report_bug (bool val) { m_report_bug = val; }
+  void set_extra_output_kind (enum diagnostics_extra_output_kind kind)
+  {
+    m_extra_output_kind = kind;
+  }
+  void set_show_cwe (bool val) { m_show_cwe = val;  }
+  void set_show_rules (bool val) { m_show_rules = val; }
+  void set_show_highlight_colors (bool val);
+  void set_path_format (enum diagnostic_path_format val)
+  {
+    m_path_format = val;
+  }
+  void set_show_path_depths (bool val) { m_show_path_depths = val; }
+  void set_show_option_requested (bool val) { m_show_option_requested = val; }
+  void set_max_errors (int val) { m_max_errors = val; }
+  void set_escape_format (enum diagnostics_escape_format val)
+  {
+    m_escape_format = val;
+  }
+
+  void set_format_decoder (printer_fn format_decoder);
+  void set_prefixing_rule (diagnostic_prefixing_rule_t rule);
+
+  /* Various accessors.  */
+  bool warning_as_error_requested_p () const
+  {
+    return m_warning_as_error_requested;
+  }
+  bool show_path_depths_p () const { return m_show_path_depths; }
+  sink &get_sink (size_t idx) const;
+  enum diagnostic_path_format get_path_format () const { return m_path_format; }
+  enum diagnostics_escape_format get_escape_format () const
+  {
+    return m_escape_format;
+  }
+
+  file_cache &
+  get_file_cache () const
+  {
+    gcc_assert (m_file_cache);
+    return *m_file_cache;
+  }
+
+  changes::change_set *get_fixits_change_set () const
+  {
+    return m_fixits_change_set;
+  }
+  const client_data_hooks *get_client_data_hooks () const
+  {
+    return m_client_data_hooks;
+  }
+
+  const logical_locations::manager *
+  get_logical_location_manager () const;
+
+  const urlifier *get_urlifier () const;
+
+  text_art::theme *get_diagram_theme () const { return m_diagrams.m_theme; }
+
+  int &diagnostic_count (enum kind kind)
+  {
+    return m_diagnostic_counters.m_count_for_kind[static_cast<size_t> (kind)];
+  }
+  int diagnostic_count (enum kind kind) const
+  {
+    return m_diagnostic_counters.get_count (kind);
+  }
+
+  /* Option-related member functions.  */
+  inline bool option_enabled_p (option_id opt_id) const
+  {
+    if (!m_option_mgr)
+      return true;
+    return m_option_mgr->option_enabled_p (opt_id);
+  }
+
+  inline char *make_option_name (option_id opt_id,
+				 enum kind orig_diag_kind,
+				 enum kind diag_kind) const
+  {
+    if (!m_option_mgr)
+      return nullptr;
+    return m_option_mgr->make_option_name (opt_id,
+					   orig_diag_kind,
+					   diag_kind);
+  }
+
+  inline char *make_option_url (option_id opt_id) const
+  {
+    if (!m_option_mgr)
+      return nullptr;
+    return m_option_mgr->make_option_url (opt_id);
+  }
+
+  void
+  set_option_manager (std::unique_ptr<option_manager> mgr,
+		      unsigned lang_mask);
+
+  unsigned get_lang_mask () const
+  {
+    return m_lang_mask;
+  }
+
+  bool diagnostic_impl (rich_location *, const metadata *,
+			option_id, const char *,
+			va_list *, enum kind) ATTRIBUTE_GCC_DIAG(5,0);
+  bool diagnostic_n_impl (rich_location *, const metadata *,
+			  option_id, unsigned HOST_WIDE_INT,
+			  const char *, const char *, va_list *,
+			  enum kind) ATTRIBUTE_GCC_DIAG(7,0);
+
+  int get_diagnostic_nesting_level () const
+  {
+    return m_diagnostic_groups.m_diagnostic_nesting_level;
+  }
+
+  char *build_indent_prefix () const;
+
+  int
+  pch_save (FILE *f)
+  {
+    return m_option_classifier.pch_save (f);
+  }
+
+  int
+  pch_restore (FILE *f)
+  {
+    return m_option_classifier.pch_restore (f);
+  }
+
+
+  void set_diagnostic_buffer (buffer *);
+  buffer *get_diagnostic_buffer () const
+  {
+    return m_diagnostic_buffer;
+  }
+  void clear_diagnostic_buffer (buffer &);
+  void flush_diagnostic_buffer (buffer &);
+
+  std::unique_ptr<pretty_printer> clone_printer () const
+  {
+    return m_reference_printer->clone ();
+  }
+
+  pretty_printer *get_reference_printer () const
+  {
+    return m_reference_printer;
+  }
+
+  void
+  add_sink (std::unique_ptr<sink>);
+
+  void remove_all_output_sinks ();
+
+  bool supports_fnotice_on_stderr_p () const;
+
+  /* Raise SIGABRT on any diagnostic of severity kind::error or higher.  */
+  void
+  set_abort_on_error (bool val)
+  {
+    m_abort_on_error = val;
+  }
+
+  /* Accessor for use in serialization, e.g. by C++ modules.  */
+  auto &
+  get_classification_history ()
+  {
+    return m_option_classifier.m_classification_history;
+  }
+
+  void set_main_input_filename (const char *filename);
+
+  void
+  set_permissive_option (option_id opt_permissive)
+  {
+    m_opt_permissive = opt_permissive;
+  }
+
+  void
+  set_fatal_errors (bool fatal_errors)
+  {
+    m_fatal_errors = fatal_errors;
+  }
+
+  void
+  set_internal_error_callback (void (*cb) (context *,
+					   const char *,
+					   va_list *))
+  {
+    m_internal_error = cb;
+  }
+
+  void
+  set_adjust_diagnostic_info_callback (void (*cb) (context *,
+						   diagnostic_info *))
+  {
+    m_adjust_diagnostic_info = cb;
+  }
+
+  void
+  inhibit_notes () { m_inhibit_notes_p = true; }
+
+  source_printing_options &
+  get_source_printing_options ()
+  {
+    return m_source_printing;
+  }
+  const source_printing_options &
+  get_source_printing_options () const
+  {
+    return m_source_printing;
+  }
+
+  void set_caret_max_width (int value);
+
+private:
+  void error_recursion () ATTRIBUTE_NORETURN;
+
+  bool diagnostic_enabled (diagnostic_info *diagnostic);
+
+  void get_any_inlining_info (diagnostic_info *diagnostic);
+
+  void check_max_errors (bool flush);
+  void action_after_output (enum kind diag_kind);
+
+  /* Data members.
+     Ideally, all of these would be private.  */
+
+private:
+  /* A reference instance of pretty_printer created by the client
+     and owned by the context.  Used for cloning when creating/adding
+     output formats.
+     Owned by the context; this would be a std::unique_ptr if
+     context had a proper ctor.  */
+  pretty_printer *m_reference_printer;
+
+  /* Cache of source code.
+     Owned by the context; this would be a std::unique_ptr if
+     context had a proper ctor.  */
+  file_cache *m_file_cache;
+
+  /* The number of times we have issued diagnostics.  */
+  counters m_diagnostic_counters;
+
+  /* True if it has been requested that warnings be treated as errors.  */
+  bool m_warning_as_error_requested;
+
+  /* The number of option indexes that can be passed to warning() et
+     al.  */
+  int m_n_opts;
+
+  /* The stack of sets of overridden diagnostic option severities.  */
+  option_classifier m_option_classifier;
+
+  /* True if we should print any CWE identifiers associated with
+     diagnostics.  */
+  bool m_show_cwe;
+
+  /* True if we should print any rules associated with diagnostics.  */
+  bool m_show_rules;
+
+  /* How should diagnostics::paths::path objects be printed.  */
+  enum diagnostic_path_format m_path_format;
+
+  /* True if we should print stack depths when printing diagnostic paths.  */
+  bool m_show_path_depths;
+
+  /* True if we should print the command line option which controls
+     each diagnostic, if known.  */
+  bool m_show_option_requested;
+
+  /* True if we should raise a SIGABRT on errors.  */
+  bool m_abort_on_error;
+
+public:
+  /* True if we should show the column number on diagnostics.  */
+  bool m_show_column;
+
+  /* True if pedwarns are errors.  */
+  bool m_pedantic_errors;
+
+  /* True if permerrors are warnings.  */
+  bool m_permissive;
+
+private:
+  /* The option to associate with turning permerrors into warnings,
+     if any.  */
+  option_id m_opt_permissive;
+
+  /* True if errors are fatal.  */
+  bool m_fatal_errors;
+
+public:
+  /* True if all warnings should be disabled.  */
+  bool m_inhibit_warnings;
+
+  /* True if warnings should be given in system headers.  */
+  bool m_warn_system_headers;
+
+private:
+  /* Maximum number of errors to report.  */
+  int m_max_errors;
+
+  /* Client-supplied callbacks for use in text output.  */
+  struct {
+    /* This function is called before any message is printed out.  It is
+       responsible for preparing message prefix and such.  For example, it
+       might say:
+       In file included from "/usr/local/include/curses.h:5:
+       from "/home/gdr/src/nifty_printer.h:56:
+       ...
+    */
+    text_starter_fn m_begin_diagnostic;
+
+    /* This function is called by diagnostic_show_locus in between
+       disjoint spans of source code, so that the context can print
+       something to indicate that a new span of source code has begun.  */
+    start_span_fn<to_text> m_text_start_span;
+    start_span_fn<to_html> m_html_start_span;
+
+    /* This function is called after the diagnostic message is printed.  */
+    text_finalizer_fn m_end_diagnostic;
+  } m_text_callbacks;
+
+  /* Client hook to report an internal error.  */
+  void (*m_internal_error) (context *, const char *, va_list *);
+
+  /* Client hook to adjust properties of the given diagnostic that we're
+     about to issue, such as its kind.  */
+  void (*m_adjust_diagnostic_info)(context *, diagnostic_info *);
+
+  /* Owned by the context; this would be a std::unique_ptr if
+     context had a proper ctor.  */
+  option_manager *m_option_mgr;
+  unsigned m_lang_mask;
+
+  /* A stack of optional hooks for adding URLs to quoted text strings in
+     diagnostics.  Only used for the main diagnostic message.
+     Typically a single one owner by the context, but can be temporarily
+     overridden by a borrowed urlifier (e.g. on-stack).  */
+  struct urlifier_stack_node
+  {
+    urlifier *m_urlifier;
+    bool m_owned;
+  };
+  auto_vec<urlifier_stack_node> *m_urlifier_stack;
+
+public:
+  /* Auxiliary data for client.  */
+  void *m_client_aux_data;
+
+  /* Used to detect that the last caret was printed at the same location.  */
+  location_t m_last_location;
+
+private:
+  int m_lock;
+
+  bool m_inhibit_notes_p;
+
+  source_printing_options m_source_printing;
+
+  /* True if -freport-bug option is used.  */
+  bool m_report_bug;
+
+  /* Used to specify additional diagnostic output to be emitted after the
+     rest of the diagnostic.  This is for implementing
+     -fdiagnostics-parseable-fixits and GCC_EXTRA_DIAGNOSTIC_OUTPUT.  */
+  enum diagnostics_extra_output_kind m_extra_output_kind;
+
+public:
+  /* What units to use when outputting the column number.  */
+  enum diagnostics_column_unit m_column_unit;
+
+  /* The origin for the column number (1-based or 0-based typically).  */
+  int m_column_origin;
+
+  /* The size of the tabstop for tab expansion.  */
+  int m_tabstop;
+
+private:
+  /* How should non-ASCII/non-printable bytes be escaped when
+     a diagnostic suggests escaping the source code on output.  */
+  enum diagnostics_escape_format m_escape_format;
+
+  /* If non-NULL, a diagnostics::changes::change_set to which fix-it hints
+     should be applied, for generating patches.
+     Owned by the context; this would be a std::unique_ptr if
+     context had a proper ctor.  */
+  changes::change_set *m_fixits_change_set;
+
+  /* Fields relating to diagnostic groups.  */
+  struct {
+    /* How many diagnostic_group instances are currently alive.  */
+    int m_group_nesting_depth;
+
+    /* How many nesting levels have been pushed within this group.  */
+    int m_diagnostic_nesting_level;
+
+    /* How many diagnostics have been emitted since the bottommost
+       diagnostic_group was pushed.  */
+    int m_emission_count;
+
+    /* The "group+diagnostic" nesting depth from which to inhibit notes.  */
+    int m_inhibiting_notes_from;
+  } m_diagnostic_groups;
+
+  void inhibit_notes_in_group (bool inhibit = true);
+  bool notes_inhibited_in_group () const;
+
+  /* The various sinks to which diagnostics are to be outputted
+     (text vs structured formats such as SARIF).
+     The sinks are owned by the context; this would be a
+     std::vector<std::unique_ptr> if context had a
+     proper ctor.  */
+  auto_vec<sink *> m_sinks;
+
+  /* Callback to set the locations of call sites along the inlining
+     stack corresponding to a diagnostic location.  Needed to traverse
+     the BLOCK_SUPERCONTEXT() chain hanging off the LOCATION_BLOCK()
+     of a diagnostic's location.  */
+  set_locations_callback_t m_set_locations_cb;
+
+  /* A bundle of hooks for providing data to the context about its client
+     e.g. version information, plugins, etc.
+     Used by SARIF output to give metadata about the client that's
+     producing diagnostics.
+     Owned by the context; this would be a std::unique_ptr if
+     context had a proper ctor.  */
+  client_data_hooks *m_client_data_hooks;
+
+  /* Support for diagrams.  */
+  struct
+  {
+    /* Theme to use when generating diagrams.
+       Can be NULL (if text art is disabled).
+       Owned by the context; this would be a std::unique_ptr if
+       context had a proper ctor.  */
+    text_art::theme *m_theme;
+
+  } m_diagrams;
+
+  /* Owned by the context.  */
+  char **m_original_argv;
+
+  /* Borrowed pointer to the active diagnostics::buffer, if any.
+     If null (the default), then diagnostics that are reported to the
+     context are immediately issued to the output format.
+     If non-null, then diagnostics that are reported to the context
+     are buffered in the buffer, and may be issued to the output format
+     later (if the buffer is flushed), moved to other buffers, or
+     discarded (if the buffer is cleared).  */
+  buffer *m_diagnostic_buffer;
+};
+
+/* Client supplied function to announce a diagnostic
+   (for text-based diagnostic output).  */
+inline text_starter_fn &
+text_starter (context *dc)
+{
+  return dc->m_text_callbacks.m_begin_diagnostic;
+}
+
+/* Client supplied function called between disjoint spans of source code,
+   so that the context can print
+   something to indicate that a new span of source code has begun.  */
+inline start_span_fn<to_text> &
+start_span (context *dc)
+{
+  return dc->m_text_callbacks.m_text_start_span;
+}
+
+/* Client supplied function called after a diagnostic message is
+   displayed (for text-based diagnostic output).  */
+inline text_finalizer_fn &
+text_finalizer (context *dc)
+{
+  return dc->m_text_callbacks.m_end_diagnostic;
+}
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_CONTEXT_H */
diff --git a/gcc/diagnostics/diagnostic-info.h b/gcc/diagnostics/diagnostic-info.h
new file mode 100644
index 0000000..052fef5
--- /dev/null
+++ b/gcc/diagnostics/diagnostic-info.h
@@ -0,0 +1,75 @@
+/* Various declarations for language-independent diagnostics subroutines.
+   Copyright (C) 2000-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_DIAGNOSTIC_INFO_H
+#define GCC_DIAGNOSTICS_DIAGNOSTIC_INFO_H
+
+namespace diagnostics {
+
+class metadata;
+
+/* A diagnostic is described by the MESSAGE to send, the FILE and LINE of
+   its context and its KIND (ice, error, warning, note, ...)  See complete
+   list in diagnostics/kinds.def.  */
+
+struct diagnostic_info
+{
+  diagnostic_info ()
+  : m_message (),
+    m_richloc (),
+    m_metadata (),
+    m_x_data (),
+    m_kind (),
+    m_option_id (),
+    m_iinfo ()
+  { }
+
+  /* Text to be formatted.  */
+  text_info m_message;
+
+  /* The location at which the diagnostic is to be reported.  */
+  rich_location *m_richloc;
+
+  /* An optional bundle of metadata associated with the diagnostic
+     (or NULL).  */
+  const metadata *m_metadata;
+
+  /* Auxiliary data for client.  */
+  void *m_x_data;
+  /* The kind of diagnostic it is about.  */
+  kind m_kind;
+  /* Which OPT_* directly controls this diagnostic.  */
+  option_id m_option_id;
+
+  /* Inlining context containing locations for each call site along
+     the inlining stack.  */
+  struct inlining_info
+  {
+    /* Locations along the inlining stack.  */
+    auto_vec<location_t, 8> m_ilocs;
+    /* The abstract origin of the location.  */
+    void *m_ao;
+    /* Set if every M_ILOCS element is in a system header.  */
+    bool m_allsyslocs;
+  } m_iinfo;
+};
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_DIAGNOSTIC_INFO_H */
diff --git a/gcc/diagnostics/diagnostics-selftests.cc b/gcc/diagnostics/diagnostics-selftests.cc
new file mode 100644
index 0000000..94a212a
--- /dev/null
+++ b/gcc/diagnostics/diagnostics-selftests.cc
@@ -0,0 +1,59 @@
+/* Selftest support for diagnostics.
+   Copyright (C) 2019-2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "selftest.h"
+#include "diagnostics/diagnostics-selftests.h"
+
+#if CHECKING_P
+
+namespace diagnostics {
+
+namespace selftest {
+
+/* Run all diagnostics-specific selftests,
+   apart from context_cc_tests, which according to
+   https://gcc.gnu.org/pipermail/gcc/2021-November/237703.html
+   has some language-specific assumptions, and thus is run from
+   c_family_tests instead.  */
+
+void
+run_diagnostics_selftests ()
+{
+  color_cc_tests ();
+  file_cache_cc_tests ();
+  source_printing_cc_tests ();
+  html_sink_cc_tests ();
+  sarif_sink_cc_tests ();
+  digraphs_cc_tests ();
+  output_spec_cc_tests ();
+  state_graphs_cc_tests ();
+  lazy_paths_cc_tests ();
+  paths_output_cc_tests ();
+  changes_cc_tests ();
+}
+
+} /* end of namespace diagnostics::selftest.  */
+
+} // namespace diagnostics
+
+#endif /* #if CHECKING_P */
diff --git a/gcc/diagnostics/diagnostics-selftests.h b/gcc/diagnostics/diagnostics-selftests.h
new file mode 100644
index 0000000..994ebad
--- /dev/null
+++ b/gcc/diagnostics/diagnostics-selftests.h
@@ -0,0 +1,55 @@
+/* Selftests for diagnostics.
+   Copyright (C) 2019-2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_SELFTESTS_H
+#define GCC_DIAGNOSTICS_SELFTESTS_H
+
+#if CHECKING_P
+
+namespace diagnostics {
+
+namespace selftest {
+
+extern void run_diagnostics_selftests ();
+
+/* Declarations for specific families of tests (by source file within
+   "diagnostics/"), in alphabetical order.  */
+
+extern void changes_cc_tests ();
+extern void color_cc_tests ();
+extern void context_cc_tests ();
+extern void digraphs_cc_tests ();
+extern void file_cache_cc_tests ();
+extern void html_sink_cc_tests ();
+extern void lazy_paths_cc_tests ();
+extern void output_spec_cc_tests ();
+extern void paths_output_cc_tests ();
+extern void sarif_sink_cc_tests ();
+extern void selftest_logical_locations_cc_tests ();
+extern void source_printing_cc_tests ();
+extern void state_graphs_cc_tests ();
+
+} /* end of namespace diagnostics::selftest.  */
+
+} // namespace diagnostics
+
+#endif /* #if CHECKING_P */
+
+#endif /* GCC_DIAGNOSTICS_SELFTESTS_H */
diff --git a/gcc/diagnostic-diagram.h b/gcc/diagnostics/diagram.h
index b1ff9a9..2d33b10 100644
--- a/gcc/diagnostic-diagram.h
+++ b/gcc/diagnostics/diagram.h
@@ -18,22 +18,24 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_DIAGRAM_H
-#define GCC_DIAGNOSTIC_DIAGRAM_H
+#ifndef GCC_DIAGNOSTICS_DIAGRAM_H
+#define GCC_DIAGNOSTICS_DIAGRAM_H
 
 namespace text_art
 {
   class canvas;
 } // namespace text_art
 
+namespace diagnostics {
+
 /* A text art diagram, along with an "alternative text" string
    describing it.  */
 
-class diagnostic_diagram
+class diagram
 {
  public:
-  diagnostic_diagram (const text_art::canvas &canvas,
-		      const char *alt_text)
+  diagram (const text_art::canvas &canvas,
+	   const char *alt_text)
   : m_canvas (canvas),
     m_alt_text (alt_text)
   {
@@ -48,4 +50,6 @@ class diagnostic_diagram
   const char *const m_alt_text;
 };
 
-#endif /* ! GCC_DIAGNOSTIC_DIAGRAM_H */
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_DIAGRAM_H */
diff --git a/gcc/diagnostics/digraphs.cc b/gcc/diagnostics/digraphs.cc
new file mode 100644
index 0000000..4a2ea4f
--- /dev/null
+++ b/gcc/diagnostics/digraphs.cc
@@ -0,0 +1,464 @@
+/* Directed graphs associated with a diagnostic.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#define INCLUDE_ALGORITHM
+#define INCLUDE_MAP
+#define INCLUDE_SET
+#define INCLUDE_STRING
+#define INCLUDE_VECTOR
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+
+#include "graphviz.h"
+#include "diagnostics/digraphs.h"
+#include "diagnostics/sarif-sink.h"
+
+#include "selftest.h"
+
+using digraph = diagnostics::digraphs::digraph;
+using digraph_node = diagnostics::digraphs::node;
+using digraph_edge = diagnostics::digraphs::edge;
+
+namespace {
+
+class conversion_to_dot
+{
+public:
+  std::unique_ptr<dot::graph>
+  make_dot_graph_from_diagnostic_graph (const digraph &);
+
+  std::unique_ptr<dot::stmt>
+  make_dot_node_from_digraph_node (const digraph_node &);
+
+  std::unique_ptr<dot::edge_stmt>
+  make_dot_edge_from_digraph_edge (const digraph_edge &);
+
+  dot::id
+  get_dot_id_for_node (const digraph_node &);
+
+  bool
+  has_edges_p (const digraph_node &);
+
+private:
+  std::set<const digraph_node *> m_nodes_with_edges;
+  std::map<const digraph_node *, dot::stmt *> m_node_map;
+};
+
+} // anonymous namespace
+
+// class conversion_to_dot
+
+std::unique_ptr<dot::graph>
+conversion_to_dot::
+make_dot_graph_from_diagnostic_graph (const diagnostics::digraphs::digraph &input_graph)
+{
+  auto output_graph = std::make_unique<dot::graph> ();
+
+  if (const char *description = input_graph.get_description ())
+    output_graph->m_stmt_list.add_attr (dot::id ("label"),
+					dot::id (description));
+
+  const int num_nodes = input_graph.get_num_nodes ();
+  const int num_edges = input_graph.get_num_edges ();
+
+  /* Determine which nodes have in-edges and out-edges.  */
+  for (int i = 0; i < num_edges; ++i)
+    {
+      const digraph_edge &input_edge = input_graph.get_edge (i);
+      m_nodes_with_edges.insert (&input_edge.get_src_node ());
+      m_nodes_with_edges.insert (&input_edge.get_dst_node ());
+    }
+
+  for (int i = 0; i < num_nodes; ++i)
+    {
+      const digraph_node &input_node = input_graph.get_node (i);
+      auto dot_node_stmt = make_dot_node_from_digraph_node (input_node);
+      output_graph->m_stmt_list.add_stmt (std::move (dot_node_stmt));
+    }
+
+  for (int i = 0; i < num_edges; ++i)
+    {
+      const digraph_edge &input_edge = input_graph.get_edge (i);
+      auto dot_edge_stmt = make_dot_edge_from_digraph_edge (input_edge);
+      output_graph->m_stmt_list.add_stmt (std::move (dot_edge_stmt));
+    }
+
+  return output_graph;
+}
+
+std::unique_ptr<dot::stmt>
+conversion_to_dot::
+make_dot_node_from_digraph_node (const diagnostics::digraphs::node &input_node)
+{
+  dot::id dot_id (get_dot_id_for_node (input_node));
+
+  /* For now, we can only do either edges or children, not both
+     ...but see https://graphviz.org/docs/attrs/compound/  */
+
+  if (has_edges_p (input_node))
+    {
+      auto output_node
+	= std::make_unique<dot::node_stmt> (std::move (dot_id));
+      m_node_map[&input_node] = output_node.get ();
+      if (const char *label = input_node.get_label ())
+	output_node->set_label (dot::id (label));
+      return output_node;
+    }
+  else
+    {
+      auto output_node = std::make_unique<dot::subgraph> (std::move (dot_id));
+      m_node_map[&input_node] = output_node.get ();
+      if (const char *label = input_node.get_label ())
+	output_node->add_attr (dot::id ("label"), dot::id (label));
+      const int num_children = input_node.get_num_children ();
+      for (int i = 0; i < num_children; ++i)
+	{
+	  const digraph_node &input_child = input_node.get_child (i);
+	  auto dot_child_stmt = make_dot_node_from_digraph_node (input_child);
+	  output_node->m_stmt_list.add_stmt (std::move (dot_child_stmt));
+	}
+      return output_node;
+    }
+}
+
+std::unique_ptr<dot::edge_stmt>
+conversion_to_dot::
+make_dot_edge_from_digraph_edge (const digraph_edge &input_edge)
+{
+  const digraph_node &src_dnode = input_edge.get_src_node ();
+  const digraph_node &dst_dnode = input_edge.get_dst_node ();
+  auto output_edge
+    = std::make_unique<dot::edge_stmt>
+    (get_dot_id_for_node (src_dnode),
+     get_dot_id_for_node (dst_dnode));
+  if (const char *label = input_edge.get_label ())
+    output_edge->set_label (dot::id (label));
+  return output_edge;
+}
+
+dot::id
+conversion_to_dot::get_dot_id_for_node (const digraph_node &input_node)
+{
+  if (has_edges_p (input_node))
+    return input_node.get_id ();
+  else
+    return std::string ("cluster_") + input_node.get_id ();
+}
+
+bool
+conversion_to_dot::has_edges_p (const digraph_node &input_node)
+{
+  return m_nodes_with_edges.find (&input_node) != m_nodes_with_edges.end ();
+}
+
+// class object
+
+const char *
+diagnostics::digraphs::object::
+get_attr (const char *key_prefix, const char *key) const
+{
+  if (!m_property_bag)
+    return nullptr;
+  std::string prefixed_key = std::string (key_prefix) + key;
+  if (json::value *jv = m_property_bag->get (prefixed_key.c_str ()))
+    if (json::string *jstr = jv->dyn_cast_string ())
+      return jstr->get_string ();
+  return nullptr;
+}
+
+void
+diagnostics::digraphs::object::
+set_attr (const char *key_prefix, const char *key, const char *value)
+{
+  set_json_attr (key_prefix, key, std::make_unique<json::string> (value));
+}
+
+void
+diagnostics::digraphs::object::
+set_json_attr (const char *key_prefix, const char *key, std::unique_ptr<json::value> value)
+{
+  std::string prefixed_key = std::string (key_prefix) + key;
+  if (!m_property_bag)
+    m_property_bag = std::make_unique<json::object> ();
+  m_property_bag->set (prefixed_key.c_str (), std::move (value));
+}
+
+// class digraph
+
+DEBUG_FUNCTION void
+diagnostics::digraphs::digraph::dump () const
+{
+  make_json_sarif_graph ()->dump ();
+}
+
+std::unique_ptr<json::object>
+diagnostics::digraphs::digraph::make_json_sarif_graph () const
+{
+  return make_sarif_graph (*this, nullptr, nullptr);
+}
+
+std::unique_ptr<dot::graph>
+diagnostics::digraphs::digraph::make_dot_graph () const
+{
+  conversion_to_dot to_dot;
+  return to_dot.make_dot_graph_from_diagnostic_graph (*this);
+}
+
+std::unique_ptr<diagnostics::digraphs::digraph>
+diagnostics::digraphs::digraph::clone () const
+{
+  auto result = std::make_unique<diagnostics::digraphs::digraph> ();
+
+  if (get_property_bag ())
+    result->set_property_bag (get_property_bag ()->clone_as_object ());
+
+  std::map<diagnostics::digraphs::node *, diagnostics::digraphs::node *> node_mapping;
+
+  for (auto &iter : m_nodes)
+    result->add_node (iter->clone (*result, node_mapping));
+  for (auto &iter : m_edges)
+    result->add_edge (iter->clone (*result, node_mapping));
+
+  return result;
+}
+
+void
+diagnostics::digraphs::digraph::add_edge (const char *id,
+					  node &src_node,
+					  node &dst_node,
+					  const char *label)
+{
+  auto e = std::make_unique<digraph_edge> (*this,
+					   id,
+					   src_node,
+					   dst_node);
+  if (label)
+    e->set_label (label);
+  add_edge (std::move (e));
+}
+
+/* Utility function for edge ids: either use EDGE_ID, or
+   generate a unique one for when we don't care about the name.
+
+   Edges in SARIF "SHALL" have an id that's unique within the graph
+   (SARIF 2.1.0 §3.41.2).  This is so that graph traversals can refer
+   to edges by id (SARIF 2.1.0's §3.43.2 edgeId property).  */
+
+std::string
+diagnostics::digraphs::digraph::make_edge_id (const char *edge_id)
+{
+  /* If we have an id, use it.  */
+  if (edge_id)
+    return edge_id;
+
+  /* Otherwise, generate a unique one of the form "edgeN".  */
+  while (true)
+    {
+      auto candidate (std::string ("edge")
+		      + std::to_string (m_next_edge_id_index++));
+      auto iter = m_id_to_edge_map.find (candidate);
+      if (iter != m_id_to_edge_map.end ())
+	{
+	  // Try again with the next index...
+	  continue;
+	}
+      return candidate;
+    }
+}
+
+// class node
+
+DEBUG_FUNCTION void
+diagnostics::digraphs::node::dump () const
+{
+  to_json_sarif_node ()->dump ();
+}
+
+std::unique_ptr<json::object>
+diagnostics::digraphs::node::to_json_sarif_node () const
+{
+  return make_sarif_node (*this, nullptr, nullptr);
+}
+
+std::unique_ptr<diagnostics::digraphs::node>
+diagnostics::digraphs::node::clone (digraph &new_graph,
+				    std::map<node *, node *> &node_mapping) const
+{
+  auto result
+    = std::make_unique<diagnostics::digraphs::node> (new_graph,
+						     get_id ());
+  node_mapping.insert ({const_cast <node *> (this), result.get ()});
+
+  result->set_logical_loc (m_logical_loc);
+
+  if (get_property_bag ())
+    result->set_property_bag (get_property_bag ()->clone_as_object ());
+
+  for (auto &iter : m_children)
+    result->add_child (iter->clone (new_graph, node_mapping));
+
+  return result;
+}
+
+// class edge
+
+std::unique_ptr<digraph_edge>
+digraph_edge::clone (digraph &new_graph,
+		     const std::map<node *, node *> &node_mapping) const
+{
+  auto iter_new_src = node_mapping.find (&m_src_node);
+  gcc_assert (iter_new_src != node_mapping.end ());
+  auto iter_new_dst = node_mapping.find (&m_dst_node);
+  gcc_assert (iter_new_dst != node_mapping.end ());
+  auto result
+    = std::make_unique<digraph_edge> (new_graph,
+				      m_id.c_str (),
+				      *iter_new_src->second,
+				      *iter_new_dst->second);
+  if (get_property_bag ())
+    result->set_property_bag (get_property_bag ()->clone_as_object ());
+
+  return result;
+}
+
+DEBUG_FUNCTION void
+diagnostics::digraphs::edge::dump () const
+{
+  to_json_sarif_edge ()->dump ();
+}
+
+std::unique_ptr<json::object>
+diagnostics::digraphs::edge::to_json_sarif_edge () const
+{
+  return make_sarif_edge (*this, nullptr);
+}
+
+#if CHECKING_P
+
+namespace diagnostics {
+namespace selftest {
+
+static void
+test_empty_graph ()
+{
+  digraph g;
+
+  {
+    auto sarif = g.make_json_sarif_graph ();
+
+    pretty_printer pp;
+    sarif->print (&pp, true);
+    ASSERT_STREQ
+      (pp_formatted_text (&pp),
+       ("{\"nodes\": [],\n"
+	" \"edges\": []}"));
+  }
+
+  {
+    auto dg = g.make_dot_graph ();
+
+    pretty_printer pp;
+    dot::writer w (pp);
+    dg->print (w);
+    ASSERT_STREQ
+      (pp_formatted_text (&pp),
+       ("digraph {\n"
+	"}\n"));
+  }
+}
+
+static void
+test_simple_graph ()
+{
+#define KEY_PREFIX "/placeholder/"
+  auto g = std::make_unique<digraph> ();
+  g->set_description ("test graph");
+  g->set_attr (KEY_PREFIX, "date", "1066");
+
+  auto a = std::make_unique<digraph_node> (*g, "a");
+  auto b = std::make_unique<digraph_node> (*g, "b");
+  b->set_attr (KEY_PREFIX, "color", "red");
+  auto c = std::make_unique<digraph_node> (*g, "c");
+  c->set_label ("I am a node label");
+
+  auto e = std::make_unique<digraph_edge> (*g, nullptr, *a, *c);
+  e->set_attr (KEY_PREFIX, "status", "copacetic");
+  e->set_label ("I am an edge label");
+  g->add_edge (std::move (e));
+
+  g->add_node (std::move (a));
+
+  b->add_child (std::move (c));
+  g->add_node (std::move (b));
+#undef KEY_PREFIX
+
+  {
+    auto sarif = g->make_json_sarif_graph ();
+
+    pretty_printer pp;
+    sarif->print (&pp, true);
+    ASSERT_STREQ
+      (pp_formatted_text (&pp),
+       ("{\"properties\": {\"/placeholder/date\": \"1066\"},\n"
+	" \"nodes\": [{\"id\": \"a\"},\n"
+	"           {\"id\": \"b\",\n"
+	"            \"properties\": {\"/placeholder/color\": \"red\"},\n"
+	"            \"children\": [{\"id\": \"c\"}]}],\n"
+	" \"edges\": [{\"id\": \"edge0\",\n"
+	"            \"properties\": {\"/placeholder/status\": \"copacetic\"},\n"
+	"            \"sourceNodeId\": \"a\",\n"
+	"            \"targetNodeId\": \"c\"}]}"));
+  }
+
+  {
+    auto dg = g->make_dot_graph ();
+
+    pretty_printer pp;
+    dot::writer w (pp);
+    dg->print (w);
+    ASSERT_STREQ
+      (pp_formatted_text (&pp),
+       ("digraph {\n"
+	"    label=\"test graph\";\n"
+	"    a;\n"
+	"    \n"
+	"    subgraph cluster_b {\n"
+	"        c [label=\"I am a node label\"];\n"
+	"\n"
+	"    };\n"
+	"    a -> c [label=\"I am an edge label\"];\n"
+	"}\n"));
+  }
+}
+
+/* Run all of the selftests within this file.  */
+
+void
+digraphs_cc_tests ()
+{
+  test_empty_graph ();
+  test_simple_graph ();
+}
+
+} // namespace diagnostics::selftest
+} // namespace diagnostics
+
+#endif /* CHECKING_P */
diff --git a/gcc/diagnostics/digraphs.h b/gcc/diagnostics/digraphs.h
new file mode 100644
index 0000000..7193ee4
--- /dev/null
+++ b/gcc/diagnostics/digraphs.h
@@ -0,0 +1,379 @@
+/* Directed graphs associated with a diagnostic.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_DIGRAPHS_H
+#define GCC_DIAGNOSTICS_DIGRAPHS_H
+
+#include "json.h"
+#include "diagnostics/logical-locations.h"
+
+class graphviz_out;
+
+class sarif_graph;
+class sarif_node;
+class sarif_edge;
+
+namespace dot { class graph; }
+
+namespace diagnostics {
+namespace digraphs {
+
+/* A family of classes: digraph, node, and edge, closely related to
+   SARIF's graph, node, and edge types (SARIF v2.1.0 sections 3.39-3.41).
+
+   Nodes can have child nodes, allowing for arbitrarily deep nesting.
+   Edges can be between any pair of nodes (potentially at different
+   nesting levels).
+
+   Digraphs, nodes, and edges also optionally have a JSON property bag,
+   allowing round-tripping of arbitrary key/value pairs through SARIF.  */
+
+class digraph;
+class node;
+class edge;
+
+/* A base class for digraph, node, and edge to allow them to have
+   an optional JSON property bag.  */
+
+class object
+{
+public:
+  const char *
+  get_attr (const char *key_prefix,
+	    const char *key) const;
+
+  void
+  set_attr (const char *key_prefix,
+	    const char *key,
+	    const char *value);
+
+  void
+  set_json_attr (const char *key_prefix,
+		 const char *key,
+		 std::unique_ptr<json::value> value);
+
+  json::object *
+  get_property_bag () const { return m_property_bag.get (); }
+
+  void
+  set_property_bag (std::unique_ptr<json::object> property_bag)
+  {
+    m_property_bag = std::move (property_bag);
+  }
+
+private:
+  std::unique_ptr<json::object> m_property_bag;
+};
+
+// A directed graph, corresponding to SARIF v2.1.0 section 3.39.
+
+class digraph : public object
+{
+ public:
+  friend class node;
+  friend class edge;
+
+  digraph () : m_next_edge_id_index (0) {}
+  virtual ~digraph () {}
+
+  const char *
+  get_description () const
+  {
+    if (!m_description)
+      return nullptr;
+    return m_description->c_str ();
+  }
+
+  void
+  set_description (const char *desc)
+  {
+    if (desc)
+      m_description = std::make_unique<std::string> (desc);
+    else
+      m_description = nullptr;
+  }
+  void
+  set_description (std::string desc)
+  {
+    m_description = std::make_unique<std::string> (std::move (desc));
+  }
+
+  node *
+  get_node_by_id (const char *id) const
+  {
+    auto iter = m_id_to_node_map.find (id);
+    if (iter == m_id_to_node_map.end ())
+      return nullptr;
+    return iter->second;
+  }
+
+  edge *
+  get_edge_by_id (const char *id) const
+  {
+    auto iter = m_id_to_edge_map.find (id);
+    if (iter == m_id_to_edge_map.end ())
+      return nullptr;
+    return iter->second;
+  }
+
+  size_t
+  get_num_nodes () const
+  {
+    return m_nodes.size ();
+  }
+
+  node &
+  get_node (size_t idx) const
+  {
+    return *m_nodes[idx].get ();
+  }
+
+  size_t
+  get_num_edges () const
+  {
+    return m_edges.size ();
+  }
+
+  edge &
+  get_edge (size_t idx) const
+  {
+    return *m_edges[idx].get ();
+  }
+
+  void
+  dump () const;
+
+  std::unique_ptr<json::object>
+  make_json_sarif_graph () const;
+
+  std::unique_ptr<dot::graph>
+  make_dot_graph () const;
+
+  void
+  add_node (std::unique_ptr<node> n)
+  {
+    gcc_assert (n);
+    m_nodes.push_back (std::move (n));
+  }
+
+  void
+  add_edge (std::unique_ptr<edge> e)
+  {
+    gcc_assert (e);
+    m_edges.push_back (std::move (e));
+  }
+
+  void
+  add_edge (const char *id,
+	    node &src_node,
+	    node &dst_node,
+	    const char *label = nullptr);
+
+  std::unique_ptr<digraph> clone () const;
+
+ private:
+  void
+  add_node_id (std::string node_id, node &new_node)
+  {
+    m_id_to_node_map.insert ({std::move (node_id), &new_node});
+  }
+  void
+  add_edge_id (std::string edge_id, edge &new_edge)
+  {
+    m_id_to_edge_map.insert ({std::move (edge_id), &new_edge});
+  }
+
+  std::string
+  make_edge_id (const char *edge_id);
+
+  std::unique_ptr<std::string> m_description;
+  std::map<std::string, node *> m_id_to_node_map;
+  std::map<std::string, edge *> m_id_to_edge_map;
+  std::vector<std::unique_ptr<node>> m_nodes;
+  std::vector<std::unique_ptr<edge>> m_edges;
+  size_t m_next_edge_id_index;
+};
+
+// A node in a directed graph, corresponding to SARIF v2.1.0 section 3.40.
+
+class node : public object
+{
+ public:
+  virtual ~node () {}
+
+  node (digraph &g, std::string id)
+  : m_id (id),
+    m_physical_loc (UNKNOWN_LOCATION)
+  {
+    g.add_node_id (std::move (id), *this);
+  }
+  node (const node &) = delete;
+
+  std::string
+  get_id () const { return m_id; }
+
+  const char *
+  get_label () const
+  {
+    if (!m_label)
+      return nullptr;
+    return m_label->c_str ();
+  }
+
+  void
+  set_label (const char *label)
+  {
+    if (label)
+      m_label = std::make_unique<std::string> (label);
+    else
+      m_label = nullptr;
+  }
+  void
+  set_label (std::string label)
+  {
+    m_label = std::make_unique<std::string> (std::move (label));
+  }
+
+  size_t
+  get_num_children () const { return m_children.size (); }
+
+  node &
+  get_child (size_t idx) const { return *m_children[idx].get (); }
+
+  void
+  add_child (std::unique_ptr<node> child)
+  {
+    gcc_assert (child);
+    m_children.push_back (std::move (child));
+  }
+
+  location_t
+  get_physical_loc () const
+  {
+    return m_physical_loc;
+  }
+
+  void
+  set_physical_loc (location_t physical_loc)
+  {
+    m_physical_loc = physical_loc;
+  }
+
+  logical_locations::key
+  get_logical_loc () const
+  {
+    return m_logical_loc;
+  }
+
+  void
+  set_logical_loc (logical_locations::key logical_loc)
+  {
+    m_logical_loc = logical_loc;
+  }
+
+  void print (graphviz_out &gv) const;
+
+  void
+  dump () const;
+
+  std::unique_ptr<json::object>
+  to_json_sarif_node () const;
+
+  std::unique_ptr<node>
+  clone (digraph &new_graph,
+	 std::map<node *, node *> &node_mapping) const;
+
+ private:
+  std::string m_id;
+  std::unique_ptr<std::string> m_label;
+  std::vector<std::unique_ptr<node>> m_children;
+  location_t m_physical_loc;
+  logical_locations::key m_logical_loc;
+};
+
+// An edge in a directed graph, corresponding to SARIF v2.1.0 section 3.41.
+
+class edge : public object
+{
+ public:
+  virtual ~edge () {}
+
+  /* SARIF requires us to provide unique edge IDs within a graph,
+     but otherwise we don't need them.
+     Pass in nullptr for the id to get the graph to generate a unique
+     edge id for us.  */
+  edge (digraph &g,
+	const char *id,
+	node &src_node,
+	node &dst_node)
+  : m_id (g.make_edge_id (id)),
+    m_src_node (src_node),
+    m_dst_node (dst_node)
+  {
+    g.add_edge_id (m_id, *this);
+  }
+
+  std::string
+  get_id () const { return m_id; }
+
+  const char *
+  get_label () const
+  {
+    if (!m_label)
+      return nullptr;
+    return m_label->c_str ();
+  }
+
+  void
+  set_label (const char *label)
+  {
+    if (label)
+      m_label = std::make_unique<std::string> (label);
+    else
+      m_label = nullptr;
+  }
+
+  node &
+  get_src_node () const { return m_src_node; }
+
+  node &
+  get_dst_node () const { return m_dst_node; }
+
+  void
+  dump () const;
+
+  std::unique_ptr<json::object>
+  to_json_sarif_edge () const;
+
+  std::unique_ptr<edge>
+  clone (digraph &new_graph,
+	 const std::map<diagnostics::digraphs::node *, diagnostics::digraphs::node *> &node_mapping) const;
+
+private:
+  std::string m_id;
+  std::unique_ptr<std::string> m_label;
+  node &m_src_node;
+  node &m_dst_node;
+};
+
+} // namespace digraphs
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_DIGRAPHS_H */
diff --git a/gcc/diagnostic-event-id.h b/gcc/diagnostics/event-id.h
index 7a45a72..167599d 100644
--- a/gcc/diagnostic-event-id.h
+++ b/gcc/diagnostics/event-id.h
@@ -1,4 +1,4 @@
-/* A class for referring to events within a diagnostic_path.
+/* A class for referring to events within a diagnostics::paths::path.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>
 
@@ -18,10 +18,10 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_EVENT_ID_H
-#define GCC_DIAGNOSTIC_EVENT_ID_H
+#ifndef GCC_DIAGNOSTICS_EVENT_ID_H
+#define GCC_DIAGNOSTICS_EVENT_ID_H
 
-/* A class for referring to events within a diagnostic_path.
+/* A class for referring to events within a diagnostics::paths::path.
 
    They are stored as 0-based offsets into the events, but
    printed (e.g. via %@) as 1-based numbers.
@@ -30,8 +30,10 @@ along with GCC; see the file COPYING3.  If not see
    which would be shown to the user as "(1)", "(2)" and "(3)".
 
    This has its own header so that pretty-print.cc can use this
-   to implement "%@" without bringing in all of diagnostic_path
-   (which e.g. refers to "tree").  */
+   to implement "%@" without bringing in all of diagnostics::paths.
+
+   This has to be in the global namespace for compatibility with
+   c-format.cc in GCC 10 onwards.  */
 
 class diagnostic_event_id_t
 {
@@ -58,15 +60,24 @@ class diagnostic_event_id_t
   int m_index; // zero-based
 };
 
+namespace diagnostics {
+namespace paths {
+
+typedef diagnostic_event_id_t event_id_t;
+
+/* A type for compactly referring to a particular thread within a
+   diagnostics::paths::path.  Typically there is just one thread per path,
+   with id 0.  */
+typedef int thread_id_t;
+
+} // namespace paths
+} // namespace diagnostics
+
+
 /* A pointer to a diagnostic_event_id_t, for use with the "%@" format
    code, which will print a 1-based representation for it, with suitable
    colorization, e.g. "(1)".
    The %@ format code requires that known_p be true for the event ID. */
 typedef diagnostic_event_id_t *diagnostic_event_id_ptr;
 
-/* A type for compactly referring to a particular thread within a
-   diagnostic_path.  Typically there is just one thread per path,
-   with id 0.  */
-typedef int diagnostic_thread_id_t;
-
-#endif /* ! GCC_DIAGNOSTIC_EVENT_ID_H */
+#endif /* ! GCC_DIAGNOSTICS_EVENT_ID_H */
diff --git a/gcc/diagnostics/file-cache.cc b/gcc/diagnostics/file-cache.cc
new file mode 100644
index 0000000..febeb03
--- /dev/null
+++ b/gcc/diagnostics/file-cache.cc
@@ -0,0 +1,1083 @@
+/* Caching input files for use by diagnostics.
+   Copyright (C) 2004-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "cpplib.h"
+#include "diagnostics/file-cache.h"
+#include "selftest.h"
+
+#ifndef HAVE_ICONV
+#define HAVE_ICONV 0
+#endif
+
+namespace diagnostics {
+
+/* Input charset configuration.  */
+static const char *default_charset_callback (const char *)
+{
+  return nullptr;
+}
+
+void
+file_cache::initialize_input_context (diagnostic_input_charset_callback ccb,
+				      bool should_skip_bom)
+{
+  m_input_context.ccb = (ccb ? ccb : default_charset_callback);
+  m_input_context.should_skip_bom = should_skip_bom;
+}
+
+/* This is a cache used by get_next_line to store the content of a
+   file to be searched for file lines.  */
+class file_cache_slot
+{
+public:
+  file_cache_slot ();
+  ~file_cache_slot ();
+
+  void dump (FILE *out, int indent) const;
+  void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
+
+  bool read_line_num (size_t line_num,
+		      char ** line, ssize_t *line_len);
+
+  /* Accessors.  */
+  const char *get_file_path () const { return m_file_path; }
+  unsigned get_use_count () const { return m_use_count; }
+  bool missing_trailing_newline_p () const
+  {
+    return m_missing_trailing_newline;
+  }
+  char_span get_full_file_content ();
+
+  void inc_use_count () { m_use_count++; }
+
+  bool create (const file_cache::input_context &in_context,
+	       const char *file_path, FILE *fp, unsigned highest_use_count);
+  void evict ();
+  void set_content (const char *buf, size_t sz);
+
+  static size_t tune (size_t line_record_size_)
+  {
+    size_t ret = line_record_size;
+    line_record_size = line_record_size_;
+    return ret;
+  }
+
+ private:
+  /* These are information used to store a line boundary.  */
+  class line_info
+  {
+  public:
+    /* The line number.  It starts from 1.  */
+    size_t line_num;
+
+    /* The position (byte count) of the beginning of the line,
+       relative to the file data pointer.  This starts at zero.  */
+    size_t start_pos;
+
+    /* The position (byte count) of the last byte of the line.  This
+       normally points to the '\n' character, or to one byte after the
+       last byte of the file, if the file doesn't contain a '\n'
+       character.  */
+    size_t end_pos;
+
+    line_info (size_t l, size_t s, size_t e)
+      : line_num (l), start_pos (s), end_pos (e)
+    {}
+
+    line_info ()
+      :line_num (0), start_pos (0), end_pos (0)
+    {}
+
+    static bool less_than(const line_info &a, const line_info &b)
+    {
+      return a.line_num < b.line_num;
+    }
+  };
+
+  bool needs_read_p () const;
+  bool needs_grow_p () const;
+  void maybe_grow ();
+  bool read_data ();
+  bool maybe_read_data ();
+  bool get_next_line (char **line, ssize_t *line_len);
+  bool read_next_line (char ** line, ssize_t *line_len);
+  bool goto_next_line ();
+
+  static const size_t buffer_size = 4 * 1024;
+  static size_t line_record_size;
+  static size_t recent_cached_lines_shift;
+
+  /* The number of time this file has been accessed.  This is used
+     to designate which file cache to evict from the cache
+     array.  */
+  unsigned m_use_count;
+
+  /* The file_path is the key for identifying a particular file in
+     the cache.  This copy is owned by the slot.  */
+  char *m_file_path;
+
+  FILE *m_fp;
+
+  /* True when an read error happened.  */
+  bool m_error;
+
+  /* This points to the content of the file that we've read so
+     far.  */
+  char *m_data;
+
+  /* The allocated buffer to be freed may start a little earlier than DATA,
+     e.g. if a UTF8 BOM was skipped at the beginning.  */
+  int m_alloc_offset;
+
+  /*  The size of the DATA array above.*/
+  size_t m_size;
+
+  /* The number of bytes read from the underlying file so far.  This
+     must be less (or equal) than SIZE above.  */
+  size_t m_nb_read;
+
+  /* The index of the beginning of the current line.  */
+  size_t m_line_start_idx;
+
+  /* The number of the previous line read.  This starts at 1.  Zero
+     means we've read no line so far.  */
+  size_t m_line_num;
+
+  /* Could this file be missing a trailing newline on its final line?
+     Initially true (to cope with empty files), set to true/false
+     as each line is read.  */
+  bool m_missing_trailing_newline;
+
+  /* This is a record of the beginning and end of the lines we've seen
+     while reading the file.  This is useful to avoid walking the data
+     from the beginning when we are asked to read a line that is
+     before LINE_START_IDX above.  When the lines exceed line_record_size
+     this is scaled down dynamically, with the line_info becoming anchors.  */
+  vec<line_info, va_heap> m_line_record;
+
+  /* A cache of the recently seen lines. This is maintained as a ring
+     buffer. */
+  vec<line_info, va_heap> m_line_recent;
+
+  /* First and last valid entry in m_line_recent.  */
+  size_t m_line_recent_last, m_line_recent_first;
+
+  void offset_buffer (int offset)
+  {
+    gcc_assert (offset < 0 ? m_alloc_offset + offset >= 0
+		: (size_t) offset <= m_size);
+    gcc_assert (m_data);
+    m_alloc_offset += offset;
+    m_data += offset;
+    m_size -= offset;
+  }
+
+};
+
+size_t file_cache_slot::line_record_size = 0;
+size_t file_cache_slot::recent_cached_lines_shift = 8;
+
+/* Tune file_cache.  */
+void
+file_cache::tune (size_t num_file_slots, size_t lines)
+{
+  if (file_cache_slot::tune (lines) != lines
+      || m_num_file_slots != num_file_slots)
+    {
+      delete[] m_file_slots;
+      m_file_slots = new file_cache_slot[num_file_slots];
+    }
+  m_num_file_slots = num_file_slots;
+}
+
+static const char *
+find_end_of_line (const char *s, size_t len);
+
+/* Lookup the cache used for the content of a given file accessed by
+   caret diagnostic.  Return the found cached file, or NULL if no
+   cached file was found.  */
+
+file_cache_slot *
+file_cache::lookup_file (const char *file_path)
+{
+  gcc_assert (file_path);
+
+  /* This will contain the found cached file.  */
+  file_cache_slot *r = NULL;
+  for (unsigned i = 0; i < m_num_file_slots; ++i)
+    {
+      file_cache_slot *c = &m_file_slots[i];
+      if (c->get_file_path () && !strcmp (c->get_file_path (), file_path))
+	{
+	  c->inc_use_count ();
+	  r = c;
+	}
+    }
+
+  if (r)
+    r->inc_use_count ();
+
+  return r;
+}
+
+/* Purge any mention of FILENAME from the cache of files used for
+   printing source code.  For use in selftests when working
+   with tempfiles.  */
+
+void
+file_cache::forcibly_evict_file (const char *file_path)
+{
+  gcc_assert (file_path);
+
+  file_cache_slot *r = lookup_file (file_path);
+  if (!r)
+    /* Not found.  */
+    return;
+
+  r->evict ();
+}
+
+/* Determine if FILE_PATH missing a trailing newline on its final line.
+   Only valid to call once all of the file has been loaded, by
+   requesting a line number beyond the end of the file.  */
+
+bool
+file_cache::missing_trailing_newline_p (const char *file_path)
+{
+  gcc_assert (file_path);
+
+  file_cache_slot *r = lookup_or_add_file (file_path);
+  return r->missing_trailing_newline_p ();
+}
+
+void
+file_cache::add_buffered_content (const char *file_path,
+				  const char *buffer,
+				  size_t sz)
+{
+  gcc_assert (file_path);
+
+  file_cache_slot *r = lookup_file (file_path);
+  if (!r)
+    {
+      unsigned highest_use_count = 0;
+      r = evicted_cache_tab_entry (&highest_use_count);
+      if (!r->create (m_input_context, file_path, nullptr, highest_use_count))
+	return;
+    }
+
+  r->set_content (buffer, sz);
+}
+
+void
+file_cache_slot::evict ()
+{
+  free (m_file_path);
+  m_file_path = NULL;
+  if (m_fp)
+    fclose (m_fp);
+  m_error = false;
+  m_fp = NULL;
+  m_nb_read = 0;
+  m_line_start_idx = 0;
+  m_line_num = 0;
+  m_line_record.truncate (0);
+  m_line_recent_first = 0;
+  m_line_recent_last = 0;
+  m_use_count = 0;
+  m_missing_trailing_newline = true;
+}
+
+/* Return the file cache that has been less used, recently, or the
+   first empty one.  If HIGHEST_USE_COUNT is non-null,
+   *HIGHEST_USE_COUNT is set to the highest use count of the entries
+   in the cache table.  */
+
+file_cache_slot*
+file_cache::evicted_cache_tab_entry (unsigned *highest_use_count)
+{
+  file_cache_slot *to_evict = &m_file_slots[0];
+  unsigned huc = to_evict->get_use_count ();
+  for (unsigned i = 1; i < m_num_file_slots; ++i)
+    {
+      file_cache_slot *c = &m_file_slots[i];
+      bool c_is_empty = (c->get_file_path () == NULL);
+
+      if (c->get_use_count () < to_evict->get_use_count ()
+	  || (to_evict->get_file_path () && c_is_empty))
+	/* We evict C because it's either an entry with a lower use
+	   count or one that is empty.  */
+	to_evict = c;
+
+      if (huc < c->get_use_count ())
+	huc = c->get_use_count ();
+
+      if (c_is_empty)
+	/* We've reached the end of the cache; subsequent elements are
+	   all empty.  */
+	break;
+    }
+
+  if (highest_use_count)
+    *highest_use_count = huc;
+
+  return to_evict;
+}
+
+/* Create the cache used for the content of a given file to be
+   accessed by caret diagnostic.  This cache is added to an array of
+   cache and can be retrieved by lookup_file_in_cache_tab.  This
+   function returns the created cache.  Note that only the last
+   m_num_file_slots files are cached.
+
+   This can return nullptr if the FILE_PATH can't be opened for
+   reading, or if the content can't be converted to the input_charset.  */
+
+file_cache_slot*
+file_cache::add_file (const char *file_path)
+{
+
+  FILE *fp = fopen (file_path, "r");
+  if (fp == NULL)
+    return NULL;
+
+  unsigned highest_use_count = 0;
+  file_cache_slot *r = evicted_cache_tab_entry (&highest_use_count);
+  if (!r->create (m_input_context, file_path, fp, highest_use_count))
+    return NULL;
+  return r;
+}
+
+/* Get a borrowed char_span to the full content of this file
+   as decoded according to the input charset, encoded as UTF-8.  */
+
+char_span
+file_cache_slot::get_full_file_content ()
+{
+  char *line;
+  ssize_t line_len;
+  while (get_next_line (&line, &line_len))
+    {
+    }
+  return char_span (m_data, m_nb_read);
+}
+
+/* Populate this slot for use on FILE_PATH and FP, dropping any
+   existing cached content within it.  */
+
+bool
+file_cache_slot::create (const file_cache::input_context &in_context,
+			 const char *file_path, FILE *fp,
+			 unsigned highest_use_count)
+{
+  m_file_path = file_path ? xstrdup (file_path) : nullptr;
+  if (m_fp)
+    fclose (m_fp);
+  m_error = false;
+  m_fp = fp;
+  if (m_alloc_offset)
+    offset_buffer (-m_alloc_offset);
+  m_nb_read = 0;
+  m_line_start_idx = 0;
+  m_line_num = 0;
+  m_line_recent_first = 0;
+  m_line_recent_last = 0;
+  m_line_record.truncate (0);
+  /* Ensure that this cache entry doesn't get evicted next time
+     add_file_to_cache_tab is called.  */
+  m_use_count = ++highest_use_count;
+  m_missing_trailing_newline = true;
+
+
+  /* Check the input configuration to determine if we need to do any
+     transformations, such as charset conversion or BOM skipping.  */
+  if (const char *input_charset = in_context.ccb (file_path))
+    {
+      /* Need a full-blown conversion of the input charset.  */
+      fclose (m_fp);
+      m_fp = NULL;
+      const cpp_converted_source cs
+	= cpp_get_converted_source (file_path, input_charset);
+      if (!cs.data)
+	return false;
+      if (m_data)
+	XDELETEVEC (m_data);
+      m_data = cs.data;
+      m_nb_read = m_size = cs.len;
+      m_alloc_offset = cs.data - cs.to_free;
+    }
+  else if (in_context.should_skip_bom)
+    {
+      if (read_data ())
+	{
+	  const int offset = cpp_check_utf8_bom (m_data, m_nb_read);
+	  offset_buffer (offset);
+	  m_nb_read -= offset;
+	}
+    }
+
+  return true;
+}
+
+void
+file_cache_slot::set_content (const char *buf, size_t sz)
+{
+  m_data = (char *)xmalloc (sz);
+  memcpy (m_data, buf, sz);
+  m_nb_read = m_size = sz;
+  m_alloc_offset = 0;
+
+  if (m_fp)
+    {
+      fclose (m_fp);
+      m_fp = nullptr;
+    }
+}
+
+/* file_cache's ctor.  */
+
+file_cache::file_cache ()
+: m_num_file_slots (16), m_file_slots (new file_cache_slot[m_num_file_slots])
+{
+  initialize_input_context (nullptr, false);
+}
+
+/* file_cache's dtor.  */
+
+file_cache::~file_cache ()
+{
+  delete[] m_file_slots;
+}
+
+void
+file_cache::dump (FILE *out, int indent) const
+{
+  for (size_t i = 0; i < m_num_file_slots; ++i)
+    {
+      fprintf (out, "%*sslot[%i]:\n", indent, "", (int)i);
+      m_file_slots[i].dump (out, indent + 2);
+    }
+}
+
+void
+file_cache::dump () const
+{
+  dump (stderr, 0);
+}
+
+/* Lookup the cache used for the content of a given file accessed by
+   caret diagnostic.  If no cached file was found, create a new cache
+   for this file, add it to the array of cached file and return
+   it.
+
+   This can return nullptr on a cache miss if FILE_PATH can't be opened for
+   reading, or if the content can't be converted to the input_charset.  */
+
+file_cache_slot*
+file_cache::lookup_or_add_file (const char *file_path)
+{
+  file_cache_slot *r = lookup_file (file_path);
+  if (r == NULL)
+    r = add_file (file_path);
+  return r;
+}
+
+/* Default constructor for a cache of file used by caret
+   diagnostic.  */
+
+file_cache_slot::file_cache_slot ()
+: m_use_count (0), m_file_path (NULL), m_fp (NULL), m_error (false), m_data (0),
+  m_alloc_offset (0), m_size (0), m_nb_read (0), m_line_start_idx (0),
+  m_line_num (0), m_missing_trailing_newline (true),
+  m_line_recent_last (0), m_line_recent_first (0)
+{
+  m_line_record.create (0);
+  m_line_recent.create (1U << recent_cached_lines_shift);
+  for (int i = 0; i < 1 << recent_cached_lines_shift; i++)
+    m_line_recent.quick_push (file_cache_slot::line_info (0, 0, 0));
+}
+
+/* Destructor for a cache of file used by caret diagnostic.  */
+
+file_cache_slot::~file_cache_slot ()
+{
+  free (m_file_path);
+  if (m_fp)
+    {
+      fclose (m_fp);
+      m_fp = NULL;
+    }
+  if (m_data)
+    {
+      offset_buffer (-m_alloc_offset);
+      XDELETEVEC (m_data);
+      m_data = 0;
+    }
+  m_line_record.release ();
+  m_line_recent.release ();
+}
+
+void
+file_cache_slot::dump (FILE *out, int indent) const
+{
+  if (!m_file_path)
+    {
+      fprintf (out, "%*s(unused)\n", indent, "");
+      return;
+    }
+  fprintf (out, "%*sfile_path: %s\n", indent, "", m_file_path);
+  fprintf (out, "%*sfp: %p\n", indent, "", (void *)m_fp);
+  fprintf (out, "%*sneeds_read_p: %i\n", indent, "", (int)needs_read_p ());
+  fprintf (out, "%*sneeds_grow_p: %i\n", indent, "", (int)needs_grow_p ());
+  fprintf (out, "%*suse_count: %i\n", indent, "", m_use_count);
+  fprintf (out, "%*ssize: %zi\n", indent, "", m_size);
+  fprintf (out, "%*snb_read: %zi\n", indent, "", m_nb_read);
+  fprintf (out, "%*sstart_line_idx: %zi\n", indent, "", m_line_start_idx);
+  fprintf (out, "%*sline_num: %zi\n", indent, "", m_line_num);
+  fprintf (out, "%*smissing_trailing_newline: %i\n",
+	   indent, "", (int)m_missing_trailing_newline);
+  fprintf (out, "%*sline records (%i):\n",
+	   indent, "", m_line_record.length ());
+  int idx = 0;
+  for (auto &line : m_line_record)
+    fprintf (out, "%*s[%i]: line %zi: byte offsets: %zi-%zi\n",
+	     indent + 2, "",
+	     idx++, line.line_num, line.start_pos, line.end_pos);
+}
+
+/* Returns TRUE iff the cache would need to be filled with data coming
+   from the file.  That is, either the cache is empty or full or the
+   current line is empty.  Note that if the cache is full, it would
+   need to be extended and filled again.  */
+
+bool
+file_cache_slot::needs_read_p () const
+{
+  return m_fp && (m_nb_read == 0
+	  || m_nb_read == m_size
+	  || (m_line_start_idx >= m_nb_read - 1));
+}
+
+/*  Return TRUE iff the cache is full and thus needs to be
+    extended.  */
+
+bool
+file_cache_slot::needs_grow_p () const
+{
+  return m_nb_read == m_size;
+}
+
+/* Grow the cache if it needs to be extended.  */
+
+void
+file_cache_slot::maybe_grow ()
+{
+  if (!needs_grow_p ())
+    return;
+
+  if (!m_data)
+    {
+      gcc_assert (m_size == 0 && m_alloc_offset == 0);
+      m_size = buffer_size;
+      m_data = XNEWVEC (char, m_size);
+    }
+  else
+    {
+      const int offset = m_alloc_offset;
+      offset_buffer (-offset);
+      m_size *= 2;
+      m_data = XRESIZEVEC (char, m_data, m_size);
+      offset_buffer (offset);
+    }
+}
+
+/*  Read more data into the cache.  Extends the cache if need be.
+    Returns TRUE iff new data could be read.  */
+
+bool
+file_cache_slot::read_data ()
+{
+  if (feof (m_fp) || ferror (m_fp))
+    return false;
+
+  maybe_grow ();
+
+  char * from = m_data + m_nb_read;
+  size_t to_read = m_size - m_nb_read;
+  size_t nb_read = fread (from, 1, to_read, m_fp);
+
+  if (ferror (m_fp))
+    {
+      m_error = true;
+      return false;
+    }
+
+  m_nb_read += nb_read;
+  return !!nb_read;
+}
+
+/* Read new data iff the cache needs to be filled with more data
+   coming from the file FP.  Return TRUE iff the cache was filled with
+   mode data.  */
+
+bool
+file_cache_slot::maybe_read_data ()
+{
+  if (!needs_read_p ())
+    return false;
+  return read_data ();
+}
+
+/* Helper function for file_cache_slot::get_next_line (), to find the end of
+   the next line.  Returns with the memchr convention, i.e. nullptr if a line
+   terminator was not found.  We need to determine line endings in the same
+   manner that libcpp does: any of \n, \r\n, or \r is a line ending.  */
+
+static const char *
+find_end_of_line (const char *s, size_t len)
+{
+  for (const auto end = s + len; s != end; ++s)
+    {
+      if (*s == '\n')
+	return s;
+      if (*s == '\r')
+	{
+	  const auto next = s + 1;
+	  if (next == end)
+	    {
+	      /* Don't find the line ending if \r is the very last character
+		 in the buffer; we do not know if it's the end of the file or
+		 just the end of what has been read so far, and we wouldn't
+		 want to break in the middle of what's actually a \r\n
+		 sequence.  Instead, we will handle the case of a file ending
+		 in a \r later.  */
+	      break;
+	    }
+	  return (*next == '\n' ? next : s);
+	}
+    }
+  return nullptr;
+}
+
+/* Read a new line from file FP, using C as a cache for the data
+   coming from the file.  Upon successful completion, *LINE is set to
+   the beginning of the line found.  *LINE points directly in the
+   line cache and is only valid until the next call of get_next_line.
+   *LINE_LEN is set to the length of the line.  Note that the line
+   does not contain any terminal delimiter.  This function returns
+   true if some data was read or process from the cache, false
+   otherwise.  Note that subsequent calls to get_next_line might
+   make the content of *LINE invalid.  */
+
+bool
+file_cache_slot::get_next_line (char **line, ssize_t *line_len)
+{
+  /* Fill the cache with data to process.  */
+  maybe_read_data ();
+
+  size_t remaining_size = m_nb_read - m_line_start_idx;
+  if (remaining_size == 0)
+    /* There is no more data to process.  */
+    return false;
+
+  const char *line_start = m_data + m_line_start_idx;
+
+  const char *next_line_start = NULL;
+  size_t len = 0;
+  const char *line_end = find_end_of_line (line_start, remaining_size);
+  if (line_end == NULL)
+    {
+      /* We haven't found an end-of-line delimiter in the cache.
+	 Fill the cache with more data from the file and look again.  */
+      while (maybe_read_data ())
+	{
+	  line_start = m_data + m_line_start_idx;
+	  remaining_size = m_nb_read - m_line_start_idx;
+	  line_end = find_end_of_line (line_start, remaining_size);
+	  if (line_end != NULL)
+	    {
+	      next_line_start = line_end + 1;
+	      break;
+	    }
+	}
+      if (line_end == NULL)
+	{
+	  /* We've loaded all the file into the cache and still no
+	     terminator.  Let's say the line ends up at one byte past the
+	     end of the file.  This is to stay consistent with the case
+	     of when the line ends up with a terminator and line_end points to
+	     that.  That consistency is useful below in the len calculation.
+
+	     If the file ends in a \r, we didn't identify it as a line
+	     terminator above, so do that now instead.  */
+	  line_end = m_data + m_nb_read;
+	  if (m_nb_read && line_end[-1] == '\r')
+	    {
+	      --line_end;
+	      m_missing_trailing_newline = false;
+	    }
+	  else
+	    m_missing_trailing_newline = true;
+	}
+      else
+	m_missing_trailing_newline = false;
+    }
+  else
+    {
+      next_line_start = line_end + 1;
+      m_missing_trailing_newline = false;
+    }
+
+  if (m_error)
+    return false;
+
+  /* At this point, we've found the end of the of line.  It either points to
+     the line terminator or to one byte after the last byte of the file.  */
+  gcc_assert (line_end != NULL);
+
+  len = line_end - line_start;
+
+  if (m_line_start_idx < m_nb_read)
+    *line = const_cast<char *> (line_start);
+
+  ++m_line_num;
+
+  /* Now update our line record so that re-reading lines from the
+     before m_line_start_idx is faster.  */
+  size_t rlen = m_line_record.length ();
+  /* Only update when beyond the previously cached region.  */
+  if (rlen == 0 || m_line_record[rlen - 1].line_num < m_line_num)
+    {
+      size_t spacing
+	= (rlen >= 2
+	   ? (m_line_record[rlen - 1].line_num
+	      - m_line_record[rlen - 2].line_num) : 1);
+      size_t delta
+	= rlen >= 1 ? m_line_num - m_line_record[rlen - 1].line_num : 1;
+
+      size_t max_size = line_record_size;
+      /* One anchor per hundred input lines.  */
+      if (max_size == 0)
+	max_size = m_line_num / 100;
+
+      /* If we're too far beyond drop half of the lines to rebalance.  */
+      if (rlen == max_size && delta >= spacing * 2)
+	{
+	  size_t j = 0;
+	  for (size_t i = 1; i < rlen; i += 2)
+	    m_line_record[j++] = m_line_record[i];
+	  m_line_record.truncate (j);
+	  rlen = j;
+	  spacing *= 2;
+	}
+
+      if (rlen < max_size && delta >= spacing)
+	{
+	  file_cache_slot::line_info li (m_line_num, m_line_start_idx,
+					 line_end - m_data);
+	  m_line_record.safe_push (li);
+	}
+    }
+
+  /* Cache recent tail lines separately for fast access. This assumes
+     most accesses do not skip backwards.  */
+  if (m_line_recent_last == m_line_recent_first
+      || m_line_recent[m_line_recent_last].line_num == m_line_num - 1)
+    {
+      size_t mask = ((size_t) 1 << recent_cached_lines_shift) - 1;
+      m_line_recent_last = (m_line_recent_last + 1) & mask;
+      if (m_line_recent_last == m_line_recent_first)
+	m_line_recent_first = (m_line_recent_first + 1) & mask;
+      m_line_recent[m_line_recent_last]
+	= file_cache_slot::line_info (m_line_num, m_line_start_idx,
+				      line_end - m_data);
+    }
+
+  /* Update m_line_start_idx so that it points to the next line to be
+     read.  */
+  if (next_line_start)
+    m_line_start_idx = next_line_start - m_data;
+  else
+    /* We didn't find any terminal '\n'.  Let's consider that the end
+       of line is the end of the data in the cache.  The next
+       invocation of get_next_line will either read more data from the
+       underlying file or return false early because we've reached the
+       end of the file.  */
+    m_line_start_idx = m_nb_read;
+
+  *line_len = len;
+
+  return true;
+}
+
+/* Consume the next bytes coming from the cache (or from its
+   underlying file if there are remaining unread bytes in the file)
+   until we reach the next end-of-line (or end-of-file).  There is no
+   copying from the cache involved.  Return TRUE upon successful
+   completion.  */
+
+bool
+file_cache_slot::goto_next_line ()
+{
+  char *l;
+  ssize_t len;
+
+  return get_next_line (&l, &len);
+}
+
+/* Read an arbitrary line number LINE_NUM from the file cached in C.
+   If the line was read successfully, *LINE points to the beginning
+   of the line in the file cache and *LINE_LEN is the length of the
+   line.  *LINE is not nul-terminated, but may contain zero bytes.
+   *LINE is only valid until the next call of read_line_num.
+   This function returns bool if a line was read.  */
+
+bool
+file_cache_slot::read_line_num (size_t line_num,
+				char ** line, ssize_t *line_len)
+{
+  gcc_assert (line_num > 0);
+
+  /* Is the line in the recent line cache?
+     This assumes the main file processing is only using
+     a single contiguous cursor with only temporary excursions.  */
+  if (m_line_recent_first != m_line_recent_last
+	&& m_line_recent[m_line_recent_first].line_num <= line_num
+	&& m_line_recent[m_line_recent_last].line_num >= line_num)
+    {
+      line_info &last = m_line_recent[m_line_recent_last];
+      size_t mask = (1U << recent_cached_lines_shift) - 1;
+      size_t idx = (m_line_recent_last - (last.line_num - line_num)) & mask;
+      line_info &recent = m_line_recent[idx];
+      gcc_assert (recent.line_num == line_num);
+      *line = m_data + recent.start_pos;
+      *line_len = recent.end_pos - recent.start_pos;
+      return true;
+    }
+
+  if (line_num <= m_line_num)
+    {
+      line_info l (line_num, 0, 0);
+      int i = m_line_record.lower_bound (l, line_info::less_than);
+      if (i == 0)
+	{
+	  m_line_start_idx = 0;
+	  m_line_num = 0;
+	}
+      else if (m_line_record[i - 1].line_num == line_num)
+	{
+	  /* We have the start/end of the line.  */
+	  *line = m_data + m_line_record[i - 1].start_pos;
+	  *line_len = m_line_record[i - 1].end_pos - m_line_record[i - 1].start_pos;
+	  return true;
+	}
+      else
+       {
+	 gcc_assert (m_line_record[i - 1].line_num < m_line_num);
+	 m_line_start_idx = m_line_record[i - 1].start_pos;
+	 m_line_num = m_line_record[i - 1].line_num - 1;
+       }
+    }
+
+  /*  Let's walk from line m_line_num up to line_num - 1, without
+      copying any line.  */
+  while (m_line_num < line_num - 1)
+    if (!goto_next_line ())
+      return false;
+
+  /* The line we want is the next one.  Let's read it.  */
+  return get_next_line (line, line_len);
+}
+
+/* Return the physical source line that corresponds to FILE_PATH/LINE.
+   The line is not nul-terminated.  The returned pointer is only
+   valid until the next call of location_get_source_line.
+   Note that the line can contain several null characters,
+   so the returned value's length has the actual length of the line.
+   If the function fails, a NULL char_span is returned.  */
+
+char_span
+file_cache::get_source_line (const char *file_path, int line)
+{
+  char *buffer = NULL;
+  ssize_t len;
+
+  if (line == 0)
+    return char_span (NULL, 0);
+
+  if (file_path == NULL)
+    return char_span (NULL, 0);
+
+  file_cache_slot *c = lookup_or_add_file (file_path);
+  if (c == NULL)
+    return char_span (NULL, 0);
+
+  bool read = c->read_line_num (line, &buffer, &len);
+  if (!read)
+    return char_span (NULL, 0);
+
+  return char_span (buffer, len);
+}
+
+char_span
+file_cache::get_source_file_content (const char *file_path)
+{
+  file_cache_slot *c = lookup_or_add_file (file_path);
+  if (c == nullptr)
+    return char_span (nullptr, 0);
+  return c->get_full_file_content ();
+}
+
+#if CHECKING_P
+
+namespace selftest {
+
+ using temp_source_file = ::selftest::temp_source_file;
+
+/* Verify reading of a specific line LINENUM in TMP, FC.  */
+
+static void
+check_line (temp_source_file &tmp, file_cache &fc, int linenum)
+{
+  char_span line = fc.get_source_line (tmp.get_filename (), linenum);
+  int n;
+  const char *b = line.get_buffer ();
+  size_t l = line.length ();
+  char buf[5];
+  ASSERT_LT (l, 5);
+  memcpy (buf, b, l);
+  buf[l] = '\0';
+  ASSERT_TRUE (sscanf (buf, "%d", &n) == 1);
+  ASSERT_EQ (n, linenum);
+}
+
+/* Test file cache replacement.  */
+
+static void
+test_replacement ()
+{
+  const int maxline = 1000;
+
+  char *vec = XNEWVEC (char, maxline * 5);
+  char *p = vec;
+  int i;
+  for (i = 1; i <= maxline; i++)
+    p += sprintf (p, "%d\n", i);
+
+  temp_source_file tmp (SELFTEST_LOCATION, ".txt", vec);
+  free (vec);
+  file_cache fc;
+
+  for (i = 2; i <= maxline; i++)
+    {
+      check_line (tmp, fc, i);
+      check_line (tmp, fc, i - 1);
+      if (i >= 10)
+	check_line (tmp, fc, i - 9);
+      if (i >= 350) /* Exceed the look behind cache.  */
+	check_line (tmp, fc, i - 300);
+    }
+  for (i = 5; i <= maxline; i += 100)
+    check_line (tmp, fc, i);
+  for (i = 1; i <= maxline; i++)
+    check_line (tmp, fc, i);
+}
+
+/* Verify reading of input files (e.g. for caret-based diagnostics).  */
+
+static void
+test_reading_source_line ()
+{
+  /* Create a tempfile and write some text to it.  */
+  temp_source_file tmp (SELFTEST_LOCATION, ".txt",
+			"01234567890123456789\n"
+			"This is the test text\n"
+			"This is the 3rd line");
+  file_cache fc;
+
+  /* Read back a specific line from the tempfile.  */
+  char_span source_line = fc.get_source_line (tmp.get_filename (), 3);
+  ASSERT_TRUE (source_line);
+  ASSERT_TRUE (source_line.get_buffer () != NULL);
+  ASSERT_EQ (20, source_line.length ());
+  ASSERT_TRUE (!strncmp ("This is the 3rd line",
+			 source_line.get_buffer (), source_line.length ()));
+
+  source_line = fc.get_source_line (tmp.get_filename (), 2);
+  ASSERT_TRUE (source_line);
+  ASSERT_TRUE (source_line.get_buffer () != NULL);
+  ASSERT_EQ (21, source_line.length ());
+  ASSERT_TRUE (!strncmp ("This is the test text",
+			 source_line.get_buffer (), source_line.length ()));
+
+  source_line = fc.get_source_line (tmp.get_filename (), 4);
+  ASSERT_FALSE (source_line);
+  ASSERT_TRUE (source_line.get_buffer () == NULL);
+}
+
+/* Verify reading from buffers (e.g. for sarif-replay).  */
+
+static void
+test_reading_source_buffer ()
+{
+  const char *text = ("01234567890123456789\n"
+		      "This is the test text\n"
+		      "This is the 3rd line");
+  const char *filename = "foo.txt";
+  file_cache fc;
+  fc.add_buffered_content (filename, text, strlen (text));
+
+  /* Read back a specific line from the tempfile.  */
+  char_span source_line = fc.get_source_line (filename, 3);
+  ASSERT_TRUE (source_line);
+  ASSERT_TRUE (source_line.get_buffer () != NULL);
+  ASSERT_EQ (20, source_line.length ());
+  ASSERT_TRUE (!strncmp ("This is the 3rd line",
+			 source_line.get_buffer (), source_line.length ()));
+
+  source_line = fc.get_source_line (filename, 2);
+  ASSERT_TRUE (source_line);
+  ASSERT_TRUE (source_line.get_buffer () != NULL);
+  ASSERT_EQ (21, source_line.length ());
+  ASSERT_TRUE (!strncmp ("This is the test text",
+			 source_line.get_buffer (), source_line.length ()));
+
+  source_line = fc.get_source_line (filename, 4);
+  ASSERT_FALSE (source_line);
+  ASSERT_TRUE (source_line.get_buffer () == NULL);
+}
+
+/* Run all of the selftests within this file.  */
+
+void
+file_cache_cc_tests ()
+{
+  test_reading_source_line ();
+  test_reading_source_buffer ();
+  test_replacement ();
+}
+
+} // namespace selftest
+
+#endif /* CHECKING_P */
+
+} // namespace diagnostics
diff --git a/gcc/diagnostics/file-cache.h b/gcc/diagnostics/file-cache.h
new file mode 100644
index 0000000..832a960
--- /dev/null
+++ b/gcc/diagnostics/file-cache.h
@@ -0,0 +1,125 @@
+/* Caching input files for use by diagnostics.
+   Copyright (C) 2004-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_FILE_CACHE_H
+#define GCC_DIAGNOSTICS_FILE_CACHE_H
+
+namespace diagnostics {
+
+/* A class capturing the bounds of a buffer, to allow for run-time
+   bounds-checking in a checked build.  */
+
+class char_span
+{
+ public:
+  char_span (const char *ptr, size_t n_elts) : m_ptr (ptr), m_n_elts (n_elts) {}
+
+  /* Test for a non-NULL pointer.  */
+  operator bool() const { return m_ptr; }
+
+  /* Get length, not including any 0-terminator (which may not be,
+     in fact, present).  */
+  size_t length () const { return m_n_elts; }
+
+  const char *get_buffer () const { return m_ptr; }
+
+  char operator[] (int idx) const
+  {
+    gcc_assert (idx >= 0);
+    gcc_assert ((size_t)idx < m_n_elts);
+    return m_ptr[idx];
+  }
+
+  char_span subspan (int offset, int n_elts) const
+  {
+    gcc_assert (offset >= 0);
+    gcc_assert (offset < (int)m_n_elts);
+    gcc_assert (n_elts >= 0);
+    gcc_assert (offset + n_elts <= (int)m_n_elts);
+    return char_span (m_ptr + offset, n_elts);
+  }
+
+  char *xstrdup () const
+  {
+    return ::xstrndup (m_ptr, m_n_elts);
+  }
+
+ private:
+  const char *m_ptr;
+  size_t m_n_elts;
+};
+
+/* Forward decl of slot within file_cache, so that the definition doesn't
+   need to be in this header.  */
+class file_cache_slot;
+
+/* A cache of source files for use when emitting diagnostics
+   (and in a few places in the C/C++ frontends).
+
+   Results are only valid until the next call to the cache, as
+   slots can be evicted.
+
+   Filenames are stored by pointer, and so must outlive the cache
+   instance.  */
+
+class file_cache
+{
+ public:
+  file_cache ();
+  ~file_cache ();
+
+  void dump (FILE *out, int indent) const;
+  void DEBUG_FUNCTION dump () const;
+
+  file_cache_slot *lookup_or_add_file (const char *file_path);
+  void forcibly_evict_file (const char *file_path);
+
+  /* See comments in diagnostic.h about the input conversion context.  */
+  struct input_context
+  {
+    diagnostic_input_charset_callback ccb;
+    bool should_skip_bom;
+  };
+  void initialize_input_context (diagnostic_input_charset_callback ccb,
+				 bool should_skip_bom);
+
+  char_span get_source_file_content (const char *file_path);
+  char_span get_source_line (const char *file_path, int line);
+  bool missing_trailing_newline_p (const char *file_path);
+
+  void add_buffered_content (const char *file_path,
+			     const char *buffer,
+			     size_t sz);
+
+  void tune (size_t num_file_slots, size_t lines);
+
+ private:
+  file_cache_slot *evicted_cache_tab_entry (unsigned *highest_use_count);
+  file_cache_slot *add_file (const char *file_path);
+  file_cache_slot *lookup_file (const char *file_path);
+
+ private:
+  size_t m_num_file_slots;
+  file_cache_slot *m_file_slots;
+  input_context m_input_context;
+};
+
+} // namespace diagnostics
+
+#endif // #ifndef GCC_DIAGNOSTICS_FILE_CACHE_H
diff --git a/gcc/diagnostic-format-html.cc b/gcc/diagnostics/html-sink.cc
index 473880f..13d6309 100644
--- a/gcc/diagnostic-format-html.cc
+++ b/gcc/diagnostics/html-sink.cc
@@ -25,35 +25,39 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "diagnostic.h"
-#include "diagnostic-metadata.h"
-#include "diagnostic-format.h"
-#include "diagnostic-format-html.h"
-#include "diagnostic-format-text.h"
-#include "diagnostic-output-file.h"
-#include "diagnostic-buffer.h"
-#include "diagnostic-path.h"
-#include "diagnostic-client-data-hooks.h"
+#include "diagnostics/metadata.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/html-sink.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/sarif-sink.h"
+#include "diagnostics/output-file.h"
+#include "diagnostics/buffering.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/client-data-hooks.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
+#include "diagnostics/selftest-context.h"
 #include "pretty-print-format-impl.h"
 #include "pretty-print-urlifier.h"
-#include "edit-context.h"
+#include "diagnostics/changes.h"
 #include "intl.h"
 #include "xml.h"
 #include "xml-printer.h"
-#include "diagnostic-state.h"
+#include "diagnostics/digraphs.h"
+#include "diagnostics/state-graphs.h"
 #include "graphviz.h"
 #include "json.h"
 #include "selftest-xml.h"
 
+namespace diagnostics {
+
 // struct html_generation_options
 
 html_generation_options::html_generation_options ()
 : m_css (true),
   m_javascript (true),
   m_show_state_diagrams (false),
-  m_show_state_diagram_xml (false),
-  m_show_state_diagram_dot_src (false)
+  m_show_state_diagrams_sarif (false),
+  m_show_state_diagrams_dot_src (false)
 {
 }
 
@@ -61,19 +65,19 @@ class html_builder;
 
 /* Concrete buffering implementation subclass for HTML output.  */
 
-class diagnostic_html_format_buffer : public diagnostic_per_format_buffer
+class html_sink_buffer : public per_sink_buffer
 {
 public:
   friend class html_builder;
-  friend class html_output_format;
+  friend class html_sink;
 
-  diagnostic_html_format_buffer (html_builder &builder)
+  html_sink_buffer (html_builder &builder)
   : m_builder (builder)
   {}
 
   void dump (FILE *out, int indent) const final override;
   bool empty_p () const final override;
-  void move_to (diagnostic_per_format_buffer &dest) final override;
+  void move_to (per_sink_buffer &dest) final override;
   void clear () final override;
   void flush () final override;
 
@@ -105,9 +109,9 @@ private:
 class html_builder
 {
 public:
-  friend class diagnostic_html_format_buffer;
+  friend class html_sink_buffer;
 
-  html_builder (diagnostic_context &context,
+  html_builder (context &dc,
 		pretty_printer &pp,
 		const line_maps *line_maps,
 		const html_generation_options &html_gen_opts);
@@ -116,9 +120,11 @@ public:
   set_main_input_filename (const char *name);
 
   void on_report_diagnostic (const diagnostic_info &diagnostic,
-			     diagnostic_t orig_diag_kind,
-			     diagnostic_html_format_buffer *buffer);
-  void emit_diagram (const diagnostic_diagram &diagram);
+			     enum kind orig_diag_kind,
+			     html_sink_buffer *buffer);
+  void emit_diagram (const diagram &d);
+  void emit_global_graph (const lazily_created<digraphs::digraph> &);
+
   void end_group ();
 
   std::unique_ptr<xml::element> take_current_diagnostic ()
@@ -136,7 +142,7 @@ public:
   }
 
   std::unique_ptr<xml::element>
-  make_element_for_metadata (const diagnostic_metadata &metadata);
+  make_element_for_metadata (const metadata &);
 
   std::unique_ptr<xml::element>
   make_element_for_patch (const diagnostic_info &diagnostic);
@@ -147,7 +153,7 @@ public:
   }
 
   std::unique_ptr<xml::node>
-  maybe_make_state_diagram (const diagnostic_event &event);
+  maybe_make_state_diagram (const paths::event &event);
 
 private:
   void
@@ -155,7 +161,7 @@ private:
 
   std::unique_ptr<xml::element>
   make_element_for_diagnostic (const diagnostic_info &diagnostic,
-			       diagnostic_t orig_diag_kind,
+			       enum kind orig_diag_kind,
 			       bool alert);
 
   std::unique_ptr<xml::element>
@@ -172,21 +178,26 @@ private:
   void
   pop_nesting_level ();
 
-  diagnostic_context &m_context;
+  void
+  add_graph (const digraphs::digraph &dg,
+	     xml::element &parent_element);
+
+  context &m_context;
   pretty_printer *m_printer;
   const line_maps *m_line_maps;
   html_generation_options m_html_gen_opts;
-  const logical_location_manager *m_logical_loc_mgr;
+  const logical_locations::manager *m_logical_loc_mgr;
 
   std::unique_ptr<xml::document> m_document;
   xml::element *m_head_element;
   xml::element *m_title_element;
+  xml::element *m_body_element;
   xml::element *m_diagnostics_element;
   std::unique_ptr<xml::element> m_cur_diagnostic_element;
   std::vector<xml::element *> m_cur_nesting_levels;
   int m_next_diag_id; // for handing out unique IDs
   json::array m_ui_focus_ids;
-  logical_location m_last_logical_location;
+  logical_locations::key m_last_logical_location;
   location_t m_last_location;
   expanded_location m_last_expanded_location;
 };
@@ -207,12 +218,12 @@ make_span (std::string class_)
   return span;
 }
 
-/* class diagnostic_html_format_buffer : public diagnostic_per_format_buffer.  */
+/* class html_sink_buffer : public per_sink_buffer.  */
 
 void
-diagnostic_html_format_buffer::dump (FILE *out, int indent) const
+html_sink_buffer::dump (FILE *out, int indent) const
 {
-  fprintf (out, "%*sdiagnostic_html_format_buffer:\n", indent, "");
+  fprintf (out, "%*shtml_sink_buffer:\n", indent, "");
   int idx = 0;
   for (auto &result : m_results)
     {
@@ -224,29 +235,29 @@ diagnostic_html_format_buffer::dump (FILE *out, int indent) const
 }
 
 bool
-diagnostic_html_format_buffer::empty_p () const
+html_sink_buffer::empty_p () const
 {
   return m_results.empty ();
 }
 
 void
-diagnostic_html_format_buffer::move_to (diagnostic_per_format_buffer &base)
+html_sink_buffer::move_to (per_sink_buffer &base)
 {
-  diagnostic_html_format_buffer &dest
-    = static_cast<diagnostic_html_format_buffer &> (base);
+  html_sink_buffer &dest
+    = static_cast<html_sink_buffer &> (base);
   for (auto &&result : m_results)
     dest.m_results.push_back (std::move (result));
   m_results.clear ();
 }
 
 void
-diagnostic_html_format_buffer::clear ()
+html_sink_buffer::clear ()
 {
   m_results.clear ();
 }
 
 void
-diagnostic_html_format_buffer::flush ()
+html_sink_buffer::flush ()
 {
   for (auto &&result : m_results)
     m_builder.m_diagnostics_element->add_child (std::move (result));
@@ -390,17 +401,18 @@ struct html_doctypedecl : public xml::doctypedecl
 
 /* html_builder's ctor.  */
 
-html_builder::html_builder (diagnostic_context &context,
+html_builder::html_builder (context &dc,
 			    pretty_printer &pp,
 			    const line_maps *line_maps,
 			    const html_generation_options &html_gen_opts)
-: m_context (context),
+: m_context (dc),
   m_printer (&pp),
   m_line_maps (line_maps),
   m_html_gen_opts (html_gen_opts),
   m_logical_loc_mgr (nullptr),
   m_head_element (nullptr),
   m_title_element (nullptr),
+  m_body_element (nullptr),
   m_diagnostics_element (nullptr),
   m_next_diag_id (0),
   m_last_location (UNKNOWN_LOCATION),
@@ -408,7 +420,7 @@ html_builder::html_builder (diagnostic_context &context,
 {
   gcc_assert (m_line_maps);
 
-  if (auto client_data_hooks = context.get_client_data_hooks ())
+  if (auto client_data_hooks = dc.get_client_data_hooks ())
     m_logical_loc_mgr = client_data_hooks->get_logical_location_manager ();
 
   m_document = std::make_unique<xml::document> ();
@@ -447,6 +459,7 @@ html_builder::html_builder (diagnostic_context &context,
 
     {
       xml::auto_print_element body (xp, "body");
+      m_body_element = xp.get_insertion_point ();
       {
 	auto diagnostics_element = make_div ("gcc-diagnostic-list");
 	m_diagnostics_element = diagnostics_element.get ();
@@ -483,10 +496,10 @@ html_builder::add_stylesheet (std::string url)
 
 void
 html_builder::on_report_diagnostic (const diagnostic_info &diagnostic,
-				    diagnostic_t orig_diag_kind,
-				    diagnostic_html_format_buffer *buffer)
+				    enum kind orig_diag_kind,
+				    html_sink_buffer *buffer)
 {
-  if (diagnostic.kind == DK_ICE || diagnostic.kind == DK_ICE_NOBT)
+  if (diagnostic.m_kind == kind::ice || diagnostic.m_kind == kind::ice_nobt)
     {
       /* Print a header for the remaining output to stderr, and
 	 return, attempting to print the usual ICE messages to
@@ -501,7 +514,7 @@ html_builder::on_report_diagnostic (const diagnostic_info &diagnostic,
   if (m_cur_diagnostic_element && nesting_level > 0)
     alert = false;
   if (!m_cur_diagnostic_element)
-    m_last_logical_location = logical_location ();
+    m_last_logical_location = logical_locations::key ();
   auto diag_element
     = make_element_for_diagnostic (diagnostic, orig_diag_kind, alert);
   if (buffer)
@@ -594,44 +607,49 @@ print_pre_source (xml::printer &xp, const char *text)
 }
 
 std::unique_ptr<xml::node>
-html_builder::maybe_make_state_diagram (const diagnostic_event &event)
+html_builder::maybe_make_state_diagram (const paths::event &event)
 {
   if (!m_html_gen_opts.m_show_state_diagrams)
     return nullptr;
 
-  /* Get XML state document; if we're going to print it later, also request
+  if (!m_logical_loc_mgr)
+    return nullptr;
+
+  /* Get state graph; if we're going to print it later, also request
      the debug version.  */
-  auto xml_state
-    = event.maybe_make_xml_state (m_html_gen_opts.m_show_state_diagram_xml);
-  if (!xml_state)
+  auto state_graph
+    = event.maybe_make_diagnostic_state_graph
+	(m_html_gen_opts.m_show_state_diagrams_sarif);
+  if (!state_graph)
     return nullptr;
 
   // Convert it to .dot AST
-  auto graph = make_dot_graph_from_xml_state (*xml_state);
-  gcc_assert (graph);
+  auto dot_graph = state_graphs::make_dot_graph (*state_graph,
+						 *m_logical_loc_mgr);
+  gcc_assert (dot_graph);
 
   auto wrapper = std::make_unique<xml::element> ("div", false);
   xml::printer xp (*wrapper);
 
-  if (m_html_gen_opts.m_show_state_diagram_xml)
+  if (m_html_gen_opts.m_show_state_diagrams_sarif)
     {
-      // For debugging, show the XML src inline:
+      // For debugging, show the SARIF src inline:
       pretty_printer pp;
-      xml_state->write_as_xml (&pp, 0, true);
+      state_graph->make_json_sarif_graph ()->print (&pp, true);
       print_pre_source (xp, pp_formatted_text (&pp));
     }
 
-  if (m_html_gen_opts.m_show_state_diagram_dot_src)
+  if (m_html_gen_opts.m_show_state_diagrams_dot_src)
     {
       // For debugging, show the dot src inline:
       pretty_printer pp;
       dot::writer w (pp);
-      graph->print (w);
+      dot_graph->print (w);
       print_pre_source (xp, pp_formatted_text (&pp));
     }
 
   // Turn the .dot into SVG and splice into place
-  auto svg = dot::make_svg_from_graph (*graph);
+  auto svg = dot::make_svg_from_graph (*dot_graph);
   if (svg)
     xp.append (std::move (svg));
 
@@ -647,7 +665,7 @@ class html_path_label_writer : public html_label_writer
 public:
   html_path_label_writer (xml::printer &xp,
 			  html_builder &builder,
-			  const diagnostic_path &path,
+			  const paths::path &path,
 			  const std::string &event_id_prefix)
   : m_xp (xp),
     m_html_builder (builder),
@@ -668,7 +686,7 @@ public:
 
   void end_label () final override
   {
-    const diagnostic_event &event
+    const paths::event &event
       = m_path.get_event (m_curr_event_id.zero_based ());
     if (auto state_doc = m_html_builder.maybe_make_state_diagram (event))
     {
@@ -696,31 +714,31 @@ private:
 
   xml::printer &m_xp;
   html_builder &m_html_builder;
-  const diagnostic_path &m_path;
+  const paths::path &m_path;
   const std::string &m_event_id_prefix;
   int m_next_event_idx;
-  diagnostic_event_id_t m_curr_event_id;
+  paths::event_id_t m_curr_event_id;
 };
 
 /* See https://pf3.patternfly.org/v3/pattern-library/widgets/#alerts */
 static const char *
-get_pf_class_for_alert_div (diagnostic_t diag_kind)
+get_pf_class_for_alert_div (enum kind diag_kind)
 {
   switch (diag_kind)
     {
-    case DK_DEBUG:
-    case DK_NOTE:
+    case kind::debug:
+    case kind::note:
       return "alert alert-info";
 
-    case DK_ANACHRONISM:
-    case DK_WARNING:
+    case kind::anachronism:
+    case kind::warning:
       return "alert alert-warning";
 
-    case DK_ERROR:
-    case DK_SORRY:
-    case DK_ICE:
-    case DK_ICE_NOBT:
-    case DK_FATAL:
+    case kind::error:
+    case kind::sorry:
+    case kind::ice:
+    case kind::ice_nobt:
+    case kind::fatal:
       return "alert alert-danger";
 
     default:
@@ -729,23 +747,23 @@ get_pf_class_for_alert_div (diagnostic_t diag_kind)
 }
 
 static const char *
-get_pf_class_for_alert_icon (diagnostic_t diag_kind)
+get_pf_class_for_alert_icon (enum kind diag_kind)
 {
   switch (diag_kind)
     {
-    case DK_DEBUG:
-    case DK_NOTE:
+    case kind::debug:
+    case kind::note:
       return "pficon pficon-info";
 
-    case DK_ANACHRONISM:
-    case DK_WARNING:
+    case kind::anachronism:
+    case kind::warning:
       return "pficon pficon-warning-triangle-o";
 
-    case DK_ERROR:
-    case DK_SORRY:
-    case DK_ICE:
-    case DK_ICE_NOBT:
-    case DK_FATAL:
+    case kind::error:
+    case kind::sorry:
+    case kind::ice:
+    case kind::ice_nobt:
+    case kind::fatal:
       return "pficon pficon-error-circle-o";
 
     default:
@@ -754,57 +772,57 @@ get_pf_class_for_alert_icon (diagnostic_t diag_kind)
 }
 
 static const char *
-get_label_for_logical_location_kind (enum logical_location_kind kind)
+get_label_for_logical_location_kind (enum logical_locations::kind kind)
 {
   switch (kind)
     {
     default:
       gcc_unreachable ();
-    case logical_location_kind::unknown:
+    case logical_locations::kind::unknown:
       return nullptr;
 
     /* Kinds within executable code.  */
-    case logical_location_kind::function:
+    case logical_locations::kind::function:
       return "Function";
-    case logical_location_kind::member:
+    case logical_locations::kind::member:
       return "Member";
-    case logical_location_kind::module_:
+    case logical_locations::kind::module_:
       return "Module";
-    case logical_location_kind::namespace_:
+    case logical_locations::kind::namespace_:
       return "Namespace";
-    case logical_location_kind::type:
+    case logical_locations::kind::type:
       return "Type";
-    case logical_location_kind::return_type:
+    case logical_locations::kind::return_type:
       return "Return type";
-    case logical_location_kind::parameter:
+    case logical_locations::kind::parameter:
       return "Parameter";
-    case logical_location_kind::variable:
+    case logical_locations::kind::variable:
       return "Variable";
 
     /* Kinds within XML or HTML documents.  */
-    case logical_location_kind::element:
+    case logical_locations::kind::element:
       return "Element";
-    case logical_location_kind::attribute:
+    case logical_locations::kind::attribute:
       return "Attribute";
-    case logical_location_kind::text:
+    case logical_locations::kind::text:
       return "Text";
-    case logical_location_kind::comment:
+    case logical_locations::kind::comment:
       return "Comment";
-    case logical_location_kind::processing_instruction:
+    case logical_locations::kind::processing_instruction:
       return "Processing Instruction";
-    case logical_location_kind::dtd:
+    case logical_locations::kind::dtd:
       return "DTD";
-    case logical_location_kind::declaration:
+    case logical_locations::kind::declaration:
       return "Declaration";
 
   /* Kinds within JSON documents.  */
-    case logical_location_kind::object:
+    case logical_locations::kind::object:
       return "Object";
-    case logical_location_kind::array:
+    case logical_locations::kind::array:
       return "Array";
-    case logical_location_kind::property:
+    case logical_locations::kind::property:
       return "Property";
-    case logical_location_kind::value:
+    case logical_locations::kind::value:
       return "Value";
     }
 }
@@ -928,7 +946,7 @@ private:
 
 std::unique_ptr<xml::element>
 html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
-					   diagnostic_t orig_diag_kind,
+					   enum kind orig_diag_kind,
 					   bool alert)
 {
   const int diag_idx = m_next_diag_id++;
@@ -952,7 +970,7 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
   diag_element->set_attr ("id", diag_id);
   if (alert)
     diag_element->set_attr ("class",
-			    get_pf_class_for_alert_div (diagnostic.kind));
+			    get_pf_class_for_alert_div (diagnostic.m_kind));
 
   xml::printer xp (*diag_element.get ());
   const size_t depth_within_alert_div = 1;
@@ -962,7 +980,7 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
   if (alert)
     {
       xp.push_tag_with_class ("span",
-			      get_pf_class_for_alert_icon (diagnostic.kind),
+			      get_pf_class_for_alert_icon (diagnostic.m_kind),
 			      true);
       xp.add_text (" ");
       xp.pop_tag ("span");
@@ -986,7 +1004,7 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
   if (show_severity)
   {
     xp.push_tag ("strong");
-    xp.add_text (_(get_diagnostic_kind_text (diagnostic.kind)));
+    xp.add_text (_(get_text_for_kind (diagnostic.m_kind)));
     xp.pop_tag ("strong");
     xp.add_text (" ");
   }
@@ -999,21 +1017,21 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
   pp_clear_output_area (m_printer);
 
   // Add any metadata as a suffix to the message
-  if (diagnostic.metadata)
+  if (diagnostic.m_metadata)
     {
       xp.add_text (" ");
-      xp.append (make_element_for_metadata (*diagnostic.metadata));
+      xp.append (make_element_for_metadata (*diagnostic.m_metadata));
     }
 
   // Add any option as a suffix to the message
 
   label_text option_text = label_text::take
-    (m_context.make_option_name (diagnostic.option_id,
-				 orig_diag_kind, diagnostic.kind));
+    (m_context.make_option_name (diagnostic.m_option_id,
+				 orig_diag_kind, diagnostic.m_kind));
   if (option_text.get ())
     {
       label_text option_url = label_text::take
-	(m_context.make_option_url (diagnostic.option_id));
+	(m_context.make_option_url (diagnostic.m_option_id));
 
       xp.add_text (" ");
       auto option_span = make_span ("gcc-option");
@@ -1045,7 +1063,7 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
       if (auto logical_loc = client_data_hooks->get_current_logical_location ())
 	if (logical_loc != m_last_logical_location)
 	  {
-	    enum logical_location_kind kind
+	    enum logical_locations::kind kind
 	      = m_logical_loc_mgr->get_kind (logical_loc);;
 	    if (const char *label = get_label_for_logical_location_kind (kind))
 	      if (const char *name_with_scope
@@ -1068,8 +1086,8 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
       if (s.line)
 	{
 	  add_labelled_value (xp, "line", "Line", std::to_string (s.line), false);
-	  diagnostic_column_policy column_policy (m_context);
-	  int converted_column = column_policy.converted_column (s);
+	  column_policy col_policy (m_context);
+	  int converted_column = col_policy.converted_column (s);
 	  if (converted_column >= 0)
 	    add_labelled_value (xp, "column", "Column",
 				std::to_string (converted_column),
@@ -1084,12 +1102,13 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
     // TODO: m_context.m_last_location should be moved into the sink
     location_t saved = m_context.m_last_location;
     m_context.m_last_location = m_last_location;
-    m_context.maybe_show_locus_as_html (*diagnostic.richloc,
-					m_context.m_source_printing,
-					diagnostic.kind,
-					xp,
-					nullptr,
-					nullptr);
+    m_context.maybe_show_locus_as_html
+      (*diagnostic.m_richloc,
+       m_context.get_source_printing_options (),
+       diagnostic.m_kind,
+       xp,
+       nullptr,
+       nullptr);
     m_context.m_last_location = saved;
     m_last_location = m_context.m_last_location;
   }
@@ -1097,7 +1116,7 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
   gcc_assert (xp.get_num_open_tags () == depth_within_alert_div);
 
   /* Execution path.  */
-  if (auto path = diagnostic.richloc->get_path ())
+  if (auto path = diagnostic.m_richloc->get_path ())
     {
       xp.push_tag ("div");
       xp.set_attr ("id", "execution-path");
@@ -1116,7 +1135,7 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
       html_path_label_writer event_label_writer (xp, *this, *path,
 						 event_id_prefix);
 
-      diagnostic_source_print_policy dspp (m_context);
+      source_print_policy dspp (m_context);
       print_path_as_html (xp, *path, m_context, &event_label_writer,
 			  dspp);
 
@@ -1125,6 +1144,15 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
 
   gcc_assert (xp.get_num_open_tags () == depth_within_alert_div);
 
+  // Try to display any per-diagnostic graphs
+  if (diagnostic.m_metadata)
+    if (auto ldg = diagnostic.m_metadata->get_lazy_digraphs ())
+      {
+	auto &digraphs = ldg->get_or_create ();
+	for (auto &dg : digraphs)
+	  add_graph (*dg, *xp.get_insertion_point ());
+      }
+
   if (auto patch_element = make_element_for_patch (diagnostic))
     {
       xp.push_tag ("div");
@@ -1142,9 +1170,9 @@ html_builder::make_element_for_diagnostic (const diagnostic_info &diagnostic,
 std::unique_ptr<xml::element>
 html_builder::make_element_for_patch (const diagnostic_info &diagnostic)
 {
-  edit_context ec (m_context.get_file_cache ());
-  ec.add_fixits (diagnostic.richloc);
-  if (char *diff = ec.generate_diff (true))
+  changes::change_set edit (m_context.get_file_cache ());
+  edit.add_fixits (diagnostic.m_richloc);
+  if (char *diff = edit.generate_diff (true))
     {
       if (strlen (diff) > 0)
 	{
@@ -1182,11 +1210,11 @@ html_builder::make_metadata_element (label_text label,
 }
 
 std::unique_ptr<xml::element>
-html_builder::make_element_for_metadata (const diagnostic_metadata &metadata)
+html_builder::make_element_for_metadata (const metadata &m)
 {
   auto span_metadata = make_span ("gcc-metadata");
 
-  int cwe = metadata.get_cwe ();
+  int cwe = m.get_cwe ();
   if (cwe)
     {
       pretty_printer pp;
@@ -1197,9 +1225,9 @@ html_builder::make_element_for_metadata (const diagnostic_metadata &metadata)
 	(make_metadata_element (std::move (label), std::move (url)));
     }
 
-  for (unsigned idx = 0; idx < metadata.get_num_rules (); ++idx)
+  for (unsigned idx = 0; idx < m.get_num_rules (); ++idx)
     {
-      auto &rule = metadata.get_rule (idx);
+      auto &rule = m.get_rule (idx);
       label_text label = label_text::take (rule.make_description ());
       label_text url = label_text::take (rule.make_url ());
       span_metadata->add_child
@@ -1209,11 +1237,11 @@ html_builder::make_element_for_metadata (const diagnostic_metadata &metadata)
   return span_metadata;
 }
 
-/* Implementation of diagnostic_context::m_diagrams.m_emission_cb
+/* Implementation of diagnostics::context::m_diagrams.m_emission_cb
    for HTML output.  */
 
 void
-html_builder::emit_diagram (const diagnostic_diagram &/*diagram*/)
+html_builder::emit_diagram (const diagram &)
 {
   /* We must be within the emission of a top-level diagnostic.  */
   gcc_assert (m_cur_diagnostic_element);
@@ -1221,6 +1249,35 @@ html_builder::emit_diagram (const diagnostic_diagram &/*diagram*/)
   // TODO: currently a no-op
 }
 
+void
+html_builder::add_graph (const digraphs::digraph &dg,
+			 xml::element &parent_element)
+{
+  if (auto dot_graph = dg.make_dot_graph ())
+    if (auto svg_element = dot::make_svg_from_graph (*dot_graph))
+      {
+	auto div = std::make_unique<xml::element> ("div", false);
+	div->set_attr ("class", "gcc-directed-graph");
+	xml::printer xp (*div);
+	if (const char *description = dg.get_description ())
+	  {
+	    xp.push_tag ("h2", true);
+	    xp.add_text (description);
+	    xp.pop_tag ("h2");
+	  }
+	xp.append (std::move (svg_element));
+	parent_element.add_child (std::move (div));
+      }
+}
+
+void
+html_builder::emit_global_graph (const lazily_created<digraphs::digraph> &ldg)
+{
+  auto &dg = ldg.get_or_create ();
+  gcc_assert (m_body_element);
+  add_graph (dg, *m_body_element);
+}
+
 /* Implementation of "end_group_cb" for HTML output.  */
 
 void
@@ -1257,10 +1314,10 @@ html_builder::flush_to_file (FILE *outf)
   fprintf (outf, "\n");
 }
 
-class html_output_format : public diagnostic_output_format
+class html_sink : public sink
 {
 public:
-  ~html_output_format ()
+  ~html_sink ()
   {
     /* Any diagnostics should have been handled by now.
        If not, then something's gone wrong with diagnostic
@@ -1272,8 +1329,8 @@ public:
 
   void dump (FILE *out, int indent) const override
   {
-    fprintf (out, "%*shtml_output_format\n", indent, "");
-    diagnostic_output_format::dump (out, indent);
+    fprintf (out, "%*shtml_sink\n", indent, "");
+    sink::dump (out, indent);
   }
 
   void
@@ -1282,15 +1339,15 @@ public:
     m_builder.set_main_input_filename (name);
   }
 
-  std::unique_ptr<diagnostic_per_format_buffer>
-  make_per_format_buffer () final override
+  std::unique_ptr<per_sink_buffer>
+  make_per_sink_buffer () final override
   {
-    return std::make_unique<diagnostic_html_format_buffer> (m_builder);
+    return std::make_unique<html_sink_buffer> (m_builder);
   }
-  void set_buffer (diagnostic_per_format_buffer *base_buffer) final override
+  void set_buffer (per_sink_buffer *base_buffer) final override
   {
-    diagnostic_html_format_buffer *buffer
-      = static_cast<diagnostic_html_format_buffer *> (base_buffer);
+    html_sink_buffer *buffer
+      = static_cast<html_sink_buffer *> (base_buffer);
     m_buffer = buffer;
   }
 
@@ -1304,13 +1361,13 @@ public:
   }
   void
   on_report_diagnostic (const diagnostic_info &diagnostic,
-			diagnostic_t orig_diag_kind) final override
+			enum kind orig_diag_kind) final override
   {
     m_builder.on_report_diagnostic (diagnostic, orig_diag_kind, m_buffer);
   }
-  void on_diagram (const diagnostic_diagram &diagram) final override
+  void on_diagram (const diagram &d) final override
   {
-    m_builder.emit_diagram (diagram);
+    m_builder.emit_diagram (d);
   }
   void after_diagnostic (const diagnostic_info &) final override
   {
@@ -1334,6 +1391,13 @@ public:
     m_builder.set_printer (*get_printer ());
   }
 
+  void
+  report_global_digraph (const lazily_created<digraphs::digraph> &ldg)
+    final override
+  {
+    m_builder.emit_global_graph (ldg);
+  }
+
   const xml::document &get_document () const
   {
     return m_builder.get_document ();
@@ -1342,41 +1406,41 @@ public:
   html_builder &get_builder () { return m_builder; }
 
 protected:
-  html_output_format (diagnostic_context &context,
-		      const line_maps *line_maps,
-		      const html_generation_options &html_gen_opts)
-  : diagnostic_output_format (context),
-    m_builder (context, *get_printer (), line_maps, html_gen_opts),
+  html_sink (context &dc,
+	     const line_maps *line_maps,
+	     const html_generation_options &html_gen_opts)
+  : sink (dc),
+    m_builder (dc, *get_printer (), line_maps, html_gen_opts),
     m_buffer (nullptr)
   {}
 
   html_builder m_builder;
-  diagnostic_html_format_buffer *m_buffer;
+  html_sink_buffer *m_buffer;
 };
 
-class html_file_output_format : public html_output_format
+class html_file_sink : public html_sink
 {
 public:
-  html_file_output_format (diagnostic_context &context,
-			   const line_maps *line_maps,
-			   const html_generation_options &html_gen_opts,
-			   diagnostic_output_file output_file)
-  : html_output_format (context, line_maps, html_gen_opts),
-    m_output_file (std::move (output_file))
+  html_file_sink (context &dc,
+		  const line_maps *line_maps,
+		  const html_generation_options &html_gen_opts,
+		  output_file output_file_)
+  : html_sink (dc, line_maps, html_gen_opts),
+    m_output_file (std::move (output_file_))
   {
     gcc_assert (m_output_file.get_open_file ());
     gcc_assert (m_output_file.get_filename ());
   }
-  ~html_file_output_format ()
+  ~html_file_sink ()
   {
     m_builder.flush_to_file (m_output_file.get_open_file ());
   }
   void dump (FILE *out, int indent) const override
   {
-    fprintf (out, "%*shtml_file_output_format: %s\n",
+    fprintf (out, "%*shtml_file_sink: %s\n",
 	     indent, "",
 	     m_output_file.get_filename ());
-    diagnostic_output_format::dump (out, indent);
+    sink::dump (out, indent);
   }
   bool machine_readable_stderr_p () const final override
   {
@@ -1384,26 +1448,26 @@ public:
   }
 
 private:
-  diagnostic_output_file m_output_file;
+  output_file m_output_file;
 };
 
 /* Attempt to open BASE_FILE_NAME.html for writing.
-   Return a non-null diagnostic_output_file,
-   or return a null diagnostic_output_file and complain to CONTEXT
+   Return a non-null output_file,
+   or return a null output_file and complain to DC
    using LINE_MAPS.  */
 
-diagnostic_output_file
-diagnostic_output_format_open_html_file (diagnostic_context &context,
-					 line_maps *line_maps,
-					 const char *base_file_name)
+output_file
+open_html_output_file (context &dc,
+		       line_maps *line_maps,
+		       const char *base_file_name)
 {
   if (!base_file_name)
     {
       rich_location richloc (line_maps, UNKNOWN_LOCATION);
-      context.emit_diagnostic_with_group
-	(DK_ERROR, richloc, nullptr, 0,
+      dc.emit_diagnostic_with_group
+	(kind::error, richloc, nullptr, 0,
 	 "unable to determine filename for HTML output");
-      return diagnostic_output_file ();
+      return output_file ();
     }
 
   label_text filename = label_text::take (concat (base_file_name,
@@ -1413,26 +1477,26 @@ diagnostic_output_format_open_html_file (diagnostic_context &context,
   if (!outf)
     {
       rich_location richloc (line_maps, UNKNOWN_LOCATION);
-      context.emit_diagnostic_with_group
-	(DK_ERROR, richloc, nullptr, 0,
+      dc.emit_diagnostic_with_group
+	(kind::error, richloc, nullptr, 0,
 	 "unable to open %qs for HTML output: %m",
 	 filename.get ());
-      return diagnostic_output_file ();
+      return output_file ();
     }
-  return diagnostic_output_file (outf, true, std::move (filename));
+  return output_file (outf, true, std::move (filename));
 }
 
-std::unique_ptr<diagnostic_output_format>
-make_html_sink (diagnostic_context &context,
+std::unique_ptr<sink>
+make_html_sink (context &dc,
 		const line_maps &line_maps,
 		const html_generation_options &html_gen_opts,
-		diagnostic_output_file output_file)
+		output_file output_file_)
 {
   auto sink
-    = std::make_unique<html_file_output_format> (context,
-						 &line_maps,
-						 html_gen_opts,
-						 std::move (output_file));
+    = std::make_unique<html_file_sink> (dc,
+					&line_maps,
+					html_gen_opts,
+					std::move (output_file_));
   sink->update_printer ();
   return sink;
 }
@@ -1489,7 +1553,7 @@ test_token_printer ()
 
   {
     token_printer_test t;
-    diagnostic_event_id_t event_id (0);
+    paths::event_id_t event_id (0);
     pp_printf (&t.m_pp, "foo %@ bar", &event_id);
     ASSERT_XML_PRINT_EQ
       (t.m_top_element,
@@ -1497,26 +1561,26 @@ test_token_printer ()
   }
 }
 
-/* A subclass of html_output_format for writing selftests.
+/* A subclass of html_sink for writing selftests.
    The XML output is cached internally, rather than written
    out to a file.  */
 
-class test_html_diagnostic_context : public test_diagnostic_context
+class test_html_context : public test_context
 {
 public:
-  test_html_diagnostic_context ()
+  test_html_context ()
   {
     html_generation_options html_gen_opts;
     html_gen_opts.m_css = false;
     html_gen_opts.m_javascript = false;
-    auto sink = std::make_unique<html_buffered_output_format> (*this,
-							       line_table,
-							       html_gen_opts);
+    auto sink = std::make_unique<html_buffered_sink> (*this,
+						      line_table,
+						      html_gen_opts);
     sink->update_printer ();
     sink->set_main_input_filename ("(main input filename)");
     m_format = sink.get (); // borrowed
 
-    set_output_format (std::move (sink));
+    set_sink (std::move (sink));
   }
 
   const xml::document &get_document () const
@@ -1530,13 +1594,13 @@ public:
   }
 
 private:
-  class html_buffered_output_format : public html_output_format
+  class html_buffered_sink : public html_sink
   {
   public:
-    html_buffered_output_format (diagnostic_context &context,
-				 const line_maps *line_maps,
-				 const html_generation_options &html_gen_opts)
-    : html_output_format (context, line_maps, html_gen_opts)
+    html_buffered_sink (context &dc,
+			const line_maps *line_maps,
+			const html_generation_options &html_gen_opts)
+    : html_sink (dc, line_maps, html_gen_opts)
     {
     }
     bool machine_readable_stderr_p () const final override
@@ -1545,20 +1609,20 @@ private:
     }
   };
 
-  html_output_format *m_format; // borrowed
+  html_sink *m_format; // borrowed
 };
 
 /* Test of reporting a diagnostic at UNKNOWN_LOCATION to a
-   diagnostic_context and examining the generated XML document.
+   diagnostics::context and examining the generated XML document.
    Verify various basic properties. */
 
 static void
 test_simple_log ()
 {
-  test_html_diagnostic_context dc;
+  test_html_context dc;
 
   rich_location richloc (line_table, UNKNOWN_LOCATION);
-  dc.report (DK_ERROR, richloc, nullptr, 0, "this is a test: %qs", "foo");
+  dc.report (kind::error, richloc, nullptr, 0, "this is a test: %qs", "foo");
 
   const xml::document &doc  = dc.get_document ();
 
@@ -1586,13 +1650,13 @@ test_simple_log ()
 static void
 test_metadata ()
 {
-  test_html_diagnostic_context dc;
+  test_html_context dc;
   html_builder &b = dc.get_builder ();
 
   {
-    diagnostic_metadata metadata;
-    metadata.add_cwe (415);
-    auto element = b.make_element_for_metadata (metadata);
+    metadata m;
+    m.add_cwe (415);
+    auto element = b.make_element_for_metadata (m);
     ASSERT_XML_PRINT_EQ
       (*element,
        "<span class=\"gcc-metadata\">"
@@ -1607,11 +1671,11 @@ test_metadata ()
   }
 
   {
-    diagnostic_metadata metadata;
-    diagnostic_metadata::precanned_rule rule ("MISC-42",
-					      "http://example.com");
-    metadata.add_rule (rule);
-    auto element = b.make_element_for_metadata (metadata);
+    metadata m;
+    metadata::precanned_rule rule ("MISC-42",
+				   "http://example.com");
+    m.add_rule (rule);
+    auto element = b.make_element_for_metadata (m);
     ASSERT_XML_PRINT_EQ
       (*element,
        "<span class=\"gcc-metadata\">"
@@ -1629,9 +1693,9 @@ test_metadata ()
 /* Run all of the selftests within this file.  */
 
 void
-diagnostic_format_html_cc_tests ()
+html_sink_cc_tests ()
 {
-  auto_fix_quotes fix_quotes;
+  ::selftest::auto_fix_quotes fix_quotes;
   test_token_printer ();
   test_simple_log ();
   test_metadata ();
@@ -1640,3 +1704,5 @@ diagnostic_format_html_cc_tests ()
 } // namespace selftest
 
 #endif /* CHECKING_P */
+
+} // namespace diagnostics
diff --git a/gcc/diagnostic-format-html.h b/gcc/diagnostics/html-sink.h
index 09a97e0..d86bde8 100644
--- a/gcc/diagnostic-format-html.h
+++ b/gcc/diagnostics/html-sink.h
@@ -18,11 +18,13 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_FORMAT_HTML_H
-#define GCC_DIAGNOSTIC_FORMAT_HTML_H
+#ifndef GCC_DIAGNOSTICS_HTML_SINK_H
+#define GCC_DIAGNOSTICS_HTML_SINK_H
 
-#include "diagnostic-format.h"
-#include "diagnostic-output-file.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/output-file.h"
+
+namespace diagnostics {
 
 struct html_generation_options
 {
@@ -36,29 +38,31 @@ struct html_generation_options
   // If true, attempt to show state diagrams at events
   bool m_show_state_diagrams;
 
-  // If true, show the XML form of the state with such diagrams
-  bool m_show_state_diagram_xml;
+  // If true, show the SARIF form of the state with such diagrams
+  bool m_show_state_diagrams_sarif;
 
   // If true, show the .dot source used for the diagram
-  bool m_show_state_diagram_dot_src;
+  bool m_show_state_diagrams_dot_src;
 };
 
-extern diagnostic_output_file
-diagnostic_output_format_open_html_file (diagnostic_context &context,
-					 line_maps *line_maps,
-					 const char *base_file_name);
+extern diagnostics::output_file
+open_html_output_file (context &dc,
+		       line_maps *line_maps,
+		       const char *base_file_name);
 
-extern std::unique_ptr<diagnostic_output_format>
-make_html_sink (diagnostic_context &context,
+extern std::unique_ptr<sink>
+make_html_sink (context &dc,
 		const line_maps &line_maps,
 		const html_generation_options &html_gen_opts,
-		diagnostic_output_file output_file);
+		output_file output_file_);
 
 extern void
 print_path_as_html (xml::printer &xp,
-		    const diagnostic_path &path,
-		    diagnostic_context &dc,
+		    const paths::path &path,
+		    context &dc,
 		    html_label_writer *event_label_writer,
-		    const diagnostic_source_print_policy &dspp);
+		    const source_print_policy &dspp);
+
+} // namespace diagnostics
 
-#endif /* ! GCC_DIAGNOSTIC_FORMAT_HTML_H */
+#endif /* ! GCC_DIAGNOSTICS_HTML_SINK_H */
diff --git a/gcc/diagnostic.def b/gcc/diagnostics/kinds.def
index ab2185c..2a0a0a6 100644
--- a/gcc/diagnostic.def
+++ b/gcc/diagnostics/kinds.def
@@ -16,40 +16,40 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-/* DK_UNSPECIFIED must be first so it has a value of zero.  We never
+/* kind::unspecified must be first so it has a value of zero.  We never
    assign this kind to an actual diagnostic, we only use this in
    variables that can hold a kind, to mean they have yet to have a
    kind specified.  I.e. they're uninitialized.  Within the diagnostic
    machinery, this kind also means "don't change the existing kind",
    meaning "no change is specified".  */
-DEFINE_DIAGNOSTIC_KIND (DK_UNSPECIFIED, "", NULL)
+DEFINE_DIAGNOSTIC_KIND (unspecified, "", NULL)
 
-/* If a diagnostic is set to DK_IGNORED, it won't get reported at all.
+/* If a diagnostic is set to kind::ignored, it won't get reported at all.
    This is used by the diagnostic machinery when it wants to disable a
    diagnostic without disabling the option which causes it.  */
-DEFINE_DIAGNOSTIC_KIND (DK_IGNORED, "", NULL)
+DEFINE_DIAGNOSTIC_KIND (ignored, "", NULL)
 
 /* The remainder are real diagnostic types.  */
-DEFINE_DIAGNOSTIC_KIND (DK_FATAL, "fatal error: ", "error")
-DEFINE_DIAGNOSTIC_KIND (DK_ICE, "internal compiler error: ", "error")
-DEFINE_DIAGNOSTIC_KIND (DK_ERROR, "error: ", "error")
-DEFINE_DIAGNOSTIC_KIND (DK_SORRY, "sorry, unimplemented: ", "error")
-DEFINE_DIAGNOSTIC_KIND (DK_WARNING, "warning: ", "warning")
-DEFINE_DIAGNOSTIC_KIND (DK_ANACHRONISM, "anachronism: ", "warning")
-DEFINE_DIAGNOSTIC_KIND (DK_NOTE, "note: ", "note")
-DEFINE_DIAGNOSTIC_KIND (DK_DEBUG, "debug: ", "note")
+DEFINE_DIAGNOSTIC_KIND (fatal, "fatal error: ", "error")
+DEFINE_DIAGNOSTIC_KIND (ice, "internal compiler error: ", "error")
+DEFINE_DIAGNOSTIC_KIND (error, "error: ", "error")
+DEFINE_DIAGNOSTIC_KIND (sorry, "sorry, unimplemented: ", "error")
+DEFINE_DIAGNOSTIC_KIND (warning, "warning: ", "warning")
+DEFINE_DIAGNOSTIC_KIND (anachronism, "anachronism: ", "warning")
+DEFINE_DIAGNOSTIC_KIND (note, "note: ", "note")
+DEFINE_DIAGNOSTIC_KIND (debug, "debug: ", "note")
 
 /* For use when using the diagnostic_show_locus machinery to show
    a range of events within a path.  */
-DEFINE_DIAGNOSTIC_KIND (DK_DIAGNOSTIC_PATH, "path: ", "path")
+DEFINE_DIAGNOSTIC_KIND (path, "path: ", "path")
 
-/* These two would be re-classified as DK_WARNING or DK_ERROR, so the
+/* These two would be re-classified as kind::warning or kind::error, so the
 prefix does not matter.  */
-DEFINE_DIAGNOSTIC_KIND (DK_PEDWARN, "pedwarn: ", NULL)
-DEFINE_DIAGNOSTIC_KIND (DK_PERMERROR, "permerror: ", NULL)
-/* This one is just for counting DK_WARNING promoted to DK_ERROR
+DEFINE_DIAGNOSTIC_KIND (pedwarn, "pedwarn: ", NULL)
+DEFINE_DIAGNOSTIC_KIND (permerror, "permerror: ", NULL)
+/* This one is just for counting kind::warning promoted to kind::error
    due to -Werror and -Werror=warning.  */
-DEFINE_DIAGNOSTIC_KIND (DK_WERROR, "error: ", NULL)
-/* This is like DK_ICE, but backtrace is not printed.  Used in the driver
+DEFINE_DIAGNOSTIC_KIND (werror, "error: ", NULL)
+/* This is like kind::ICE, but backtrace is not printed.  Used in the driver
    when reporting fatal signal in the compiler.  */
-DEFINE_DIAGNOSTIC_KIND (DK_ICE_NOBT, "internal compiler error: ", "error")
+DEFINE_DIAGNOSTIC_KIND (ice_nobt, "internal compiler error: ", "error")
diff --git a/gcc/diagnostics/kinds.h b/gcc/diagnostics/kinds.h
new file mode 100644
index 0000000..7b4a168
--- /dev/null
+++ b/gcc/diagnostics/kinds.h
@@ -0,0 +1,45 @@
+/* An enum used to discriminate severities of diagnostics.
+   Copyright (C) 1998-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_KINDS_H
+#define GCC_DIAGNOSTICS_KINDS_H
+
+namespace diagnostics {
+
+/* Constants used to discriminate diagnostics.  */
+enum class kind
+{
+#define DEFINE_DIAGNOSTIC_KIND(K, msgid, C) K,
+#include "diagnostics/kinds.def"
+#undef DEFINE_DIAGNOSTIC_KIND
+  last_diagnostic_kind,
+  /* This is used for tagging pragma pops in the diagnostic
+     classification history chain.  */
+  pop,
+  /* This is used internally to note that a diagnostic is enabled
+     without mandating any specific type.  */
+  any
+};
+
+extern const char *get_text_for_kind (enum diagnostics::kind);
+extern const char *get_color_for_kind (enum diagnostics::kind);
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_KINDS_H */
diff --git a/gcc/lazy-diagnostic-path.cc b/gcc/diagnostics/lazy-paths.cc
index 34743e3..4934651 100644
--- a/gcc/lazy-diagnostic-path.cc
+++ b/gcc/diagnostics/lazy-paths.cc
@@ -23,60 +23,58 @@ along with GCC; see the file COPYING3.  If not see
 #define INCLUDE_VECTOR
 #include "system.h"
 #include "coretypes.h"
-#include "tree.h"
-#include "version.h"
-#include "intl.h"
 #include "diagnostic.h"
-#include "lazy-diagnostic-path.h"
+#include "diagnostics/lazy-paths.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
-#include "simple-diagnostic-path.h"
-#include "gcc-rich-location.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/selftest-context.h"
+#include "diagnostics/selftest-paths.h"
+#include "diagnostics/text-sink.h"
 
-/* class lazy_diagnostic_path : public diagnostic_path.  */
+using namespace diagnostics::paths;
 
-/* Implementation of diagnostic_path vfuncs in terms of a lazily-generated
+/* class lazy_path : public path.  */
+
+/* Implementation of path vfuncs in terms of a lazily-generated
    path.  */
 
 unsigned
-lazy_diagnostic_path::num_events () const
+lazy_path::num_events () const
 {
   lazily_generate_path ();
   return m_inner_path->num_events ();
 }
 
-const diagnostic_event &
-lazy_diagnostic_path::get_event (int idx) const
+const event &
+lazy_path::get_event (int idx) const
 {
   lazily_generate_path ();
   return m_inner_path->get_event (idx);
 }
 
 unsigned
-lazy_diagnostic_path::num_threads () const
+lazy_path::num_threads () const
 {
   lazily_generate_path ();
   return m_inner_path->num_threads ();
 }
 
-const diagnostic_thread &
-lazy_diagnostic_path::get_thread (diagnostic_thread_id_t idx) const
+const thread &
+lazy_path::get_thread (thread_id_t idx) const
 {
   lazily_generate_path ();
   return m_inner_path->get_thread (idx);
 }
 
 bool
-lazy_diagnostic_path::same_function_p (int event_idx_a,
-					   int event_idx_b) const
+lazy_path::same_function_p (int event_idx_a,
+			    int event_idx_b) const
 {
   lazily_generate_path ();
   return m_inner_path->same_function_p (event_idx_a, event_idx_b);
 }
 
 void
-lazy_diagnostic_path::lazily_generate_path () const
+lazy_path::lazily_generate_path () const
 {
   if (!m_inner_path)
     m_inner_path = make_inner_path ();
@@ -85,30 +83,30 @@ lazy_diagnostic_path::lazily_generate_path () const
 
 #if CHECKING_P
 
+namespace diagnostics {
 namespace selftest {
 
-class test_lazy_path : public lazy_diagnostic_path
+using auto_fix_quotes = ::selftest::auto_fix_quotes;
+
+class test_lazy_path : public lazy_path
 {
 public:
   test_lazy_path (pretty_printer &pp)
-  : lazy_diagnostic_path (m_logical_loc_mgr),
+  : lazy_path (m_logical_loc_mgr),
     m_pp (pp)
   {
   }
-  std::unique_ptr<diagnostic_path> make_inner_path () const final override
+  std::unique_ptr<path> make_inner_path () const final override
   {
-    tree fntype_void_void
-      = build_function_type_array (void_type_node, 0, nullptr);
-    tree fndecl_foo = build_fn_decl ("foo", fntype_void_void);
     auto path
-      = std::make_unique<simple_diagnostic_path> (m_logical_loc_mgr,
-						  &m_pp);
-    path->add_event (UNKNOWN_LOCATION, fndecl_foo, 0, "first %qs", "free");
-    path->add_event (UNKNOWN_LOCATION, fndecl_foo, 0, "double %qs", "free");
+      = std::make_unique<paths::selftest::test_path> (m_logical_loc_mgr,
+						      &m_pp);
+    path->add_event (UNKNOWN_LOCATION, "foo", 0, "first %qs", "free");
+    path->add_event (UNKNOWN_LOCATION, "foo", 0, "double %qs", "free");
     return path;
   }
 private:
-  const tree_logical_location_manager m_logical_loc_mgr;
+  mutable logical_locations::selftest::test_manager m_logical_loc_mgr;
   pretty_printer &m_pp;
 };
 
@@ -127,26 +125,26 @@ test_intraprocedural_path (pretty_printer *event_pp)
 		"double `free'");
 }
 
-/* Implementation of diagnostic_option_manager for which all
+/* Implementation of diagnostics::option_manager for which all
    options are disabled, for use in selftests.
-   Note that this is *not* called for diagnostic_option_id (0), which
+   Note that this is *not* called for option_id (0), which
    means "always warn"  */
 
-class all_warnings_disabled : public diagnostic_option_manager
+class all_warnings_disabled : public diagnostics::option_manager
 {
 public:
-  int option_enabled_p (diagnostic_option_id) const final override
+  int option_enabled_p (diagnostics::option_id) const final override
   {
     /* Treat all options as disabled.  */
     return 0;
   }
-  char *make_option_name (diagnostic_option_id,
-			  diagnostic_t,
-			  diagnostic_t) const final override
+  char *make_option_name (diagnostics::option_id,
+			  enum kind,
+			  enum kind) const final override
   {
     return nullptr;
   }
-  char *make_option_url (diagnostic_option_id) const final override
+  char *make_option_url (diagnostics::option_id) const final override
   {
     return nullptr;
   }
@@ -155,10 +153,10 @@ public:
 static void
 test_emission (pretty_printer *event_pp)
 {
-  struct test_rich_location : public gcc_rich_location
+  struct test_rich_location : public rich_location
   {
     test_rich_location (pretty_printer &event_pp)
-    : gcc_rich_location (UNKNOWN_LOCATION),
+    : rich_location (line_table, UNKNOWN_LOCATION),
       m_path (event_pp)
     {
       set_path (&m_path);
@@ -169,16 +167,16 @@ test_emission (pretty_printer *event_pp)
   /* Verify that we don't bother generating the inner path if the warning
      is skipped.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.set_option_manager (std::make_unique<all_warnings_disabled> (), 0);
 
     test_rich_location rich_loc (*event_pp);
     ASSERT_FALSE (rich_loc.m_path.generated_p ());
 
-    diagnostic_option_id option_id (42); // has to be non-zero
+    diagnostics::option_id opt_id (42); // has to be non-zero
     bool emitted
-      = dc.emit_diagnostic_with_group (DK_WARNING, rich_loc, nullptr,
-				       option_id,
+      = dc.emit_diagnostic_with_group (kind::warning, rich_loc, nullptr,
+				       opt_id,
 				       "this warning should be skipped");
     ASSERT_FALSE (emitted);
     ASSERT_FALSE (rich_loc.m_path.generated_p ());
@@ -187,23 +185,23 @@ test_emission (pretty_printer *event_pp)
   /* Verify that we *do* generate the inner path for a diagnostic that
      is emitted, such as an error.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
 
     test_rich_location rich_loc (*event_pp);
     ASSERT_FALSE (rich_loc.m_path.generated_p ());
 
     bool emitted
-      = dc.emit_diagnostic_with_group (DK_ERROR, rich_loc, nullptr, 0,
+      = dc.emit_diagnostic_with_group (kind::error, rich_loc, nullptr, 0,
 				       "this is a test");
     ASSERT_TRUE (emitted);
     ASSERT_TRUE (rich_loc.m_path.generated_p ());
 
     /* Verify that the path works as expected.  */
     dc.set_path_format (DPF_INLINE_EVENTS);
-    diagnostic_text_output_format sink (dc);
-    pp_buffer (sink.get_printer ())->m_flush_p = false;
-    sink.print_path (rich_loc.m_path);
-    ASSERT_STREQ (pp_formatted_text (sink.get_printer ()),
+    diagnostics::text_sink sink_ (dc);
+    pp_buffer (sink_.get_printer ())->m_flush_p = false;
+    sink_.print_path (rich_loc.m_path);
+    ASSERT_STREQ (pp_formatted_text (sink_.get_printer ()),
 		  "  `foo': event 1\n"
 		  " (1): first `free'\n"
 		  "  `foo': event 2\n"
@@ -214,7 +212,7 @@ test_emission (pretty_printer *event_pp)
 /* Run all of the selftests within this file.  */
 
 void
-lazy_diagnostic_path_cc_tests ()
+lazy_paths_cc_tests ()
 {
   /* In a few places we use the global dc's printer to determine
      colorization so ensure this off during the tests.  */
@@ -232,6 +230,7 @@ lazy_diagnostic_path_cc_tests ()
   pp_show_color (global_pp) = saved_show_color;
 }
 
-} // namespace selftest
+} // namespace diagnostics::selftest
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
diff --git a/gcc/lazy-diagnostic-path.h b/gcc/diagnostics/lazy-paths.h
index 9609cab..01b921b 100644
--- a/gcc/lazy-diagnostic-path.h
+++ b/gcc/diagnostics/lazy-paths.h
@@ -18,13 +18,16 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_LAZY_DIAGNOSTIC_PATH_H
-#define GCC_LAZY_DIAGNOSTIC_PATH_H
+#ifndef GCC_DIAGNOSTICS_LAZY_PATHS_H
+#define GCC_DIAGNOSTICS_LAZY_PATHS_H
 
-#include "diagnostic-path.h"
+#include "diagnostics/paths.h"
 
-/* An implementation of diagnostic_path which has a trivial ctor
-   and lazily creates another diagnostic_path the first time the path
+namespace diagnostics {
+namespace paths {
+
+/* An implementation of diagnostics::paths::path which has a trivial ctor
+   and lazily creates another path the first time the path
    is queried, deferring to this inner path for all queries.
 
    Use this to avoid expensive path creation logic when creating
@@ -32,16 +35,16 @@ along with GCC; see the file COPYING3.  If not see
    is actually used by a diagnostic, and thus avoided for warnings that
    are disabled.  */
 
-class lazy_diagnostic_path : public diagnostic_path
+class lazy_path : public path
 {
  public:
-  virtual ~lazy_diagnostic_path () {}
+  virtual ~lazy_path () {}
 
   unsigned num_events () const final override;
-  const diagnostic_event & get_event (int idx) const final override;
+  const event & get_event (int idx) const final override;
   unsigned num_threads () const final override;
-  const diagnostic_thread &
-  get_thread (diagnostic_thread_id_t) const final override;
+  const thread &
+  get_thread (thread_id_t) const final override;
   bool
   same_function_p (int event_idx_a,
 		   int event_idx_b) const final override;
@@ -49,16 +52,19 @@ class lazy_diagnostic_path : public diagnostic_path
   bool generated_p () const { return m_inner_path != nullptr; }
 
 protected:
-  lazy_diagnostic_path (const logical_location_manager &logical_loc_mgr)
-  : diagnostic_path (logical_loc_mgr)
+  lazy_path (const logical_locations::manager &logical_loc_mgr)
+  : path (logical_loc_mgr)
   {
   }
 
  private:
   void lazily_generate_path () const;
-  virtual std::unique_ptr<diagnostic_path> make_inner_path () const = 0;
+  virtual std::unique_ptr<path> make_inner_path () const = 0;
 
-  mutable std::unique_ptr<diagnostic_path> m_inner_path;
+  mutable std::unique_ptr<path> m_inner_path;
 };
 
-#endif /* ! GCC_LAZY_DIAGNOSTIC_PATH_H */
+} // namespace paths
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_LAZY_PATHS_H */
diff --git a/gcc/logical-location.h b/gcc/diagnostics/logical-locations.h
index 3522934..b52a9b4 100644
--- a/gcc/logical-location.h
+++ b/gcc/diagnostics/logical-locations.h
@@ -18,18 +18,21 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_LOGICAL_LOCATION_H
-#define GCC_LOGICAL_LOCATION_H
+#ifndef GCC_DIAGNOSTICS_LOGICAL_LOCATIONS_H
+#define GCC_DIAGNOSTICS_LOGICAL_LOCATIONS_H
 
 #include "label-text.h"
 
+namespace diagnostics {
+namespace logical_locations {
+
 /* An enum for discriminating between different kinds of logical location
    for a diagnostic.
 
    Roughly corresponds to logicalLocation's "kind" property in SARIF v2.1.0
    (section 3.33.7).  */
 
-enum class logical_location_kind
+enum class kind
 {
   unknown,
 
@@ -68,78 +71,84 @@ enum class logical_location_kind
    libgdiagnostics internals), and without requiring heap allocation of an
    interface class when emitting a diagnostic.
 
-   To do this, we split the implementation into logical_location, which is
-   a wrapper around a (const void *), and logical_location_manager which
+   To do this, we split the implementation into logical_locations::key, which is
+   a wrapper around a (const void *), and logical_locations::manager which
    is provided by the client and has vfunc hooks for interpreting
-   logical_location instances.
+   key instances.
 
-   Every logical_location is associated with a logical_location_manager and
-   only has meaning in relation to that manager.
+   Every logical_locations::key is associated with a logical_locations::manager
+   and only has meaning in relation to that manager.
 
-   A "nullptr" within a logical_location means "no logical location".
+   A "nullptr" within a key means "no logical location".
 
    See tree-logical-location.h for concrete subclasses relating to trees,
    where the pointer is a const_tree.
 
-   See selftest-logical-location.h for a concrete subclass for selftests.  */
+   See diagnostics/selftest-logical-locations.h for a concrete subclass for
+   selftests.  */
 
-/* Abstract base class for giving meaning to logical_location values.
-   Typically there will just be one client-provided instance, of a
-   client-specific subclass.  */
+/* Extrinsic state for identifying a specific logical location.
+   This will be our logical location type.
+   This only makes sense with respect to a specific manager.
+   e.g. for a tree-based one it's a wrapper around "tree".
+
+   "nullptr" means "no logical location".
+
+   Note that there is no integration with GCC's garbage collector and thus
+   keys can't be long-lived.  */
 
-class logical_location_manager
+class key
 {
 public:
-  /* Extrinsic state for identifying a specific logical location.
-     This will be our logical_location type.
-     This only makes sense with respect to a specific manager.
-     e.g. for a tree-based one it's a wrapper around "tree".
-     "nullptr" means "no logical location".  */
-  class key
+  key () : m_ptr (nullptr) {}
+
+  static key from_ptr (const void *ptr)
   {
-  public:
-    key () : m_ptr (nullptr) {}
+    return key (ptr);
+  }
 
-    static key from_ptr (const void *ptr)
-    {
-      return key (ptr);
-    }
+  operator bool () const
+  {
+    return m_ptr != nullptr;
+  }
 
-    operator bool () const
-    {
-      return m_ptr != nullptr;
-    }
+  template <typename T>
+  T cast_to () const { return static_cast<T> (m_ptr); }
 
-    template <typename T>
-    T cast_to () const { return static_cast<T> (m_ptr); }
+  bool
+  operator== (const key &other) const
+  {
+    return m_ptr == other.m_ptr;
+  }
 
-    bool
-    operator== (const key &other) const
-    {
-      return m_ptr == other.m_ptr;
-    }
+  bool
+  operator!= (const key &other) const
+  {
+    return m_ptr != other.m_ptr;
+  }
 
-    bool
-    operator!= (const key &other) const
-    {
-      return m_ptr != other.m_ptr;
-    }
+  bool
+  operator< (const key &other) const
+  {
+    return m_ptr < other.m_ptr;
+  }
 
-    bool
-    operator< (const key &other) const
-    {
-      return m_ptr < other.m_ptr;
-    }
+private:
+  explicit key (const void *ptr) : m_ptr (ptr) {}
 
-  private:
-    explicit key (const void *ptr) : m_ptr (ptr) {}
+  const void *m_ptr;
+};
 
-    const void *m_ptr;
-  };
+/* Abstract base class for giving meaning to keys.
+   Typically there will just be one client-provided instance, of a
+   client-specific subclass.  */
 
-  virtual ~logical_location_manager () {}
+class manager
+{
+public:
+  virtual ~manager () {}
 
-  /* vfuncs for interpreting logical_location values.  */
+  /* vfuncs for interpreting keys.  */
 
   /* Get a string (or NULL) for K suitable for use by the SARIF logicalLocation
      "name" property (SARIF v2.1.0 section 3.33.4).  */
@@ -154,7 +163,7 @@ public:
   virtual const char *get_internal_name (key k) const = 0;
 
   /* Get what kind of SARIF logicalLocation K is (if any).  */
-  virtual enum logical_location_kind get_kind (key k) const = 0;
+  virtual enum kind get_kind (key k) const = 0;
 
   /* Get a string for location K in a form suitable for path output.  */
   virtual label_text get_name_for_path_output (key k) const = 0;
@@ -165,11 +174,7 @@ public:
   bool function_p (key k) const;
 };
 
-/* A logical location is a key for a given logical_location_manager.
-
-   Note that there is no integration with GCC's garbage collector and thus
-   logical_location instances can't be long-lived.  */
-
-typedef logical_location_manager::key logical_location;
+} // namespace diagnostics::logical_locations
+} // namespace diagnostics
 
-#endif /* GCC_LOGICAL_LOCATION_H.  */
+#endif /* GCC_DIAGNOSTICS_LOGICAL_LOCATIONS_H.  */
diff --git a/gcc/diagnostic-macro-unwinding.cc b/gcc/diagnostics/macro-unwinding.cc
index aa1b816..66bad1c 100644
--- a/gcc/diagnostic-macro-unwinding.cc
+++ b/gcc/diagnostics/macro-unwinding.cc
@@ -22,10 +22,12 @@ along with GCC; see the file COPYING3.  If not see
 #include "coretypes.h"
 #include "tree.h"
 #include "diagnostic.h"
-#include "diagnostic-macro-unwinding.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/macro-unwinding.h"
+#include "diagnostics/text-sink.h"
 #include "intl.h"
 
+namespace diagnostics {
+
 /* This is a pair made of a location and the line map it originated
    from.  It's used in the maybe_unwind_expanded_macro_loc function
    below.  */
@@ -73,7 +75,7 @@ struct loc_map_pair
    function.  */
 
 void
-maybe_unwind_expanded_macro_loc (diagnostic_text_output_format &text_output,
+maybe_unwind_expanded_macro_loc (text_sink &text_output,
                                  location_t where)
 {
   const struct line_map *map;
@@ -215,8 +217,11 @@ maybe_unwind_expanded_macro_loc (diagnostic_text_output_format &text_output,
     that is similar to what is done for function call stacks, or
     template instantiation contexts.  */
 void
-virt_loc_aware_diagnostic_finalizer (diagnostic_text_output_format &text_output,
-				     const diagnostic_info *diagnostic)
+virt_loc_aware_text_finalizer (text_sink &text_output,
+			       const diagnostic_info *diagnostic)
 {
-  maybe_unwind_expanded_macro_loc (text_output, diagnostic_location (diagnostic));
+  maybe_unwind_expanded_macro_loc (text_output,
+				   diagnostic_location (diagnostic));
 }
+
+} // namespace diagnostics
diff --git a/gcc/diagnostic-macro-unwinding.h b/gcc/diagnostics/macro-unwinding.h
index ef1e55d..1f28d58 100644
--- a/gcc/diagnostic-macro-unwinding.h
+++ b/gcc/diagnostics/macro-unwinding.h
@@ -17,13 +17,17 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_MACRO_UNWINDING_H
-#define GCC_DIAGNOSTIC_MACRO_UNWINDING_H
+#ifndef GCC_DIAGNOSTICS_MACRO_UNWINDING_H
+#define GCC_DIAGNOSTICS_MACRO_UNWINDING_H
 
-void virt_loc_aware_diagnostic_finalizer (diagnostic_text_output_format &,
-					  const diagnostic_info *);
+namespace diagnostics {
 
-extern void maybe_unwind_expanded_macro_loc (diagnostic_text_output_format &,
+extern void virt_loc_aware_text_finalizer (text_sink &,
+					   const diagnostic_info *);
+
+extern void maybe_unwind_expanded_macro_loc (text_sink &,
 					     location_t where);
 
-#endif /* ! GCC_DIAGNOSTIC_MACRO_UNWINDING_H */
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_MACRO_UNWINDING_H */
diff --git a/gcc/diagnostic-metadata.h b/gcc/diagnostics/metadata.h
index baeeb6e..c28f982 100644
--- a/gcc/diagnostic-metadata.h
+++ b/gcc/diagnostics/metadata.h
@@ -18,20 +18,31 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_METADATA_H
-#define GCC_DIAGNOSTIC_METADATA_H
+#ifndef GCC_DIAGNOSTICS_METADATA_H
+#define GCC_DIAGNOSTICS_METADATA_H
 
-class sarif_object;
+#include "lazily-created.h"
+
+namespace diagnostics {
+
+  class sarif_object;
+  namespace digraphs { class digraph; }
 
 /* A bundle of additional metadata that can be associated with a
    diagnostic.
 
    This supports an optional CWE identifier, and zero or more
-   "rules".  */
+   "rules".
+
+   Additionally, this provides a place to associate a diagnostic
+   with zero or more directed graphs.  */
 
-class diagnostic_metadata
+class metadata
 {
  public:
+  using lazy_digraphs
+  = lazily_created<std::vector<std::unique_ptr<digraphs::digraph>>>;
+
   /* Abstract base class for referencing a rule that has been violated,
      such as within a coding standard, or within a specification.  */
   class rule
@@ -64,8 +75,8 @@ class diagnostic_metadata
     const char *m_url;
   };
 
-  diagnostic_metadata () : m_cwe (0) {}
-  virtual ~diagnostic_metadata () {}
+  metadata () : m_cwe (0), m_lazy_digraphs (nullptr) {}
+  virtual ~metadata () {}
 
   /* Hook for SARIF output to allow for adding diagnostic-specific
      properties to  the result object's property bag.  */
@@ -87,9 +98,27 @@ class diagnostic_metadata
   unsigned get_num_rules () const { return m_rules.length (); }
   const rule &get_rule (unsigned idx) const { return *(m_rules[idx]); }
 
+  void
+  set_lazy_digraphs (const lazy_digraphs *lazy_digraphs_)
+  {
+    m_lazy_digraphs = lazy_digraphs_;
+  }
+
+  const lazy_digraphs *
+  get_lazy_digraphs () const
+  {
+    return m_lazy_digraphs;
+  }
+
  private:
   int m_cwe;
   auto_vec<const rule *> m_rules;
+
+  /* An optional way to create directed graphs associated with the
+     diagnostic, for the sinks that support this (e.g. SARIF).  */
+  const lazy_digraphs *m_lazy_digraphs;
 };
 
-#endif /* ! GCC_DIAGNOSTIC_METADATA_H */
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_METADATA_H */
diff --git a/gcc/diagnostics/option-classifier.cc b/gcc/diagnostics/option-classifier.cc
new file mode 100644
index 0000000..f98fd55
--- /dev/null
+++ b/gcc/diagnostics/option-classifier.cc
@@ -0,0 +1,222 @@
+/* Stacks of set of classifications of diagnostics.
+   Copyright (C) 1999-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "version.h"
+#include "diagnostic.h"
+
+namespace diagnostics {
+
+void
+option_classifier::init (int n_opts)
+{
+  m_n_opts = n_opts;
+  m_classify_diagnostic = XNEWVEC (enum kind, n_opts);
+  for (int i = 0; i < n_opts; i++)
+    m_classify_diagnostic[i] = kind::unspecified;
+  m_push_list = vNULL;
+  m_classification_history = vNULL;
+}
+
+void
+option_classifier::fini ()
+{
+  XDELETEVEC (m_classify_diagnostic);
+  m_classify_diagnostic = nullptr;
+  m_classification_history.release ();
+  m_push_list.release ();
+}
+
+/* Save the diagnostics::option_classifier state to F for PCH
+   output.  Returns 0 on success, -1 on error.  */
+
+int
+option_classifier::pch_save (FILE *f)
+{
+  unsigned int lengths[2] = { m_classification_history.length (),
+			      m_push_list.length () };
+  if (fwrite (lengths, sizeof (lengths), 1, f) != 1
+      || (lengths[0]
+	  && fwrite (m_classification_history.address (),
+		     sizeof (classification_change_t),
+		     lengths[0], f) != lengths[0])
+      || (lengths[1]
+	  && fwrite (m_push_list.address (), sizeof (int),
+		     lengths[1], f) != lengths[1]))
+    return -1;
+  return 0;
+}
+
+/* Read the diagnostics::option_classifier state from F for PCH
+   read.  Returns 0 on success, -1 on error.  */
+
+int
+option_classifier::pch_restore (FILE *f)
+{
+  unsigned int lengths[2];
+  if (fread (lengths, sizeof (lengths), 1, f) != 1)
+    return -1;
+  gcc_checking_assert (m_classification_history.is_empty ());
+  gcc_checking_assert (m_push_list.is_empty ());
+  m_classification_history.safe_grow (lengths[0]);
+  m_push_list.safe_grow (lengths[1]);
+  if ((lengths[0]
+       && fread (m_classification_history.address (),
+		 sizeof (classification_change_t),
+		 lengths[0], f) != lengths[0])
+      || (lengths[1]
+	  && fread (m_push_list.address (), sizeof (int),
+		    lengths[1], f) != lengths[1]))
+    return -1;
+  return 0;
+}
+
+/* Save all diagnostic classifications in a stack.  */
+
+void
+option_classifier::push ()
+{
+  m_push_list.safe_push (m_classification_history.length ());
+}
+
+/* Restore the topmost classification set off the stack.  If the stack
+   is empty, revert to the state based on command line parameters.  */
+
+void
+option_classifier::pop (location_t where)
+{
+  int jump_to;
+
+  if (!m_push_list.is_empty ())
+    jump_to = m_push_list.pop ();
+  else
+    jump_to = 0;
+
+  classification_change_t v = { where, jump_to, kind::pop };
+  m_classification_history.safe_push (v);
+}
+
+/* Interface to specify diagnostic kind overrides.  Returns the
+   previous setting, or kind::unspecified if the parameters are out of
+   range.  If OPTION_ID is zero, the new setting is for all the
+   diagnostics.  */
+
+enum kind
+option_classifier::classify_diagnostic (const context *dc,
+					option_id opt_id,
+					enum kind new_kind,
+					location_t where)
+{
+  enum kind old_kind;
+
+  if (opt_id.m_idx < 0
+      || opt_id.m_idx >= m_n_opts
+      || new_kind >= kind::last_diagnostic_kind)
+    return kind::unspecified;
+
+  old_kind = m_classify_diagnostic[opt_id.m_idx];
+
+  /* Handle pragmas separately, since we need to keep track of *where*
+     the pragmas were.  */
+  if (where != UNKNOWN_LOCATION)
+    {
+      unsigned i;
+
+      /* Record the command-line status, so we can reset it back on kind::pop. */
+      if (old_kind == kind::unspecified)
+	{
+	  old_kind = (!dc->option_enabled_p (opt_id)
+		      ? kind::ignored : kind::any);
+	  m_classify_diagnostic[opt_id.m_idx] = old_kind;
+	}
+
+      classification_change_t *p;
+      FOR_EACH_VEC_ELT_REVERSE (m_classification_history, i, p)
+	if (p->option == opt_id.m_idx)
+	  {
+	    old_kind = p->kind;
+	    break;
+	  }
+
+      classification_change_t v
+	= { where, opt_id.m_idx, new_kind };
+      m_classification_history.safe_push (v);
+    }
+  else
+    m_classify_diagnostic[opt_id.m_idx] = new_kind;
+
+  return old_kind;
+}
+
+/* Update the kind of DIAGNOSTIC based on its location(s), including
+   any of those in its inlining stack, relative to any
+     #pragma GCC diagnostic
+   directives recorded within this object.
+
+   Return the new kind of DIAGNOSTIC if it was updated, or kind::unspecified
+   otherwise.  */
+
+enum kind
+option_classifier::
+update_effective_level_from_pragmas (diagnostic_info *diagnostic) const
+{
+  if (m_classification_history.is_empty ())
+    return kind::unspecified;
+
+  /* Iterate over the locations, checking the diagnostic disposition
+     for the diagnostic at each.  If it's explicitly set as opposed
+     to unspecified, update the disposition for this instance of
+     the diagnostic and return it.  */
+  for (location_t loc: diagnostic->m_iinfo.m_ilocs)
+    {
+      /* FIXME: Stupid search.  Optimize later. */
+      unsigned int i;
+      classification_change_t *p;
+      FOR_EACH_VEC_ELT_REVERSE (m_classification_history, i, p)
+	{
+	  location_t pragloc = p->location;
+	  if (!linemap_location_before_p (line_table, pragloc, loc))
+	    continue;
+
+	  if (p->kind == kind::pop)
+	    {
+	      /* Move on to the next region.  */
+	      i = p->option;
+	      continue;
+	    }
+
+	  option_id opt_id = p->option;
+	  /* The option 0 is for all the diagnostics.  */
+	  if (opt_id == 0 || opt_id == diagnostic->m_option_id)
+	    {
+	      enum kind kind = p->kind;
+	      if (kind != diagnostics::kind::unspecified)
+		diagnostic->m_kind = kind;
+	      return kind;
+	    }
+	}
+    }
+
+  return kind::unspecified;
+}
+
+} // namespace diagnostics
diff --git a/gcc/diagnostics/option-classifier.h b/gcc/diagnostics/option-classifier.h
new file mode 100644
index 0000000..3b16c74
--- /dev/null
+++ b/gcc/diagnostics/option-classifier.h
@@ -0,0 +1,110 @@
+/* Stacks of set of classifications of diagnostics.
+   Copyright (C) 2000-2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_OPTION_CLASSIFIER_H
+#define GCC_DIAGNOSTICS_OPTION_CLASSIFIER_H
+
+namespace diagnostics {
+
+/*  Forward declarations.  */
+class context;
+
+/* A stack of sets of classifications: each entry in the stack is
+   a mapping from option index to diagnostic severity that can be changed
+   via pragmas.  The stack can be pushed and popped.  */
+
+class option_classifier
+{
+public:
+  void init (int n_opts);
+  void fini ();
+
+  /* Save all diagnostic classifications in a stack.  */
+  void push ();
+
+  /* Restore the topmost classification set off the stack.  If the stack
+     is empty, revert to the state based on command line parameters.  */
+  void pop (location_t where);
+
+  bool option_unspecified_p (option_id opt_id) const
+  {
+    return get_current_override (opt_id) == kind::unspecified;
+  }
+
+  enum kind get_current_override (option_id opt_id) const
+  {
+    gcc_assert (opt_id.m_idx < m_n_opts);
+    return m_classify_diagnostic[opt_id.m_idx];
+  }
+
+  enum kind
+  classify_diagnostic (const context *context,
+		       option_id opt_id,
+		       enum kind new_kind,
+		       location_t where);
+
+  enum kind
+  update_effective_level_from_pragmas (diagnostic_info *diagnostic) const;
+
+  int pch_save (FILE *);
+  int pch_restore (FILE *);
+
+private:
+  /* Each time a diagnostic's classification is changed with a pragma,
+     we record the change and the location of the change in an array of
+     these structs.  */
+  struct classification_change_t
+  {
+    location_t location;
+
+    /* For kind::pop, this is the index of the corresponding push (as stored
+       in m_push_list).
+       Otherwise, this is an option index.  */
+    int option;
+
+    enum kind kind;
+  };
+
+  int m_n_opts;
+
+  /* For each option index that can be passed to warning() et al
+     (OPT_* from options.h when using this code with the core GCC
+     options), this array may contain a new kind that the diagnostic
+     should be changed to before reporting, or kind::unspecified to leave
+     it as the reported kind, or kind::ignored to not report it at
+     all.  */
+  enum kind *m_classify_diagnostic;
+
+  /* History of all changes to the classifications above.  This list
+     is stored in location-order, so we can search it, either
+     binary-wise or end-to-front, to find the most recent
+     classification for a given diagnostic, given the location of the
+     diagnostic.  */
+  vec<classification_change_t> m_classification_history;
+
+  /* For context::get_classification_history, declared later.  */
+  friend class context;
+
+  /* For pragma push/pop.  */
+  vec<int> m_push_list;
+};
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_OPTION_CLASSIFIER_H */
diff --git a/gcc/diagnostics/option-id.h b/gcc/diagnostics/option-id.h
new file mode 100644
index 0000000..4132775
--- /dev/null
+++ b/gcc/diagnostics/option-id.h
@@ -0,0 +1,49 @@
+/* Declaration of struct diagnostics::option_id.
+   Copyright (C) 2024-2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_OPTION_ID_H
+#define GCC_DIAGNOSTICS_OPTION_ID_H
+
+namespace diagnostics {
+
+/* A class to use for the ID of an option that controls
+   a particular diagnostic.
+   This is just a wrapper around "int", but better documents
+   the intent of the code.  */
+
+struct option_id
+{
+  option_id () : m_idx (0) {}
+
+  option_id (int idx) : m_idx (idx) {}
+  /* Ideally we'd take an enum opt_code here, but we don't
+     want to depend on its decl.  */
+
+  bool operator== (option_id other) const
+  {
+    return m_idx == other.m_idx;
+  }
+
+  int m_idx;
+};
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_OPTION_ID_H */
diff --git a/gcc/diagnostic-output-file.h b/gcc/diagnostics/output-file.h
index a0b2e1b..f936387 100644
--- a/gcc/diagnostic-output-file.h
+++ b/gcc/diagnostics/output-file.h
@@ -18,22 +18,24 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_OUTPUT_FILE_H
-#define GCC_DIAGNOSTIC_OUTPUT_FILE_H
+#ifndef GCC_DIAGNOSTICS_OUTPUT_FILE_H
+#define GCC_DIAGNOSTICS_OUTPUT_FILE_H
+
+namespace diagnostics {
 
 /* RAII class for wrapping a FILE * that could be borrowed or owned,
    along with the underlying filename.  */
 
-class diagnostic_output_file
+class output_file
 {
 public:
-  diagnostic_output_file ()
+  output_file ()
   : m_outf (nullptr),
     m_owned (false),
     m_filename ()
   {
   }
-  diagnostic_output_file (FILE *outf, bool owned, label_text filename)
+  output_file (FILE *outf, bool owned, label_text filename)
   : m_outf (outf),
     m_owned (owned),
     m_filename (std::move (filename))
@@ -42,7 +44,7 @@ public:
     if (m_owned)
       gcc_assert (m_outf);
   }
-  ~diagnostic_output_file ()
+  ~output_file ()
   {
     if (m_owned)
       {
@@ -50,8 +52,8 @@ public:
 	fclose (m_outf);
       }
   }
-  diagnostic_output_file (const diagnostic_output_file &other) = delete;
-  diagnostic_output_file (diagnostic_output_file &&other)
+  output_file (const output_file &other) = delete;
+  output_file (output_file &&other)
   : m_outf (other.m_outf),
     m_owned (other.m_owned),
     m_filename (std::move (other.m_filename))
@@ -63,10 +65,10 @@ public:
     if (m_owned)
       gcc_assert (m_outf);
   }
-  diagnostic_output_file &
-  operator= (const diagnostic_output_file &other) = delete;
-  diagnostic_output_file &
-  operator= (diagnostic_output_file &&other)
+  output_file &
+  operator= (const output_file &other) = delete;
+  output_file &
+  operator= (output_file &&other)
   {
     if (m_owned)
       {
@@ -91,8 +93,8 @@ public:
   FILE *get_open_file () const { return m_outf; }
   const char *get_filename () const { return m_filename.get (); }
 
-  static diagnostic_output_file
-  try_to_open (diagnostic_context &context,
+  static output_file
+  try_to_open (context &dc,
 	       line_maps *line_maps,
 	       const char *base_file_name,
 	       const char *extension,
@@ -104,4 +106,6 @@ private:
   label_text m_filename;
 };
 
-#endif /* ! GCC_DIAGNOSTIC_OUTPUT_FILE_H */
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_OUTPUT_FILE_H */
diff --git a/gcc/diagnostic-output-spec.cc b/gcc/diagnostics/output-spec.cc
index 25ef86f..83f128c 100644
--- a/gcc/diagnostic-output-spec.cc
+++ b/gcc/diagnostics/output-spec.cc
@@ -32,20 +32,21 @@ along with GCC; see the file COPYING3.  If not see
 #include "version.h"
 #include "intl.h"
 #include "diagnostic.h"
-#include "diagnostic-color.h"
-#include "diagnostic-format.h"
-#include "diagnostic-format-html.h"
-#include "diagnostic-format-text.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/color.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/html-sink.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/sarif-sink.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
+#include "diagnostics/selftest-context.h"
 #include "pretty-print-markup.h"
-#include "diagnostic-output-spec.h"
+#include "diagnostics/output-spec.h"
 
 /* A namespace for handling the DSL of the arguments of
    -fdiagnostics-add-output= and -fdiagnostics-set-output=.  */
 
-namespace diagnostics_output_spec {
+namespace diagnostics {
+namespace output_spec {
 
 /* Decls.  */
 
@@ -56,7 +57,7 @@ struct scheme_name_and_params
 };
 
 /* Class for parsing the arguments of -fdiagnostics-add-output= and
-   -fdiagnostics-set-output=, and making diagnostic_output_format
+   -fdiagnostics-set-output=, and making sink
    instances (or issuing errors).  */
 
 class output_factory
@@ -72,9 +73,9 @@ public:
 
     const std::string &get_scheme_name () const { return m_scheme_name; }
 
-    virtual std::unique_ptr<diagnostic_output_format>
+    virtual std::unique_ptr<sink>
     make_sink (const context &ctxt,
-	       diagnostic_context &dc,
+	       diagnostics::context &dc,
 	       const char *unparsed_arg,
 	       const scheme_name_and_params &parsed_arg) const = 0;
 
@@ -145,9 +146,9 @@ public:
 
   output_factory ();
 
-  std::unique_ptr<diagnostic_output_format>
+  std::unique_ptr<sink>
   make_sink (const context &ctxt,
-	     diagnostic_context &dc,
+	     diagnostics::context &dc,
 	     const char *unparsed_arg,
 	     const scheme_name_and_params &parsed_arg);
 
@@ -162,9 +163,9 @@ class text_scheme_handler : public output_factory::scheme_handler
 public:
   text_scheme_handler () : scheme_handler ("text") {}
 
-  std::unique_ptr<diagnostic_output_format>
+  std::unique_ptr<sink>
   make_sink (const context &ctxt,
-	     diagnostic_context &dc,
+	     diagnostics::context &dc,
 	     const char *unparsed_arg,
 	     const scheme_name_and_params &parsed_arg) const final override;
 };
@@ -174,16 +175,16 @@ class sarif_scheme_handler : public output_factory::scheme_handler
 public:
   sarif_scheme_handler () : scheme_handler ("sarif") {}
 
-  std::unique_ptr<diagnostic_output_format>
+  std::unique_ptr<sink>
   make_sink (const context &ctxt,
-	     diagnostic_context &dc,
+	     diagnostics::context &dc,
 	     const char *unparsed_arg,
 	     const scheme_name_and_params &parsed_arg) const final override;
 
 private:
   static sarif_generation_options
   make_sarif_gen_opts (enum sarif_version version,
-		       bool xml_state);
+		       bool state_graph);
 
   static std::unique_ptr<sarif_serialization_format>
   make_sarif_serialization_object (enum sarif_serialization_kind);
@@ -194,9 +195,9 @@ class html_scheme_handler : public output_factory::scheme_handler
 public:
   html_scheme_handler () : scheme_handler ("experimental-html") {}
 
-  std::unique_ptr<diagnostic_output_format>
+  std::unique_ptr<sink>
   make_sink (const context &ctxt,
-	     diagnostic_context &dc,
+	     diagnostics::context &dc,
 	     const char *unparsed_arg,
 	     const scheme_name_and_params &parsed_arg) const final override;
 };
@@ -241,16 +242,16 @@ context::report_missing_key (const char *unparsed_arg,
      get_option_name (), scheme_name.c_str (), key.c_str (), metavar);
 }
 
-diagnostic_output_file
+output_file
 context::open_output_file (label_text &&filename) const
 {
   FILE *outf = fopen (filename.get (), "w");
   if (!outf)
     {
       report_error ("unable to open %qs: %m", filename.get ());
-      return diagnostic_output_file (nullptr, false, std::move (filename));
+      return output_file (nullptr, false, std::move (filename));
     }
-  return diagnostic_output_file (outf, true, std::move (filename));
+  return output_file (outf, true, std::move (filename));
 }
 
 static std::unique_ptr<scheme_name_and_params>
@@ -303,15 +304,15 @@ parse (const context &ctxt, const char *unparsed_arg)
   return std::make_unique<scheme_name_and_params> (std::move (result));
 }
 
-std::unique_ptr<diagnostic_output_format>
+std::unique_ptr<sink>
 context::parse_and_make_sink (const char *unparsed_arg,
-			      diagnostic_context &dc)
+			      diagnostics::context &dc)
 {
-  auto parsed_arg = diagnostics_output_spec::parse (*this, unparsed_arg);
+  auto parsed_arg = parse (*this, unparsed_arg);
   if (!parsed_arg)
     return nullptr;
 
-  diagnostics_output_spec::output_factory factory;
+  output_factory factory;
   return factory.make_sink (*this, dc, unparsed_arg, *parsed_arg);
 }
 
@@ -335,9 +336,9 @@ output_factory::get_scheme_handler (const std::string &scheme_name)
   return nullptr;
 }
 
-std::unique_ptr<diagnostic_output_format>
+std::unique_ptr<sink>
 output_factory::make_sink (const context &ctxt,
-			   diagnostic_context &dc,
+			   diagnostics::context &dc,
 			   const char *unparsed_arg,
 			   const scheme_name_and_params &parsed_arg)
 {
@@ -360,9 +361,9 @@ output_factory::make_sink (const context &ctxt,
 
 /* class text_scheme_handler : public output_factory::scheme_handler.  */
 
-std::unique_ptr<diagnostic_output_format>
+std::unique_ptr<sink>
 text_scheme_handler::make_sink (const context &ctxt,
-				diagnostic_context &dc,
+				diagnostics::context &dc,
 				const char *unparsed_arg,
 				const scheme_name_and_params &parsed_arg) const
 {
@@ -412,7 +413,7 @@ text_scheme_handler::make_sink (const context &ctxt,
       return nullptr;
     }
 
-  auto sink = std::make_unique<diagnostic_text_output_format> (dc);
+  auto sink = std::make_unique<diagnostics::text_sink> (dc);
   sink->set_show_nesting (show_nesting);
   sink->set_show_locations_in_nesting (show_locations_in_nesting);
   sink->set_show_nesting_levels (show_levels);
@@ -421,9 +422,9 @@ text_scheme_handler::make_sink (const context &ctxt,
 
 /* class sarif_scheme_handler : public output_factory::scheme_handler.  */
 
-std::unique_ptr<diagnostic_output_format>
+std::unique_ptr<sink>
 sarif_scheme_handler::make_sink (const context &ctxt,
-				 diagnostic_context &dc,
+				 diagnostics::context &dc,
 				 const char *unparsed_arg,
 				 const scheme_name_and_params &parsed_arg) const
 {
@@ -431,7 +432,7 @@ sarif_scheme_handler::make_sink (const context &ctxt,
   enum sarif_serialization_kind serialization_kind
     = sarif_serialization_kind::json;
   enum sarif_version version = sarif_version::v2_1_0;
-  bool xml_state = false;
+  bool state_graph = false;
   for (auto& iter : parsed_arg.m_kvs)
     {
       const std::string &key = iter.first;
@@ -469,10 +470,10 @@ sarif_scheme_handler::make_sink (const context &ctxt,
 	    return nullptr;
 	  continue;
 	}
-      if (key == "xml-state")
+      if (key == "state-graphs")
 	{
 	  if (!parse_bool_value (ctxt, unparsed_arg, key, value,
-				 xml_state))
+				 state_graph))
 	    return nullptr;
 	  continue;
 	}
@@ -481,16 +482,16 @@ sarif_scheme_handler::make_sink (const context &ctxt,
       auto_vec<const char *> known_keys;
       known_keys.safe_push ("file");
       known_keys.safe_push ("serialization");
+      known_keys.safe_push ("state-graphs");
       known_keys.safe_push ("version");
-      known_keys.safe_push ("xml-state");
       ctxt.report_unknown_key (unparsed_arg, key, get_scheme_name (),
 			       known_keys);
       return nullptr;
     }
 
-  diagnostic_output_file output_file;
+  output_file output_file_;
   if (filename.get ())
-    output_file = ctxt.open_output_file (std::move (filename));
+    output_file_ = ctxt.open_output_file (std::move (filename));
   else
     // Default filename
     {
@@ -503,17 +504,16 @@ sarif_scheme_handler::make_sink (const context &ctxt,
 				   "FILENAME");
 	  return nullptr;
 	}
-      output_file
-	= diagnostic_output_format_open_sarif_file
-	    (dc,
-	     ctxt.get_affected_location_mgr (),
-	     basename,
-	     serialization_kind);
+      output_file_
+	= open_sarif_output_file (dc,
+				  ctxt.get_affected_location_mgr (),
+				  basename,
+				  serialization_kind);
     }
-  if (!output_file)
+  if (!output_file_)
     return nullptr;
 
-  auto sarif_gen_opts = make_sarif_gen_opts (version, xml_state);
+  auto sarif_gen_opts = make_sarif_gen_opts (version, state_graph);
 
   auto serialization_obj = make_sarif_serialization_object (serialization_kind);
 
@@ -521,17 +521,17 @@ sarif_scheme_handler::make_sink (const context &ctxt,
 			       *ctxt.get_affected_location_mgr (),
 			       std::move (serialization_obj),
 			       sarif_gen_opts,
-			       std::move (output_file));
+			       std::move (output_file_));
   return sink;
 }
 
 sarif_generation_options
 sarif_scheme_handler::make_sarif_gen_opts (enum sarif_version version,
-					   bool xml_state)
+					   bool state_graph)
 {
   sarif_generation_options sarif_gen_opts;
   sarif_gen_opts.m_version = version;
-  sarif_gen_opts.m_xml_state = xml_state;
+  sarif_gen_opts.m_state_graph = state_graph;
   return sarif_gen_opts;
 }
 
@@ -551,9 +551,9 @@ make_sarif_serialization_object (enum sarif_serialization_kind kind)
 
 /* class html_scheme_handler : public output_factory::scheme_handler.  */
 
-std::unique_ptr<diagnostic_output_format>
+std::unique_ptr<sink>
 html_scheme_handler::make_sink (const context &ctxt,
-				diagnostic_context &dc,
+				diagnostics::context &dc,
 				const char *unparsed_arg,
 				const scheme_name_and_params &parsed_arg) const
 {
@@ -561,8 +561,8 @@ html_scheme_handler::make_sink (const context &ctxt,
   label_text filename;
   bool javascript = true;
   bool show_state_diagrams = false;
-  bool show_state_diagram_xml = false;
-  bool show_state_diagram_dot_src = false;
+  bool show_state_diagrams_sarif = false;
+  bool show_state_diagrams_dot_src = false;
   for (auto& iter : parsed_arg.m_kvs)
     {
       const std::string &key = iter.first;
@@ -593,17 +593,17 @@ html_scheme_handler::make_sink (const context &ctxt,
 	    return nullptr;
 	  continue;
 	}
-      if (key == "show-state-diagram-dot-src")
+      if (key == "show-state-diagrams-dot-src")
 	{
 	  if (!parse_bool_value (ctxt, unparsed_arg, key, value,
-				 show_state_diagram_dot_src))
+				 show_state_diagrams_dot_src))
 	    return nullptr;
 	  continue;
 	}
-      if (key == "show-state-diagram-xml")
+      if (key == "show-state-diagrams-sarif")
 	{
 	  if (!parse_bool_value (ctxt, unparsed_arg, key, value,
-				 show_state_diagram_xml))
+				 show_state_diagrams_sarif))
 	    return nullptr;
 	  continue;
 	}
@@ -615,15 +615,15 @@ html_scheme_handler::make_sink (const context &ctxt,
       known_keys.safe_push ("javascript");
       known_keys.safe_push ("show-state-diagrams");
       known_keys.safe_push ("show-state-diagram-dot-src");
-      known_keys.safe_push ("show-state-diagram-xml");
+      known_keys.safe_push ("show-state-diagram-sarif");
       ctxt.report_unknown_key (unparsed_arg, key, get_scheme_name (),
 			       known_keys);
       return nullptr;
     }
 
-  diagnostic_output_file output_file;
+  output_file output_file_;
   if (filename.get ())
-    output_file = ctxt.open_output_file (std::move (filename));
+    output_file_ = ctxt.open_output_file (std::move (filename));
   else
     // Default filename
     {
@@ -636,35 +636,37 @@ html_scheme_handler::make_sink (const context &ctxt,
 				   "FILENAME");
 	  return nullptr;
 	}
-      output_file
-	= diagnostic_output_format_open_html_file
+      output_file_
+	= open_html_output_file
 	    (dc,
 	     ctxt.get_affected_location_mgr (),
 	     basename);
     }
-  if (!output_file)
+  if (!output_file_)
     return nullptr;
 
   html_generation_options html_gen_opts;
   html_gen_opts.m_css = css;
   html_gen_opts.m_javascript = javascript;
   html_gen_opts.m_show_state_diagrams = show_state_diagrams;
-  html_gen_opts.m_show_state_diagram_xml = show_state_diagram_xml;
-  html_gen_opts.m_show_state_diagram_dot_src = show_state_diagram_dot_src;
+  html_gen_opts.m_show_state_diagrams_sarif = show_state_diagrams_sarif;
+  html_gen_opts.m_show_state_diagrams_dot_src = show_state_diagrams_dot_src;
 
   auto sink = make_html_sink (dc,
 			      *ctxt.get_affected_location_mgr (),
 			      html_gen_opts,
-			      std::move (output_file));
+			      std::move (output_file_));
   return sink;
 }
 
-} // namespace diagnostics_output_spec
+} // namespace output_spec
 
 #if CHECKING_P
 
 namespace selftest {
 
+using auto_fix_quotes = ::selftest::auto_fix_quotes;
+
 /* RAII class to temporarily override "progname" to the
    string "PROGNAME".  */
 
@@ -688,18 +690,18 @@ private:
 
 struct parser_test
 {
-  class test_spec_context : public diagnostics_output_spec::gcc_spec_context
+  class test_spec_context : public diagnostics::output_spec::dc_spec_context
   {
   public:
-    test_spec_context (diagnostic_context &dc,
+    test_spec_context (diagnostics::context &dc,
 		       line_maps *location_mgr,
 		       location_t loc,
 		       const char *option_name)
-    : gcc_spec_context (dc,
-			location_mgr,
-			location_mgr,
-			loc,
-			option_name)
+    : dc_spec_context (dc,
+		       location_mgr,
+		       location_mgr,
+		       loc,
+		       option_name)
     {
     }
 
@@ -713,15 +715,15 @@ struct parser_test
   parser_test ()
   : m_dc (),
     m_ctxt (m_dc, line_table, UNKNOWN_LOCATION, "-fOPTION="),
-    m_fmt (m_dc.get_output_format (0))
+    m_fmt (m_dc.get_sink (0))
   {
     pp_buffer (m_fmt.get_printer ())->m_flush_p = false;
   }
 
-  std::unique_ptr<diagnostics_output_spec::scheme_name_and_params>
+  std::unique_ptr<diagnostics::output_spec::scheme_name_and_params>
   parse (const char *unparsed_arg)
   {
-    return diagnostics_output_spec::parse (m_ctxt, unparsed_arg);
+    return diagnostics::output_spec::parse (m_ctxt, unparsed_arg);
   }
 
   bool execution_failed_p () const
@@ -736,9 +738,9 @@ struct parser_test
   }
 
 private:
-  test_diagnostic_context m_dc;
+  diagnostics::selftest::test_context m_dc;
   test_spec_context m_ctxt;
-  diagnostic_output_format &m_fmt;
+  diagnostics::sink &m_fmt;
 };
 
 /* Selftests.  */
@@ -838,12 +840,13 @@ test_output_arg_parsing ()
 /* Run all of the selftests within this file.  */
 
 void
-diagnostic_output_spec_cc_tests ()
+output_spec_cc_tests ()
 {
   test_output_arg_parsing ();
 }
 
-} // namespace selftest
-
+} // namespace diagnostics::selftest
 
 #endif /* #if CHECKING_P */
+
+} // namespace diagnostics
diff --git a/gcc/diagnostic-output-spec.h b/gcc/diagnostics/output-spec.h
index e02cdfe..c84d237 100644
--- a/gcc/diagnostic-output-spec.h
+++ b/gcc/diagnostics/output-spec.h
@@ -18,13 +18,14 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_OUTPUT_SPEC_H
-#define GCC_DIAGNOSTIC_OUTPUT_SPEC_H
+#ifndef GCC_DIAGNOSTICS_OUTPUT_SPEC_H
+#define GCC_DIAGNOSTICS_OUTPUT_SPEC_H
 
-#include "diagnostic-format.h"
-#include "diagnostic-output-file.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/output-file.h"
 
-namespace diagnostics_output_spec {
+namespace diagnostics {
+namespace output_spec {
 
 /* An abstract base class for handling the DSL of -fdiagnostics-add-output=
    and -fdiagnostics-set-output=.  */
@@ -32,9 +33,9 @@ namespace diagnostics_output_spec {
 class context
 {
  public:
-  std::unique_ptr<diagnostic_output_format>
+  std::unique_ptr<sink>
   parse_and_make_sink (const char *,
-		       diagnostic_context &dc);
+		       diagnostics::context &dc);
 
   void
   report_error (const char *gmsgid, ...) const
@@ -52,7 +53,7 @@ class context
 		      const std::string &scheme_name,
 		      const char *metavar) const;
 
-  diagnostic_output_file
+  output_file
   open_output_file (label_text &&filename) const;
 
   const char *
@@ -81,16 +82,16 @@ protected:
   line_maps *m_affected_location_mgr;
 };
 
-/* A subclass that implements reporting errors via a diagnostic_context.  */
+/* A subclass that implements reporting errors via a diagnostics::context.  */
 
-struct gcc_spec_context : public diagnostics_output_spec::context
+struct dc_spec_context : public output_spec::context
 {
 public:
-  gcc_spec_context (diagnostic_context &dc,
-		    line_maps *affected_location_mgr,
-		    line_maps *control_location_mgr,
-		    location_t loc,
-		    const char *option_name)
+  dc_spec_context (diagnostics::context &dc,
+		   line_maps *affected_location_mgr,
+		   line_maps *control_location_mgr,
+		   location_t loc,
+		   const char *option_name)
   : context (option_name, affected_location_mgr),
     m_dc (dc),
     m_control_location_mgr (control_location_mgr),
@@ -102,15 +103,16 @@ public:
   {
     m_dc.begin_group ();
     rich_location richloc (m_control_location_mgr, m_loc);
-    m_dc.diagnostic_impl (&richloc, nullptr, -1, gmsgid, ap, DK_ERROR);
+    m_dc.diagnostic_impl (&richloc, nullptr, -1, gmsgid, ap, kind::error);
     m_dc.end_group ();
   }
 
-  diagnostic_context &m_dc;
+  diagnostics::context &m_dc;
   line_maps *m_control_location_mgr;
   location_t m_loc;
 };
 
-} // namespace diagnostics_output_spec
+} // namespace output_spec
+} // namespace diagnostics
 
-#endif
+#endif // #ifndef GCC_DIAGNOSTICS_OUTPUT_SPEC_H
diff --git a/gcc/diagnostic-path-output.cc b/gcc/diagnostics/paths-output.cc
index 439fd33..a3ac9a0 100644
--- a/gcc/diagnostic-path-output.cc
+++ b/gcc/diagnostics/paths-output.cc
@@ -26,20 +26,20 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "diagnostic.h"
-#include "diagnostic-macro-unwinding.h"
+#include "diagnostics/macro-unwinding.h"
 #include "intl.h"
-#include "diagnostic-path.h"
+#include "diagnostics/paths.h"
 #include "gcc-rich-location.h"
-#include "diagnostic-color.h"
-#include "diagnostic-event-id.h"
-#include "diagnostic-label-effects.h"
+#include "diagnostics/color.h"
+#include "diagnostics/event-id.h"
+#include "diagnostics/source-printing-effects.h"
 #include "pretty-print-markup.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
-#include "selftest-diagnostic-path.h"
+#include "diagnostics/selftest-context.h"
+#include "diagnostics/selftest-paths.h"
 #include "text-art/theme.h"
-#include "diagnostic-format-text.h"
-#include "diagnostic-format-html.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/html-sink.h"
 #include "xml.h"
 #include "xml-printer.h"
 
@@ -54,17 +54,20 @@ along with GCC; see the file COPYING3.  If not see
 
 namespace {
 
+using namespace diagnostics;
+using namespace diagnostics::paths;
+
 /* A bundle of state for printing a path.  */
 
 class path_print_policy
 {
 public:
-  path_print_policy (const diagnostic_text_output_format &text_output)
+  path_print_policy (const diagnostics::text_sink &text_output)
   : m_source_policy (text_output.get_context ())
   {
   }
 
-  path_print_policy (const diagnostic_context &dc)
+  path_print_policy (const diagnostics::context &dc)
   : m_source_policy (dc)
   {
   }
@@ -75,26 +78,26 @@ public:
     return m_source_policy.get_diagram_theme ();
   }
 
-  const diagnostic_source_print_policy &
+  const diagnostics::source_print_policy &
   get_source_policy () const { return m_source_policy; }
 
 private:
-  diagnostic_source_print_policy m_source_policy;
+  diagnostics::source_print_policy m_source_policy;
 };
 
 /* Subclass of range_label for showing a particular event
-   when showing a consecutive run of events within a diagnostic_path as
+   when showing a consecutive run of events within a diagnostic path as
    labelled ranges within one gcc_rich_location.  */
 
 class path_label : public range_label
 {
  public:
-  path_label (const diagnostic_path &path,
+  path_label (const path &path_,
 	      const pretty_printer &ref_pp,
 	      unsigned start_idx,
 	      bool colorize,
 	      bool allow_emojis)
-  : m_path (path),
+  : m_path (path_),
     m_ref_pp (ref_pp),
     m_start_idx (start_idx), m_effects (*this),
     m_colorize (colorize), m_allow_emojis (allow_emojis)
@@ -103,18 +106,17 @@ class path_label : public range_label
   label_text get_text (unsigned range_idx) const final override
   {
     unsigned event_idx = m_start_idx + range_idx;
-    const diagnostic_event &event = m_path.get_event (event_idx);
+    const event &ev = m_path.get_event (event_idx);
 
-    const diagnostic_event::meaning meaning (event.get_meaning ());
+    const event::meaning meaning (ev.get_meaning ());
 
     auto pp = m_ref_pp.clone ();
     pp_show_color (pp.get ()) = m_colorize;
-    diagnostic_event_id_t event_id (event_idx);
-
+    event_id_t event_id (event_idx);
     pp_printf (pp.get (), "%@", &event_id);
     pp_space (pp.get ());
 
-    if (meaning.m_verb == diagnostic_event::verb::danger
+    if (meaning.m_verb == event::verb::danger
 	&& m_allow_emojis)
       {
 	pp_unicode_character (pp.get (), 0x26A0); /* U+26A0 WARNING SIGN.  */
@@ -128,14 +130,15 @@ class path_label : public range_label
 	pp_string (pp.get (), "  ");
       }
 
-    event.print_desc (*pp.get ());
+    ev.print_desc (*pp.get ());
 
     label_text result
       = label_text::take (xstrdup (pp_formatted_text (pp.get ())));
     return result;
   }
 
-  const label_effects *get_effects (unsigned /*range_idx*/) const final override
+  const diagnostics::label_effects *
+  get_effects (unsigned /*range_idx*/) const final override
   {
     return &m_effects;
   }
@@ -150,28 +153,28 @@ class path_label : public range_label
     }
     bool has_in_edge (unsigned range_idx) const final override
     {
-      if (const diagnostic_event *prev_event
+      if (const event *prev_event
 	    = m_path_label.get_prev_event (range_idx))
 	return prev_event->connect_to_next_event_p ();
       return false;
     }
     bool has_out_edge (unsigned range_idx) const final override
     {
-      const diagnostic_event &event = m_path_label.get_event (range_idx);
-      return event.connect_to_next_event_p ();
+      const event &ev = m_path_label.get_event (range_idx);
+      return ev.connect_to_next_event_p ();
     }
 
   private:
     const path_label &m_path_label;
   };
 
-  const diagnostic_event &get_event (unsigned range_idx) const
+  const event &get_event (unsigned range_idx) const
   {
     unsigned event_idx = m_start_idx + range_idx;
     return m_path.get_event (event_idx);
   }
 
-  const diagnostic_event *get_prev_event (unsigned range_idx) const
+  const event *get_prev_event (unsigned range_idx) const
   {
     if (m_start_idx + range_idx == 0)
       return nullptr;
@@ -179,7 +182,7 @@ class path_label : public range_label
     return &m_path.get_event (event_idx);
   }
 
-  const diagnostic_path &m_path;
+  const path &m_path;
   const pretty_printer &m_ref_pp;
   unsigned m_start_idx;
   path_label_effects m_effects;
@@ -188,20 +191,20 @@ class path_label : public range_label
 };
 
 /* Return true if E1 and E2 can be consolidated into the same run of events
-   when printing a diagnostic_path.  */
+   when printing a diagnostic path.  */
 
 static bool
-can_consolidate_events (const diagnostic_path &path,
-			const diagnostic_event &e1,
+can_consolidate_events (const path &p,
+			const event &e1,
 			unsigned ev1_idx,
-			const diagnostic_event &e2,
+			const event &e2,
 			unsigned ev2_idx,
 			bool check_locations)
 {
   if (e1.get_thread_id () != e2.get_thread_id ())
     return false;
 
-  if (!path.same_function_p (ev1_idx, ev2_idx))
+  if (!p.same_function_p (ev1_idx, ev2_idx))
     return false;
 
   if (e1.get_stack_depth () != e2.get_stack_depth ())
@@ -231,16 +234,16 @@ struct event_range;
 struct path_summary;
 class thread_event_printer;
 
-/* A bundle of information about all of the events in a diagnostic_path
+/* A bundle of information about all of the events in a diagnostic path
    relating to a specific path, for use by path_summary.  */
 
 class per_thread_summary
 {
 public:
-  per_thread_summary (const diagnostic_path &path,
-		      const logical_location_manager &logical_loc_mgr,
+  per_thread_summary (const path &path_,
+		      const logical_locations::manager &logical_loc_mgr,
 		      label_text name, unsigned swimlane_idx)
-  : m_path (path),
+  : m_path (path_),
     m_logical_loc_mgr (logical_loc_mgr),
     m_name (std::move (name)),
     m_swimlane_idx (swimlane_idx),
@@ -267,8 +270,8 @@ private:
   friend class thread_event_printer;
   friend struct event_range;
 
-  const diagnostic_path &m_path;
-  const logical_location_manager &m_logical_loc_mgr;
+  const path &m_path;
+  const logical_locations::manager &m_logical_loc_mgr;
 
   const label_text m_name;
 
@@ -279,7 +282,7 @@ private:
   // The event ranges specific to this thread:
   auto_vec<event_range *> m_event_ranges;
 
-  const diagnostic_event *m_last_event;
+  const event *m_last_event;
 
   int m_min_depth;
   int m_max_depth;
@@ -291,7 +294,7 @@ private:
 struct stack_frame
 {
   stack_frame (std::unique_ptr<stack_frame> parent,
-	       logical_location logical_loc,
+	       logical_locations::key logical_loc,
 	       int stack_depth)
   : m_parent (std::move (parent)),
     m_logical_loc (logical_loc),
@@ -299,7 +302,7 @@ struct stack_frame
   {}
 
   std::unique_ptr<stack_frame> m_parent;
-  logical_location m_logical_loc;
+  logical_locations::key m_logical_loc;
   const int m_stack_depth;
 };
 
@@ -309,9 +312,9 @@ struct stack_frame
 static std::unique_ptr<stack_frame>
 begin_html_stack_frame (xml::printer &xp,
 			std::unique_ptr<stack_frame> parent,
-			logical_location logical_loc,
+			logical_locations::key logical_loc,
 			int stack_depth,
-			const logical_location_manager *logical_loc_mgr)
+			const logical_locations::manager *logical_loc_mgr)
 {
   if (logical_loc)
     {
@@ -398,7 +401,7 @@ emit_svg_arrow (xml::printer &xp, int old_depth, int new_depth)
   xp.add_raw (pp_formatted_text (pp));
 }
 
-/* A range of consecutive events within a diagnostic_path, all within the
+/* A range of consecutive events within a diagnostic path, all within the
    same thread, and with the same fndecl and stack_depth, and which are suitable
    to print with a single call to diagnostic_show_locus.  */
 struct event_range
@@ -423,7 +426,7 @@ struct event_range
        the events already on this source line, *and* a new event at COLUMN.  */
     bool
     can_add_label_for_event_p (bool has_in_edge,
-			       const diagnostic_event *prev_event,
+			       const event *prev_event,
 			       bool has_out_edge,
 			       int column) const
     {
@@ -488,20 +491,20 @@ struct event_range
     int m_max_label_source_column;
   };
 
-  event_range (const diagnostic_path &path,
+  event_range (const path &path_,
 	       const pretty_printer &ref_pp,
 	       unsigned start_idx,
-	       const diagnostic_event &initial_event,
+	       const event &initial_event,
 	       per_thread_summary &t,
 	       bool show_event_links,
 	       bool colorize_labels,
 	       bool allow_emojis)
-  : m_path (path),
+  : m_path (path_),
     m_initial_event (initial_event),
     m_logical_loc (initial_event.get_logical_location ()),
     m_stack_depth (initial_event.get_stack_depth ()),
     m_start_idx (start_idx), m_end_idx (start_idx),
-    m_path_label (path, ref_pp,
+    m_path_label (path_, ref_pp,
 		  start_idx, colorize_labels, allow_emojis),
     m_richloc (initial_event.get_location (), &m_path_label, nullptr),
     m_thread_id (initial_event.get_thread_id ()),
@@ -516,7 +519,7 @@ struct event_range
 	per_source_line_info &source_line_info
 	  = get_per_source_line_info (exploc.line);
 
-	const diagnostic_event *prev_thread_event = t.m_last_event;
+	const event *prev_thread_event = t.m_last_event;
 	const bool has_in_edge
 	  = (prev_thread_event
 	     ? prev_thread_event->connect_to_next_event_p ()
@@ -540,7 +543,7 @@ struct event_range
   }
 
   bool maybe_add_event (const path_print_policy &policy,
-			const diagnostic_event &new_ev,
+			const event &new_ev,
 			unsigned new_ev_idx,
 			bool check_rich_locations)
   {
@@ -557,7 +560,7 @@ struct event_range
       (line_table, new_ev.get_location (), LOCATION_ASPECT_CARET);
     per_source_line_info &source_line_info
       = get_per_source_line_info (exploc.line);
-    const diagnostic_event *prev_event = nullptr;
+    const event *prev_event = nullptr;
     if (new_ev_idx > 0)
       prev_event = &m_path.get_event (new_ev_idx - 1);
     const bool has_in_edge = (prev_event
@@ -591,24 +594,24 @@ struct event_range
      call to diagnostic_show_locus.  */
 
   void print_as_text (pretty_printer &pp,
-		      diagnostic_text_output_format &text_output,
-		      diagnostic_source_effect_info *effect_info)
+		      diagnostics::text_sink &text_output,
+		      diagnostics::source_effect_info *effect_info)
   {
     location_t initial_loc = m_initial_event.get_location ();
 
-    diagnostic_context &dc = text_output.get_context ();
+    diagnostics::context &dc = text_output.get_context ();
 
     /* Emit a span indicating the filename (and line/column) if the
        line has changed relative to the last call to
        diagnostic_show_locus.  */
-    if (dc.m_source_printing.enabled)
+    if (dc.get_source_printing_options ().enabled)
       {
 	expanded_location exploc
 	  = linemap_client_expand_location_to_spelling_point
 	  (line_table, initial_loc, LOCATION_ASPECT_CARET);
 	if (exploc.file != LOCATION_FILE (dc.m_last_location))
 	  {
-	    diagnostic_location_print_policy loc_policy (text_output);
+	    diagnostics::location_print_policy loc_policy (text_output);
 	    loc_policy.print_text_span_start (dc, pp, exploc);
 	  }
       }
@@ -624,7 +627,7 @@ struct event_range
       {
 	for (unsigned i = m_start_idx; i <= m_end_idx; i++)
 	  {
-	    const diagnostic_event &iter_event = m_path.get_event (i);
+	    const event &iter_event = m_path.get_event (i);
 	    diagnostic_event_id_t event_id (i);
 	    pp_printf (&pp, " %@: ", &event_id);
 	    iter_event.print_desc (pp);
@@ -635,7 +638,7 @@ struct event_range
 
     /* Call diagnostic_show_locus to show the events using labels.  */
     diagnostic_show_locus (&dc, text_output.get_source_printing_options (),
-			   &m_richloc, DK_DIAGNOSTIC_PATH, &pp,
+			   &m_richloc, diagnostics::kind::path, &pp,
 			   effect_info);
 
     /* If we have a macro expansion, show the expansion to the user.  */
@@ -650,8 +653,8 @@ struct event_range
      call to diagnostic_show_locus_as_html.  */
 
   void print_as_html (xml::printer &xp,
-		      diagnostic_context &dc,
-		      diagnostic_source_effect_info *effect_info,
+		      diagnostics::context &dc,
+		      diagnostics::source_effect_info *effect_info,
 		      html_label_writer *event_label_writer)
   {
     location_t initial_loc = m_initial_event.get_location ();
@@ -659,14 +662,14 @@ struct event_range
     /* Emit a span indicating the filename (and line/column) if the
        line has changed relative to the last call to
        diagnostic_show_locus.  */
-    if (dc.m_source_printing.enabled)
+    if (dc.get_source_printing_options ().enabled)
       {
 	expanded_location exploc
 	  = linemap_client_expand_location_to_spelling_point
 	  (line_table, initial_loc, LOCATION_ASPECT_CARET);
 	if (exploc.file != LOCATION_FILE (dc.m_last_location))
 	  {
-	    diagnostic_location_print_policy loc_policy (dc);
+	    diagnostics::location_print_policy loc_policy (dc);
 	    loc_policy.print_html_span_start (dc, xp, exploc);
 	  }
       }
@@ -682,7 +685,7 @@ struct event_range
       {
 	for (unsigned i = m_start_idx; i <= m_end_idx; i++)
 	  {
-	    const diagnostic_event &iter_event = m_path.get_event (i);
+	    const event &iter_event = m_path.get_event (i);
 	    diagnostic_event_id_t event_id (i);
 	    pretty_printer pp;
 	    pp_printf (&pp, " %@: ", &event_id);
@@ -698,29 +701,29 @@ struct event_range
 
     /* Call diagnostic_show_locus_as_html to show the source,
        showing events using labels.  */
-    diagnostic_show_locus_as_html (&dc, dc.m_source_printing,
-				   &m_richloc, DK_DIAGNOSTIC_PATH, xp,
+    diagnostic_show_locus_as_html (&dc, dc.get_source_printing_options (),
+				   &m_richloc, diagnostics::kind::path, xp,
 				   effect_info, event_label_writer);
 
     // TODO: show macro expansions
   }
 
-  const diagnostic_path &m_path;
-  const diagnostic_event &m_initial_event;
-  logical_location m_logical_loc;
+  const path &m_path;
+  const event &m_initial_event;
+  logical_locations::key m_logical_loc;
   int m_stack_depth;
   unsigned m_start_idx;
   unsigned m_end_idx;
   path_label m_path_label;
   gcc_rich_location m_richloc;
-  diagnostic_thread_id_t m_thread_id;
+  thread_id_t m_thread_id;
   per_thread_summary &m_per_thread_summary;
   hash_map<int_hash<int, -1, -2>,
 	   per_source_line_info> m_source_line_info_map;
   bool m_show_event_links;
 };
 
-/* A struct for grouping together the events in a diagnostic_path into
+/* A struct for grouping together the events in a path into
    ranges of events, partitioned by thread and by stack frame (i.e. by fndecl
    and stack depth).  */
 
@@ -728,19 +731,19 @@ struct path_summary
 {
   path_summary (const path_print_policy &policy,
 		const pretty_printer &ref_pp,
-		const diagnostic_path &path,
+		const path &path_,
 		bool check_rich_locations,
 		bool colorize = false,
 		bool show_event_links = true);
 
-  const logical_location_manager &get_logical_location_manager () const
+  const logical_locations::manager &get_logical_location_manager () const
   {
     return m_logical_loc_mgr;
   }
   unsigned get_num_ranges () const { return m_ranges.length (); }
   bool multithreaded_p () const { return m_per_thread_summary.length () > 1; }
 
-  const per_thread_summary &get_events_for_thread_id (diagnostic_thread_id_t tid)
+  const per_thread_summary &get_events_for_thread_id (thread_id_t tid)
   {
     per_thread_summary **slot = m_thread_id_to_events.get (tid);
     gcc_assert (slot);
@@ -748,23 +751,23 @@ struct path_summary
     return **slot;
   }
 
-  const logical_location_manager &m_logical_loc_mgr;
+  const logical_locations::manager &m_logical_loc_mgr;
   auto_delete_vec <event_range> m_ranges;
   auto_delete_vec <per_thread_summary> m_per_thread_summary;
-  hash_map<int_hash<diagnostic_thread_id_t, -1, -2>,
+  hash_map<int_hash<thread_id_t, -1, -2>,
 	   per_thread_summary *> m_thread_id_to_events;
 
 private:
   per_thread_summary &
-  get_or_create_events_for_thread_id (const diagnostic_path &path,
-				      diagnostic_thread_id_t tid)
+  get_or_create_events_for_thread_id (const path &path_,
+				      thread_id_t tid)
   {
     if (per_thread_summary **slot = m_thread_id_to_events.get (tid))
       return **slot;
 
-    const diagnostic_thread &thread = path.get_thread (tid);
+    const thread &thread = path_.get_thread (tid);
     per_thread_summary *pts
-      = new per_thread_summary (path,
+      = new per_thread_summary (path_,
 				m_logical_loc_mgr,
 				thread.get_name (false),
 				m_per_thread_summary.length ());
@@ -798,40 +801,40 @@ per_thread_summary::interprocedural_p () const
 
 path_summary::path_summary (const path_print_policy &policy,
 			    const pretty_printer &ref_pp,
-			    const diagnostic_path &path,
+			    const path &path_,
 			    bool check_rich_locations,
 			    bool colorize,
 			    bool show_event_links)
-: m_logical_loc_mgr (path.get_logical_location_manager ())
+: m_logical_loc_mgr (path_.get_logical_location_manager ())
 {
-  const unsigned num_events = path.num_events ();
+  const unsigned num_events = path_.num_events ();
 
   event_range *cur_event_range = nullptr;
   for (unsigned idx = 0; idx < num_events; idx++)
     {
-      const diagnostic_event &event = path.get_event (idx);
-      const diagnostic_thread_id_t thread_id = event.get_thread_id ();
+      const event &ev = path_.get_event (idx);
+      const thread_id_t thread_id = ev.get_thread_id ();
       per_thread_summary &pts
-	= get_or_create_events_for_thread_id (path, thread_id);
+	= get_or_create_events_for_thread_id (path_, thread_id);
 
-      pts.update_depth_limits (event.get_stack_depth ());
+      pts.update_depth_limits (ev.get_stack_depth ());
 
       if (cur_event_range)
 	if (cur_event_range->maybe_add_event (policy,
-					      event,
+					      ev,
 					      idx, check_rich_locations))
 	  continue;
 
       auto theme = policy.get_diagram_theme ();
       const bool allow_emojis = theme ? theme->emojis_p () : false;
-      cur_event_range = new event_range (path, ref_pp,
-					 idx, event, pts,
+      cur_event_range = new event_range (path_, ref_pp,
+					 idx, ev, pts,
 					 show_event_links,
 					 colorize,
 					 allow_emojis);
       m_ranges.safe_push (cur_event_range);
       pts.m_event_ranges.safe_push (cur_event_range);
-      pts.m_last_event = &event;
+      pts.m_last_event = &ev;
     }
 }
 
@@ -881,11 +884,11 @@ public:
   }
 
   void
-  print_swimlane_for_event_range_as_text (diagnostic_text_output_format &text_output,
+  print_swimlane_for_event_range_as_text (diagnostics::text_sink &text_output,
 					  pretty_printer *pp,
-					  const logical_location_manager &logical_loc_mgr,
+					  const logical_locations::manager &logical_loc_mgr,
 					  event_range *range,
-					  diagnostic_source_effect_info *effect_info)
+					  diagnostics::source_effect_info *effect_info)
   {
     gcc_assert (pp);
     const char *const line_color = "path";
@@ -1041,11 +1044,11 @@ public:
   }
 
   void
-  print_swimlane_for_event_range_as_html (diagnostic_context &dc,
+  print_swimlane_for_event_range_as_html (diagnostics::context &dc,
 					  xml::printer &xp,
 					  html_label_writer *event_label_writer,
 					  event_range *range,
-					  diagnostic_source_effect_info *effect_info)
+					  diagnostics::source_effect_info *effect_info)
   {
     range->print_as_html (xp, dc, effect_info, event_label_writer);
     m_num_printed++;
@@ -1107,7 +1110,7 @@ private:
 
 static void
 print_path_summary_as_text (const path_summary &ps,
-			    diagnostic_text_output_format &text_output,
+			    diagnostics::text_sink &text_output,
 			    bool show_depths)
 {
   pretty_printer *const pp = text_output.get_printer ();
@@ -1135,7 +1138,7 @@ print_path_summary_as_text (const path_summary &ps,
       thread_event_printer &tep = thread_event_printers[swimlane_idx];
       /* Wire up any trailing out-edge from previous range to leading in-edge
 	 of this range.  */
-      diagnostic_source_effect_info effect_info;
+      diagnostics::source_effect_info effect_info;
       effect_info.m_leading_in_edge_column = last_out_edge_column;
       tep.print_swimlane_for_event_range_as_text
 	(text_output, pp,
@@ -1149,7 +1152,7 @@ print_path_summary_as_text (const path_summary &ps,
 
 static void
 print_path_summary_as_html (const path_summary &ps,
-			    diagnostic_context &dc,
+			    diagnostics::context &dc,
 			    xml::printer &xp,
 			    html_label_writer *event_label_writer,
 			    bool show_depths)
@@ -1158,7 +1161,7 @@ print_path_summary_as_html (const path_summary &ps,
   for (auto t : ps.m_per_thread_summary)
     thread_event_printers.push_back (thread_event_printer (*t, show_depths));
 
-  const logical_location_manager *logical_loc_mgr
+  const logical_locations::manager *logical_loc_mgr
     = dc.get_logical_location_manager ();
 
   xp.push_tag_with_class ("div", "event-ranges", false);
@@ -1173,7 +1176,7 @@ print_path_summary_as_html (const path_summary &ps,
       const int swimlane_idx
 	= range->m_per_thread_summary.get_swimlane_index ();
 
-      const logical_location this_logical_loc = range->m_logical_loc;
+      const logical_locations::key this_logical_loc = range->m_logical_loc;
       const int this_depth = range->m_stack_depth;
       if (curr_frame)
 	{
@@ -1261,7 +1264,7 @@ print_path_summary_as_html (const path_summary &ps,
       thread_event_printer &tep = thread_event_printers[swimlane_idx];
       /* Wire up any trailing out-edge from previous range to leading in-edge
 	 of this range.  */
-      diagnostic_source_effect_info effect_info;
+      diagnostics::source_effect_info effect_info;
       effect_info.m_leading_in_edge_column = last_out_edge_column;
       tep.print_swimlane_for_event_range_as_html (dc, xp, event_label_writer,
 						  range, &effect_info);
@@ -1284,8 +1287,8 @@ print_path_summary_as_html (const path_summary &ps,
 class element_event_desc : public pp_element
 {
 public:
-  element_event_desc (const diagnostic_event &event)
-  : m_event (event)
+  element_event_desc (const event &event_)
+  : m_event (event_)
   {
   }
 
@@ -1297,15 +1300,15 @@ public:
   }
 
 private:
-  const diagnostic_event &m_event;
+  const event &m_event;
 };
 
 /* Print PATH according to the context's path_format.  */
 
 void
-diagnostic_text_output_format::print_path (const diagnostic_path &path)
+diagnostics::text_sink::print_path (const path &path_)
 {
-  const unsigned num_events = path.num_events ();
+  const unsigned num_events = path_.num_events ();
 
   switch (get_context ().get_path_format ())
     {
@@ -1316,36 +1319,36 @@ diagnostic_text_output_format::print_path (const diagnostic_path &path)
     case DPF_SEPARATE_EVENTS:
       {
 	/* A note per event.  */
-	auto &logical_loc_mgr = path.get_logical_location_manager ();
+	auto &logical_loc_mgr = path_.get_logical_location_manager ();
 	for (unsigned i = 0; i < num_events; i++)
 	  {
-	    const diagnostic_event &event = path.get_event (i);
-	    element_event_desc e_event_desc (event);
+	    const event &ev = path_.get_event (i);
+	    element_event_desc e_event_desc (ev);
 	    diagnostic_event_id_t event_id (i);
 	    if (get_context ().show_path_depths_p ())
 	      {
-		int stack_depth = event.get_stack_depth ();
+		int stack_depth = ev.get_stack_depth ();
 		/* -fdiagnostics-path-format=separate-events doesn't print
 		   fndecl information, so with -fdiagnostics-show-path-depths
 		   print the fndecls too, if any.  */
-		if (logical_location logical_loc
-		      = event.get_logical_location ())
+		if (logical_locations::key logical_loc
+		      = ev.get_logical_location ())
 		  {
 		    label_text name
 		      (logical_loc_mgr.get_name_for_path_output (logical_loc));
-		    inform (event.get_location (),
+		    inform (ev.get_location (),
 			    "%@ %e (fndecl %qs, depth %i)",
 			    &event_id, &e_event_desc,
 			    name.get (), stack_depth);
 		  }
 		else
-		  inform (event.get_location (),
+		  inform (ev.get_location (),
 			  "%@ %e (depth %i)",
 			  &event_id, &e_event_desc,
 			  stack_depth);
 	      }
 	    else
-	      inform (event.get_location (),
+	      inform (ev.get_location (),
 		      "%@ %e", &event_id, &e_event_desc);
 	  }
       }
@@ -1361,7 +1364,7 @@ diagnostic_text_output_format::print_path (const diagnostic_path &path)
 	const bool show_event_links = m_source_printing.show_event_links_p;
 	path_summary summary (policy,
 			      *pp,
-			      path,
+			      path_,
 			      check_rich_locations,
 			      colorize,
 			      show_event_links);
@@ -1376,25 +1379,25 @@ diagnostic_text_output_format::print_path (const diagnostic_path &path)
     }
 }
 
-/* Print PATH as HTML to XP, using DC and DSPP for settings.
+/* Print PATH_ as HTML to XP, using DC and DSPP for settings.
    If non-null, use EVENT_LABEL_WRITER when writing events.  */
 
 void
-print_path_as_html (xml::printer &xp,
-		    const diagnostic_path &path,
-		    diagnostic_context &dc,
-		    html_label_writer *event_label_writer,
-		    const diagnostic_source_print_policy &dspp)
+diagnostics::print_path_as_html (xml::printer &xp,
+				 const path &path_,
+				 context &dc,
+				 html_label_writer *event_label_writer,
+				 const source_print_policy &dspp)
 {
   path_print_policy policy (dc);
   const bool check_rich_locations = true;
   const bool colorize = false;
-  const diagnostic_source_printing_options &source_printing_opts
+  const source_printing_options &source_printing_opts
     = dspp.get_options ();
   const bool show_event_links = source_printing_opts.show_event_links_p;
   path_summary summary (policy,
 			*dc.get_reference_printer (),
-			path,
+			path_,
 			check_rich_locations,
 			colorize,
 			show_event_links);
@@ -1404,18 +1407,27 @@ print_path_as_html (xml::printer &xp,
 
 #if CHECKING_P
 
+namespace diagnostics {
+namespace paths {
 namespace selftest {
 
-/* Return true iff all events in PATH have locations for which column data
+using location = ::selftest::location;
+using line_table_case = ::selftest::line_table_case;
+using line_table_test = ::selftest::line_table_test;
+using temp_source_file = ::selftest::temp_source_file;
+
+using test_context = diagnostics::selftest::test_context;
+
+/* Return true iff all events in PATH_ have locations for which column data
    is available, so that selftests that require precise string output can
    bail out for awkward line_table cases.  */
 
 static bool
-path_events_have_column_data_p (const diagnostic_path &path)
+path_events_have_column_data_p (const path &path_)
 {
-  for (unsigned idx = 0; idx < path.num_events (); idx++)
+  for (unsigned idx = 0; idx < path_.num_events (); idx++)
     {
-      location_t event_loc = path.get_event (idx).get_location ();
+      location_t event_loc = path_.get_event (idx).get_location ();
       if (line_table->get_pure_location (event_loc)
 	  > LINE_MAP_MAX_LOCATION_WITH_COLS)
 	return false;
@@ -1432,11 +1444,12 @@ path_events_have_column_data_p (const diagnostic_path &path)
 static void
 test_empty_path (pretty_printer *event_pp)
 {
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   ASSERT_FALSE (path.interprocedural_p ());
 
-  test_diagnostic_context dc;
-  diagnostic_text_output_format text_output (dc);
+  test_context dc;
+  text_sink text_output (dc);
   path_print_policy policy (text_output);
   path_summary summary (policy, *event_pp, path, false);
   ASSERT_EQ (summary.get_num_ranges (), 0);
@@ -1451,15 +1464,16 @@ test_empty_path (pretty_printer *event_pp)
 static void
 test_intraprocedural_path (pretty_printer *event_pp)
 {
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   const char *const funcname = "foo";
   path.add_event (UNKNOWN_LOCATION, funcname, 0, "first %qs", "free");
   path.add_event (UNKNOWN_LOCATION, funcname, 0, "double %qs", "free");
 
   ASSERT_FALSE (path.interprocedural_p ());
 
-  test_diagnostic_context dc;
-  diagnostic_text_output_format text_output (dc);
+  selftest::test_context dc;
+  text_sink text_output (dc);
   path_print_policy policy (text_output);
   path_summary summary (policy, *event_pp, path, false, false, false);
   ASSERT_EQ (summary.get_num_ranges (), 1);
@@ -1476,7 +1490,8 @@ test_intraprocedural_path (pretty_printer *event_pp)
 static void
 test_interprocedural_path_1 (pretty_printer *event_pp)
 {
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_entry ("test", 0);
   path.add_call ("test", 0, "make_boxed_int");
   path.add_call ("make_boxed_int", 1, "wrapped_malloc");
@@ -1495,8 +1510,8 @@ test_interprocedural_path_1 (pretty_printer *event_pp)
   ASSERT_TRUE (path.interprocedural_p ());
 
   {
-    test_diagnostic_context dc;
-    diagnostic_text_output_format text_output (dc, nullptr, false);
+    selftest::test_context dc;
+    text_sink text_output (dc, nullptr, false);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, false);
     ASSERT_EQ (summary.get_num_ranges (), 9);
@@ -1556,9 +1571,9 @@ test_interprocedural_path_1 (pretty_printer *event_pp)
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE);
-    diagnostic_text_output_format text_output (dc);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, false);
     print_path_summary_as_text (summary, text_output, true);
@@ -1623,7 +1638,8 @@ test_interprocedural_path_1 (pretty_printer *event_pp)
 static void
 test_interprocedural_path_2 (pretty_printer *event_pp)
 {
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_entry ("foo", 0);
   path.add_call ("foo", 0, "bar");
   path.add_call ("bar", 1, "baz");
@@ -1634,8 +1650,8 @@ test_interprocedural_path_2 (pretty_printer *event_pp)
   ASSERT_TRUE (path.interprocedural_p ());
 
   {
-    test_diagnostic_context dc;
-    diagnostic_text_output_format text_output (dc);
+    selftest::test_context dc;
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, false);
     ASSERT_EQ (summary.get_num_ranges (), 5);
@@ -1670,9 +1686,9 @@ test_interprocedural_path_2 (pretty_printer *event_pp)
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE);
-    diagnostic_text_output_format text_output (dc);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, false);
     print_path_summary_as_text (summary, text_output, true);
@@ -1707,12 +1723,13 @@ test_interprocedural_path_2 (pretty_printer *event_pp)
 }
 
 /* Verify that print_path_summary is sane in the face of a recursive
-   diagnostic_path.  */
+   diagnostic path.  */
 
 static void
 test_recursion (pretty_printer *event_pp)
 {
- test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_entry ("factorial", 0);
   for (int depth = 0; depth < 3; depth++)
     path.add_call ("factorial", depth, "factorial");
@@ -1721,10 +1738,10 @@ test_recursion (pretty_printer *event_pp)
   ASSERT_TRUE (path.interprocedural_p ());
 
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
 
-    diagnostic_text_output_format text_output (dc);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, false);
     ASSERT_EQ (summary.get_num_ranges (), 4);
@@ -1753,10 +1770,10 @@ test_recursion (pretty_printer *event_pp)
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE);
 
-    diagnostic_text_output_format text_output (dc);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, false);
     print_path_summary_as_text (summary, text_output, true);
@@ -1789,7 +1806,7 @@ test_recursion (pretty_printer *event_pp)
 class control_flow_test
 {
 public:
-  control_flow_test (const location &loc,
+  control_flow_test (const selftest::location &loc,
 		     const line_table_case &case_,
 		     const char *content)
   : m_tmp_file (loc, ".c", content,
@@ -1858,7 +1875,8 @@ test_control_flow_1 (const line_table_case &case_,
   const location_t conditional = t.get_line_and_column (3, 7);
   const location_t cfg_dest = t.get_line_and_column (5, 10);
 
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_event (conditional, nullptr, 0,
 		  "following %qs branch (when %qs is NULL)...",
 		  "false", "p");
@@ -1875,10 +1893,10 @@ test_control_flow_1 (const line_table_case &case_,
 
 
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_event_links_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -1901,10 +1919,10 @@ test_control_flow_1 (const line_table_case &case_,
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_event_links_p = false;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (false);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -1924,11 +1942,11 @@ test_control_flow_1 (const line_table_case &case_,
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_line_numbers_p = true;
-    dc.m_source_printing.show_event_links_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_line_numbers (true);
+    dc.show_event_links (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -1951,11 +1969,11 @@ test_control_flow_1 (const line_table_case &case_,
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_line_numbers_p = true;
-    dc.m_source_printing.show_event_links_p = false;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_line_numbers (true);
+    dc.show_event_links (false);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -1975,10 +1993,10 @@ test_control_flow_1 (const line_table_case &case_,
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE);
-    dc.m_source_printing.show_event_links_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -2001,11 +2019,11 @@ test_control_flow_1 (const line_table_case &case_,
        pp_formatted_text (text_output.get_printer ()));
   }
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_UNICODE);
-    dc.m_source_printing.show_event_links_p = true;
-    dc.m_source_printing.show_line_numbers_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    dc.show_line_numbers (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -2055,7 +2073,8 @@ test_control_flow_2 (const line_table_case &case_,
   const location_t loop_body_start = t.get_line_and_columns (5, 12, 17);
   const location_t loop_body_end = t.get_line_and_columns (5, 5, 9, 17);
 
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_event (iter_test, nullptr, 0, "infinite loop here");
 
   path.add_event (iter_test, nullptr, 0, "looping from here...");
@@ -2072,11 +2091,11 @@ test_control_flow_2 (const line_table_case &case_,
     return;
 
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_event_links_p = true;
-    dc.m_source_printing.show_line_numbers_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    dc.show_line_numbers (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -2142,7 +2161,8 @@ test_control_flow_3 (const line_table_case &case_,
   const location_t iter_test = t.get_line_and_column (3, 19);
   const location_t iter_next = t.get_line_and_columns (3, 22, 24);
 
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_event (iter_test, nullptr, 0, "infinite loop here");
 
   path.add_event (iter_test, nullptr, 0, "looping from here...");
@@ -2159,11 +2179,11 @@ test_control_flow_3 (const line_table_case &case_,
     return;
 
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_event_links_p = true;
-    dc.m_source_printing.show_line_numbers_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    dc.show_line_numbers (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -2208,7 +2228,8 @@ assert_cfg_edge_path_streq (const location &loc,
 			    const location_t dst_loc,
 			    const char *expected_str)
 {
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   path.add_event (src_loc, nullptr, 0, "from here...");
   path.connect_to_next_event ();
 
@@ -2217,11 +2238,11 @@ assert_cfg_edge_path_streq (const location &loc,
   if (!path_events_have_column_data_p (path))
     return;
 
-  test_diagnostic_context dc;
+  selftest::test_context dc;
   dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-  dc.m_source_printing.show_event_links_p = true;
-  dc.m_source_printing.show_line_numbers_p = true;
-  diagnostic_text_output_format text_output (dc);
+  dc.show_event_links (true);
+  dc.show_line_numbers (true);
+  text_sink text_output (dc);
   path_print_policy policy (text_output);
   path_summary summary (policy, *event_pp, path, true);
   print_path_summary_as_text (summary, text_output, false);
@@ -2512,7 +2533,8 @@ test_control_flow_5 (const line_table_case &case_,
 
   control_flow_test t (SELFTEST_LOCATION, case_, content);
 
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   /* (1) */
   path.add_event (t.get_line_and_column (1, 6), nullptr, 0,
 		  "following %qs branch (when %qs is non-NULL)...",
@@ -2541,11 +2563,11 @@ test_control_flow_5 (const line_table_case &case_,
     return;
 
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_event_links_p = true;
-    dc.m_source_printing.show_line_numbers_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    dc.show_line_numbers (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -2601,7 +2623,8 @@ test_control_flow_6 (const line_table_case &case_,
 
   control_flow_test t (SELFTEST_LOCATION, case_, content);
 
-  test_diagnostic_path path (event_pp);
+  logical_locations::selftest::test_manager logical_loc_mgr;
+  test_path path (logical_loc_mgr, event_pp);
   /* (1) */
   path.add_event (t.get_line_and_columns (6, 25, 35), nullptr, 0,
 		  "allocated here");
@@ -2630,11 +2653,11 @@ test_control_flow_6 (const line_table_case &case_,
     return;
 
   {
-    test_diagnostic_context dc;
+    selftest::test_context dc;
     dc.set_text_art_charset (DIAGNOSTICS_TEXT_ART_CHARSET_ASCII);
-    dc.m_source_printing.show_event_links_p = true;
-    dc.m_source_printing.show_line_numbers_p = true;
-    diagnostic_text_output_format text_output (dc);
+    dc.show_event_links (true);
+    dc.show_line_numbers (true);
+    text_sink text_output (dc);
     path_print_policy policy (text_output);
     path_summary summary (policy, *event_pp, path, true);
     print_path_summary_as_text (summary, text_output, false);
@@ -2687,24 +2710,30 @@ control_flow_tests (const line_table_case &case_)
   test_control_flow_6 (case_, &pp);
 }
 
+} // namespace diagnostics::paths::selftest
+} // namespace diagnostics::paths
+
+namespace selftest { // diagnostics::selftest
+
 /* Run all of the selftests within this file.  */
 
 void
-diagnostic_path_output_cc_tests ()
+paths_output_cc_tests ()
 {
   pretty_printer pp;
   pp_show_color (&pp) = false;
 
-  auto_fix_quotes fix_quotes;
-  test_empty_path (&pp);
-  test_intraprocedural_path (&pp);
-  test_interprocedural_path_1 (&pp);
-  test_interprocedural_path_2 (&pp);
-  test_recursion (&pp);
-  for_each_line_table_case (control_flow_tests);
+  ::selftest::auto_fix_quotes fix_quotes;
+  diagnostics::paths::selftest::test_empty_path (&pp);
+  diagnostics::paths::selftest::test_intraprocedural_path (&pp);
+  diagnostics::paths::selftest::test_interprocedural_path_1 (&pp);
+  diagnostics::paths::selftest::test_interprocedural_path_2 (&pp);
+  diagnostics::paths::selftest::test_recursion (&pp);
+  for_each_line_table_case (diagnostics::paths::selftest::control_flow_tests);
 }
 
-} // namespace selftest
+} // namespace diagnostics::selftest
+} // namespace diagnostics
 
 #if __GNUC__ >= 10
 #  pragma GCC diagnostic pop
diff --git a/gcc/diagnostic-path.cc b/gcc/diagnostics/paths.cc
index 3e169da..bc769c4 100644
--- a/gcc/diagnostic-path.cc
+++ b/gcc/diagnostics/paths.cc
@@ -26,8 +26,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "diagnostic.h"
-#include "diagnostic-path.h"
-#include "xml.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/state-graphs.h"
 
 /* Disable warnings about missing quoting in GCC diagnostics for the print
    calls below.  */
@@ -36,12 +36,15 @@ along with GCC; see the file COPYING3.  If not see
 #  pragma GCC diagnostic ignored "-Wformat-diag"
 #endif
 
-/* class diagnostic_event.  */
+using namespace diagnostics;
+using namespace diagnostics::paths;
 
-/* struct diagnostic_event::meaning.  */
+/* class diagnostics::paths::event.  */
+
+/* struct event::meaning.  */
 
 void
-diagnostic_event::meaning::dump_to_pp (pretty_printer *pp) const
+event::meaning::dump_to_pp (pretty_printer *pp) const
 {
   bool need_comma = false;
   pp_character (pp, '{');
@@ -71,7 +74,7 @@ diagnostic_event::meaning::dump_to_pp (pretty_printer *pp) const
    threadFlowLocation "kinds" property (SARIF v2.1.0 section 3.38.8).  */
 
 const char *
-diagnostic_event::meaning::maybe_get_verb_str (enum verb v)
+event::meaning::maybe_get_verb_str (enum verb v)
 {
   switch (v)
     {
@@ -102,7 +105,7 @@ diagnostic_event::meaning::maybe_get_verb_str (enum verb v)
    threadFlowLocation "kinds" property (SARIF v2.1.0 section 3.38.8).  */
 
 const char *
-diagnostic_event::meaning::maybe_get_noun_str (enum noun n)
+event::meaning::maybe_get_noun_str (enum noun n)
 {
   switch (n)
     {
@@ -129,7 +132,7 @@ diagnostic_event::meaning::maybe_get_noun_str (enum noun n)
    threadFlowLocation "kinds" property (SARIF v2.1.0 section 3.38.8).  */
 
 const char *
-diagnostic_event::meaning::maybe_get_property_str (enum property p)
+event::meaning::maybe_get_property_str (enum property p)
 {
   switch (p)
     {
@@ -148,7 +151,7 @@ diagnostic_event::meaning::maybe_get_property_str (enum property p)
    (for debugging/logging purposes).  */
 
 label_text
-diagnostic_event::get_desc (pretty_printer &ref_pp) const
+event::get_desc (pretty_printer &ref_pp) const
 {
   auto pp = ref_pp.clone ();
   pp_show_color (pp.get ()) = false;
@@ -156,31 +159,31 @@ diagnostic_event::get_desc (pretty_printer &ref_pp) const
   return label_text::take (xstrdup (pp_formatted_text (pp.get ())));
 }
 
-// Base implementation of diagnostic_event::maybe_make_xml_state
+// Base implementation of event::maybe_make_diagnostic_state_graph
 
-std::unique_ptr<xml::document>
-diagnostic_event::maybe_make_xml_state (bool) const
+std::unique_ptr<digraphs::digraph>
+event::maybe_make_diagnostic_state_graph (bool) const
 {
-  // Don't attempt to make a state document:
+  // Don't attempt to make a state graph:
   return nullptr;
 }
 
-/* class diagnostic_path.  */
+/* class diagnostics::paths::path.  */
 
-/* Subroutine of diagnostic_path::interprocedural_p.
+/* Subroutine of path::interprocedural_p.
    Look for the first event in this path that is within a function
    i.e. has a non-null logical location for which function_p is true.
    If found, write its index to *OUT_IDX and return true.
    Otherwise return false.  */
 
 bool
-diagnostic_path::get_first_event_in_a_function (unsigned *out_idx) const
+path::get_first_event_in_a_function (unsigned *out_idx) const
 {
   const unsigned num = num_events ();
   for (unsigned i = 0; i < num; i++)
     {
-      const diagnostic_event &event = get_event (i);
-      if (logical_location logical_loc = event.get_logical_location ())
+      const event &event = get_event (i);
+      if (logical_locations::key logical_loc = event.get_logical_location ())
 	if (m_logical_loc_mgr.function_p (logical_loc))
 	  {
 	    *out_idx = i;
@@ -194,14 +197,14 @@ diagnostic_path::get_first_event_in_a_function (unsigned *out_idx) const
    function, or false if it is purely intraprocedural.  */
 
 bool
-diagnostic_path::interprocedural_p () const
+path::interprocedural_p () const
 {
   /* Ignore leading events that are outside of any function.  */
   unsigned first_fn_event_idx;
   if (!get_first_event_in_a_function (&first_fn_event_idx))
     return false;
 
-  const diagnostic_event &first_fn_event = get_event (first_fn_event_idx);
+  const event &first_fn_event = get_event (first_fn_event_idx);
   int first_fn_stack_depth = first_fn_event.get_stack_depth ();
 
   const unsigned num = num_events ();
@@ -218,10 +221,10 @@ diagnostic_path::interprocedural_p () const
 /* Print PATH by emitting a dummy "note" associated with it.  */
 
 DEBUG_FUNCTION
-void debug (diagnostic_path *path)
+void debug (path *p)
 {
   rich_location richloc (line_table, UNKNOWN_LOCATION);
-  richloc.set_path (path);
+  richloc.set_path (p);
   inform (&richloc, "debug path");
 }
 
diff --git a/gcc/diagnostic-path.h b/gcc/diagnostics/paths.h
index e68f768..d30c420 100644
--- a/gcc/diagnostic-path.h
+++ b/gcc/diagnostics/paths.h
@@ -18,19 +18,28 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_PATH_H
-#define GCC_DIAGNOSTIC_PATH_H
+#ifndef GCC_DIAGNOSTICS_PATHS_H
+#define GCC_DIAGNOSTICS_PATHS_H
 
 #include "diagnostic.h" /* for ATTRIBUTE_GCC_DIAG.  */
-#include "diagnostic-event-id.h"
-#include "logical-location.h"
-
-namespace xml { class document; }
-
-class sarif_builder;
-class sarif_object;
-
-/* A diagnostic_path is an optional additional piece of metadata associated
+#include "diagnostics/event-id.h"
+#include "diagnostics/logical-locations.h"
+
+namespace diagnostics {
+  namespace digraphs {
+    class digraph;
+  } // namespace digraphs
+  namespace logical_locations {
+    class manager;
+  } // logical_locations
+  class sarif_builder;
+  class sarif_object;
+} //namespace diagnostics
+
+namespace diagnostics {
+namespace paths {
+
+/* A diagnostics::paths::path is an optional additional piece of metadata associated
    with a diagnostic (via its rich_location).
 
    It describes a sequence of events predicted by the compiler that
@@ -68,9 +77,9 @@ class sarif_object;
 /* Abstract base classes, describing events within a path, and the paths
    themselves.  */
 
-/* One event within a diagnostic_path.  */
+/* One event within a path.  */
 
-class diagnostic_event
+class event
 {
  public:
   /* Enums for giving a sense of what this event means.
@@ -142,7 +151,7 @@ class diagnostic_event
     enum property m_property;
   };
 
-  virtual ~diagnostic_event () {}
+  virtual ~event () {}
 
   virtual location_t get_location () const = 0;
 
@@ -154,7 +163,7 @@ class diagnostic_event
   virtual void print_desc (pretty_printer &pp) const = 0;
 
   /* Get a logical location for this event, or null if there is none.  */
-  virtual logical_location get_logical_location () const = 0;
+  virtual logical_locations::key get_logical_location () const = 0;
 
   virtual meaning get_meaning () const = 0;
 
@@ -162,7 +171,7 @@ class diagnostic_event
      next event (e.g. to highlight control flow).  */
   virtual bool connect_to_next_event_p () const = 0;
 
-  virtual diagnostic_thread_id_t get_thread_id () const = 0;
+  virtual thread_id_t get_thread_id () const = 0;
 
   /* Hook for SARIF output to allow for adding diagnostic-specific
      properties to the threadFlowLocation object's property bag.  */
@@ -173,36 +182,36 @@ class diagnostic_event
   }
 
   /* Hook for capturing state at this event, potentially for visualizing
-     in HTML output.  */
-  virtual std::unique_ptr<xml::document>
-  maybe_make_xml_state (bool debug) const;
+     in HTML output, or for adding to SARIF.  */
+  virtual std::unique_ptr<digraphs::digraph>
+  maybe_make_diagnostic_state_graph (bool debug) const;
 
   label_text get_desc (pretty_printer &ref_pp) const;
 };
 
 /* Abstract base class representing a thread of execution within
-   a diagnostic_path.
-   Each diagnostic_event is associated with one thread.
-   Typically there is just one thread per diagnostic_path. */
+   a diagnostics::paths::path.
+   Each event is associated with one thread.
+   Typically there is just one thread per diagnostics::paths::path. */
 
-class diagnostic_thread
+class thread
 {
 public:
-  virtual ~diagnostic_thread () {}
+  virtual ~thread () {}
   virtual label_text get_name (bool can_colorize) const = 0;
 };
 
 /* Abstract base class for getting at a sequence of events.  */
 
-class diagnostic_path
+class path
 {
  public:
-  virtual ~diagnostic_path () {}
+  virtual ~path () {}
   virtual unsigned num_events () const = 0;
-  virtual const diagnostic_event & get_event (int idx) const = 0;
+  virtual const event & get_event (int idx) const = 0;
   virtual unsigned num_threads () const = 0;
-  virtual const diagnostic_thread &
-  get_thread (diagnostic_thread_id_t) const = 0;
+  virtual const thread &
+  get_thread (thread_id_t) const = 0;
 
   /* Return true iff the two events are both within the same function,
      or both outside of any function.  */
@@ -213,13 +222,13 @@ class diagnostic_path
   bool interprocedural_p () const;
   bool multithreaded_p () const;
 
-  const logical_location_manager &get_logical_location_manager () const
+  const logical_locations::manager &get_logical_location_manager () const
   {
     return m_logical_loc_mgr;
   }
 
 protected:
-  diagnostic_path (const logical_location_manager &logical_loc_mgr)
+  path (const logical_locations::manager &logical_loc_mgr)
   : m_logical_loc_mgr (logical_loc_mgr)
   {
   }
@@ -227,12 +236,15 @@ protected:
 private:
   bool get_first_event_in_a_function (unsigned *out_idx) const;
 
-  const logical_location_manager &m_logical_loc_mgr;
+  const logical_locations::manager &m_logical_loc_mgr;
 };
 
+} // namespace paths
+} // namespace diagnostics
+
 /* Concrete subclasses of the above can be found in
    simple-diagnostic-path.h.  */
 
-extern void debug (diagnostic_path *path);
+extern void debug (diagnostics::paths::path *path);
 
-#endif /* ! GCC_DIAGNOSTIC_PATH_H */
+#endif /* ! GCC_DIAGNOSTICS_PATHS_H */
diff --git a/gcc/diagnostic-format-sarif.cc b/gcc/diagnostics/sarif-sink.cc
index 14cdbc2..4738ae9 100644
--- a/gcc/diagnostic-format-sarif.cc
+++ b/gcc/diagnostics/sarif-sink.cc
@@ -27,23 +27,25 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "diagnostic.h"
-#include "diagnostic-metadata.h"
-#include "diagnostic-path.h"
-#include "diagnostic-format.h"
-#include "diagnostic-buffer.h"
+#include "diagnostics/metadata.h"
+#include "diagnostics/digraphs.h"
+#include "diagnostics/state-graphs.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/buffering.h"
 #include "json.h"
 #include "cpplib.h"
-#include "logical-location.h"
-#include "diagnostic-client-data-hooks.h"
-#include "diagnostic-diagram.h"
+#include "diagnostics/logical-locations.h"
+#include "diagnostics/client-data-hooks.h"
+#include "diagnostics/diagram.h"
 #include "text-art/canvas.h"
-#include "diagnostic-format-sarif.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/sarif-sink.h"
+#include "diagnostics/text-sink.h"
 #include "ordered-hash-map.h"
 #include "sbitmap.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
-#include "selftest-diagnostic-show-locus.h"
+#include "diagnostics/selftest-context.h"
+#include "diagnostics/selftest-source-printing.h"
 #include "selftest-json.h"
 #include "text-range-label.h"
 #include "pretty-print-format-impl.h"
@@ -52,6 +54,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "backtrace.h"
 #include "xml.h"
 
+namespace diagnostics {
+
 /* A json::array where the values are "unique" as per
    SARIF v2.1.0 section 3.7.3 ("Array properties with unique values").  */
 
@@ -448,9 +452,9 @@ public:
 
   void
   on_nested_diagnostic (const diagnostic_info &diagnostic,
-			diagnostic_t orig_diag_kind,
+			enum kind orig_diag_kind,
 			sarif_builder &builder);
-  void on_diagram (const diagnostic_diagram &diagram,
+  void on_diagram (const diagram &d,
 		   sarif_builder &builder);
 
 private:
@@ -530,22 +534,22 @@ public:
   unsigned get_index_within_parent () const { return m_idx_within_parent; }
 
   sarif_thread_flow &
-  get_or_append_thread_flow (const diagnostic_thread &thread,
-			     diagnostic_thread_id_t thread_id);
+  get_or_append_thread_flow (const paths::thread &thread,
+			     paths::thread_id_t thread_id);
 
   sarif_thread_flow &
-  get_thread_flow (diagnostic_thread_id_t thread_id);
+  get_thread_flow (paths::thread_id_t thread_id);
 
   void add_location (sarif_thread_flow_location &);
 
   sarif_thread_flow_location &
-  get_thread_flow_loc_obj (diagnostic_event_id_t event_id) const;
+  get_thread_flow_loc_obj (paths::event_id_t event_id) const;
 
 private:
   sarif_result &m_parent;
   const unsigned m_idx_within_parent;
 
-  hash_map<int_hash<diagnostic_thread_id_t, -1, -2>,
+  hash_map<int_hash<paths::thread_id_t, -1, -2>,
 	   sarif_thread_flow *> m_thread_id_map; // borrowed ptr
   json::array *m_thread_flows_arr; // borrowed
 
@@ -561,7 +565,7 @@ class sarif_thread_flow : public sarif_object
 {
 public:
   sarif_thread_flow (sarif_code_flow &parent,
-		     const diagnostic_thread &thread,
+		     const paths::thread &thread,
 		     unsigned idx_within_parent);
 
   sarif_code_flow &get_parent () const { return m_parent; }
@@ -658,20 +662,20 @@ public:
   render (const sarif_builder &builder) const = 0;
 };
 
-/* Concrete buffering implementation subclass for JSON output.  */
+/* Concrete buffering implementation subclass for SARIF output.  */
 
-class diagnostic_sarif_format_buffer : public diagnostic_per_format_buffer
+class sarif_sink_buffer : public per_sink_buffer
 {
 public:
-  friend class sarif_output_format;
+  friend class sarif_sink;
 
-  diagnostic_sarif_format_buffer (sarif_builder &builder)
+  sarif_sink_buffer (sarif_builder &builder)
   : m_builder (builder)
   {}
 
   void dump (FILE *out, int indent) const final override;
   bool empty_p () const final override;
-  void move_to (diagnostic_per_format_buffer &dest) final override;
+  void move_to (per_sink_buffer &dest) final override;
   void clear () final override;
   void flush () final override;
 
@@ -741,15 +745,15 @@ sarif_serialization_format_json::write_to_file (FILE *outf,
    - doesn't capture -Werror cleanly
    - doesn't capture inlining information (can SARIF handle this?)
    - doesn't capture macro expansion information (can SARIF handle this?).
-   - doesn't capture any diagnostic_metadata::rules associated with
+   - doesn't capture any diagnostics::metadata::rules associated with
      a diagnostic.  */
 
 class sarif_builder
 {
 public:
-  friend class diagnostic_sarif_format_buffer;
+  friend class sarif_sink_buffer;
 
-  sarif_builder (diagnostic_context &context,
+  sarif_builder (diagnostics::context &dc,
 		 pretty_printer &printer,
 		 const line_maps *line_maps,
 		 std::unique_ptr<sarif_serialization_format> serialization_format,
@@ -761,7 +765,7 @@ public:
     m_printer = &printer;
   }
 
-  const logical_location_manager *
+  const logical_locations::manager *
   get_logical_location_manager () const
   {
     return m_logical_loc_mgr;
@@ -771,11 +775,14 @@ public:
   set_main_input_filename (const char *name);
 
   void on_report_diagnostic (const diagnostic_info &diagnostic,
-			     diagnostic_t orig_diag_kind,
-			     diagnostic_sarif_format_buffer *buffer);
-  void emit_diagram (const diagnostic_diagram &diagram);
+			     enum kind orig_diag_kind,
+			     sarif_sink_buffer *buffer);
+  void emit_diagram (const diagram &d);
   void end_group ();
 
+  void
+  report_global_digraph (const lazily_created<digraphs::digraph> &);
+
   std::unique_ptr<sarif_result> take_current_result ()
   {
     return std::move (m_cur_group_result);
@@ -789,9 +796,9 @@ public:
 		      const diagnostic_info &diagnostic,
 		      enum diagnostic_artifact_role role);
   std::unique_ptr<sarif_location>
-  make_location_object (sarif_location_manager &loc_mgr,
+  make_location_object (sarif_location_manager *loc_mgr,
 			const rich_location &rich_loc,
-			logical_location logical_loc,
+			logical_locations::key logical_loc,
 			enum diagnostic_artifact_role role);
   std::unique_ptr<sarif_location>
   make_location_object (sarif_location_manager &loc_mgr,
@@ -800,7 +807,7 @@ public:
   std::unique_ptr<sarif_message>
   make_message_object (const char *msg) const;
   std::unique_ptr<sarif_message>
-  make_message_object_for_diagram (const diagnostic_diagram &diagram);
+  make_message_object_for_diagram (const diagram &d);
   std::unique_ptr<sarif_artifact_content>
   maybe_make_artifact_content_object (const char *filename) const;
 
@@ -813,7 +820,7 @@ public:
     return m_current_code_flow;
   }
 
-  diagnostic_context &get_context () const { return m_context; }
+  diagnostics::context &get_context () const { return m_context; }
   pretty_printer *get_printer () const { return m_printer; }
   token_printer &get_token_printer () { return m_token_printer; }
   enum sarif_version get_version () const { return m_sarif_gen_opts.m_version; }
@@ -829,7 +836,7 @@ public:
   const sarif_generation_options &get_opts () const { return m_sarif_gen_opts; }
 
   std::unique_ptr<sarif_logical_location>
-  make_minimal_sarif_logical_location (logical_location);
+  make_minimal_sarif_logical_location (logical_locations::key);
 
 private:
   class sarif_token_printer : public token_printer
@@ -847,7 +854,7 @@ private:
 
   std::unique_ptr<sarif_result>
   make_result_object (const diagnostic_info &diagnostic,
-		      diagnostic_t orig_diag_kind,
+		      enum kind orig_diag_kind,
 		      unsigned idx_within_parent);
   void
   add_any_include_chain (sarif_location_manager &loc_mgr,
@@ -855,22 +862,22 @@ private:
 			 location_t where);
   void
   set_any_logical_locs_arr (sarif_location &location_obj,
-			    logical_location logical_loc);
+			    logical_locations::key logical_loc);
   std::unique_ptr<sarif_location>
   make_location_object (sarif_location_manager &loc_mgr,
-			const diagnostic_event &event,
+			const paths::event &event,
 			enum diagnostic_artifact_role role);
   std::unique_ptr<sarif_code_flow>
   make_code_flow_object (sarif_result &result,
 			 unsigned idx_within_parent,
-			 const diagnostic_path &path);
+			 const paths::path &path);
   void
   populate_thread_flow_location_object (sarif_result &result,
 					sarif_thread_flow_location &thread_flow_loc_obj,
-					const diagnostic_event &event,
+					const paths::event &event,
 					int event_execution_idx);
   std::unique_ptr<json::array>
-  maybe_make_kinds_array (diagnostic_event::meaning m) const;
+  maybe_make_kinds_array (paths::event::meaning m) const;
   std::unique_ptr<sarif_physical_location>
   maybe_make_physical_location_object (location_t loc,
 				       enum diagnostic_artifact_role role,
@@ -890,7 +897,7 @@ private:
   make_region_object_for_hint (const fixit_hint &hint) const;
 
   int
-  ensure_sarif_logical_location_for (logical_location k);
+  ensure_sarif_logical_location_for (logical_locations::key k);
 
   std::unique_ptr<sarif_multiformat_message_string>
   make_multiformat_message_string (const char *msg) const;
@@ -911,7 +918,7 @@ private:
   make_tool_component_reference_object_for_cwe () const;
   std::unique_ptr<sarif_reporting_descriptor>
   make_reporting_descriptor_object_for_warning (const diagnostic_info &diagnostic,
-						diagnostic_t orig_diag_kind,
+						enum kind orig_diag_kind,
 						const char *option_text);
   std::unique_ptr<sarif_reporting_descriptor>
   make_reporting_descriptor_object_for_cwe_id (int cwe_id) const;
@@ -943,12 +950,12 @@ private:
   std::unique_ptr<json::object>
   make_stack_from_backtrace ();
 
-  diagnostic_context &m_context;
+  diagnostics::context &m_context;
   pretty_printer *m_printer;
   const line_maps *m_line_maps;
   sarif_token_printer m_token_printer;
 
-  const logical_location_manager *m_logical_loc_mgr;
+  const logical_locations::manager *m_logical_loc_mgr;
 
   /* The JSON object for the invocation object.  */
   std::unique_ptr<sarif_invocation> m_invocation_obj;
@@ -974,6 +981,8 @@ private:
 
   std::unique_ptr<sarif_array_of_unique<sarif_logical_location>> m_cached_logical_locs;
 
+  std::unique_ptr<sarif_array_of_unique<sarif_graph>> m_run_graphs;
+
   int m_tabstop;
 
   std::unique_ptr<sarif_serialization_format> m_serialization_format;
@@ -1053,10 +1062,10 @@ sarif_invocation::add_notification_for_ice (const diagnostic_info &diagnostic,
 void
 sarif_invocation::prepare_to_flush (sarif_builder &builder)
 {
-  const diagnostic_context &context = builder.get_context ();
+  const context &dc = builder.get_context ();
 
   /* "executionSuccessful" property (SARIF v2.1.0 section 3.20.14).  */
-  if (context.execution_failed_p ())
+  if (dc.execution_failed_p ())
     m_success = false;
   set_bool ("executionSuccessful", m_success);
 
@@ -1065,7 +1074,7 @@ sarif_invocation::prepare_to_flush (sarif_builder &builder)
 
   /* Call client hook, allowing it to create a custom property bag for
      this object (SARIF v2.1.0 section 3.8) e.g. for recording time vars.  */
-  if (auto client_data_hooks = context.get_client_data_hooks ())
+  if (auto client_data_hooks = dc.get_client_data_hooks ())
     client_data_hooks->add_sarif_invocation_properties (*this);
 
   // "endTimeUtc" property (SARIF v2.1.0 section 3.20.8);
@@ -1298,15 +1307,15 @@ sarif_location_manager::process_worklist_item (sarif_builder &builder,
 
 void
 sarif_result::on_nested_diagnostic (const diagnostic_info &diagnostic,
-				    diagnostic_t /*orig_diag_kind*/,
+				    enum kind /*orig_diag_kind*/,
 				    sarif_builder &builder)
 {
   /* We don't yet generate meaningful logical locations for notes;
      sometimes these will related to current_function_decl, but
      often they won't.  */
   auto location_obj
-    = builder.make_location_object (*this, *diagnostic.richloc,
-				    logical_location (),
+    = builder.make_location_object (this, *diagnostic.m_richloc,
+				    logical_locations::key (),
 				    diagnostic_artifact_role::result_file);
   auto message_obj
     = builder.make_message_object (pp_formatted_text (builder.get_printer ()));
@@ -1329,11 +1338,11 @@ sarif_result::on_nested_diagnostic (const diagnostic_info &diagnostic,
    (SARIF v2.1.0 section 3.28.5).  */
 
 void
-sarif_result::on_diagram (const diagnostic_diagram &diagram,
+sarif_result::on_diagram (const diagram &d,
 			  sarif_builder &builder)
 {
   auto location_obj = std::make_unique<sarif_location> ();
-  auto message_obj = builder.make_message_object_for_diagram (diagram);
+  auto message_obj = builder.make_message_object_for_diagram (d);
   location_obj->set<sarif_message> ("message", std::move (message_obj));
 
   add_related_location (std::move (location_obj), builder);
@@ -1565,8 +1574,8 @@ sarif_code_flow::sarif_code_flow (sarif_result &parent,
 }
 
 sarif_thread_flow &
-sarif_code_flow::get_or_append_thread_flow (const diagnostic_thread &thread,
-					    diagnostic_thread_id_t thread_id)
+sarif_code_flow::get_or_append_thread_flow (const paths::thread &thread,
+					    paths::thread_id_t thread_id)
 {
   sarif_thread_flow **slot = m_thread_id_map.get (thread_id);
   if (slot)
@@ -1582,7 +1591,7 @@ sarif_code_flow::get_or_append_thread_flow (const diagnostic_thread &thread,
 }
 
 sarif_thread_flow &
-sarif_code_flow::get_thread_flow (diagnostic_thread_id_t thread_id)
+sarif_code_flow::get_thread_flow (paths::thread_id_t thread_id)
 {
   sarif_thread_flow **slot = m_thread_id_map.get (thread_id);
   gcc_assert (slot); // it must already have one
@@ -1596,7 +1605,7 @@ sarif_code_flow::add_location (sarif_thread_flow_location &tfl_obj)
 }
 
 sarif_thread_flow_location &
-sarif_code_flow::get_thread_flow_loc_obj (diagnostic_event_id_t event_id) const
+sarif_code_flow::get_thread_flow_loc_obj (paths::event_id_t event_id) const
 {
   gcc_assert (event_id.known_p ());
   gcc_assert ((size_t)event_id.zero_based () < m_all_tfl_objs.size ());
@@ -1608,7 +1617,7 @@ sarif_code_flow::get_thread_flow_loc_obj (diagnostic_event_id_t event_id) const
 /* class sarif_thread_flow : public sarif_object.  */
 
 sarif_thread_flow::sarif_thread_flow (sarif_code_flow &parent,
-				      const diagnostic_thread &thread,
+				      const paths::thread &thread,
 				      unsigned idx_within_parent)
 : m_parent (parent),
   m_idx_within_parent (idx_within_parent)
@@ -1643,19 +1652,19 @@ sarif_thread_flow::add_location ()
 
 /* sarif_builder's ctor.  */
 
-sarif_builder::sarif_builder (diagnostic_context &context,
+sarif_builder::sarif_builder (diagnostics::context &dc,
 			      pretty_printer &printer,
 			      const line_maps *line_maps,
 			      std::unique_ptr<sarif_serialization_format> serialization_format,
 			      const sarif_generation_options &sarif_gen_opts)
-: m_context (context),
+: m_context (dc),
   m_printer (&printer),
   m_line_maps (line_maps),
   m_token_printer (*this),
   m_logical_loc_mgr (nullptr),
   m_invocation_obj
     (std::make_unique<sarif_invocation> (*this,
-					 context.get_original_argv ())),
+					 dc.get_original_argv ())),
   m_results_array (new json::array ()),
   m_cur_group_result (nullptr),
   m_seen_any_relative_paths (false),
@@ -1663,7 +1672,9 @@ sarif_builder::sarif_builder (diagnostic_context &context,
   m_rules_arr (new json::array ()),
   m_cached_logical_locs
     (std::make_unique<sarif_array_of_unique<sarif_logical_location>> ()),
-  m_tabstop (context.m_tabstop),
+  m_run_graphs
+    (std::make_unique<sarif_array_of_unique<sarif_graph>> ()),
+  m_tabstop (dc.m_tabstop),
   m_serialization_format (std::move (serialization_format)),
   m_sarif_gen_opts (sarif_gen_opts),
   m_next_result_idx (0),
@@ -1672,7 +1683,7 @@ sarif_builder::sarif_builder (diagnostic_context &context,
   gcc_assert (m_line_maps);
   gcc_assert (m_serialization_format);
 
-  if (auto client_data_hooks = context.get_client_data_hooks ())
+  if (auto client_data_hooks = dc.get_client_data_hooks ())
     m_logical_loc_mgr = client_data_hooks->get_logical_location_manager ();
 }
 
@@ -1725,10 +1736,10 @@ bt_callback (void *data, uintptr_t pc, const char *filename, int lineno,
   if (filename == nullptr && function == nullptr)
     return 0;
 
-  /* Skip functions in diagnostic.cc or diagnostic-global-context.cc.  */
+  /* Skip functions in context.cc or diagnostic-global-context.cc.  */
   if (closure->m_frames_arr->size () == 0
       && filename != nullptr
-      && (strcmp (lbasename (filename), "diagnostic.cc") == 0
+      && (strcmp (lbasename (filename), "context.cc") == 0
 	  || strcmp (lbasename (filename),
 		     "diagnostic-global-context.cc") == 0))
     return 0;
@@ -1829,12 +1840,12 @@ sarif_builder::set_main_input_filename (const char *name)
 
 void
 sarif_builder::on_report_diagnostic (const diagnostic_info &diagnostic,
-				     diagnostic_t orig_diag_kind,
-				     diagnostic_sarif_format_buffer *buffer)
+				     enum kind orig_diag_kind,
+				     sarif_sink_buffer *buffer)
 {
   pp_output_formatted_text (m_printer, m_context.get_urlifier ());
 
-  if (diagnostic.kind == DK_ICE || diagnostic.kind == DK_ICE_NOBT)
+  if (diagnostic.m_kind == kind::ice || diagnostic.m_kind == kind::ice_nobt)
     {
       std::unique_ptr<json::object> stack = make_stack_from_backtrace ();
       m_invocation_obj->add_notification_for_ice (diagnostic, *this,
@@ -1872,15 +1883,15 @@ sarif_builder::on_report_diagnostic (const diagnostic_info &diagnostic,
     }
 }
 
-/* Implementation of diagnostic_context::m_diagrams.m_emission_cb
+/* Implementation of diagnostics::context::m_diagrams.m_emission_cb
    for SARIF output.  */
 
 void
-sarif_builder::emit_diagram (const diagnostic_diagram &diagram)
+sarif_builder::emit_diagram (const diagram &d)
 {
   /* We must be within the emission of a top-level diagnostic.  */
   gcc_assert (m_cur_group_result);
-  m_cur_group_result->on_diagram (diagram, *this);
+  m_cur_group_result->on_diagram (d, *this);
 }
 
 /* Implementation of "end_group_cb" for SARIF output.  */
@@ -1895,6 +1906,17 @@ sarif_builder::end_group ()
     }
 }
 
+void
+sarif_builder::
+report_global_digraph (const lazily_created<digraphs::digraph> &ldg)
+{
+  auto &dg = ldg.get_or_create ();
+
+  /* Presumably the location manager must be nullptr; see
+     https://github.com/oasis-tcs/sarif-spec/issues/712  */
+  m_run_graphs->append (make_sarif_graph (dg, this, nullptr));
+}
+
 /* Create a top-level object, and add it to all the results
    (and other entities) we've seen so far, moving ownership
    to the object.  */
@@ -1927,16 +1949,16 @@ sarif_builder::flush_to_file (FILE *outf)
    Return nullptr if there isn't one.  */
 
 static const char *
-maybe_get_sarif_level (diagnostic_t diag_kind)
+maybe_get_sarif_level (enum kind diag_kind)
 {
   switch (diag_kind)
     {
-    case DK_WARNING:
+    case kind::warning:
       return "warning";
-    case DK_ERROR:
+    case kind::error:
       return "error";
-    case DK_NOTE:
-    case DK_ANACHRONISM:
+    case kind::note:
+    case kind::anachronism:
       return "note";
     default:
       return nullptr;
@@ -1948,10 +1970,10 @@ maybe_get_sarif_level (diagnostic_t diag_kind)
    have anything better to use.  */
 
 static char *
-make_rule_id_for_diagnostic_kind (diagnostic_t diag_kind)
+make_rule_id_for_diagnostic_kind (enum kind diag_kind)
 {
   /* Lose the trailing ": ".  */
-  const char *kind_text = get_diagnostic_kind_text (diag_kind);
+  const char *kind_text = get_text_for_kind (diag_kind);
   size_t len = strlen (kind_text);
   gcc_assert (len > 2);
   gcc_assert (kind_text[len - 2] == ':');
@@ -1965,7 +1987,7 @@ make_rule_id_for_diagnostic_kind (diagnostic_t diag_kind)
 
 std::unique_ptr<sarif_result>
 sarif_builder::make_result_object (const diagnostic_info &diagnostic,
-				   diagnostic_t orig_diag_kind,
+				   enum kind orig_diag_kind,
 				   unsigned idx_within_parent)
 {
   auto result_obj = std::make_unique<sarif_result> (idx_within_parent);
@@ -1973,8 +1995,8 @@ sarif_builder::make_result_object (const diagnostic_info &diagnostic,
   /* "ruleId" property (SARIF v2.1.0 section 3.27.5).  */
   /* Ideally we'd have an option_name for these.  */
   if (char *option_text
-	= m_context.make_option_name (diagnostic.option_id,
-				      orig_diag_kind, diagnostic.kind))
+	= m_context.make_option_name (diagnostic.m_option_id,
+				      orig_diag_kind, diagnostic.m_kind))
     {
       /* Lazily create reportingDescriptor objects for and add to m_rules_arr.
 	 Set ruleId referencing them.  */
@@ -2004,10 +2026,10 @@ sarif_builder::make_result_object (const diagnostic_info &diagnostic,
       free (rule_id);
     }
 
-  if (diagnostic.metadata)
+  if (diagnostic.m_metadata)
     {
       /* "taxa" property (SARIF v2.1.0 section 3.27.8).  */
-      if (int cwe_id = diagnostic.metadata->get_cwe ())
+      if (int cwe_id = diagnostic.m_metadata->get_cwe ())
 	{
 	  auto taxa_arr = std::make_unique<json::array> ();
 	  taxa_arr->append<sarif_reporting_descriptor_reference>
@@ -2015,13 +2037,13 @@ sarif_builder::make_result_object (const diagnostic_info &diagnostic,
 	  result_obj->set<json::array> ("taxa", std::move (taxa_arr));
 	}
 
-      diagnostic.metadata->maybe_add_sarif_properties (*result_obj);
+      diagnostic.m_metadata->maybe_add_sarif_properties (*result_obj);
 
-      /* We don't yet support diagnostic_metadata::rule.  */
+      /* We don't yet support diagnostics::metadata::rule.  */
     }
 
   /* "level" property (SARIF v2.1.0 section 3.27.10).  */
-  if (const char *sarif_level = maybe_get_sarif_level (diagnostic.kind))
+  if (const char *sarif_level = maybe_get_sarif_level (diagnostic.m_kind))
     result_obj->set_string ("level", sarif_level);
 
   /* "message" property (SARIF v2.1.0 section 3.27.11).  */
@@ -2038,7 +2060,7 @@ sarif_builder::make_result_object (const diagnostic_info &diagnostic,
 			 diagnostic_artifact_role::result_file));
 
   /* "codeFlows" property (SARIF v2.1.0 section 3.27.18).  */
-  if (const diagnostic_path *path = diagnostic.richloc->get_path ())
+  if (const paths::path *path = diagnostic.m_richloc->get_path ())
     {
       auto code_flows_arr = std::make_unique<json::array> ();
       const unsigned code_flow_index = 0;
@@ -2049,12 +2071,25 @@ sarif_builder::make_result_object (const diagnostic_info &diagnostic,
       result_obj->set<json::array> ("codeFlows", std::move (code_flows_arr));
     }
 
+  // "graphs" property (SARIF v2.1.0 section 3.27.19).  */
+  if (diagnostic.m_metadata)
+    if (auto ldg = diagnostic.m_metadata->get_lazy_digraphs ())
+      {
+	auto &digraphs = ldg->get_or_create ();
+	auto graphs_arr = std::make_unique<json::array> ();
+	for (auto &iter : digraphs)
+	  graphs_arr->append (make_sarif_graph (*iter, this,
+						result_obj.get ()));
+	if (graphs_arr->size () > 0)
+	  result_obj->set<json::array> ("graphs", std::move (graphs_arr));
+      }
+
   /* The "relatedLocations" property (SARIF v2.1.0 section 3.27.22) is
      set up later, if any nested diagnostics occur within this diagnostic
      group.  */
 
   /* "fixes" property (SARIF v2.1.0 section 3.27.30).  */
-  const rich_location *richloc = diagnostic.richloc;
+  const rich_location *richloc = diagnostic.m_richloc;
   if (richloc->get_num_fixit_hints ())
     {
       auto fix_arr = std::make_unique<json::array> ();
@@ -2071,7 +2106,7 @@ sarif_builder::make_result_object (const diagnostic_info &diagnostic,
 std::unique_ptr<sarif_reporting_descriptor>
 sarif_builder::
 make_reporting_descriptor_object_for_warning (const diagnostic_info &diagnostic,
-					      diagnostic_t /*orig_diag_kind*/,
+					      enum kind /*orig_diag_kind*/,
 					      const char *option_text)
 {
   auto reporting_desc = std::make_unique<sarif_reporting_descriptor> ();
@@ -2083,7 +2118,7 @@ make_reporting_descriptor_object_for_warning (const diagnostic_info &diagnostic,
      it seems redundant compared to "id".  */
 
   /* "helpUri" property (SARIF v2.1.0 section 3.49.12).  */
-  if (char *option_url = m_context.make_option_url (diagnostic.option_id))
+  if (char *option_url = m_context.make_option_url (diagnostic.m_option_id))
     {
       reporting_desc->set_string ("helpUri", option_url);
       free (option_url);
@@ -2171,12 +2206,12 @@ sarif_builder::make_locations_arr (sarif_location_manager &loc_mgr,
 				   enum diagnostic_artifact_role role)
 {
   auto locations_arr = std::make_unique<json::array> ();
-  logical_location logical_loc;
+  logical_locations::key logical_loc;
   if (auto client_data_hooks = m_context.get_client_data_hooks ())
     logical_loc = client_data_hooks->get_current_logical_location ();
 
   auto location_obj
-    = make_location_object (loc_mgr, *diagnostic.richloc, logical_loc, role);
+    = make_location_object (&loc_mgr, *diagnostic.m_richloc, logical_loc, role);
   /* Don't add entirely empty location objects to the array.  */
   if (!location_obj->is_empty ())
     locations_arr->append<sarif_location> (std::move (location_obj));
@@ -2191,7 +2226,7 @@ sarif_builder::make_locations_arr (sarif_location_manager &loc_mgr,
 void
 sarif_builder::
 set_any_logical_locs_arr (sarif_location &location_obj,
-			  logical_location logical_loc)
+			  logical_locations::key logical_loc)
 {
   if (!logical_loc)
     return;
@@ -2210,12 +2245,14 @@ set_any_logical_locs_arr (sarif_location &location_obj,
 /* Make a "location" object (SARIF v2.1.0 section 3.28) for RICH_LOC
    and LOGICAL_LOC.
    Use LOC_MGR for any locations that need "id" values, and for
-   any worklist items.  */
+   any worklist items.
+   Note that we might not always have a LOC_MGR; see
+   https://github.com/oasis-tcs/sarif-spec/issues/712  */
 
 std::unique_ptr<sarif_location>
-sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
+sarif_builder::make_location_object (sarif_location_manager *loc_mgr,
 				     const rich_location &rich_loc,
-				     logical_location logical_loc,
+				     logical_locations::key logical_loc,
 				     enum diagnostic_artifact_role role)
 {
   class escape_nonascii_renderer : public content_renderer
@@ -2230,21 +2267,22 @@ sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
     std::unique_ptr<sarif_multiformat_message_string>
     render (const sarif_builder &builder) const final override
     {
-      diagnostic_context dc;
+      diagnostics::context dc;
       diagnostic_initialize (&dc, 0);
-      dc.m_source_printing.enabled = true;
-      dc.m_source_printing.colorize_source_p = false;
-      dc.m_source_printing.show_labels_p = true;
-      dc.m_source_printing.show_line_numbers_p = true;
+      auto &source_printing_opts = dc.get_source_printing_options ();
+      source_printing_opts.enabled = true;
+      source_printing_opts.colorize_source_p = false;
+      source_printing_opts.show_labels_p = true;
+      source_printing_opts.show_line_numbers_p = true;
 
       rich_location my_rich_loc (m_richloc);
       my_rich_loc.set_escape_on_output (true);
 
-      diagnostic_source_print_policy source_policy (dc);
+      source_print_policy source_policy (dc);
       dc.set_escape_format (m_escape_format);
-      diagnostic_text_output_format text_output (dc);
+      text_sink text_output (dc);
       source_policy.print (*text_output.get_printer (),
-			   my_rich_loc, DK_ERROR, nullptr);
+			   my_rich_loc, kind::error, nullptr);
 
       const char *buf = pp_formatted_text (text_output.get_printer ());
       std::unique_ptr<sarif_multiformat_message_string> result
@@ -2309,8 +2347,8 @@ sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
 
 	/* Add related locations for any secondary locations in RICH_LOC
 	   that don't have labels (and thus aren't added to "annotations"). */
-	if (i > 0 && !handled)
-	  loc_mgr.add_relationship_to_worklist
+	if (loc_mgr && i > 0 && !handled)
+	  loc_mgr->add_relationship_to_worklist
 	    (*location_obj.get (),
 	     sarif_location_manager::worklist_item::kind::unlabelled_secondary_location,
 	     range->m_loc);
@@ -2321,7 +2359,8 @@ sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
 				      std::move (annotations_arr));
   }
 
-  add_any_include_chain (loc_mgr, *location_obj.get (), loc);
+  if (loc_mgr)
+    add_any_include_chain (*loc_mgr, *location_obj.get (), loc);
 
   /* A flag for hinting that the diagnostic involves issues at the
      level of character encodings (such as homoglyphs, or misleading
@@ -2340,7 +2379,7 @@ sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
 /* If WHERE was #included from somewhere, add a worklist item
    to LOC_MGR to lazily add a location for the #include location,
    and relationships between it and the LOCATION_OBJ.
-   Compare with diagnostic_context::report_current_module, but rather
+   Compare with diagnostics::context::report_current_module, but rather
    than iterating the current chain, we add the next edge and iterate
    in the worklist, so that edges are only added once.  */
 
@@ -2392,11 +2431,11 @@ sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
 }
 
 /* Make a "location" object (SARIF v2.1.0 section 3.28) for EVENT
-   within a diagnostic_path.  */
+   within a paths::path.  */
 
 std::unique_ptr<sarif_location>
 sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
-				     const diagnostic_event &event,
+				     const paths::event &event,
 				     enum diagnostic_artifact_role role)
 {
   auto location_obj = std::make_unique<sarif_location> ();
@@ -2409,7 +2448,7 @@ sarif_builder::make_location_object (sarif_location_manager &loc_mgr,
 						std::move (phs_loc_obj));
 
   /* "logicalLocations" property (SARIF v2.1.0 section 3.28.4).  */
-  logical_location logical_loc = event.get_logical_location ();
+  logical_locations::key logical_loc = event.get_logical_location ();
   set_any_logical_locs_arr (*location_obj, logical_loc);
 
   /* "message" property (SARIF v2.1.0 section 3.28.5).  */
@@ -2730,57 +2769,59 @@ sarif_builder::make_region_object_for_hint (const fixit_hint &hint) const
    Return nullptr if unknown.  */
 
 static const char *
-maybe_get_sarif_kind (enum logical_location_kind kind)
+maybe_get_sarif_kind (enum logical_locations::kind kind)
 {
+  using namespace logical_locations;
+
   switch (kind)
     {
     default:
       gcc_unreachable ();
-    case logical_location_kind::unknown:
+    case logical_locations::kind::unknown:
       return nullptr;
 
     /* Kinds within executable code.  */
-    case logical_location_kind::function:
+    case logical_locations::kind::function:
       return "function";
-    case logical_location_kind::member:
+    case logical_locations::kind::member:
       return "member";
-    case logical_location_kind::module_:
+    case logical_locations::kind::module_:
       return "module";
-    case logical_location_kind::namespace_:
+    case logical_locations::kind::namespace_:
       return "namespace";
-    case logical_location_kind::type:
+    case logical_locations::kind::type:
       return "type";
-    case logical_location_kind::return_type:
+    case logical_locations::kind::return_type:
       return "returnType";
-    case logical_location_kind::parameter:
+    case logical_locations::kind::parameter:
       return "parameter";
-    case logical_location_kind::variable:
+    case logical_locations::kind::variable:
       return "variable";
 
     /* Kinds within XML or HTML documents.  */
-    case logical_location_kind::element:
+    case logical_locations::kind::element:
       return "element";
-    case logical_location_kind::attribute:
+    case logical_locations::kind::attribute:
       return "attribute";
-    case logical_location_kind::text:
+    case logical_locations::kind::text:
       return "text";
-    case logical_location_kind::comment:
+    case logical_locations::kind::comment:
       return "comment";
-    case logical_location_kind::processing_instruction:
+    case logical_locations::kind::processing_instruction:
       return "processingInstruction";
-    case logical_location_kind::dtd:
+    case logical_locations::kind::dtd:
       return "dtd";
-    case logical_location_kind::declaration:
+    case logical_locations::kind::declaration:
       return "declaration";
 
     /* Kinds within JSON documents.  */
-    case logical_location_kind::object:
+    case logical_locations::kind::object:
       return "object";
-    case logical_location_kind::array:
+    case logical_locations::kind::array:
       return "array";
-    case logical_location_kind::property:
+    case logical_locations::kind::property:
       return "property";
-    case logical_location_kind::value:
+    case logical_locations::kind::value:
       return "value";
     }
 }
@@ -2792,20 +2833,156 @@ maybe_get_sarif_kind (enum logical_location_kind kind)
 void
 sarif_property_bag::set_logical_location (const char *property_name,
 					  sarif_builder &builder,
-					  logical_location logical_loc)
+					  logical_locations::key logical_loc)
 {
   set<sarif_logical_location>
     (property_name,
      builder.make_minimal_sarif_logical_location (logical_loc));
 }
 
+static void
+copy_any_property_bag (const digraphs::object &input_obj,
+		       sarif_object &output_obj)
+{
+  if (input_obj.get_property_bag ())
+    {
+      const json::object &old_bag = *input_obj.get_property_bag ();
+      sarif_property_bag &new_bag = output_obj.get_or_create_properties ();
+      for (size_t i = 0; i < old_bag.get_num_keys (); ++i)
+	{
+	  const char *key = old_bag.get_key (i);
+	  json::value *val = old_bag.get (key);
+	  new_bag.set (key, val->clone ());
+	}
+    }
+}
+
+std::unique_ptr<sarif_graph>
+make_sarif_graph (const digraphs::digraph &g,
+		  sarif_builder *builder,
+		  sarif_location_manager *sarif_location_mgr)
+{
+  auto result = std::make_unique<sarif_graph> ();
+
+  // 3.39.2 description property
+  if (const char *desc = g.get_description ())
+    if (builder)
+      result->set<sarif_message> ("description",
+				  builder->make_message_object (desc));
+
+  copy_any_property_bag (g, *result);
+
+  // 3.39.3 nodes property
+  auto nodes_arr = std::make_unique<json::array> ();
+  const int num_nodes = g.get_num_nodes ();
+  for (int i = 0; i < num_nodes; ++i)
+    nodes_arr->append (make_sarif_node (g.get_node (i),
+					builder,
+					sarif_location_mgr));
+  result->set ("nodes", std::move (nodes_arr));
+
+  // 3.39.4 edges property
+  auto edges_arr = std::make_unique<json::array> ();
+  const int num_edges = g.get_num_edges ();
+  for (int i = 0; i < num_edges; ++i)
+    edges_arr->append (make_sarif_edge (g.get_edge (i), builder));
+  result->set ("edges", std::move (edges_arr));
+
+  return result;
+}
+
+std::unique_ptr<sarif_node>
+make_sarif_node (const digraphs::node &n,
+		 sarif_builder *builder,
+		 sarif_location_manager *sarif_location_mgr)
+{
+  auto result = std::make_unique<sarif_node> ();
+
+  // 3.40.2 id property
+  result->set_string ("id", n.get_id ().c_str ());
+
+  copy_any_property_bag (n, *result);
+
+  // 3.40.3 label property
+  if (const char *label = n.get_label ())
+    if (builder)
+      result->set<sarif_message> ("label",
+				  builder->make_message_object (label));
+
+  // 3.40.4 location property
+  if (n.get_logical_loc ()
+      || n.get_physical_loc () != UNKNOWN_LOCATION)
+    if (builder)
+      {
+	rich_location rich_loc
+	  (line_table, n.get_physical_loc ());
+	auto loc_obj
+	  = builder->make_location_object
+	      (sarif_location_mgr,
+	       rich_loc,
+	       n.get_logical_loc (),
+	       diagnostic_artifact_role::scanned_file);
+	result->set<sarif_location> ("location",
+				     std::move (loc_obj));
+    }
+
+  // 3.40.5 children property
+  if (const int num_children = n.get_num_children ())
+    {
+      auto children_arr = std::make_unique<json::array> ();
+      for (int i = 0; i < num_children; ++i)
+	children_arr->append (make_sarif_node (n.get_child (i),
+					       builder,
+					       sarif_location_mgr));
+      result->set ("children", std::move (children_arr));
+    }
+
+  return result;
+}
+
+std::unique_ptr<sarif_edge>
+make_sarif_edge (const digraphs::edge &e,
+		 sarif_builder *builder)
+{
+  auto result = std::make_unique<sarif_edge> ();
+
+  // 3.41.2 id property
+  result->set_string ("id", e.get_id ().c_str ());
+
+  copy_any_property_bag (e, *result);
+
+  // 3.41.3 label property
+  if (const char *label = e.get_label ())
+    if (builder)
+      result->set<sarif_message> ("label",
+				  builder->make_message_object (label));
+
+  // 3.41.4 sourceNodeId property
+  result->set_string ("sourceNodeId", e.get_src_node ().get_id ().c_str ());
+
+  // 3.41.5 targetNodeId property
+  result->set_string ("targetNodeId", e.get_dst_node ().get_id ().c_str ());
+
+  return result;
+}
+
+void
+sarif_property_bag::set_graph (const char *property_name,
+			       sarif_builder &builder,
+			       sarif_location_manager *sarif_location_mgr,
+			       const digraphs::digraph &g)
+{
+  set<sarif_graph> (property_name,
+		    make_sarif_graph (g, &builder, sarif_location_mgr));
+}
+
 /* Ensure that m_cached_logical_locs has a "logicalLocation" object
    (SARIF v2.1.0 section 3.33) for K, and return its index within the
    array.  */
 
 int
 sarif_builder::
-ensure_sarif_logical_location_for (logical_location k)
+ensure_sarif_logical_location_for (logical_locations::key k)
 {
   gcc_assert (m_logical_loc_mgr);
 
@@ -2823,7 +3000,7 @@ ensure_sarif_logical_location_for (logical_location k)
     sarif_logical_loc->set_string ("decoratedName", internal_name);
 
   /* "kind" property (SARIF v2.1.0 section 3.33.7).  */
-  enum logical_location_kind kind = m_logical_loc_mgr->get_kind (k);
+  enum logical_locations::kind kind = m_logical_loc_mgr->get_kind (k);
   if (const char *sarif_kind_str = maybe_get_sarif_kind (kind))
     sarif_logical_loc->set_string ("kind", sarif_kind_str);
 
@@ -2849,7 +3026,7 @@ ensure_sarif_logical_location_for (logical_location k)
 
 std::unique_ptr<sarif_logical_location>
 sarif_builder::
-make_minimal_sarif_logical_location (logical_location logical_loc)
+make_minimal_sarif_logical_location (logical_locations::key logical_loc)
 {
   gcc_assert (m_logical_loc_mgr);
 
@@ -2874,7 +3051,7 @@ make_minimal_sarif_logical_location (logical_location logical_loc)
 
 label_text
 make_sarif_url_for_event (const sarif_code_flow *code_flow,
-			  diagnostic_event_id_t event_id)
+			  paths::event_id_t event_id)
 {
   gcc_assert (event_id.known_p ());
 
@@ -2908,7 +3085,7 @@ make_sarif_url_for_event (const sarif_code_flow *code_flow,
 std::unique_ptr<sarif_code_flow>
 sarif_builder::make_code_flow_object (sarif_result &result,
 				      unsigned idx_within_parent,
-				      const diagnostic_path &path)
+				      const paths::path &path)
 {
   auto code_flow_obj
     = std::make_unique <sarif_code_flow> (result, idx_within_parent);
@@ -2920,8 +3097,8 @@ sarif_builder::make_code_flow_object (sarif_result &result,
      SARIF file.  */
   for (unsigned i = 0; i < path.num_events (); i++)
     {
-      const diagnostic_event &event = path.get_event (i);
-      const diagnostic_thread_id_t thread_id = event.get_thread_id ();
+      const paths::event &event = path.get_event (i);
+      const paths::thread_id_t thread_id = event.get_thread_id ();
 
       sarif_thread_flow &thread_flow_obj
 	= code_flow_obj->get_or_append_thread_flow (path.get_thread (thread_id),
@@ -2933,7 +3110,7 @@ sarif_builder::make_code_flow_object (sarif_result &result,
   m_current_code_flow = code_flow_obj.get ();
   for (unsigned i = 0; i < path.num_events (); i++)
     {
-      const diagnostic_event &event = path.get_event (i);
+      const paths::event &event = path.get_event (i);
       sarif_thread_flow_location &thread_flow_loc_obj
 	= code_flow_obj->get_thread_flow_loc_obj (i);
       populate_thread_flow_location_object (result,
@@ -2953,24 +3130,28 @@ void
 sarif_builder::
 populate_thread_flow_location_object (sarif_result &result,
 				      sarif_thread_flow_location &tfl_obj,
-				      const diagnostic_event &ev,
+				      const paths::event &ev,
 				      int event_execution_idx)
 {
-  /* Give diagnostic_event subclasses a chance to add custom properties
+  /* Give paths::event subclasses a chance to add custom properties
      via a property bag.  */
   ev.maybe_add_sarif_properties (*this, tfl_obj);
 
-  if (get_opts ().m_xml_state)
-    if (auto xml_state = ev.maybe_make_xml_state (true))
+  if (get_opts ().m_state_graph)
+    if (auto state_graph = ev.maybe_make_diagnostic_state_graph (true))
       {
 	sarif_property_bag &props = tfl_obj.get_or_create_properties ();
 
-	pretty_printer pp;
-	xml_state->write_as_xml (&pp, 0, true);
-
-#define PROPERTY_PREFIX "gcc/diagnostic_event/"
-	props.set_string (PROPERTY_PREFIX "xml_state",
-			  pp_formatted_text (&pp));
+#define PROPERTY_PREFIX "gcc/diagnostics/paths/event/"
+	props.set_graph (PROPERTY_PREFIX "state_graph",
+			 *this,
+			 /* Use RESULT for any related locations in the graph's
+			    nodes.
+			    It's not clear if this is correct; see:
+			    https://github.com/oasis-tcs/sarif-spec/issues/712
+			 */
+			 &result,
+			 *state_graph);
 #undef PROPERTY_PREFIX
       }
 
@@ -2980,7 +3161,7 @@ populate_thread_flow_location_object (sarif_result &result,
      make_location_object (result, ev, diagnostic_artifact_role::traced_file));
 
   /* "kinds" property (SARIF v2.1.0 section 3.38.8).  */
-  diagnostic_event::meaning m = ev.get_meaning ();
+  paths::event::meaning m = ev.get_meaning ();
   if (auto kinds_arr = maybe_make_kinds_array (m))
     tfl_obj.set<json::array> ("kinds", std::move (kinds_arr));
 
@@ -3003,22 +3184,25 @@ populate_thread_flow_location_object (sarif_result &result,
    Otherwise, return nullptr.  */
 
 std::unique_ptr<json::array>
-sarif_builder::maybe_make_kinds_array (diagnostic_event::meaning m) const
+sarif_builder::
+maybe_make_kinds_array (paths::event::meaning m) const
 {
-  if (m.m_verb == diagnostic_event::verb::unknown
-      && m.m_noun == diagnostic_event::noun::unknown
-      && m.m_property == diagnostic_event::property::unknown)
+  using namespace paths;
+
+  if (m.m_verb == event::verb::unknown
+      && m.m_noun == event::noun::unknown
+      && m.m_property == event::property::unknown)
     return nullptr;
 
   auto kinds_arr = std::make_unique<json::array> ();
   if (const char *verb_str
-	= diagnostic_event::meaning::maybe_get_verb_str (m.m_verb))
+	= event::meaning::maybe_get_verb_str (m.m_verb))
     kinds_arr->append_string (verb_str);
   if (const char *noun_str
-	= diagnostic_event::meaning::maybe_get_noun_str (m.m_noun))
+	= event::meaning::maybe_get_noun_str (m.m_noun))
     kinds_arr->append_string (noun_str);
   if (const char *property_str
-	= diagnostic_event::meaning::maybe_get_property_str (m.m_property))
+	= event::meaning::maybe_get_property_str (m.m_property))
     kinds_arr->append_string (property_str);
   return kinds_arr;
 }
@@ -3069,18 +3253,18 @@ sarif_builder::make_message_object (const char *msg) const
   return message_obj;
 }
 
-/* Make a "message" object (SARIF v2.1.0 section 3.11) for DIAGRAM.
+/* Make a "message" object (SARIF v2.1.0 section 3.11) for D.
    We emit the diagram as a code block within the Markdown part
    of the message.  */
 
 std::unique_ptr<sarif_message>
-sarif_builder::make_message_object_for_diagram (const diagnostic_diagram &diagram)
+sarif_builder::make_message_object_for_diagram (const diagram &d)
 {
   auto message_obj = std::make_unique<sarif_message> ();
 
   /* "text" property (SARIF v2.1.0 section 3.11.8).  */
   set_string_property_escaping_braces (*message_obj,
-				       "text", diagram.get_alt_text ());
+				       "text", d.get_alt_text ());
 
   pretty_printer *const pp = m_printer;
   char *saved_prefix = pp_take_prefix (pp);
@@ -3089,7 +3273,7 @@ sarif_builder::make_message_object_for_diagram (const diagnostic_diagram &diagra
   /* "To produce a code block in Markdown, simply indent every line of
      the block by at least 4 spaces or 1 tab."
      Here we use 4 spaces.  */
-  diagram.get_canvas ().print_to_pp (pp, "    ");
+  d.get_canvas ().print_to_pp (pp, "    ");
   pp_set_prefix (pp, saved_prefix);
 
   /* "markdown" property (SARIF v2.1.0 section 3.11.9).  */
@@ -3234,6 +3418,11 @@ make_run_object (std::unique_ptr<sarif_invocation> invocation_obj,
 				 std::move (m_cached_logical_locs));
     }
 
+  // "graphs" property (SARIF v2.1.0 3.14.20)
+  if (m_run_graphs->size () > 0)
+    run_obj->set<json::array> ("graphs",
+			       std::move (m_run_graphs));
+
   return run_obj;
 }
 
@@ -3257,7 +3446,8 @@ sarif_builder::make_tool_object ()
 	class my_plugin_visitor : public client_version_info :: plugin_visitor
 	{
 	public:
-	  void on_plugin (const diagnostic_client_plugin_info &p) final override
+	  void
+	  on_plugin (const client_plugin_info &p) final override
 	  {
 	    /* Create a "toolComponent" object (SARIF v2.1.0 section 3.19)
 	       for the plugin.  */
@@ -3608,12 +3798,12 @@ sarif_builder::make_artifact_content_object (const char *text) const
   return content_obj;
 }
 
-/* class diagnostic_sarif_format_buffer : public diagnostic_per_format_buffer.  */
+/* class sarif_sink_buffer : public per_sink_buffer.  */
 
 void
-diagnostic_sarif_format_buffer::dump (FILE *out, int indent) const
+sarif_sink_buffer::dump (FILE *out, int indent) const
 {
-  fprintf (out, "%*sdiagnostic_sarif_format_buffer:\n", indent, "");
+  fprintf (out, "%*ssarif_sink_buffer:\n", indent, "");
   int idx = 0;
   for (auto &result : m_results)
     {
@@ -3625,29 +3815,29 @@ diagnostic_sarif_format_buffer::dump (FILE *out, int indent) const
 }
 
 bool
-diagnostic_sarif_format_buffer::empty_p () const
+sarif_sink_buffer::empty_p () const
 {
   return m_results.empty ();
 }
 
 void
-diagnostic_sarif_format_buffer::move_to (diagnostic_per_format_buffer &base)
+sarif_sink_buffer::move_to (per_sink_buffer &base)
 {
-  diagnostic_sarif_format_buffer &dest
-    = static_cast<diagnostic_sarif_format_buffer &> (base);
+  sarif_sink_buffer &dest
+    = static_cast<sarif_sink_buffer &> (base);
   for (auto &&result : m_results)
     dest.m_results.push_back (std::move (result));
   m_results.clear ();
 }
 
 void
-diagnostic_sarif_format_buffer::clear ()
+sarif_sink_buffer::clear ()
 {
   m_results.clear ();
 }
 
 void
-diagnostic_sarif_format_buffer::flush ()
+sarif_sink_buffer::flush ()
 {
   for (auto &&result : m_results)
     {
@@ -3657,10 +3847,10 @@ diagnostic_sarif_format_buffer::flush ()
   m_results.clear ();
 }
 
-class sarif_output_format : public diagnostic_output_format
+class sarif_sink : public sink
 {
 public:
-  ~sarif_output_format ()
+  ~sarif_sink ()
   {
     /* Any sarifResult objects should have been handled by now.
        If not, then something's gone wrong with diagnostic
@@ -3672,8 +3862,8 @@ public:
 
   void dump (FILE *out, int indent) const override
   {
-    fprintf (out, "%*ssarif_output_format\n", indent, "");
-    diagnostic_output_format::dump (out, indent);
+    fprintf (out, "%*ssarif_sink\n", indent, "");
+    sink::dump (out, indent);
   }
 
   void
@@ -3682,15 +3872,15 @@ public:
     m_builder.set_main_input_filename (name);
   }
 
-  std::unique_ptr<diagnostic_per_format_buffer>
-  make_per_format_buffer () final override
+  std::unique_ptr<per_sink_buffer>
+  make_per_sink_buffer () final override
   {
-    return std::make_unique<diagnostic_sarif_format_buffer> (m_builder);
+    return std::make_unique<sarif_sink_buffer> (m_builder);
   }
-  void set_buffer (diagnostic_per_format_buffer *base_buffer) final override
+  void set_buffer (per_sink_buffer *base_buffer) final override
   {
-    diagnostic_sarif_format_buffer *buffer
-      = static_cast<diagnostic_sarif_format_buffer *> (base_buffer);
+    sarif_sink_buffer *buffer
+      = static_cast<sarif_sink_buffer *> (base_buffer);
     m_buffer = buffer;
   }
 
@@ -3726,53 +3916,60 @@ public:
   }
   void
   on_report_diagnostic (const diagnostic_info &diagnostic,
-			diagnostic_t orig_diag_kind) final override
+			enum kind orig_diag_kind) final override
   {
     m_builder.on_report_diagnostic (diagnostic, orig_diag_kind, m_buffer);
   }
-  void on_diagram (const diagnostic_diagram &diagram) final override
+  void on_diagram (const diagram &d) final override
   {
-    m_builder.emit_diagram (diagram);
+    m_builder.emit_diagram (d);
   }
   void after_diagnostic (const diagnostic_info &) final override
   {
     /* No-op.  */
   }
 
+  void
+  report_global_digraph (const lazily_created<digraphs::digraph> &ldg)
+    final override
+  {
+    m_builder.report_global_digraph (ldg);
+  }
+
   sarif_builder &get_builder () { return m_builder; }
 
   size_t num_results () const { return m_builder.num_results (); }
   sarif_result &get_result (size_t idx) { return m_builder.get_result (idx); }
 
 protected:
-  sarif_output_format (diagnostic_context &context,
-		       const line_maps *line_maps,
-		       std::unique_ptr<sarif_serialization_format> serialization_format,
-		       const sarif_generation_options &sarif_gen_opts)
-  : diagnostic_output_format (context),
-    m_builder (context, *get_printer (), line_maps,
+  sarif_sink (context &dc,
+	      const line_maps *line_maps,
+	      std::unique_ptr<sarif_serialization_format> serialization_format,
+	      const sarif_generation_options &sarif_gen_opts)
+  : sink (dc),
+    m_builder (dc, *get_printer (), line_maps,
 	       std::move (serialization_format), sarif_gen_opts),
     m_buffer (nullptr)
   {}
 
   sarif_builder m_builder;
-  diagnostic_sarif_format_buffer *m_buffer;
+  sarif_sink_buffer *m_buffer;
 };
 
-class sarif_stream_output_format : public sarif_output_format
+class sarif_stream_sink : public sarif_sink
 {
 public:
-  sarif_stream_output_format (diagnostic_context &context,
-			      const line_maps *line_maps,
-			      std::unique_ptr<sarif_serialization_format> serialization_format,
-			      const sarif_generation_options &sarif_gen_opts,
-			      FILE *stream)
-  : sarif_output_format (context, line_maps,
-			 std::move (serialization_format), sarif_gen_opts),
+  sarif_stream_sink (context &dc,
+		     const line_maps *line_maps,
+		     std::unique_ptr<sarif_serialization_format> serialization_format,
+		     const sarif_generation_options &sarif_gen_opts,
+		     FILE *stream)
+  : sarif_sink (dc, line_maps,
+		std::move (serialization_format), sarif_gen_opts),
     m_stream (stream)
   {
   }
-  ~sarif_stream_output_format ()
+  ~sarif_stream_sink ()
   {
     m_builder.flush_to_file (m_stream);
   }
@@ -3784,31 +3981,32 @@ private:
   FILE *m_stream;
 };
 
-class sarif_file_output_format : public sarif_output_format
+class sarif_file_sink : public sarif_sink
 {
 public:
-  sarif_file_output_format (diagnostic_context &context,
-			    const line_maps *line_maps,
-			    std::unique_ptr<sarif_serialization_format> serialization_format,
-			    const sarif_generation_options &sarif_gen_opts,
-			    diagnostic_output_file output_file)
-  : sarif_output_format (context, line_maps,
-			 std::move (serialization_format), sarif_gen_opts),
-    m_output_file (std::move (output_file))
+  sarif_file_sink (context &dc,
+		   const line_maps *line_maps,
+		   std::unique_ptr<sarif_serialization_format> serialization_format,
+		   const sarif_generation_options &sarif_gen_opts,
+		   output_file output_file_)
+  : sarif_sink (dc, line_maps,
+		std::move (serialization_format),
+		sarif_gen_opts),
+    m_output_file (std::move (output_file_))
   {
     gcc_assert (m_output_file.get_open_file ());
     gcc_assert (m_output_file.get_filename ());
   }
-  ~sarif_file_output_format ()
+  ~sarif_file_sink ()
   {
     m_builder.flush_to_file (m_output_file.get_open_file ());
   }
   void dump (FILE *out, int indent) const override
   {
-    fprintf (out, "%*ssarif_file_output_format: %s\n",
+    fprintf (out, "%*ssarif_file_sink: %s\n",
 	     indent, "",
 	     m_output_file.get_filename ());
-    diagnostic_output_format::dump (out, indent);
+    sink::dump (out, indent);
   }
   bool machine_readable_stderr_p () const final override
   {
@@ -3816,7 +4014,7 @@ public:
   }
 
 private:
-  diagnostic_output_file m_output_file;
+  output_file m_output_file;
 };
 
 /* Print the start of an embedded link to PP, as per 3.11.6.  */
@@ -3939,52 +4137,52 @@ sarif_builder::sarif_token_printer::print_tokens (pretty_printer *pp,
    to a file).
    Return a reference to *FMT.  */
 
-static diagnostic_output_format &
-diagnostic_output_format_init_sarif (diagnostic_context &context,
-				     std::unique_ptr<sarif_output_format> fmt)
+static sink &
+init_sarif_sink (context &dc,
+		 std::unique_ptr<sarif_sink> fmt)
 {
   gcc_assert (fmt);
-  diagnostic_output_format &out = *fmt;
+  sink &out = *fmt;
 
   fmt->update_printer ();
 
-  context.set_output_format (std::move (fmt));
+  dc.set_sink (std::move (fmt));
 
   return out;
 }
 
-/* Populate CONTEXT in preparation for SARIF output to stderr.
+/* Populate DC in preparation for SARIF output to stderr.
    Return a reference to the new sink.  */
 
-diagnostic_output_format &
-diagnostic_output_format_init_sarif_stderr (diagnostic_context &context,
-					    const line_maps *line_maps,
-					    bool formatted)
+sink &
+init_sarif_stderr (context &dc,
+		   const line_maps *line_maps,
+		   bool formatted)
 {
   gcc_assert (line_maps);
   const sarif_generation_options sarif_gen_opts;
   auto serialization
     = std::make_unique<sarif_serialization_format_json> (formatted);
-  return diagnostic_output_format_init_sarif
-    (context,
-     std::make_unique<sarif_stream_output_format> (context,
-						   line_maps,
-						   std::move (serialization),
-						   sarif_gen_opts,
-						   stderr));
+  return init_sarif_sink
+    (dc,
+     std::make_unique<sarif_stream_sink> (dc,
+					  line_maps,
+					  std::move (serialization),
+					  sarif_gen_opts,
+					  stderr));
 }
 
 /* Attempt to open "BASE_FILE_NAME""EXTENSION" for writing.
-   Return a non-null diagnostic_output_file,
-   or return a null diagnostic_output_file and complain to CONTEXT
+   Return a non-null output_file,
+   or return a null output_file and complain to DC
    using LINE_MAPS.  */
 
-diagnostic_output_file
-diagnostic_output_file::try_to_open (diagnostic_context &context,
-				     line_maps *line_maps,
-				     const char *base_file_name,
-				     const char *extension,
-				     bool is_binary)
+output_file
+output_file::try_to_open (context &dc,
+			  line_maps *line_maps,
+			  const char *base_file_name,
+			  const char *extension,
+			  bool is_binary)
 {
   gcc_assert (extension);
   gcc_assert (extension[0] == '.');
@@ -3992,10 +4190,10 @@ diagnostic_output_file::try_to_open (diagnostic_context &context,
   if (!base_file_name)
     {
       rich_location richloc (line_maps, UNKNOWN_LOCATION);
-      context.emit_diagnostic_with_group
-	(DK_ERROR, richloc, nullptr, 0,
+      dc.emit_diagnostic_with_group
+	(kind::error, richloc, nullptr, 0,
 	 "unable to determine filename for SARIF output");
-      return diagnostic_output_file ();
+      return output_file ();
     }
 
   label_text filename = label_text::take (concat (base_file_name,
@@ -4005,25 +4203,25 @@ diagnostic_output_file::try_to_open (diagnostic_context &context,
   if (!outf)
     {
       rich_location richloc (line_maps, UNKNOWN_LOCATION);
-      context.emit_diagnostic_with_group
-	(DK_ERROR, richloc, nullptr, 0,
+      dc.emit_diagnostic_with_group
+	(kind::error, richloc, nullptr, 0,
 	 "unable to open %qs for diagnostic output: %m",
 	 filename.get ());
-      return diagnostic_output_file ();
+      return output_file ();
     }
-  return diagnostic_output_file (outf, true, std::move (filename));
+  return output_file (outf, true, std::move (filename));
 }
 
 /* Attempt to open BASE_FILE_NAME.sarif for writing JSON.
-   Return a non-null diagnostic_output_file,
-   or return a null diagnostic_output_file and complain to CONTEXT
+   Return a non-null output_file,
+   or return a null output_file and complain to DC
    using LINE_MAPS.  */
 
-diagnostic_output_file
-diagnostic_output_format_open_sarif_file (diagnostic_context &context,
-					  line_maps *line_maps,
-					  const char *base_file_name,
-					  enum sarif_serialization_kind serialization_kind)
+output_file
+open_sarif_output_file (context &dc,
+			line_maps *line_maps,
+			const char *base_file_name,
+			enum sarif_serialization_kind serialization_kind)
 {
   const char *suffix;
   bool is_binary;
@@ -4037,78 +4235,78 @@ diagnostic_output_format_open_sarif_file (diagnostic_context &context,
       break;
     }
 
-  return diagnostic_output_file::try_to_open (context,
-					      line_maps,
-					      base_file_name,
-					      suffix,
-					      is_binary);
+  return output_file::try_to_open (dc,
+				   line_maps,
+				   base_file_name,
+				   suffix,
+				   is_binary);
 }
 
-/* Populate CONTEXT in preparation for SARIF output to a file named
+/* Populate DC in preparation for SARIF output to a file named
    BASE_FILE_NAME.sarif.
    Return a reference to the new sink.  */
 
-diagnostic_output_format &
-diagnostic_output_format_init_sarif_file (diagnostic_context &context,
-					  line_maps *line_maps,
-					  bool formatted,
-					  const char *base_file_name)
+sink &
+init_sarif_file (context &dc,
+		 line_maps *line_maps,
+		 bool formatted,
+		 const char *base_file_name)
 {
   gcc_assert (line_maps);
 
-  diagnostic_output_file output_file
-    = diagnostic_output_format_open_sarif_file (context,
-						line_maps,
-						base_file_name,
-						sarif_serialization_kind::json);
+  output_file output_file_
+    = open_sarif_output_file (dc,
+			      line_maps,
+			      base_file_name,
+			      sarif_serialization_kind::json);
   auto serialization
     = std::make_unique<sarif_serialization_format_json> (formatted);
 
   const sarif_generation_options sarif_gen_opts;
-  return diagnostic_output_format_init_sarif
-    (context,
-     std::make_unique<sarif_file_output_format> (context,
-						 line_maps,
-						 std::move (serialization),
-						 sarif_gen_opts,
-						 std::move (output_file)));
+  return init_sarif_sink
+    (dc,
+     std::make_unique<sarif_file_sink> (dc,
+					line_maps,
+					std::move (serialization),
+					sarif_gen_opts,
+					std::move (output_file_)));
 }
 
-/* Populate CONTEXT in preparation for SARIF output to STREAM.
+/* Populate DC in preparation for SARIF output to STREAM.
    Return a reference to the new sink.  */
 
-diagnostic_output_format &
-diagnostic_output_format_init_sarif_stream (diagnostic_context &context,
-					    const line_maps *line_maps,
-					    bool formatted,
-					    FILE *stream)
+sink &
+init_sarif_stream (context &dc,
+		   const line_maps *line_maps,
+		   bool formatted,
+		   FILE *stream)
 {
   gcc_assert (line_maps);
   const sarif_generation_options sarif_gen_opts;
   auto serialization
     = std::make_unique<sarif_serialization_format_json> (formatted);
-  return diagnostic_output_format_init_sarif
-    (context,
-     std::make_unique<sarif_stream_output_format> (context,
-						   line_maps,
-						   std::move (serialization),
-						   sarif_gen_opts,
-						   stream));
+  return init_sarif_sink
+    (dc,
+     std::make_unique<sarif_stream_sink> (dc,
+					  line_maps,
+					  std::move (serialization),
+					  sarif_gen_opts,
+					  stream));
 }
 
-std::unique_ptr<diagnostic_output_format>
-make_sarif_sink (diagnostic_context &context,
+std::unique_ptr<sink>
+make_sarif_sink (context &dc,
 		 const line_maps &line_maps,
 		 std::unique_ptr<sarif_serialization_format> serialization,
 		 const sarif_generation_options &sarif_gen_opts,
-		 diagnostic_output_file output_file)
+		 output_file output_file_)
 {
   auto sink
-    = std::make_unique<sarif_file_output_format> (context,
-						  &line_maps,
-						  std::move (serialization),
-						  sarif_gen_opts,
-						  std::move (output_file));
+    = std::make_unique<sarif_file_sink> (dc,
+					 &line_maps,
+					 std::move (serialization),
+					 sarif_gen_opts,
+					 std::move (output_file_));
   sink->update_printer ();
   return sink;
 }
@@ -4117,7 +4315,7 @@ make_sarif_sink (diagnostic_context &context,
 
 sarif_generation_options::sarif_generation_options ()
 : m_version (sarif_version::v2_1_0),
-  m_xml_state (false)
+  m_state_graph (false)
 {
 }
 
@@ -4125,6 +4323,9 @@ sarif_generation_options::sarif_generation_options ()
 
 namespace selftest {
 
+using auto_fix_quotes = ::selftest::auto_fix_quotes;
+using line_table_case = ::selftest::line_table_case;
+
 static void
 test_sarif_array_of_unique_1 ()
 {
@@ -4195,45 +4396,44 @@ test_sarif_array_of_unique_2 ()
   ASSERT_JSON_INT_PROPERTY_EQ (arr[1], "index", 1);
 }
 
-/* A subclass of sarif_output_format for writing selftests.
+/* A subclass of sarif_sink for writing selftests.
    The JSON output is cached internally, rather than written
    out to a file.  */
 
-class test_sarif_diagnostic_context : public test_diagnostic_context
+class test_sarif_diagnostic_context : public test_context
 {
 public:
   test_sarif_diagnostic_context (const char *main_input_filename,
 				 const sarif_generation_options &sarif_gen_opts)
   {
-    auto format = std::make_unique<buffered_output_format> (*this,
-							    line_table,
-							    true,
-							    sarif_gen_opts);
-    m_format = format.get (); // borrowed
-    diagnostic_output_format_init_sarif (*this, std::move (format));
-    m_format->set_main_input_filename (main_input_filename);
+    auto sink_ = std::make_unique<buffered_sink> (*this,
+						  line_table,
+						  true,
+						  sarif_gen_opts);
+    m_sink = sink_.get (); // borrowed
+    init_sarif_sink (*this, std::move (sink_));
+    m_sink->set_main_input_filename (main_input_filename);
   }
 
   std::unique_ptr<sarif_log> flush_to_object ()
   {
-    return m_format->flush_to_object ();
+    return m_sink->flush_to_object ();
   }
 
-  size_t num_results () const { return m_format->num_results (); }
-  sarif_result &get_result (size_t idx) { return m_format->get_result (idx); }
+  size_t num_results () const { return m_sink->num_results (); }
+  sarif_result &get_result (size_t idx) { return m_sink->get_result (idx); }
 
 private:
-  class buffered_output_format : public sarif_output_format
+  class buffered_sink : public sarif_sink
   {
   public:
-    buffered_output_format (diagnostic_context &context,
-			    const line_maps *line_maps,
-			    bool formatted,
-			    const sarif_generation_options &sarif_gen_opts)
-    : sarif_output_format (context, line_maps,
-			   std::make_unique<sarif_serialization_format_json>
-			     (formatted),
-			   sarif_gen_opts)
+    buffered_sink (context &dc,
+		   const line_maps *line_maps,
+		   bool formatted,
+		   const sarif_generation_options &sarif_gen_opts)
+    : sarif_sink (dc, line_maps,
+		  std::make_unique<sarif_serialization_format_json> (formatted),
+		  sarif_gen_opts)
     {
     }
     bool machine_readable_stderr_p () const final override
@@ -4246,7 +4446,7 @@ private:
     }
   };
 
-  buffered_output_format *m_format; // borrowed
+  buffered_sink *m_sink; // borrowed
 };
 
 /* Test making a sarif_location for a complex rich_location
@@ -4254,16 +4454,16 @@ private:
 
 static void
 test_make_location_object (const sarif_generation_options &sarif_gen_opts,
-			   const line_table_case &case_)
+			   const ::selftest::line_table_case &case_)
 {
-  diagnostic_show_locus_fixture_one_liner_utf8 f (case_);
+  source_printing_fixture_one_liner_utf8 f (case_);
   location_t line_end = linemap_position_for_column (line_table, 31);
 
   /* Don't attempt to run the tests if column data might be unavailable.  */
   if (line_end > LINE_MAP_MAX_LOCATION_WITH_COLS)
     return;
 
-  test_diagnostic_context dc;
+  test_context dc;
   pretty_printer pp;
   sarif_builder builder
     (dc, pp, line_table,
@@ -4299,7 +4499,7 @@ test_make_location_object (const sarif_generation_options &sarif_gen_opts,
 
   std::unique_ptr<sarif_location> location_obj
     = builder.make_location_object
-	(result, richloc, logical_location (),
+	(&result, richloc, logical_locations::key (),
 	 diagnostic_artifact_role::analysis_target);
   ASSERT_NE (location_obj, nullptr);
 
@@ -4376,7 +4576,7 @@ test_make_location_object (const sarif_generation_options &sarif_gen_opts,
 }
 
 /* Test of reporting a diagnostic at UNKNOWN_LOCATION to a
-   diagnostic_context and examining the generated sarif_log.
+   diagnostics::context and examining the generated sarif_log.
    Verify various basic properties. */
 
 static void
@@ -4385,7 +4585,7 @@ test_simple_log (const sarif_generation_options &sarif_gen_opts)
   test_sarif_diagnostic_context dc ("MAIN_INPUT_FILENAME", sarif_gen_opts);
 
   rich_location richloc (line_table, UNKNOWN_LOCATION);
-  dc.report (DK_ERROR, richloc, nullptr, 0, "this is a test: %i", 42);
+  dc.report (kind::error, richloc, nullptr, 0, "this is a test: %i", 42);
 
   auto log_ptr = dc.flush_to_object ();
 
@@ -4507,7 +4707,7 @@ test_simple_log_2 (const sarif_generation_options &sarif_gen_opts,
     /* 000000000111111
        123456789012345.  */
     = "unsinged int i;\n";
-  diagnostic_show_locus_fixture f (case_, content);
+  source_printing_fixture f (case_, content);
   location_t line_end = linemap_position_for_column (line_table, 31);
 
   /* Don't attempt to run the tests if column data might be unavailable.  */
@@ -4522,7 +4722,7 @@ test_simple_log_2 (const sarif_generation_options &sarif_gen_opts,
 		     linemap_position_for_column (line_table, 8));
 
   rich_location richloc (line_table, typo_loc);
-  dc.report (DK_ERROR, richloc, nullptr, 0,
+  dc.report (kind::error, richloc, nullptr, 0,
 	     "did you misspell %qs again?",
 	     "unsigned");
 
@@ -4650,7 +4850,7 @@ test_message_with_embedded_link (const sarif_generation_options &sarif_gen_opts)
   {
     test_sarif_diagnostic_context dc ("test.c", sarif_gen_opts);
     rich_location richloc (line_table, UNKNOWN_LOCATION);
-    dc.report (DK_ERROR, richloc, nullptr, 0,
+    dc.report (kind::error, richloc, nullptr, 0,
 	       "before %{text%} after",
 	       "http://example.com");
     std::unique_ptr<sarif_log> log = dc.flush_to_object ();
@@ -4673,7 +4873,7 @@ test_message_with_embedded_link (const sarif_generation_options &sarif_gen_opts)
 #  pragma GCC diagnostic push
 #  pragma GCC diagnostic ignored "-Wformat-diag"
 #endif
-    dc.report (DK_ERROR, richloc, nullptr, 0,
+    dc.report (kind::error, richloc, nullptr, 0,
 	       "Prohibited term used in %{para[0]\\spans[2]%}.",
 	       "1");
 #if __GNUC__ >= 10
@@ -4707,7 +4907,7 @@ test_message_with_embedded_link (const sarif_generation_options &sarif_gen_opts)
     test_sarif_diagnostic_context dc ("test.c", sarif_gen_opts);
     dc.push_owned_urlifier (std::make_unique<test_urlifier> ());
     rich_location richloc (line_table, UNKNOWN_LOCATION);
-    dc.report (DK_ERROR, richloc, nullptr, 0,
+    dc.report (kind::error, richloc, nullptr, 0,
 	       "foo %<-foption%> %<unrecognized%> bar");
     std::unique_ptr<sarif_log> log = dc.flush_to_object ();
 
@@ -4728,7 +4928,7 @@ test_message_with_braces (const sarif_generation_options &sarif_gen_opts)
   {
     test_sarif_diagnostic_context dc ("test.c", sarif_gen_opts);
     rich_location richloc (line_table, UNKNOWN_LOCATION);
-    dc.report (DK_ERROR, richloc, nullptr, 0,
+    dc.report (kind::error, richloc, nullptr, 0,
 	       "open brace: %qs close brace: %qs",
 	       "{", "}");
     std::unique_ptr<sarif_log> log = dc.flush_to_object ();
@@ -4745,26 +4945,26 @@ test_buffering (const sarif_generation_options &sarif_gen_opts)
 {
   test_sarif_diagnostic_context dc ("test.c", sarif_gen_opts);
 
-  diagnostic_buffer buf_a (dc);
-  diagnostic_buffer buf_b (dc);
+  diagnostics::buffer buf_a (dc);
+  diagnostics::buffer buf_b (dc);
 
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
 
-  ASSERT_EQ (dc.diagnostic_count (DK_ERROR), 0);
-  ASSERT_EQ (buf_a.diagnostic_count (DK_ERROR), 0);
-  ASSERT_EQ (buf_b.diagnostic_count (DK_ERROR), 0);
+  ASSERT_EQ (dc.diagnostic_count (kind::error), 0);
+  ASSERT_EQ (buf_a.diagnostic_count (kind::error), 0);
+  ASSERT_EQ (buf_b.diagnostic_count (kind::error), 0);
   ASSERT_EQ (dc.num_results (), 0);
   ASSERT_TRUE (buf_a.empty_p ());
   ASSERT_TRUE (buf_b.empty_p ());
 
   /* Unbuffered diagnostic.  */
   {
-    dc.report (DK_ERROR, rich_loc, nullptr, 0,
+    dc.report (kind::error, rich_loc, nullptr, 0,
 	       "message 1");
 
-    ASSERT_EQ (dc.diagnostic_count (DK_ERROR), 1);
-    ASSERT_EQ (buf_a.diagnostic_count (DK_ERROR), 0);
-    ASSERT_EQ (buf_b.diagnostic_count (DK_ERROR), 0);
+    ASSERT_EQ (dc.diagnostic_count (kind::error), 1);
+    ASSERT_EQ (buf_a.diagnostic_count (kind::error), 0);
+    ASSERT_EQ (buf_b.diagnostic_count (kind::error), 0);
     ASSERT_EQ (dc.num_results (), 1);
     sarif_result &result_obj = dc.get_result (0);
     auto message_obj = get_message_from_result (result_obj);
@@ -4777,11 +4977,11 @@ test_buffering (const sarif_generation_options &sarif_gen_opts)
   /* Buffer diagnostic into buffer A.  */
   {
     dc.set_diagnostic_buffer (&buf_a);
-    dc.report (DK_ERROR, rich_loc, nullptr, 0,
+    dc.report (kind::error, rich_loc, nullptr, 0,
 	       "message in buffer a");
-    ASSERT_EQ (dc.diagnostic_count (DK_ERROR), 1);
-    ASSERT_EQ (buf_a.diagnostic_count (DK_ERROR), 1);
-    ASSERT_EQ (buf_b.diagnostic_count (DK_ERROR), 0);
+    ASSERT_EQ (dc.diagnostic_count (kind::error), 1);
+    ASSERT_EQ (buf_a.diagnostic_count (kind::error), 1);
+    ASSERT_EQ (buf_b.diagnostic_count (kind::error), 0);
     ASSERT_EQ (dc.num_results (), 1);
     ASSERT_FALSE (buf_a.empty_p ());
     ASSERT_TRUE (buf_b.empty_p ());
@@ -4790,11 +4990,11 @@ test_buffering (const sarif_generation_options &sarif_gen_opts)
   /* Buffer diagnostic into buffer B.  */
   {
     dc.set_diagnostic_buffer (&buf_b);
-    dc.report (DK_ERROR, rich_loc, nullptr, 0,
+    dc.report (kind::error, rich_loc, nullptr, 0,
 	       "message in buffer b");
-    ASSERT_EQ (dc.diagnostic_count (DK_ERROR), 1);
-    ASSERT_EQ (buf_a.diagnostic_count (DK_ERROR), 1);
-    ASSERT_EQ (buf_b.diagnostic_count (DK_ERROR), 1);
+    ASSERT_EQ (dc.diagnostic_count (kind::error), 1);
+    ASSERT_EQ (buf_a.diagnostic_count (kind::error), 1);
+    ASSERT_EQ (buf_b.diagnostic_count (kind::error), 1);
     ASSERT_EQ (dc.num_results (), 1);
     ASSERT_FALSE (buf_a.empty_p ());
     ASSERT_FALSE (buf_b.empty_p ());
@@ -4803,9 +5003,9 @@ test_buffering (const sarif_generation_options &sarif_gen_opts)
   /* Flush buffer B to dc.  */
   {
     dc.flush_diagnostic_buffer (buf_b);
-    ASSERT_EQ (dc.diagnostic_count (DK_ERROR), 2);
-    ASSERT_EQ (buf_a.diagnostic_count (DK_ERROR), 1);
-    ASSERT_EQ (buf_b.diagnostic_count (DK_ERROR), 0);
+    ASSERT_EQ (dc.diagnostic_count (kind::error), 2);
+    ASSERT_EQ (buf_a.diagnostic_count (kind::error), 1);
+    ASSERT_EQ (buf_b.diagnostic_count (kind::error), 0);
     ASSERT_EQ (dc.num_results (), 2);
     sarif_result &result_1_obj = dc.get_result (1);
     auto message_1_obj = get_message_from_result (result_1_obj);
@@ -4818,9 +5018,9 @@ test_buffering (const sarif_generation_options &sarif_gen_opts)
   /* Clear buffer A.  */
   {
     dc.clear_diagnostic_buffer (buf_a);
-    ASSERT_EQ (dc.diagnostic_count (DK_ERROR), 2);
-    ASSERT_EQ (buf_a.diagnostic_count (DK_ERROR), 0);
-    ASSERT_EQ (buf_b.diagnostic_count (DK_ERROR), 0);
+    ASSERT_EQ (dc.diagnostic_count (kind::error), 2);
+    ASSERT_EQ (buf_a.diagnostic_count (kind::error), 0);
+    ASSERT_EQ (buf_b.diagnostic_count (kind::error), 0);
     ASSERT_EQ (dc.num_results (), 2);
     ASSERT_TRUE (buf_a.empty_p ());
     ASSERT_TRUE (buf_b.empty_p ());
@@ -4857,7 +5057,7 @@ run_line_table_case_tests_per_version (const line_table_case &case_)
 /* Run all of the selftests within this file.  */
 
 void
-diagnostic_format_sarif_cc_tests ()
+sarif_sink_cc_tests ()
 {
   test_sarif_array_of_unique_1 ();
   test_sarif_array_of_unique_2 ();
@@ -4871,6 +5071,8 @@ diagnostic_format_sarif_cc_tests ()
   for_each_line_table_case (run_line_table_case_tests_per_version);
 }
 
-} // namespace selftest
+} // namespace diagnostics::selftest
 
 #endif /* CHECKING_P */
+
+} // namespace diagnostics
diff --git a/gcc/diagnostic-format-sarif.h b/gcc/diagnostics/sarif-sink.h
index c3ae330..9f8a73f 100644
--- a/gcc/diagnostic-format-sarif.h
+++ b/gcc/diagnostics/sarif-sink.h
@@ -18,13 +18,21 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_FORMAT_SARIF_H
-#define GCC_DIAGNOSTIC_FORMAT_SARIF_H
+#ifndef GCC_DIAGNOSTICS_SARIF_SINK_H
+#define GCC_DIAGNOSTICS_SARIF_SINK_H
 
 #include "json.h"
-#include "diagnostic-format.h"
-#include "diagnostic-output-file.h"
-#include "logical-location.h"
+#include "diagnostics/sink.h"
+#include "diagnostics/output-file.h"
+#include "diagnostics/logical-locations.h"
+
+namespace diagnostics {
+
+namespace digraphs {
+  class digraph;
+  class node;
+  class edge;
+}
 
 /* Enum for choosing what format to serializing the generated SARIF into.  */
 
@@ -35,26 +43,26 @@ enum class sarif_serialization_kind
    num_values
 };
 
-extern diagnostic_output_file
-diagnostic_output_format_open_sarif_file (diagnostic_context &context,
-					  line_maps *line_maps,
-					  const char *base_file_name,
-					  enum sarif_serialization_kind serialization_kind);
-
-extern diagnostic_output_format &
-diagnostic_output_format_init_sarif_stderr (diagnostic_context &context,
-					    const line_maps *line_maps,
-					    bool formatted);
-extern diagnostic_output_format &
-diagnostic_output_format_init_sarif_file (diagnostic_context &context,
-					  line_maps *line_maps,
-					  bool formatted,
-					  const char *base_file_name);
-extern diagnostic_output_format &
-diagnostic_output_format_init_sarif_stream (diagnostic_context &context,
-					    const line_maps *line_maps,
-					    bool formatted,
-					    FILE *stream);
+extern output_file
+open_sarif_output_file (context &dc,
+			line_maps *line_maps,
+			const char *base_file_name,
+			enum sarif_serialization_kind serialization_kind);
+
+extern sink &
+init_sarif_stderr (context &dc,
+		   const line_maps *line_maps,
+		   bool formatted);
+extern sink &
+init_sarif_file (context &dc,
+		 line_maps *line_maps,
+		 bool formatted,
+		 const char *base_file_name);
+extern sink &
+init_sarif_stream (context &dc,
+		   const line_maps *line_maps,
+		   bool formatted,
+		   FILE *stream);
 
 /* Abstract base class for handling JSON output vs other kinds of
    serialization of the json tree.  */
@@ -101,17 +109,18 @@ struct sarif_generation_options
   sarif_generation_options ();
 
   enum sarif_version m_version;
-  bool m_xml_state;
+  bool m_state_graph;
 };
 
-extern std::unique_ptr<diagnostic_output_format>
-make_sarif_sink (diagnostic_context &context,
+extern std::unique_ptr<sink>
+make_sarif_sink (context &dc,
 		 const line_maps &line_maps,
 		 std::unique_ptr<sarif_serialization_format> serialization_format,
 		 const sarif_generation_options &sarif_gen_opts,
-		 diagnostic_output_file output_file);
+		 output_file output_file_);
 
 class sarif_builder;
+class sarif_location_manager;
 
 /* Concrete subclass of json::object for SARIF property bags
    (SARIF v2.1.0 section 3.8).  */
@@ -121,7 +130,11 @@ class sarif_property_bag : public json::object
 public:
   void set_logical_location (const char *property_name,
 			     sarif_builder &,
-			     logical_location logical_loc);
+			     logical_locations::key logical_loc);
+  void set_graph (const char *property_name,
+		  sarif_builder &,
+		  sarif_location_manager *sarif_location_mgr,
+		  const digraphs::digraph &g);
 };
 
 /* Concrete subclass of json::object for SARIF objects that can
@@ -136,4 +149,41 @@ public:
   sarif_property_bag &get_or_create_properties ();
 };
 
-#endif /* ! GCC_DIAGNOSTIC_FORMAT_SARIF_H */
+/* Subclass of sarif_object for SARIF "graph" objects
+   (SARIF v2.1.0 section 3.39).  */
+
+class sarif_graph : public sarif_object
+{
+};
+
+/* Subclass of sarif_object for SARIF "node" objects
+   (SARIF v2.1.0 section 3.40).  */
+
+class sarif_node : public sarif_object
+{
+};
+
+/* Subclass of sarif_object for SARIF "edge" objects
+   (SARIF v2.1.0 section 3.41).  */
+
+class sarif_edge : public sarif_object
+{
+};
+
+extern std::unique_ptr<sarif_graph>
+make_sarif_graph (const digraphs::digraph &g,
+		  sarif_builder *builder,
+		  sarif_location_manager *sarif_location_mgr);
+
+extern std::unique_ptr<sarif_node>
+make_sarif_node (const digraphs::node &n,
+		 sarif_builder *builder,
+		 sarif_location_manager *sarif_location_mgr);
+
+extern std::unique_ptr<sarif_edge>
+make_sarif_edge (const digraphs::edge &e,
+		 sarif_builder *builder);
+
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_SARIF_SINK_H */
diff --git a/gcc/selftest-diagnostic.cc b/gcc/diagnostics/selftest-context.cc
index eeee2eb..2b6dd0b 100644
--- a/gcc/selftest-diagnostic.cc
+++ b/gcc/diagnostics/selftest-context.cc
@@ -21,61 +21,65 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "diagnostic.h"
-#include "diagnostic-format.h"
+#include "diagnostics/sink.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
+#include "diagnostics/selftest-context.h"
 
 /* The selftest code should entirely disappear in a production
    configuration, hence we guard all of it with #if CHECKING_P.  */
 
 #if CHECKING_P
 
+namespace diagnostics {
 namespace selftest {
 
-/* Implementation of class selftest::test_diagnostic_context.  */
+/* Implementation of class diagnostics::selftest::test_context.  */
 
-test_diagnostic_context::test_diagnostic_context ()
+test_context::test_context ()
 {
   diagnostic_initialize (this, 0);
   pp_show_color (get_reference_printer ()) = false;
-  m_source_printing.enabled = true;
-  m_source_printing.show_labels_p = true;
+
+  auto &source_printing_opts = get_source_printing_options ();
+  source_printing_opts.enabled = true;
+  source_printing_opts.show_labels_p = true;
   m_show_column = true;
-  diagnostic_start_span (this) = start_span_cb;
-  m_source_printing.min_margin_width = 6;
-  m_source_printing.max_width = 80;
-  pp_buffer (get_output_format (0).get_printer ())->m_flush_p = false;
+  start_span (this) = start_span_cb;
+  source_printing_opts.min_margin_width = 6;
+  source_printing_opts.max_width = 80;
+  pp_buffer (get_sink (0).get_printer ())->m_flush_p = false;
 }
 
-test_diagnostic_context::~test_diagnostic_context ()
+test_context::~test_context ()
 {
   diagnostic_finish (this);
 }
 
-/* Implementation of diagnostic_start_span_fn, hiding the
+/* Implementation of diagnostics::start_span_fn, hiding the
    real filename (to avoid printing the names of tempfiles).  */
 
 void
-test_diagnostic_context::
-start_span_cb (const diagnostic_location_print_policy &loc_policy,
-	       to_text &sink,
+test_context::
+start_span_cb (const location_print_policy &loc_policy,
+	       to_text &html_or_text,
 	       expanded_location exploc)
 {
   exploc.file = "FILENAME";
-  default_diagnostic_start_span_fn<to_text> (loc_policy, sink, exploc);
+  default_start_span_fn<to_text>
+    (loc_policy, html_or_text, exploc);
 }
 
 bool
-test_diagnostic_context::report (diagnostic_t kind,
-				 rich_location &richloc,
-				 const diagnostic_metadata *metadata,
-				 diagnostic_option_id option,
-				 const char * fmt, ...)
+test_context::report (enum kind kind,
+		      rich_location &richloc,
+		      const metadata *metadata_,
+		      option_id opt_id,
+		      const char * fmt, ...)
 {
   va_list ap;
   va_start (ap, fmt);
   begin_group ();
-  bool result = diagnostic_impl (&richloc, metadata, option, fmt, &ap, kind);
+  bool result = diagnostic_impl (&richloc, metadata_, opt_id, fmt, &ap, kind);
   end_group ();
   va_end (ap);
   return result;
@@ -85,15 +89,16 @@ test_diagnostic_context::report (diagnostic_t kind,
    Return the text buffer from the printer.  */
 
 const char *
-test_diagnostic_context::test_show_locus (rich_location &richloc)
+test_context::test_show_locus (rich_location &richloc)
 {
   pretty_printer *pp = get_reference_printer ();
   gcc_assert (pp);
-  diagnostic_source_print_policy source_policy (*this);
-  source_policy.print (*pp, richloc, DK_ERROR, nullptr);
+  source_print_policy source_policy (*this);
+  source_policy.print (*pp, richloc, kind::error, nullptr);
   return pp_formatted_text (pp);
 }
 
 } // namespace selftest
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
diff --git a/gcc/selftest-diagnostic.h b/gcc/diagnostics/selftest-context.h
index 4a43310..cfba997 100644
--- a/gcc/selftest-diagnostic.h
+++ b/gcc/diagnostics/selftest-context.h
@@ -17,29 +17,30 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_SELFTEST_DIAGNOSTIC_H
-#define GCC_SELFTEST_DIAGNOSTIC_H
+#ifndef GCC_DIAGNOSTICS_SELFTEST_CONTEXT_H
+#define GCC_DIAGNOSTICS_SELFTEST_CONTEXT_H
 
 /* The selftest code should entirely disappear in a production
    configuration, hence we guard all of it with #if CHECKING_P.  */
 
 #if CHECKING_P
 
+namespace diagnostics {
 namespace selftest {
 
-/* Convenience subclass of diagnostic_context for testing
+/* Convenience subclass of diagnostics::context for testing
    the diagnostic subsystem.  */
 
-class test_diagnostic_context : public diagnostic_context
+class test_context : public context
 {
  public:
-  test_diagnostic_context ();
-  ~test_diagnostic_context ();
+  test_context ();
+  ~test_context ();
 
-  /* Implementation of diagnostic_start_span_fn, hiding the
+  /* Implementation of diagnostics::start_span_fn, hiding the
      real filename (to avoid printing the names of tempfiles).  */
   static void
-  start_span_cb (const diagnostic_location_print_policy &,
+  start_span_cb (const location_print_policy &,
 		 to_text &sink,
 		 expanded_location exploc);
 
@@ -47,17 +48,46 @@ class test_diagnostic_context : public diagnostic_context
      should only be called on a context that uses a non-standard formatter
      that e.g. gathers the results in memory, rather than emits to stderr.  */
   bool
-  report (diagnostic_t kind,
+  report (enum kind kind,
 	  rich_location &richloc,
-	  const diagnostic_metadata *metadata,
-	  diagnostic_option_id option,
+	  const metadata *,
+	  option_id opt_id,
 	  const char * fmt, ...) ATTRIBUTE_GCC_DIAG(6,7);
 
   const char *test_show_locus (rich_location &richloc);
+
+  /* Setters for the context's source_printing_options
+     for use in selftests.  */
+  void colorize_source (bool val)
+  {
+    get_source_printing_options ().colorize_source_p = val;
+  }
+  void show_labels (bool val)
+  {
+    get_source_printing_options ().show_labels_p = val;
+  }
+  void show_line_numbers (bool val)
+  {
+    get_source_printing_options ().show_line_numbers_p = val;
+  }
+  void show_ruler (bool val)
+  {
+    get_source_printing_options ().show_ruler_p = val;
+  }
+  void show_event_links (bool val)
+  {
+    get_source_printing_options ().show_event_links_p = val;
+  }
+  void set_caret_char (unsigned idx, char ch)
+  {
+    gcc_assert (idx < rich_location::STATICALLY_ALLOCATED_RANGES);
+    get_source_printing_options ().caret_chars[idx] = ch;
+  }
 };
 
 } // namespace selftest
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
 
-#endif /* GCC_SELFTEST_DIAGNOSTIC_H */
+#endif /* GCC_DIAGNOSTICS_SELFTEST_CONTEXT_H */
diff --git a/gcc/selftest-logical-location.cc b/gcc/diagnostics/selftest-logical-locations.cc
index 63b9fb4..8ba4233 100644
--- a/gcc/selftest-logical-location.cc
+++ b/gcc/diagnostics/selftest-logical-locations.cc
@@ -1,4 +1,4 @@
-/* Concrete subclass of logical_location for use in selftests.
+/* Concrete subclass of logical_locations::manager for use in selftests.
    Copyright (C) 2024-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -22,22 +22,24 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "selftest.h"
-#include "selftest-logical-location.h"
+#include "diagnostics/selftest-logical-locations.h"
 
 #if CHECKING_P
 
+namespace diagnostics {
+namespace logical_locations {
 namespace selftest {
 
-/* class test_logical_location_manager : public logical_location_manager.  */
+/* class test_manager : public manager.  */
 
-test_logical_location_manager::~test_logical_location_manager ()
+test_manager::~test_manager ()
 {
   for (auto iter : m_name_to_item_map)
     delete iter.second;
 }
 
 const char *
-test_logical_location_manager::get_short_name (key k) const
+test_manager::get_short_name (key k) const
 {
   auto item = item_from_key (k);
   if (!item)
@@ -46,43 +48,43 @@ test_logical_location_manager::get_short_name (key k) const
 }
 
 const char *
-test_logical_location_manager::get_name_with_scope (key k) const
+test_manager::get_name_with_scope (key k) const
 {
   auto item = item_from_key (k);
   return item->m_name;
 }
 
 const char *
-test_logical_location_manager::get_internal_name (key k) const
+test_manager::get_internal_name (key k) const
 {
   auto item = item_from_key (k);
   return item->m_name;
 }
 
-enum logical_location_kind
-test_logical_location_manager::get_kind (key k) const
+enum diagnostics::logical_locations::kind
+test_manager::get_kind (key k) const
 {
   auto item = item_from_key (k);
   return item->m_kind;
 }
 
 label_text
-test_logical_location_manager::get_name_for_path_output (key k) const
+test_manager::get_name_for_path_output (key k) const
 {
   auto item = item_from_key (k);
   return label_text::borrow (item->m_name);
 }
 
-logical_location
-test_logical_location_manager::
+diagnostics::logical_locations::key
+test_manager::
 logical_location_from_funcname (const char *funcname)
 {
   const item *i = item_from_funcname (funcname);
   return key::from_ptr (i);
 }
 
-const test_logical_location_manager::item *
-test_logical_location_manager::item_from_funcname (const char *funcname)
+const test_manager::item *
+test_manager::item_from_funcname (const char *funcname)
 {
   if (!funcname)
     return nullptr;
@@ -90,7 +92,7 @@ test_logical_location_manager::item_from_funcname (const char *funcname)
   if (item **slot = m_name_to_item_map.get (funcname))
     return *slot;
 
-  item *i = new item (logical_location_kind::function, funcname);
+  item *i = new item (kind::function, funcname);
   m_name_to_item_map.put (funcname, i);
   return i;
 }
@@ -98,14 +100,14 @@ test_logical_location_manager::item_from_funcname (const char *funcname)
 /* Run all of the selftests within this file.  */
 
 void
-selftest_logical_location_cc_tests ()
+selftest_logical_locations_cc_tests ()
 {
-  test_logical_location_manager mgr;
+  test_manager mgr;
 
   ASSERT_FALSE (mgr.logical_location_from_funcname (nullptr));
 
-  logical_location loc_foo = mgr.logical_location_from_funcname ("foo");
-  logical_location loc_bar = mgr.logical_location_from_funcname ("bar");
+  key loc_foo = mgr.logical_location_from_funcname ("foo");
+  key loc_bar = mgr.logical_location_from_funcname ("bar");
 
   ASSERT_NE (loc_foo, loc_bar);
 
@@ -113,6 +115,8 @@ selftest_logical_location_cc_tests ()
   ASSERT_STREQ (mgr.get_short_name (loc_bar), "bar");
 }
 
-} // namespace selftest
+} // namespace diagnostics::logical_locations::selftest
+} // namespace diagnostics::logical_locations
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
diff --git a/gcc/selftest-logical-location.h b/gcc/diagnostics/selftest-logical-locations.h
index d9bf38f..c14a282 100644
--- a/gcc/selftest-logical-location.h
+++ b/gcc/diagnostics/selftest-logical-locations.h
@@ -1,4 +1,4 @@
-/* Concrete subclass of logical_location for use in selftests.
+/* Concrete subclass of logical_locations::manager for use in selftests.
    Copyright (C) 2024-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
 
@@ -18,56 +18,58 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_SELFTEST_LOGICAL_LOCATION_H
-#define GCC_SELFTEST_LOGICAL_LOCATION_H
+#ifndef GCC_DIAGNOSTICS_SELFTEST_LOGICAL_LOCATIONS_H
+#define GCC_DIAGNOSTICS_SELFTEST_LOGICAL_LOCATIONS_H
 
-#include "logical-location.h"
+#include "diagnostics/logical-locations.h"
 
 /* The selftest code should entirely disappear in a production
    configuration, hence we guard all of it with #if CHECKING_P.  */
 
 #if CHECKING_P
 
+namespace diagnostics {
+namespace logical_locations {
 namespace selftest {
 
-/* Concrete subclass of logical_location_manager for use in selftests.  */
+/* Concrete subclass of logical_locations::manager for use in selftests.  */
 
-class test_logical_location_manager : public logical_location_manager
+class test_manager : public manager
 {
 public:
-  ~test_logical_location_manager ();
+  ~test_manager ();
 
   const char *get_short_name (key) const final override;
   const char *get_name_with_scope (key) const final override;
   const char *get_internal_name (key) const final override;
-  enum logical_location_kind get_kind (key) const final override;
+  kind get_kind (key) const final override;
   label_text get_name_for_path_output (key) const final override;
   key get_parent (key) const final override
   {
     return key ();
   }
 
-  logical_location
+  key
   logical_location_from_funcname (const char *funcname);
 
 private:
   struct item
   {
-    item (enum logical_location_kind kind,
+    item (kind kind_,
 	  const char *name)
-    : m_kind (kind),
+    : m_kind (kind_),
       m_name (name)
     {
     }
 
-    enum logical_location_kind m_kind;
+    kind m_kind;
     const char *m_name;
   };
 
   const item *
   item_from_funcname (const char *funcname);
 
-  static const item *item_from_key (logical_location k)
+  static const item *item_from_key (key k)
   {
     return k.cast_to<const item *> ();
   }
@@ -75,9 +77,10 @@ private:
   hash_map<nofree_string_hash, item *> m_name_to_item_map;
 };
 
-} // namespace selftest
+} // namespace diagnostics::logical_locations::selftest
+} // namespace diagnostics::logical_locations::
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
 
-
-#endif /* GCC_SELFTEST_LOGICAL_LOCATION_H.  */
+#endif /* GCC_DIAGNOSTICS_SELFTEST_LOGICAL_LOCATIONS_H.  */
diff --git a/gcc/selftest-diagnostic-path.cc b/gcc/diagnostics/selftest-paths.cc
index 04372e7..56ce7ff 100644
--- a/gcc/selftest-diagnostic-path.cc
+++ b/gcc/diagnostics/selftest-paths.cc
@@ -24,66 +24,69 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "version.h"
-#include "demangle.h"
-#include "backtrace.h"
 #include "diagnostic.h"
-#include "selftest-diagnostic-path.h"
+#include "diagnostics/selftest-paths.h"
 
 #if CHECKING_P
 
+namespace diagnostics {
+namespace paths {
 namespace selftest {
 
-/* class test_diagnostic_path : public diagnostic_path.  */
+/* class test_path : public diagnostics::paths::path.  */
 
-test_diagnostic_path::test_diagnostic_path (pretty_printer *event_pp)
-: diagnostic_path (m_test_logical_loc_mgr),
+test_path::
+test_path (logical_locations::selftest::test_manager &logical_loc_mgr,
+	   pretty_printer *event_pp)
+: path (logical_loc_mgr),
+  m_test_logical_loc_mgr (logical_loc_mgr),
   m_event_pp (event_pp)
 {
   add_thread ("main");
 }
 
-/* Implementation of diagnostic_path::num_events vfunc for
-   test_diagnostic_path: simply get the number of events in the vec.  */
+/* Implementation of path::num_events vfunc for
+   test_path: simply get the number of events in the vec.  */
 
 unsigned
-test_diagnostic_path::num_events () const
+test_path::num_events () const
 {
   return m_events.length ();
 }
 
-/* Implementation of diagnostic_path::get_event vfunc for
-   test_diagnostic_path: simply return the event in the vec.  */
+/* Implementation of path::get_event vfunc for
+   test_path: simply return the event in the vec.  */
 
-const diagnostic_event &
-test_diagnostic_path::get_event (int idx) const
+const event &
+test_path::get_event (int idx) const
 {
   return *m_events[idx];
 }
 
 unsigned
-test_diagnostic_path::num_threads () const
+test_path::num_threads () const
 {
   return m_threads.length ();
 }
 
-const diagnostic_thread &
-test_diagnostic_path::get_thread (diagnostic_thread_id_t idx) const
+const thread &
+test_path::get_thread (thread_id_t idx) const
 {
   return *m_threads[idx];
 }
 
 bool
-test_diagnostic_path::same_function_p (int event_idx_a,
-				       int event_idx_b) const
+test_path::same_function_p (int event_idx_a,
+			    int event_idx_b) const
 {
   return (m_events[event_idx_a]->get_logical_location ()
 	  == m_events[event_idx_b]->get_logical_location ());
 }
 
-diagnostic_thread_id_t
-test_diagnostic_path::add_thread (const char *name)
+thread_id_t
+test_path::add_thread (const char *name)
 {
-  m_threads.safe_push (new test_diagnostic_thread (name));
+  m_threads.safe_push (new test_thread (name));
   return m_threads.length () - 1;
 }
 
@@ -95,11 +98,11 @@ test_diagnostic_path::add_thread (const char *name)
 
    Return the id of the new event.  */
 
-diagnostic_event_id_t
-test_diagnostic_path::add_event (location_t loc,
-				 const char *funcname,
-				 int depth,
-				 const char *fmt, ...)
+event_id_t
+test_path::add_event (location_t loc,
+		      const char *funcname,
+		      int depth,
+		      const char *fmt, ...)
 {
   pretty_printer *pp = m_event_pp;
   pp_clear_output_area (pp);
@@ -117,24 +120,24 @@ test_diagnostic_path::add_event (location_t loc,
 
   va_end (ap);
 
-  test_diagnostic_event *new_event
-    = new test_diagnostic_event (loc,
-				 logical_location_from_funcname (funcname),
-				 depth,
-				 pp_formatted_text (pp));
+  test_event *new_event
+    = new test_event (loc,
+		      logical_location_from_funcname (funcname),
+		      depth,
+		      pp_formatted_text (pp));
   m_events.safe_push (new_event);
 
   pp_clear_output_area (pp);
 
-  return diagnostic_event_id_t (m_events.length () - 1);
+  return event_id_t (m_events.length () - 1);
 }
 
-diagnostic_event_id_t
-test_diagnostic_path::add_thread_event (diagnostic_thread_id_t thread_id,
-					location_t loc,
-					const char *funcname,
-					int depth,
-					const char *fmt, ...)
+event_id_t
+test_path::add_thread_event (thread_id_t thread_id,
+			     location_t loc,
+			     const char *funcname,
+			     int depth,
+			     const char *fmt, ...)
 {
   pretty_printer *pp = m_event_pp;
   pp_clear_output_area (pp);
@@ -153,52 +156,52 @@ test_diagnostic_path::add_thread_event (diagnostic_thread_id_t thread_id,
 
   va_end (ap);
 
-  test_diagnostic_event *new_event
-    = new test_diagnostic_event (loc,
-				 logical_location_from_funcname (funcname),
-				 depth,
-				 pp_formatted_text (pp),
-				 thread_id);
+  test_event *new_event
+    = new test_event (loc,
+		      logical_location_from_funcname (funcname),
+		      depth,
+		      pp_formatted_text (pp),
+		      thread_id);
   m_events.safe_push (new_event);
 
   pp_clear_output_area (pp);
 
-  return diagnostic_event_id_t (m_events.length () - 1);
+  return event_id_t (m_events.length () - 1);
 }
 
 /* Mark the most recent event on this path (which must exist) as being
    connected to the next one to be added.  */
 
 void
-test_diagnostic_path::connect_to_next_event ()
+test_path::connect_to_next_event ()
 {
   gcc_assert (m_events.length () > 0);
   m_events[m_events.length () - 1]->connect_to_next_event ();
 }
 
 void
-test_diagnostic_path::add_entry (const char *callee_name,
-				 int stack_depth,
-				 diagnostic_thread_id_t thread_id)
+test_path::add_entry (const char *callee_name,
+		      int stack_depth,
+		      thread_id_t thread_id)
 {
   add_thread_event (thread_id, UNKNOWN_LOCATION, callee_name, stack_depth,
 		    "entering %qs", callee_name);
 }
 
 void
-test_diagnostic_path::add_return (const char *caller_name,
-				  int stack_depth,
-				  diagnostic_thread_id_t thread_id)
+test_path::add_return (const char *caller_name,
+		       int stack_depth,
+		       thread_id_t thread_id)
 {
   add_thread_event (thread_id, UNKNOWN_LOCATION, caller_name, stack_depth,
 		    "returning to %qs", caller_name);
 }
 
 void
-test_diagnostic_path::add_call (const char *caller_name,
-				int caller_stack_depth,
-				const char *callee_name,
-				diagnostic_thread_id_t thread_id)
+test_path::add_call (const char *caller_name,
+		     int caller_stack_depth,
+		     const char *callee_name,
+		     thread_id_t thread_id)
 {
   add_thread_event (thread_id, UNKNOWN_LOCATION,
 		    caller_name, caller_stack_depth,
@@ -206,22 +209,22 @@ test_diagnostic_path::add_call (const char *caller_name,
   add_entry (callee_name, caller_stack_depth + 1, thread_id);
 }
 
-logical_location
-test_diagnostic_path::logical_location_from_funcname (const char *funcname)
+logical_locations::key
+test_path::logical_location_from_funcname (const char *funcname)
 {
   return m_test_logical_loc_mgr.logical_location_from_funcname (funcname);
 }
 
-/* struct test_diagnostic_event.  */
+/* struct test_event.  */
 
-/* test_diagnostic_event's ctor.  */
+/* test_event's ctor.  */
 
-test_diagnostic_event::
-test_diagnostic_event (location_t loc,
-		       logical_location logical_loc,
-		       int depth,
-		       const char *desc,
-		       diagnostic_thread_id_t thread_id)
+test_event::
+test_event (location_t loc,
+	    logical_location logical_loc,
+	    int depth,
+	    const char *desc,
+	    thread_id_t thread_id)
 : m_loc (loc),
   m_logical_loc (logical_loc),
   m_depth (depth), m_desc (xstrdup (desc)),
@@ -230,13 +233,15 @@ test_diagnostic_event (location_t loc,
 {
 }
 
-/* test_diagnostic_event's dtor.  */
+/* test_event's dtor.  */
 
-test_diagnostic_event::~test_diagnostic_event ()
+test_event::~test_event ()
 {
   free (m_desc);
 }
 
-} // namespace selftest
+} // namespace diagnostics::paths::selftest
+} // namespace diagnostics::paths
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
diff --git a/gcc/selftest-diagnostic-path.h b/gcc/diagnostics/selftest-paths.h
index 8829943..fe628f6 100644
--- a/gcc/selftest-diagnostic-path.h
+++ b/gcc/diagnostics/selftest-paths.h
@@ -18,17 +18,19 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_SELFTEST_DIAGNOSTIC_PATH_H
-#define GCC_SELFTEST_DIAGNOSTIC_PATH_H
+#ifndef GCC_DIAGNOSTICS_SELFTEST_PATHS_H
+#define GCC_DIAGNOSTICS_SELFTEST_PATHS_H
 
-#include "diagnostic-path.h"
-#include "selftest-logical-location.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/selftest-logical-locations.h"
 
 /* The selftest code should entirely disappear in a production
    configuration, hence we guard all of it with #if CHECKING_P.  */
 
 #if CHECKING_P
 
+namespace diagnostics {
+namespace paths {
 namespace selftest {
 
 /* Concrete subclasses of the abstract base classes
@@ -36,17 +38,20 @@ namespace selftest {
 
    This code should have no dependency on "tree".  */
 
-/* An implementation of diagnostic_event.  */
+/* An implementation of diagnostics::paths::event.  */
 
-class test_diagnostic_event : public diagnostic_event
+class test_event : public event
 {
  public:
-  test_diagnostic_event (location_t loc,
-			 logical_location logical_loc,
-			 int depth,
-			 const char *desc,
-			 diagnostic_thread_id_t thread_id = 0);
-  ~test_diagnostic_event ();
+  using logical_location = logical_locations::key;
+  using thread_id_t = paths::thread_id_t;
+
+  test_event (location_t loc,
+	      logical_location logical_loc,
+	      int depth,
+	      const char *desc,
+	      thread_id_t thread_id = 0);
+  ~test_event ();
 
   location_t get_location () const final override { return m_loc; }
   int get_stack_depth () const final override { return m_depth; }
@@ -66,7 +71,7 @@ class test_diagnostic_event : public diagnostic_event
   {
     return m_connected_to_next_event;
   }
-  diagnostic_thread_id_t get_thread_id () const final override
+  thread_id_t get_thread_id () const final override
   {
     return m_thread_id;
   }
@@ -82,15 +87,15 @@ class test_diagnostic_event : public diagnostic_event
   int m_depth;
   char *m_desc; // has been formatted; doesn't get i18n-ed
   bool m_connected_to_next_event;
-  diagnostic_thread_id_t m_thread_id;
+  thread_id_t m_thread_id;
 };
 
-/* A simple implementation of diagnostic_thread.  */
+/* A simple implementation of diagnostics::paths::thread.  */
 
-class test_diagnostic_thread : public diagnostic_thread
+class test_thread : public thread
 {
 public:
-  test_diagnostic_thread (const char *name) : m_name (name) {}
+  test_thread (const char *name) : m_name (name) {}
   label_text get_name (bool) const final override
   {
     return label_text::borrow (m_name);
@@ -100,33 +105,34 @@ private:
   const char *m_name; // has been i18n-ed and formatted
 };
 
-/* A concrete subclass of diagnostic_path for implementing selftests
-   - a vector of test_diagnostic_event instances
+/* A concrete subclass of diagnostics::paths::path for implementing selftests
+   - a vector of test_event instances
    - adds member functions for adding test event
    - does no translation of its events
    - has no dependency on "tree".  */
 
-class test_diagnostic_path : public diagnostic_path
+class test_path : public path
 {
  public:
-  test_diagnostic_path (pretty_printer *event_pp);
+  test_path (logical_locations::selftest::test_manager &logical_loc_mgr,
+	     pretty_printer *event_pp);
 
   unsigned num_events () const final override;
-  const diagnostic_event & get_event (int idx) const final override;
+  const event & get_event (int idx) const final override;
   unsigned num_threads () const final override;
-  const diagnostic_thread &
-  get_thread (diagnostic_thread_id_t) const final override;
+  const thread &
+  get_thread (thread_id_t) const final override;
   bool
   same_function_p (int event_idx_a,
 		   int event_idx_b) const final override;
 
-  diagnostic_thread_id_t add_thread (const char *name);
+  thread_id_t add_thread (const char *name);
 
-  diagnostic_event_id_t add_event (location_t loc, const char *funcname, int depth,
-				   const char *fmt, ...)
+  event_id_t add_event (location_t loc, const char *funcname, int depth,
+			const char *fmt, ...)
     ATTRIBUTE_GCC_DIAG(5,6);
-  diagnostic_event_id_t
-  add_thread_event (diagnostic_thread_id_t thread_id,
+  event_id_t
+  add_thread_event (thread_id_t thread_id,
 		    location_t loc, const char *funcname, int depth,
 		    const char *fmt, ...)
     ATTRIBUTE_GCC_DIAG(6,7);
@@ -134,28 +140,30 @@ class test_diagnostic_path : public diagnostic_path
   void connect_to_next_event ();
 
   void add_entry (const char *callee_name, int stack_depth,
-		  diagnostic_thread_id_t thread_id = 0);
+		  thread_id_t thread_id = 0);
   void add_return (const char *caller_name, int stack_depth,
-		   diagnostic_thread_id_t thread_id = 0);
+		   thread_id_t thread_id = 0);
   void add_call (const char *caller_name,
 		 int caller_stack_depth,
 		 const char *callee_name,
-		 diagnostic_thread_id_t thread_id = 0);
+		 thread_id_t thread_id = 0);
 
  private:
-  logical_location
+  logical_locations::key
   logical_location_from_funcname (const char *funcname);
 
-  test_logical_location_manager m_test_logical_loc_mgr;
-  auto_delete_vec<test_diagnostic_thread> m_threads;
-  auto_delete_vec<test_diagnostic_event> m_events;
+  logical_locations::selftest::test_manager &m_test_logical_loc_mgr;
+  auto_delete_vec<test_thread> m_threads;
+  auto_delete_vec<test_event> m_events;
 
   /* (for use by add_event).  */
   pretty_printer *m_event_pp;
 };
 
-} // namespace selftest
+} // namespace diagnostics::paths::selftest
+} // namespace diagnostics::paths
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
 
-#endif /* ! GCC_SELFTEST_DIAGNOSTIC_PATH_H */
+#endif /* ! GCC_DIAGNOSTICS_SELFTEST_PATHS_H */
diff --git a/gcc/selftest-diagnostic-show-locus.h b/gcc/diagnostics/selftest-source-printing.h
index 37a18b2..451c120 100644
--- a/gcc/selftest-diagnostic-show-locus.h
+++ b/gcc/diagnostics/selftest-source-printing.h
@@ -17,16 +17,18 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_SELFTEST_DIAGNOSTIC_SHOW_LOCUS_H
-#define GCC_SELFTEST_DIAGNOSTIC_SHOW_LOCUS_H
+#ifndef GCC_DIAGNOSTICS_SELFTEST_SOURCE_PRINTING_H
+#define GCC_DIAGNOSTICS_SELFTEST_SOURCE_PRINTING_H
 
 #include "selftest.h"
+#include "diagnostics/file-cache.h"
 
 /* The selftest code should entirely disappear in a production
    configuration, hence we guard all of it with #if CHECKING_P.  */
 
 #if CHECKING_P
 
+namespace diagnostics {
 namespace selftest {
 
 /* RAII class for use in selftests involving diagnostic_show_locus.
@@ -37,10 +39,10 @@ namespace selftest {
      push a line_map starting at the first line of the temporary file
    - provide a file_cache.  */
 
-struct diagnostic_show_locus_fixture
+struct source_printing_fixture
 {
-  diagnostic_show_locus_fixture (const line_table_case &case_,
-				 const char *content);
+  source_printing_fixture (const ::selftest::line_table_case &case_,
+			   const char *content);
 
   const char *get_filename () const
   {
@@ -48,8 +50,8 @@ struct diagnostic_show_locus_fixture
   }
 
   const char *m_content;
-  temp_source_file m_tmp_source_file;
-  line_table_test m_ltt;
+  ::selftest::temp_source_file m_tmp_source_file;
+  ::selftest::line_table_test m_ltt;
   file_cache m_fc;
 };
 
@@ -69,14 +71,15 @@ struct diagnostic_show_locus_fixture
    Here SS represents the two display columns for the U+1F602 emoji and
    P represents the one display column for the U+03C0 pi symbol.  */
 
-struct diagnostic_show_locus_fixture_one_liner_utf8
-  : public diagnostic_show_locus_fixture
+struct source_printing_fixture_one_liner_utf8
+  : public source_printing_fixture
 {
-  diagnostic_show_locus_fixture_one_liner_utf8 (const line_table_case &case_);
+  source_printing_fixture_one_liner_utf8 (const ::selftest::line_table_case &case_);
 };
 
-} // namespace selftest
+} // namespace diagnostics::selftest
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
 
-#endif /* GCC_SELFTEST_DIAGNOSTIC_SHOW_LOCUS_H */
+#endif /* GCC_DIAGNOSTICS_SELFTEST_SOURCE_PRINTING_H */
diff --git a/gcc/diagnostic-format.h b/gcc/diagnostics/sink.h
index 07db2cd..ac4e0fb64 100644
--- a/gcc/diagnostic-format.h
+++ b/gcc/diagnostics/sink.h
@@ -18,21 +18,23 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_FORMAT_H
-#define GCC_DIAGNOSTIC_FORMAT_H
+#ifndef GCC_DIAGNOSTICS_SINK_H
+#define GCC_DIAGNOSTICS_SINK_H
 
 #include "diagnostic.h"
 
-class diagnostic_per_format_buffer;
+namespace diagnostics {
+
+class per_sink_buffer;
 
 /* Abstract base class for a particular output format for diagnostics;
    each value of -fdiagnostics-output-format= will have its own
    implementation.  */
 
-class diagnostic_output_format
+class sink
 {
 public:
-  virtual ~diagnostic_output_format () {}
+  virtual ~sink () {}
 
   virtual void dump (FILE *out, int indent) const;
 
@@ -40,16 +42,16 @@ public:
      e.g. for titles of HTML, for SARIF's artifact metadata.  */
   virtual void set_main_input_filename (const char *) {}
 
-  /* Vfunc for making an appropriate diagnostic_per_format_buffer
+  /* Vfunc for making an appropriate per_sink_buffer
      subclass for this format.  */
-  virtual std::unique_ptr<diagnostic_per_format_buffer>
-  make_per_format_buffer () = 0;
+  virtual std::unique_ptr<per_sink_buffer>
+  make_per_sink_buffer () = 0;
 
-  /* Vfunc to be called when call a diagnostic_buffer is set on
-     a diagnostic_context, to update this format.  The per_format_buffer
-     will be one created by make_per_format_buffer above and thus be
+  /* Vfunc to be called when call a diagnostics::buffer is set on
+     a diagnostics::context, to update this format.  The per_sink_buffer
+     will be one created by make_per_sink_buffer above and thus be
      of the correct subclass.  */
-  virtual void set_buffer (diagnostic_per_format_buffer *) = 0;
+  virtual void set_buffer (per_sink_buffer *) = 0;
 
   virtual void on_begin_group () = 0;
   virtual void on_end_group () = 0;
@@ -57,22 +59,25 @@ public:
   /* Vfunc with responsibility for phase 3 of formatting the message
      and "printing" the result.  */
   virtual void on_report_diagnostic (const diagnostic_info &,
-				     diagnostic_t orig_diag_kind) = 0;
+				     enum kind orig_diag_kind) = 0;
 
   virtual void on_report_verbatim (text_info &);
 
-  virtual void on_diagram (const diagnostic_diagram &diagram) = 0;
+  virtual void on_diagram (const diagram &diag) = 0;
   virtual void after_diagnostic (const diagnostic_info &) = 0;
   virtual bool machine_readable_stderr_p () const = 0;
   virtual bool follows_reference_printer_p () const = 0;
 
-  /* Vfunc called when the diagnostic_context changes its
+  /* Vfunc called when the diagnostics::context changes its
      reference printer (either to a new subclass of pretty_printer
      or when color/url options change).
      Subclasses should update their m_printer accordingly.  */
   virtual void update_printer () = 0;
 
-  diagnostic_context &get_context () const { return m_context; }
+  virtual void
+  report_global_digraph (const lazily_created<digraphs::digraph> &) = 0;
+
+  context &get_context () const { return m_context; }
   pretty_printer *get_printer () const { return m_printer.get (); }
 
   text_art::theme *get_diagram_theme () const
@@ -83,21 +88,23 @@ public:
   void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
 
 protected:
-  diagnostic_output_format (diagnostic_context &context)
-  : m_context (context),
-    m_printer (context.clone_printer ())
+  sink (context &dc)
+  : m_context (dc),
+    m_printer (dc.clone_printer ())
   {}
 
 protected:
-  diagnostic_context &m_context;
+  context &m_context;
   std::unique_ptr<pretty_printer> m_printer;
 };
 
 extern void
-diagnostic_output_format_init (diagnostic_context &,
-			       const char *main_input_filename_,
-			       const char *base_file_name,
-			       enum diagnostics_output_format,
-			       bool json_formatting);
+output_format_init (context &,
+		    const char *main_input_filename_,
+		    const char *base_file_name,
+		    enum diagnostics_output_format,
+		    bool json_formatting);
+
+} // namespace diagnostics
 
-#endif /* ! GCC_DIAGNOSTIC_FORMAT_H */
+#endif /* ! GCC_DIAGNOSTICS_SINK_H */
diff --git a/gcc/diagnostic-label-effects.h b/gcc/diagnostics/source-printing-effects.h
index 1d877b0d..3c4b1bd 100644
--- a/gcc/diagnostic-label-effects.h
+++ b/gcc/diagnostics/source-printing-effects.h
@@ -18,8 +18,10 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_LABEL_EFFECTS_H
-#define GCC_DIAGNOSTIC_LABEL_EFFECTS_H
+#ifndef GCC_DIAGNOSTICS_SOURCE_PRINTING_EFFECTS_H
+#define GCC_DIAGNOSTICS_SOURCE_PRINTING_EFFECTS_H
+
+namespace diagnostics {
 
 /* Abstract base class for describing special effects when printing
    a label when quoting source code.  */
@@ -37,10 +39,10 @@ public:
 
 /* A class to hold state when quoting a run of lines of source code.  */
 
-class diagnostic_source_effect_info
+class source_effect_info
 {
 public:
-  diagnostic_source_effect_info ()
+  source_effect_info ()
   : m_leading_in_edge_column (-1),
     m_trailing_out_edge_column (-1)
   {
@@ -55,4 +57,6 @@ public:
   int m_trailing_out_edge_column;
 };
 
-#endif /* GCC_DIAGNOSTIC_LABEL_EFFECTS_H */
+} // namespace diagnostics
+
+#endif /* GCC_DIAGNOSTICS_SOURCE_PRINTING_EFFECTS_H */
diff --git a/gcc/diagnostic-show-locus.cc b/gcc/diagnostics/source-printing.cc
index da7637c..94b1c2d 100644
--- a/gcc/diagnostic-show-locus.cc
+++ b/gcc/diagnostics/source-printing.cc
@@ -25,20 +25,19 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "version.h"
-#include "demangle.h"
 #include "intl.h"
-#include "backtrace.h"
 #include "diagnostic.h"
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
 #include "gcc-rich-location.h"
 #include "text-range-label.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
-#include "selftest-diagnostic-show-locus.h"
+#include "diagnostics/selftest-context.h"
+#include "diagnostics/selftest-source-printing.h"
 #include "cpplib.h"
 #include "text-art/types.h"
 #include "text-art/theme.h"
-#include "diagnostic-label-effects.h"
+#include "diagnostics/source-printing-effects.h"
+#include "diagnostics/file-cache.h"
 #include "xml.h"
 #include "xml-printer.h"
 
@@ -96,7 +95,7 @@ class colorizer
  public:
   colorizer (pretty_printer &pp,
 	     const rich_location &richloc,
-	     diagnostic_t diagnostic_kind);
+	     enum diagnostics::kind diagnostic_kind);
   ~colorizer ();
 
   void set_range (int range_idx)
@@ -116,7 +115,7 @@ class colorizer
        two colors for the secondary locations.
        But if we're printing a run of events in a diagnostic path, that
        makes no sense, so print all of them with the same colorization.  */
-    if (m_diagnostic_kind == DK_DIAGNOSTIC_PATH)
+    if (m_diagnostic_kind == diagnostics::kind::path)
       set_state (0);
     else
       set_state (range_idx);
@@ -141,7 +140,7 @@ class colorizer
 
   pretty_printer &m_pp;
   const rich_location &m_richloc;
-  diagnostic_t m_diagnostic_kind;
+  enum diagnostics::kind m_diagnostic_kind;
   int m_current_state;
   const char *m_range1;
   const char *m_range2;
@@ -203,7 +202,7 @@ enum column_unit {
 class exploc_with_display_col : public expanded_location
 {
  public:
-  exploc_with_display_col (file_cache &fc,
+  exploc_with_display_col (diagnostics::file_cache &fc,
 			   const expanded_location &exploc,
 			   const cpp_char_column_policy &policy,
 			   enum location_aspect aspect)
@@ -375,9 +374,9 @@ struct char_display_policy : public cpp_char_column_policy
  public:
   char_display_policy (int tabstop,
 		       int (*width_cb) (cppchar_t c),
-		       void (*print_text_cb) (to_text &sink,
+		       void (*print_text_cb) (diagnostics::to_text &text_out,
 					      const cpp_decoded_char &cp),
-		       void (*print_html_cb) (to_html &sink,
+		       void (*print_html_cb) (diagnostics::to_html &html_out,
 					      const cpp_decoded_char &cp))
   : cpp_char_column_policy (tabstop, width_cb),
     m_print_text_cb (print_text_cb),
@@ -385,21 +384,23 @@ struct char_display_policy : public cpp_char_column_policy
   {
   }
 
-  void (*m_print_text_cb) (to_text &sink,
+  void (*m_print_text_cb) (diagnostics::to_text &text_out,
 			   const cpp_decoded_char &cp);
-  void (*m_print_html_cb) (to_html &sink,
+  void (*m_print_html_cb) (diagnostics::to_html &html_out,
 			   const cpp_decoded_char &cp);
 };
 
-template <typename Sink> class layout_printer;
+template <typename TextOrHtml> class layout_printer;
 
 } // anonymous namespace
 
+namespace diagnostics {
+
 /* This code is written generically to write either:
    - text, to a pretty_printer, potentially with colorization codes, or
    - html, to an xml::printer, with nested HTML tags.
 
-   This is handled via a "Sink" template, which is either to_text
+   This is handled via a "TextOrHtml" template, which is either to_text
    or to_html.  */
 
 /* Writing text output.  */
@@ -519,8 +520,8 @@ struct to_text
   }
 
   void
-  invoke_start_span_fn (const diagnostic_source_print_policy &source_policy,
-			const diagnostic_location_print_policy &loc_policy,
+  invoke_start_span_fn (const source_print_policy &source_policy,
+			const location_print_policy &loc_policy,
 			const expanded_location &exploc)
   {
     source_policy.get_text_start_span_fn () (loc_policy, *this, exploc);
@@ -548,8 +549,8 @@ struct to_html
   struct auto_check_tag_nesting : public xml::auto_check_tag_nesting
   {
   public:
-    auto_check_tag_nesting (to_html &sink)
-    : xml::auto_check_tag_nesting (sink.m_xp)
+    auto_check_tag_nesting (to_html &out)
+    : xml::auto_check_tag_nesting (out.m_xp)
     {
     }
   };
@@ -654,8 +655,8 @@ struct to_html
   }
 
   void
-  invoke_start_span_fn (const diagnostic_source_print_policy &source_policy,
-			const diagnostic_location_print_policy &loc_policy,
+  invoke_start_span_fn (const source_print_policy &source_policy,
+			const location_print_policy &loc_policy,
 			const expanded_location &exploc)
   {
     source_policy.get_html_start_span_fn () (loc_policy, *this, exploc);
@@ -687,66 +688,67 @@ private:
 };
 
 void
-diagnostic_location_print_policy::
-print_text_span_start (const diagnostic_context &dc,
+location_print_policy::
+print_text_span_start (const diagnostics::context &dc,
 		       pretty_printer &pp,
 		       const expanded_location &exploc)
 {
-  to_text sink (pp, nullptr);
-  diagnostic_source_print_policy source_policy (dc);
-  source_policy.get_text_start_span_fn () (*this, sink, exploc);
+  to_text out (pp, nullptr);
+  source_print_policy source_policy (dc);
+  source_policy.get_text_start_span_fn () (*this, out, exploc);
 }
 
 void
-diagnostic_location_print_policy::
-print_html_span_start (const diagnostic_context &dc,
+location_print_policy::
+print_html_span_start (const diagnostics::context &dc,
 		       xml::printer &xp,
 		       const expanded_location &exploc)
 {
-  to_html sink (xp, nullptr, nullptr);
-  diagnostic_source_print_policy source_policy (dc);
-  source_policy.get_html_start_span_fn () (*this, sink, exploc);
+  to_html out (xp, nullptr, nullptr);
+  source_print_policy source_policy (dc);
+  source_policy.get_html_start_span_fn () (*this, out, exploc);
 }
 
 pretty_printer *
-get_printer (to_text &sink)
+get_printer (to_text &out)
 {
-  return &sink.m_pp;
+  return &out.m_pp;
 }
 
 template<>
 void
-default_diagnostic_start_span_fn<to_text> (const diagnostic_location_print_policy &loc_policy,
-					   to_text &sink,
-					   expanded_location exploc)
+default_start_span_fn<to_text> (const location_print_policy &loc_policy,
+				to_text &out,
+				expanded_location exploc)
 {
-  const diagnostic_column_policy &column_policy
-    = loc_policy.get_column_policy ();
+  const column_policy &column_policy_ = loc_policy.get_column_policy ();
   label_text text
-    = column_policy.get_location_text (exploc,
-				       loc_policy.show_column_p (),
-				       pp_show_color (&sink.m_pp));
-  pp_string (&sink.m_pp, text.get ());
-  pp_newline (&sink.m_pp);
+    = column_policy_.get_location_text (exploc,
+					loc_policy.show_column_p (),
+					pp_show_color (&out.m_pp));
+  pp_string (&out.m_pp, text.get ());
+  pp_newline (&out.m_pp);
 }
 
 template<>
 void
-default_diagnostic_start_span_fn<to_html> (const diagnostic_location_print_policy &loc_policy,
-					   to_html &sink,
-					   expanded_location exploc)
+default_start_span_fn<to_html> (const location_print_policy &loc_policy,
+				to_html &out,
+				expanded_location exploc)
 {
-  const diagnostic_column_policy &column_policy
+  const column_policy &column_policy_
     = loc_policy.get_column_policy ();
   label_text text
-    = column_policy.get_location_text (exploc,
-				       loc_policy.show_column_p (),
-				       false);
-  sink.m_xp.push_tag_with_class ("span", "location", true);
-  sink.m_xp.add_text (text.get ());
-  sink.m_xp.pop_tag ("span");
+    = column_policy_.get_location_text (exploc,
+					loc_policy.show_column_p (),
+					false);
+  out.m_xp.push_tag_with_class ("span", "location", true);
+  out.m_xp.add_text (text.get ());
+  out.m_xp.pop_tag ("span");
 }
 
+} // namespace diagnostics
+
 namespace {
 
 /* A class to control the overall layout when printing a diagnostic.
@@ -761,12 +763,12 @@ namespace {
 class layout
 {
  public:
-  friend class layout_printer<to_text>;
-  friend class layout_printer<to_html>;
+  friend class layout_printer<diagnostics::to_text>;
+  friend class layout_printer<diagnostics::to_html>;
 
-  layout (const diagnostic_source_print_policy &source_policy,
+  layout (const diagnostics::source_print_policy &source_policy,
 	  const rich_location &richloc,
-	  diagnostic_source_effect_info *effect_info = nullptr);
+	  diagnostics::source_effect_info *effect_info = nullptr);
 
   bool maybe_add_location_range (const location_range *loc_range,
 				 unsigned original_idx,
@@ -811,12 +813,12 @@ class layout
  private:
   bool compatible_locations_p (location_t loc_a, location_t loc_b) const;
 
-  const diagnostic_source_printing_options &m_options;
+  const diagnostics::source_printing_options &m_options;
   const line_maps *m_line_table;
-  file_cache &m_file_cache;
+  diagnostics::file_cache &m_file_cache;
   const text_art::ascii_theme m_fallback_theme;
   const text_art::theme &m_theme;
-  diagnostic_source_effect_info *m_effect_info;
+  diagnostics::source_effect_info *m_effect_info;
   char_display_policy m_char_policy;
   location_t m_primary_loc;
   exploc_with_display_col m_exploc;
@@ -838,19 +840,19 @@ enum class margin_kind
 };
 
 /* A bundle of state for printing a particular layout
-   to a particular Sink (either to_text or to_html).  */
-template <typename Sink>
+   to a particular TextOrHtml (either to_text or to_html).  */
+template <typename TextOrHtml>
 class layout_printer
 {
 public:
-  layout_printer (Sink &sink,
+  layout_printer (TextOrHtml &text_or_html,
 		  const layout &layout,
 		  bool is_diagnostic_path);
 
-  void print (const diagnostic_source_print_policy &source_policy);
+  void print (const diagnostics::source_print_policy &source_policy);
 
 private:
-  const diagnostic_source_printing_options &
+  const diagnostics::source_printing_options &
   get_options () const { return m_layout.m_options; }
 
   const text_art::theme &
@@ -880,7 +882,7 @@ private:
   void set_outside_range ();
 
 private:
-  Sink &m_sink;
+  TextOrHtml &m_text_or_html;
   const layout &m_layout;
   bool m_is_diagnostic_path;
 
@@ -944,7 +946,7 @@ private:
 
 colorizer::colorizer (pretty_printer &pp,
 		      const rich_location &richloc,
-		      diagnostic_t diagnostic_kind) :
+		      enum diagnostics::kind diagnostic_kind) :
   m_pp (pp),
   m_richloc (richloc),
   m_diagnostic_kind (diagnostic_kind),
@@ -1022,7 +1024,7 @@ colorizer::begin_state (int state)
       pp_string
 	(&m_pp,
 	 colorize_start (pp_show_color (&m_pp),
-			 diagnostic_get_color_for_kind (m_diagnostic_kind)));
+			 get_color_for_kind (m_diagnostic_kind)));
       break;
 
     case 1:
@@ -1206,7 +1208,8 @@ layout_range::has_in_edge () const
 {
   if (!m_label)
     return false;
-  const label_effects *effects = m_label->get_effects (m_original_idx);
+  const diagnostics::label_effects *effects
+    = m_label->get_effects (m_original_idx);
   if (!effects)
     return false;
 
@@ -1220,7 +1223,8 @@ layout_range::has_out_edge () const
 {
   if (!m_label)
     return false;
-  const label_effects *effects = m_label->get_effects (m_original_idx);
+  const diagnostics::label_effects *effects
+    = m_label->get_effects (m_original_idx);
   if (!effects)
     return false;
 
@@ -1247,7 +1251,7 @@ static cpp_char_column_policy def_policy ()
    e.g. in test_diagnostic_show_locus_one_liner_utf8().  */
 
 static layout_range
-make_range (file_cache &fc,
+make_range (diagnostics::file_cache &fc,
 	    int start_line, int start_col, int end_line, int end_col)
 {
   const expanded_location start_exploc
@@ -1276,7 +1280,7 @@ make_range (file_cache &fc,
 static void
 test_layout_range_for_single_point ()
 {
-  file_cache fc;
+  diagnostics::file_cache fc;
   layout_range point = make_range (fc, 7, 10, 7, 10);
 
   /* Tests for layout_range::contains_point.  */
@@ -1313,7 +1317,7 @@ test_layout_range_for_single_point ()
 static void
 test_layout_range_for_single_line ()
 {
-  file_cache fc;
+  diagnostics::file_cache fc;
   layout_range example_a = make_range (fc, 2, 22, 2, 38);
 
   /* Tests for layout_range::contains_point.  */
@@ -1356,7 +1360,7 @@ test_layout_range_for_single_line ()
 static void
 test_layout_range_for_multiple_lines ()
 {
-  file_cache fc;
+  diagnostics::file_cache fc;
   layout_range example_b = make_range (fc, 3, 14, 5, 8);
 
   /* Tests for layout_range::contains_point.  */
@@ -1567,9 +1571,9 @@ fixit_cmp (const void *p_a, const void *p_b)
 /* Callback for char_display_policy::m_print_cb for printing source chars
    when not escaping the source.  */
 
-template <class Sink>
+template <class TextOrHtml>
 static void
-default_print_decoded_ch (Sink &sink,
+default_print_decoded_ch (TextOrHtml &text_or_html,
 			  const cpp_decoded_char &decoded_ch)
 {
   for (const char *ptr = decoded_ch.m_start_byte;
@@ -1577,11 +1581,11 @@ default_print_decoded_ch (Sink &sink,
     {
       if (*ptr == '\0' || *ptr == '\r')
 	{
-	  sink.add_space ();
+	  text_or_html.add_space ();
 	  continue;
 	}
 
-      sink.add_utf8_byte (*ptr);
+      text_or_html.add_utf8_byte (*ptr);
     }
 }
 
@@ -1609,9 +1613,9 @@ escape_as_bytes_width (cppchar_t ch)
 /* Callback for char_display_policy::m_print_cb for printing source chars
    when escaping with DIAGNOSTICS_ESCAPE_FORMAT_BYTES.  */
 
-template <typename Sink>
+template <typename TextOrHtml>
 static void
-escape_as_bytes_print (Sink &sink,
+escape_as_bytes_print (TextOrHtml &text_or_html,
 		       const cpp_decoded_char &decoded_ch)
 {
   if (!decoded_ch.m_valid_ch)
@@ -1621,14 +1625,14 @@ escape_as_bytes_print (Sink &sink,
 	{
 	  char buf[16];
 	  sprintf (buf, "<%02x>", (unsigned char)*iter);
-	  sink.add_text (buf);
+	  text_or_html.add_text (buf);
 	}
       return;
     }
 
   cppchar_t ch = decoded_ch.m_ch;
   if (ch < 0x80 && ISPRINT (ch))
-    sink.add_character (ch);
+    text_or_html.add_character (ch);
   else
     {
       for (const char *iter = decoded_ch.m_start_byte;
@@ -1636,7 +1640,7 @@ escape_as_bytes_print (Sink &sink,
 	{
 	  char buf[16];
 	  sprintf (buf, "<%02x>", (unsigned char)*iter);
-	  sink.add_text (buf);
+	  text_or_html.add_text (buf);
 	}
     }
 }
@@ -1666,40 +1670,40 @@ escape_as_unicode_width (cppchar_t ch)
 /* Callback for char_display_policy::m_print_cb for printing source chars
    when escaping with DIAGNOSTICS_ESCAPE_FORMAT_UNICODE.  */
 
-template <typename Sink>
+template <typename TextOrHtml>
 static void
-escape_as_unicode_print (Sink &sink,
+escape_as_unicode_print (TextOrHtml &text_or_html,
 			 const cpp_decoded_char &decoded_ch)
 {
   if (!decoded_ch.m_valid_ch)
     {
-      escape_as_bytes_print<Sink> (sink, decoded_ch);
+      escape_as_bytes_print<TextOrHtml> (text_or_html, decoded_ch);
       return;
     }
 
   cppchar_t ch = decoded_ch.m_ch;
   if (ch < 0x80 && ISPRINT (ch))
-    sink.add_character (ch);
+    text_or_html.add_character (ch);
   else
     {
       char buf[16];
       sprintf (buf, "<U+%04X>", ch);
-      sink.add_text (buf);
+      text_or_html.add_text (buf);
     }
 }
 
 /* Populate a char_display_policy based on SOURCE_POLICY and RICHLOC.  */
 
 static char_display_policy
-make_char_policy (const diagnostic_source_print_policy &source_policy,
+make_char_policy (const diagnostics::source_print_policy &source_policy,
 		  const rich_location &richloc)
 {
   /* The default is to not escape non-ASCII bytes.  */
   char_display_policy result
     (source_policy.get_column_policy ().get_tabstop (),
      cpp_wcwidth,
-     default_print_decoded_ch<to_text>,
-     default_print_decoded_ch<to_html>);
+     default_print_decoded_ch<diagnostics::to_text>,
+     default_print_decoded_ch<diagnostics::to_html>);
 
   /* If the diagnostic suggests escaping non-ASCII bytes, then
      use policy from user-supplied options.  */
@@ -1712,13 +1716,15 @@ make_char_policy (const diagnostic_source_print_policy &source_policy,
 	  gcc_unreachable ();
 	case DIAGNOSTICS_ESCAPE_FORMAT_UNICODE:
 	  result.m_width_cb = escape_as_unicode_width;
-	  result.m_print_text_cb = escape_as_unicode_print<to_text>;
-	  result.m_print_html_cb = escape_as_unicode_print<to_html>;
+	  result.m_print_text_cb
+	    = escape_as_unicode_print<diagnostics::to_text>;
+	  result.m_print_html_cb
+	    = escape_as_unicode_print<diagnostics::to_html>;
 	  break;
 	case DIAGNOSTICS_ESCAPE_FORMAT_BYTES:
 	  result.m_width_cb = escape_as_bytes_width;
-	  result.m_print_text_cb = escape_as_bytes_print<to_text>;
-	  result.m_print_html_cb = escape_as_bytes_print<to_html>;
+	  result.m_print_text_cb = escape_as_bytes_print<diagnostics::to_text>;
+	  result.m_print_html_cb = escape_as_bytes_print<diagnostics::to_html>;
 	  break;
 	}
     }
@@ -1735,11 +1741,11 @@ make_char_policy (const diagnostic_source_print_policy &source_policy,
    Determine the range of lines that we will print, splitting them
    up into an ordered list of disjoint spans of contiguous line numbers.
    Determine m_x_offset_display, to ensure that the primary caret
-   will fit within the max_width provided by the diagnostic_context.  */
+   will fit within the max_width provided by the diagnostics::context.  */
 
-layout::layout (const diagnostic_source_print_policy &source_policy,
+layout::layout (const diagnostics::source_print_policy &source_policy,
 		const rich_location &richloc,
-		diagnostic_source_effect_info *effect_info)
+		diagnostics::source_effect_info *effect_info)
 : m_options (source_policy.get_options ()),
   m_line_table (richloc.get_line_table ()),
   m_file_cache (source_policy.get_file_cache ()),
@@ -1927,25 +1933,25 @@ layout::will_show_line_p (linenum_type row) const
 
 template<>
 void
-layout_printer<to_text>::print_gap_in_line_numbering ()
+layout_printer<diagnostics::to_text>::print_gap_in_line_numbering ()
 {
   gcc_assert (m_layout.m_options.show_line_numbers_p);
 
-  m_sink.emit_text_prefix ();
+  m_text_or_html.emit_text_prefix ();
 
   for (int i = 0; i < m_layout.get_linenum_width () + 1; i++)
-    m_sink.add_character ('.');
+    m_text_or_html.add_character ('.');
 
-  m_sink.add_newline ();
+  m_text_or_html.add_newline ();
 }
 
 template<>
 void
-layout_printer<to_html>::print_gap_in_line_numbering ()
+layout_printer<diagnostics::to_html>::print_gap_in_line_numbering ()
 {
   gcc_assert (m_layout.m_options.show_line_numbers_p);
 
-  m_sink.add_raw_html
+  m_text_or_html.add_raw_html
     ("<tbody class=\"line-span-jump\">\n"
      "<tr class=\"line-span-jump-row\">"
      "<td class=\"linenum-gap\">[...]</td>"
@@ -2169,7 +2175,7 @@ layout::calculate_linenum_width ()
   int highest_line = last_span->m_last_line;
   if (highest_line < 0)
     highest_line = 0;
-  m_linenum_width = num_digits (highest_line);
+  m_linenum_width = diagnostics::num_digits (highest_line);
   /* If we're showing jumps in the line-numbering, allow at least 3 chars.  */
   if (m_line_spans.length () > 1)
     m_linenum_width = MAX (m_linenum_width, 3);
@@ -2196,8 +2202,9 @@ layout::calculate_x_offset_display ()
       return;
     }
 
-  const char_span line = m_file_cache.get_source_line (m_exploc.file,
-						       m_exploc.line);
+  const diagnostics::char_span line
+    = m_file_cache.get_source_line (m_exploc.file,
+				    m_exploc.line);
   if (!line)
     {
       /* Nothing to do, we couldn't find the source line.  */
@@ -2264,32 +2271,32 @@ layout::calculate_x_offset_display ()
    colorization and tab expansion, this function tracks the line position in
    both byte and display column units.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 line_bounds
-layout_printer<Sink>::print_source_line (linenum_type row,
+layout_printer<TextOrHtml>::print_source_line (linenum_type row,
 					 const char *line,
 					 int line_bytes)
 {
-  m_sink.colorize_text_ensure_normal ();
-  m_sink.push_html_tag ("tr", true);
-  m_sink.emit_text_prefix ();
+  m_text_or_html.colorize_text_ensure_normal ();
+  m_text_or_html.push_html_tag ("tr", true);
+  m_text_or_html.emit_text_prefix ();
   if (m_layout.m_options.show_line_numbers_p)
     {
-      m_sink.push_html_tag_with_class ("td", "linenum", true);
-      int width = num_digits (row);
+      m_text_or_html.push_html_tag_with_class ("td", "linenum", true);
+      int width = diagnostics::num_digits (row);
       for (int i = 0; i < m_layout.get_linenum_width () - width; i++)
-	m_sink.add_space ();
+	m_text_or_html.add_space ();
       char buf[20];
       sprintf (buf, "%i", row);
-      m_sink.add_text (buf);
-      if (Sink::is_text ())
-	m_sink.add_text (" |");
-      m_sink.pop_html_tag ("td");
+      m_text_or_html.add_text (buf);
+      if (TextOrHtml::is_text ())
+	m_text_or_html.add_text (" |");
+      m_text_or_html.pop_html_tag ("td");
     }
 
-  m_sink.push_html_tag_with_class ("td", "left-margin", true);
+  m_text_or_html.push_html_tag_with_class ("td", "left-margin", true);
   print_leftmost_column ();
-  m_sink.pop_html_tag ("td");
+  m_text_or_html.pop_html_tag ("td");
 
   /* We will stop printing the source line at any trailing whitespace.  */
   line_bytes = get_line_bytes_without_trailing_whitespace (line,
@@ -2300,7 +2307,7 @@ layout_printer<Sink>::print_source_line (linenum_type row,
      tab expansion, and for implementing m_x_offset_display.  */
   cpp_display_width_computation dw (line, line_bytes, m_layout.m_char_policy);
 
-  m_sink.push_html_tag_with_class ("td", "source", true);
+  m_text_or_html.push_html_tag_with_class ("td", "source", true);
 
   /* Skip the first m_x_offset_display display columns.  In case the leading
      portion that will be skipped ends with a character with wcwidth > 1, then
@@ -2312,7 +2319,7 @@ layout_printer<Sink>::print_source_line (linenum_type row,
   for (int skipped_display_cols
 	 = dw.advance_display_cols (m_layout.m_x_offset_display);
        skipped_display_cols > m_layout.m_x_offset_display; --skipped_display_cols)
-    m_sink.add_space ();
+    m_text_or_html.add_space ();
 
   /* Print the line and compute the line_bounds.  */
   line_bounds lbounds;
@@ -2356,7 +2363,7 @@ layout_printer<Sink>::print_source_line (linenum_type row,
 	  /* The returned display width is the number of spaces into which the
 	     tab should be expanded.  */
 	  for (int i = 0; i != this_display_width; ++i)
-	    m_sink.add_space ();
+	    m_text_or_html.add_space ();
 	  continue;
 	}
 
@@ -2370,7 +2377,7 @@ layout_printer<Sink>::print_source_line (linenum_type row,
 	}
 
       /* Output the character.  */
-      m_sink.print_decoded_char (m_layout.m_char_policy, cp);
+      m_text_or_html.print_decoded_char (m_layout.m_char_policy, cp);
       c = dw.next_byte ();
     }
   set_outside_range ();
@@ -2399,9 +2406,9 @@ layout::should_print_annotation_line_p (linenum_type row) const
 /* Print the leftmost column after the margin, which is used for showing
    links between labels (e.g. for CFG edges in execution paths).  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::print_leftmost_column ()
+layout_printer<TextOrHtml>::print_leftmost_column ()
 {
   if (!get_options ().show_event_links_p)
     gcc_assert (m_link_lhs_state == link_lhs_state::none);
@@ -2411,33 +2418,33 @@ layout_printer<Sink>::print_leftmost_column ()
     default:
       gcc_unreachable ();
     case link_lhs_state::none:
-      m_sink.add_space ();
+      m_text_or_html.add_space ();
       break;
     case link_lhs_state::rewinding_to_lhs:
       {
-	m_sink.colorize_text_for_cfg_edge ();
+	m_text_or_html.colorize_text_for_cfg_edge ();
 	const cppchar_t ch = get_theme ().get_cppchar
 	  (text_art::theme::cell_kind::CFG_FROM_LEFT_TO_DOWN);
-	m_sink.add_character (ch);
-	m_sink.colorize_text_ensure_normal ();
+	m_text_or_html.add_character (ch);
+	m_text_or_html.colorize_text_ensure_normal ();
       }
       break;
     case link_lhs_state::at_lhs:
       {
-	m_sink.colorize_text_for_cfg_edge ();
+	m_text_or_html.colorize_text_for_cfg_edge ();
 	const cppchar_t ch = get_theme ().get_cppchar
 	  (text_art::theme::cell_kind::CFG_DOWN);
-	m_sink.add_character (ch);
-	m_sink.colorize_text_ensure_normal ();
+	m_text_or_html.add_character (ch);
+	m_text_or_html.colorize_text_ensure_normal ();
       }
       break;
     case link_lhs_state::indenting_to_dest:
       {
-	m_sink.colorize_text_for_cfg_edge ();
+	m_text_or_html.colorize_text_for_cfg_edge ();
 	const cppchar_t ch = get_theme ().get_cppchar
 	  (text_art::theme::cell_kind::CFG_FROM_DOWN_TO_RIGHT);
-	m_sink.add_character (ch);
-	m_sink.colorize_text_ensure_normal ();
+	m_text_or_html.add_character (ch);
+	m_text_or_html.colorize_text_ensure_normal ();
       }
       break;
     }
@@ -2450,16 +2457,16 @@ layout_printer<Sink>::print_leftmost_column ()
    After any left margin, print a leftmost column, which is used for
    showing links between labels (e.g. for CFG edges in execution paths).
 
-   For text sinks, this also first prints the text prefix.
-   For html sinks, this also pushes <tr> and <td> open tags, where the
+   For text output, this also first prints the text prefix.
+   For html output, this also pushes <tr> and <td> open tags, where the
    <td> is for the coming annotations.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::start_annotation_line (enum margin_kind margin)
+layout_printer<TextOrHtml>::start_annotation_line (enum margin_kind margin)
 {
-  m_sink.emit_text_prefix ();
-  m_sink.push_html_tag ("tr", true);
+  m_text_or_html.emit_text_prefix ();
+  m_text_or_html.push_html_tag ("tr", true);
 
   char margin_char = (margin == margin_kind::insertion
 		      ? '+'
@@ -2469,29 +2476,29 @@ layout_printer<Sink>::start_annotation_line (enum margin_kind margin)
     {
       /* Print the margin.  If MARGIN_CHAR != ' ', then print up to 3
 	 of it, right-aligned, padded with spaces.  */
-      m_sink.push_html_tag_with_class ("td", "linenum", true);
+      m_text_or_html.push_html_tag_with_class ("td", "linenum", true);
       int i;
       for (i = 0; i < m_layout.m_linenum_width - 3; i++)
-	m_sink.add_space ();
+	m_text_or_html.add_space ();
       for (; i < m_layout.m_linenum_width; i++)
-	m_sink.add_character (margin_char);
-      if (Sink::is_text ())
-	m_sink.add_text (" |");
-      m_sink.pop_html_tag ("td");
+	m_text_or_html.add_character (margin_char);
+      if (TextOrHtml::is_text ())
+	m_text_or_html.add_text (" |");
+      m_text_or_html.pop_html_tag ("td");
     }
 
-  m_sink.push_html_tag_with_class ("td", "left-margin", true);
+  m_text_or_html.push_html_tag_with_class ("td", "left-margin", true);
   if (margin == margin_kind::insertion)
-    m_sink.add_character (margin_char);
+    m_text_or_html.add_character (margin_char);
   else
     print_leftmost_column ();
-  m_sink.pop_html_tag ("td");
+  m_text_or_html.pop_html_tag ("td");
 
-  m_sink.push_html_tag_with_class ("td",
-				   (margin == margin_kind::ruler
-				    ? "ruler"
-				    : "annotation"),
-				   true);
+  m_text_or_html.push_html_tag_with_class ("td",
+					   (margin == margin_kind::ruler
+					    ? "ruler"
+					    : "annotation"),
+					   true);
 }
 
 /* End a source or annotation line: text implementation.
@@ -2499,10 +2506,10 @@ layout_printer<Sink>::start_annotation_line (enum margin_kind margin)
 
 template<>
 void
-layout_printer<to_text>::end_line ()
+layout_printer<diagnostics::to_text>::end_line ()
 {
-  m_sink.colorize_text_ensure_normal ();
-  m_sink.add_newline ();
+  m_text_or_html.colorize_text_ensure_normal ();
+  m_text_or_html.add_newline ();
 }
 
 /* End a source or annotation line: HTML implementation.
@@ -2510,18 +2517,18 @@ layout_printer<to_text>::end_line ()
 
 template<>
 void
-layout_printer<to_html>::end_line ()
+layout_printer<diagnostics::to_html>::end_line ()
 {
-  m_sink.pop_html_tag ("td");
-  m_sink.pop_html_tag ("tr");
+  m_text_or_html.pop_html_tag ("td");
+  m_text_or_html.pop_html_tag ("tr");
 }
 
 /* Handle the various transitions between being-in-range and
    not-being-in-a-range, and between ranges.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::set_in_range (int range_idx)
+layout_printer<TextOrHtml>::set_in_range (int range_idx)
 {
   if (m_was_in_range_p)
     {
@@ -2541,9 +2548,9 @@ layout_printer<Sink>::set_in_range (int range_idx)
   m_last_range_idx = range_idx;
 }
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::set_outside_range ()
+layout_printer<TextOrHtml>::set_outside_range ()
 {
   if (m_was_in_range_p)
     /* transition from "in a range" to "not in a range".  */
@@ -2554,9 +2561,9 @@ layout_printer<Sink>::set_outside_range ()
 /* Print a line consisting of the caret/underlines for the given
    source line.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::print_annotation_line (linenum_type row,
+layout_printer<TextOrHtml>::print_annotation_line (linenum_type row,
 					     const line_bounds lbounds)
 {
   int x_bound = m_layout.get_x_bound_for_row (row,
@@ -2590,15 +2597,15 @@ layout_printer<Sink>::print_annotation_line (linenum_type row,
 		caret_char = get_options ().caret_chars[state.range_idx];
 	      else
 		caret_char = '^';
-	      m_sink.add_character (caret_char);
+	      m_text_or_html.add_character (caret_char);
 	    }
 	  else
-	    m_sink.add_character ('~');
+	    m_text_or_html.add_character ('~');
 	}
       else
 	{
 	  /* Not in a range.  */
-	  m_sink.add_character (' ');
+	  m_text_or_html.add_character (' ');
 	}
     }
 
@@ -2697,54 +2704,54 @@ public:
 
 template<>
 void
-layout_printer<to_text>::begin_label (int range_idx,
-				      bool is_label_text)
+layout_printer<diagnostics::to_text>::begin_label (int range_idx,
+						   bool is_label_text)
 {
   /* Colorize the text, unless it's for labels for events in a
-     diagnostic_path.  */
+     diagnostic path.  */
   if (is_label_text && m_is_diagnostic_path)
     return;
 
-  gcc_assert (m_sink.m_colorizer);
-  m_sink.m_colorizer->set_range (range_idx);
+  gcc_assert (m_text_or_html.m_colorizer);
+  m_text_or_html.m_colorizer->set_range (range_idx);
 }
 
 template<>
 void
-layout_printer<to_html>::begin_label (int range_idx,
-				      bool is_label_text)
+layout_printer<diagnostics::to_html>::begin_label (int range_idx,
+						   bool is_label_text)
 {
-  if (is_label_text && m_sink.m_html_label_writer)
-    m_sink.m_html_label_writer->begin_label ();
+  if (is_label_text && m_text_or_html.m_html_label_writer)
+    m_text_or_html.m_html_label_writer->begin_label ();
 
   if (const char *highlight_color
-	= m_sink.get_highlight_color_for_range_idx (range_idx))
-    m_sink.m_xp.push_tag_with_class ("span", highlight_color);
+	= m_text_or_html.get_highlight_color_for_range_idx (range_idx))
+    m_text_or_html.m_xp.push_tag_with_class ("span", highlight_color);
 }
 
 template<>
 void
-layout_printer<to_text>::end_label (int, bool)
+layout_printer<diagnostics::to_text>::end_label (int, bool)
 {
-  m_sink.colorize_text_ensure_normal ();
+  m_text_or_html.colorize_text_ensure_normal ();
 }
 
 template<>
 void
-layout_printer<to_html>::end_label (int range_idx,
-				    bool is_label_text)
+layout_printer<diagnostics::to_html>::end_label (int range_idx,
+						 bool is_label_text)
 {
-  if (m_sink.get_highlight_color_for_range_idx (range_idx))
-    m_sink.m_xp.pop_tag ("span");
+  if (m_text_or_html.get_highlight_color_for_range_idx (range_idx))
+    m_text_or_html.m_xp.pop_tag ("span");
 
-  if (is_label_text && m_sink.m_html_label_writer)
-    m_sink.m_html_label_writer->end_label ();
+  if (is_label_text && m_text_or_html.m_html_label_writer)
+    m_text_or_html.m_html_label_writer->end_label ();
 }
 
 /* Print any labels in this row.  */
-template <typename Sink>
+template <typename TextOrHtml>
 void
-layout_printer<Sink>::print_any_labels (linenum_type row)
+layout_printer<TextOrHtml>::print_any_labels (linenum_type row)
 {
   int i;
   auto_vec<line_label> labels;
@@ -2885,20 +2892,20 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
 		       . ^~~~~~~~~~~~~
 		       . this text.  */
 		    gcc_assert (get_options ().show_event_links_p);
-		    m_sink.colorize_text_for_cfg_edge ();
+		    m_text_or_html.colorize_text_for_cfg_edge ();
 		    const cppchar_t right= get_theme ().get_cppchar
 		      (text_art::theme::cell_kind::CFG_RIGHT);
 		    while (column < label->m_column - 1)
 		      {
-			m_sink.add_character (right);
+			m_text_or_html.add_character (right);
 			column++;
 		      }
 		    if (column == label->m_column - 1)
 		      {
-			m_sink.add_character ('>');
+			m_text_or_html.add_character ('>');
 			column++;
 		      }
-		    m_sink.colorize_text_ensure_normal ();
+		    m_text_or_html.colorize_text_ensure_normal ();
 		    m_link_lhs_state = link_lhs_state::none;
 		    label_line_with_in_edge = -1;
 		  }
@@ -2907,7 +2914,7 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
 		gcc_assert (column == label->m_column);
 
 		begin_label (label->m_state_idx, true);
-		m_sink.add_text (label->m_text.m_buffer);
+		m_text_or_html.add_text (label->m_text.m_buffer);
 		end_label (label->m_state_idx, true);
 
 		column += label->m_display_width;
@@ -2924,13 +2931,13 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
 		      (text_art::theme::cell_kind::CFG_RIGHT);
 		    const cppchar_t from_right_to_down= get_theme ().get_cppchar
 		      (text_art::theme::cell_kind::CFG_FROM_RIGHT_TO_DOWN);
-		    m_sink.colorize_text_for_cfg_edge ();
-		    m_sink.add_space ();
-		    m_sink.add_character (right);
-		    m_sink.add_character ('>');
-		    m_sink.add_character (right);
-		    m_sink.add_character (from_right_to_down);
-		    m_sink.colorize_text_ensure_normal ();
+		    m_text_or_html.colorize_text_for_cfg_edge ();
+		    m_text_or_html.add_space ();
+		    m_text_or_html.add_character (right);
+		    m_text_or_html.add_character ('>');
+		    m_text_or_html.add_character (right);
+		    m_text_or_html.add_character (from_right_to_down);
+		    m_text_or_html.colorize_text_ensure_normal ();
 		    column += 5;
 		    m_link_rhs_column = column - 1;
 		  }
@@ -2940,7 +2947,7 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
 		gcc_assert (column <= label->m_column);
 		move_to_column (&column, label->m_column, true);
 		begin_label (label->m_state_idx, false);
-		m_sink.add_character ('|');
+		m_text_or_html.add_character ('|');
 		end_label (label->m_state_idx, false);
 		column++;
 	      }
@@ -2951,11 +2958,11 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
 	if (m_link_rhs_column != -1 && column < m_link_rhs_column)
 	  {
 	    move_to_column (&column, m_link_rhs_column, true);
-	    m_sink.colorize_text_for_cfg_edge ();
+	    m_text_or_html.colorize_text_for_cfg_edge ();
 	    const cppchar_t down= get_theme ().get_cppchar
 	      (text_art::theme::cell_kind::CFG_DOWN);
-	    m_sink.add_character (down);
-	    m_sink.colorize_text_ensure_normal ();
+	    m_text_or_html.add_character (down);
+	    m_text_or_html.colorize_text_ensure_normal ();
 	  }
 
 	end_line ();
@@ -2969,10 +2976,10 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
       int column = 1 + m_layout.m_x_offset_display;
       start_annotation_line (margin_kind::normal);
       move_to_column (&column, m_link_rhs_column, true);
-      m_sink.colorize_text_for_cfg_edge ();
+      m_text_or_html.colorize_text_for_cfg_edge ();
       const cppchar_t down= get_theme ().get_cppchar
 	(text_art::theme::cell_kind::CFG_DOWN);
-      m_sink.add_character (down);
+      m_text_or_html.add_character (down);
       end_line ();
     }
 
@@ -2990,9 +2997,9 @@ layout_printer<Sink>::print_any_labels (linenum_type row)
    They are printed on lines of their own, before the source line
    itself, with a leading '+'.  */
 
-template <typename Sink>
+template <typename TextOrHtml>
 void
-layout_printer<Sink>::print_leading_fixits (linenum_type row)
+layout_printer<TextOrHtml>::print_leading_fixits (linenum_type row)
 {
   for (unsigned int i = 0; i < m_layout.m_fixit_hints.length (); i++)
     {
@@ -3012,14 +3019,14 @@ layout_printer<Sink>::print_leading_fixits (linenum_type row)
 	     and the inserted line with "insert" colorization
 	     helps them stand out from each other, and from
 	     the surrounding text.  */
-	  m_sink.colorize_text_ensure_normal ();
+	  m_text_or_html.colorize_text_ensure_normal ();
 	  start_annotation_line (margin_kind::insertion);
-	  m_sink.colorize_text_for_fixit_insert ();
+	  m_text_or_html.colorize_text_for_fixit_insert ();
 	  /* Print all but the trailing newline of the fix-it hint.
 	     We have to print the newline separately to avoid
 	     getting additional pp prefixes printed.  */
 	  for (size_t i = 0; i < hint->get_length () - 1; i++)
-	    m_sink.add_character (hint->get_string ()[i]);
+	    m_text_or_html.add_character (hint->get_string ()[i]);
 	  end_line ();
 	}
     }
@@ -3154,7 +3161,7 @@ public:
 
 /* Get the range of bytes or display columns that HINT would affect.  */
 static column_range
-get_affected_range (file_cache &fc,
+get_affected_range (diagnostics::file_cache &fc,
 		    const cpp_char_column_policy &policy,
 		    const fixit_hint *hint, enum column_unit col_unit)
 {
@@ -3184,7 +3191,7 @@ get_affected_range (file_cache &fc,
 /* Get the range of display columns that would be printed for HINT.  */
 
 static column_range
-get_printed_columns (file_cache &fc,
+get_printed_columns (diagnostics::file_cache &fc,
 		     const cpp_char_column_policy &policy,
 		     const fixit_hint *hint)
 {
@@ -3246,7 +3253,7 @@ public:
     m_display_cols = cpp_display_width (m_text, m_byte_length, m_policy);
   }
 
-  void overwrite (int dst_offset, const char_span &src_span)
+  void overwrite (int dst_offset, const diagnostics::char_span &src_span)
   {
     gcc_assert (dst_offset >= 0);
     gcc_assert (dst_offset + src_span.length () < m_alloc_sz);
@@ -3307,7 +3314,7 @@ correction::ensure_terminated ()
 class line_corrections
 {
 public:
-  line_corrections (file_cache &fc,
+  line_corrections (diagnostics::file_cache &fc,
 		    const char_display_policy &policy,
 		    const char *filename,
 		    linenum_type row)
@@ -3318,7 +3325,7 @@ public:
 
   void add_hint (const fixit_hint *hint);
 
-  file_cache &m_file_cache;
+  diagnostics::file_cache &m_file_cache;
   const char_display_policy &m_policy;
   const char *m_filename;
   linenum_type m_row;
@@ -3341,9 +3348,12 @@ line_corrections::~line_corrections ()
 class source_line
 {
 public:
-  source_line (file_cache &fc, const char *filename, int line);
+  source_line (diagnostics::file_cache &fc, const char *filename, int line);
 
-  char_span as_span () { return char_span (chars, width); }
+  diagnostics::char_span as_span ()
+  {
+    return diagnostics::char_span (chars, width);
+  }
 
   const char *chars;
   int width;
@@ -3351,9 +3361,11 @@ public:
 
 /* source_line's ctor.  */
 
-source_line::source_line (file_cache &fc, const char *filename, int line)
+source_line::source_line (diagnostics::file_cache &fc,
+			  const char *filename,
+			  int line)
 {
-  char_span span = fc.get_source_line (filename, line);
+  diagnostics::char_span span = fc.get_source_line (filename, line);
   chars = span.get_buffer ();
   width = span.length ();
 }
@@ -3417,9 +3429,10 @@ line_corrections::add_hint (const fixit_hint *hint)
 		(old_byte_len,
 		 line.as_span ().subspan (between.start - 1,
 					  between.finish + 1 - between.start));
-	      last_correction->overwrite (old_byte_len + between_byte_len,
-					  char_span (hint->get_string (),
-						     hint->get_length ()));
+	      last_correction->overwrite
+		(old_byte_len + between_byte_len,
+		 diagnostics::char_span (hint->get_string (),
+					 hint->get_length ()));
 	      last_correction->m_byte_length = new_byte_len;
 	      last_correction->ensure_terminated ();
 	      last_correction->m_affected_bytes.finish
@@ -3450,11 +3463,11 @@ line_corrections::add_hint (const fixit_hint *hint)
    Fix-it hints that insert new lines are handled separately,
    in layout::print_leading_fixits.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::print_trailing_fixits (linenum_type row)
+layout_printer<TextOrHtml>::print_trailing_fixits (linenum_type row)
 {
-  typename Sink::auto_check_tag_nesting sentinel (m_sink);
+  typename TextOrHtml::auto_check_tag_nesting sentinel (m_text_or_html);
 
   /* Build a list of correction instances for the line,
      potentially consolidating hints (for the sake of readability).  */
@@ -3490,9 +3503,9 @@ layout_printer<Sink>::print_trailing_fixits (linenum_type row)
 	  /* This assumes the insertion just affects one line.  */
 	  int start_column = c->m_printed_columns.start;
 	  move_to_column (&column, start_column, true);
-	  m_sink.colorize_text_for_fixit_insert ();
-	  m_sink.add_text (c->m_text);
-	  m_sink.colorize_text_ensure_normal ();
+	  m_text_or_html.colorize_text_for_fixit_insert ();
+	  m_text_or_html.add_text (c->m_text);
+	  m_text_or_html.colorize_text_ensure_normal ();
 	  column += c->m_display_cols;
 	}
       else
@@ -3508,10 +3521,10 @@ layout_printer<Sink>::print_trailing_fixits (linenum_type row)
 	      || c->m_byte_length == 0)
 	    {
 	      move_to_column (&column, start_column, true);
-	      m_sink.colorize_text_for_fixit_delete ();
+	      m_text_or_html.colorize_text_for_fixit_delete ();
 	      for (; column <= finish_column; column++)
-		m_sink.add_character ('-');
-	      m_sink.colorize_text_ensure_normal ();
+		m_text_or_html.add_character ('-');
+	      m_text_or_html.colorize_text_ensure_normal ();
 	    }
 	  /* Print the replacement text.  REPLACE also covers
 	     removals, so only do this extra work (potentially starting
@@ -3519,9 +3532,9 @@ layout_printer<Sink>::print_trailing_fixits (linenum_type row)
 	  if (c->m_byte_length > 0)
 	    {
 	      move_to_column (&column, start_column, true);
-	      m_sink.colorize_text_for_fixit_insert ();
-	      m_sink.add_text (c->m_text);
-	      m_sink.colorize_text_ensure_normal ();
+	      m_text_or_html.colorize_text_for_fixit_insert ();
+	      m_text_or_html.add_text (c->m_text);
+	      m_text_or_html.colorize_text_ensure_normal ();
 	      column += c->m_display_cols;
 	    }
 	}
@@ -3627,9 +3640,9 @@ layout::get_x_bound_for_row (linenum_type row, int caret_column,
    and updating *COLUMN.  If ADD_LEFT_MARGIN, then print the (empty)
    left margin after any newline.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::move_to_column (int *column,
+layout_printer<TextOrHtml>::move_to_column (int *column,
 				      int dest_column,
 				      bool add_left_margin)
 {
@@ -3646,9 +3659,9 @@ layout_printer<Sink>::move_to_column (int *column,
     {
       /* For debugging column issues, it can be helpful to replace this
 	 add_space call with
-	   m_sink.add_character ('0' + (*column % 10));
+	   m_text_or_html.add_character ('0' + (*column % 10));
 	 to visualize the changing value of "*column".  */
-      m_sink.add_space ();
+      m_text_or_html.add_space ();
       (*column)++;
     }
 }
@@ -3656,11 +3669,11 @@ layout_printer<Sink>::move_to_column (int *column,
 /* For debugging layout issues, render a ruler giving column numbers
    (after the 1-column indent).  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::show_ruler (int max_column)
+layout_printer<TextOrHtml>::show_ruler (int max_column)
 {
-  m_sink.push_html_tag_with_class("thead", "ruler", false);
+  m_text_or_html.push_html_tag_with_class("thead", "ruler", false);
 
   /* Hundreds.  */
   if (max_column > 99)
@@ -3670,9 +3683,9 @@ layout_printer<Sink>::show_ruler (int max_column)
 	   column <= max_column;
 	   ++column)
 	if (column % 10 == 0)
-	  m_sink.add_character ('0' + (column / 100) % 10);
+	  m_text_or_html.add_character ('0' + (column / 100) % 10);
 	else
-	  m_sink.add_space ();
+	  m_text_or_html.add_space ();
       end_line ();
     }
 
@@ -3682,9 +3695,9 @@ layout_printer<Sink>::show_ruler (int max_column)
        column <= max_column;
        ++column)
     if (column % 10 == 0)
-      m_sink.add_character ('0' + (column / 10) % 10);
+      m_text_or_html.add_character ('0' + (column / 10) % 10);
     else
-      m_sink.add_space ();
+      m_text_or_html.add_space ();
   end_line ();
 
   /* Units.  */
@@ -3692,23 +3705,23 @@ layout_printer<Sink>::show_ruler (int max_column)
   for (int column = 1 + m_layout.m_x_offset_display;
        column <= max_column;
        ++column)
-    m_sink.add_character ('0' + (column % 10));
+    m_text_or_html.add_character ('0' + (column % 10));
   end_line ();
 
-  m_sink.pop_html_tag("thead"); // thead
+  m_text_or_html.pop_html_tag("thead"); // thead
 }
 
 /* Print leading fix-its (for new lines inserted before the source line)
    then the source line, followed by an annotation line
    consisting of any caret/underlines, then any fixits.
    If the source line can't be read, print nothing.  */
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::print_line (linenum_type row)
+layout_printer<TextOrHtml>::print_line (linenum_type row)
 {
-  typename Sink::auto_check_tag_nesting sentinel (m_sink);
+  typename TextOrHtml::auto_check_tag_nesting sentinel (m_text_or_html);
 
-  char_span line
+  diagnostics::char_span line
     = m_layout.m_file_cache.get_source_line (m_layout.m_exploc.file, row);
   if (!line)
     return;
@@ -3730,9 +3743,9 @@ layout_printer<Sink>::print_line (linenum_type row)
    showing the link entering at the top right and emerging
    at the bottom left.  */
 
-template<typename Sink>
+template<typename TextOrHtml>
 void
-layout_printer<Sink>::print_any_right_to_left_edge_lines ()
+layout_printer<TextOrHtml>::print_any_right_to_left_edge_lines ()
 {
   if (m_link_rhs_column == -1)
     /* Can also happen if the out-edge had UNKNOWN_LOCATION.  */
@@ -3745,25 +3758,25 @@ layout_printer<Sink>::print_any_right_to_left_edge_lines ()
 
   int column = 1 + m_layout.m_x_offset_display;
   move_to_column (&column, m_link_rhs_column, true);
-  m_sink.colorize_text_for_cfg_edge ();
+  m_text_or_html.colorize_text_for_cfg_edge ();
   const cppchar_t down= get_theme ().get_cppchar
     (text_art::theme::cell_kind::CFG_DOWN);
-  m_sink.add_character (down);
+  m_text_or_html.add_character (down);
   end_line ();
 
   /* Print the line with "┌──────────────────────────────────────────┘".  */
   m_link_lhs_state = link_lhs_state::rewinding_to_lhs;
   start_annotation_line (margin_kind::normal);
-  m_sink.colorize_text_for_cfg_edge ();
+  m_text_or_html.colorize_text_for_cfg_edge ();
   const cppchar_t left= get_theme ().get_cppchar
     (text_art::theme::cell_kind::CFG_LEFT);
   for (int column = 1 + m_layout.m_x_offset_display;
        column < m_link_rhs_column;
        ++column)
-    m_sink.add_character (left);
+    m_text_or_html.add_character (left);
   const cppchar_t from_down_to_left = get_theme ().get_cppchar
     (text_art::theme::cell_kind::CFG_FROM_DOWN_TO_LEFT);
-  m_sink.add_character (from_down_to_left);
+  m_text_or_html.add_character (from_down_to_left);
   end_line ();
 
   /* We now have a link line on the LHS,
@@ -3772,11 +3785,11 @@ layout_printer<Sink>::print_any_right_to_left_edge_lines ()
   m_link_rhs_column = -1;
 }
 
-template<typename Sink>
-layout_printer<Sink>::layout_printer (Sink &sink,
+template<typename TextOrHtml>
+layout_printer<TextOrHtml>::layout_printer (TextOrHtml &text_or_html,
 				      const layout &layout,
 				      bool is_diagnostic_path)
-: m_sink (sink),
+: m_text_or_html (text_or_html),
   m_layout (layout),
   m_is_diagnostic_path (is_diagnostic_path),
   m_was_in_range_p (false),
@@ -3801,7 +3814,7 @@ layout_printer<Sink>::layout_printer (Sink &sink,
 
 bool
 gcc_rich_location::
-add_location_if_nearby (const diagnostic_source_print_policy &policy,
+add_location_if_nearby (const diagnostics::source_print_policy &policy,
 			location_t loc,
 			bool restrict_to_current_line_spans,
 			const range_label *label)
@@ -3825,27 +3838,29 @@ add_location_if_nearby (const diagnostic_source_print_policy &policy,
 
 bool
 gcc_rich_location::
-add_location_if_nearby (const diagnostic_context &dc,
+add_location_if_nearby (const diagnostics::context &dc,
 			location_t loc,
 			bool restrict_to_current_line_spans,
 			const range_label *label)
 {
-  diagnostic_source_print_policy source_policy (dc,
-						dc.m_source_printing);
+  diagnostics::source_print_policy
+    source_policy (dc,
+		   dc.get_source_printing_options ());
   return add_location_if_nearby (source_policy, loc,
 				 restrict_to_current_line_spans, label);
 }
 
+namespace diagnostics {
 
-/* As per diagnostic_source_print_policy::print, but don't print anything
+/* As per diagnostics::source_print_policy::print, but don't print anything
    if source printing is disabled, or if the location hasn't changed.  */
 
 void
-diagnostic_context::maybe_show_locus (const rich_location &richloc,
-				      const diagnostic_source_printing_options &opts,
-				      diagnostic_t diagnostic_kind,
-				      pretty_printer &pp,
-				      diagnostic_source_effect_info *effects)
+context::maybe_show_locus (const rich_location &richloc,
+			   const source_printing_options &opts,
+			   enum kind diagnostic_kind,
+			   pretty_printer &pp,
+			   source_effect_info *effects)
 {
   const location_t loc = richloc.get_loc ();
   /* Do nothing if source-printing has been disabled.  */
@@ -3866,7 +3881,7 @@ diagnostic_context::maybe_show_locus (const rich_location &richloc,
 
   m_last_location = loc;
 
-  diagnostic_source_print_policy source_policy (*this, opts);
+  source_print_policy source_policy (*this, opts);
   source_policy.print (pp, richloc, diagnostic_kind, effects);
 }
 
@@ -3874,12 +3889,12 @@ diagnostic_context::maybe_show_locus (const rich_location &richloc,
    If non-null, use LABEL_WRITER when writing labelled ranges.  */
 
 void
-diagnostic_context::maybe_show_locus_as_html (const rich_location &richloc,
-					      const diagnostic_source_printing_options &opts,
-					      diagnostic_t diagnostic_kind,
-					      xml::printer &xp,
-					      diagnostic_source_effect_info *effects,
-					      html_label_writer *label_writer)
+context::maybe_show_locus_as_html (const rich_location &richloc,
+				   const source_printing_options &opts,
+				   enum kind diagnostic_kind,
+				   xml::printer &xp,
+				   source_effect_info *effects,
+				   html_label_writer *label_writer)
 {
   const location_t loc = richloc.get_loc ();
   /* Do nothing if source-printing has been disabled.  */
@@ -3900,13 +3915,13 @@ diagnostic_context::maybe_show_locus_as_html (const rich_location &richloc,
 
   m_last_location = loc;
 
-  diagnostic_source_print_policy source_policy (*this, opts);
+  source_print_policy source_policy (*this, opts);
   source_policy.print_as_html (xp, richloc, diagnostic_kind, effects,
 			       label_writer);
 }
 
-diagnostic_source_print_policy::
-diagnostic_source_print_policy (const diagnostic_context &dc)
+source_print_policy::
+source_print_policy (const context &dc)
 : m_options (dc.m_source_printing),
   m_location_policy (dc),
   m_text_start_span_cb (dc.m_text_callbacks.m_text_start_span),
@@ -3917,9 +3932,8 @@ diagnostic_source_print_policy (const diagnostic_context &dc)
 {
 }
 
-diagnostic_source_print_policy::
-diagnostic_source_print_policy (const diagnostic_context &dc,
-				const diagnostic_source_printing_options &opts)
+source_print_policy::source_print_policy (const context &dc,
+					  const source_printing_options &opts)
 : m_options (opts),
   m_location_policy (dc),
   m_text_start_span_cb (dc.m_text_callbacks.m_text_start_span),
@@ -3936,17 +3950,16 @@ diagnostic_source_print_policy (const diagnostic_context &dc,
    If EFFECTS is non-null, then use and update it.  */
 
 void
-diagnostic_source_print_policy::print (pretty_printer &pp,
-				       const rich_location &richloc,
-				       diagnostic_t diagnostic_kind,
-				       diagnostic_source_effect_info *effects)
-  const
+source_print_policy::print (pretty_printer &pp,
+			    const rich_location &richloc,
+			    enum kind diagnostic_kind,
+			    source_effect_info *effects) const
 {
   layout layout (*this, richloc, effects);
   colorizer col (pp, richloc, diagnostic_kind);
-  to_text sink (pp, col);
-  layout_printer<to_text> lp (sink, layout,
-			      diagnostic_kind == DK_DIAGNOSTIC_PATH);
+  to_text text_or_html (pp, col);
+  layout_printer<to_text> lp (text_or_html, layout,
+			      diagnostic_kind == diagnostics::kind::path);
   lp.print (*this);
 }
 
@@ -3954,28 +3967,30 @@ diagnostic_source_print_policy::print (pretty_printer &pp,
    If non-null, use LABEL_WRITER when writing labelled ranges.  */
 
 void
-diagnostic_source_print_policy::print_as_html (xml::printer &xp,
-					       const rich_location &richloc,
-					       diagnostic_t diagnostic_kind,
-					       diagnostic_source_effect_info *effects,
-					       html_label_writer *label_writer)
-  const
+source_print_policy::print_as_html (xml::printer &xp,
+				    const rich_location &richloc,
+				    enum kind diagnostic_kind,
+				    source_effect_info *effects,
+				    html_label_writer *label_writer) const
 {
   layout layout (*this, richloc, effects);
-  to_html sink (xp, &richloc, label_writer);
-  layout_printer<to_html> lp (sink, layout,
-			      diagnostic_kind == DK_DIAGNOSTIC_PATH);
+  to_html text_or_html (xp, &richloc, label_writer);
+  layout_printer<to_html> lp (text_or_html, layout,
+			      diagnostic_kind == diagnostics::kind::path);
   xml::auto_check_tag_nesting sentinel (xp);
   lp.print (*this);
 }
 
-template <typename Sink>
+} // namespace diagnostics
+
+template <typename TextOrHtml>
 void
-layout_printer<Sink>::print (const diagnostic_source_print_policy &source_policy)
+layout_printer<TextOrHtml>::
+print (const diagnostics::source_print_policy &source_policy)
 {
-  typename Sink::auto_check_tag_nesting sentinel (m_sink);
+  typename TextOrHtml::auto_check_tag_nesting sentinel (m_text_or_html);
 
-  m_sink.push_html_tag_with_class ("table", "locus", false);
+  m_text_or_html.push_html_tag_with_class ("table", "locus", false);
 
   if (get_options ().show_ruler_p)
     show_ruler (m_layout.m_x_offset_display + get_options ().max_width);
@@ -3998,13 +4013,13 @@ layout_printer<Sink>::print (const diagnostic_source_print_policy &source_policy
 	    {
 	      expanded_location exploc
 		= m_layout.get_expanded_location (line_span);
-	      const diagnostic_location_print_policy &
+	      const diagnostics::location_print_policy &
 		loc_policy = source_policy.get_location_policy ();
-	      m_sink.invoke_start_span_fn (source_policy, loc_policy, exploc);
+	      m_text_or_html.invoke_start_span_fn (source_policy, loc_policy, exploc);
 	    }
 	}
 
-      m_sink.push_html_tag_with_class ("tbody", "line-span", false);
+      m_text_or_html.push_html_tag_with_class ("tbody", "line-span", false);
 
       /* Iterate over the lines within this span (using linenum_arith_t to
 	 avoid overflow with 0xffffffff causing an infinite loop).  */
@@ -4013,30 +4028,37 @@ layout_printer<Sink>::print (const diagnostic_source_print_policy &source_policy
 	   row <= last_line; row++)
 	print_line (row);
 
-      m_sink.pop_html_tag ("tbody");
+      m_text_or_html.pop_html_tag ("tbody");
     }
 
   if (auto effect_info = m_layout.m_effect_info)
     effect_info->m_trailing_out_edge_column = m_link_rhs_column;
 
-  m_sink.pop_html_tag ("table");
+  m_text_or_html.pop_html_tag ("table");
 }
 
 #if CHECKING_P
 
+namespace diagnostics {
 namespace selftest {
 
+using test_context = diagnostics::selftest::test_context;
+
+using line_table_case = ::selftest::line_table_case;
+using line_table_test = ::selftest::line_table_test;
+using temp_source_file = ::selftest::temp_source_file;
+
 static std::unique_ptr<xml::node>
 make_element_for_locus (const rich_location &rich_loc,
-			diagnostic_t kind,
-			diagnostic_context &dc)
+			enum kind kind,
+			diagnostics::context &dc)
 {
   dc.m_last_location = UNKNOWN_LOCATION;
 
   xml::element wrapper ("wrapper", false);
   xml::printer xp (wrapper);
   dc.maybe_show_locus_as_html (rich_loc,
-			       dc.m_source_printing,
+			       dc.get_source_printing_options (),
 			       kind,
 			       xp,
 			       nullptr,
@@ -4049,8 +4071,8 @@ make_element_for_locus (const rich_location &rich_loc,
 
 static label_text
 make_raw_html_for_locus (const rich_location &rich_loc,
-			 diagnostic_t kind,
-			 diagnostic_context &dc)
+			 enum kind kind,
+			 diagnostics::context &dc)
 {
   auto node = make_element_for_locus (rich_loc, kind, dc);
   pretty_printer pp;
@@ -4061,9 +4083,9 @@ make_raw_html_for_locus (const rich_location &rich_loc,
 
 /* Selftests for diagnostic_show_locus.  */
 
-diagnostic_show_locus_fixture::
-diagnostic_show_locus_fixture (const line_table_case &case_,
-			       const char *content)
+source_printing_fixture::
+source_printing_fixture (const line_table_case &case_,
+			 const char *content)
 : m_content (content),
   m_tmp_source_file (SELFTEST_LOCATION, ".c", content),
   m_ltt (case_),
@@ -4076,10 +4098,10 @@ diagnostic_show_locus_fixture (const line_table_case &case_,
 /* Populate a char_display_policy based on DC and RICHLOC.  */
 
 static char_display_policy
-make_char_policy (const diagnostic_context &dc,
+make_char_policy (const diagnostics::context &dc,
 		  const rich_location &richloc)
 {
-  diagnostic_source_print_policy source_policy (dc);
+  diagnostics::source_print_policy source_policy (dc);
   return ::make_char_policy (source_policy, richloc);
 }
 
@@ -4101,7 +4123,7 @@ test_display_widths ()
 
   /* No escaping.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     char_display_policy policy (make_char_policy (dc, richloc));
     ASSERT_EQ (cpp_display_width (pi, strlen (pi), policy), 1);
     ASSERT_EQ (cpp_display_width (emoji, strlen (emoji), policy), 2);
@@ -4112,7 +4134,7 @@ test_display_widths ()
   richloc.set_escape_on_output (true);
 
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_UNICODE);
     char_display_policy policy (make_char_policy (dc, richloc));
     ASSERT_EQ (cpp_display_width (pi, strlen (pi), policy), 8);
@@ -4124,7 +4146,7 @@ test_display_widths ()
   }
 
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_BYTES);
     char_display_policy policy (make_char_policy (dc, richloc));
     ASSERT_EQ (cpp_display_width (pi, strlen (pi), policy), 8);
@@ -4149,13 +4171,14 @@ test_offset_impl (int caret_byte_col, int max_width,
 		  int expected_x_offset_display,
 		  int left_margin = test_left_margin)
 {
-  test_diagnostic_context dc;
-  dc.m_source_printing.max_width = max_width;
-  /* diagnostic_context::min_margin_width sets the minimum space reserved for
+  test_context dc;
+  auto &source_printing_opts = dc.get_source_printing_options ();
+  source_printing_opts.max_width = max_width;
+  /* min_margin_width sets the minimum space reserved for
      the line number plus one space after.  */
-  dc.m_source_printing.min_margin_width = left_margin - test_linenum_sep + 1;
-  dc.m_source_printing.show_line_numbers_p = true;
-  diagnostic_source_print_policy source_policy (dc);
+  source_printing_opts.min_margin_width = left_margin - test_linenum_sep + 1;
+  dc.show_line_numbers (true);
+  diagnostics::source_print_policy source_policy (dc);
   rich_location richloc (line_table,
 			 linemap_position_for_column (line_table,
 						      caret_byte_col));
@@ -4188,7 +4211,7 @@ test_layout_x_offset_display_utf8 (const line_table_case &case_)
      no multibyte characters earlier on the line.  */
   const int emoji_col = 102;
 
-  diagnostic_show_locus_fixture f (case_, content);
+  source_printing_fixture f (case_, content);
 
   linemap_add (line_table, LC_ENTER, false, f.get_filename (), 1);
 
@@ -4202,7 +4225,7 @@ test_layout_x_offset_display_utf8 (const line_table_case &case_)
   ASSERT_EQ (1, LOCATION_LINE (line_end));
   ASSERT_EQ (line_bytes, LOCATION_COLUMN (line_end));
 
-  char_span lspan = f.m_fc.get_source_line (f.get_filename (), 1);
+  diagnostics::char_span lspan = f.m_fc.get_source_line (f.get_filename (), 1);
   ASSERT_EQ (line_display_cols,
 	     cpp_display_width (lspan.get_buffer (), lspan.length (),
 				def_policy ()));
@@ -4265,21 +4288,22 @@ test_layout_x_offset_display_utf8 (const line_table_case &case_)
 
   /* Test that the source line is offset as expected when printed.  */
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.max_width = small_width - 6;
-    dc.m_source_printing.min_margin_width
+    test_context dc;
+    auto &source_printing_opts = dc.get_source_printing_options ();
+    source_printing_opts.max_width = small_width - 6;
+    source_printing_opts.min_margin_width
       = test_left_margin - test_linenum_sep + 1;
-    dc.m_source_printing.show_line_numbers_p = true;
-    dc.m_source_printing.show_ruler_p = true;
-    diagnostic_source_print_policy policy (dc);
+    dc.show_line_numbers (true);
+    dc.show_ruler (true);
+    diagnostics::source_print_policy policy (dc);
     rich_location richloc (line_table,
 			   linemap_position_for_column (line_table,
 							emoji_col));
     layout test_layout (policy, richloc, nullptr);
     colorizer col (*dc.get_reference_printer (),
-		   richloc, DK_ERROR);
-    to_text sink (*dc.get_reference_printer (), col);
-    layout_printer<to_text> lp (sink, test_layout, false);
+		   richloc, diagnostics::kind::error);
+    diagnostics::to_text text_or_html (*dc.get_reference_printer (), col);
+    layout_printer<diagnostics::to_text> lp (text_or_html, test_layout, false);
     lp.print (policy);
     ASSERT_STREQ ("     |         1         \n"
 		  "     |         1         \n"
@@ -4295,21 +4319,22 @@ test_layout_x_offset_display_utf8 (const line_table_case &case_)
      the first emoji in the middle of the UTF-8 sequence.  Check that we replace
      it with a padding space in this case.  */
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.max_width = small_width - 5;
-    dc.m_source_printing.min_margin_width
+    test_context dc;
+    auto &source_printing_opts = dc.get_source_printing_options ();
+    source_printing_opts.max_width = small_width - 5;
+    source_printing_opts.min_margin_width
       = test_left_margin - test_linenum_sep + 1;
-    dc.m_source_printing.show_line_numbers_p = true;
-    dc.m_source_printing.show_ruler_p = true;
-    diagnostic_source_print_policy policy (dc);
+    dc.show_line_numbers (true);
+    dc.show_ruler (true);
+    diagnostics::source_print_policy policy (dc);
     rich_location richloc (line_table,
 			   linemap_position_for_column (line_table,
 							emoji_col + 2));
     layout test_layout (dc, richloc, nullptr);
     colorizer col (*dc.get_reference_printer (),
-		   richloc, DK_ERROR);
-    to_text sink (*dc.get_reference_printer (), col);
-    layout_printer<to_text> lp (sink, test_layout, false);
+		   richloc, diagnostics::kind::error);
+    diagnostics::to_text text_or_html (*dc.get_reference_printer (), col);
+    layout_printer<diagnostics::to_text> lp (text_or_html, test_layout, false);
     lp.print (policy);
     ASSERT_STREQ ("     |        1         1 \n"
 		  "     |        1         2 \n"
@@ -4354,7 +4379,7 @@ test_layout_x_offset_display_tab (const line_table_case &case_)
   ASSERT_EQ (7, extra_width[10]);
 
   temp_source_file tmp (SELFTEST_LOCATION, ".c", content);
-  file_cache fc;
+  diagnostics::file_cache fc;
   line_table_test ltt (case_);
 
   linemap_add (line_table, LC_ENTER, false, tmp.get_filename (), 1);
@@ -4366,7 +4391,7 @@ test_layout_x_offset_display_tab (const line_table_case &case_)
     return;
 
   /* Check that cpp_display_width handles the tabs as expected.  */
-  char_span lspan = fc.get_source_line (tmp.get_filename (), 1);
+  diagnostics::char_span lspan = fc.get_source_line (tmp.get_filename (), 1);
   ASSERT_EQ ('\t', *(lspan.get_buffer () + (tab_col - 1)));
   for (int tabstop = 1; tabstop != num_tabstops; ++tabstop)
     {
@@ -4386,14 +4411,15 @@ test_layout_x_offset_display_tab (const line_table_case &case_)
 						      tab_col + 1));
   for (int tabstop = 1; tabstop != num_tabstops; ++tabstop)
     {
-      test_diagnostic_context dc;
+      test_context dc;
       dc.m_tabstop = tabstop;
-      diagnostic_source_print_policy policy (dc);
+      diagnostics::source_print_policy policy (dc);
       layout test_layout (policy, richloc, nullptr);
       colorizer col (*dc.get_reference_printer (),
-		     richloc, DK_ERROR);
-      to_text sink (*dc.get_reference_printer (), col);
-      layout_printer<to_text> lp (sink, test_layout, false);
+		     richloc, diagnostics::kind::error);
+      diagnostics::to_text text_or_html (*dc.get_reference_printer (), col);
+      layout_printer<diagnostics::to_text> lp
+	(text_or_html, test_layout, false);
       lp.print (policy);
       const char *out = pp_formatted_text (dc.get_reference_printer ());
       ASSERT_EQ (nullptr, strchr (out, '\t'));
@@ -4409,19 +4435,21 @@ test_layout_x_offset_display_tab (const line_table_case &case_)
      over.  */
   for (int tabstop = 1; tabstop != num_tabstops; ++tabstop)
     {
-      test_diagnostic_context dc;
+      test_context dc;
       dc.m_tabstop = tabstop;
       static const int small_width = 24;
-      dc.m_source_printing.max_width = small_width - 4;
-      dc.m_source_printing.min_margin_width
+      auto &source_printing_opts = dc.get_source_printing_options ();
+      source_printing_opts.max_width = small_width - 4;
+      source_printing_opts.min_margin_width
 	= test_left_margin - test_linenum_sep + 1;
-      dc.m_source_printing.show_line_numbers_p = true;
-      diagnostic_source_print_policy policy (dc);
+      dc.show_line_numbers (true);
+      diagnostics::source_print_policy policy (dc);
       layout test_layout (policy, richloc, nullptr);
       colorizer col (*dc.get_reference_printer (),
-		     richloc, DK_ERROR);
-      to_text sink (*dc.get_reference_printer (), col);
-      layout_printer<to_text> lp (sink, test_layout, false);
+		     richloc, diagnostics::kind::error);
+      diagnostics::to_text text_or_html (*dc.get_reference_printer (), col);
+      layout_printer<diagnostics::to_text> lp
+	(text_or_html, test_layout, false);
       lp.print (policy);
 
       /* We have arranged things so that two columns will be printed before
@@ -4447,7 +4475,7 @@ test_layout_x_offset_display_tab (const line_table_case &case_)
 static void
 test_diagnostic_show_locus_unknown_location ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   rich_location richloc (line_table, UNKNOWN_LOCATION);
   ASSERT_STREQ ("", dc.test_show_locus (richloc));
 }
@@ -4467,7 +4495,7 @@ test_diagnostic_show_locus_unknown_location ()
 static void
 test_one_liner_simple_caret ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 10);
   rich_location richloc (line_table, caret);
   ASSERT_STREQ (" foo = bar.field;\n"
@@ -4481,7 +4509,7 @@ test_one_liner_simple_caret ()
 static void
 test_one_liner_no_column ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 0);
   rich_location richloc (line_table, caret);
   ASSERT_STREQ (" foo = bar.field;\n",
@@ -4493,7 +4521,7 @@ test_one_liner_no_column ()
 static void
 test_one_liner_caret_and_range ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 10);
   location_t start = linemap_position_for_column (line_table, 7);
   location_t finish = linemap_position_for_column (line_table, 15);
@@ -4504,8 +4532,8 @@ test_one_liner_caret_and_range ()
 		dc.test_show_locus (richloc));
 
   {
-    test_diagnostic_context dc;
-    auto out = make_raw_html_for_locus (richloc, DK_ERROR, dc);
+    test_context dc;
+    auto out = make_raw_html_for_locus (richloc, diagnostics::kind::error, dc);
      ASSERT_STREQ
        ("<table class=\"locus\">\n"
 	"  <tbody class=\"line-span\">\n"
@@ -4516,9 +4544,9 @@ test_one_liner_caret_and_range ()
 	out.get ());
   }
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.show_line_numbers_p = true;
-    auto out = make_raw_html_for_locus (richloc, DK_ERROR, dc);
+    test_context dc;
+    dc.show_line_numbers (true);
+    auto out = make_raw_html_for_locus (richloc, diagnostics::kind::error, dc);
      ASSERT_STREQ
        ("<table class=\"locus\">\n"
 	"  <tbody class=\"line-span\">\n"
@@ -4535,24 +4563,24 @@ test_one_liner_caret_and_range ()
 static void
 test_one_liner_multiple_carets_and_ranges ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t foo
     = make_location (linemap_position_for_column (line_table, 2),
 		     linemap_position_for_column (line_table, 1),
 		     linemap_position_for_column (line_table, 3));
-  dc.m_source_printing.caret_chars[0] = 'A';
+  dc.set_caret_char (0, 'A');
 
   location_t bar
     = make_location (linemap_position_for_column (line_table, 8),
 		     linemap_position_for_column (line_table, 7),
 		     linemap_position_for_column (line_table, 9));
-  dc.m_source_printing.caret_chars[1] = 'B';
+  dc.set_caret_char (1, 'B');
 
   location_t field
     = make_location (linemap_position_for_column (line_table, 13),
 		     linemap_position_for_column (line_table, 11),
 		     linemap_position_for_column (line_table, 15));
-  dc.m_source_printing.caret_chars[2] = 'C';
+  dc.set_caret_char (2, 'C');
 
   rich_location richloc (line_table, foo);
   richloc.add_range (bar, SHOW_RANGE_WITH_CARET);
@@ -4567,7 +4595,7 @@ test_one_liner_multiple_carets_and_ranges ()
 static void
 test_one_liner_fixit_insert_before ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 7);
   rich_location richloc (line_table, caret);
   richloc.add_fixit_insert_before ("&");
@@ -4582,7 +4610,7 @@ test_one_liner_fixit_insert_before ()
 static void
 test_one_liner_fixit_insert_after ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t start = linemap_position_for_column (line_table, 1);
   location_t finish = linemap_position_for_column (line_table, 3);
   location_t foo = make_location (start, start, finish);
@@ -4609,7 +4637,7 @@ test_one_liner_fixit_remove ()
 
   /* Normal.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  "          ^~~~~~\n"
 		  "          ------\n",
@@ -4618,7 +4646,7 @@ test_one_liner_fixit_remove ()
 
   /* Test of adding a prefix.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     pp_prefixing_rule (dc.get_reference_printer ()) = DIAGNOSTICS_SHOW_PREFIX_EVERY_LINE;
     pp_set_prefix (dc.get_reference_printer (), xstrdup ("TEST PREFIX:"));
     ASSERT_STREQ ("TEST PREFIX: foo = bar.field;\n"
@@ -4629,9 +4657,10 @@ test_one_liner_fixit_remove ()
 
   /* Normal, with ruler.  */
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.show_ruler_p = true;
-    dc.m_source_printing.max_width = 104;
+    test_context dc;
+    auto &source_printing_opts = dc.get_source_printing_options ();
+    dc.show_ruler (true);
+    source_printing_opts.max_width = 104;
     ASSERT_STREQ ("          0         0         0         0         0         0         0         0         0         1    \n"
 		  "          1         2         3         4         5         6         7         8         9         0    \n"
 		  " 12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234\n"
@@ -4643,9 +4672,10 @@ test_one_liner_fixit_remove ()
 
   /* Test of adding a prefix, with ruler.  */
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.show_ruler_p = true;
-    dc.m_source_printing.max_width = 50;
+    test_context dc;
+    auto &source_printing_opts = dc.get_source_printing_options ();
+    dc.show_ruler (true);
+    source_printing_opts.max_width = 50;
     pp_prefixing_rule (dc.get_reference_printer ()) = DIAGNOSTICS_SHOW_PREFIX_EVERY_LINE;
     pp_set_prefix (dc.get_reference_printer (), xstrdup ("TEST PREFIX:"));
     ASSERT_STREQ ("TEST PREFIX:          1         2         3         4         5\n"
@@ -4658,10 +4688,11 @@ test_one_liner_fixit_remove ()
 
   /* Test of adding a prefix, with ruler and line numbers.  */
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.show_ruler_p = true;
-    dc.m_source_printing.max_width = 50;
-    dc.m_source_printing.show_line_numbers_p = true;
+    test_context dc;
+    auto &source_printing_opts = dc.get_source_printing_options ();
+    dc.show_ruler (true);
+    source_printing_opts.max_width = 50;
+    dc.show_line_numbers (true);
     pp_prefixing_rule (dc.get_reference_printer ()) = DIAGNOSTICS_SHOW_PREFIX_EVERY_LINE;
     pp_set_prefix (dc.get_reference_printer (), xstrdup ("TEST PREFIX:"));
     ASSERT_STREQ ("TEST PREFIX:      |          1         2         3         4         5\n"
@@ -4678,7 +4709,7 @@ test_one_liner_fixit_remove ()
 static void
 test_one_liner_fixit_replace ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t start = linemap_position_for_column (line_table, 11);
   location_t finish = linemap_position_for_column (line_table, 15);
   location_t field = make_location (start, start, finish);
@@ -4696,7 +4727,7 @@ test_one_liner_fixit_replace ()
 static void
 test_one_liner_fixit_replace_non_equal_range ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 5);
   location_t start = linemap_position_for_column (line_table, 11);
   location_t finish = linemap_position_for_column (line_table, 15);
@@ -4721,7 +4752,7 @@ test_one_liner_fixit_replace_non_equal_range ()
 static void
 test_one_liner_fixit_replace_equal_secondary_range ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 5);
   location_t start = linemap_position_for_column (line_table, 11);
   location_t finish = linemap_position_for_column (line_table, 15);
@@ -4762,7 +4793,7 @@ test_one_liner_fixit_validation_adhoc_locations ()
     /* It should not have been discarded by the validator.  */
     ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  "       ^~~~~~~~~~                               "
 		  "          "
@@ -4788,7 +4819,7 @@ test_one_liner_fixit_validation_adhoc_locations ()
     /* It should not have been discarded by the validator.  */
     ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  "       ^~~~~~~~~~                               "
 		  "          "
@@ -4825,7 +4856,7 @@ test_one_liner_fixit_validation_adhoc_locations ()
     /* It should not have been discarded by the validator.  */
     ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  "       ^~~~~~~~~~                               "
 		  "          "
@@ -4849,7 +4880,7 @@ test_one_liner_fixit_validation_adhoc_locations ()
 static void
 test_one_liner_many_fixits_1 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 5);
   rich_location richloc (line_table, equals);
   for (int i = 0; i < 19; i++)
@@ -4867,7 +4898,7 @@ test_one_liner_many_fixits_1 ()
 static void
 test_one_liner_many_fixits_2 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 5);
   rich_location richloc (line_table, equals);
   for (int i = 0; i < 19; i++)
@@ -4910,7 +4941,7 @@ test_one_liner_labels ()
     richloc.add_range (field, SHOW_RANGE_WITHOUT_CARET, &label2);
 
     {
-      test_diagnostic_context dc;
+      test_context dc;
       ASSERT_STREQ (" foo = bar.field;\n"
 		    " ^~~   ~~~ ~~~~~\n"
 		    " |     |   |\n"
@@ -4920,8 +4951,8 @@ test_one_liner_labels ()
 
     /* Verify that we can disable label-printing.  */
     {
-      test_diagnostic_context dc;
-      dc.m_source_printing.show_labels_p = false;
+      test_context dc;
+      dc.show_labels (false);
       ASSERT_STREQ (" foo = bar.field;\n"
 		    " ^~~   ~~~ ~~~~~\n",
 		    dc.test_show_locus (richloc));
@@ -4937,7 +4968,7 @@ test_one_liner_labels ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label1);
     richloc.add_range (field, SHOW_RANGE_WITHOUT_CARET, &label2);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  " ^~~   ~~~ ~~~~~\n"
 		  " |     |   |\n"
@@ -4947,9 +4978,10 @@ test_one_liner_labels ()
 		  dc.test_show_locus (richloc));
 
     {
-      test_diagnostic_context dc;
-      dc.m_source_printing.show_line_numbers_p = true;
-      auto out = make_raw_html_for_locus (richloc, DK_ERROR, dc);
+      test_context dc;
+      dc.show_line_numbers (true);
+      auto out
+	= make_raw_html_for_locus (richloc, diagnostics::kind::error, dc);
       ASSERT_STREQ
 	("<table class=\"locus\">\n"
 	 "  <tbody class=\"line-span\">\n"
@@ -4975,7 +5007,7 @@ test_one_liner_labels ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label1);
     richloc.add_range (field, SHOW_RANGE_WITHOUT_CARET, &label2);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  " ^~~   ~~~ ~~~~~\n"
 		  " |     |   |\n"
@@ -4993,7 +5025,7 @@ test_one_liner_labels ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label1);
     richloc.add_range (foo, SHOW_RANGE_WITHOUT_CARET, &label2);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  " ~~~   ~~~ ^~~~~\n"
 		  " |     |   |\n"
@@ -5011,7 +5043,7 @@ test_one_liner_labels ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label1);
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label2);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  "       ^~~\n"
 		  "       |\n"
@@ -5045,7 +5077,7 @@ test_one_liner_labels ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label_1c);
     richloc.add_range (foo, SHOW_RANGE_WITHOUT_CARET, &label_2c);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  " ~~~   ~~~ ^~~~~\n"
 		  " |     |   |\n"
@@ -5067,7 +5099,7 @@ test_one_liner_labels ()
     text_range_label label (nullptr);
     gcc_rich_location richloc (bar, &label, nullptr);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" foo = bar.field;\n"
 		  "       ^~~\n",
 		  dc.test_show_locus (richloc));
@@ -5087,7 +5119,7 @@ test_diagnostic_show_locus_one_liner (const line_table_case &case_)
      ....................1234567890123456.  */
   const char *content = "foo = bar.field;\n";
 
-  diagnostic_show_locus_fixture f (case_, content);
+  source_printing_fixture f (case_, content);
 
   location_t line_end = linemap_position_for_column (line_table, 16);
 
@@ -5117,7 +5149,7 @@ test_diagnostic_show_locus_one_liner (const line_table_case &case_)
 
 /* Version of all one-liner tests exercising multibyte awareness.
    These are all called from test_diagnostic_show_locus_one_liner,
-   which uses diagnostic_show_locus_fixture_one_liner_utf8 to create
+   which uses source_printing_fixture_one_liner_utf8 to create
    the test file; see the notes in diagnostic-show-locus-selftest.h.
 
    Note: all of the below asserts would be easier to read if we used UTF-8
@@ -5132,7 +5164,7 @@ test_diagnostic_show_locus_one_liner (const line_table_case &case_)
 static void
 test_one_liner_simple_caret_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 18);
   rich_location richloc (line_table, caret);
   ASSERT_STREQ (" \xf0\x9f\x98\x82"
@@ -5148,7 +5180,7 @@ test_one_liner_simple_caret_utf8 ()
 static void
 test_one_liner_caret_and_range_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 18);
   location_t start = linemap_position_for_column (line_table, 12);
   location_t finish = linemap_position_for_column (line_table, 30);
@@ -5168,24 +5200,24 @@ test_one_liner_caret_and_range_utf8 ()
 static void
 test_one_liner_multiple_carets_and_ranges_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t foo
     = make_location (linemap_position_for_column (line_table, 7),
 		     linemap_position_for_column (line_table, 1),
 		     linemap_position_for_column (line_table, 8));
-  dc.m_source_printing.caret_chars[0] = 'A';
+  dc.set_caret_char (0, 'A');
 
   location_t bar
     = make_location (linemap_position_for_column (line_table, 16),
 		     linemap_position_for_column (line_table, 12),
 		     linemap_position_for_column (line_table, 17));
-  dc.m_source_printing.caret_chars[1] = 'B';
+  dc.set_caret_char (1, 'B');
 
   location_t field
     = make_location (linemap_position_for_column (line_table, 26),
 		     linemap_position_for_column (line_table, 19),
 		     linemap_position_for_column (line_table, 30));
-  dc.m_source_printing.caret_chars[2] = 'C';
+  dc.set_caret_char (2, 'C');
   rich_location richloc (line_table, foo);
   richloc.add_range (bar, SHOW_RANGE_WITH_CARET);
   richloc.add_range (field, SHOW_RANGE_WITH_CARET);
@@ -5203,7 +5235,7 @@ test_one_liner_multiple_carets_and_ranges_utf8 ()
 static void
 test_one_liner_fixit_insert_before_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t caret = linemap_position_for_column (line_table, 12);
   rich_location richloc (line_table, caret);
   richloc.add_fixit_insert_before ("&");
@@ -5222,7 +5254,7 @@ test_one_liner_fixit_insert_before_utf8 ()
 static void
 test_one_liner_fixit_insert_after_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t start = linemap_position_for_column (line_table, 1);
   location_t finish = linemap_position_for_column (line_table, 8);
   location_t foo = make_location (start, start, finish);
@@ -5243,7 +5275,7 @@ test_one_liner_fixit_insert_after_utf8 ()
 static void
 test_one_liner_fixit_remove_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t start = linemap_position_for_column (line_table, 18);
   location_t finish = linemap_position_for_column (line_table, 30);
   location_t dot = make_location (start, start, finish);
@@ -5264,7 +5296,7 @@ test_one_liner_fixit_remove_utf8 ()
 static void
 test_one_liner_fixit_replace_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t start = linemap_position_for_column (line_table, 19);
   location_t finish = linemap_position_for_column (line_table, 30);
   location_t field = make_location (start, start, finish);
@@ -5287,7 +5319,7 @@ test_one_liner_fixit_replace_utf8 ()
 static void
 test_one_liner_fixit_replace_non_equal_range_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 10);
   location_t start = linemap_position_for_column (line_table, 19);
   location_t finish = linemap_position_for_column (line_table, 30);
@@ -5317,7 +5349,7 @@ test_one_liner_fixit_replace_non_equal_range_utf8 ()
 static void
 test_one_liner_fixit_replace_equal_secondary_range_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 10);
   location_t start = linemap_position_for_column (line_table, 19);
   location_t finish = linemap_position_for_column (line_table, 30);
@@ -5363,7 +5395,7 @@ test_one_liner_fixit_validation_adhoc_locations_utf8 ()
     /* It should not have been discarded by the validator.  */
     ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" \xf0\x9f\x98\x82"
 		     "_foo = \xcf\x80"
 			     "_bar.\xf0\x9f\x98\x82"
@@ -5393,7 +5425,7 @@ test_one_liner_fixit_validation_adhoc_locations_utf8 ()
     /* It should not have been discarded by the validator.  */
     ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" \xf0\x9f\x98\x82"
 		     "_foo = \xcf\x80"
 			     "_bar.\xf0\x9f\x98\x82"
@@ -5434,7 +5466,7 @@ test_one_liner_fixit_validation_adhoc_locations_utf8 ()
     /* It should not have been discarded by the validator.  */
     ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" \xf0\x9f\x98\x82"
 		     "_foo = \xcf\x80"
 			     "_bar.\xf0\x9f\x98\x82"
@@ -5462,7 +5494,7 @@ test_one_liner_fixit_validation_adhoc_locations_utf8 ()
 static void
 test_one_liner_many_fixits_1_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 10);
   rich_location richloc (line_table, equals);
   for (int i = 0; i < 19; i++)
@@ -5485,7 +5517,7 @@ test_one_liner_many_fixits_1_utf8 ()
 static void
 test_one_liner_many_fixits_2_utf8 ()
 {
-  test_diagnostic_context dc;
+  test_context dc;
   location_t equals = linemap_position_for_column (line_table, 10);
   rich_location richloc (line_table, equals);
   const int nlocs = 19;
@@ -5546,7 +5578,7 @@ test_one_liner_labels_utf8 ()
     richloc.add_range (field, SHOW_RANGE_WITHOUT_CARET, &label2);
 
     {
-      test_diagnostic_context dc;
+      test_context dc;
       ASSERT_STREQ (" \xf0\x9f\x98\x82"
 		       "_foo = \xcf\x80"
 			       "_bar.\xf0\x9f\x98\x82"
@@ -5572,7 +5604,7 @@ test_one_liner_labels_utf8 ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label1);
     richloc.add_range (field, SHOW_RANGE_WITHOUT_CARET, &label2);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" \xf0\x9f\x98\x82"
 		     "_foo = \xcf\x80"
 			     "_bar.\xf0\x9f\x98\x82"
@@ -5596,7 +5628,7 @@ test_one_liner_labels_utf8 ()
     richloc.add_range (bar, SHOW_RANGE_WITHOUT_CARET, &label1);
     richloc.add_range (field, SHOW_RANGE_WITHOUT_CARET, &label2);
 
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" \xf0\x9f\x98\x82"
 		     "_foo = \xcf\x80"
 			     "_bar.\xf0\x9f\x98\x82"
@@ -5621,7 +5653,7 @@ test_one_liner_labels_utf8 ()
     richloc.set_escape_on_output (true);
 
     {
-      test_diagnostic_context dc;
+      test_context dc;
       dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_UNICODE);
       ASSERT_STREQ (" <U+1F602>_foo = <U+03C0>_bar.<U+1F602>_field<U+03C0>;\n"
 		    " ^~~~~~~~~~~~~   ~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~~~\n"
@@ -5632,7 +5664,7 @@ test_one_liner_labels_utf8 ()
 		    dc.test_show_locus (richloc));
     }
     {
-      test_diagnostic_context dc;
+      test_context dc;
       dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_BYTES);
       ASSERT_STREQ
 	(" <f0><9f><98><82>_foo = <cf><80>_bar.<f0><9f><98><82>_field<cf><80>;\n"
@@ -5651,8 +5683,8 @@ test_one_liner_labels_utf8 ()
 static void
 test_one_liner_colorized_utf8 ()
 {
-  test_diagnostic_context dc;
-  dc.m_source_printing.colorize_source_p = true;
+  test_context dc;
+  dc.colorize_source (true);
   diagnostic_color_init (&dc, DIAGNOSTICS_COLOR_YES);
   const location_t pi = linemap_position_for_column (line_table, 12);
   rich_location richloc (line_table, pi);
@@ -5675,9 +5707,9 @@ static const char * const one_liner_utf8_content
      1111222233334444567890122223333456789999000011112222345678999900001
      Byte columns.  */
 
-diagnostic_show_locus_fixture_one_liner_utf8::
-diagnostic_show_locus_fixture_one_liner_utf8 (const line_table_case &case_)
-: diagnostic_show_locus_fixture (case_, one_liner_utf8_content)
+source_printing_fixture_one_liner_utf8::
+source_printing_fixture_one_liner_utf8 (const line_table_case &case_)
+: source_printing_fixture (case_, one_liner_utf8_content)
 {
 }
 
@@ -5686,7 +5718,7 @@ diagnostic_show_locus_fixture_one_liner_utf8 (const line_table_case &case_)
 static void
 test_diagnostic_show_locus_one_liner_utf8 (const line_table_case &case_)
 {
-  diagnostic_show_locus_fixture_one_liner_utf8 f (case_);
+  source_printing_fixture_one_liner_utf8 f (case_);
 
   location_t line_end = linemap_position_for_column (line_table, 31);
 
@@ -5698,7 +5730,7 @@ test_diagnostic_show_locus_one_liner_utf8 (const line_table_case &case_)
   ASSERT_EQ (1, LOCATION_LINE (line_end));
   ASSERT_EQ (31, LOCATION_COLUMN (line_end));
 
-  char_span lspan = f.m_fc.get_source_line (f.get_filename (), 1);
+  diagnostics::char_span lspan = f.m_fc.get_source_line (f.get_filename (), 1);
   ASSERT_EQ (25, cpp_display_width (lspan.get_buffer (), lspan.length (),
 				    def_policy ()));
   ASSERT_EQ (25, location_compute_display_column (f.m_fc,
@@ -5755,7 +5787,7 @@ test_add_location_if_nearby (const line_table_case &case_)
   /* Test of add_location_if_nearby on the same line as the
      primary location.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     const location_t missing_close_brace_1_39
       = linemap_position_for_line_and_column (line_table, ord_map, 1, 39);
     const location_t matching_open_brace_1_18
@@ -5773,7 +5805,7 @@ test_add_location_if_nearby (const line_table_case &case_)
   /* Test of add_location_if_nearby on a different line to the
      primary location.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     const location_t missing_close_brace_6_1
       = linemap_position_for_line_and_column (line_table, ord_map, 6, 1);
     const location_t matching_open_brace_3_1
@@ -5822,7 +5854,7 @@ test_diagnostic_show_locus_fixit_lines (const line_table_case &case_)
 
   /* The one-liner case (line 2).  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     const location_t x
       = linemap_position_for_line_and_column (line_table, ord_map, 2, 24);
     const location_t colon
@@ -5841,7 +5873,7 @@ test_diagnostic_show_locus_fixit_lines (const line_table_case &case_)
      span starts are printed due to the gap between the span at line 3
      and that at line 6).  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     const location_t y
       = linemap_position_for_line_and_column (line_table, ord_map, 3, 24);
     const location_t colon
@@ -5869,8 +5901,8 @@ test_diagnostic_show_locus_fixit_lines (const line_table_case &case_)
     rich_location richloc (line_table, colon);
     richloc.add_fixit_insert_before (y, ".");
     richloc.add_fixit_replace (colon, "=");
-    test_diagnostic_context dc;
-    dc.m_source_printing.show_line_numbers_p = true;
+    test_context dc;
+    dc.show_line_numbers (true);
     ASSERT_STREQ ("    3 |                        y\n"
 		  "      |                        .\n"
 		  "......\n"
@@ -6043,7 +6075,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
   const char *content
     = ("  foo *f = (foo *)ptr->field;\n");
   temp_source_file tmp (SELFTEST_LOCATION, ".C", content);
-  file_cache fc;
+  diagnostics::file_cache fc;
   line_table_test ltt (case_);
 
   const line_map_ordinary *ord_map
@@ -6075,7 +6107,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
 
   /* Example where 3 fix-it hints are printed as one.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_replace (open_paren, "const_cast<");
     richloc.add_fixit_replace (close_paren, "> (");
@@ -6144,7 +6176,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
 
   /* Example where two are consolidated during printing.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_replace (open_paren, "CAST (");
     richloc.add_fixit_replace (close_paren, ") (");
@@ -6160,7 +6192,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
 
   /* Example where none are consolidated during printing.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_replace (open_paren, "CST (");
     richloc.add_fixit_replace (close_paren, ") (");
@@ -6176,7 +6208,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
 
   /* Example of deletion fix-it hints.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_insert_before (open_paren, "(bar *)");
     source_range victim = {open_paren, close_paren};
@@ -6195,7 +6227,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
 
   /* Example of deletion fix-it hints that would overlap.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_insert_before (open_paren, "(longer *)");
     source_range victim = {expr_start, expr_finish};
@@ -6214,7 +6246,7 @@ test_overlapped_fixit_printing (const line_table_case &case_)
 
   /* Example of insertion fix-it hints that would overlap.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_insert_before (open_paren, "LONGER THAN THE CAST");
     richloc.add_fixit_insert_after (close_paren, "TEST");
@@ -6280,8 +6312,8 @@ test_overlapped_fixit_printing_utf8 (const line_table_case &case_)
 
   /* Example where 3 fix-it hints are printed as one.  */
   {
-    test_diagnostic_context dc;
-    file_cache &fc = dc.get_file_cache ();
+    test_context dc;
+    diagnostics::file_cache &fc = dc.get_file_cache ();
     rich_location richloc (line_table, expr);
     richloc.add_fixit_replace (open_paren, "const_cast<");
     richloc.add_fixit_replace (close_paren, "> (");
@@ -6356,7 +6388,7 @@ test_overlapped_fixit_printing_utf8 (const line_table_case &case_)
 
   /* Example where two are consolidated during printing.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_replace (open_paren, "CAST (");
     richloc.add_fixit_replace (close_paren, ") (");
@@ -6375,7 +6407,7 @@ test_overlapped_fixit_printing_utf8 (const line_table_case &case_)
 
   /* Example where none are consolidated during printing.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_replace (open_paren, "CST (");
     richloc.add_fixit_replace (close_paren, ") (");
@@ -6394,7 +6426,7 @@ test_overlapped_fixit_printing_utf8 (const line_table_case &case_)
 
   /* Example of deletion fix-it hints.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_insert_before (open_paren, "(bar\xf0\x9f\x98\x82 *)");
     source_range victim = {open_paren, close_paren};
@@ -6417,7 +6449,7 @@ test_overlapped_fixit_printing_utf8 (const line_table_case &case_)
 
   /* Example of deletion fix-it hints that would overlap.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_insert_before (open_paren, "(long\xf0\x9f\x98\x82 *)");
     source_range victim = {expr_start, expr_finish};
@@ -6441,7 +6473,7 @@ test_overlapped_fixit_printing_utf8 (const line_table_case &case_)
 
   /* Example of insertion fix-it hints that would overlap.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, expr);
     richloc.add_fixit_insert_before
       (open_paren, "L\xf0\x9f\x98\x82NGER THAN THE CAST");
@@ -6505,8 +6537,8 @@ test_overlapped_fixit_printing_2 (const line_table_case &case_)
 
   /* Two insertions, in the wrong order.  */
   {
-    test_diagnostic_context dc;
-    file_cache &fc = dc.get_file_cache ();
+    test_context dc;
+    diagnostics::file_cache &fc = dc.get_file_cache ();
 
     rich_location richloc (line_table, col_20);
     richloc.add_fixit_insert_before (col_23, "{");
@@ -6534,7 +6566,7 @@ test_overlapped_fixit_printing_2 (const line_table_case &case_)
   /* Various overlapping insertions, some occurring "out of order"
      (reproducing the fix-it hints from PR c/81405).  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     rich_location richloc (line_table, col_20);
 
     richloc.add_fixit_insert_before (col_20, "{{");
@@ -6588,7 +6620,7 @@ test_fixit_insert_containing_newline (const line_table_case &case_)
 
     /* Without line numbers.  */
     {
-      test_diagnostic_context dc;
+      test_context dc;
       ASSERT_STREQ ("       x = a;\n"
 		    "+      break;\n"
 		    "     case 'b':\n"
@@ -6598,8 +6630,8 @@ test_fixit_insert_containing_newline (const line_table_case &case_)
 
     /* With line numbers.  */
     {
-      test_diagnostic_context dc;
-      dc.m_source_printing.show_line_numbers_p = true;
+      test_context dc;
+      dc.show_line_numbers (true);
       ASSERT_STREQ ("    2 |       x = a;\n"
 		    "  +++ |+      break;\n"
 		    "    3 |     case 'b':\n"
@@ -6614,7 +6646,7 @@ test_fixit_insert_containing_newline (const line_table_case &case_)
     rich_location richloc (line_table, case_loc);
     richloc.add_fixit_insert_before (case_start, "break;\n");
     ASSERT_TRUE (richloc.seen_impossible_fixit_p ());
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ ("     case 'b':\n"
 		  "     ^~~~~~~~~\n",
 		  dc.test_show_locus (richloc));
@@ -6661,7 +6693,7 @@ test_fixit_insert_containing_newline_2 (const line_table_case &case_)
     return;
 
   {
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ ("FILENAME:1:1:\n"
 		  "+#include <stdio.h>\n"
 		  " test (int ch)\n"
@@ -6674,8 +6706,8 @@ test_fixit_insert_containing_newline_2 (const line_table_case &case_)
   /* With line-numbering, the line spans are close enough to be
      consolidated, since it makes little sense to skip line 2.  */
   {
-    test_diagnostic_context dc;
-    dc.m_source_printing.show_line_numbers_p = true;
+    test_context dc;
+    dc.show_line_numbers (true);
     ASSERT_STREQ ("  +++ |+#include <stdio.h>\n"
 		  "    1 | test (int ch)\n"
 		  "    2 | {\n"
@@ -6717,7 +6749,7 @@ test_fixit_replace_containing_newline (const line_table_case &case_)
   if (finish > LINE_MAP_MAX_LOCATION_WITH_COLS)
     return;
 
-  test_diagnostic_context dc;
+  test_context dc;
   ASSERT_STREQ (" foo = bar ();\n"
 		"             ^\n",
 		dc.test_show_locus (richloc));
@@ -6760,7 +6792,7 @@ test_fixit_deletion_affecting_newline (const line_table_case &case_)
   if (finish > LINE_MAP_MAX_LOCATION_WITH_COLS)
     return;
 
-  test_diagnostic_context dc;
+  test_context dc;
   ASSERT_STREQ (" foo = bar (\n"
 		"          ~^\n"
 		"       );\n"
@@ -6800,7 +6832,7 @@ test_tab_expansion (const line_table_case &case_)
      into 11 spaces.  Recall that print_line() also puts one space before
      everything too.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.m_tabstop = tabstop;
     rich_location richloc (line_table,
 			   linemap_position_for_column (line_table,
@@ -6813,7 +6845,7 @@ test_tab_expansion (const line_table_case &case_)
   /* Confirm the display width was tracked correctly across the internal tab
      as well.  */
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.m_tabstop = tabstop;
     rich_location richloc (line_table,
 			   linemap_position_for_column (line_table,
@@ -6854,14 +6886,14 @@ test_escaping_bytes_1 (const line_table_case &case_)
   richloc.add_range (v_loc);
 
   {
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ (" before \1\2\3\v\x80\xff""after\n"
 		  "       ^   ~\n",
 		  dc.test_show_locus (richloc));
   }
   richloc.set_escape_on_output (true);
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_UNICODE);
     ASSERT_STREQ
       (" before<U+0000><U+0001><U+0002><U+0003><U+000B><80><ff>after\n"
@@ -6869,7 +6901,7 @@ test_escaping_bytes_1 (const line_table_case &case_)
        dc.test_show_locus (richloc));
   }
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_BYTES);
     ASSERT_STREQ (" before<00><01><02><03><0b><80><ff>after\n"
 		  "       ^~~~            ~~~~\n",
@@ -6904,21 +6936,21 @@ test_escaping_bytes_2 (const line_table_case &case_)
   gcc_rich_location richloc (nul_loc);
 
   {
-    test_diagnostic_context dc;
+    test_context dc;
     ASSERT_STREQ ("  after\n"
 		  " ^\n",
 		  dc.test_show_locus (richloc));
   }
   richloc.set_escape_on_output (true);
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_UNICODE);
     ASSERT_STREQ (" <U+0000>after\n"
 		  " ^~~~~~~~\n",
 		  dc.test_show_locus (richloc));
   }
   {
-    test_diagnostic_context dc;
+    test_context dc;
     dc.set_escape_format (DIAGNOSTICS_ESCAPE_FORMAT_BYTES);
     ASSERT_STREQ (" <00>after\n"
 		  " ^~~~\n",
@@ -6957,9 +6989,10 @@ test_line_numbers_multiline_range ()
     = linemap_position_for_line_and_column (line_table, ord_map, 11, 4);
   location_t loc = make_location (caret, start, finish);
 
-  test_diagnostic_context dc;
-  dc.m_source_printing.show_line_numbers_p = true;
-  dc.m_source_printing.min_margin_width = 0;
+  test_context dc;
+  dc.show_line_numbers (true);
+  auto &source_printing_opts = dc.get_source_printing_options ();
+  source_printing_opts.min_margin_width = 0;
   gcc_rich_location richloc (loc);
   ASSERT_STREQ (" 9 | this is line 9\n"
 		"   |         ~~~~~~\n"
@@ -6973,7 +7006,7 @@ test_line_numbers_multiline_range ()
 /* Run all of the selftests within this file.  */
 
 void
-diagnostic_show_locus_cc_tests ()
+source_printing_cc_tests ()
 {
   test_line_span ();
 
@@ -7009,7 +7042,8 @@ diagnostic_show_locus_cc_tests ()
   test_line_numbers_multiline_range ();
 }
 
-} // namespace selftest
+} // namespace diagnostics::selftest
+} // namespace diagnostics
 
 #endif /* #if CHECKING_P */
 
diff --git a/gcc/diagnostics/state-graphs-to-dot.cc b/gcc/diagnostics/state-graphs-to-dot.cc
new file mode 100644
index 0000000..2d80e6b
--- /dev/null
+++ b/gcc/diagnostics/state-graphs-to-dot.cc
@@ -0,0 +1,551 @@
+/* Creating GraphViz .dot files from diagnostic state graphs.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#define INCLUDE_ALGORITHM
+#define INCLUDE_MAP
+#define INCLUDE_SET
+#define INCLUDE_STRING
+#define INCLUDE_VECTOR
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+
+#include "diagnostics/state-graphs.h"
+#include "graphviz.h"
+#include "xml.h"
+#include "xml-printer.h"
+#include "intl.h"
+
+using namespace diagnostics;
+using namespace diagnostics::state_graphs;
+
+static int
+get_depth (const digraphs::node &n)
+{
+  int deepest_child = 0;
+  for (size_t i = 0; i < n.get_num_children (); ++i)
+      deepest_child = std::max (deepest_child,
+				get_depth (n.get_child (i)));
+  return deepest_child + 1;
+}
+
+static const char *
+get_color_for_dynalloc_state (enum node_dynalloc_state dynalloc_st)
+{
+  switch (dynalloc_st)
+    {
+    default:
+      gcc_unreachable ();
+      break;
+    case node_dynalloc_state::unknown:
+    case node_dynalloc_state::nonnull:
+      return nullptr;
+
+    case node_dynalloc_state::unchecked:
+      return "#ec7a08"; // pf-orange-400
+
+    case node_dynalloc_state::freed:
+      return "#cc0000"; // pf-red-100
+    }
+}
+
+static void
+set_color_for_dynalloc_state (dot::attr_list &attrs,
+			      enum node_dynalloc_state state)
+{
+  if (const char *color = get_color_for_dynalloc_state (state))
+    attrs.add (dot::id ("color"), dot::id (color));
+}
+
+class state_diagram : public dot::graph
+{
+public:
+  state_diagram (const diagnostics::digraphs::digraph &input_state_graph,
+		 const logical_locations::manager &logical_loc_mgr)
+  : m_logical_loc_mgr (logical_loc_mgr)
+  {
+    // "node [shape=plaintext]\n"
+    {
+      auto attr_stmt
+	= std::make_unique<dot::attr_stmt> (dot::attr_stmt::kind::node);
+      attr_stmt->m_attrs.add (dot::id ("shape"), dot::id ("plaintext"));
+      add_stmt (std::move (attr_stmt));
+    }
+
+    /* Determine which nodes are involved in edges.  */
+    for (size_t i = 0; i < input_state_graph.get_num_edges (); ++i)
+      {
+	auto &edge = input_state_graph.get_edge (i);
+	m_src_nodes.insert (&edge.get_src_node ());
+	m_dst_nodes.insert (&edge.get_dst_node ());
+      }
+
+    /* Recurse down the nodes in the state graph, creating subgraphs
+       and then eventually creating nodes, and recursively
+       creating XML tables, and adding ports for the endpoints of edges
+       where needed.  */
+
+    auto root_cluster
+      = std::make_unique<dot::subgraph> (dot::id ("cluster_memory_regions"));
+    for (size_t i = 0; i < input_state_graph.get_num_nodes (); ++i)
+      on_input_state_node (*root_cluster,
+			   state_node_ref (input_state_graph.get_node (i)));
+    add_stmt (std::move (root_cluster));
+
+    /* Now create dot edges for edges in input_stage_graph.  */
+    for (size_t i = 0; i < input_state_graph.get_num_edges (); ++i)
+      {
+	auto &edge = input_state_graph.get_edge (i);
+	auto &src_node = edge.get_src_node ();
+	auto &dst_node = edge.get_dst_node ();
+
+	auto src_port_id = m_src_node_to_port_id.find (&src_node);
+	if (src_port_id == m_src_node_to_port_id.end ())
+	  continue;
+	auto dst_port_id = m_dst_node_to_port_id.find (&dst_node);
+	if (dst_port_id == m_dst_node_to_port_id.end ())
+	  continue;
+
+	auto e = std::make_unique<dot::edge_stmt> (src_port_id->second,
+						   dst_port_id->second);
+	set_color_for_dynalloc_state
+	  (e->m_attrs, state_node_ref (dst_node).get_dynalloc_state ());
+
+	add_stmt (std::move (e));
+      }
+  }
+
+private:
+  struct pending_edge
+  {
+    dot::node_id m_src_node_id;
+    std::string m_dst_region_id;
+  };
+
+  dot::id
+  get_id_for_region (const char *region_id)
+  {
+    gcc_assert (region_id);
+    return std::string ("cluster_region_") + region_id;
+  }
+
+  dot::id
+  make_id (state_node_ref state_node, bool cluster)
+  {
+    std::string input_node_id = state_node.m_node.get_id ();
+    if (cluster)
+      return std::string ("cluster_") + input_node_id;
+    else
+      return input_node_id;
+  }
+
+  bool
+  starts_node_p (state_node_ref state_node)
+  {
+    switch (state_node.get_node_kind ())
+      {
+      default:
+	return false;
+
+      case node_kind::stack:
+	/* We want all frames in the stack in the same table,
+	   so they are grouped.  */
+      case node_kind::dynalloc_buffer:
+      case node_kind::variable:
+	return true;
+      }
+  }
+
+  const char *
+  get_label_for_node (state_node_ref state_node)
+  {
+    switch (state_node.get_node_kind ())
+      {
+      default:
+	return nullptr;
+
+      case node_kind::globals:
+	return _("Globals");
+      case node_kind::code:
+	return _("Code");
+      case node_kind::stack:
+	return _("Stack");
+      case node_kind::heap_:
+	return _("Heap");
+      }
+  }
+
+  void
+  on_input_state_node (dot::subgraph &parent_subgraph,
+		       state_node_ref state_node)
+  {
+    dot::id sg_id = make_id (state_node, true);
+
+    if (starts_node_p (state_node))
+      {
+	// Create node with table
+	xml::element table ("table", false);
+	xml::printer xp (table);
+	xp.set_attr ("border", "0");
+	xp.set_attr ("cellborder", "1");
+	xp.set_attr ("cellspacing", "0");
+
+	const int max_depth = get_depth (state_node.m_node);
+	const int num_columns = max_depth + 2;
+
+	dot::id id_of_dot_node = make_id (state_node, false);
+	on_node_in_table (id_of_dot_node, xp, state_node,
+			  max_depth, 0, num_columns);
+
+	auto node = std::make_unique<dot::node_stmt> (std::move (id_of_dot_node));
+	node->m_attrs.add (dot::id ("shape"),
+			   dot::id ("plaintext"));
+
+	// xml must be done by now
+
+	node->m_attrs.add (dot::id ("label"),
+			   dot::id (table));
+
+	parent_subgraph.m_stmt_list.add_stmt (std::move (node));
+      }
+    else
+      {
+	auto child_subgraph = std::make_unique<dot::subgraph> (std::move (sg_id));
+
+	if (const char *label = get_label_for_node (state_node))
+	  child_subgraph->add_attr (dot::id ("label"), dot::id (label));
+
+	// recurse:
+	for (size_t i = 0; i < state_node.m_node.get_num_children (); ++i)
+	  on_input_state_node (*child_subgraph,
+			       state_node.m_node.get_child (i));
+	parent_subgraph.m_stmt_list.add_stmt (std::move (child_subgraph));
+      }
+  }
+
+  enum class style { h1, h2 };
+
+  void
+  add_title_tr (const dot::id &id_of_dot_node,
+		xml::printer &xp,
+		int num_columns,
+		state_node_ref state_node,
+		std::string heading,
+		enum style styl,
+		enum node_dynalloc_state dynalloc_state)
+  {
+    xp.push_tag ("tr", true);
+    xp.push_tag ("td", false);
+    xp.set_attr ("colspan", std::to_string (num_columns));
+    xp.set_attr ("cellpadding", "5");
+
+    const char *bgcolor;
+    const char *color;
+    if (const char *c = get_color_for_dynalloc_state (dynalloc_state))
+      {
+	bgcolor = c;
+	color = "white";
+      }
+    else
+      switch (styl)
+	{
+	default:
+	  gcc_unreachable ();
+	case style::h1:
+	  // from diagnostics/html-sink.cc: HTML_STYLE .linenum
+	  bgcolor = "#0088ce";
+	  color = "white";
+	  break;
+	case style::h2:
+	  // from diagnostics/html-sink.cc: HTML_STYLE .events-hdr
+	  bgcolor = "#393f44"; // pf-black-800
+	  color = "white";
+	  break;
+	}
+
+    xp.set_attr ("bgcolor", bgcolor);
+    xp.push_tag ("font", false);
+    xp.set_attr ("color", color);
+    if (heading == "")
+      heading = " ";
+    xp.add_text (std::move (heading));
+    xp.pop_tag ("font");
+
+    maybe_add_dst_port (id_of_dot_node, xp, state_node);
+
+    xp.pop_tag ("td");
+    xp.pop_tag ("tr");
+  }
+
+  /* Recursively add <TR> to XP for STATE_NODE and its descendents.  */
+  void
+  on_node_in_table (const dot::id &id_of_dot_node,
+		    xml::printer &xp,
+		    state_node_ref state_node,
+		    int max_depth,
+		    int depth,
+		    int num_columns)
+  {
+    bool recurse = true;
+    auto input_node_kind = state_node.get_node_kind ();
+
+    switch (input_node_kind)
+      {
+      case node_kind::padding:
+      case node_kind::other:
+	return;
+
+      case node_kind::stack:
+	add_title_tr (id_of_dot_node, xp, num_columns, state_node, "Stack",
+		      style::h1,
+		      node_dynalloc_state::unknown);
+	break;
+      case node_kind::stack_frame:
+	if (auto logical_loc = state_node.get_logical_loc ())
+	  if (const char *function
+		= m_logical_loc_mgr.get_short_name (logical_loc))
+	    add_title_tr (id_of_dot_node, xp, num_columns, state_node,
+			  std::string ("Frame: ") + function,
+			  style::h2,
+			  node_dynalloc_state::unknown);
+	break;
+      case node_kind::dynalloc_buffer:
+	{
+	  enum node_dynalloc_state dynalloc_st
+	    = state_node.get_dynalloc_state ();
+	  const char *extents = state_node.get_dynamic_extents ();
+	  const char *type = state_node.get_type ();
+	  pretty_printer pp;
+	  switch (dynalloc_st)
+	    {
+	    default:
+	      gcc_unreachable ();
+
+	    case node_dynalloc_state::unknown:
+	    case node_dynalloc_state::nonnull:
+	      if (type)
+		{
+		if (extents)
+		  pp_printf (&pp, "%s (%s byte allocation)",
+			     type, extents);
+		else
+		  pp_printf (&pp, "%s", type);
+		}
+	      else
+		{
+		  if (extents)
+		    pp_printf (&pp, "%s byte allocation",
+			       extents);
+		}
+	      break;
+
+	    case node_dynalloc_state::unchecked:
+	      if (type)
+		{
+		  if (extents)
+		    pp_printf (&pp, "%s (unchecked %s byte allocation)",
+			       type, extents);
+		}
+	      else
+		{
+		  if (extents)
+		    pp_printf (&pp, "Unchecked %s byte allocation",
+			       extents);
+		}
+	      break;
+
+	    case node_dynalloc_state::freed:
+	      // TODO: show deallocator
+	      // TODO: show deallocation event
+	      pp_printf (&pp, "Freed buffer");
+	      break;
+	    }
+	  maybe_add_dst_port (id_of_dot_node, xp, state_node);
+	  add_title_tr (id_of_dot_node, xp, num_columns, state_node,
+			pp_formatted_text (&pp),
+			style::h2,
+			dynalloc_st);
+	}
+	break;
+
+      default:
+	{
+	  xp.push_tag ("tr", true);
+
+	  maybe_add_dst_port (id_of_dot_node, xp, state_node);
+
+	  if (depth > 0)
+	    {
+	      /* Indent, by create a <td> spanning "depth" columns.  */
+	      xp.push_tag ("td", false);
+	      xp.set_attr ("colspan", std::to_string (depth));
+	      xp.add_text (" "); // graphviz doesn't like <td/>
+	      xp.pop_tag ("td");
+	    }
+
+	  switch (input_node_kind)
+	    {
+	    default:
+	      break;
+	    case node_kind::variable:
+	      {
+		const char *name = state_node.get_name ();
+		gcc_assert (name);
+		xp.push_tag ("td", false);
+		maybe_add_dst_port (id_of_dot_node, xp, state_node);
+		push_src_text (xp);
+		xp.add_text (name);
+		pop_src_text (xp);
+		xp.pop_tag ("td");
+	      }
+	      break;
+	    case node_kind::element:
+	      {
+		const char *index = state_node.get_index ();
+		gcc_assert (index);
+		xp.push_tag ("td", false);
+		maybe_add_dst_port (id_of_dot_node, xp, state_node);
+		push_src_text (xp);
+		xp.add_text ("[");
+		xp.add_text (index);
+		xp.add_text ("]");
+		pop_src_text (xp);
+		xp.pop_tag ("td");
+	      }
+	      break;
+	    case node_kind::field:
+	      {
+		const char *name = state_node.get_name ();
+		gcc_assert (name);
+		xp.push_tag ("td", false);
+		maybe_add_dst_port (id_of_dot_node, xp, state_node);
+		push_src_text (xp);
+		xp.add_text (".");
+		xp.add_text (name);
+		pop_src_text (xp);
+		xp.pop_tag ("td");
+	      }
+	      break;
+	    }
+
+	  if (const char *type = state_node.get_type ())
+	    {
+	      xp.push_tag ("td", false);
+	      xp.set_attr ("align", "right");
+	      push_src_text (xp);
+	      xp.add_text (type);
+	      pop_src_text (xp);
+	      xp.pop_tag ("td");
+	    }
+
+	  if (const char *value = state_node.get_value ())
+	    {
+	      xp.push_tag ("td", false);
+	      xp.set_attr ("align", "left");
+	      maybe_add_src_port (id_of_dot_node, xp, state_node);
+	      push_src_text (xp);
+	      xp.add_text (value);
+	      pop_src_text (xp);
+	      xp.pop_tag ("td");
+	      recurse = false;
+	    }
+	  xp.pop_tag ("tr");
+	}
+	break;
+      }
+
+    if (recurse)
+      for (size_t i = 0; i < state_node.m_node.get_num_children (); ++i)
+	on_node_in_table (id_of_dot_node, xp,
+			  state_node.m_node.get_child (i),
+			  max_depth, depth + 1, num_columns);
+  }
+
+  void
+  push_src_text (xml::printer &xp)
+  {
+    xp.push_tag ("font");
+    xp.set_attr ("color", "blue");
+  }
+
+  void
+  pop_src_text (xml::printer &xp)
+  {
+    xp.pop_tag ("font");
+  }
+
+  /* If STATE_NODE is in m_src_nodes, add a port to XP for possible
+     incoming edges to use.  */
+
+  void
+  maybe_add_src_port (const dot::id &id_of_dot_node,
+		      xml::printer &xp,
+		      state_node_ref state_node)
+  {
+    auto iter = m_src_nodes.find (&state_node.m_node);
+    if (iter == m_src_nodes.end ())
+      return;
+
+    dot::id src_id = make_id (state_node, false);
+    dot::node_id node_id (id_of_dot_node,
+			  dot::port (src_id,
+				     dot::compass_pt::e));
+    m_src_node_to_port_id.insert ({&state_node.m_node, node_id});
+    xp.set_attr ("port", src_id.m_str);
+  }
+
+  /* If STATE_NODE is in m_dst_nodes, add a port to XP for possible
+     incoming edges to use.  */
+
+  void
+  maybe_add_dst_port (const dot::id &id_of_dot_node,
+		      xml::printer &xp,
+		      state_node_ref state_node)
+  {
+    auto iter = m_dst_nodes.find (&state_node.m_node);
+    if (iter == m_dst_nodes.end ())
+      return;
+
+    dot::id dst_id = make_id (state_node, false);
+    dot::node_id node_id (id_of_dot_node,
+			  dot::port (dst_id/*,
+					     dot::compass_pt::w*/));
+    m_dst_node_to_port_id.insert ({&state_node.m_node, node_id});
+    xp.set_attr ("port", dst_id.m_str);
+  }
+
+private:
+  const logical_locations::manager &m_logical_loc_mgr;
+
+  /* All nodes involved in edges (and thus will need a port).  */
+  std::set<digraphs::node *> m_src_nodes;
+  std::set<digraphs::node *> m_dst_nodes;
+
+  std::map<digraphs::node *, dot::node_id> m_src_node_to_port_id;
+  std::map<digraphs::node *, dot::node_id> m_dst_node_to_port_id;
+};
+
+std::unique_ptr<dot::graph>
+state_graphs::
+make_dot_graph (const digraphs::digraph &state_graph,
+		const logical_locations::manager &logical_loc_mgr)
+{
+  return std::make_unique<state_diagram> (state_graph, logical_loc_mgr);
+}
diff --git a/gcc/diagnostics/state-graphs.cc b/gcc/diagnostics/state-graphs.cc
new file mode 100644
index 0000000..5941c41
--- /dev/null
+++ b/gcc/diagnostics/state-graphs.cc
@@ -0,0 +1,156 @@
+/* Extensions to diagnostics::digraphs to support state graphs.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#define INCLUDE_ALGORITHM
+#define INCLUDE_MAP
+#define INCLUDE_SET
+#define INCLUDE_STRING
+#define INCLUDE_VECTOR
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+
+#include "diagnostics/state-graphs.h"
+#include "selftest.h"
+
+using namespace diagnostics::state_graphs;
+
+const char * const node_kind_strs[] = {
+  "globals",
+  "code",
+  "function",
+  "stack",
+  "stack-frame",
+  "heap",
+  "thread-local",
+  "dynalloc-buffer",
+  "variable",
+  "field",
+  "padding",
+  "element",
+  "other",
+};
+
+const char *
+diagnostics::state_graphs::node_kind_to_str (enum node_kind k)
+{
+  return node_kind_strs[static_cast<int> (k)];
+}
+
+// struct state_node_ref
+
+enum node_kind
+state_node_ref::get_node_kind () const
+{
+  const char *value = get_attr ("kind");
+  if (!value)
+    return node_kind::other;
+
+  for (size_t i = 0; i < ARRAY_SIZE (node_kind_strs); ++i)
+    if (!strcmp (node_kind_strs[i], value))
+      return static_cast<enum node_kind> (i);
+
+  return node_kind::other;
+}
+
+void
+state_node_ref::set_node_kind (enum node_kind k)
+{
+  set_attr ("kind", node_kind_to_str (k));
+}
+
+const char * const dynalloc_state_strs[] = {
+  "unknown",
+  "nonnull",
+  "unchecked",
+  "freed"
+};
+
+enum node_dynalloc_state
+state_node_ref::get_dynalloc_state () const
+{
+  const char *value = get_attr ("dynalloc-state");
+  if (!value)
+    return node_dynalloc_state::unknown;
+
+  for (size_t i = 0; i < ARRAY_SIZE (dynalloc_state_strs); ++i)
+    if (!strcmp (dynalloc_state_strs[i], value))
+      return static_cast<enum node_dynalloc_state> (i);
+  
+  return node_dynalloc_state::unknown;
+}
+
+void
+state_node_ref::set_dynalloc_state (enum node_dynalloc_state s) const
+{
+  set_attr ("dynalloc-state",
+	    dynalloc_state_strs[static_cast <size_t> (s)]);
+}
+
+const char *
+state_node_ref::get_dynamic_extents () const
+{
+  return m_node.get_attr (STATE_NODE_PREFIX, "dynamic-extents");
+}
+
+void
+state_node_ref::set_json_attr (const char *key,
+			       std::unique_ptr<json::value> value) const
+{
+  m_node.set_json_attr (STATE_NODE_PREFIX, key, std::move (value));
+}
+
+#if CHECKING_P
+
+namespace diagnostics {
+namespace selftest {
+
+static void
+test_node_attrs ()
+{
+  digraphs::digraph g;
+  digraphs::node n (g, "a");
+  state_node_ref node_ref (n);
+
+  ASSERT_EQ (node_ref.get_node_kind (), node_kind::other);
+  node_ref.set_node_kind (node_kind::stack);
+  ASSERT_EQ (node_ref.get_node_kind (), node_kind::stack);
+
+  ASSERT_EQ (node_ref.get_dynalloc_state (), node_dynalloc_state::unknown);
+  node_ref.set_dynalloc_state (node_dynalloc_state::freed);
+  ASSERT_EQ (node_ref.get_dynalloc_state (), node_dynalloc_state::freed);
+
+  ASSERT_EQ (node_ref.get_type (), nullptr);
+  node_ref.set_type ("const char *");
+  ASSERT_STREQ (node_ref.get_type (), "const char *");
+}
+
+/* Run all of the selftests within this file.  */
+
+void
+state_graphs_cc_tests ()
+{
+  test_node_attrs ();
+}
+
+} // namespace diagnostics::selftest
+} // namespace diagnostics
+
+#endif /* CHECKING_P */
diff --git a/gcc/diagnostics/state-graphs.h b/gcc/diagnostics/state-graphs.h
new file mode 100644
index 0000000..ad18f82
--- /dev/null
+++ b/gcc/diagnostics/state-graphs.h
@@ -0,0 +1,156 @@
+/* Extensions to diagnostics::digraphs to support state graphs.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_DIAGNOSTICS_STATE_GRAPHS_H
+#define GCC_DIAGNOSTICS_STATE_GRAPHS_H
+
+#include "diagnostics/digraphs.h"
+#include "diagnostics/logical-locations.h"
+
+/* diagnostics::digraphs provides support for directed graphs.
+
+   diagnostics::state_graphs provides a way to extend these graphs
+   for representing "state graphs" i.e. a representation of the state
+   of memory inside a program, for use e.g. by -fanalyzer.
+
+   Specifically, nodes represent memory regions, and we use property bags
+   in these nodes to stash extra properties (e.g. what kind of memory region
+   a node is e.g. stack vs heap).  */
+
+class sarif_graph;
+namespace dot { class graph; }
+
+namespace diagnostics {
+namespace state_graphs {
+
+enum class node_kind
+{
+  // Memory regions
+  globals,
+  code,
+  function, // code within a particular function
+  stack,
+  stack_frame,
+  heap_,
+  thread_local_,
+
+  /* Dynamically-allocated buffer,
+     on heap or stack (depending on parent).  */
+  dynalloc_buffer,
+
+  variable,
+
+  field, // field within a struct or union
+  padding, // padding bits in a struct or union
+  element, // element within an array
+
+  other // anything else
+};
+
+extern const char *
+node_kind_to_str (enum node_kind);
+
+enum class node_dynalloc_state
+{
+  unknown,
+  nonnull,
+  unchecked,
+  freed
+};
+
+/* Prefixes to use in SARIF property bags.  */
+#define STATE_GRAPH_PREFIX "gcc/diagnostic_state_graph/"
+#define STATE_NODE_PREFIX "gcc/diagnostic_state_node/"
+#define STATE_EDGE_PREFIX "gcc/diagnostic_state_edge/"
+
+/* A wrapper around a node that gets/sets attributes, using
+   the node's property bag for storage, so that the data roundtrips
+   through SARIF.  */
+
+struct state_node_ref
+{
+  state_node_ref (diagnostics::digraphs::node &node)
+  : m_node (node)
+  {}
+
+  enum node_kind
+  get_node_kind () const;
+  void
+  set_node_kind (enum node_kind);
+
+  // For node_kind::stack_frame, this will be the function
+  logical_locations::key
+  get_logical_loc () const
+  {
+    return m_node.get_logical_loc ();
+  }
+
+  // For node_kind::dynalloc_buffer
+  enum node_dynalloc_state
+  get_dynalloc_state () const;
+
+  void
+  set_dynalloc_state (enum node_dynalloc_state) const;
+
+  const char *
+  get_dynamic_extents () const;
+
+  const char *
+  get_name () const { return get_attr ("name"); }
+  void
+  set_name (const char *name) const { set_attr ("name", name); }
+
+  const char *
+  get_type () const { return get_attr ("type"); }
+  void
+  set_type (const char *type) const { set_attr ("type", type); }
+
+  const char *
+  get_value () const { return get_attr ("value"); }
+
+  const char *
+  get_index () const { return get_attr ("index"); }
+
+  const char *
+  get_attr (const char *key) const
+  {
+    return m_node.get_attr (STATE_NODE_PREFIX, key);
+  }
+
+  void
+  set_attr (const char *key, const char *value) const
+  {
+    return m_node.set_attr (STATE_NODE_PREFIX, key, value);
+  }
+
+  void
+  set_json_attr (const char *key, std::unique_ptr<json::value> value) const;
+
+  diagnostics::digraphs::node &m_node;
+};
+
+extern std::unique_ptr<dot::graph>
+make_dot_graph (const diagnostics::digraphs::digraph &state_graph,
+		const logical_locations::manager &logical_loc_mgr);
+
+} // namespace state_graphs
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_STATE_GRAPHS_H */
diff --git a/gcc/diagnostic-format-text.cc b/gcc/diagnostics/text-sink.cc
index 7ecbe5f..bcf91cf 100644
--- a/gcc/diagnostic-format-text.cc
+++ b/gcc/diagnostics/text-sink.cc
@@ -25,14 +25,14 @@ along with GCC; see the file COPYING3.  If not see
 #include "version.h"
 #include "intl.h"
 #include "diagnostic.h"
-#include "diagnostic-color.h"
-#include "diagnostic-url.h"
-#include "diagnostic-metadata.h"
-#include "diagnostic-path.h"
-#include "diagnostic-client-data-hooks.h"
-#include "diagnostic-diagram.h"
-#include "diagnostic-format-text.h"
-#include "diagnostic-buffer.h"
+#include "diagnostics/color.h"
+#include "diagnostics/url.h"
+#include "diagnostics/metadata.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/client-data-hooks.h"
+#include "diagnostics/diagram.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/buffering.h"
 #include "text-art/theme.h"
 
 /* Disable warnings about quoting issues in the pp_xxx calls below
@@ -42,54 +42,55 @@ along with GCC; see the file COPYING3.  If not see
 #  pragma GCC diagnostic ignored "-Wformat-diag"
 #endif
 
-/* Concrete buffering implementation subclass for JSON output.  */
+namespace diagnostics {
 
-class diagnostic_text_format_buffer : public diagnostic_per_format_buffer
+/* Concrete buffering implementation subclass for text output.  */
+
+class text_sink_buffer : public per_sink_buffer
 {
 public:
-  friend class diagnostic_text_output_format;
+  friend class text_sink;
 
-  diagnostic_text_format_buffer (diagnostic_output_format &format);
+  text_sink_buffer (sink &sink_);
 
   void dump (FILE *out, int indent) const final override;
 
   bool empty_p () const final override;
-  void move_to (diagnostic_per_format_buffer &dest) final override;
+  void move_to (per_sink_buffer &dest) final override;
   void clear () final override;
   void flush () final override;
 
 private:
-  diagnostic_output_format &m_format;
+  sink &m_sink;
   output_buffer m_output_buffer;
 };
 
-/* class diagnostic_text_format_buffer : public diagnostic_per_format_buffer.  */
+/* class text_sink_buffer : public per_sink_buffer.  */
 
-diagnostic_text_format_buffer::
-diagnostic_text_format_buffer (diagnostic_output_format &format)
-: m_format (format)
+text_sink_buffer::text_sink_buffer (sink &sink_)
+: m_sink (sink_)
 {
   m_output_buffer.m_flush_p = false;
 }
 
 void
-diagnostic_text_format_buffer::dump (FILE *out, int indent) const
+text_sink_buffer::dump (FILE *out, int indent) const
 {
-  fprintf (out, "%*sdiagnostic_text_format_buffer:\n", indent, "");
+  fprintf (out, "%*stext_sink_buffer:\n", indent, "");
   m_output_buffer.dump (out, indent + 2);
 }
 
 bool
-diagnostic_text_format_buffer::empty_p () const
+text_sink_buffer::empty_p () const
 {
   return output_buffer_last_position_in_text (&m_output_buffer) == nullptr;
 }
 
 void
-diagnostic_text_format_buffer::move_to (diagnostic_per_format_buffer &base_dest)
+text_sink_buffer::move_to (per_sink_buffer &base_dest)
 {
-  diagnostic_text_format_buffer &dest
-    = static_cast<diagnostic_text_format_buffer &> (base_dest);
+  text_sink_buffer &dest
+    = static_cast<text_sink_buffer &> (base_dest);
   const char *str = output_buffer_formatted_text (&m_output_buffer);
   output_buffer_append_r (&dest.m_output_buffer, str, strlen (str));
 
@@ -99,9 +100,9 @@ diagnostic_text_format_buffer::move_to (diagnostic_per_format_buffer &base_dest)
 }
 
 void
-diagnostic_text_format_buffer::clear ()
+text_sink_buffer::clear ()
 {
-  pretty_printer *const pp = m_format.get_printer ();
+  pretty_printer *const pp = m_sink.get_printer ();
   output_buffer *const old_output_buffer = pp_buffer (pp);
 
   pp_buffer (pp) = &m_output_buffer;
@@ -113,9 +114,9 @@ diagnostic_text_format_buffer::clear ()
 }
 
 void
-diagnostic_text_format_buffer::flush ()
+text_sink_buffer::flush ()
 {
-  pretty_printer *const pp = m_format.get_printer ();
+  pretty_printer *const pp = m_sink.get_printer ();
   output_buffer *const old_output_buffer = pp_buffer (pp);
 
   pp_buffer (pp) = &m_output_buffer;
@@ -126,12 +127,12 @@ diagnostic_text_format_buffer::flush ()
   pp_buffer (pp) = old_output_buffer;
 }
 
-/* class diagnostic_text_output_format : public diagnostic_output_format.  */
+/* class diagnostics::text_sink : public diagnostics::sink.  */
 
-diagnostic_text_output_format::~diagnostic_text_output_format ()
+text_sink::~text_sink ()
 {
   /* Some of the errors may actually have been warnings.  */
-  if (m_context.diagnostic_count (DK_WERROR))
+  if (m_context.diagnostic_count (kind::werror))
     {
       pretty_printer *pp = get_printer ();
       /* -Werror was given.  */
@@ -155,13 +156,13 @@ diagnostic_text_output_format::~diagnostic_text_output_format ()
 }
 
 void
-diagnostic_text_output_format::dump (FILE *out, int indent) const
+text_sink::dump (FILE *out, int indent) const
 {
-  fprintf (out, "%*sdiagnostic_text_output_format\n", indent, "");
+  fprintf (out, "%*stext_sink\n", indent, "");
   fprintf (out, "%*sm_follows_reference_printer: %s\n",
 	   indent, "",
 	   m_follows_reference_printer ? "true" : "false");
-  diagnostic_output_format::dump (out, indent);
+  sink::dump (out, indent);
   fprintf (out, "%*ssaved_output_buffer:\n", indent + 2, "");
   if (m_saved_output_buffer)
     m_saved_output_buffer->dump (out, indent + 4);
@@ -170,10 +171,10 @@ diagnostic_text_output_format::dump (FILE *out, int indent) const
 }
 
 void
-diagnostic_text_output_format::set_buffer (diagnostic_per_format_buffer *base)
+text_sink::set_buffer (per_sink_buffer *base)
 {
-  diagnostic_text_format_buffer * const buffer
-    = static_cast<diagnostic_text_format_buffer *> (base);
+  text_sink_buffer * const buffer
+    = static_cast<text_sink_buffer *> (base);
 
   pretty_printer *const pp = get_printer ();
 
@@ -189,23 +190,22 @@ diagnostic_text_output_format::set_buffer (diagnostic_per_format_buffer *base)
     }
 }
 
-std::unique_ptr<diagnostic_per_format_buffer>
-diagnostic_text_output_format::make_per_format_buffer ()
+std::unique_ptr<per_sink_buffer>
+text_sink::make_per_sink_buffer ()
 {
-  return std::make_unique<diagnostic_text_format_buffer> (*this);
+  return std::make_unique<text_sink_buffer> (*this);
 }
 
-/* Implementation of diagnostic_output_format::on_report_diagnostic vfunc
+/* Implementation of diagnostics::sink::on_report_diagnostic vfunc
    for GCC's standard textual output.  */
 
 void
-diagnostic_text_output_format::
-on_report_diagnostic (const diagnostic_info &diagnostic,
-		      diagnostic_t orig_diag_kind)
+text_sink::on_report_diagnostic (const diagnostic_info &diagnostic,
+				 enum kind orig_diag_kind)
 {
   pretty_printer *pp = get_printer ();
 
-  (*diagnostic_text_starter (&m_context)) (*this, &diagnostic);
+  (*text_starter (&m_context)) (*this, &diagnostic);
 
   pp_output_formatted_text (pp, m_context.get_urlifier ());
 
@@ -242,23 +242,23 @@ on_report_diagnostic (const diagnostic_info &diagnostic,
 	}
     }
 
-  (*diagnostic_text_finalizer (&m_context)) (*this,
-					     &diagnostic,
-					     orig_diag_kind);
+  (*text_finalizer (&m_context)) (*this,
+				  &diagnostic,
+				  orig_diag_kind);
 
   if (m_show_nesting && m_show_locations_in_nesting)
     get_context ().m_last_location = diagnostic_location (&diagnostic);
 }
 
 void
-diagnostic_text_output_format::on_report_verbatim (text_info &text)
+text_sink::on_report_verbatim (text_info &text)
 {
   pp_format_verbatim (get_printer (), &text);
   pp_newline_and_flush (get_printer ());
 }
 
 void
-diagnostic_text_output_format::on_diagram (const diagnostic_diagram &diagram)
+text_sink::on_diagram (const diagnostics::diagram &d)
 {
   pretty_printer *const pp = get_printer ();
 
@@ -267,17 +267,17 @@ diagnostic_text_output_format::on_diagram (const diagnostic_diagram &diagram)
   /* Use a newline before and after and a two-space indent
      to make the diagram stand out a little from the wall of text.  */
   pp_newline (pp);
-  diagram.get_canvas ().print_to_pp (pp, "  ");
+  d.get_canvas ().print_to_pp (pp, "  ");
   pp_newline (pp);
   pp_set_prefix (pp, saved_prefix);
   pp_flush (pp);
 }
 
 void
-diagnostic_text_output_format::
+text_sink::
 after_diagnostic (const diagnostic_info &diagnostic)
 {
-  if (const diagnostic_path *path = diagnostic.richloc->get_path ())
+  if (const paths::path *path = diagnostic.m_richloc->get_path ())
     print_path (*path);
 }
 
@@ -289,16 +289,15 @@ after_diagnostic (const diagnostic_info &diagnostic)
 
    The caller is responsible for freeing the memory.  */
 char *
-diagnostic_text_output_format::
-build_prefix (const diagnostic_info &diagnostic) const
+text_sink::build_prefix (const diagnostic_info &diagnostic) const
 {
-  gcc_assert (diagnostic.kind < DK_LAST_DIAGNOSTIC_KIND);
+  gcc_assert (diagnostic.m_kind < kind::last_diagnostic_kind);
 
-  const char *text = _(get_diagnostic_kind_text (diagnostic.kind));
+  const char *text = _(get_text_for_kind (diagnostic.m_kind));
   const char *text_cs = "", *text_ce = "";
   pretty_printer *pp = get_printer ();
 
-  if (const char *color_name = diagnostic_get_color_for_kind (diagnostic.kind))
+  if (const char *color_name = get_color_for_kind (diagnostic.m_kind))
     {
       text_cs = colorize_start (pp_show_color (pp), color_name);
       text_ce = colorize_stop (pp_show_color (pp));
@@ -311,7 +310,7 @@ build_prefix (const diagnostic_info &diagnostic) const
 
       /* Reduce verbosity of nested diagnostics by not printing "note: "
 	 all the time.  */
-      if (diagnostic.kind == DK_NOTE)
+      if (diagnostic.m_kind == kind::note)
 	return indent_prefix;
 
       char *result = build_message_string ("%s%s%s%s", indent_prefix,
@@ -330,7 +329,7 @@ build_prefix (const diagnostic_info &diagnostic) const
 
 /* Same as build_prefix, but only the source FILE is given.  */
 char *
-diagnostic_text_output_format::file_name_as_prefix (const char *f) const
+text_sink::file_name_as_prefix (const char *f) const
 {
   pretty_printer *const pp = get_printer ();
   const char *locus_cs
@@ -354,7 +353,7 @@ get_bullet_point_unichar (bool unicode)
 /* Return true if DC's theme supports unicode characters.  */
 
 static bool
-use_unicode_p (const diagnostic_context &dc)
+use_unicode_p (const context &dc)
 {
   if (text_art::theme *theme = dc.get_diagram_theme ())
     return theme->unicode_p ();
@@ -366,7 +365,7 @@ use_unicode_p (const diagnostic_context &dc)
    nested diagnostics.  */
 
 static unsigned
-get_bullet_point_unichar (diagnostic_context &dc)
+get_bullet_point_unichar (context &dc)
 {
   return get_bullet_point_unichar (use_unicode_p (dc));
 }
@@ -382,7 +381,7 @@ get_bullet_point_unichar (diagnostic_context &dc)
    The caller is responsible for freeing the memory.  */
 
 char *
-diagnostic_text_output_format::build_indent_prefix (bool with_bullet) const
+text_sink::build_indent_prefix (bool with_bullet) const
 {
   if (!m_show_nesting)
     return xstrdup ("");
@@ -407,18 +406,18 @@ diagnostic_text_output_format::build_indent_prefix (bool with_bullet) const
 /* Add a purely textual note with text GMSGID and with LOCATION.  */
 
 void
-diagnostic_text_output_format::append_note (location_t location,
-					    const char * gmsgid, ...)
+text_sink::append_note (location_t location,
+			const char * gmsgid, ...)
 {
-  diagnostic_context *context = &get_context ();
+  context *dc = &get_context ();
 
   diagnostic_info diagnostic;
   va_list ap;
   rich_location richloc (line_table, location);
 
   va_start (ap, gmsgid);
-  diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, DK_NOTE);
-  if (context->m_inhibit_notes_p)
+  diagnostic_set_info (&diagnostic, gmsgid, &ap, &richloc, kind::note);
+  if (dc->m_inhibit_notes_p)
     {
       va_end (ap);
       return;
@@ -426,25 +425,24 @@ diagnostic_text_output_format::append_note (location_t location,
   pretty_printer *pp = get_printer ();
   char *saved_prefix = pp_take_prefix (pp);
   pp_set_prefix (pp, build_prefix (diagnostic));
-  pp_format (pp, &diagnostic.message);
+  pp_format (pp, &diagnostic.m_message);
   pp_output_formatted_text (pp);
   pp_destroy_prefix (pp);
   pp_set_prefix (pp, saved_prefix);
   pp_newline (pp);
-  diagnostic_show_locus (context, get_source_printing_options (),
-			 &richloc, DK_NOTE, pp);
+  diagnostic_show_locus (dc, get_source_printing_options (),
+			 &richloc, kind::note, pp);
   va_end (ap);
 }
 
 bool
-diagnostic_text_output_format::follows_reference_printer_p () const
+text_sink::follows_reference_printer_p () const
 {
   return m_follows_reference_printer;
 }
 
 void
-diagnostic_text_output_format::
-update_printer ()
+text_sink::update_printer ()
 {
   pretty_printer *copy_from_pp
     = (m_follows_reference_printer
@@ -469,18 +467,18 @@ update_printer ()
    description of the security issue.  */
 
 void
-diagnostic_text_output_format::print_any_cwe (const diagnostic_info &diagnostic)
+text_sink::print_any_cwe (const diagnostic_info &diagnostic)
 {
-  if (!diagnostic.metadata)
+  if (!diagnostic.m_metadata)
     return;
 
-  int cwe = diagnostic.metadata->get_cwe ();
+  int cwe = diagnostic.m_metadata->get_cwe ();
   if (cwe)
     {
       pretty_printer * const pp = get_printer ();
       char *saved_prefix = pp_take_prefix (pp);
       pp_string (pp, " [");
-      const char *kind_color = diagnostic_get_color_for_kind (diagnostic.kind);
+      const char *kind_color = get_color_for_kind (diagnostic.m_kind);
       pp_string (pp, colorize_start (pp_show_color (pp), kind_color));
       if (pp->supports_urls_p ())
 	{
@@ -504,23 +502,21 @@ diagnostic_text_output_format::print_any_cwe (const diagnostic_info &diagnostic)
    with any URL provided by the rule.  */
 
 void
-diagnostic_text_output_format::
-print_any_rules (const diagnostic_info &diagnostic)
+text_sink::print_any_rules (const diagnostic_info &diagnostic)
 {
-  if (!diagnostic.metadata)
+  if (!diagnostic.m_metadata)
     return;
 
-  for (unsigned idx = 0; idx < diagnostic.metadata->get_num_rules (); idx++)
+  for (unsigned idx = 0; idx < diagnostic.m_metadata->get_num_rules (); idx++)
     {
-      const diagnostic_metadata::rule &rule
-	= diagnostic.metadata->get_rule (idx);
+      const diagnostics::metadata::rule &rule
+	= diagnostic.m_metadata->get_rule (idx);
       if (char *desc = rule.make_description ())
 	{
 	  pretty_printer * const pp = get_printer ();
 	  char *saved_prefix = pp_take_prefix (pp);
 	  pp_string (pp, " [");
-	  const char *kind_color
-	    = diagnostic_get_color_for_kind (diagnostic.kind);
+	  const char *kind_color = get_color_for_kind (diagnostic.m_kind);
 	  pp_string (pp, colorize_start (pp_show_color (pp), kind_color));
 	  char *url = nullptr;
 	  if (pp->supports_urls_p ())
@@ -542,25 +538,23 @@ print_any_rules (const diagnostic_info &diagnostic)
     }
 }
 
-/* Print any metadata about the option used to control DIAGNOSTIC to CONTEXT's
-   printer, e.g. " [-Werror=uninitialized]".
-   Subroutine of diagnostic_context::report_diagnostic.  */
+/* Print any metadata about the option used to control DIAGNOSTIC to
+   the context's printer, e.g. " [-Werror=uninitialized]".  */
 
 void
-diagnostic_text_output_format::
-print_option_information (const diagnostic_info &diagnostic,
-			  diagnostic_t orig_diag_kind)
+text_sink::print_option_information (const diagnostic_info &diagnostic,
+				     enum kind orig_diag_kind)
 {
   if (char *option_text
-      = m_context.make_option_name (diagnostic.option_id,
-				    orig_diag_kind, diagnostic.kind))
+      = m_context.make_option_name (diagnostic.m_option_id,
+				    orig_diag_kind, diagnostic.m_kind))
     {
       char *option_url = nullptr;
       pretty_printer * const pp = get_printer ();
       if (pp->supports_urls_p ())
-	option_url = m_context.make_option_url (diagnostic.option_id);
+	option_url = m_context.make_option_url (diagnostic.m_option_id);
       pp_string (pp, " [");
-      const char *kind_color = diagnostic_get_color_for_kind (diagnostic.kind);
+      const char *kind_color = get_color_for_kind (diagnostic.m_kind);
       pp_string (pp, colorize_start (pp_show_color (pp), kind_color));
       if (option_url)
 	pp_begin_url (pp, option_url);
@@ -579,7 +573,7 @@ print_option_information (const diagnostic_info &diagnostic,
 /* Only dump the "In file included from..." stack once for each file.  */
 
 bool
-diagnostic_text_output_format::includes_seen_p (const line_map_ordinary *map)
+text_sink::includes_seen_p (const line_map_ordinary *map)
 {
   /* No include path for main.  */
   if (MAIN_FILE_P (map))
@@ -602,13 +596,12 @@ diagnostic_text_output_format::includes_seen_p (const line_map_ordinary *map)
 }
 
 label_text
-diagnostic_text_output_format::
-get_location_text (const expanded_location &s) const
+text_sink::get_location_text (const expanded_location &s) const
 {
-  diagnostic_column_policy column_policy (get_context ());
-  return column_policy.get_location_text (s,
-					  show_column_p (),
-					  pp_show_color (get_printer ()));
+  column_policy column_policy_ (get_context ());
+  return column_policy_.get_location_text (s,
+					   show_column_p (),
+					   pp_show_color (get_printer ()));
 }
 
 /* Helpers for writing lang-specific starters/finalizers for text output.  */
@@ -636,7 +629,7 @@ maybe_line_and_column (int line, int col)
 }
 
 void
-diagnostic_text_output_format::report_current_module (location_t where)
+text_sink::report_current_module (location_t where)
 {
   pretty_printer *pp = get_printer ();
   const line_map_ordinary *map = nullptr;
@@ -704,8 +697,8 @@ diagnostic_text_output_format::report_current_module (location_t where)
 }
 
 void
-default_diagnostic_text_starter (diagnostic_text_output_format &text_output,
-				 const diagnostic_info *diagnostic)
+default_text_starter (text_sink &text_output,
+		      const diagnostic_info *diagnostic)
 {
   text_output.report_current_module (diagnostic_location (diagnostic));
   pretty_printer *const pp = text_output.get_printer ();
@@ -713,9 +706,9 @@ default_diagnostic_text_starter (diagnostic_text_output_format &text_output,
 }
 
 void
-default_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
-				   const diagnostic_info *diagnostic,
-				   diagnostic_t)
+default_text_finalizer (text_sink &text_output,
+			const diagnostic_info *diagnostic,
+			enum kind)
 {
   pretty_printer *const pp = text_output.get_printer ();
   char *saved_prefix = pp_take_prefix (pp);
@@ -723,7 +716,7 @@ default_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
   pp_newline (pp);
   diagnostic_show_locus (&text_output.get_context (),
 			 text_output.get_source_printing_options (),
-			 diagnostic->richloc, diagnostic->kind, pp);
+			 diagnostic->m_richloc, diagnostic->m_kind, pp);
   pp_set_prefix (pp, saved_prefix);
   pp_flush (pp);
 }
@@ -731,3 +724,5 @@ default_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
 #if __GNUC__ >= 10
 #  pragma GCC diagnostic pop
 #endif
+
+} // namespace diagnostics
diff --git a/gcc/diagnostic-format-text.h b/gcc/diagnostics/text-sink.h
index 64f8e13..5c60976 100644
--- a/gcc/diagnostic-format-text.h
+++ b/gcc/diagnostics/text-sink.h
@@ -18,49 +18,51 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_FORMAT_TEXT_H
-#define GCC_DIAGNOSTIC_FORMAT_TEXT_H
+#ifndef GCC_DIAGNOSTICS_TEXT_SINK_H
+#define GCC_DIAGNOSTICS_TEXT_SINK_H
 
-#include "diagnostic-format.h"
+#include "diagnostics/sink.h"
 
-/* Subclass of diagnostic_output_format for classic text-based output
+namespace diagnostics {
+
+/* Subclass of diagnostics::sink for classic text-based output
    to stderr.
 
-   Uses diagnostic_context.m_text_callbacks to provide client-specific
+   Uses diagnostics::context.m_text_callbacks to provide client-specific
    textual output (e.g. include paths, macro expansions, etc).  */
 
-class diagnostic_text_output_format : public diagnostic_output_format
+class text_sink : public sink
 {
 public:
-  diagnostic_text_output_format (diagnostic_context &context,
-				 diagnostic_source_printing_options *source_printing = nullptr,
-				 bool follows_reference_printer = false)
-  : diagnostic_output_format (context),
+  text_sink (context &dc,
+	     source_printing_options *source_printing = nullptr,
+	     bool follows_reference_printer = false)
+  : sink (dc),
     m_saved_output_buffer (nullptr),
-    m_column_policy (context),
+    m_column_policy (dc),
     m_last_module (nullptr),
     m_includes_seen (nullptr),
     m_source_printing (source_printing
 		       ? *source_printing
-		       : context.m_source_printing),
+		       : dc.get_source_printing_options ()),
     m_follows_reference_printer (follows_reference_printer),
     m_show_nesting (false),
     m_show_nesting_levels (false)
   {}
-  ~diagnostic_text_output_format ();
+  ~text_sink ();
 
   void dump (FILE *out, int indent) const override;
 
-  std::unique_ptr<diagnostic_per_format_buffer>
-  make_per_format_buffer () final override;
-  void set_buffer (diagnostic_per_format_buffer *) final override;
+  std::unique_ptr<per_sink_buffer>
+  make_per_sink_buffer () final override;
+  void set_buffer (per_sink_buffer *) final override;
 
   void on_begin_group () override {}
   void on_end_group () override {}
   void on_report_diagnostic (const diagnostic_info &,
-			     diagnostic_t orig_diag_kind) override;
+			     enum kind orig_diag_kind) override;
   void on_report_verbatim (text_info &) final override;
-  void on_diagram (const diagnostic_diagram &diagram) override;
+  void on_diagram (const diagram &d) override;
   void after_diagnostic (const diagnostic_info &) override;
   bool machine_readable_stderr_p () const final override
   {
@@ -70,6 +72,13 @@ public:
 
   void update_printer () override;
 
+  void
+  report_global_digraph (const lazily_created<digraphs::digraph> &)
+    final override
+  {
+    // no-op for text
+  }
+
   /* Helpers for writing lang-specific starters/finalizers for text output.  */
   char *build_prefix (const diagnostic_info &) const;
   void report_current_module (location_t where);
@@ -81,15 +90,15 @@ public:
 
   char *build_indent_prefix (bool with_bullet) const;
 
-  void print_path (const diagnostic_path &path);
+  void print_path (const paths::path &path);
 
   bool show_column_p () const { return get_context ().m_show_column; }
 
-  const diagnostic_column_policy &get_column_policy () const
+  const column_policy &get_column_policy () const
   {
     return m_column_policy;
   }
-  diagnostic_location_print_policy get_location_print_policy () const;
+  location_print_policy get_location_print_policy () const;
 
   bool show_nesting_p () const { return m_show_nesting; }
   bool show_locations_in_nesting_p () const
@@ -109,11 +118,11 @@ public:
 
   label_text get_location_text (const expanded_location &s) const;
 
-  diagnostic_source_printing_options &get_source_printing_options ()
+  source_printing_options &get_source_printing_options ()
   {
     return m_source_printing;
   }
-  const diagnostic_source_printing_options &get_source_printing_options () const
+  const source_printing_options &get_source_printing_options () const
   {
     return m_source_printing;
   }
@@ -122,14 +131,14 @@ protected:
   void print_any_cwe (const diagnostic_info &diagnostic);
   void print_any_rules (const diagnostic_info &diagnostic);
   void print_option_information (const diagnostic_info &diagnostic,
-				 diagnostic_t orig_diag_kind);
+				 enum kind orig_diag_kind);
 
   bool includes_seen_p (const line_map_ordinary *map);
 
-  /* For handling diagnostic_buffer.  */
+  /* For handling diagnostics::buffer.  */
   output_buffer *m_saved_output_buffer;
 
-  diagnostic_column_policy m_column_policy;
+  column_policy m_column_policy;
 
   /* Used to detect when the input file stack has changed since last
      described.  */
@@ -139,10 +148,10 @@ protected:
      include path for.  */
   hash_set<location_t, false, location_hash> *m_includes_seen;
 
-  diagnostic_source_printing_options &m_source_printing;
+  source_printing_options &m_source_printing;
 
   /* If true, this is the initial default text output format created
-     when the diagnostic_context was created, and, in particular, before
+     when the diagnostics::context was created, and, in particular, before
      initializations of color and m_url_format.  Hence this should follow
      the dc's reference printer for these.
      If false, this text output was created after the dc was created, and
@@ -162,4 +171,6 @@ protected:
   bool m_show_nesting_levels;
 };
 
-#endif /* ! GCC_DIAGNOSTIC_FORMAT_TEXT_H */
+} // namespace diagnostics
+
+#endif /* ! GCC_DIAGNOSTICS_TEXT_SINK_H */
diff --git a/gcc/diagnostic-url.h b/gcc/diagnostics/url.h
index d54775b..89efa36 100644
--- a/gcc/diagnostic-url.h
+++ b/gcc/diagnostics/url.h
@@ -17,8 +17,8 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
-#ifndef GCC_DIAGNOSTIC_URL_H
-#define GCC_DIAGNOSTIC_URL_H
+#ifndef GCC_DIAGNOSTICS_URL_H
+#define GCC_DIAGNOSTICS_URL_H
 
 /* Whether to add URLs to diagnostics:
    - DIAGNOSTICS_URL_NO: never
@@ -49,4 +49,4 @@ const diagnostic_url_format URL_FORMAT_DEFAULT = URL_FORMAT_BEL;
 
 extern diagnostic_url_format determine_url_format (diagnostic_url_rule_t);
 
-#endif /* ! GCC_DIAGNOSTIC_URL_H */
+#endif /* ! GCC_DIAGNOSTICS_URL_H */
diff --git a/gcc/doc/analyzer.texi b/gcc/doc/analyzer.texi
index 4dd14c8..87a2a40 100644
--- a/gcc/doc/analyzer.texi
+++ b/gcc/doc/analyzer.texi
@@ -432,7 +432,7 @@ binop_svalue (mult_expr, initial_svalue(‘size_t’, decl_region(frame_region(â
 @subsection Analyzer Paths
 
 We need to explain to the user what the problem is, and to persuade them
-that there really is a problem.  Hence having a @code{diagnostic_path}
+that there really is a problem.  Hence having a @code{diagnostics::paths::path}
 isn't just an incidental detail of the analyzer; it's required.
 
 Paths ought to be:
@@ -491,7 +491,7 @@ and notes if it is infeasible.
 
 The above gives us a shortest feasible @code{exploded_path} through the
 @code{exploded_graph} (a list of @code{exploded_edge *}).  We use this
-@code{exploded_path} to build a @code{diagnostic_path} (a list of
+@code{exploded_path} to build a @code{diagnostics::paths::path} (a list of
 @strong{events} for the diagnostic subsystem) - specifically a
 @code{checker_path}.
 
@@ -624,17 +624,17 @@ which writes out the diagnostics in HTML form, and generates SVG state
 diagrams visualizing the state of memory at each event (inspired by the
 "ddd" debugger).  These can be seen by pressing 'j' and 'k' to single-step
 forward and backward through events.  Note that these SVG diagrams are
-created from an intermediate XML representation generated from
-@code{program_state} objects.  The XML representation can be easier to
+created from an intermediate SARIF directed graph representation generated from
+@code{program_state} objects.  The SARIF representation can be easier to
 read - for example, rather than storing the contents of memory via byte
 offsets, it uses fields for structs and element indexes for arrays,
 recursively.  However it is a different representation, and thus bugs could
 be hidden by this transformation.  Generating the SVG diagrams requires
 an invocation of "dot" per event, so it noticeably slows down diagnostic
-emission, hence the opt-in command-line flag.  The XML and ``dot''
+emission, hence the opt-in command-line flag.  The SARIF and ``dot''
 representations can be seen by @code{__analyzer_dump_xml} and
 @code{__analyzer_dump_dot} below (writing them to stderr), or by adding
-@code{show-state-diagrams-xml=yes} and
+@code{show-state-diagrams-sarif=yes} and
 @code{show-state-diagrams-dot-src=yes} to the html sink, which shows
 them within the generated HTML next to the generated SVG.
 
@@ -792,9 +792,9 @@ will emit a warning describing the state of the 2nd argument
 a name matching the 1st argument (which must be a string literal).
 This is for use when debugging, and may be of use in DejaGnu tests.
 
-@item __analyzer_dump_xml
+@item __analyzer_dump_sarif
 @smallexample
-__analyzer_dump_xml ();
+__analyzer_dump_sarif ();
 @end smallexample
 
 will dump the copious information about the analyzer's state each time it
diff --git a/gcc/doc/avr-mmcu.texi b/gcc/doc/avr-mmcu.texi
index feb7725..5efcc81 100644
--- a/gcc/doc/avr-mmcu.texi
+++ b/gcc/doc/avr-mmcu.texi
@@ -50,15 +50,15 @@
 
 @item @anchor{avrxmega2}avrxmega2
 ``XMEGA'' devices with more than 8@tie{}KiB and up to 64@tie{}KiB of program memory.
-@*@var{mcu}@tie{}= @code{atxmega8e5}, @code{atxmega16a4}, @code{atxmega16a4u}, @code{atxmega16c4}, @code{atxmega16d4}, @code{atxmega16e5}, @code{atxmega32a4}, @code{atxmega32a4u}, @code{atxmega32c3}, @code{atxmega32c4}, @code{atxmega32d3}, @code{atxmega32d4}, @code{atxmega32e5}, @code{avr64da28}, @code{avr64da32}, @code{avr64da48}, @code{avr64da64}, @code{avr64db28}, @code{avr64db32}, @code{avr64db48}, @code{avr64db64}, @code{avr64dd14}, @code{avr64dd20}, @code{avr64dd28}, @code{avr64dd32}, @code{avr64du28}, @code{avr64du32}, @code{avr64ea28}, @code{avr64ea32}, @code{avr64ea48}, @code{avr64sd28}, @code{avr64sd32}, @code{avr64sd48}.
+@*@var{mcu}@tie{}= @code{atxmega8e5}, @code{atxmega16a4}, @code{atxmega16a4u}, @code{atxmega16c4}, @code{atxmega16d4}, @code{atxmega16e5}, @code{atxmega32a4}, @code{atxmega32a4u}, @code{atxmega32c3}, @code{atxmega32c4}, @code{atxmega32d3}, @code{atxmega32d4}, @code{atxmega32e5}, @code{avr64da28}, @code{avr64da28s}, @code{avr64da32}, @code{avr64da32s}, @code{avr64da48}, @code{avr64da48s}, @code{avr64da64}, @code{avr64da64s}, @code{avr64db28}, @code{avr64db32}, @code{avr64db48}, @code{avr64db64}, @code{avr64dd14}, @code{avr64dd20}, @code{avr64dd28}, @code{avr64dd32}, @code{avr64du28}, @code{avr64du32}, @code{avr64ea28}, @code{avr64ea32}, @code{avr64ea48}, @code{avr64sd28}, @code{avr64sd32}, @code{avr64sd48}.
 
 @item @anchor{avrxmega3}avrxmega3
 ``XMEGA'' devices with up to 64@tie{}KiB of combined program memory and RAM, and with program memory visible in the RAM address space.
-@*@var{mcu}@tie{}= @code{attiny202}, @code{attiny204}, @code{attiny212}, @code{attiny214}, @code{attiny402}, @code{attiny404}, @code{attiny406}, @code{attiny412}, @code{attiny414}, @code{attiny416}, @code{attiny416auto}, @code{attiny417}, @code{attiny424}, @code{attiny426}, @code{attiny427}, @code{attiny804}, @code{attiny806}, @code{attiny807}, @code{attiny814}, @code{attiny816}, @code{attiny817}, @code{attiny824}, @code{attiny826}, @code{attiny827}, @code{attiny1604}, @code{attiny1606}, @code{attiny1607}, @code{attiny1614}, @code{attiny1616}, @code{attiny1617}, @code{attiny1624}, @code{attiny1626}, @code{attiny1627}, @code{attiny3214}, @code{attiny3216}, @code{attiny3217}, @code{attiny3224}, @code{attiny3226}, @code{attiny3227}, @code{atmega808}, @code{atmega809}, @code{atmega1608}, @code{atmega1609}, @code{atmega3208}, @code{atmega3209}, @code{atmega4808}, @code{atmega4809}, @code{avr16dd14}, @code{avr16dd20}, @code{avr16dd28}, @code{avr16dd32}, @code{avr16du14}, @code{avr16du20}, @code{avr16du28}, @code{avr16du32}, @code{avr16ea28}, @code{avr16ea32}, @code{avr16ea48}, @code{avr16eb14}, @code{avr16eb20}, @code{avr16eb28}, @code{avr16eb32}, @code{avr32da28}, @code{avr32da32}, @code{avr32da48}, @code{avr32db28}, @code{avr32db32}, @code{avr32db48}, @code{avr32dd14}, @code{avr32dd20}, @code{avr32dd28}, @code{avr32dd32}, @code{avr32du14}, @code{avr32du20}, @code{avr32du28}, @code{avr32du32}, @code{avr32ea28}, @code{avr32ea32}, @code{avr32ea48}, @code{avr32sd20}, @code{avr32sd28}, @code{avr32sd32}.
+@*@var{mcu}@tie{}= @code{attiny202}, @code{attiny204}, @code{attiny212}, @code{attiny214}, @code{attiny402}, @code{attiny404}, @code{attiny406}, @code{attiny412}, @code{attiny414}, @code{attiny416}, @code{attiny416auto}, @code{attiny417}, @code{attiny424}, @code{attiny426}, @code{attiny427}, @code{attiny804}, @code{attiny806}, @code{attiny807}, @code{attiny814}, @code{attiny816}, @code{attiny817}, @code{attiny824}, @code{attiny826}, @code{attiny827}, @code{attiny1604}, @code{attiny1606}, @code{attiny1607}, @code{attiny1614}, @code{attiny1616}, @code{attiny1617}, @code{attiny1624}, @code{attiny1626}, @code{attiny1627}, @code{attiny3214}, @code{attiny3216}, @code{attiny3217}, @code{attiny3224}, @code{attiny3226}, @code{attiny3227}, @code{atmega808}, @code{atmega809}, @code{atmega1608}, @code{atmega1609}, @code{atmega3208}, @code{atmega3209}, @code{atmega4808}, @code{atmega4809}, @code{avr16dd14}, @code{avr16dd20}, @code{avr16dd28}, @code{avr16dd32}, @code{avr16du14}, @code{avr16du20}, @code{avr16du28}, @code{avr16du32}, @code{avr16ea28}, @code{avr16ea32}, @code{avr16ea48}, @code{avr16eb14}, @code{avr16eb20}, @code{avr16eb28}, @code{avr16eb32}, @code{avr32da28}, @code{avr32da28s}, @code{avr32da32}, @code{avr32da32s}, @code{avr32da48}, @code{avr32da48s}, @code{avr32db28}, @code{avr32db32}, @code{avr32db48}, @code{avr32dd14}, @code{avr32dd20}, @code{avr32dd28}, @code{avr32dd32}, @code{avr32du14}, @code{avr32du20}, @code{avr32du28}, @code{avr32du32}, @code{avr32ea28}, @code{avr32ea32}, @code{avr32ea48}, @code{avr32sd20}, @code{avr32sd28}, @code{avr32sd32}.
 
 @item @anchor{avrxmega4}avrxmega4
 ``XMEGA'' devices with more than 64@tie{}KiB and up to 128@tie{}KiB of program memory.
-@*@var{mcu}@tie{}= @code{atxmega64a3}, @code{atxmega64a3u}, @code{atxmega64a4u}, @code{atxmega64b1}, @code{atxmega64b3}, @code{atxmega64c3}, @code{atxmega64d3}, @code{atxmega64d4}, @code{avr128da28}, @code{avr128da32}, @code{avr128da48}, @code{avr128da64}, @code{avr128db28}, @code{avr128db32}, @code{avr128db48}, @code{avr128db64}.
+@*@var{mcu}@tie{}= @code{atxmega64a3}, @code{atxmega64a3u}, @code{atxmega64a4u}, @code{atxmega64b1}, @code{atxmega64b3}, @code{atxmega64c3}, @code{atxmega64d3}, @code{atxmega64d4}, @code{avr128da28}, @code{avr128da28s}, @code{avr128da32}, @code{avr128da32s}, @code{avr128da48}, @code{avr128da48s}, @code{avr128da64}, @code{avr128da64s}, @code{avr128db28}, @code{avr128db32}, @code{avr128db48}, @code{avr128db64}.
 
 @item @anchor{avrxmega5}avrxmega5
 ``XMEGA'' devices with more than 64@tie{}KiB and up to 128@tie{}KiB of program memory and more than 64@tie{}KiB of RAM.
diff --git a/gcc/doc/cpp.texi b/gcc/doc/cpp.texi
index fc60792..cc1656f 100644
--- a/gcc/doc/cpp.texi
+++ b/gcc/doc/cpp.texi
@@ -3252,7 +3252,7 @@ helps people match the @samp{#endif} to the corresponding @samp{#ifdef}.
 Older programs sometimes put @var{MACRO} directly after the
 @samp{#endif} without enclosing it in a comment.  This is invalid code
 according to the C standard.  CPP accepts it with a warning.  It
-never affects which @samp{#ifndef} the @samp{#endif} matches.
+never affects which @samp{#ifdef} the @samp{#endif} matches.
 
 @findex #ifndef
 Sometimes you wish to use some code if a macro is @emph{not} defined.
diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi
index 70adf2d..224d619 100644
--- a/gcc/doc/extend.texi
+++ b/gcc/doc/extend.texi
@@ -5654,12 +5654,12 @@ You can specify the kind of interrupt to be handled by adding an optional
 parameter to the interrupt attribute like this:
 
 @smallexample
-void f (void) __attribute__ ((interrupt ("user")));
+void f (void) __attribute__ ((interrupt ("supervisor")));
 @end smallexample
 
-Permissible values for this parameter are @code{user}, @code{supervisor},
-and @code{machine}.  If there is no parameter, then it defaults to
-@code{machine}.
+Permissible values for this parameter are @code{supervisor},
+@code{machine}, and @code{rnmi}.  If there is no parameter, then it
+defaults to @code{machine}.
 
 @cindex @code{riscv_vector_cc} function attribute, RISC-V
 @item riscv_vector_cc
@@ -7138,11 +7138,9 @@ The @code{aligned} attribute can also be used for functions
 @cindex @code{counted_by} variable attribute
 @item counted_by (@var{count})
 The @code{counted_by} attribute may be attached to the C99 flexible array
-member, or a pointer field of a structure.  It indicates that the number
-of the elements of the array that is held by the flexible array member
-field, or is pointed to by the pointer field, is given by the field
-"@var{count}" in the same structure as the flexible array member or the
-pointer field.
+member of a structure.  It indicates that the number of the elements of the
+array is given by the field "@var{count}" in the same structure as the
+flexible array member.
 
 This attribute is available only in C for now.
 In C++ this attribute is ignored.
@@ -7163,22 +7161,8 @@ struct P @{
 @end smallexample
 
 @noindent
-specifies that the @code{array} is a flexible array member whose number
-of elements is given by the field @code{count} in the same structure.
-
-@smallexample
-struct PP @{
-  size_t count2;
-  char other1;
-  char *array2 __attribute__ ((counted_by (count2)));
-  int other2;
-@} *pp;
-@end smallexample
-
-@noindent
-specifies that the @code{array2} is an array that is pointed by the
-pointer field, and its number of elements is given by the field
-@code{count2} in the same structure.
+specifies that the @code{array} is a flexible array member whose number of
+elements is given by the field @code{count} in the same structure.
 
 The field that represents the number of the elements should have an
 integer type.  Otherwise, the compiler reports an error and ignores
@@ -7187,12 +7171,6 @@ the attribute.
 When the field that represents the number of the elements is assigned a
 negative integer value, the compiler treats the value as zero.
 
-The @code{counted_by} attribute is not allowed for a pointer to @code{void},
-a pointer to function, or a pointer to a structure or union that includes
-a flexible array member.  However, it is allowed for a pointer to
-non-void incomplete structure or union types, as long as the type could
-be completed before the first reference to the pointer.
-
 An explicit @code{counted_by} annotation defines a relationship between
 two objects, @code{p->array} and @code{p->count}, and there are the
 following requirements on the relationship between this pair:
@@ -7208,13 +7186,6 @@ available all the time.  This relationship must hold even after any of
 these related objects are updated during the program.
 @end itemize
 
-In addition to the above requirements, there is one more requirement
-between this pair if and only if @code{p->array} is an array that is
-pointed by the pointer field:
-
-@code{p->array} and @code{p->count} can only be changed by changing the
-whole structure at the same time.
-
 It's the programmer's responsibility to make sure the above requirements to
 be kept all the time.  Otherwise the compiler reports warnings and
 the results of the array bound sanitizer and the
@@ -7236,8 +7207,6 @@ In the above, @code{ref1} uses @code{val1} as the number of the elements in
 @code{p->array}, and @code{ref2} uses @code{val2} as the number of elements
 in @code{p->array}.
 
-Note, however, the above feature is not valid for the pointer field.
-
 @cindex @code{alloc_size} variable attribute
 @item alloc_size (@var{position})
 @itemx alloc_size (@var{position-1}, @var{position-2})
@@ -12806,6 +12775,7 @@ the two, as explained in the sections below.
 @menu
 * Global Register Variables::   Variables declared at global scope.
 * Local Register Variables::    Variables declared within a function.
+* Hard Register Constraints::   Operands forced into specific machine registers.
 @end menu
 
 @node Global Register Variables
@@ -13011,6 +12981,167 @@ with slightly different characteristics (@pxref{MIPS Coprocessors,,
 Defining coprocessor specifics for MIPS targets, gccint, 
 GNU Compiler Collection (GCC) Internals}).
 
+@node Hard Register Constraints
+@subsubsection Hard Register Constraints
+
+Similar to register @code{asm} but still distinct, hard register constraints
+are another way to force operands of inline @code{asm} into specific machine
+registers.  In contrast to register @code{asm} where a variable is bound to a
+machine register, a hard register constraint binds an @code{asm} operand to a
+machine register.  Assume in the following that @code{r4} is a general-purpose
+register, @code{f5} a floating-point register, and @code{v6} a vector register
+for some target.
+
+@smallexample
+int x;
+int y __attribute__ ((vector_size (16)));
+@dots{}
+asm ("some instructions"
+     : "=@{r4@}" (x)
+     : "@{f5@}" (42.0), "@{v6@}" (y));
+@end smallexample
+
+For the inline @code{asm}, variable @code{x} is bound to register @code{r4},
+and @code{y} is loaded to @code{v6}.  Furthermore, constant @code{42.0} is
+loaded into floating-point register @code{f5}.
+
+A key difference between register @code{asm} and hard register constraints is
+that the latter are specified at the point where they are supposed to
+materialize, namely at inline @code{asm}, which may lead to more readable code.
+
+@subsubheading Usage
+
+Each input operand is loaded into the register specified by its corresponding
+hard register constraint.  Furthermore, each hard register must be used at most
+once among an alternative for inputs.  This renders hard register constraints
+more strict compared to register @code{asm} where multiple inputs may share a
+register as for example in
+
+@smallexample
+int x;
+register int y asm ("0") = @dots{};
+asm ("" : "=r" (x) : "r" (y), "r" (y));
+@end smallexample
+
+or even
+
+@smallexample
+register int x asm ("0") = 42;
+register int y asm ("0") = 24;
+asm ("" : "=r" (x) : "r" (x), "r" (y));
+@end smallexample
+
+The analogue for hard register constraints is invalid in order to prevent
+subtle bugs.
+
+Likewise, two outputs must not share a register among an alternative.  That
+means, the following example is invalid
+
+@smallexample
+int x, y;
+asm ("" : "=@{r4@}" (x), "=@{r4@}" (y));  // invalid
+@end smallexample
+
+which also aligns with register @code{asm}.  Despite that, each output must
+refer to a distinct object if a hard register constraint is involved.  For
+example, in the following, object @code{x} is assigned two registers.
+
+@smallexample
+int x;
+asm ("" : "=r" (x), "=r" (x));
+@end smallexample
+
+This is not allowed for hard register constraints in order to prevent subtle
+bugs.  Even if only one output operand has a hard register constraint, the code
+is rejected since the allocation for the object is still ambiguous.
+
+@smallexample
+int x;
+asm ("" : "=r" (x), "=@{1@}" (x));  // invalid
+@end smallexample
+
+The type of an operand must be supported by the corresponding machine register.
+
+A hard register constraint may refer to any general, floating-point, or vector
+register except a fixed register as e.g.@: the stack-pointer register.  The set
+of allowed registers is target dependent analogue to register @code{asm}.
+Furthermore, the referenced register must be a valid register name of the
+target.  Note, on some targets, a single register may be referred to by
+different names where each name specifies the length of the register.  For
+example, on x86_64 the register names @code{rcx}, @code{ecx}, and @code{cx} all
+refer to the same register but in different sizes.  If any of those names is
+used for a hard register constraint, the actual size of a register is
+determined by its corresponding operand.  For example
+
+@smallexample
+long x;
+asm ("mov\t$42, %0" : "=@{ecx@}" (x));
+@end smallexample
+
+Although the hard register constraint refers to register @code{ecx}, the actual
+register will be @code{rcx} since on x86_64 a @code{long} is 8 byte in total.
+This aligns with register @code{asm} where you could have
+
+@smallexample
+register long x asm ("ecx");
+@end smallexample
+
+@subsubheading Interaction with Register @code{asm}
+
+A mixture of both constructs as for example
+
+@smallexample
+register int x asm ("r4") = 42;
+int y;
+asm ("" : "=@{r5@}" (y) : "r" (x));
+@end smallexample
+
+is valid.
+
+If an operand is a register @code{asm} and the corresponding constraint a hard
+register, then both must refer to the same register.  That means
+
+@smallexample
+register int x asm ("r4");
+asm ("" : "=@{r4@}" (x));
+@end smallexample
+
+is valid and
+
+@smallexample
+register int x asm ("r4");
+asm ("" : "=@{r5@}" (x));  // invalid
+@end smallexample
+
+is invalid.
+
+Note, register @code{asm} may not only be clobbered by function calls but also
+by inline @code{asm} in conjunction with hard register constraints.  For
+example, in the following
+
+@smallexample
+register int x asm ("r5") = 42;
+int y;
+asm ("" : "=@{r5@}" (y));
+asm ("" : "+r" (x));
+@end smallexample
+
+variable @code{x} materializes before the very first inline @code{asm} which
+writes to register @code{r5} and therefore clobbers @code{x} which in turn is
+read by the subsequent inline @code{asm}.
+
+@subsubheading Limitations
+
+At the moment fixed registers are not supported for hard register constraints.
+Thus, idioms like
+
+@smallexample
+register void *x asm ("rsp");
+asm ("" : "=r" (x));
+@end smallexample
+
+are not supported for hard register constraints.  This might be lifted.
+
 @node Size of an asm
 @subsection Size of an @code{asm}
 
@@ -15553,7 +15684,7 @@ are 128-bit.  Only supported on targets when 128-bit types are supported.
 Returns the calculated 8-bit bit-reversed CRC using the initial CRC (8-bit),
 data (8-bit) and the polynomial (8-bit).
 @var{crc} is the initial CRC, @var{data} is the data and
-@var{poly} is the polynomial without leading 1.
+@var{poly} is the polynomial without leading 1. @var{poly} is required to be a compile-time constant.
 Table-based or clmul-based CRC may be used for the
 calculation, depending on the target architecture.
 @enddefbuiltin
@@ -15608,7 +15739,7 @@ is 32-bit.
 Returns the calculated 8-bit bit-forward CRC using the initial CRC (8-bit),
 data (8-bit) and the polynomial (8-bit).
 @var{crc} is the initial CRC, @var{data} is the data and
-@var{poly} is the polynomial without leading 1.
+@var{poly} is the polynomial without leading 1. @var{poly} is required to be a compile-time constant.
 Table-based or clmul-based CRC may be used for the
 calculation, depending on the target architecture.
 @enddefbuiltin
diff --git a/gcc/doc/install.texi b/gcc/doc/install.texi
index 80ee2cd..f3f1445 100644
--- a/gcc/doc/install.texi
+++ b/gcc/doc/install.texi
@@ -302,7 +302,7 @@ released on 2017-05-06.
 The gcobol documentation is maintained as manpages using troff
 mdoc. GNU groff is required to convert them to PDF format.  Conversion
 to HTML is done with mandoc, available at
-@uref{http://mdocml.bsd.lv/}.
+@uref{https://mandoc.bsd.lv}.
 
 Because ISO COBOL defines strict requirements for numerical precision,
 gcobol requires hardware with 128-bit computation instructions.  This
@@ -2667,6 +2667,17 @@ target binutils supports @code{Intel CET} instructions and disabled
 otherwise.  In this case, the target libraries are configured to get
 additional @option{-fcf-protection} option.
 
+@item --enable-x86-64-mfentry
+@itemx --disable-x86-64-mfentry
+Enable @option {-mfentry} by default on x86-64 to put the profiling
+counter call, @code{__fentry__}, before the prologue so that @option{-pg}
+can be used with @option{-fshrink-wrap} which is enabled at @option{-O1}.
+This configure option is 64-bit only because @code{__fentry__} doesn't
+support PIC in 32-bit mode.
+
+@option{--enable-x86-64-mfentry=auto} is default.  @option{-mfentry} is
+enabled on Linux/x86-64 by default.
+
 @item --with-riscv-attribute=@samp{yes}, @samp{no} or @samp{default}
 Generate RISC-V attribute by default, in order to record extra build
 information in object.
@@ -5153,13 +5164,6 @@ respects, this target is the same as the
 @anchor{windows}
 @heading Microsoft Windows
 
-@subheading Intel 16-bit versions
-The 16-bit versions of Microsoft Windows, such as Windows 3.1, are not
-supported.
-
-However, the 32-bit port has limited support for Microsoft
-Windows 3.11 in the Win32s environment, as a target only.  See below.
-
 @subheading Intel 32-bit versions
 The 32-bit versions of Windows, including Windows 95, Windows NT, Windows
 XP, and Windows Vista, are supported by several different target
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 7640e7d..c1e708b 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -353,7 +353,7 @@ Objective-C and Objective-C++ Dialects}.
 -Wno-builtin-macro-redefined  -Wc90-c99-compat  -Wc99-c11-compat
 -Wc11-c23-compat  -Wc23-c2y-compat
 -Wc++-compat  -Wc++11-compat  -Wc++14-compat  -Wc++17-compat
--Wc++20-compat
+-Wc++20-compat  -Wc++26-compat
 -Wno-c++11-extensions  -Wno-c++14-extensions -Wno-c++17-extensions
 -Wno-c++20-extensions  -Wno-c++23-extensions
 -Wcalloc-transposed-args
@@ -433,7 +433,8 @@ Objective-C and Objective-C++ Dialects}.
 -Wunsuffixed-float-constants
 -Wunterminated-string-initialization
 -Wunused
--Wunused-but-set-parameter  -Wunused-but-set-variable
+-Wunused-but-set-parameter  -Wunused-but-set-parameter=@var{n}
+-Wunused-but-set-variable  -Wunused-but-set-variable=@var{n}
 -Wunused-const-variable  -Wunused-const-variable=@var{n}
 -Wunused-function  -Wunused-label  -Wunused-local-typedefs
 -Wunused-macros
@@ -914,7 +915,7 @@ Objective-C and Objective-C++ Dialects}.
 @emph{AVR Options} (@ref{AVR Options})
 @gccoptlist{-mmcu=@var{mcu}  -mabsdata  -maccumulate-args  -mcvt
 -mbranch-cost=@var{cost}  -mfuse-add=@var{level}  -mfuse-move=@var{level}
--mcall-prologues  -mgas-isr-prologues  -mint8  -mflmap
+-mfuse-move2  -mcall-prologues  -mgas-isr-prologues  -mint8  -mflmap
 -mdouble=@var{bits}  -mlong-double=@var{bits}  -mno-call-main
 -mn_flash=@var{size}  -mfract-convert-truncate  -mno-interrupts
 -mmain-is-OS_task  -mrelax  -mrmw  -mstrict-X  -mtiny-stack
@@ -3816,6 +3817,23 @@ Warn when a type with an ABI tag is used in a context that does not
 have that ABI tag.  See @ref{C++ Attributes} for more information
 about ABI tags.
 
+@opindex Wabbreviated-auto-in-template-arg
+@opindex Wno-abbreviated-auto-in-template-arg
+@item -Wno-abbreviated-auto-in-template-arg
+Disable the error for an @code{auto} placeholder type used within a
+template argument list to declare a C++20 abbreviated function
+template, e.g.
+
+@smallexample
+void f(S<auto>);
+@end smallexample
+
+This feature was proposed in the Concepts TS, but was not adopted into
+C++20; in the standard, a placeholder in a parameter declaration must
+appear as a decl-specifier.  The error can also be reduced to a
+warning by @option{-fpermissive} or
+@option{-Wno-error=abbreviated-auto-in-template-arg}.
+
 @opindex Wcomma-subscript
 @opindex Wno-comma-subscript
 @item -Wcomma-subscript @r{(C++ and Objective-C++ only)}
@@ -6126,10 +6144,10 @@ end-users, and subject to change or removal without notice:
 
 @table @gcctabopt
 
-@item xml-state=@r{[}yes@r{|}no@r{]}
+@item state-graphs=@r{[}yes@r{|}no@r{]}
 This is a debugging feature and defaults to @code{no}.
-If @code{xml-state=yes}, then attempt to capture detailed state information
-from @option{-fanalyzer} in the generated SARIF.
+If @code{state-graphs=yes}, then attempt to capture detailed state
+information from @option{-fanalyzer} in the generated SARIF.
 
 @end table
 
@@ -6177,11 +6195,11 @@ then if @code{show-state-diagrams=yes},
 the generated state diagrams will also show the .dot source input to
 GraphViz used for the diagram.
 
-@item show-state-diagrams-xml=@r{[}yes@r{|}no@r{]}
+@item show-state-diagrams-sarif=@r{[}yes@r{|}no@r{]}
 This is a debugging feature and defaults to @code{no}.
-If @code{show-state-diagrams-xml=yes}
+If @code{show-state-diagrams-sarif=yes}
 then if @code{show-state-diagrams=yes}, the generated state diagrams will
-also show an XML representation of the state.
+also show a SARIF representation of the state.
 
 @end table
 
@@ -6443,6 +6461,7 @@ only by this flag, but it also downgrades some C and C++ diagnostics
 that have their own flag:
 
 @gccoptlist{
+-Wabbreviated-auto-in-template-arg @r{(C++ and Objective-C++ only)}
 -Wdeclaration-missing-parameter-type @r{(C and Objective-C only)}
 -Wimplicit-function-declaration @r{(C and Objective-C only)}
 -Wimplicit-int @r{(C and Objective-C only)}
@@ -7903,27 +7922,89 @@ statement.
 @opindex Wunused-but-set-parameter
 @opindex Wno-unused-but-set-parameter
 @item -Wunused-but-set-parameter
+@option{-Wunused-but-set-parameter} is the same as
+@option{-Wunused-but-set-parameter=3} and
+@option{-Wno-unused-but-set-parameter} is the same as
+@option{-Wunused-but-set-parameter=0}.
+
+@opindex Wunused-but-set-parameter=
+@item -Wunused-but-set-parameter=@var{n}
 Warn whenever a function parameter is assigned to, but otherwise unused
 (aside from its declaration).
 
 To suppress this warning use the @code{unused} attribute
 (@pxref{Variable Attributes}).
 
-This warning is also enabled by @option{-Wunused} together with
-@option{-Wextra}.
+@option{-Wunused-but-set-parameter=0} disables the warning.
+With @option{-Wunused-but-set-parameter=1} all uses except initialization
+and left hand side of assignment which is not further used disable the
+warning.
+With @option{-Wunused-but-set-parameter=2} additionally uses of parameter
+in @code{++} and @code{--} operators don't count as uses.
+And finally with @option{-Wunused-but-set-parameter=3} additionally
+uses in @var{parm} @code{@var{@@}=} @var{rhs} outside of @var{rhs} don't
+count as uses.  See @option{-Wunused-but-set-variable=@var{n}} option for
+examples.
+
+This @option{-Wunused-but-set-parameter=3} warning is also enabled by
+@option{-Wunused} together with @option{-Wextra}.
 
 @opindex Wunused-but-set-variable
 @opindex Wno-unused-but-set-variable
 @item -Wunused-but-set-variable
+@option{-Wunused-but-set-variable} is the same as
+@option{-Wunused-but-set-variable=3} and
+@option{-Wno-unused-but-set-variable} is the same as
+@option{-Wunused-but-set-variable=0}.
+
+@opindex Wunused-but-set-variable=
+@item -Wunused-but-set-variable=@var{n}
 Warn whenever a local variable is assigned to, but otherwise unused
 (aside from its declaration).
-This warning is enabled by @option{-Wall}.
+This @option{-Wunused-but-set-variable=3} warning is enabled by @option{-Wall}.
 
 To suppress this warning use the @code{unused} attribute
 (@pxref{Variable Attributes}).
 
-This warning is also enabled by @option{-Wunused}, which is enabled
-by @option{-Wall}.
+@option{-Wunused-but-set-variable=0} disables the warning.
+With @option{-Wunused-but-set-variable=1} all uses except initialization
+and left hand side of assignment which is not further used disable the
+warning.
+With @option{-Wunused-but-set-variable=2} additionally uses of variable
+in @code{++} and @code{--} operators don't count as uses.
+And finally with @option{-Wunused-but-set-variable=3} additionally
+uses in @var{parm} @code{@var{@@}=} @var{rhs} outside of @var{rhs} don't
+count as uses.
+
+This @option{-Wunused-but-set-variable=3} warning is also enabled by
+@option{-Wunused}, which is enabled by @option{-Wall}.
+
+@smallexample
+void foo (void)
+@{
+  int a = 1; // @option{-Wunused-variable} warning
+  int b = 0; // Warning for @var{n} >= 1
+  b = 1; b = 2;
+  int c = 0; // Warning for @var{n} >= 2
+  ++c; c--; --c; c++;
+  int d = 0; // Warning for @var{n} >= 3
+  d += 4;
+  int e = 0; // No warning, cast to void
+  (void) e;
+  int f = 0; // No warning, f used
+  int g = f = 5;
+  (void) g;
+  int h = 0; // No warning, preincrement used
+  int i = ++h;
+  (void) i;
+  int j = 0; // No warning, postdecrement used
+  int k = j--;
+  (void) k;
+  int l = 0; // No warning, l used
+  int m = l |= 2;
+  (void) m;
+@}
+@end smallexample
 
 @opindex Wunused-function
 @opindex Wno-unused-function
@@ -9672,6 +9753,12 @@ and ISO C++ 2017.  This warning is enabled by @option{-Wall}.
 Warn about C++ constructs whose meaning differs between ISO C++ 2017
 and ISO C++ 2020.  This warning is enabled by @option{-Wall}.
 
+@opindex Wc++26-compat
+@opindex Wno-c++26-compat
+@item -Wc++26-compat @r{(C++ and Objective-C++ only)}
+Warn about C++ constructs whose meaning differs between ISO C++ 2023
+and upcoming ISO C++ 2026.  This warning is enabled by @option{-Wall}.
+
 @opindex Wc++11-extensions
 @opindex Wno-c++11-extensions
 @item -Wno-c++11-extensions @r{(C++ and Objective-C++ only)}
@@ -25023,6 +25110,10 @@ Valid values for @var{level} are in the range @code{0} @dots{} @code{23}
 which is a 3:2:2:2 mixed radix value.  Each digit controls some
 aspect of the optimization.
 
+@opindex mfuse-move2
+@item -mfuse-move2
+Run a post combine optimization pass that tries to fuse move instructions.
+
 @opindex mstrict-X
 @item -mstrict-X
 Use address register @code{X} in a way proposed by the hardware.  This means
@@ -31283,31 +31374,14 @@ When the RISC-V specifications define an extension as depending on other
 extensions, GCC will implicitly add the dependent extensions to the enabled
 extension set if they weren't added explicitly.
 
-@opindex mcpu
-@item -mcpu=@var{processor-string}
-Use architecture of and optimize the output for the given processor, specified
-by particular CPU name.
-Permissible values for this option are: @samp{mips-p8700}, @samp{sifive-e20},
-@samp{sifive-e21}, @samp{sifive-e24}, @samp{sifive-e31}, @samp{sifive-e34},
-@samp{sifive-e76}, @samp{sifive-s21}, @samp{sifive-s51}, @samp{sifive-s54},
-@samp{sifive-s76}, @samp{sifive-u54}, @samp{sifive-u74}, @samp{sifive-x280},
-@samp{sifive-xp450}, @samp{sifive-x670}, @samp{thead-c906}, @samp{tt-ascalon-d8},
-@samp{xiangshan-nanhu}, @samp{xiangshan-kunminghu}, @samp{xt-c908}, @samp{xt-c908v},
-@samp{xt-c910}, @samp{xt-c910v2}, @samp{xt-c920}, @samp{xt-c920v2}.
+@include riscv-mcpu.texi
 
 Note that @option{-mcpu} does not override @option{-march} or @option{-mtune}.
 
-@opindex mtune
-@item -mtune=@var{processor-string}
-Optimize the output for the given processor, specified by microarchitecture or
-particular CPU name.  Permissible values for this option are:
-@samp{generic-ooo}, @samp{mips-p8700}, @samp{rocket}, @samp{sifive-3-series},
-@samp{sifive-5-series}, @samp{sifive-7-series}, @samp{size},
-@samp{sifive-p400-series}, @samp{sifive-p600-series}, and all valid options for
-@option{-mcpu=}.
+@include riscv-mtune.texi
 
 When @option{-mtune=} is not specified, use the setting from @option{-mcpu},
-the default is @samp{rocket} if both are not specified.
+the default is @samp{generic} if both are not specified.
 
 The @samp{size} choice is not intended for use by end-users.  This is used
 when @option{-Os} is specified.  It overrides the instruction cost info
@@ -34868,9 +34942,9 @@ Intel Panther Lake CPU with 64-bit extensions, MOVBE, MMX, SSE, SSE2, SSE3,
 SSSE3, SSE4.1, SSE4.2, POPCNT, AES, PREFETCHW, PCLMUL, RDRND, XSAVE, XSAVEC,
 XSAVES, XSAVEOPT, FSGSBASE, PTWRITE, RDPID, SGX, GFNI-SSE, CLWB, MOVDIRI,
 MOVDIR64B, WAITPKG, ADCX, AVX, AVX2, BMI, BMI2, F16C, FMA, LZCNT, PCONFIG, PKU,
-VAES, VPCLMULQDQ, SERIALIZE, HRESET, KL, WIDEKL, AVX-VNNI, UINTR, AVXIFMA,
-AVXVNNIINT8, AVXNECONVERT, CMPCCXADD, AVXVNNIINT16, SHA512, SM3, SM4 and
-PREFETCHI instruction set support.
+VAES, VPCLMULQDQ, SERIALIZE, HRESET, AVX-VNNI, UINTR, AVXIFMA, AVXVNNIINT8,
+AVXNECONVERT, CMPCCXADD, AVXVNNIINT16, SHA512, SM3, SM4 and PREFETCHI
+instruction set support.
 
 @item sapphirerapids
 @itemx emeraldrapids
@@ -34973,9 +35047,9 @@ Intel Clearwater Forest CPU with 64-bit extensions, MOVBE, MMX, SSE, SSE2,
 SSE3, SSSE3, SSE4.1, SSE4.2, POPCNT, AES, PREFETCHW, PCLMUL, RDRND, XSAVE,
 XSAVEC, XSAVES, XSAVEOPT, FSGSBASE, PTWRITE, RDPID, SGX, GFNI-SSE, CLWB,
 MOVDIRI, MOVDIR64B, CLDEMOTE, WAITPKG, ADCX, AVX, AVX2, BMI, BMI2, F16C, FMA,
-LZCNT, PCONFIG, PKU, VAES, VPCLMULQDQ, SERIALIZE, HRESET, KL, WIDEKL, AVX-VNNI,
-ENQCMD, UINTR, AVXIFMA, AVXVNNIINT8, AVXNECONVERT, CMPCCXADD, AVXVNNIINT16,
-SHA512, SM3, SM4, USER_MSR and PREFETCHI instruction set support.
+LZCNT, PCONFIG, PKU, VAES, VPCLMULQDQ, SERIALIZE, HRESET, AVX-VNNI, ENQCMD,
+UINTR, AVXIFMA, AVXVNNIINT8, AVXNECONVERT, CMPCCXADD, AVXVNNIINT16, SHA512,
+SM3, SM4, USER_MSR and PREFETCHI instruction set support.
 
 @item k6
 AMD K6 CPU with MMX instruction set support.
diff --git a/gcc/doc/libgdiagnostics/topics/compatibility.rst b/gcc/doc/libgdiagnostics/topics/compatibility.rst
index 10adcc5..0ca41a3 100644
--- a/gcc/doc/libgdiagnostics/topics/compatibility.rst
+++ b/gcc/doc/libgdiagnostics/topics/compatibility.rst
@@ -178,6 +178,8 @@ acccessing values within a :type:`diagnostic_logical_location`:
 
   * :func:`diagnostic_logical_location_get_decorated_name`
 
+.. _LIBGDIAGNOSTICS_ABI_2:
+
 ``LIBGDIAGNOSTICS_ABI_2``
 -------------------------
 ``LIBGDIAGNOSTICS_ABI_2`` covers the addition of these functions for
@@ -186,3 +188,77 @@ supporting command-line options and SARIF playback:
   * :func:`diagnostic_manager_add_sink_from_spec`
 
   * :func:`diagnostic_manager_set_analysis_target`
+
+.. _LIBGDIAGNOSTICS_ABI_3:
+
+``LIBGDIAGNOSTICS_ABI_3``
+-------------------------
+
+``LIBGDIAGNOSTICS_ABI_3`` covers the addition of these functions for
+working with directed graphs:
+
+  * :func:`diagnostic_manager_new_graph`
+
+  * :func:`diagnostic_manager_take_global_graph`
+
+  * :func:`diagnostic_take_graph`
+
+  * :func:`diagnostic_graph_release`
+
+  * :func:`diagnostic_graph_set_description`
+
+  * :func:`diagnostic_graph_add_node`
+
+  * :func:`diagnostic_graph_add_edge`
+
+  * :func:`diagnostic_graph_get_node_by_id`
+
+  * :func:`diagnostic_graph_get_edge_by_id`
+
+  * :func:`diagnostic_node_set_label`
+
+  * :func:`diagnostic_node_set_location`
+
+  * :func:`diagnostic_node_set_logical_location`
+
+.. _LIBGDIAGNOSTICS_ABI_4:
+
+``LIBGDIAGNOSTICS_ABI_4``
+-------------------------
+
+``LIBGDIAGNOSTICS_ABI_4`` covers the addition of these functions for
+working with :type:`diagnostic_message_buffer`.
+
+  * :func:`diagnostic_message_buffer_new`
+
+  * :func:`diagnostic_message_buffer_release`
+
+  * :func:`diagnostic_message_buffer_append_str`
+
+  * :func:`diagnostic_message_buffer_append_text`
+
+  * :func:`diagnostic_message_buffer_append_byte`
+
+  * :func:`diagnostic_message_buffer_append_printf`
+
+  * :func:`diagnostic_message_buffer_append_event_id`
+
+  * :func:`diagnostic_message_buffer_begin_url`
+
+  * :func:`diagnostic_message_buffer_end_url`
+
+  * :func:`diagnostic_message_buffer_begin_quote`
+
+  * :func:`diagnostic_message_buffer_end_quote`
+
+  * :func:`diagnostic_message_buffer_begin_color`
+
+  * :func:`diagnostic_message_buffer_end_color`
+
+  * :func:`diagnostic_message_buffer_dump`
+
+  * :func:`diagnostic_finish_via_msg_buf`
+
+  * :func:`diagnostic_add_location_with_label_via_msg_buf`
+
+  * :func:`diagnostic_execution_path_add_event_via_msg_buf`
diff --git a/gcc/doc/libgdiagnostics/topics/diagnostic-manager.rst b/gcc/doc/libgdiagnostics/topics/diagnostic-manager.rst
index 0390704..c94d19e 100644
--- a/gcc/doc/libgdiagnostics/topics/diagnostic-manager.rst
+++ b/gcc/doc/libgdiagnostics/topics/diagnostic-manager.rst
@@ -63,17 +63,17 @@ Responsibilities include:
 				       diagnostic_manager *control_mgr)
 
    This function can be used to support option processing similar to GCC's
-   :option:`-fdiagnostics-add-output=`.  This allows command-line tools to
-   support the same domain-specific language for specifying output sink
-   as GCC does.
-
-   The function will attempt to parse :param:`spec` as if it were
-   an argument to GCC's :option:`-fdiagnostics-add-output=OUTPUT-SPEC`.
-   If successful, it will add an output sink to :param:`affected_mgr` and return zero.
-   Otherwise, it will emit an error diagnostic to :param:`control_mgr` and
+   `-fdiagnostics-add-output= <https://gcc.gnu.org/onlinedocs/gcc/Diagnostic-Message-Formatting-Options.html#index-fdiagnostics-add-output>`_.
+   This allows command-line tools to support the same domain-specific
+   language for specifying output sinks as GCC does.
+
+   The function will attempt to parse ``spec`` as if it were
+   an argument to GCC's `-fdiagnostics-add-output= <https://gcc.gnu.org/onlinedocs/gcc/Diagnostic-Message-Formatting-Options.html#index-fdiagnostics-add-output>`_.
+   If successful, it will add an output sink to ``affected_mgr`` and return zero.
+   Otherwise, it will emit an error diagnostic to ``control_mgr`` and
    return non-zero.
 
-   :param:`affected_mgr` and :param:`control_mgr` can be the same manager,
+   ``affected_mgr`` and ``control_mgr`` can be the same manager,
    or be different managers.
 
    This function was added in :ref:`LIBGDIAGNOSTICS_ABI_2`; you can
@@ -83,14 +83,14 @@ Responsibilities include:
 
       #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_manager_add_sink_from_spec
 
-
 .. function:: void diagnostic_manager_set_analysis_target (diagnostic_manager *mgr, \
 	                                                   const diagnostic_file *file)
 
-   This function sets the "main input file" of :param:`mgr` to be
-   :param:`file`.
+   This function sets the "main input file" of ``mgr`` to be
+   ``file``.
    This affects the :code:`<title>` of generated HTML and
-   the :code:`role` of the artifact in SARIF output (SARIF v2.1.0 section 3.24.6).
+   the :code:`role` of the :code:`artifact` in SARIF output
+   (`SARIF v2.1.0 section 3.24.6 <https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/sarif-v2.1.0-errata01-os-complete.html#_Toc141790867>`_).
 
    This function was added in :ref:`LIBGDIAGNOSTICS_ABI_2`; you can
    test for its presence using
diff --git a/gcc/doc/libgdiagnostics/topics/diagnostics.rst b/gcc/doc/libgdiagnostics/topics/diagnostics.rst
index 3d24da0..7454c6e 100644
--- a/gcc/doc/libgdiagnostics/topics/diagnostics.rst
+++ b/gcc/doc/libgdiagnostics/topics/diagnostics.rst
@@ -105,6 +105,24 @@ Diagnostics are
 
    All three parameters must be non-NULL.
 
+.. function:: void diagnostic_finish_via_msg_buf (diagnostic *diag, \
+			       diagnostic_message_buffer *msg_buf)
+
+   This is equivalent to :func:`diagnostic_finish`, but using a message
+   buffer rather than a format string and variadic arguments.
+
+   ``diag`` and ``msg_buf`` must both be non-NULL.
+
+   Calling this function transfers ownership of ``msg_buf`` to the
+   diagnostic - do not call :func:`diagnostic_message_buffer_release` on
+   it.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
 
 Diagnostic groups
 *****************
diff --git a/gcc/doc/libgdiagnostics/topics/execution-paths.rst b/gcc/doc/libgdiagnostics/topics/execution-paths.rst
index 321503f..8381c45 100644
--- a/gcc/doc/libgdiagnostics/topics/execution-paths.rst
+++ b/gcc/doc/libgdiagnostics/topics/execution-paths.rst
@@ -88,6 +88,28 @@ cross-references between events.  In particular FIXME
    Equivalent to :func:`diagnostic_execution_path_add_event`, but using a
    :type:`va_list` rather than directly taking variadic arguments.
 
+.. function:: diagnostic_event_id diagnostic_execution_path_add_event_via_msg_buf (diagnostic_execution_path *path, \
+						 const diagnostic_physical_location *physical_loc, \
+						 const diagnostic_logical_location *logical_loc, \
+						 unsigned stack_depth,
+						 diagnostic_message_buffer *msg_buf)
+
+   This is equivalent to :func:`diagnostic_execution_path_add_event` but
+   using a message buffer rather than a format string and variadic
+   arguments.
+
+   ``path`` and ``msg_buf`` must both be non-NULL.
+
+   Calling this function transfers ownership of ``msg_buf`` to the
+   path - do not call :func:`diagnostic_message_buffer_release` on it.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
 Paths are printed to text sinks, and for SARIF sinks each path is added as
 a ``codeFlow`` object (see SARIF 2.1.0
 `§3.36 codeFlow object <https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/sarif-v2.1.0-errata01-os-complete.html#_Toc141790990>`_).
diff --git a/gcc/doc/libgdiagnostics/topics/graphs.rst b/gcc/doc/libgdiagnostics/topics/graphs.rst
new file mode 100644
index 0000000..b976013
--- /dev/null
+++ b/gcc/doc/libgdiagnostics/topics/graphs.rst
@@ -0,0 +1,197 @@
+.. Copyright (C) 2025 Free Software Foundation, Inc.
+   Originally contributed by David Malcolm <dmalcolm@redhat.com>
+
+   This is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see
+   <https://www.gnu.org/licenses/>.
+
+.. default-domain:: c
+
+Graphs
+======
+
+.. type:: diagnostic_graph
+
+SARIF has support for capturing directed graphs (such as callgraphs
+and control flow graphs), both at the level of the run as a whole,
+and at the level of individual results.
+
+libgdiagnostics supports this with the following entrypoints, allowing
+directed graphs to be
+
+* created (with :func:`diagnostic_manager_new_graph`)
+
+* reported "globally" (with :func:`diagnostic_manager_take_global_graph`)
+
+* reported as part of a :type:`diagnostic` (with :func:`diagnostic_take_graph`), or
+
+* discarded (with :func:`diagnostic_graph_release`).
+
+.. function:: diagnostic_graph * diagnostic_manager_new_graph (diagnostic_manager *manager)
+
+   Create a new directed graph.
+
+   The resulting graph is owned by the caller and must have one of
+   :func:`diagnostic_manager_take_global_graph`,
+   :func:`diagnostic_take_graph`,
+   or :func:`diagnostic_graph_release` called on it to avoid leaks.
+
+   The parameter ``manager`` must be non-null.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: void diagnostic_manager_take_global_graph (diagnostic_manager *manager, \
+				      diagnostic_graph *graph)
+
+   Report this graph "globally", taking ownership of it.
+   This won't appear in text sinks, but in SARIF sinks the graph will be
+   added to theRun.graphs (SARIF v2.1.0 3.14.20).
+
+   Parameters ``manager`` and ``graph`` must both be non-null.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: void diagnostic_take_graph (diagnostic *diag, \
+		      diagnostic_graph *graph)
+
+   Add this graph to ``diag``, transferring ownership of it to ``diag``.
+   This won't appear in text sinks, but in SARIF sinks the graph will be
+   added to theResult.graphs (SARIF v2.1.0 3.27.19).
+
+   Parameters ``diag`` and ``graph`` must both be non-null.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: void diagnostic_graph_release (diagnostic_graph *graph)
+
+   Release ``graph`` which must still be owned by the caller
+   i.e. it must *not* have had
+   :func:`diagnostic_manager_take_global_graph` or
+   :func:`diagnostic_take_graph` called on it.
+
+   Parameters ``graph`` can be null.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+
+.. function:: void diagnostic_graph_set_description (diagnostic_graph *graph, \
+				 const char *description)
+
+   Set the description of ``graph`` for use in the value of the
+   SARIF ``description`` property (SARIF v2.1.0 section 3.39.2).
+
+   The parameter ``graph`` must be non-null.
+   The parameter ``description`` can be null, for clearing any existing
+   description.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: diagnostic_node * diagnostic_graph_add_node (diagnostic_graph *graph, \
+			   const char *node_id, \
+			   diagnostic_node *parent_node)
+
+   Create and add a new node within ``graph`` with the given `id``.
+   The id must be unique within nodes in ``graph``.
+
+   The parameters ``graph`` and ``id`` must be non-null.
+
+   ``parent_node`` can be NULL (for a top-level node in the graph),
+   or non-null for a child node, allowing for arbitrary nesting of
+   nodes.
+
+   The new node is owned by ``graph``.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: diagnostic_edge * diagnostic_graph_add_edge (diagnostic_graph *graph, \
+			   const char *edge_id, \
+			   diagnostic_node *src_node, \
+			   diagnostic_node *dst_node, \
+			   const char *label)
+
+   Create and add a new edge within ``graph``.
+
+   The parameters ``graph``, ``src_node`` and ``dest_node``
+   must be non-null.
+
+   If non-null, then ``edge_id`` must be unique within ``graph``;
+   if ``edge_id`` is null then a unique id of the form "edge0", "edge1",
+   etc will be used automatically.
+
+   If non-null, then ``label`` will be used for the value of the
+   SARIF ``label`` property (SARIF v2.1.0 section 3.41.3).
+
+   The new edge is owned by ``graph``.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: diagnostic_node *diagnostic_graph_get_node_by_id (diagnostic_graph *graph, \
+				 const char *node_id)
+
+   Get the node in ``graph`` with the given id, or null.
+
+   The parameters ``graph`` and ``node_id`` must be non-null.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: diagnostic_edge *diagnostic_graph_get_edge_by_id (diagnostic_graph *graph, \
+				 const char *edge_id)
+
+   Get the edge in ``graph`` with the given id, or null.
+
+   The parameters ``graph`` and ``edge_id`` must be non-null.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+
+.. type:: diagnostic_node
+
+.. function:: void diagnostic_node_set_label (diagnostic_node *node, \
+			   const char *label)
+
+   Set the label of ``node`` for use in the value of the
+   SARIF ``label`` property (SARIF v2.1.0 section 3.40.3).
+
+   The parameter ``node`` must be non-null.
+   The parameter ``label`` can be null, for clearing any existing
+   label.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: void diagnostic_node_set_location (diagnostic_node *node, \
+			      const diagnostic_physical_location *loc)
+
+   Set the physical location of ``node``, if any.
+
+   The parameter ``node`` must be non-null.
+   The parameter ``loc`` can be null, for clearing any existing
+   location.
+
+   If set, the value will be used by SARIF sinks within the
+   ``location`` property  (SARIF v2.1.0 section 3.40.4).
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
+
+.. function:: void diagnostic_node_set_logical_location (diagnostic_node *node, \
+				      const diagnostic_logical_location *logical_loc)
+
+   Set the logical location of ``node``, if any.
+
+   The parameter ``node`` must be non-null.
+   The parameter ``logical_loc`` _can be null, for clearing any existing
+   location.
+
+   If set, the value will be used by SARIF sinks within the
+   ``location`` property  (SARIF v2.1.0 section 3.40.4).
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`.
diff --git a/gcc/doc/libgdiagnostics/topics/index.rst b/gcc/doc/libgdiagnostics/topics/index.rst
index 6eb3ed6..437ee05 100644
--- a/gcc/doc/libgdiagnostics/topics/index.rst
+++ b/gcc/doc/libgdiagnostics/topics/index.rst
@@ -28,6 +28,7 @@ Topic reference
    diagnostic-manager.rst
    diagnostics.rst
    message-formatting.rst
+   message-buffers.rst
    physical-locations.rst
    logical-locations.rst
    metadata.rst
@@ -35,5 +36,6 @@ Topic reference
    execution-paths.rst
    text-output.rst
    sarif.rst
+   graphs.rst
    ux.rst
    compatibility.rst
diff --git a/gcc/doc/libgdiagnostics/topics/logical-locations.rst b/gcc/doc/libgdiagnostics/topics/logical-locations.rst
index 184b563..294d396 100644
--- a/gcc/doc/libgdiagnostics/topics/logical-locations.rst
+++ b/gcc/doc/libgdiagnostics/topics/logical-locations.rst
@@ -120,7 +120,7 @@ source location
    "equal" input values on the same :type:`diagnostic_manager` will return
    the same instance of :type:`diagnostic_logical_location`.  "Equal" here
    includes different string buffers that compare as equal with
-   :func:``strcmp`.
+   :func:`strcmp`.
 
 .. function:: void diagnostic_manager_debug_dump_logical_location (const diagnostic_manager *diag_mgr, \
                                                                    const diagnostic_logical_location *loc, \
@@ -147,8 +147,9 @@ Accessors
 
 The following functions can be used to access the data that was passed to a
 :type:`diagnostic_logical_location` when it was created.  In each case, the
-``loc`` parameter must be non-NULL.  :type:`const char *` values will point
-at copies of the original buffer.
+``loc`` parameter must be non-NULL.  The return values will point
+at *copies* of the original buffer owned by the
+:type:`diagnostic_logical_location`, or be null.
 
 .. function:: enum diagnostic_logical_location_kind_t diagnostic_logical_location_get_kind (const diagnostic_logical_location *loc)
 
diff --git a/gcc/doc/libgdiagnostics/topics/message-buffers.rst b/gcc/doc/libgdiagnostics/topics/message-buffers.rst
new file mode 100644
index 0000000..c6f5851
--- /dev/null
+++ b/gcc/doc/libgdiagnostics/topics/message-buffers.rst
@@ -0,0 +1,310 @@
+.. Copyright (C) 2025 Free Software Foundation, Inc.
+   Originally contributed by David Malcolm <dmalcolm@redhat.com>
+
+   This is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see
+   <https://www.gnu.org/licenses/>.
+
+.. default-domain:: c
+
+Message buffers
+===============
+
+.. type:: diagnostic_message_buffer
+
+A :type:`diagnostic_message_buffer` is a buffer into which text can be
+accumulated, before being used:
+
+* as the message of a diagnostic, using :func:`diagnostic_finish_via_msg_buf`
+
+* as the text of a label for a :type:`diagnostic_physical_location` using
+  :func:`diagnostic_add_location_with_label_via_msg_buf`
+
+* as the text of an event within a :type:`diagnostic_execution_path` using
+  :func:`diagnostic_execution_path_add_event_via_msg_buf`
+
+This is to allow more flexible creation of messages than a "format string
+plus variadic arguments" API.
+
+.. function:: diagnostic_message_buffer * diagnostic_message_buffer_new (void)
+
+   This function creates a new :type:`diagnostic_message_buffer`.
+
+   The caller is responsible for cleaning it up, either by handing it off
+   to one of the API entrypoints that takes ownership of it (such as
+   :func:`diagnostic_finish_via_msg_buf`), or by calling
+   :func:`diagnostic_message_buffer_release` on it.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_release (diagnostic_message_buffer *msg_buf)
+
+   This function releases ``msg_buf``.
+
+   Typically you don't need to call this, but instead will pass the
+   buffer to one of the API entrypoints that takes over ownership of
+   it (such as :func:`diagnostic_finish_via_msg_buf`); calling it
+   after this would lead to a double-free bug, as you no longer "own"
+   the buffer.
+
+   ``msg_buf`` must be non-NULL.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_append_str (diagnostic_message_buffer *msg_buf, \
+				      const char *p)
+
+   This function appends the null-terminated string ``p`` to the buffer.
+   The string is assumed to be UTF-8 encoded.
+
+   ``msg_buf`` and ``p`` must both be non-NULL.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_append_text (diagnostic_message_buffer *msg_buf, \
+	       const char *p, \
+	       size_t len)
+
+   This function appends ``len`` bytes from ``p`` to the buffer.
+   The bytes are assumed to be UTF-8 encoded.
+
+   ``msg_buf`` and ``p`` must both be non-NULL.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_append_byte (diagnostic_message_buffer *msg_buf,\
+				       char ch)
+
+   This function appends ``ch`` to the buffer.  This should be either
+   ASCII, or part of UTF-8 encoded text.
+
+   ``msg_buf`` must be non-NULL.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_append_printf (diagnostic_message_buffer *msg_buf, \
+					 const char *fmt, ...)
+
+   This function appends a formatted string to the buffer, using the
+   formatting rules for ``printf``.
+
+   The string is assumed to be UTF-8 encoded.
+
+   ``msg_buf`` and ``fmt`` must both be non-NULL.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_append_event_id (diagnostic_message_buffer *msg_buf, \
+					   diagnostic_event_id event_id)
+
+   This function appends a :type:`diagnostic_event_id` to the buffer.
+
+   ``msg_buf`` must be non-NULL.
+
+   For text output, the event will be printed in the form ``(1)``.
+
+   This is analogous to the
+   :doc:`%@ message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+Hyperlink support
+*****************
+
+.. function:: void diagnostic_message_buffer_begin_url (diagnostic_message_buffer *msg_buf, \
+				     const char *url)
+
+   This function indicates the beginning of a run of text that should be
+   associated with the given URL.  The run of text should be closed with
+   a matching call to :func:`diagnostic_message_buffer_end_url`.
+
+   ``msg_buf`` and ``url`` must both be non-NULL.
+
+   For text output in a suitably modern terminal, the run of text will
+   be emitted as a clickable hyperlink to the URL.
+
+   For SARIF sinks, the run of text will be emitted using SARIF's
+   embedded link syntax.
+
+   This is analogous to the
+   :doc:`%{ message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_end_url (diagnostic_message_buffer *msg_buf)
+
+   This function ends a run of text within the buffer started with
+   :func:`diagnostic_message_buffer_begin_url`.
+
+   ``msg_buf`` must be non-NULL.
+
+   This is analogous to the
+   :doc:`%} message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+Quoted text
+***********
+
+.. function:: void diagnostic_message_buffer_begin_quote (diagnostic_message_buffer *msg_buf)
+
+   This function indicates the beginning of a run of text that should be
+   printed in quotes.  The run of text should be closed with
+   a matching call to :func:`diagnostic_message_buffer_end_quote`.
+
+   ``msg_buf`` must be non-NULL.
+
+   For text output in a suitably modern terminal, the run of text will
+   appear in bold.
+   be emitted as a clickable hyperlink to the URL.
+
+   For SARIF sinks, the run of text will be emitted using SARIF's
+   embedded link syntax.
+
+   This is analogous to the
+   ``%<``:doc:`message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_end_url (diagnostic_message_buffer *msg_buf)
+
+   This function ends a run of text within the buffer started with
+   :func:`diagnostic_message_buffer_begin_url`.
+
+   ``msg_buf`` must be non-NULL.
+
+   This is analogous to the
+   :doc:`%> message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+Color
+*****
+
+.. function:: void diagnostic_message_buffer_begin_color (diagnostic_message_buffer *msg_buf, \
+				     const char *color)
+
+   This function indicates the beginning of a run of text that should be
+   colorized as the given color.  The run of text should be closed with
+   a matching call to :func:`diagnostic_message_buffer_end_color`.
+
+   The precise set of available color names is currently undocumented.
+
+   ``msg_buf`` and ``color`` must both be non-NULL.
+
+   For text output in a suitable terminal, the run of text will
+   be colorized.
+
+   For SARIF sinks, the run of text will be emitted using SARIF's
+   embedded link syntax.
+
+   This is analogous to the
+   :doc:`%r message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+.. function:: void diagnostic_message_buffer_end_color (diagnostic_message_buffer *msg_buf)
+
+   This function ends a run of text within the buffer started with
+   :func:`diagnostic_message_buffer_begin_color`.
+
+   ``msg_buf`` must be non-NULL.
+
+   This is analogous to the
+   :doc:`%R message formatting code <message-formatting>`.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+Debugging a message buffer
+**************************
+
+.. function:: void diagnostic_message_buffer_dump (const diagnostic_message_buffer *msg_buf, \
+				FILE *outf)
+
+   This function writes a representation of the contents of ``msg_buf``
+   to ``outf``, for debugging.
+
+   ``msg_buf`` can be NULL or non-NULL.
+   ``outf`` must be non-NULL.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_4`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
diff --git a/gcc/doc/libgdiagnostics/topics/message-formatting.rst b/gcc/doc/libgdiagnostics/topics/message-formatting.rst
index 7064b70..803feba 100644
--- a/gcc/doc/libgdiagnostics/topics/message-formatting.rst
+++ b/gcc/doc/libgdiagnostics/topics/message-formatting.rst
@@ -23,6 +23,11 @@ Message formatting
 Various libgdiagnostics entrypoints take a format string and
 variadic arguments.
 
+.. note::
+
+   See also :type:`diagnostic_message_buffer`, which offers an
+   alternative way to build up messages.
+
 The format strings take codes prefixed by ``%``, or ``%q`` to put
 the result in quotes.  For example::
 
diff --git a/gcc/doc/libgdiagnostics/topics/physical-locations.rst b/gcc/doc/libgdiagnostics/topics/physical-locations.rst
index 099e27e..be8e7eb 100644
--- a/gcc/doc/libgdiagnostics/topics/physical-locations.rst
+++ b/gcc/doc/libgdiagnostics/topics/physical-locations.rst
@@ -284,3 +284,23 @@ This diagnostic has three locations
             |   ~~ ^ ~~~~~
             |   |    |
             |   int  const char *
+
+.. function:: void diagnostic_add_location_with_label_via_msg_buf (diagnostic *diag, \
+						const diagnostic_physical_location *loc, \
+						diagnostic_message_buffer *msg_buf)
+
+   This is equivalent to :func:`diagnostic_add_location_with_label` but
+   using a message buffer rather than a text string.
+
+   ``diag`` and ``msg_buf`` must both be non-NULL.
+
+   Calling this function transfers ownership of ``msg_buf`` to the
+   diagnostic - do not call :func:`diagnostic_message_buffer_release` on
+   it.
+
+   This function was added in :ref:`LIBGDIAGNOSTICS_ABI_3`; you can
+   test for its presence using
+
+   .. code-block:: c
+
+      #ifdef LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
diff --git a/gcc/doc/libgdiagnostics/tutorial/02-physical-locations.rst b/gcc/doc/libgdiagnostics/tutorial/02-physical-locations.rst
index 95b95ca..e64b668 100644
--- a/gcc/doc/libgdiagnostics/tutorial/02-physical-locations.rst
+++ b/gcc/doc/libgdiagnostics/tutorial/02-physical-locations.rst
@@ -112,10 +112,10 @@ leading to output like this::
    foo.c:17: error: can't find 'foo.h'"
       17 | #include <foo.h>
 
-where libgdiagnostics will attempt to load the source file and
+where libgdiagnostics will attempt to load ``foo.c`` and
 quote the pertinent line.
 
-If libgdiagnostics cannot open the file, it will merely print::
+If libgdiagnostics cannot open ``foo.c``, it will merely print::
 
    foo.c:17: error: can't find 'foo.h'
 
@@ -160,14 +160,15 @@ On compiling and running the program, we should get this output::
       17 | #include <foo.h>
          |           ^~~~~
 
-where libgdiagnostics will attempt to load the source file and
+where libgdiagnostics will attempt to load ``foo.c`` and
 underling the pertinent part of the given line.
 
-If libgdiagnostics cannot open the file, it will merely print::
+If libgdiagnostics cannot open ``foo.c``, it will merely print::
 
    foo.c:17:8: error: can't find 'foo.h'
 
-A range can span multiple lines within the same file.
+A range can span multiple lines within the same file, but cannot
+span multiple files.
 
 As before, you can use :func:`diagnostic_manager_debug_dump_location` to
 dump the locations.  For the above example::
diff --git a/gcc/doc/libgdiagnostics/tutorial/07-execution-paths.rst b/gcc/doc/libgdiagnostics/tutorial/07-execution-paths.rst
index 0ac8bf0..9147171 100644
--- a/gcc/doc/libgdiagnostics/tutorial/07-execution-paths.rst
+++ b/gcc/doc/libgdiagnostics/tutorial/07-execution-paths.rst
@@ -134,8 +134,6 @@ Here's the above example in full:
    :end-before:  end full example
 
 
-Moving on
-*********
-
-That's the end of the tutorial.  For more information on libgdiagnostics, see
-the :doc:`topic guide <../topics/index>`.
+See the :doc:`guide to execution paths <../topics/execution-paths>`
+for more information, or go on to
+:doc:`the next section of the tutorial <08-message-buffers>`.
diff --git a/gcc/doc/libgdiagnostics/tutorial/08-message-buffers.rst b/gcc/doc/libgdiagnostics/tutorial/08-message-buffers.rst
new file mode 100644
index 0000000..a83c50c
--- /dev/null
+++ b/gcc/doc/libgdiagnostics/tutorial/08-message-buffers.rst
@@ -0,0 +1,75 @@
+.. Copyright (C) 2025 Free Software Foundation, Inc.
+   Originally contributed by David Malcolm <dmalcolm@redhat.com>
+
+   This is free software: you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see
+   <https://www.gnu.org/licenses/>.
+
+.. default-domain:: c
+
+Tutorial part 8: message buffers
+================================
+
+In previous examples, we finished a diagnostic with a call to
+:func:`diagnostic_finish`, which takes a format string and arguments
+to determine the text message of the diagnostic.
+
+Sometimes this approach is inconvenient, such as where you might want to
+build up a message programatically from a series of components.
+Additionally, you might have existing code that uses ``fprintf``, whereas
+:func:`diagnostic_finish` has its
+:doc:`own formatting conventions <../topics/message-formatting>` which are
+:strong:`not` the same as printf.
+
+For this reason libgdiagnostics (from ``LIBGDIAGNOSTICS_ABI_3`` onwards)
+supports :type:`diagnostic_message_buffer`, which can be used to accumulate a
+message before using it.
+
+You create a :type:`diagnostic_message_buffer` using
+:func:`diagnostic_message_buffer_new`.
+
+There are various API entrypoints for accumulating text into the buffer.
+
+For example:
+
+.. literalinclude:: ../../../testsuite/libgdiagnostics.dg/test-message-buffer.c
+   :language: c
+   :start-after: /* begin quoted source */
+   :end-before:  /* end quoted source */
+
+Running this will produce this text output::
+
+.. code-block:: console
+
+  $ ./test-message-buffer.c.exe
+  ./test-message-buffer.c.exe: error: this is a string; foo; int: 42 str: mostly harmless; this is a link  'this is quoted' highlight A highlight B (1).
+
+where in a suitably-capable terminal if a text sink is directly
+connected to a tty:
+
+* the ``this is a link`` will be a clickable hyperlink
+  (and the URL will be captured in SARIF output).
+
+* the quoted text will be in bold
+
+* the ``highlight A`` and ``highlight B`` text will be colorized
+
+* the event ID will be colorized (and will be a URL in SARIF output
+  if used within a :type:`diagnostic_execution_path`).
+
+
+Moving on
+*********
+
+That's the end of the tutorial.  For more information on libgdiagnostics, see
+the :doc:`topic guide <../topics/index>`.
diff --git a/gcc/doc/libgdiagnostics/tutorial/index.rst b/gcc/doc/libgdiagnostics/tutorial/index.rst
index 172a28c..09a15e9 100644
--- a/gcc/doc/libgdiagnostics/tutorial/index.rst
+++ b/gcc/doc/libgdiagnostics/tutorial/index.rst
@@ -30,3 +30,4 @@ The following tutorial gives an overview of how to use libgdiagnostics.
    05-warnings.rst
    06-fix-it-hints.rst
    07-execution-paths.rst
+   08-message-buffers.rst
diff --git a/gcc/doc/md.texi b/gcc/doc/md.texi
index 28159b2..aba93f6 100644
--- a/gcc/doc/md.texi
+++ b/gcc/doc/md.texi
@@ -1366,6 +1366,12 @@ as for @samp{<} apply.
 A register operand is allowed provided that it is in a general
 register.
 
+@cindex hard registers in constraint
+@item @samp{@{r@}}
+An operand is bound to hard register @samp{r} which may be any general,
+floating-point, or vector register except a fixed register like a stack pointer
+register.  The set of fixed registers is target dependent.
+
 @cindex constants in constraints
 @cindex @samp{i} in constraint
 @item @samp{i}
diff --git a/gcc/doc/riscv-ext.texi b/gcc/doc/riscv-ext.texi
index c3ed1bf..572b70e 100644
--- a/gcc/doc/riscv-ext.texi
+++ b/gcc/doc/riscv-ext.texi
@@ -714,4 +714,8 @@
 @tab 1.0
 @tab Ventana integer conditional operations extension
 
+@item xmipscmov
+@tab 1.0
+@tab Mips conditional move extension
+
 @end multitable
diff --git a/gcc/doc/riscv-mcpu.texi b/gcc/doc/riscv-mcpu.texi
new file mode 100644
index 0000000..6753e51
--- /dev/null
+++ b/gcc/doc/riscv-mcpu.texi
@@ -0,0 +1,69 @@
+@c Copyright (C) 2025 Free Software Foundation, Inc.
+@c This is part of the GCC manual.
+@c For copying conditions, see the file gcc/doc/include/fdl.texi.
+
+@c This file is generated automatically using
+@c  gcc/config/riscv/gen-riscv-mcpu-texi.cc from:
+@c       gcc/config/riscv/riscv-cores.def
+
+@c Please *DO NOT* edit manually.
+
+@samp{Core Name}
+
+@opindex mcpu
+@item -mcpu=@var{processor-string}
+Use architecture of and optimize the output for the given processor, specified
+by particular CPU name. Permissible values for this option are:
+
+
+@samp{sifive-e20},
+
+@samp{sifive-e21},
+
+@samp{sifive-e24},
+
+@samp{sifive-e31},
+
+@samp{sifive-e34},
+
+@samp{sifive-e76},
+
+@samp{sifive-s21},
+
+@samp{sifive-s51},
+
+@samp{sifive-s54},
+
+@samp{sifive-s76},
+
+@samp{sifive-u54},
+
+@samp{sifive-u74},
+
+@samp{sifive-x280},
+
+@samp{sifive-p450},
+
+@samp{sifive-p670},
+
+@samp{thead-c906},
+
+@samp{xt-c908},
+
+@samp{xt-c908v},
+
+@samp{xt-c910},
+
+@samp{xt-c910v2},
+
+@samp{xt-c920},
+
+@samp{xt-c920v2},
+
+@samp{tt-ascalon-d8},
+
+@samp{xiangshan-nanhu},
+
+@samp{xiangshan-kunminghu},
+
+@samp{mips-p8700}.
diff --git a/gcc/doc/riscv-mtune.texi b/gcc/doc/riscv-mtune.texi
new file mode 100644
index 0000000..a2a4d3e
--- /dev/null
+++ b/gcc/doc/riscv-mtune.texi
@@ -0,0 +1,59 @@
+@c Copyright (C) 2025 Free Software Foundation, Inc.
+@c This is part of the GCC manual.
+@c For copying conditions, see the file gcc/doc/include/fdl.texi.
+
+@c This file is generated automatically using
+@c  gcc/config/riscv/gen-riscv-mtune-texi.cc from:
+@c       gcc/config/riscv/riscv-cores.def
+
+@c Please *DO NOT* edit manually.
+
+@samp{Tune Name}
+
+@opindex mtune
+@item -mtune=@var{processor-string}
+Optimize the output for the given processor, specified by microarchitecture or
+particular CPU name.  Permissible values for this option are:
+
+
+@samp{generic},
+
+@samp{rocket},
+
+@samp{sifive-3-series},
+
+@samp{sifive-5-series},
+
+@samp{sifive-7-series},
+
+@samp{sifive-p400-series},
+
+@samp{sifive-p600-series},
+
+@samp{tt-ascalon-d8},
+
+@samp{thead-c906},
+
+@samp{xt-c908},
+
+@samp{xt-c908v},
+
+@samp{xt-c910},
+
+@samp{xt-c910v2},
+
+@samp{xt-c920},
+
+@samp{xt-c920v2},
+
+@samp{xiangshan-nanhu},
+
+@samp{xiangshan-kunminghu},
+
+@samp{generic-ooo},
+
+@samp{size},
+
+@samp{mips-p8700},
+
+and all valid options for @option{-mcpu=}.
diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
index 6c5586e..a919304 100644
--- a/gcc/doc/sourcebuild.texi
+++ b/gcc/doc/sourcebuild.texi
@@ -2373,6 +2373,15 @@ whether it does so by default).
 @itemx aarch64_sve1024_hw
 @itemx aarch64_sve2048_hw
 Like @code{aarch64_sve_hw}, but also test for an exact hardware vector length.
+@item aarch64_sve2_hw
+AArch64 target that is able to generate and execute SVE2 code (regardless of
+whether it does so by default).
+@item aarch64_sve2p1_hw
+AArch64 target that is able to generate and execute SVE2.1 code (regardless of
+whether it does so by default).
+@item aarch64_sme_hw
+AArch64 target that is able to generate and execute SME code (regardless of
+whether it does so by default).
 
 @item aarch64_fjcvtzs_hw
 AArch64 target that is able to generate and execute armv8.3-a FJCVTZS
diff --git a/gcc/doc/tm.texi b/gcc/doc/tm.texi
index 5e30564..215552c 100644
--- a/gcc/doc/tm.texi
+++ b/gcc/doc/tm.texi
@@ -6382,12 +6382,14 @@ return type of the vectorized function shall be of vector type
 @var{vec_type_out} and the argument types should be @var{vec_type_in}.
 @end deftypefn
 
-@deftypefn {Target Hook} bool TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT (machine_mode @var{mode}, const_tree @var{type}, int @var{misalignment}, bool @var{is_packed})
+@deftypefn {Target Hook} bool TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT (machine_mode @var{mode}, const_tree @var{type}, int @var{misalignment}, bool @var{is_packed}, bool @var{is_gather_scatter})
 This hook should return true if the target supports misaligned vector
 store/load of a specific factor denoted in the @var{misalignment}
 parameter.  The vector store/load should be of machine mode @var{mode} and
-the elements in the vectors should be of type @var{type}.  @var{is_packed}
-parameter is true if the memory access is defined in a packed struct.
+the elements in the vectors should be of type @var{type}.  The
+@var{is_packed} parameter is true if the misalignment is unknown and the
+memory access is defined in a packed struct.  @var{is_gather_scatter} is true
+if the load/store is a gather or scatter.
 @end deftypefn
 
 @deftypefn {Target Hook} machine_mode TARGET_VECTORIZE_PREFERRED_SIMD_MODE (scalar_mode @var{mode})
@@ -6511,6 +6513,15 @@ The default is @code{NULL_TREE} which means to not vectorize scatter
 stores.
 @end deftypefn
 
+@deftypefn {Target Hook} bool TARGET_VECTORIZE_PREFER_GATHER_SCATTER (machine_mode @var{mode}, int @var{scale}, unsigned int @var{group_size})
+This hook returns TRUE if gather loads or scatter stores are cheaper on
+this target than a sequence of elementwise loads or stores.  The @var{mode}
+and @var{scale} correspond to the @code{gather_load} and
+@code{scatter_store} instruction patterns.  The @var{group_size} is the
+number of scalar elements in each scalar loop iteration that are to be
+combined into the vector.
+@end deftypefn
+
 @deftypefn {Target Hook} int TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN (struct cgraph_node *@var{}, struct cgraph_simd_clone *@var{}, @var{tree}, @var{int}, @var{bool})
 This hook should set @var{vecsize_mangle}, @var{vecsize_int}, @var{vecsize_float}
 fields in @var{simd_clone} structure pointed by @var{clone_info} argument and also
diff --git a/gcc/doc/tm.texi.in b/gcc/doc/tm.texi.in
index eccc4d8..b03ad4c 100644
--- a/gcc/doc/tm.texi.in
+++ b/gcc/doc/tm.texi.in
@@ -4311,6 +4311,8 @@ address;  but often a machine-dependent strategy can generate better code.
 
 @hook TARGET_VECTORIZE_BUILTIN_SCATTER
 
+@hook TARGET_VECTORIZE_PREFER_GATHER_SCATTER
+
 @hook TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN
 
 @hook TARGET_SIMD_CLONE_ADJUST
diff --git a/gcc/errors.cc b/gcc/errors.cc
index 6d1ecc7..fb6b19d 100644
--- a/gcc/errors.cc
+++ b/gcc/errors.cc
@@ -19,7 +19,8 @@ along with GCC; see the file COPYING3.  If not see
 
 /* warning, error, and fatal.  These definitions are suitable for use
    in the generator programs; the compiler has a more elaborate suite
-   of diagnostic printers, found in diagnostic.cc.  */
+   of diagnostic printers, found in diagnostic-global-context.cc using
+   the diagnostics/ subdirectory.  */
 
 #ifdef HOST_GENERATOR_FILE
 #include "config.h"
@@ -126,7 +127,7 @@ trim_filename (const char *name)
 
 /* "Fancy" abort.  Reports where in the compiler someone gave up.
    This file is used only by build programs, so we're not as polite as
-   the version in diagnostic.cc.  */
+   the version in diagnostics/context.cc.  */
 void
 fancy_abort (const char *file, int line, const char *func)
 {
diff --git a/gcc/expr.cc b/gcc/expr.cc
index ac4fdfa..3f2b121 100644
--- a/gcc/expr.cc
+++ b/gcc/expr.cc
@@ -13206,6 +13206,8 @@ constant_byte_string (tree arg, tree *ptr_offset, tree *mem_size, tree *decl,
 	     of the expected type and size.  */
 	  if (!initsize)
 	    initsize = integer_zero_node;
+	  else if (!tree_fits_uhwi_p (initsize))
+	    return NULL_TREE;
 
 	  unsigned HOST_WIDE_INT size = tree_to_uhwi (initsize);
 	  if (size > (unsigned HOST_WIDE_INT) INT_MAX)
diff --git a/gcc/ext-dce.cc b/gcc/ext-dce.cc
index afe7afe..67ec92a 100644
--- a/gcc/ext-dce.cc
+++ b/gcc/ext-dce.cc
@@ -651,9 +651,8 @@ ext_dce_process_uses (rtx_insn *insn, rtx obj,
 
 	  /* ?!? How much of this should mirror SET handling, potentially
 	     being shared?   */
-	  if (SUBREG_P (dst) && SUBREG_BYTE (dst).is_constant ())
+	  if (SUBREG_P (dst) && subreg_lsb (dst).is_constant (&bit))
 	    {
-	      bit = subreg_lsb (dst).to_constant ();
 	      if (bit >= HOST_BITS_PER_WIDE_INT)
 		bit = HOST_BITS_PER_WIDE_INT - 1;
 	      dst = SUBREG_REG (dst);
@@ -758,7 +757,7 @@ ext_dce_process_uses (rtx_insn *insn, rtx obj,
 		     and process the inner object.  */
 		  if (paradoxical_subreg_p (y))
 		    y = XEXP (y, 0);
-		  else if (SUBREG_P (y) && SUBREG_BYTE (y).is_constant ())
+		  else if (SUBREG_P (y) && subreg_lsb (y).is_constant (&bit))
 		    {
 		      /* If !TRULY_NOOP_TRUNCATION_MODES_P, the mode
 			 change performed by Y would normally need to be a
@@ -775,8 +774,6 @@ ext_dce_process_uses (rtx_insn *insn, rtx obj,
 				    GET_MODE (SUBREG_REG (y))))))
 			break;
 
-		      bit = subreg_lsb (y).to_constant ();
-
 		      /* If this is a wide object (more bits than we can fit
 			 in a HOST_WIDE_INT), then just break from the SET
 			 context.   That will cause the iterator to walk down
diff --git a/gcc/final.cc b/gcc/final.cc
index a4dbab7..afcb0bb 100644
--- a/gcc/final.cc
+++ b/gcc/final.cc
@@ -83,6 +83,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "function-abi.h"
 #include "common/common-target.h"
 #include "diagnostic.h"
+#include "diagnostics/file-cache.h"
 
 #include "dwarf2out.h"
 
@@ -2116,7 +2117,7 @@ asm_show_source (const char *filename, int linenum)
   if (!filename)
     return;
 
-  char_span line
+  diagnostics::char_span line
     = global_dc->get_file_cache ().get_source_line (filename, linenum);
   if (!line)
     return;
diff --git a/gcc/fold-const.cc b/gcc/fold-const.cc
index 4749257..8867540 100644
--- a/gcc/fold-const.cc
+++ b/gcc/fold-const.cc
@@ -15224,7 +15224,7 @@ bool
 tree_expr_nonnegative_warnv_p (tree t, bool *strict_overflow_p, int depth)
 {
   enum tree_code code;
-  if (t == error_mark_node)
+  if (error_operand_p (t))
     return false;
 
   code = TREE_CODE (t);
diff --git a/gcc/fortran/ChangeLog b/gcc/fortran/ChangeLog
index 0ea9c39..d75d64f 100644
--- a/gcc/fortran/ChangeLog
+++ b/gcc/fortran/ChangeLog
@@ -1,3 +1,272 @@
+2025-08-01  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* trans-decl.cc (gfc_trans_deferred_vars): Fix closing brace in
+	a comment.
+
+2025-07-31  Mikael Morin  <morin-mikael@orange.fr>
+
+	PR fortran/121342
+	* trans-expr.cc (gfc_conv_subref_array_arg): Remove offset
+	update.
+	(gfc_conv_procedure_call): For polymorphic functions, move the
+	scalarizer descriptor information...
+	* trans-array.cc (gfc_add_loop_ss_code): ... here, and evaluate
+	the bounds to fresh variables.
+	(get_class_info_from_ss): Remove offset update.
+	(gfc_conv_ss_startstride): Don't set a zero value for function
+	result upper bounds.
+	(late_set_loop_bounds): New.
+	(gfc_conv_loop_setup): If the bounds of a function result have
+	been set, and no other array provided loop bounds for a
+	dimension, use the function result bounds as loop bounds for
+	that dimension.
+	(gfc_set_delta): Don't skip delta setting for polymorphic
+	function results.
+
+2025-07-30  Mikael Morin  <morin-mikael@orange.fr>
+
+	* trans-array.cc (gfc_array_init_size): Remove the nelems
+	argument.
+	(gfc_array_allocate): Update caller.  Remove the nelems
+	argument.
+	* trans-stmt.cc (gfc_trans_allocate): Update caller.  Remove the
+	nelems variable.
+	* trans-array.h (gfc_array_allocate): Update prototype.
+
+2025-07-30  Yuao Ma  <c8ef@outlook.com>
+
+	* check.cc (gfc_check_split): Argument check for SPLIT.
+	* gfortran.h (enum gfc_isym_id): Define GFC_ISYM_SPLIT.
+	* intrinsic.cc (add_subroutines): Register SPLIT intrinsic.
+	* intrinsic.h (gfc_check_split): New decl.
+	(gfc_resolve_split): Ditto.
+	* intrinsic.texi: SPLIT documentation.
+	* iresolve.cc (gfc_resolve_split): Add resolved_sym for SPLIT.
+	* trans-decl.cc (gfc_build_intrinsic_function_decls): Add decl for
+	SPLIT in libgfortran.
+	* trans-intrinsic.cc (conv_intrinsic_split): SPLIT codegen.
+	(gfc_conv_intrinsic_subroutine): Handle SPLIT case.
+	* trans.h (GTY): Declare gfor_fndecl_string_split{, _char4}.
+
+2025-07-27  Mikael Morin  <mikael@gcc.gnu.org>
+
+	PR fortran/121185
+	* trans-expr.cc (gfc_trans_assignment_1): Use the same condition
+	to set the is_alloc_lhs flag and to decide to generate
+	reallocation code.  Add explicit call to gfc_fix_class_refs
+	before evaluating the condition.
+
+2025-07-27  Mikael Morin  <mikael@gcc.gnu.org>
+
+	PR fortran/121185
+	* trans-array.cc (set_factored_descriptor_value): Also trigger
+	the saving of the previously selected reference on encountering
+	an INDIRECT_REF.  Extract the saving code...
+	(save_ref): ... here as a new function.
+
+2025-07-27  Mikael Morin  <mikael@gcc.gnu.org>
+
+	PR fortran/121185
+	* trans-expr.cc (gfc_get_class_from_expr): Give up class
+	container lookup on the second COMPONENT_REF after an array
+	descriptor.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* error.cc: Make diagnostics::context::m_source_printing private.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* cpp.cc: Update usage of "diagnostic_info" to explicitly refer to
+	"diagnostics::diagnostic_info".
+	* error.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* cpp.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* error.cc: Likewise.
+	* options.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* cpp.cc: Update for renaming of
+	diagnostic_option_id to diagnostics::option_id.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* error.cc: Update for move of diagnostic-color.h to
+	diagnostics/color.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* error.cc: Update for diagnostic_context becoming
+	diagnostics::context.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* error.cc: Update to add "m_" prefix to fields of
+	diagnostic_info throughout.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* error.cc: Update for move of diagnostics output formats into
+	namespace "diagnostics" as "sinks".
+	* gfortran.h: Likewise.
+
+2025-07-23  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/121203
+	* trans-expr.cc (gfc_conv_procedure_call): Obtain the character
+	length of an assumed character length procedure from the typespec
+	of the actual argument even if there is no explicit interface.
+
+2025-07-21  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* trans-decl.cc (gfc_trans_deferred_vars): Fix indentation.
+
+2025-07-21  Andre Vehreschild  <vehre@gcc.gnu.org>
+
+	PR fortran/119106
+	* expr.cc (simplify_constructor): Do not simplify constants.
+	(gfc_simplify_expr): Continue to simplify expression when an
+	iterator is present.
+
+2025-07-21  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* trans-array.cc (gfc_conv_ss_descriptor): Move the descriptor
+	expression initialisation...
+	(set_factored_descriptor_value): ... to this new function.
+	Before initialisation, walk the reference expression passed as
+	argument and save some of its subexpressions to a variable.
+	(substitute_t): New struct.
+	(maybe_substitute_expr): New function.
+	(substitute_subexpr_in_expr): New function.
+
+2025-07-18  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/121145
+	* trans-expr.cc (gfc_conv_procedure_call): Do not create pointer
+	check for proc-pointer actual passed to optional dummy.
+
+2025-07-16  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/121060
+	* interface.cc (matching_typebound_op): Defer determination of
+	specific procedure until resolution by returning NULL.
+
+2025-07-16  Steve Kargl  <sgk@troutmask.apl.washington.edu>
+
+	* decl.cc (gfc_match_import): Correct minor whitespace snafu
+	and fix NULL pointer dereferences in two places.
+
+2025-07-15  Kwok Cheung Yeung  <kcyeung@baylibre.com>
+
+	PR fortran/104428
+	* trans-openmp.cc (gfc_trans_omp_declare_variant): Check that proc_st
+	is non-NULL before dereferencing.  Add line number to error message.
+
+2025-07-15  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* gfortran.h (gfc_symbol): Remove field allocated_in_scope.
+	* trans-array.cc (gfc_array_allocate): Don't set it.
+	(gfc_alloc_allocatable_for_assignment): Likewise.
+	Generate the unallocated descriptor bounds initialisation
+	before the opening of the reallocation code block.  Create a
+	variable and use it as additional condition to the unallocated
+	descriptor bounds initialisation.
+
+2025-07-15  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* trans-array.cc (gfc_conv_ss_descriptor): Don't evaluate
+	offset and data to a variable if is_alloc_lhs is set.  Move the
+	existing evaluation decision condition for data...
+	(save_descriptor_data): ... here as a new predicate.
+	(evaluate_bound): Add argument save_value.  Omit the evaluation
+	of the value to a variable if that argument isn't set.
+	(gfc_conv_expr_descriptor): Update caller.
+	(gfc_conv_section_startstride): Update caller.  Set save_value
+	if is_alloc_lhs is not set.  Omit the evaluation of stride to a
+	variable if save_value isn't set.
+	(gfc_set_delta): Omit the evaluation of delta to a variable
+	if is_alloc_lhs is set.
+	(gfc_is_reallocatable_lhs): Return false if flag_realloc_lhs
+	isn't set.
+	(gfc_alloc_allocatable_for_assignment): Don't update
+	the variables that may be stored in saved_offset, delta, and
+	data.  Call instead...
+	(update_reallocated_descriptor): ... this new procedure.
+	* trans-expr.cc (gfc_trans_assignment_1): Don't omit setting the
+	is_alloc_lhs flag if the right hand side is an intrinsic
+	function.  Clear the flag if the right hand side is scalar.
+
+2025-07-15  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* trans-expr.cc (gfc_trans_assignment_1): Generate array
+	reallocation code before entering the scalarisation loops.
+
+2025-07-15  Filip Kastl  <fkastl@suse.cz>
+
+	* resolve.cc (resolve_select_type): Fix indentation.
+
+2025-07-12  Tobias Burnus  <tburnus@baylibre.com>
+
+	* invoke.texi (-Wsurprising): Note about OpenACC warning
+	related to PARAMATER.
+	* openmp.cc (resolve_omp_clauses, gfc_resolve_oacc_declare):
+	Accept PARAMETER for OpenACC but add surprising warning.
+	* trans-openmp.cc (gfc_trans_omp_variable_list,
+	gfc_trans_omp_clauses): Ignore PARAMETER inside clauses.
+
+2025-07-11  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/106135
+	* decl.cc (build_sym): Emit an error if a symbol associated by
+	an IMPORT, ONLY or IMPORT, all statement is being redeclared.
+	(gfc_match_import): Parse and check the F2018 versions of the
+	IMPORT statement. For scopes other than and interface body, if
+	the symbol cannot be found in the host scope, generate it and
+	set it up such that gfc_fixup_sibling_symbols can transfer its
+	'imported attribute' if it turnes out to be a not yet parsed
+	procedure. Test for violations of C897-8100.
+	* gfortran.h : Add 'import_only' to the gfc_symtree structure.
+	Add the enum, 'importstate', which is used for values the new
+	field 'import_state' in gfc_namespace.
+	* parse.cc (gfc_fixup_sibling_symbols): Transfer the attribute
+	'imported' to the new symbol.
+	* resolve.cc (check_sym_import_status, check_import_status):
+	New functions to test symbols and expressions for violations of
+	F2018:C8102.
+	(resolve_call): Test the 'resolved_sym' against C8102 by a call
+	to 'check_sym_import_status'.
+	(gfc_resolve_expr): If the expression is OK and an IMPORT
+	statement has been registered in the current scope, test C102
+	by calling 'check_import_status'.
+	(resolve_select_type): Test the declared derived type in TYPE
+	IS and CLASS IS statements.
+
+2025-07-08  Andre Vehreschild  <vehre@gcc.gnu.org>
+
+	PR fortran/120637
+	* class.cc (finalize_component): Return true, when a finalizable
+	component was detect and do not free it.
+
+2025-07-07  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* trans-intrinsic.cc (conv_intrinsic_move_alloc): Add pre and
+	post code for the FROM and TO arguments.
+
+2025-07-04  Martin Jambor  <mjambor@suse.cz>
+
+	* io.cc (format_asterisk): Add a brace around static initialization
+	location part of the field locus.
+
+2025-07-03  Andre Vehreschild  <vehre@gcc.gnu.org>
+
+	PR fortran/120843
+	* resolve.cc (resolve_operator): Remove conformability check,
+	because it is not in the standard.
+
 2025-07-01  Harald Anlauf  <anlauf@gmx.de>
 
 	* coarray.cc (check_add_new_component): Treat pure and elemental
diff --git a/gcc/fortran/check.cc b/gcc/fortran/check.cc
index 838d523..8626526 100644
--- a/gcc/fortran/check.cc
+++ b/gcc/fortran/check.cc
@@ -5559,6 +5559,27 @@ gfc_check_scan (gfc_expr *x, gfc_expr *y, gfc_expr *z, gfc_expr *kind)
   return true;
 }
 
+bool
+gfc_check_split (gfc_expr *string, gfc_expr *set, gfc_expr *pos, gfc_expr *back)
+{
+  if (!type_check (string, 0, BT_CHARACTER))
+    return false;
+
+  if (!type_check (set, 1, BT_CHARACTER))
+    return false;
+
+  if (!type_check (pos, 2, BT_INTEGER) || !scalar_check (pos, 2))
+    return false;
+
+  if (back != NULL
+      && (!type_check (back, 3, BT_LOGICAL) || !scalar_check (back, 3)))
+    return false;
+
+  if (!same_type_check (string, 0, set, 1))
+    return false;
+
+  return true;
+}
 
 bool
 gfc_check_secnds (gfc_expr *r)
diff --git a/gcc/fortran/class.cc b/gcc/fortran/class.cc
index df18601..a1c6faf 100644
--- a/gcc/fortran/class.cc
+++ b/gcc/fortran/class.cc
@@ -1034,7 +1034,7 @@ comp_is_finalizable (gfc_component *comp)
    of calling the appropriate finalizers, coarray deregistering, and
    deallocation of allocatable subcomponents.  */
 
-static void
+static bool
 finalize_component (gfc_expr *expr, gfc_symbol *derived, gfc_component *comp,
 		    gfc_symbol *stat, gfc_symbol *fini_coarray, gfc_code **code,
 		    gfc_namespace *sub_ns)
@@ -1044,14 +1044,14 @@ finalize_component (gfc_expr *expr, gfc_symbol *derived, gfc_component *comp,
   gfc_was_finalized *f;
 
   if (!comp_is_finalizable (comp))
-    return;
+    return false;
 
   /* If this expression with this component has been finalized
      already in this namespace, there is nothing to do.  */
   for (f = sub_ns->was_finalized; f; f = f->next)
     {
       if (f->e == expr && f->c == comp)
-	return;
+	return false;
     }
 
   e = gfc_copy_expr (expr);
@@ -1208,8 +1208,6 @@ finalize_component (gfc_expr *expr, gfc_symbol *derived, gfc_component *comp,
       final_wrap->ext.actual->next->next = gfc_get_actual_arglist ();
       final_wrap->ext.actual->next->next->expr = fini_coarray_expr;
 
-
-
       if (*code)
 	{
 	  (*code)->next = final_wrap;
@@ -1221,11 +1219,14 @@ finalize_component (gfc_expr *expr, gfc_symbol *derived, gfc_component *comp,
   else
     {
       gfc_component *c;
+      bool ret = false;
 
       for (c = comp->ts.u.derived->components; c; c = c->next)
-	finalize_component (e, comp->ts.u.derived, c, stat, fini_coarray, code,
-			    sub_ns);
-      gfc_free_expr (e);
+	ret |= finalize_component (e, comp->ts.u.derived, c, stat, fini_coarray,
+				   code, sub_ns);
+      /* Only free the expression, if it has never been used.  */
+      if (!ret)
+	gfc_free_expr (e);
     }
 
   /* Record that this was finalized already in this namespace.  */
@@ -1234,6 +1235,7 @@ finalize_component (gfc_expr *expr, gfc_symbol *derived, gfc_component *comp,
   sub_ns->was_finalized->e = expr;
   sub_ns->was_finalized->c = comp;
   sub_ns->was_finalized->next = f;
+  return true;
 }
 
 
@@ -2314,6 +2316,7 @@ finish_assumed_rank:
     {
       gfc_symbol *stat;
       gfc_code *block = NULL;
+      gfc_expr *ptr_expr;
 
       if (!ptr)
 	{
@@ -2359,14 +2362,15 @@ finish_assumed_rank:
 					     sub_ns);
       block = block->next;
 
+      ptr_expr = gfc_lval_expr_from_sym (ptr);
       for (comp = derived->components; comp; comp = comp->next)
 	{
 	  if (comp == derived->components && derived->attr.extension
 	      && ancestor_wrapper && ancestor_wrapper->expr_type != EXPR_NULL)
 	    continue;
 
-	  finalize_component (gfc_lval_expr_from_sym (ptr), derived, comp,
-			      stat, fini_coarray, &block, sub_ns);
+	  finalize_component (ptr_expr, derived, comp, stat, fini_coarray,
+			      &block, sub_ns);
 	  if (!last_code->block->next)
 	    last_code->block->next = block;
 	}
diff --git a/gcc/fortran/cpp.cc b/gcc/fortran/cpp.cc
index 1b70420..15ecc7d 100644
--- a/gcc/fortran/cpp.cc
+++ b/gcc/fortran/cpp.cc
@@ -1063,7 +1063,7 @@ cb_used_define (cpp_reader *pfile, location_t line ATTRIBUTE_UNUSED,
 /* Return the gcc option code associated with the reason for a cpp
    message, or 0 if none.  */
 
-static diagnostic_option_id
+static diagnostics::option_id
 cb_cpp_diagnostic_cpp_option (enum cpp_warning_reason reason)
 {
   const struct cpp_reason_option_codes_t *entry;
@@ -1088,8 +1088,8 @@ cb_cpp_diagnostic (cpp_reader *pfile ATTRIBUTE_UNUSED,
 		   rich_location *richloc,
 		   const char *msg, va_list *ap)
 {
-  diagnostic_info diagnostic;
-  diagnostic_t dlevel;
+  diagnostics::diagnostic_info diagnostic;
+  enum diagnostics::kind dlevel;
   bool save_warn_system_headers = global_dc->m_warn_system_headers;
   bool ret;
 
@@ -1099,22 +1099,22 @@ cb_cpp_diagnostic (cpp_reader *pfile ATTRIBUTE_UNUSED,
       global_dc->m_warn_system_headers = 1;
       /* Fall through.  */
     case CPP_DL_WARNING:
-      dlevel = DK_WARNING;
+      dlevel = diagnostics::kind::warning;
       break;
     case CPP_DL_PEDWARN:
-      dlevel = DK_PEDWARN;
+      dlevel = diagnostics::kind::pedwarn;
       break;
     case CPP_DL_ERROR:
-      dlevel = DK_ERROR;
+      dlevel = diagnostics::kind::error;
       break;
     case CPP_DL_ICE:
-      dlevel = DK_ICE;
+      dlevel = diagnostics::kind::ice;
       break;
     case CPP_DL_NOTE:
-      dlevel = DK_NOTE;
+      dlevel = diagnostics::kind::note;
       break;
     case CPP_DL_FATAL:
-      dlevel = DK_FATAL;
+      dlevel = diagnostics::kind::fatal;
       break;
     default:
       gcc_unreachable ();
diff --git a/gcc/fortran/decl.cc b/gcc/fortran/decl.cc
index 69acd2d..af42575 100644
--- a/gcc/fortran/decl.cc
+++ b/gcc/fortran/decl.cc
@@ -1723,13 +1723,17 @@ build_sym (const char *name, int elem, gfc_charlen *cl, bool cl_deferred,
   symbol_attribute attr;
   gfc_symbol *sym;
   int upper;
-  gfc_symtree *st;
+  gfc_symtree *st, *host_st = NULL;
 
   /* Symbols in a submodule are host associated from the parent module or
      submodules. Therefore, they can be overridden by declarations in the
      submodule scope. Deal with this by attaching the existing symbol to
      a new symtree and recycling the old symtree with a new symbol...  */
   st = gfc_find_symtree (gfc_current_ns->sym_root, name);
+  if (((st && st->import_only) || (gfc_current_ns->import_state == IMPORT_ALL))
+      && gfc_current_ns->parent)
+    host_st = gfc_find_symtree (gfc_current_ns->parent->sym_root, name);
+
   if (st != NULL && gfc_state_stack->state == COMP_SUBMODULE
       && st->n.sym != NULL
       && st->n.sym->attr.host_assoc && st->n.sym->attr.used_in_submodule)
@@ -1742,6 +1746,20 @@ build_sym (const char *name, int elem, gfc_charlen *cl, bool cl_deferred,
       sym->refs++;
       gfc_set_sym_referenced (sym);
     }
+  /* ...Check that F2018 IMPORT, ONLY and IMPORT, ALL statements, within the
+     current scope are not violated by local redeclarations. Note that there is
+     no need to guard for std >= F2018 because import_only and IMPORT_ALL are
+     only set for these standards.  */
+  else if (host_st && host_st->n.sym
+	   && host_st->n.sym != gfc_current_ns->proc_name
+	   && !(st && st->n.sym
+		&& (st->n.sym->attr.dummy || st->n.sym->attr.result)))
+    {
+      gfc_error ("F2018: C8102 %s at %L is already imported by an %s "
+		 "statement and must not be re-declared", name, var_locus,
+		 (st && st->import_only) ? "IMPORT, ONLY" : "IMPORT, ALL");
+      return false;
+    }
   /* ...Otherwise generate a new symtree and new symbol.  */
   else if (gfc_get_symbol (name, NULL, &sym, var_locus))
     return false;
@@ -5100,6 +5118,54 @@ error:
 }
 
 
+/* Match the IMPORT statement.  IMPORT was added to F2003 as
+
+   R1209 import-stmt  is IMPORT [[ :: ] import-name-list ]
+
+   C1210 (R1209) The IMPORT statement is allowed only in an interface-body.
+
+   C1211 (R1209) Each import-name shall be the name of an entity in the
+		 host scoping unit.
+
+   under the description of an interface block. Under F2008, IMPORT was
+   split out of the interface block description to 12.4.3.3 and C1210
+   became
+
+   C1210 (R1209) The IMPORT statement is allowed only in an interface-body
+		 that is not a module procedure interface body.
+
+   Finally, F2018, section 8.8, has changed the IMPORT statement to
+
+   R867 import-stmt  is IMPORT [[ :: ] import-name-list ]
+		     or IMPORT, ONLY : import-name-list
+		     or IMPORT, NONE
+		     or IMPORT, ALL
+
+   C896 (R867) An IMPORT statement shall not appear in the scoping unit of
+		a main-program, external-subprogram, module, or block-data.
+
+   C897 (R867) Each import-name shall be the name of an entity in the host
+		scoping unit.
+
+   C898  If any IMPORT statement in a scoping unit has an ONLY specifier,
+	 all IMPORT statements in that scoping unit shall have an ONLY
+	 specifier.
+
+   C899  IMPORT, NONE shall not appear in the scoping unit of a submodule.
+
+   C8100 If an IMPORT, NONE or IMPORT, ALL statement appears in a scoping
+	 unit, no other IMPORT statement shall appear in that scoping unit.
+
+   C8101 Within an interface body, an entity that is accessed by host
+	 association shall be accessible by host or use association within
+	 the host scoping unit, or explicitly declared prior to the interface
+	 body.
+
+   C8102 An entity whose name appears as an import-name or which is made
+	 accessible by an IMPORT, ALL statement shall not appear in any
+	 context described in 19.5.1.4 that would cause the host entity
+	 of that name to be inaccessible.  */
+
 match
 gfc_match_import (void)
 {
@@ -5107,16 +5173,28 @@ gfc_match_import (void)
   match m;
   gfc_symbol *sym;
   gfc_symtree *st;
+  bool f2018_allowed = gfc_option.allow_std & ~GFC_STD_OPT_F08;;
+  importstate current_import_state = gfc_current_ns->import_state;
 
-  if (gfc_current_ns->proc_name == NULL
-      || gfc_current_ns->proc_name->attr.if_source != IFSRC_IFBODY)
+  if (!f2018_allowed
+      && (gfc_current_ns->proc_name == NULL
+	  || gfc_current_ns->proc_name->attr.if_source != IFSRC_IFBODY))
     {
       gfc_error ("IMPORT statement at %C only permitted in "
 		 "an INTERFACE body");
       return MATCH_ERROR;
     }
+  else if (f2018_allowed
+	   && (!gfc_current_ns->parent || gfc_current_ns->is_block_data))
+    goto C897;
+
+  if (f2018_allowed
+      && (current_import_state == IMPORT_ALL
+	  || current_import_state == IMPORT_NONE))
+    goto C8100;
 
-  if (gfc_current_ns->proc_name->attr.module_procedure)
+  if (gfc_current_ns->proc_name
+      && gfc_current_ns->proc_name->attr.module_procedure)
     {
       gfc_error ("F2008: C1210 IMPORT statement at %C is not permitted "
 		 "in a module procedure interface body");
@@ -5126,20 +5204,65 @@ gfc_match_import (void)
   if (!gfc_notify_std (GFC_STD_F2003, "IMPORT statement at %C"))
     return MATCH_ERROR;
 
+  gfc_current_ns->import_state = IMPORT_NOT_SET;
+  if (f2018_allowed)
+    {
+      if (gfc_match (" , none") == MATCH_YES)
+	{
+	  if (current_import_state == IMPORT_ONLY)
+	    goto C898;
+	  if (gfc_current_state () == COMP_SUBMODULE)
+	    goto C899;
+	  gfc_current_ns->import_state = IMPORT_NONE;
+	}
+      else if (gfc_match (" , only :") == MATCH_YES)
+	{
+	  if (current_import_state != IMPORT_NOT_SET
+	      && current_import_state != IMPORT_ONLY)
+	    goto C898;
+	  gfc_current_ns->import_state = IMPORT_ONLY;
+	}
+      else if (gfc_match (" , all") == MATCH_YES)
+	{
+	  if (current_import_state == IMPORT_ONLY)
+	    goto C898;
+	  gfc_current_ns->import_state = IMPORT_ALL;
+	}
+
+      if (current_import_state != IMPORT_NOT_SET
+	  && (gfc_current_ns->import_state == IMPORT_NONE
+	      || gfc_current_ns->import_state == IMPORT_ALL))
+	goto C8100;
+    }
+
+  /* F2008 IMPORT<eos> is distinct from F2018 IMPORT, ALL.  */
   if (gfc_match_eos () == MATCH_YES)
     {
-      /* All host variables should be imported.  */
-      gfc_current_ns->has_import_set = 1;
+      /* This is the F2008 variant.  */
+      if (gfc_current_ns->import_state == IMPORT_NOT_SET)
+	{
+	  if (current_import_state == IMPORT_ONLY)
+	    goto C898;
+	  gfc_current_ns->import_state = IMPORT_F2008;
+	}
+
+      /* Host variables should be imported.  */
+      if (gfc_current_ns->import_state != IMPORT_NONE)
+	gfc_current_ns->has_import_set = 1;
       return MATCH_YES;
     }
 
-  if (gfc_match (" ::") == MATCH_YES)
+  if (gfc_match (" ::") == MATCH_YES
+      && gfc_current_ns->import_state != IMPORT_ONLY)
     {
       if (gfc_match_eos () == MATCH_YES)
-	{
-	   gfc_error ("Expecting list of named entities at %C");
-	   return MATCH_ERROR;
-	}
+	goto expecting_list;
+      gfc_current_ns->import_state = IMPORT_F2008;
+    }
+  else if (gfc_current_ns->import_state == IMPORT_ONLY)
+    {
+      if (gfc_match_eos () == MATCH_YES)
+	goto expecting_list;
     }
 
   for(;;)
@@ -5149,13 +5272,15 @@ gfc_match_import (void)
       switch (m)
 	{
 	case MATCH_YES:
-	  if (gfc_current_ns->parent !=  NULL
+	  if (gfc_current_ns->parent != NULL
 	      && gfc_find_symbol (name, gfc_current_ns->parent, 1, &sym))
 	    {
 	       gfc_error ("Type name %qs at %C is ambiguous", name);
 	       return MATCH_ERROR;
 	    }
-	  else if (!sym && gfc_current_ns->proc_name->ns->parent !=  NULL
+	  else if (!sym
+		   && gfc_current_ns->proc_name
+		   && gfc_current_ns->proc_name->ns->parent
 		   && gfc_find_symbol (name,
 				       gfc_current_ns->proc_name->ns->parent,
 				       1, &sym))
@@ -5166,12 +5291,29 @@ gfc_match_import (void)
 
 	  if (sym == NULL)
 	    {
-	      gfc_error ("Cannot IMPORT %qs from host scoping unit "
-			 "at %C - does not exist.", name);
-	      return MATCH_ERROR;
+	      if (gfc_current_ns->proc_name
+		  && gfc_current_ns->proc_name->attr.if_source == IFSRC_IFBODY)
+		{
+		  gfc_error ("Cannot IMPORT %qs from host scoping unit "
+			     "at %C - does not exist.", name);
+		  return MATCH_ERROR;
+		}
+	      else
+		{
+		  /* This might be a procedure that has not yet been parsed. If
+		     so gfc_fixup_sibling_symbols will replace this symbol with
+		     that of the procedure.  */
+		  gfc_get_sym_tree (name, gfc_current_ns, &st, false,
+				    &gfc_current_locus);
+		  st->n.sym->refs++;
+		  st->n.sym->attr.imported = 1;
+		  st->import_only = 1;
+		  goto next_item;
+		}
 	    }
 
-	  if (gfc_find_symtree (gfc_current_ns->sym_root, name))
+	  st = gfc_find_symtree (gfc_current_ns->sym_root, name);
+	  if (st && st->n.sym && st->n.sym->attr.imported)
 	    {
 	      gfc_warning (0, "%qs is already IMPORTed from host scoping unit "
 			   "at %C", name);
@@ -5182,6 +5324,7 @@ gfc_match_import (void)
 	  st->n.sym = sym;
 	  sym->refs++;
 	  sym->attr.imported = 1;
+	  st->import_only = 1;
 
 	  if (sym->attr.generic && (sym = gfc_find_dt_in_generic (sym)))
 	    {
@@ -5193,6 +5336,7 @@ gfc_match_import (void)
 	      st->n.sym = sym;
 	      sym->refs++;
 	      sym->attr.imported = 1;
+	      st->import_only = 1;
 	    }
 
 	  goto next_item;
@@ -5216,6 +5360,34 @@ gfc_match_import (void)
 syntax:
   gfc_error ("Syntax error in IMPORT statement at %C");
   return MATCH_ERROR;
+
+C897:
+  gfc_error ("F2018: C897 IMPORT statement at %C cannot appear in a main "
+	     "program, an external subprogram, a module or block data");
+  return MATCH_ERROR;
+
+C898:
+  gfc_error ("F2018: C898 IMPORT statement at %C is not permitted because "
+	     "a scoping unit has an ONLY specifier, can only have IMPORT "
+	     "with an ONLY specifier");
+  return MATCH_ERROR;
+
+C899:
+  gfc_error ("F2018: C899 IMPORT, NONE shall not appear in the scoping unit"
+	     " of a submodule as at %C");
+  return MATCH_ERROR;
+
+C8100:
+  gfc_error ("F2018: C8100 IMPORT statement at %C is not permitted because "
+	     "%s has already been declared, which must be unique in the "
+	     "scoping unit",
+	     gfc_current_ns->import_state == IMPORT_ALL ? "IMPORT, ALL" :
+							  "IMPORT, NONE");
+  return MATCH_ERROR;
+
+expecting_list:
+  gfc_error ("Expecting list of named entities at %C");
+  return MATCH_ERROR;
 }
 
 
diff --git a/gcc/fortran/error.cc b/gcc/fortran/error.cc
index 004a4b2..ebf9e61 100644
--- a/gcc/fortran/error.cc
+++ b/gcc/fortran/error.cc
@@ -31,9 +31,9 @@ along with GCC; see the file COPYING3.  If not see
 #include "gfortran.h"
 
 #include "diagnostic.h"
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
 #include "tree-diagnostic.h" /* tree_diagnostics_defaults */
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 
 static int suppress_errors = 0;
 
@@ -43,7 +43,7 @@ static bool warnings_not_errors = false;
 static bool buffered_p;
 
 static gfc_error_buffer *error_buffer;
-static diagnostic_buffer *pp_error_buffer, *pp_warning_buffer;
+static diagnostics::buffer *pp_error_buffer, *pp_warning_buffer;
 
 gfc_error_buffer::gfc_error_buffer ()
 : flag (false), buffer (*global_dc)
@@ -228,7 +228,7 @@ gfc_print_wide_char (gfc_char_t c)
    it to global_dc.  */
 
 static void
-gfc_clear_diagnostic_buffer (diagnostic_buffer *this_buffer)
+gfc_clear_diagnostic_buffer (diagnostics::buffer *this_buffer)
 {
   gcc_assert (this_buffer);
   global_dc->clear_diagnostic_buffer (*this_buffer);
@@ -237,13 +237,13 @@ gfc_clear_diagnostic_buffer (diagnostic_buffer *this_buffer)
 /* The currently-printing diagnostic, for use by gfc_format_decoder,
    for colorizing %C and %L.  */
 
-static diagnostic_info *curr_diagnostic;
+static diagnostics::diagnostic_info *curr_diagnostic;
 
 /* A helper function to call diagnostic_report_diagnostic, while setting
    curr_diagnostic for the duration of the call.  */
 
 static bool
-gfc_report_diagnostic (diagnostic_info *diagnostic)
+gfc_report_diagnostic (diagnostics::diagnostic_info *diagnostic)
 {
   gcc_assert (diagnostic != NULL);
   curr_diagnostic = diagnostic;
@@ -261,9 +261,9 @@ gfc_warning (int opt, const char *gmsgid, va_list ap)
   va_list argp;
   va_copy (argp, ap);
 
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
-  diagnostic_buffer *old_buffer = global_dc->get_diagnostic_buffer ();
+  diagnostics::buffer *old_buffer = global_dc->get_diagnostic_buffer ();
   gcc_assert (!old_buffer);
 
   gfc_clear_diagnostic_buffer (pp_warning_buffer);
@@ -272,8 +272,8 @@ gfc_warning (int opt, const char *gmsgid, va_list ap)
     global_dc->set_diagnostic_buffer (pp_warning_buffer);
 
   diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
-		       DK_WARNING);
-  diagnostic.option_id = opt;
+		       diagnostics::kind::warning);
+  diagnostic.m_option_id = opt;
   bool ret = gfc_report_diagnostic (&diagnostic);
 
   if (buffered_p)
@@ -441,7 +441,7 @@ gfc_format_decoder (pretty_printer *pp, text_info *text, const char *spec,
 	const char *color
 	  = (loc_num
 	     ? "range1"
-	     : diagnostic_get_color_for_kind (curr_diagnostic->kind));
+	     : diagnostics::get_color_for_kind (curr_diagnostic->m_kind));
 	pp_string (pp, colorize_start (pp_show_color (pp), color));
 	pp_string (pp, result[loc_num]);
 	pp_string (pp, colorize_stop (pp_show_color (pp)));
@@ -460,8 +460,8 @@ gfc_format_decoder (pretty_printer *pp, text_info *text, const char *spec,
 /* Return a malloc'd string describing the kind of diagnostic.  The
    caller is responsible for freeing the memory.  */
 static char *
-gfc_diagnostic_build_kind_prefix (diagnostic_context *context,
-				  const diagnostic_info *diagnostic)
+gfc_diagnostic_build_kind_prefix (diagnostics::context *context,
+				  const diagnostics::diagnostic_info *diagnostic)
 {
   static const char *const diagnostic_kind_text[] = {
 #define DEFINE_DIAGNOSTIC_KIND(K, T, C) (T),
@@ -475,15 +475,16 @@ gfc_diagnostic_build_kind_prefix (diagnostic_context *context,
 #undef DEFINE_DIAGNOSTIC_KIND
     NULL
   };
-  gcc_assert (diagnostic->kind < DK_LAST_DIAGNOSTIC_KIND);
-  const char *text = _(diagnostic_kind_text[diagnostic->kind]);
+  const int diag_kind_idx = static_cast<int> (diagnostic->m_kind);
+  gcc_assert (diagnostic->m_kind < diagnostics::kind::last_diagnostic_kind);
+  const char *text = _(diagnostic_kind_text[diag_kind_idx]);
   const char *text_cs = "", *text_ce = "";
   pretty_printer *const pp = context->get_reference_printer ();
 
-  if (diagnostic_kind_color[diagnostic->kind])
+if (diagnostic_kind_color[diag_kind_idx])
     {
       text_cs = colorize_start (pp_show_color (pp),
-				diagnostic_kind_color[diagnostic->kind]);
+				diagnostic_kind_color[diag_kind_idx]);
       text_ce = colorize_stop (pp_show_color (pp));
     }
   return build_message_string ("%s%s:%s ", text_cs, text, text_ce);
@@ -492,7 +493,7 @@ gfc_diagnostic_build_kind_prefix (diagnostic_context *context,
 /* Return a malloc'd string describing a location.  The caller is
    responsible for freeing the memory.  */
 static char *
-gfc_diagnostic_build_locus_prefix (const diagnostic_location_print_policy &loc_policy,
+gfc_diagnostic_build_locus_prefix (const diagnostics::location_print_policy &loc_policy,
 				   expanded_location s,
 				   bool colorize)
 {
@@ -511,7 +512,7 @@ gfc_diagnostic_build_locus_prefix (const diagnostic_location_print_policy &loc_p
 /* Return a malloc'd string describing two locations.  The caller is
    responsible for freeing the memory.  */
 static char *
-gfc_diagnostic_build_locus_prefix (const diagnostic_location_print_policy &loc_policy,
+gfc_diagnostic_build_locus_prefix (const diagnostics::location_print_policy &loc_policy,
 				   expanded_location s, expanded_location s2,
 				   bool colorize)
 {
@@ -548,16 +549,16 @@ gfc_diagnostic_build_locus_prefix (const diagnostic_location_print_policy &loc_p
        [locus of primary range]: Error: Some error at (1) and (2)
 */
 static void
-gfc_diagnostic_text_starter (diagnostic_text_output_format &text_output,
-			     const diagnostic_info *diagnostic)
+gfc_diagnostic_text_starter (diagnostics::text_sink &text_output,
+			     const diagnostics::diagnostic_info *diagnostic)
 {
-  diagnostic_context *const context = &text_output.get_context ();
+  diagnostics::context *const context = &text_output.get_context ();
   pretty_printer *const pp = text_output.get_printer ();
   char * kind_prefix = gfc_diagnostic_build_kind_prefix (context, diagnostic);
 
   expanded_location s1 = diagnostic_expand_location (diagnostic);
   expanded_location s2;
-  bool one_locus = diagnostic->richloc->get_num_locations () < 2;
+  bool one_locus = diagnostic->m_richloc->get_num_locations () < 2;
   bool same_locus = false;
 
   if (!one_locus)
@@ -566,13 +567,13 @@ gfc_diagnostic_text_starter (diagnostic_text_output_format &text_output,
       same_locus = diagnostic_same_line (context, s1, s2);
     }
 
-  diagnostic_location_print_policy loc_policy (text_output);
+  diagnostics::location_print_policy loc_policy (text_output);
   const bool colorize = pp_show_color (pp);
   char * locus_prefix = (one_locus || !same_locus)
     ? gfc_diagnostic_build_locus_prefix (loc_policy, s1, colorize)
     : gfc_diagnostic_build_locus_prefix (loc_policy, s1, s2, colorize);
 
-  if (!context->m_source_printing.enabled
+  if (!context->get_source_printing_options ().enabled
       || diagnostic_location (diagnostic, 0) <= BUILTINS_LOCATION
       || diagnostic_location (diagnostic, 0) == context->m_last_location)
     {
@@ -608,7 +609,7 @@ gfc_diagnostic_text_starter (diagnostic_text_output_format &text_output,
       pp_newline (pp);
       diagnostic_show_locus (context,
 			     text_output.get_source_printing_options (),
-			     diagnostic->richloc, diagnostic->kind,
+			     diagnostic->m_richloc, diagnostic->m_kind,
 			     pp);
       /* If the caret line was shown, the prefix does not contain the
 	 locus.  */
@@ -617,11 +618,11 @@ gfc_diagnostic_text_starter (diagnostic_text_output_format &text_output,
 }
 
 static void
-gfc_diagnostic_start_span (const diagnostic_location_print_policy &loc_policy,
-			   to_text &sink,
+gfc_diagnostic_start_span (const diagnostics::location_print_policy &loc_policy,
+			   diagnostics::to_text &sink,
 			   expanded_location exploc)
 {
-  pretty_printer *pp = get_printer (sink);
+  pretty_printer *pp = diagnostics::get_printer (sink);
   const bool colorize = pp_show_color (pp);
   char *locus_prefix
     = gfc_diagnostic_build_locus_prefix (loc_policy, exploc, colorize);
@@ -634,9 +635,9 @@ gfc_diagnostic_start_span (const diagnostic_location_print_policy &loc_policy,
 
 
 static void
-gfc_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
-			       const diagnostic_info *diagnostic ATTRIBUTE_UNUSED,
-			       diagnostic_t orig_diag_kind ATTRIBUTE_UNUSED)
+gfc_diagnostic_text_finalizer (diagnostics::text_sink &text_output,
+			       const diagnostics::diagnostic_info *,
+			       enum diagnostics::kind orig_diag_kind ATTRIBUTE_UNUSED)
 {
   pretty_printer *const pp = text_output.get_printer ();
   pp_destroy_prefix (pp);
@@ -650,13 +651,14 @@ bool
 gfc_warning_now_at (location_t loc, int opt, const char *gmsgid, ...)
 {
   va_list argp;
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, loc);
   bool ret;
 
   va_start (argp, gmsgid);
-  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc, DK_WARNING);
-  diagnostic.option_id = opt;
+  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
+		       diagnostics::kind::warning);
+  diagnostic.m_option_id = opt;
   ret = gfc_report_diagnostic (&diagnostic);
   va_end (argp);
   return ret;
@@ -668,14 +670,14 @@ bool
 gfc_warning_now (int opt, const char *gmsgid, ...)
 {
   va_list argp;
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
   bool ret;
 
   va_start (argp, gmsgid);
   diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
-		       DK_WARNING);
-  diagnostic.option_id = opt;
+		       diagnostics::kind::warning);
+  diagnostic.m_option_id = opt;
   ret = gfc_report_diagnostic (&diagnostic);
   va_end (argp);
   return ret;
@@ -687,14 +689,14 @@ bool
 gfc_warning_internal (int opt, const char *gmsgid, ...)
 {
   va_list argp;
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
   bool ret;
 
   va_start (argp, gmsgid);
   diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
-		       DK_WARNING);
-  diagnostic.option_id = opt;
+		       diagnostics::kind::warning);
+  diagnostic.m_option_id = opt;
   ret = gfc_report_diagnostic (&diagnostic);
   va_end (argp);
   return ret;
@@ -706,13 +708,14 @@ void
 gfc_error_now (const char *gmsgid, ...)
 {
   va_list argp;
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
 
   error_buffer->flag = true;
 
   va_start (argp, gmsgid);
-  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc, DK_ERROR);
+  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
+		       diagnostics::kind::error);
   gfc_report_diagnostic (&diagnostic);
   va_end (argp);
 }
@@ -724,11 +727,12 @@ void
 gfc_fatal_error (const char *gmsgid, ...)
 {
   va_list argp;
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
 
   va_start (argp, gmsgid);
-  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc, DK_FATAL);
+  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
+		       diagnostics::kind::fatal);
   gfc_report_diagnostic (&diagnostic);
   va_end (argp);
 
@@ -776,9 +780,9 @@ gfc_error_opt (int opt, const char *gmsgid, va_list ap)
       return;
     }
 
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location richloc (line_table, UNKNOWN_LOCATION);
-  diagnostic_buffer *old_buffer = global_dc->get_diagnostic_buffer ();
+  diagnostics::buffer *old_buffer = global_dc->get_diagnostic_buffer ();
   gcc_assert (!old_buffer);
 
   gfc_clear_diagnostic_buffer (pp_error_buffer);
@@ -786,7 +790,8 @@ gfc_error_opt (int opt, const char *gmsgid, va_list ap)
   if (buffered_p)
     global_dc->set_diagnostic_buffer (pp_error_buffer);
 
-  diagnostic_set_info (&diagnostic, gmsgid, &argp, &richloc, DK_ERROR);
+  diagnostic_set_info (&diagnostic, gmsgid, &argp, &richloc,
+		       diagnostics::kind::error);
   gfc_report_diagnostic (&diagnostic);
 
   if (buffered_p)
@@ -823,7 +828,7 @@ gfc_internal_error (const char *gmsgid, ...)
 {
   int e, w;
   va_list argp;
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location rich_loc (line_table, UNKNOWN_LOCATION);
 
   gfc_get_errors (&w, &e);
@@ -831,7 +836,8 @@ gfc_internal_error (const char *gmsgid, ...)
     exit(EXIT_FAILURE);
 
   va_start (argp, gmsgid);
-  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc, DK_ICE);
+  diagnostic_set_info (&diagnostic, gmsgid, &argp, &rich_loc,
+		       diagnostics::kind::ice);
   gfc_report_diagnostic (&diagnostic);
   va_end (argp);
 
@@ -885,8 +891,8 @@ static void
 gfc_move_error_buffer_from_to (gfc_error_buffer * buffer_from,
 			       gfc_error_buffer * buffer_to)
 {
-  diagnostic_buffer * from = &(buffer_from->buffer);
-  diagnostic_buffer * to =  &(buffer_to->buffer);
+  diagnostics::buffer * from = &(buffer_from->buffer);
+  diagnostics::buffer * to =  &(buffer_to->buffer);
 
   buffer_to->flag = buffer_from->flag;
   buffer_from->flag = false;
@@ -950,13 +956,14 @@ gfc_errors_to_warnings (bool f)
 void
 gfc_diagnostics_init (void)
 {
-  diagnostic_text_starter (global_dc) = gfc_diagnostic_text_starter;
-  diagnostic_start_span (global_dc) = gfc_diagnostic_start_span;
-  diagnostic_text_finalizer (global_dc) = gfc_diagnostic_text_finalizer;
+  diagnostics::text_starter (global_dc) = gfc_diagnostic_text_starter;
+  diagnostics::start_span (global_dc) = gfc_diagnostic_start_span;
+  diagnostics::text_finalizer (global_dc) = gfc_diagnostic_text_finalizer;
   global_dc->set_format_decoder (gfc_format_decoder);
-  global_dc->m_source_printing.caret_chars[0] = '1';
-  global_dc->m_source_printing.caret_chars[1] = '2';
-  pp_warning_buffer = new diagnostic_buffer (*global_dc);
+  auto &source_printing_opts = global_dc->get_source_printing_options ();
+  source_printing_opts.caret_chars[0] = '1';
+  source_printing_opts.caret_chars[1] = '2';
+  pp_warning_buffer = new diagnostics::buffer (*global_dc);
   error_buffer = new gfc_error_buffer ();
   pp_error_buffer = &(error_buffer->buffer);
 }
@@ -967,10 +974,11 @@ gfc_diagnostics_finish (void)
   tree_diagnostics_defaults (global_dc);
   /* We still want to use the gfc starter and finalizer, not the tree
      defaults.  */
-  diagnostic_text_starter (global_dc) = gfc_diagnostic_text_starter;
-  diagnostic_text_finalizer (global_dc) = gfc_diagnostic_text_finalizer;
-  global_dc->m_source_printing.caret_chars[0] = '^';
-  global_dc->m_source_printing.caret_chars[1] = '^';
+  diagnostics::text_starter (global_dc) = gfc_diagnostic_text_starter;
+  diagnostics::text_finalizer (global_dc) = gfc_diagnostic_text_finalizer;
+  auto &source_printing_opts = global_dc->get_source_printing_options ();
+  source_printing_opts.caret_chars[0] = '^';
+  source_printing_opts.caret_chars[1] = '^';
   delete error_buffer;
   error_buffer = nullptr;
   pp_error_buffer = nullptr;
diff --git a/gcc/fortran/expr.cc b/gcc/fortran/expr.cc
index b0495b7..b8d04ff 100644
--- a/gcc/fortran/expr.cc
+++ b/gcc/fortran/expr.cc
@@ -1372,7 +1372,7 @@ simplify_constructor (gfc_constructor_base base, int type)
 	      || !gfc_simplify_expr (c->iterator->step, type)))
 	return false;
 
-      if (c->expr)
+      if (c->expr && c->expr->expr_type != EXPR_CONSTANT)
 	{
 	  /* Try and simplify a copy.  Replace the original if successful
 	     but keep going through the constructor at all costs.  Not
@@ -2469,7 +2469,8 @@ gfc_simplify_expr (gfc_expr *p, int type)
 	{
 	  if (!simplify_parameter_variable (p, type))
 	    return false;
-	  break;
+	  if (!iter_stack)
+	    break;
 	}
 
       if (type == 1)
diff --git a/gcc/fortran/gfortran.h b/gcc/fortran/gfortran.h
index 6848bd1..d9dcd1b 100644
--- a/gcc/fortran/gfortran.h
+++ b/gcc/fortran/gfortran.h
@@ -729,6 +729,8 @@ enum gfc_isym_id
   GFC_ISYM_COSPI,
   GFC_ISYM_SINPI,
   GFC_ISYM_TANPI,
+
+  GFC_ISYM_SPLIT,
 };
 
 enum init_local_logical
@@ -2028,10 +2030,6 @@ typedef struct gfc_symbol
   /* Set if this should be passed by value, but is not a VALUE argument
      according to the Fortran standard.  */
   unsigned pass_as_value:1;
-  /* Set if an allocatable array variable has been allocated in the current
-     scope. Used in the suppression of uninitialized warnings in reallocation
-     on assignment.  */
-  unsigned allocated_in_scope:1;
   /* Set if an external dummy argument is called with different argument lists.
      This is legal in Fortran, but can cause problems with autogenerated
      C prototypes for C23.  */
@@ -2188,6 +2186,7 @@ typedef struct gfc_symtree
     gfc_omp_udr *omp_udr;
   }
   n;
+  unsigned import_only:1;
 }
 gfc_symtree;
 
@@ -2215,6 +2214,17 @@ typedef struct gfc_was_finalized {
 }
 gfc_was_finalized;
 
+
+  /* Flag F2018 import status */
+enum importstate
+{ IMPORT_NOT_SET = 0,	/* Default condition.  */
+  IMPORT_F2008,		/* Old style IMPORT.  */
+  IMPORT_ONLY,		/* Import list used.  */
+  IMPORT_NONE,		/* No host association.  Unique in scoping unit.  */
+  IMPORT_ALL		/* Must be unique in the scoping unit.  */
+};
+
+
 /* A namespace describes the contents of procedure, module, interface block
    or BLOCK construct.  */
 /* ??? Anything else use these?  */
@@ -2328,6 +2338,10 @@ typedef struct gfc_namespace
   /* Set to 1 if namespace is an interface body with "IMPORT" used.  */
   unsigned has_import_set:1;
 
+  /* Flag F2018 import status */
+  ENUM_BITFIELD (importstate) import_state :3;
+
+
   /* Set to 1 if the namespace uses "IMPLICIT NONE (export)".  */
   unsigned has_implicit_none_export:1;
 
@@ -3582,11 +3596,11 @@ bool gfc_notify_std (int, const char *, ...) ATTRIBUTE_GCC_GFC(2,3);
 #define gfc_syntax_error(ST)	\
   gfc_error ("Syntax error in %s statement at %C", gfc_ascii_statement (ST));
 
-#include "diagnostic-buffer.h"  /* For diagnostic_buffer.  */
+#include "diagnostics/buffering.h"  /* For diagnostics::buffer.  */
 struct gfc_error_buffer
 {
   bool flag;
-  diagnostic_buffer buffer;
+  diagnostics::buffer buffer;
 
   gfc_error_buffer();
 };
diff --git a/gcc/fortran/interface.cc b/gcc/fortran/interface.cc
index f74fbf0..d08f683 100644
--- a/gcc/fortran/interface.cc
+++ b/gcc/fortran/interface.cc
@@ -4781,6 +4781,13 @@ matching_typebound_op (gfc_expr** tb_base,
 		gfc_actual_arglist* argcopy;
 		bool matches;
 
+		/* If expression matching comes here during parsing, eg. when
+		   parsing ASSOCIATE, generic TBPs have not yet been resolved
+		   and g->specific will not have been set. Wait for expression
+		   resolution by returning NULL.  */
+		if (!g->specific && !gfc_current_ns->resolved)
+		  return NULL;
+
 		gcc_assert (g->specific);
 		if (g->specific->error)
 		  continue;
diff --git a/gcc/fortran/intrinsic.cc b/gcc/fortran/intrinsic.cc
index 9e07627..c99a7a8 100644
--- a/gcc/fortran/intrinsic.cc
+++ b/gcc/fortran/intrinsic.cc
@@ -3933,6 +3933,14 @@ add_subroutines (void)
 	      pt, BT_INTEGER, di, OPTIONAL, INTENT_IN,
 	      gt, BT_INTEGER, di, OPTIONAL, INTENT_OUT);
 
+  add_sym_4s ("split", GFC_ISYM_SPLIT, CLASS_PURE,
+	      BT_UNKNOWN, 0, GFC_STD_F2023,
+	      gfc_check_split, NULL, gfc_resolve_split,
+	      "string", BT_CHARACTER, dc, REQUIRED, INTENT_IN,
+	      "set", BT_CHARACTER, dc, REQUIRED, INTENT_IN,
+	      "pos", BT_INTEGER, di, REQUIRED, INTENT_INOUT,
+	      "back", BT_LOGICAL, dl, OPTIONAL, INTENT_IN);
+
   /* The following subroutines are part of ISO_C_BINDING.  */
 
   add_sym_3s ("c_f_pointer", GFC_ISYM_C_F_POINTER, CLASS_IMPURE, BT_UNKNOWN, 0,
diff --git a/gcc/fortran/intrinsic.h b/gcc/fortran/intrinsic.h
index fd54588..8a0ab93 100644
--- a/gcc/fortran/intrinsic.h
+++ b/gcc/fortran/intrinsic.h
@@ -215,6 +215,7 @@ bool gfc_check_mvbits (gfc_expr *, gfc_expr *, gfc_expr *, gfc_expr *,
 bool gfc_check_random_init (gfc_expr *, gfc_expr *);
 bool gfc_check_random_number (gfc_expr *);
 bool gfc_check_random_seed (gfc_expr *, gfc_expr *, gfc_expr *);
+bool gfc_check_split (gfc_expr *, gfc_expr *, gfc_expr *, gfc_expr *);
 bool gfc_check_dtime_etime_sub (gfc_expr *, gfc_expr *);
 bool gfc_check_fgetputc_sub (gfc_expr *, gfc_expr *, gfc_expr *);
 bool gfc_check_fgetput_sub (gfc_expr *, gfc_expr *);
@@ -693,6 +694,7 @@ void gfc_resolve_link_sub (gfc_code *);
 void gfc_resolve_symlnk_sub (gfc_code *);
 void gfc_resolve_signal_sub (gfc_code *);
 void gfc_resolve_sleep_sub (gfc_code *);
+void gfc_resolve_split (gfc_code *);
 void gfc_resolve_stat_sub (gfc_code *);
 void gfc_resolve_system_clock (gfc_code *);
 void gfc_resolve_system_sub (gfc_code *);
diff --git a/gcc/fortran/intrinsic.texi b/gcc/fortran/intrinsic.texi
index 3103da3..a24b234 100644
--- a/gcc/fortran/intrinsic.texi
+++ b/gcc/fortran/intrinsic.texi
@@ -313,6 +313,7 @@ Some basic guidelines for editing this document:
 * @code{SIZEOF}:        SIZEOF,    Determine the size in bytes of an expression
 * @code{SLEEP}:         SLEEP,     Sleep for the specified number of seconds
 * @code{SPACING}:       SPACING,   Smallest distance between two numbers of a given type
+* @code{SPLIT}:         SPLIT,     Parse a string into tokens, one at a time.
 * @code{SPREAD}:        SPREAD,    Add a dimension to an array 
 * @code{SQRT}:          SQRT,      Square-root function
 * @code{SRAND}:         SRAND,     Reinitialize the random number generator
@@ -14203,6 +14204,69 @@ Fortran 90 and later
 
 
 
+@node SPLIT
+@section @code{SPLIT} --- Parse a string into tokens, one at a time
+@fnindex SPLIT
+@cindex string, split
+
+@table @asis
+@item @emph{Synopsis}:
+@code{RESULT = SPLIT(STRING, SET, POS [, BACK])}
+
+@item @emph{Description}:
+Updates the integer @var{POS} to the position of the next (or previous)
+separator in @var{STRING}.
+
+If @var{BACK} is absent or is present with the value false, @var{POS} is
+assigned the position of the leftmost token delimiter in @var{STRING} whose
+position is greater than @var{POS}, or if there is no such character, it is
+assigned a value one greater than the length of @var{STRING}. This identifies
+a token with starting position one greater than the value of @var{POS} on
+invocation, and ending position one less than the value of @var{POS} on return.
+
+If @var{BACK} is present with the value true, @var{POS} is assigned the
+position of the rightmost token delimiter in @var{STRING} whose position is
+less than @var{POS}, or if there is no such character, it is assigned the value
+zero. This identifies a token with ending position one less than the value of
+@var{POS} on invocation, and starting position one greater than the value of
+@var{POS} on return.
+
+@item @emph{Class}:
+Subroutine
+
+@item @emph{Arguments}:
+@multitable @columnfractions .15 .70
+@item @var{STRING} @tab Shall be of type @code{CHARACTER}.
+@item @var{SET}    @tab Shall be of type @code{CHARACTER}.
+@item @var{POS}    @tab Shall be of type @code{INTEGER}.
+@item @var{BACK}   @tab (Optional) Shall be of type @code{LOGICAL}.
+@end multitable
+
+@item @emph{Example}:
+@smallexample
+character(len=:), allocatable :: input
+character(len=2) :: set = ', '
+integer :: p
+input = "one,last example"
+p = 0
+do
+  if (p > len(input)) exit
+  istart = p + 1
+  call split(input, set, p)
+  iend = p - 1
+  print '(t7, a)', input(istart:iend)
+end do
+@end smallexample
+
+@item @emph{Standard}:
+Fortran 2023
+
+@item @emph{See also}:
+@ref{SCAN}
+@end table
+
+
+
 @node SPREAD
 @section @code{SPREAD} --- Add a dimension to an array
 @fnindex SPREAD
diff --git a/gcc/fortran/invoke.texi b/gcc/fortran/invoke.texi
index da085d1..0b893e8 100644
--- a/gcc/fortran/invoke.texi
+++ b/gcc/fortran/invoke.texi
@@ -1170,6 +1170,10 @@ A @code{CHARACTER} variable is declared with negative length.
 With @option{-fopenmp}, for fixed-form source code, when an @code{omx}
 vendor-extension sentinel is encountered. (The equivalent @code{ompx},
 used in free-form source code, is diagnosed by default.)
+
+@item
+With @option{-fopenacc}, when using named constances with clauses that
+take a variable as doing so has no effect.
 @end itemize
 
 @opindex Wtabs
diff --git a/gcc/fortran/io.cc b/gcc/fortran/io.cc
index 7466d8f..4d28c2c 100644
--- a/gcc/fortran/io.cc
+++ b/gcc/fortran/io.cc
@@ -29,7 +29,7 @@ along with GCC; see the file COPYING3.  If not see
 
 gfc_st_label
 format_asterisk = {0, NULL, NULL, -1, ST_LABEL_FORMAT, ST_LABEL_FORMAT, NULL,
-		   0, {NULL, NULL}, NULL, 0};
+		   0, {NULL, {NULL}}, NULL, 0};
 
 typedef struct
 {
diff --git a/gcc/fortran/iresolve.cc b/gcc/fortran/iresolve.cc
index 1001309..da354ab 100644
--- a/gcc/fortran/iresolve.cc
+++ b/gcc/fortran/iresolve.cc
@@ -3863,6 +3863,19 @@ gfc_resolve_sleep_sub (gfc_code *c)
   c->resolved_sym = gfc_get_intrinsic_sub_symbol (name);
 }
 
+void
+gfc_resolve_split (gfc_code *c)
+{
+  const char *name;
+  gfc_expr *string;
+
+  string = c->ext.actual->expr;
+  if (string->ts.type == BT_CHARACTER && string->ts.kind == 4)
+    name = "__split_char4";
+  else
+    name = "__split";
+  c->resolved_sym = gfc_get_intrinsic_sub_symbol (name);
+}
 
 /* G77 compatibility function srand().  */
 
diff --git a/gcc/fortran/openmp.cc b/gcc/fortran/openmp.cc
index fe0a47a..f1acc00 100644
--- a/gcc/fortran/openmp.cc
+++ b/gcc/fortran/openmp.cc
@@ -8895,15 +8895,21 @@ resolve_omp_clauses (gfc_code *code, gfc_omp_clauses *omp_clauses,
 	if (list == OMP_LIST_MAP
 	    && n->sym->attr.flavor == FL_PARAMETER)
 	  {
+	    /* OpenACC since 3.4 permits for Fortran named constants, but
+	       permits removing then as optimization is not needed and such
+	       ignore them. Likewise below for FIRSTPRIVATE.  */
 	    if (openacc)
-	      gfc_error ("Object %qs is not a variable at %L; parameters"
-			 " cannot be and need not be copied", n->sym->name,
-			 &n->where);
+	      gfc_warning (OPT_Wsurprising, "Clause for object %qs at %L is "
+			   "ignored as parameters need not be copied",
+			   n->sym->name, &n->where);
 	    else
 	      gfc_error ("Object %qs is not a variable at %L; parameters"
 			 " cannot be and need not be mapped", n->sym->name,
 			 &n->where);
 	  }
+	else if (openacc && n->sym->attr.flavor == FL_PARAMETER)
+	  gfc_warning (OPT_Wsurprising, "Clause for object %qs at %L is ignored"
+		       " as it is a parameter", n->sym->name, &n->where);
 	else if (list != OMP_LIST_USES_ALLOCATORS)
 	  gfc_error ("Object %qs is not a variable at %L", n->sym->name,
 		     &n->where);
@@ -12756,9 +12762,21 @@ gfc_resolve_oacc_declare (gfc_namespace *ns)
 		&& (n->sym->attr.flavor != FL_PROCEDURE
 		    || n->sym->result != n->sym))
 	      {
-		gfc_error ("Object %qs is not a variable at %L",
-			   n->sym->name, &oc->loc);
-		continue;
+		if (n->sym->attr.flavor != FL_PARAMETER)
+		  {
+		    gfc_error ("Object %qs is not a variable at %L",
+			       n->sym->name, &oc->loc);
+		    continue;
+		  }
+		/* Note that OpenACC 3.4 permits name constants, but the
+		   implementation is permitted to ignore the clause;
+		   as semantically, device_resident kind of makes sense
+		   (and the wording with it is a bit odd), the warning
+		   is suppressed.  */
+		if (list != OMP_LIST_DEVICE_RESIDENT)
+		  gfc_warning (OPT_Wsurprising, "Object %qs at %L is ignored as"
+			       " parameters need not be copied", n->sym->name,
+			       &oc->loc);
 	      }
 
 	    if (n->expr && n->expr->ref->type == REF_ARRAY)
diff --git a/gcc/fortran/options.cc b/gcc/fortran/options.cc
index d3c9066..821a8c8 100644
--- a/gcc/fortran/options.cc
+++ b/gcc/fortran/options.cc
@@ -406,7 +406,8 @@ gfc_post_options (const char **pfilename)
       if (warn_line_truncation && !OPTION_SET_P (warnings_are_errors)
 	  && option_unspecified_p (OPT_Wline_truncation))
 	diagnostic_classify_diagnostic (global_dc, OPT_Wline_truncation,
-					DK_ERROR, UNKNOWN_LOCATION);
+					diagnostics::kind::error,
+					UNKNOWN_LOCATION);
     }
   else
     {
diff --git a/gcc/fortran/parse.cc b/gcc/fortran/parse.cc
index 8d4ca39..847ff37 100644
--- a/gcc/fortran/parse.cc
+++ b/gcc/fortran/parse.cc
@@ -6793,6 +6793,7 @@ gfc_fixup_sibling_symbols (gfc_symbol *sym, gfc_namespace *siblings)
   gfc_namespace *ns;
   gfc_symtree *st;
   gfc_symbol *old_sym;
+  bool imported;
 
   for (ns = siblings; ns; ns = ns->sibling)
     {
@@ -6808,6 +6809,7 @@ gfc_fixup_sibling_symbols (gfc_symbol *sym, gfc_namespace *siblings)
 	goto fixup_contained;
 
       old_sym = st->n.sym;
+      imported = old_sym->attr.imported == 1;
       if (old_sym->ns == ns
 	    && !old_sym->attr.contained
 
@@ -6834,7 +6836,8 @@ gfc_fixup_sibling_symbols (gfc_symbol *sym, gfc_namespace *siblings)
 	  /* Replace it with the symbol from the parent namespace.  */
 	  st->n.sym = sym;
 	  sym->refs++;
-
+	  if (imported)
+	    sym->attr.imported = 1;
 	  gfc_release_symbol (old_sym);
 	}
 
diff --git a/gcc/fortran/resolve.cc b/gcc/fortran/resolve.cc
index 4a6e951..c33bd17 100644
--- a/gcc/fortran/resolve.cc
+++ b/gcc/fortran/resolve.cc
@@ -3919,10 +3919,153 @@ found:
 }
 
 
+
+static bool
+check_sym_import_status (gfc_symbol *sym, gfc_symtree *s, gfc_expr *e,
+			 gfc_code *c, gfc_namespace *ns)
+{
+  locus *here;
+
+  /* If the type has been imported then its vtype functions are OK.  */
+  if (e && e->expr_type == EXPR_FUNCTION && sym->attr.vtype)
+    return true;
+
+  if (e)
+    here = &e->where;
+  else
+    here = &c->loc;
+
+  if (s && !s->import_only)
+    s = gfc_find_symtree (ns->sym_root, sym->name);
+
+  if (ns->import_state == IMPORT_ONLY
+      && sym->ns != ns
+      && (!s || !s->import_only))
+    {
+      gfc_error ("F2018: C8102 %qs at %L is host associated but does not "
+		 "appear in an IMPORT or IMPORT, ONLY list", sym->name, here);
+      return false;
+    }
+  else if (ns->import_state == IMPORT_NONE
+	   && sym->ns != ns)
+    {
+      gfc_error ("F2018: C8102 %qs at %L is host associated in a scope that "
+		 "has IMPORT, NONE", sym->name, here);
+      return false;
+    }
+  return true;
+}
+
+
+static bool
+check_import_status (gfc_expr *e)
+{
+  gfc_symtree *st;
+  gfc_ref *ref;
+  gfc_symbol *sym, *der;
+  gfc_namespace *ns = gfc_current_ns;
+
+  switch (e->expr_type)
+    {
+      case EXPR_VARIABLE:
+      case EXPR_FUNCTION:
+      case EXPR_SUBSTRING:
+	sym = e->symtree ? e->symtree->n.sym : NULL;
+
+	/* Check the symbol itself.  */
+	if (sym
+	    && !(ns->proc_name
+		 && (sym == ns->proc_name))
+	    && !check_sym_import_status (sym, e->symtree, e, NULL, ns))
+	  return false;
+
+	/* Check the declared derived type.  */
+	if (sym->ts.type == BT_DERIVED)
+	  {
+	    der = sym->ts.u.derived;
+	    st = gfc_find_symtree (ns->sym_root, der->name);
+
+	    if (!check_sym_import_status (der, st, e, NULL, ns))
+	      return false;
+	  }
+	else if (sym->ts.type == BT_CLASS && !UNLIMITED_POLY (sym))
+	  {
+	    der = CLASS_DATA (sym) ? CLASS_DATA (sym)->ts.u.derived
+				   : sym->ts.u.derived;
+	    st = gfc_find_symtree (ns->sym_root, der->name);
+
+	    if (!check_sym_import_status (der, st, e, NULL, ns))
+	      return false;
+	  }
+
+	/* Check the declared derived types of component references.  */
+	for (ref = e->ref; ref; ref = ref->next)
+	  if (ref->type == REF_COMPONENT)
+	    {
+	      gfc_component *c = ref->u.c.component;
+	      if (c->ts.type == BT_DERIVED)
+		{
+		  der = c->ts.u.derived;
+		  st = gfc_find_symtree (ns->sym_root, der->name);
+		  if (!check_sym_import_status (der, st, e, NULL, ns))
+		    return false;
+		}
+	      else if (c->ts.type == BT_CLASS && !UNLIMITED_POLY (c))
+		{
+		  der = CLASS_DATA (c) ? CLASS_DATA (c)->ts.u.derived
+				       : c->ts.u.derived;
+		  st = gfc_find_symtree (ns->sym_root, der->name);
+		  if (!check_sym_import_status (der, st, e, NULL, ns))
+		    return false;
+		}
+	    }
+
+	break;
+
+      case EXPR_ARRAY:
+      case EXPR_STRUCTURE:
+	/* Check the declared derived type.  */
+	if (e->ts.type == BT_DERIVED)
+	  {
+	    der = e->ts.u.derived;
+	    st = gfc_find_symtree (ns->sym_root, der->name);
+
+	    if (!check_sym_import_status (der, st, e, NULL, ns))
+	      return false;
+	  }
+	else if (e->ts.type == BT_CLASS && !UNLIMITED_POLY (e))
+	  {
+	    der = CLASS_DATA (e) ? CLASS_DATA (e)->ts.u.derived
+				   : e->ts.u.derived;
+	    st = gfc_find_symtree (ns->sym_root, der->name);
+
+	    if (!check_sym_import_status (der, st, e, NULL, ns))
+	      return false;
+	  }
+
+	break;
+
+/* Either not applicable or resolved away
+      case EXPR_OP:
+      case EXPR_UNKNOWN:
+      case EXPR_CONSTANT:
+      case EXPR_NULL:
+      case EXPR_COMPCALL:
+      case EXPR_PPC: */
+
+      default:
+	break;
+    }
+
+  return true;
+}
+
+
 /* Resolve a subroutine call.  Although it was tempting to use the same code
    for functions, subroutines and functions are stored differently and this
    makes things awkward.  */
 
+
 static bool
 resolve_call (gfc_code *c)
 {
@@ -4080,6 +4223,11 @@ resolve_call (gfc_code *c)
 		 "Using subroutine %qs at %L is deprecated",
 		 c->resolved_sym->name, &c->loc);
 
+  csym = c->resolved_sym ? c->resolved_sym : csym;
+  if (t && gfc_current_ns->import_state != IMPORT_NOT_SET && !c->resolved_isym
+      && csym != gfc_current_ns->proc_name)
+    return check_sym_import_status (csym, c->symtree, NULL, c, gfc_current_ns);
+
   return t;
 }
 
@@ -7792,6 +7940,7 @@ fixup_unique_dummy (gfc_expr *e)
     e->symtree = st;
 }
 
+
 /* Resolve an expression.  That is, make sure that types of operands agree
    with their operators, intrinsic operators are converted to function calls
    for overloaded types and unresolved function references are resolved.  */
@@ -7919,6 +8068,9 @@ gfc_resolve_expr (gfc_expr *e)
       && UNLIMITED_POLY (e->symtree->n.sym))
     e->do_not_resolve_again = 1;
 
+  if (t && gfc_current_ns->import_state != IMPORT_NOT_SET)
+    t = check_import_status (e);
+
   return t;
 }
 
@@ -10572,6 +10724,7 @@ resolve_select_type (gfc_code *code, gfc_namespace *old_ns)
   int rank = 0, corank = 0;
   gfc_ref* ref = NULL;
   gfc_expr *selector_expr = NULL;
+  gfc_code *old_code = code;
 
   ns = code->ext.block.ns;
   if (code->expr2)
@@ -10860,6 +11013,18 @@ resolve_select_type (gfc_code *code, gfc_namespace *old_ns)
 	 when this case is actually true, so build a new ASSOCIATE
 	 that does precisely this here (instead of using the
 	 'global' one).  */
+
+      /* First check the derived type import status.  */
+      if (gfc_current_ns->import_state != IMPORT_NOT_SET
+	  && (c->ts.type == BT_DERIVED || c->ts.type == BT_CLASS))
+	{
+	  st = gfc_find_symtree (gfc_current_ns->sym_root,
+				 c->ts.u.derived->name);
+	  if (!check_sym_import_status (c->ts.u.derived, st, NULL, old_code,
+					gfc_current_ns))
+	    error++;
+	}
+
       const char * var_name = gfc_var_name_for_select_type_temp (orig_expr1);
       if (c->ts.type == BT_CLASS)
 	snprintf (name, sizeof (name), "__tmp_class_%s_%s",
diff --git a/gcc/fortran/trans-array.cc b/gcc/fortran/trans-array.cc
index 7be2d7b..990aaaf 100644
--- a/gcc/fortran/trans-array.cc
+++ b/gcc/fortran/trans-array.cc
@@ -1426,12 +1426,6 @@ get_class_info_from_ss (stmtblock_t * pre, gfc_ss *ss, tree *eltype,
 	  tmp2 = gfc_class_len_get (class_expr);
 	  gfc_add_modify (pre, tmp, tmp2);
 	}
-
-      if (rhs_function)
-	{
-	  tmp = gfc_class_data_get (class_expr);
-	  gfc_conv_descriptor_offset_set (pre, tmp, gfc_index_zero_node);
-	}
     }
   else if (rhs_ss->info->data.array.descriptor)
    {
@@ -3372,18 +3366,51 @@ gfc_add_loop_ss_code (gfc_loopinfo * loop, gfc_ss * ss, bool subscript,
 	  break;
 
 	case GFC_SS_FUNCTION:
-	  /* Array function return value.  We call the function and save its
-	     result in a temporary for use inside the loop.  */
-	  gfc_init_se (&se, NULL);
-	  se.loop = loop;
-	  se.ss = ss;
-	  if (gfc_is_class_array_function (expr))
-	    expr->must_finalize = 1;
-	  gfc_conv_expr (&se, expr);
-	  gfc_add_block_to_block (&outer_loop->pre, &se.pre);
-	  gfc_add_block_to_block (&outer_loop->post, &se.post);
-	  gfc_add_block_to_block (&outer_loop->post, &se.finalblock);
-	  ss_info->string_length = se.string_length;
+	  {
+	    /* Array function return value.  We call the function and save its
+	       result in a temporary for use inside the loop.  */
+	    gfc_init_se (&se, NULL);
+	    se.loop = loop;
+	    se.ss = ss;
+	    bool class_func = gfc_is_class_array_function (expr);
+	    if (class_func)
+	      expr->must_finalize = 1;
+	    gfc_conv_expr (&se, expr);
+	    gfc_add_block_to_block (&outer_loop->pre, &se.pre);
+	    if (class_func
+		&& se.expr
+		&& GFC_CLASS_TYPE_P (TREE_TYPE (se.expr)))
+	      {
+		tree tmp = gfc_class_data_get (se.expr);
+		info->descriptor = tmp;
+		info->data = gfc_conv_descriptor_data_get (tmp);
+		info->offset = gfc_conv_descriptor_offset_get (tmp);
+		for (gfc_ss *s = ss; s; s = s->parent)
+		  for (int n = 0; n < s->dimen; n++)
+		    {
+		      int dim = s->dim[n];
+		      tree tree_dim = gfc_rank_cst[dim];
+
+		      tree start;
+		      start = gfc_conv_descriptor_lbound_get (tmp, tree_dim);
+		      start = gfc_evaluate_now (start, &outer_loop->pre);
+		      info->start[dim] = start;
+
+		      tree end;
+		      end = gfc_conv_descriptor_ubound_get (tmp, tree_dim);
+		      end = gfc_evaluate_now (end, &outer_loop->pre);
+		      info->end[dim] = end;
+
+		      tree stride;
+		      stride = gfc_conv_descriptor_stride_get (tmp, tree_dim);
+		      stride = gfc_evaluate_now (stride, &outer_loop->pre);
+		      info->stride[dim] = stride;
+		    }
+	      }
+	    gfc_add_block_to_block (&outer_loop->post, &se.post);
+	    gfc_add_block_to_block (&outer_loop->post, &se.finalblock);
+	    ss_info->string_length = se.string_length;
+	  }
 	  break;
 
 	case GFC_SS_CONSTRUCTOR:
@@ -3420,6 +3447,183 @@ gfc_add_loop_ss_code (gfc_loopinfo * loop, gfc_ss * ss, bool subscript,
 }
 
 
+/* Given an array descriptor expression DESCR and its data pointer DATA, decide
+   whether to either save the data pointer to a variable and use the variable or
+   use the data pointer expression directly without any intermediary variable.
+   */
+
+static bool
+save_descriptor_data (tree descr, tree data)
+{
+  return !(DECL_P (data)
+	   || (TREE_CODE (data) == ADDR_EXPR
+	       && DECL_P (TREE_OPERAND (data, 0)))
+	   || (GFC_DESCRIPTOR_TYPE_P (TREE_TYPE (descr))
+	       && TREE_CODE (descr) == COMPONENT_REF
+	       && GFC_CLASS_TYPE_P (TREE_TYPE (TREE_OPERAND (descr, 0)))));
+}
+
+
+/* Type of the DATA argument passed to walk_tree by substitute_subexpr_in_expr
+   and used by maybe_substitute_expr.  */
+
+typedef struct
+{
+  tree target, repl;
+}
+substitute_t;
+
+
+/* Check if the expression in *TP is equal to the substitution target provided
+   in DATA->TARGET and replace it with DATA->REPL in that case.   This is a
+   callback function for use with walk_tree.  */
+
+static tree
+maybe_substitute_expr (tree *tp, int *walk_subtree, void *data)
+{
+  substitute_t *subst = (substitute_t *) data;
+  if (*tp == subst->target)
+    {
+      *tp = subst->repl;
+      *walk_subtree = 0;
+    }
+
+  return NULL_TREE;
+}
+
+
+/* Substitute in EXPR any occurence of TARGET with REPLACEMENT.  */
+
+static void
+substitute_subexpr_in_expr (tree target, tree replacement, tree expr)
+{
+  substitute_t subst;
+  subst.target = target;
+  subst.repl = replacement;
+
+  walk_tree (&expr, maybe_substitute_expr, &subst, nullptr);
+}
+
+
+/* Save REF to a fresh variable in all of REPLACEMENT_ROOTS, appending extra
+   code to CODE.  Before returning, add REF to REPLACEMENT_ROOTS and clear
+   REF.  */
+
+static void
+save_ref (tree &code, tree &ref, vec<tree> &replacement_roots)
+{
+  stmtblock_t tmp_block;
+  gfc_init_block (&tmp_block);
+  tree var = gfc_evaluate_now (ref, &tmp_block);
+  gfc_add_expr_to_block (&tmp_block, code);
+  code = gfc_finish_block (&tmp_block);
+
+  unsigned i;
+  tree repl_root;
+  FOR_EACH_VEC_ELT (replacement_roots, i, repl_root)
+    substitute_subexpr_in_expr (ref, var, repl_root);
+
+  replacement_roots.safe_push (ref);
+  ref = NULL_TREE;
+}
+
+
+/* Save the descriptor reference VALUE to storage pointed by DESC_PTR.  Before
+   that, try to factor subexpressions of VALUE to variables, adding extra code
+   to BLOCK.
+
+   The candidate references to factoring are dereferenced pointers because they
+   are cheap to copy and array descriptors because they are often the base of
+   multiple subreferences.  */
+
+static void
+set_factored_descriptor_value (tree *desc_ptr, tree value, stmtblock_t *block)
+{
+  /* As the reference is processed from outer to inner, variable definitions
+     will be generated in reversed order, so can't be put directly in BLOCK.
+     We use TMP_BLOCK instead.  */
+  tree accumulated_code = NULL_TREE;
+
+  /* The current candidate to factoring.  */
+  tree saveable_ref = NULL_TREE;
+
+  /* The root expressions in which we look for subexpressions to replace with
+     variables.  */
+  auto_vec<tree> replacement_roots;
+  replacement_roots.safe_push (value);
+
+  tree data_ref = value;
+  tree next_ref = NULL_TREE;
+
+  /* If the candidate reference is not followed by a subreference, it can't be
+     saved to a variable as it may be reallocatable, and we have to keep the
+     parent reference to be able to store the new pointer value in case of
+     reallocation.  */
+  bool maybe_reallocatable = true;
+
+  while (true)
+    {
+      if (!maybe_reallocatable
+	  && GFC_DESCRIPTOR_TYPE_P (TREE_TYPE (data_ref)))
+	saveable_ref = data_ref;
+
+      if (TREE_CODE (data_ref) == INDIRECT_REF)
+	{
+	  next_ref = TREE_OPERAND (data_ref, 0);
+
+	  if (!maybe_reallocatable)
+	    {
+	      if (saveable_ref != NULL_TREE && saveable_ref != data_ref)
+		{
+		  /* A reference worth saving has been seen, and now the pointer
+		     to the current reference is also worth saving.  If the
+		     previous reference to save wasn't the current one, do save
+		     it now.  Otherwise drop it as we prefer saving the
+		     pointer.  */
+		  save_ref (accumulated_code, saveable_ref, replacement_roots);
+		}
+
+	      /* Don't evaluate the pointer to a variable yet; do it only if the
+		 variable would be significantly more simple than the reference
+		 it replaces.  That is if the reference contains anything
+		 different from NOPs, COMPONENTs and DECLs.  */
+	      saveable_ref = next_ref;
+	    }
+	}
+      else if (TREE_CODE (data_ref) == COMPONENT_REF)
+	{
+	  maybe_reallocatable = false;
+	  next_ref = TREE_OPERAND (data_ref, 0);
+	}
+      else if (TREE_CODE (data_ref) == NOP_EXPR)
+	next_ref = TREE_OPERAND (data_ref, 0);
+      else
+	{
+	  if (DECL_P (data_ref))
+	    break;
+
+	  if (TREE_CODE (data_ref) == ARRAY_REF)
+	    {
+	      maybe_reallocatable = false;
+	      next_ref = TREE_OPERAND (data_ref, 0);
+	    }
+
+	  if (saveable_ref != NULL_TREE)
+	    /* We have seen a reference worth saving.  Do it now.  */
+	    save_ref (accumulated_code, saveable_ref, replacement_roots);
+
+	  if (TREE_CODE (data_ref) != ARRAY_REF)
+	    break;
+	}
+
+      data_ref = next_ref;
+    }
+
+  *desc_ptr = value;
+  gfc_add_expr_to_block (block, accumulated_code);
+}
+
+
 /* Translate expressions for the descriptor and data pointer of a SS.  */
 /*GCC ARRAYS*/
 
@@ -3440,7 +3644,7 @@ gfc_conv_ss_descriptor (stmtblock_t * block, gfc_ss * ss, int base)
   se.descriptor_only = 1;
   gfc_conv_expr_lhs (&se, ss_info->expr);
   gfc_add_block_to_block (block, &se.pre);
-  info->descriptor = se.expr;
+  set_factored_descriptor_value (&info->descriptor, se.expr, block);
   ss_info->string_length = se.string_length;
   ss_info->class_container = se.class_container;
 
@@ -3466,17 +3670,14 @@ gfc_conv_ss_descriptor (stmtblock_t * block, gfc_ss * ss, int base)
          Otherwise we must evaluate it now to avoid breaking dependency
 	 analysis by pulling the expressions for elemental array indices
 	 inside the loop.  */
-      if (!(DECL_P (tmp)
-	    || (TREE_CODE (tmp) == ADDR_EXPR
-		&& DECL_P (TREE_OPERAND (tmp, 0)))
-	    || (GFC_DESCRIPTOR_TYPE_P (TREE_TYPE (se.expr))
-		&& TREE_CODE (se.expr) == COMPONENT_REF
-		&& GFC_CLASS_TYPE_P (TREE_TYPE (TREE_OPERAND (se.expr, 0))))))
+      if (save_descriptor_data (se.expr, tmp) && !ss->is_alloc_lhs)
 	tmp = gfc_evaluate_now (tmp, block);
       info->data = tmp;
 
       tmp = gfc_conv_array_offset (se.expr);
-      info->offset = gfc_evaluate_now (tmp, block);
+      if (!ss->is_alloc_lhs)
+	tmp = gfc_evaluate_now (tmp, block);
+      info->offset = tmp;
 
       /* Make absolutely sure that the saved_offset is indeed saved
 	 so that the variable is still accessible after the loops
@@ -4769,13 +4970,12 @@ gfc_trans_scalarized_loop_boundary (gfc_loopinfo * loop, stmtblock_t * body)
 
 static void
 evaluate_bound (stmtblock_t *block, tree *bounds, gfc_expr ** values,
-		tree desc, int dim, bool lbound, bool deferred)
+		tree desc, int dim, bool lbound, bool deferred, bool save_value)
 {
   gfc_se se;
   gfc_expr * input_val = values[dim];
   tree *output = &bounds[dim];
 
-
   if (input_val)
     {
       /* Specified section bound.  */
@@ -4801,7 +5001,8 @@ evaluate_bound (stmtblock_t *block, tree *bounds, gfc_expr ** values,
       *output = lbound ? gfc_conv_array_lbound (desc, dim) :
 			 gfc_conv_array_ubound (desc, dim);
     }
-  *output = gfc_evaluate_now (*output, block);
+  if (save_value)
+    *output = gfc_evaluate_now (*output, block);
 }
 
 
@@ -4834,18 +5035,18 @@ gfc_conv_section_startstride (stmtblock_t * block, gfc_ss * ss, int dim)
 	      || ar->dimen_type[dim] == DIMEN_THIS_IMAGE);
   desc = info->descriptor;
   stride = ar->stride[dim];
-
+  bool save_value = !ss->is_alloc_lhs;
 
   /* Calculate the start of the range.  For vector subscripts this will
      be the range of the vector.  */
   evaluate_bound (block, info->start, ar->start, desc, dim, true,
-		  ar->as->type == AS_DEFERRED);
+		  ar->as->type == AS_DEFERRED, save_value);
 
   /* Similarly calculate the end.  Although this is not used in the
      scalarizer, it is needed when checking bounds and where the end
      is an expression with side-effects.  */
   evaluate_bound (block, info->end, ar->end, desc, dim, false,
-		  ar->as->type == AS_DEFERRED);
+		  ar->as->type == AS_DEFERRED, save_value);
 
 
   /* Calculate the stride.  */
@@ -4856,7 +5057,11 @@ gfc_conv_section_startstride (stmtblock_t * block, gfc_ss * ss, int dim)
       gfc_init_se (&se, NULL);
       gfc_conv_expr_type (&se, stride, gfc_array_index_type);
       gfc_add_block_to_block (block, &se.pre);
-      info->stride[dim] = gfc_evaluate_now (se.expr, block);
+      tree value = se.expr;
+      if (save_value)
+	info->stride[dim] = gfc_evaluate_now (value, block);
+      else
+	info->stride[dim] = value;
     }
 }
 
@@ -5205,7 +5410,8 @@ done:
 	      int dim = ss->dim[n];
 
 	      info->start[dim]  = gfc_index_zero_node;
-	      info->end[dim]    = gfc_index_zero_node;
+	      if (ss_info->type != GFC_SS_FUNCTION)
+		info->end[dim]    = gfc_index_zero_node;
 	      info->stride[dim] = gfc_index_one_node;
 	    }
 	  break;
@@ -5890,6 +6096,46 @@ set_loop_bounds (gfc_loopinfo *loop)
 }
 
 
+/* Last attempt to set the loop bounds, in case they depend on an allocatable
+   function result.  */
+
+static void
+late_set_loop_bounds (gfc_loopinfo *loop)
+{
+  int n, dim;
+  gfc_array_info *info;
+  gfc_ss **loopspec;
+
+  loopspec = loop->specloop;
+
+  for (n = 0; n < loop->dimen; n++)
+    {
+      /* Set the extents of this range.  */
+      if (loop->from[n] == NULL_TREE
+	  || loop->to[n] == NULL_TREE)
+	{
+	  /* We should have found the scalarization loop specifier.  If not,
+	     that's bad news.  */
+	  gcc_assert (loopspec[n]);
+
+	  info = &loopspec[n]->info->data.array;
+	  dim = loopspec[n]->dim[n];
+
+	  if (loopspec[n]->info->type == GFC_SS_FUNCTION
+	      && info->start[dim]
+	      && info->end[dim])
+	    {
+	      loop->from[n] = info->start[dim];
+	      loop->to[n] = info->end[dim];
+	    }
+	}
+    }
+
+  for (loop = loop->nested; loop; loop = loop->next)
+    late_set_loop_bounds (loop);
+}
+
+
 /* Initialize the scalarization loop.  Creates the loop variables.  Determines
    the range of the loop variables.  Creates a temporary if required.
    Also generates code for scalar expressions which have been
@@ -5908,6 +6154,8 @@ gfc_conv_loop_setup (gfc_loopinfo * loop, locus * where)
      allocating the temporary.  */
   gfc_add_loop_ss_code (loop, loop->ss, false, where);
 
+  late_set_loop_bounds (loop);
+
   tmp_ss = loop->temp_ss;
   /* If we want a temporary then create it.  */
   if (tmp_ss != NULL)
@@ -5964,9 +6212,11 @@ gfc_set_delta (gfc_loopinfo *loop)
       gfc_ss_type ss_type;
 
       ss_type = ss->info->type;
-      if (ss_type != GFC_SS_SECTION
-	  && ss_type != GFC_SS_COMPONENT
-	  && ss_type != GFC_SS_CONSTRUCTOR)
+      if (!(ss_type == GFC_SS_SECTION
+	    || ss_type == GFC_SS_COMPONENT
+	    || ss_type == GFC_SS_CONSTRUCTOR
+	    || (ss_type == GFC_SS_FUNCTION
+		&& gfc_is_class_array_function (ss->info->expr))))
 	continue;
 
       info = &ss->info->data.array;
@@ -5991,7 +6241,10 @@ gfc_set_delta (gfc_loopinfo *loop)
 				     gfc_array_index_type,
 				     info->start[dim], tmp);
 
-	      info->delta[dim] = gfc_evaluate_now (tmp, &outer_loop->pre);
+	      if (ss->is_alloc_lhs)
+		info->delta[dim] = tmp;
+	      else
+		info->delta[dim] = gfc_evaluate_now (tmp, &outer_loop->pre);
 	    }
 	}
     }
@@ -6115,8 +6368,8 @@ static tree
 gfc_array_init_size (tree descriptor, int rank, int corank, tree * poffset,
 		     gfc_expr ** lower, gfc_expr ** upper, stmtblock_t * pblock,
 		     stmtblock_t * descriptor_block, tree * overflow,
-		     tree expr3_elem_size, tree *nelems, gfc_expr *expr3,
-		     tree expr3_desc, bool e3_has_nodescriptor, gfc_expr *expr,
+		     tree expr3_elem_size, gfc_expr *expr3, tree expr3_desc,
+		     bool e3_has_nodescriptor, gfc_expr *expr,
 		     tree *element_size, bool explicit_ts)
 {
   tree type;
@@ -6392,7 +6645,6 @@ gfc_array_init_size (tree descriptor, int rank, int corank, tree * poffset,
   if (rank == 0)
     return *element_size;
 
-  *nelems = gfc_evaluate_now (stride, pblock);
   stride = fold_convert (size_type_node, stride);
 
   /* First check for overflow. Since an array of type character can
@@ -6481,9 +6733,8 @@ retrieve_last_ref (gfc_ref **ref_in, gfc_ref **prev_ref_in)
 bool
 gfc_array_allocate (gfc_se * se, gfc_expr * expr, tree status, tree errmsg,
 		    tree errlen, tree label_finish, tree expr3_elem_size,
-		    tree *nelems, gfc_expr *expr3, tree e3_arr_desc,
-		    bool e3_has_nodescriptor, gfc_omp_namelist *omp_alloc,
-		    bool explicit_ts)
+		    gfc_expr *expr3, tree e3_arr_desc, bool e3_has_nodescriptor,
+		    gfc_omp_namelist *omp_alloc, bool explicit_ts)
 {
   tree tmp;
   tree pointer;
@@ -6614,7 +6865,7 @@ gfc_array_allocate (gfc_se * se, gfc_expr * expr, tree status, tree errmsg,
 			      coarray ? ref->u.ar.as->corank : 0,
 			      &offset, lower, upper,
 			      &se->pre, &set_descriptor_block, &overflow,
-			      expr3_elem_size, nelems, expr3, e3_arr_desc,
+			      expr3_elem_size, expr3, e3_arr_desc,
 			      e3_has_nodescriptor, expr, &element_size,
 			      explicit_ts);
 
@@ -6779,8 +7030,6 @@ gfc_array_allocate (gfc_se * se, gfc_expr * expr, tree status, tree errmsg,
   else
       gfc_add_expr_to_block (&se->pre, set_descriptor);
 
-  expr->symtree->n.sym->allocated_in_scope = 1;
-
   return true;
 }
 
@@ -8470,7 +8719,7 @@ gfc_conv_expr_descriptor (gfc_se *se, gfc_expr *expr)
 	  gcc_assert (n == codim - 1);
 	  evaluate_bound (&loop.pre, info->start, ar->start,
 			  info->descriptor, n + ndim, true,
-			  ar->as->type == AS_DEFERRED);
+			  ar->as->type == AS_DEFERRED, true);
 	  loop.from[n + loop.dimen] = info->start[n + ndim];
 	}
       else
@@ -11206,6 +11455,9 @@ gfc_is_reallocatable_lhs (gfc_expr *expr)
   gfc_ref * ref;
   gfc_symbol *sym;
 
+  if (!flag_realloc_lhs)
+    return false;
+
   if (!expr->ref)
     return false;
 
@@ -11330,6 +11582,55 @@ concat_str_length (gfc_expr* expr)
 }
 
 
+/* Among the scalarization chain of LOOP, find the element associated with an
+   allocatable array on the lhs of an assignment and evaluate its fields
+   (bounds, offset, etc) to new variables, putting the new code in BLOCK.  This
+   function is to be called after putting the reallocation code in BLOCK and
+   before the beginning of the scalarization loop body.
+
+   The fields to be saved are expected to hold on entry to the function
+   expressions referencing the array descriptor.  Especially the expressions
+   shouldn't be already temporary variable references as the value saved before
+   reallocation would be incorrect after reallocation.
+   At the end of the function, the expressions have been replaced with variable
+   references.  */
+
+static void
+update_reallocated_descriptor (stmtblock_t *block, gfc_loopinfo *loop)
+{
+  for (gfc_ss *s = loop->ss; s != gfc_ss_terminator; s = s->loop_chain)
+    {
+      if (!s->is_alloc_lhs)
+	continue;
+
+      gcc_assert (s->info->type == GFC_SS_SECTION);
+      gfc_array_info *info = &s->info->data.array;
+
+#define SAVE_VALUE(value) \
+	      do \
+		{ \
+		  value = gfc_evaluate_now (value, block); \
+		} \
+	      while (0)
+
+      if (save_descriptor_data (info->descriptor, info->data))
+	SAVE_VALUE (info->data);
+      SAVE_VALUE (info->offset);
+      info->saved_offset = info->offset;
+      for (int i = 0; i < s->dimen; i++)
+	{
+	  int dim = s->dim[i];
+	  SAVE_VALUE (info->start[dim]);
+	  SAVE_VALUE (info->end[dim]);
+	  SAVE_VALUE (info->stride[dim]);
+	  SAVE_VALUE (info->delta[dim]);
+	}
+
+#undef SAVE_VALUE
+    }
+}
+
+
 /* Allocate the lhs of an assignment to an allocatable array, otherwise
    reallocate it.  */
 
@@ -11368,7 +11669,6 @@ gfc_alloc_allocatable_for_assignment (gfc_loopinfo *loop,
   tree lbd;
   tree class_expr2 = NULL_TREE;
   int n;
-  int dim;
   gfc_array_spec * as;
   bool coarray = (flag_coarray == GFC_FCOARRAY_LIB
 		  && gfc_caf_attr (expr1, true).codimension);
@@ -11423,14 +11723,61 @@ gfc_alloc_allocatable_for_assignment (gfc_loopinfo *loop,
       && !expr2->value.function.isym)
     expr2->ts.u.cl->backend_decl = rss->info->string_length;
 
-  gfc_start_block (&fblock);
-
   /* Since the lhs is allocatable, this must be a descriptor type.
      Get the data and array size.  */
   desc = linfo->descriptor;
   gcc_assert (GFC_DESCRIPTOR_TYPE_P (TREE_TYPE (desc)));
   array1 = gfc_conv_descriptor_data_get (desc);
 
+  /* If the data is null, set the descriptor bounds and offset.  This suppresses
+     the maybe used uninitialized warning.  Note that the always false variable
+     prevents this block from ever being executed, and makes sure that the
+     optimizers are able to remove it.  Component references are not subject to
+     the warnings, so we don't uselessly complicate the generated code for them.
+     */
+  for (ref = expr1->ref; ref; ref = ref->next)
+    if (ref->type == REF_COMPONENT)
+      break;
+
+  if (!ref)
+    {
+      stmtblock_t unalloc_init_block;
+      gfc_init_block (&unalloc_init_block);
+      tree guard = gfc_create_var (logical_type_node, "unallocated_init_guard");
+      gfc_add_modify (&unalloc_init_block, guard, logical_false_node);
+
+      gfc_start_block (&loop_pre_block);
+      for (n = 0; n < expr1->rank; n++)
+	{
+	  gfc_conv_descriptor_lbound_set (&loop_pre_block, desc,
+					  gfc_rank_cst[n],
+					  gfc_index_one_node);
+	  gfc_conv_descriptor_ubound_set (&loop_pre_block, desc,
+					  gfc_rank_cst[n],
+					  gfc_index_zero_node);
+	  gfc_conv_descriptor_stride_set (&loop_pre_block, desc,
+					  gfc_rank_cst[n],
+					  gfc_index_zero_node);
+	}
+
+      tmp = gfc_conv_descriptor_offset (desc);
+      gfc_add_modify (&loop_pre_block, tmp, gfc_index_zero_node);
+
+      tmp = fold_build2_loc (input_location, EQ_EXPR,
+			     logical_type_node, array1,
+			     build_int_cst (TREE_TYPE (array1), 0));
+      tmp = fold_build2_loc (input_location, TRUTH_ANDIF_EXPR,
+			     logical_type_node, tmp, guard);
+      tmp = build3_v (COND_EXPR, tmp,
+		      gfc_finish_block (&loop_pre_block),
+		      build_empty_stmt (input_location));
+      gfc_prepend_expr_to_block (&loop->pre, tmp);
+      gfc_prepend_expr_to_block (&loop->pre,
+				 gfc_finish_block (&unalloc_init_block));
+    }
+
+  gfc_start_block (&fblock);
+
   if (expr2)
     desc2 = rss->info->data.array.descriptor;
   else
@@ -11543,45 +11890,6 @@ gfc_alloc_allocatable_for_assignment (gfc_loopinfo *loop,
 			 array1, build_int_cst (TREE_TYPE (array1), 0));
   cond_null= gfc_evaluate_now (cond_null, &fblock);
 
-  /* If the data is null, set the descriptor bounds and offset. This suppresses
-     the maybe used uninitialized warning and forces the use of malloc because
-     the size is zero in all dimensions. Note that this block is only executed
-     if the lhs is unallocated and is only applied once in any namespace.
-     Component references are not subject to the warnings.  */
-  for (ref = expr1->ref; ref; ref = ref->next)
-    if (ref->type == REF_COMPONENT)
-      break;
-
-  if (!expr1->symtree->n.sym->allocated_in_scope && !ref)
-    {
-      gfc_start_block (&loop_pre_block);
-      for (n = 0; n < expr1->rank; n++)
-	{
-	  gfc_conv_descriptor_lbound_set (&loop_pre_block, desc,
-					  gfc_rank_cst[n],
-					  gfc_index_one_node);
-	  gfc_conv_descriptor_ubound_set (&loop_pre_block, desc,
-					  gfc_rank_cst[n],
-					  gfc_index_zero_node);
-	  gfc_conv_descriptor_stride_set (&loop_pre_block, desc,
-					  gfc_rank_cst[n],
-					  gfc_index_zero_node);
-	}
-
-      tmp = gfc_conv_descriptor_offset (desc);
-      gfc_add_modify (&loop_pre_block, tmp, gfc_index_zero_node);
-
-      tmp = fold_build2_loc (input_location, EQ_EXPR,
-			     logical_type_node, array1,
-			     build_int_cst (TREE_TYPE (array1), 0));
-      tmp = build3_v (COND_EXPR, tmp,
-		      gfc_finish_block (&loop_pre_block),
-		      build_empty_stmt (input_location));
-      gfc_prepend_expr_to_block (&loop->pre, tmp);
-
-      expr1->symtree->n.sym->allocated_in_scope = 1;
-    }
-
   tmp = build3_v (COND_EXPR, cond_null,
 		  build1_v (GOTO_EXPR, jump_label1),
 		  build_empty_stmt (input_location));
@@ -11736,21 +12044,6 @@ gfc_alloc_allocatable_for_assignment (gfc_loopinfo *loop,
      running offset.  Use the saved_offset instead.  */
   tmp = gfc_conv_descriptor_offset (desc);
   gfc_add_modify (&fblock, tmp, offset);
-  if (linfo->saved_offset
-      && VAR_P (linfo->saved_offset))
-    gfc_add_modify (&fblock, linfo->saved_offset, tmp);
-
-  /* Now set the deltas for the lhs.  */
-  for (n = 0; n < expr1->rank; n++)
-    {
-      tmp = gfc_conv_descriptor_lbound_get (desc, gfc_rank_cst[n]);
-      dim = lss->dim[n];
-      tmp = fold_build2_loc (input_location, MINUS_EXPR,
-			     gfc_array_index_type, tmp,
-			     loop->from[dim]);
-      if (linfo->delta[dim] && VAR_P (linfo->delta[dim]))
-	gfc_add_modify (&fblock, linfo->delta[dim], tmp);
-    }
 
   /* Take into account _len of unlimited polymorphic entities, so that span
      for array descriptors and allocation sizes are computed correctly.  */
@@ -11972,18 +12265,18 @@ gfc_alloc_allocatable_for_assignment (gfc_loopinfo *loop,
   tmp = build3_v (COND_EXPR, cond_null, alloc_expr, realloc_expr);
   gfc_add_expr_to_block (&fblock, tmp);
 
-  /* Make sure that the scalarizer data pointer is updated.  */
-  if (linfo->data && VAR_P (linfo->data))
-    {
-      tmp = gfc_conv_descriptor_data_get (desc);
-      gfc_add_modify (&fblock, linfo->data, tmp);
-    }
-
   /* Add the label for same shape lhs and rhs.  */
   tmp = build1_v (LABEL_EXPR, jump_label2);
   gfc_add_expr_to_block (&fblock, tmp);
 
-  return gfc_finish_block (&fblock);
+  tree realloc_code = gfc_finish_block (&fblock);
+
+  stmtblock_t result_block;
+  gfc_init_block (&result_block);
+  gfc_add_expr_to_block (&result_block, realloc_code);
+  update_reallocated_descriptor (&result_block, loop);
+
+  return gfc_finish_block (&result_block);
 }
 
 
diff --git a/gcc/fortran/trans-array.h b/gcc/fortran/trans-array.h
index 1bb3294..29098fd 100644
--- a/gcc/fortran/trans-array.h
+++ b/gcc/fortran/trans-array.h
@@ -20,9 +20,8 @@ along with GCC; see the file COPYING3.  If not see
 
 /* Generate code to initialize and allocate an array.  Statements are added to
    se, which should contain an expression for the array descriptor.  */
-bool gfc_array_allocate (gfc_se *, gfc_expr *, tree, tree, tree, tree,
-			 tree, tree *, gfc_expr *, tree, bool,
-			 gfc_omp_namelist *, bool);
+bool gfc_array_allocate (gfc_se *, gfc_expr *, tree, tree, tree, tree, tree,
+			 gfc_expr *, tree, bool, gfc_omp_namelist *, bool);
 
 /* Allow the bounds of a loop to be set from a callee's array spec.  */
 void gfc_set_loop_bounds_from_array_spec (gfc_interface_mapping *,
diff --git a/gcc/fortran/trans-decl.cc b/gcc/fortran/trans-decl.cc
index 43bd7be..3b49b18 100644
--- a/gcc/fortran/trans-decl.cc
+++ b/gcc/fortran/trans-decl.cc
@@ -197,6 +197,7 @@ tree gfor_fndecl_string_scan;
 tree gfor_fndecl_string_verify;
 tree gfor_fndecl_string_trim;
 tree gfor_fndecl_string_minmax;
+tree gfor_fndecl_string_split;
 tree gfor_fndecl_adjustl;
 tree gfor_fndecl_adjustr;
 tree gfor_fndecl_select_string;
@@ -208,6 +209,7 @@ tree gfor_fndecl_string_scan_char4;
 tree gfor_fndecl_string_verify_char4;
 tree gfor_fndecl_string_trim_char4;
 tree gfor_fndecl_string_minmax_char4;
+tree gfor_fndecl_string_split_char4;
 tree gfor_fndecl_adjustl_char4;
 tree gfor_fndecl_adjustr_char4;
 tree gfor_fndecl_select_string_char4;
@@ -3569,6 +3571,12 @@ gfc_build_intrinsic_function_decls (void)
 	build_pointer_type (pchar1_type_node), integer_type_node,
 	integer_type_node);
 
+  gfor_fndecl_string_split = gfc_build_library_function_decl_with_spec (
+    get_identifier (PREFIX ("string_split")), ". . R . R . . ",
+    gfc_charlen_type_node, 6, gfc_charlen_type_node, pchar1_type_node,
+    gfc_charlen_type_node, pchar1_type_node, gfc_charlen_type_node,
+    gfc_logical4_type_node);
+
   gfor_fndecl_adjustl = gfc_build_library_function_decl_with_spec (
 	get_identifier (PREFIX("adjustl")), ". W . R ",
 	void_type_node, 3, pchar1_type_node, gfc_charlen_type_node,
@@ -3641,6 +3649,12 @@ gfc_build_intrinsic_function_decls (void)
 	build_pointer_type (pchar4_type_node), integer_type_node,
 	integer_type_node);
 
+  gfor_fndecl_string_split_char4 = gfc_build_library_function_decl_with_spec (
+    get_identifier (PREFIX ("string_split_char4")), ". . R . R . . ",
+    gfc_charlen_type_node, 6, gfc_charlen_type_node, pchar4_type_node,
+    gfc_charlen_type_node, pchar4_type_node, gfc_charlen_type_node,
+    gfc_logical4_type_node);
+
   gfor_fndecl_adjustl_char4 = gfc_build_library_function_decl_with_spec (
 	get_identifier (PREFIX("adjustl_char4")), ". W . R ",
 	void_type_node, 3, pchar4_type_node, gfc_charlen_type_node,
@@ -4773,14 +4787,14 @@ gfc_trans_deferred_vars (gfc_symbol * proc_sym, gfc_wrapped_block * block)
       /* Nullify explicit return class arrays on entry.  */
       tree type;
       tmp = get_proc_result (proc_sym);
-	if (tmp && GFC_CLASS_TYPE_P (TREE_TYPE (tmp)))
-	  {
-	    gfc_start_block (&init);
-	    tmp = gfc_class_data_get (tmp);
-	    type = TREE_TYPE (gfc_conv_descriptor_data_get (tmp));
-	    gfc_conv_descriptor_data_set (&init, tmp, build_int_cst (type, 0));
-	    gfc_add_init_cleanup (block, gfc_finish_block (&init), NULL_TREE);
-	  }
+      if (tmp && GFC_CLASS_TYPE_P (TREE_TYPE (tmp)))
+	{
+	  gfc_start_block (&init);
+	  tmp = gfc_class_data_get (tmp);
+	  type = TREE_TYPE (gfc_conv_descriptor_data_get (tmp));
+	  gfc_conv_descriptor_data_set (&init, tmp, build_int_cst (type, 0));
+	  gfc_add_init_cleanup (block, gfc_finish_block (&init), NULL_TREE);
+	}
     }
 
 
@@ -5326,7 +5340,7 @@ gfc_trans_deferred_vars (gfc_symbol * proc_sym, gfc_wrapped_block * block)
 	continue;
       /* 'omp allocate( {purpose: allocator, value: align},
 			{purpose: init-stmtlist, value: cleanup-stmtlist},
-			{purpose: size-var, value: last-size-expr}}
+			{purpose: size-var, value: last-size-expr} )
 	  where init-stmt/cleanup-stmt is the STATEMENT list to find the
 	  try-final block; last-size-expr is to find the location after
 	  which to add the code and 'size-var' is for the proper size, cf.
diff --git a/gcc/fortran/trans-expr.cc b/gcc/fortran/trans-expr.cc
index 3e0d763..ec24084 100644
--- a/gcc/fortran/trans-expr.cc
+++ b/gcc/fortran/trans-expr.cc
@@ -714,6 +714,8 @@ gfc_get_class_from_expr (tree expr)
 {
   tree tmp;
   tree type;
+  bool array_descr_found = false;
+  bool comp_after_descr_found = false;
 
   for (tmp = expr; tmp; tmp = TREE_OPERAND (tmp, 0))
     {
@@ -725,6 +727,8 @@ gfc_get_class_from_expr (tree expr)
 	{
 	  if (GFC_CLASS_TYPE_P (type))
 	    return tmp;
+	  if (GFC_DESCRIPTOR_TYPE_P (type))
+	    array_descr_found = true;
 	  if (type != TYPE_CANONICAL (type))
 	    type = TYPE_CANONICAL (type);
 	  else
@@ -732,6 +736,23 @@ gfc_get_class_from_expr (tree expr)
 	}
       if (VAR_P (tmp) || TREE_CODE (tmp) == PARM_DECL)
 	break;
+
+      /* Avoid walking up the reference chain too far.  For class arrays, the
+	 array descriptor is a direct component (through a pointer) of the class
+	 container.  So there is exactly one COMPONENT_REF between a class
+	 container and its child array descriptor.  After seeing an array
+	 descriptor, we can give up on the second COMPONENT_REF we see, if no
+	 class container was found until that point.  */
+      if (array_descr_found)
+	{
+	  if (comp_after_descr_found)
+	    {
+	      if (TREE_CODE (tmp) == COMPONENT_REF)
+		return NULL_TREE;
+	    }
+	  else if (TREE_CODE (tmp) == COMPONENT_REF)
+	    comp_after_descr_found = true;
+	}
     }
 
   if (POINTER_TYPE_P (TREE_TYPE (tmp)))
@@ -5464,16 +5485,6 @@ gfc_conv_subref_array_arg (gfc_se *se, gfc_expr * expr, int g77,
   /* Translate the expression.  */
   gfc_conv_expr (&rse, expr);
 
-  /* Reset the offset for the function call since the loop
-     is zero based on the data pointer.  Note that the temp
-     comes first in the loop chain since it is added second.  */
-  if (gfc_is_class_array_function (expr))
-    {
-      tmp = loop.ss->loop_chain->info->data.array.descriptor;
-      gfc_conv_descriptor_offset_set (&loop.pre, tmp,
-				      gfc_index_zero_node);
-    }
-
   gfc_conv_tmp_array_ref (&lse);
 
   if (intent != INTENT_OUT)
@@ -7909,21 +7920,21 @@ gfc_conv_procedure_call (gfc_se * se, gfc_symbol * sym,
 	  se->ss->info->class_container = arg1_cntnr;
 	}
 
-      if (fsym && e)
+      /* Obtain the character length of an assumed character length procedure
+	 from the typespec of the actual argument.  */
+      if (e
+	  && parmse.string_length == NULL_TREE
+	  && e->ts.type == BT_PROCEDURE
+	  && e->symtree->n.sym->ts.type == BT_CHARACTER
+	  && e->symtree->n.sym->ts.u.cl->length != NULL
+	  && e->symtree->n.sym->ts.u.cl->length->expr_type == EXPR_CONSTANT)
 	{
-	  /* Obtain the character length of an assumed character length
-	     length procedure from the typespec.  */
-	  if (fsym->ts.type == BT_CHARACTER
-	      && parmse.string_length == NULL_TREE
-	      && e->ts.type == BT_PROCEDURE
-	      && e->symtree->n.sym->ts.type == BT_CHARACTER
-	      && e->symtree->n.sym->ts.u.cl->length != NULL
-	      && e->symtree->n.sym->ts.u.cl->length->expr_type == EXPR_CONSTANT)
-	    {
-	      gfc_conv_const_charlen (e->symtree->n.sym->ts.u.cl);
-	      parmse.string_length = e->symtree->n.sym->ts.u.cl->backend_decl;
-	    }
+	  gfc_conv_const_charlen (e->symtree->n.sym->ts.u.cl);
+	  parmse.string_length = e->symtree->n.sym->ts.u.cl->backend_decl;
+	}
 
+      if (fsym && e)
+	{
 	  /* Obtain the character length for a NULL() actual with a character
 	     MOLD argument.  Otherwise substitute a suitable dummy length.
 	     Here we handle non-optional dummies of non-bind(c) procedures.  */
@@ -8159,7 +8170,8 @@ gfc_conv_procedure_call (gfc_se * se, gfc_symbol * sym,
 		msg = xasprintf ("Pointer actual argument '%s' is not "
 				 "associated", e->symtree->n.sym->name);
 	      else if (attr.proc_pointer && !e->value.function.actual
-		       && (fsym == NULL || !fsym_attr.proc_pointer))
+		       && (fsym == NULL
+			   || (!fsym_attr.proc_pointer && !fsym_attr.optional)))
 		msg = xasprintf ("Proc-pointer actual argument '%s' is not "
 				 "associated", e->symtree->n.sym->name);
 	      else
@@ -8842,28 +8854,9 @@ gfc_conv_procedure_call (gfc_se * se, gfc_symbol * sym,
 	  && se->expr && GFC_CLASS_TYPE_P (TREE_TYPE (se->expr))
 	  && expr->must_finalize)
 	{
-	  int n;
-	  if (se->ss && se->ss->loop)
-	    {
-	      gfc_add_block_to_block (&se->ss->loop->pre, &se->pre);
-	      se->expr = gfc_evaluate_now (se->expr, &se->ss->loop->pre);
-	      tmp = gfc_class_data_get (se->expr);
-	      info->descriptor = tmp;
-	      info->data = gfc_conv_descriptor_data_get (tmp);
-	      info->offset = gfc_conv_descriptor_offset_get (tmp);
-	      for (n = 0; n < se->ss->loop->dimen; n++)
-		{
-		  tree dim = gfc_rank_cst[n];
-		  se->ss->loop->to[n] = gfc_conv_descriptor_ubound_get (tmp, dim);
-		  se->ss->loop->from[n] = gfc_conv_descriptor_lbound_get (tmp, dim);
-		}
-	    }
-	  else
-	    {
-	      /* TODO Eliminate the doubling of temporaries. This
-		 one is necessary to ensure no memory leakage.  */
-	      se->expr = gfc_evaluate_now (se->expr, &se->pre);
-	    }
+	  /* TODO Eliminate the doubling of temporaries.  This
+	     one is necessary to ensure no memory leakage.  */
+	  se->expr = gfc_evaluate_now (se->expr, &se->pre);
 
 	  /* Finalize the result, if necessary.  */
 	  attr = expr->value.function.esym
@@ -12870,16 +12863,19 @@ gfc_trans_assignment_1 (gfc_expr * expr1, gfc_expr * expr2, bool init_flag,
   gfc_init_se (&lse, NULL);
   gfc_init_se (&rse, NULL);
 
+  gfc_fix_class_refs (expr1);
+
+  realloc_flag = flag_realloc_lhs
+		 && gfc_is_reallocatable_lhs (expr1)
+		 && expr2->rank
+		 && !is_runtime_conformable (expr1, expr2);
+
   /* Walk the lhs.  */
   lss = gfc_walk_expr (expr1);
-  if (gfc_is_reallocatable_lhs (expr1))
+  if (realloc_flag)
     {
       lss->no_bounds_check = 1;
-      if (!(expr2->expr_type == EXPR_FUNCTION
-	    && expr2->value.function.isym != NULL
-	    && !(expr2->value.function.isym->elemental
-		 || expr2->value.function.isym->conversion)))
-	lss->is_alloc_lhs = 1;
+      lss->is_alloc_lhs = 1;
     }
   else
     lss->no_bounds_check = expr1->no_bounds_check;
@@ -12927,11 +12923,6 @@ gfc_trans_assignment_1 (gfc_expr * expr1, gfc_expr * expr2, bool init_flag,
 
   assoc_assign = is_assoc_assign (expr1, expr2);
 
-  realloc_flag = flag_realloc_lhs
-		 && gfc_is_reallocatable_lhs (expr1)
-		 && expr2->rank
-		 && !is_runtime_conformable (expr1, expr2);
-
   /* Only analyze the expressions for coarray properties, when in coarray-lib
      mode.  Avoid false-positive uninitialized diagnostics with initializing
      the codimension flag unconditionally.  */
@@ -12943,6 +12934,7 @@ gfc_trans_assignment_1 (gfc_expr * expr1, gfc_expr * expr2, bool init_flag,
       rhs_caf_attr = gfc_caf_attr (expr2, false, &rhs_refs_comp);
     }
 
+  tree reallocation = NULL_TREE;
   if (lss != gfc_ss_terminator)
     {
       /* The assignment needs scalarization.  */
@@ -12961,8 +12953,12 @@ gfc_trans_assignment_1 (gfc_expr * expr1, gfc_expr * expr2, bool init_flag,
       /* Walk the rhs.  */
       rss = gfc_walk_expr (expr2);
       if (rss == gfc_ss_terminator)
-	/* The rhs is scalar.  Add a ss for the expression.  */
-	rss = gfc_get_scalar_ss (gfc_ss_terminator, expr2);
+	{
+	  /* The rhs is scalar.  Add a ss for the expression.  */
+	  rss = gfc_get_scalar_ss (gfc_ss_terminator, expr2);
+	  lss->is_alloc_lhs = 0;
+	}
+
       /* When doing a class assign, then the handle to the rhs needs to be a
 	 pointer to allow for polymorphism.  */
       if (is_poly_assign && expr2->rank == 0 && !UNLIMITED_POLY (expr2))
@@ -13011,6 +13007,15 @@ gfc_trans_assignment_1 (gfc_expr * expr1, gfc_expr * expr2, bool init_flag,
 	  ompws_flags |= OMPWS_SCALARIZER_WS | OMPWS_SCALARIZER_BODY;
 	}
 
+      /* F2003: Allocate or reallocate lhs of allocatable array.  */
+      if (realloc_flag)
+	{
+	  realloc_lhs_warning (expr1->ts.type, true, &expr1->where);
+	  ompws_flags &= ~OMPWS_SCALARIZER_WS;
+	  reallocation = gfc_alloc_allocatable_for_assignment (&loop, expr1,
+							       expr2);
+	}
+
       /* Start the scalarized loop body.  */
       gfc_start_scalarized_body (&loop, &body);
     }
@@ -13319,15 +13324,8 @@ gfc_trans_assignment_1 (gfc_expr * expr1, gfc_expr * expr2, bool init_flag,
 	  gfc_add_expr_to_block (&body, tmp);
 	}
 
-      /* F2003: Allocate or reallocate lhs of allocatable array.  */
-      if (realloc_flag)
-	{
-	  realloc_lhs_warning (expr1->ts.type, true, &expr1->where);
-	  ompws_flags &= ~OMPWS_SCALARIZER_WS;
-	  tmp = gfc_alloc_allocatable_for_assignment (&loop, expr1, expr2);
-	  if (tmp != NULL_TREE)
-	    gfc_add_expr_to_block (&loop.code[expr1->rank - 1], tmp);
-	}
+      if (reallocation != NULL_TREE)
+	gfc_add_expr_to_block (&loop.code[loop.dimen - 1], reallocation);
 
       if (maybe_workshare)
 	ompws_flags &= ~OMPWS_SCALARIZER_BODY;
diff --git a/gcc/fortran/trans-intrinsic.cc b/gcc/fortran/trans-intrinsic.cc
index f1bfd3e..f68ceb1 100644
--- a/gcc/fortran/trans-intrinsic.cc
+++ b/gcc/fortran/trans-intrinsic.cc
@@ -3466,6 +3466,74 @@ else
   return gfc_finish_block (&block);
 }
 
+static tree
+conv_intrinsic_split (gfc_code *code)
+{
+  stmtblock_t block, post_block;
+  gfc_se se;
+  gfc_expr *string_expr, *set_expr, *pos_expr, *back_expr;
+  tree string, string_len;
+  tree set, set_len;
+  tree pos, pos_for_call;
+  tree back;
+  tree fndecl, call;
+
+  string_expr = code->ext.actual->expr;
+  set_expr = code->ext.actual->next->expr;
+  pos_expr = code->ext.actual->next->next->expr;
+  back_expr = code->ext.actual->next->next->next->expr;
+
+  gfc_start_block (&block);
+  gfc_init_block (&post_block);
+
+  gfc_init_se (&se, NULL);
+  gfc_conv_expr (&se, string_expr);
+  gfc_conv_string_parameter (&se);
+  gfc_add_block_to_block (&block, &se.pre);
+  gfc_add_block_to_block (&post_block, &se.post);
+  string = se.expr;
+  string_len = se.string_length;
+
+  gfc_init_se (&se, NULL);
+  gfc_conv_expr (&se, set_expr);
+  gfc_conv_string_parameter (&se);
+  gfc_add_block_to_block (&block, &se.pre);
+  gfc_add_block_to_block (&post_block, &se.post);
+  set = se.expr;
+  set_len = se.string_length;
+
+  gfc_init_se (&se, NULL);
+  gfc_conv_expr (&se, pos_expr);
+  gfc_add_block_to_block (&block, &se.pre);
+  gfc_add_block_to_block (&post_block, &se.post);
+  pos = se.expr;
+  pos_for_call = fold_convert (gfc_charlen_type_node, pos);
+
+  if (back_expr)
+    {
+      gfc_init_se (&se, NULL);
+      gfc_conv_expr (&se, back_expr);
+      gfc_add_block_to_block (&block, &se.pre);
+      gfc_add_block_to_block (&post_block, &se.post);
+      back = se.expr;
+    }
+  else
+    back = logical_false_node;
+
+  if (string_expr->ts.kind == 1)
+    fndecl = gfor_fndecl_string_split;
+  else if (string_expr->ts.kind == 4)
+    fndecl = gfor_fndecl_string_split_char4;
+  else
+    gcc_unreachable ();
+
+  call = build_call_expr_loc (input_location, fndecl, 6, string_len, string,
+			      set_len, set, pos_for_call, back);
+  gfc_add_modify (&block, pos, fold_convert (TREE_TYPE (pos), call));
+
+  gfc_add_block_to_block (&block, &post_block);
+  return gfc_finish_block (&block);
+}
 
 /* Return a character string containing the tty name.  */
 
@@ -13101,6 +13169,8 @@ conv_intrinsic_move_alloc (gfc_code *code)
     }
   gfc_conv_expr_descriptor (&to_se, to_expr);
   gfc_conv_expr_descriptor (&from_se, from_expr);
+  gfc_add_block_to_block (&block, &to_se.pre);
+  gfc_add_block_to_block (&block, &from_se.pre);
 
   /* For coarrays, call SYNC ALL if TO is already deallocated as MOVE_ALLOC
      is an image control "statement", cf. IR F08/0040 in 12-006A.  */
@@ -13174,6 +13244,9 @@ conv_intrinsic_move_alloc (gfc_code *code)
   if (fin_label)
     gfc_add_expr_to_block (&block, build1_v (LABEL_EXPR, fin_label));
 
+  gfc_add_block_to_block (&block, &to_se.post);
+  gfc_add_block_to_block (&block, &from_se.post);
+
   return gfc_finish_block (&block);
 }
 
@@ -13256,6 +13329,10 @@ gfc_conv_intrinsic_subroutine (gfc_code *code)
       res = conv_intrinsic_system_clock (code);
       break;
 
+    case GFC_ISYM_SPLIT:
+      res = conv_intrinsic_split (code);
+      break;
+
     default:
       res = NULL_TREE;
       break;
diff --git a/gcc/fortran/trans-openmp.cc b/gcc/fortran/trans-openmp.cc
index a2e70fc..278e91c 100644
--- a/gcc/fortran/trans-openmp.cc
+++ b/gcc/fortran/trans-openmp.cc
@@ -2792,8 +2792,13 @@ gfc_trans_omp_variable_list (enum omp_clause_code code,
 			     gfc_omp_namelist *namelist, tree list,
 			     bool declare_simd)
 {
+  /* PARAMETER (named constants) are excluded as OpenACC 3.4 permits them now
+     as 'var' but permits compilers to ignore them.  In expressions, it should
+     have been replaced by the value (and this function should not be called
+     anyway) and for var-using clauses, they should just be skipped.  */
   for (; namelist != NULL; namelist = namelist->next)
-    if (namelist->sym->attr.referenced || declare_simd)
+    if ((namelist->sym->attr.referenced || declare_simd)
+	&& namelist->sym->attr.flavor != FL_PARAMETER)
       {
 	tree t = gfc_trans_omp_variable (namelist->sym, declare_simd);
 	if (t != error_mark_node)
@@ -4029,7 +4034,8 @@ gfc_trans_omp_clauses (stmtblock_t *block, gfc_omp_clauses *clauses,
 	case OMP_LIST_MAP:
 	  for (; n != NULL; n = n->next)
 	    {
-	      if (!n->sym->attr.referenced)
+	      if (!n->sym->attr.referenced
+		  || n->sym->attr.flavor == FL_PARAMETER)
 		continue;
 
 	      location_t map_loc = gfc_get_location (&n->where);
@@ -4986,7 +4992,8 @@ gfc_trans_omp_clauses (stmtblock_t *block, gfc_omp_clauses *clauses,
 	case OMP_LIST_CACHE:
 	  for (; n != NULL; n = n->next)
 	    {
-	      if (!n->sym->attr.referenced)
+	      if (!n->sym->attr.referenced
+		  && n->sym->attr.flavor != FL_PARAMETER)
 		continue;
 
 	      switch (list)
@@ -9707,11 +9714,12 @@ gfc_trans_omp_declare_variant (gfc_namespace *ns, gfc_namespace *parent_ns)
 	{
 	  gfc_symtree *proc_st;
 	  gfc_find_sym_tree (variant_proc_name, gfc_current_ns, 1, &proc_st);
-	  variant_proc_sym = proc_st->n.sym;
+	  variant_proc_sym = proc_st ? proc_st->n.sym : NULL;
 	}
       if (variant_proc_sym == NULL)
 	{
-	  gfc_error ("Cannot find symbol %qs", variant_proc_name);
+	  gfc_error ("Cannot find symbol %qs at %L", variant_proc_name,
+						     &odv->where);
 	  continue;
 	}
       set_selectors = omp_check_context_selector
diff --git a/gcc/fortran/trans-stmt.cc b/gcc/fortran/trans-stmt.cc
index f105401..b4ddf75 100644
--- a/gcc/fortran/trans-stmt.cc
+++ b/gcc/fortran/trans-stmt.cc
@@ -6710,7 +6710,6 @@ gfc_trans_allocate (gfc_code * code, gfc_omp_namelist *omp_allocate)
   stmtblock_t block;
   stmtblock_t post;
   stmtblock_t final_block;
-  tree nelems;
   bool upoly_expr, tmp_expr3_len_flag = false, al_len_needs_set, is_coarray;
   bool needs_caf_sync, caf_refs_comp;
   bool e3_has_nodescriptor = false;
@@ -7242,7 +7241,6 @@ gfc_trans_allocate (gfc_code * code, gfc_omp_namelist *omp_allocate)
 	 to handle the complete array allocation.  Only the element size
 	 needs to be provided, which is done most of the time by the
 	 pre-evaluation step.  */
-      nelems = NULL_TREE;
       if (expr3_len && (code->expr3->ts.type == BT_CHARACTER
 			|| code->expr3->ts.type == BT_CLASS))
 	{
@@ -7313,9 +7311,8 @@ gfc_trans_allocate (gfc_code * code, gfc_omp_namelist *omp_allocate)
 
 	}
 
-      if (!gfc_array_allocate (&se, expr, stat, errmsg, errlen,
-			       label_finish, tmp, &nelems,
-			       e3rhs ? e3rhs : code->expr3,
+      if (!gfc_array_allocate (&se, expr, stat, errmsg, errlen, label_finish,
+			       tmp, e3rhs ? e3rhs : code->expr3,
 			       e3_is == E3_DESC ? expr3 : NULL_TREE,
 			       e3_has_nodescriptor, omp_alloc_item,
 			       code->ext.alloc.ts.type != BT_UNKNOWN))
diff --git a/gcc/fortran/trans.h b/gcc/fortran/trans.h
index 461b0cd..40680e9 100644
--- a/gcc/fortran/trans.h
+++ b/gcc/fortran/trans.h
@@ -961,6 +961,7 @@ extern GTY(()) tree gfor_fndecl_string_scan;
 extern GTY(()) tree gfor_fndecl_string_verify;
 extern GTY(()) tree gfor_fndecl_string_trim;
 extern GTY(()) tree gfor_fndecl_string_minmax;
+extern GTY(()) tree gfor_fndecl_string_split;
 extern GTY(()) tree gfor_fndecl_adjustl;
 extern GTY(()) tree gfor_fndecl_adjustr;
 extern GTY(()) tree gfor_fndecl_select_string;
@@ -972,6 +973,7 @@ extern GTY(()) tree gfor_fndecl_string_scan_char4;
 extern GTY(()) tree gfor_fndecl_string_verify_char4;
 extern GTY(()) tree gfor_fndecl_string_trim_char4;
 extern GTY(()) tree gfor_fndecl_string_minmax_char4;
+extern GTY(()) tree gfor_fndecl_string_split_char4;
 extern GTY(()) tree gfor_fndecl_adjustl_char4;
 extern GTY(()) tree gfor_fndecl_adjustr_char4;
 extern GTY(()) tree gfor_fndecl_select_string_char4;
diff --git a/gcc/function.cc b/gcc/function.cc
index 48167b0c..2b77bbd 100644
--- a/gcc/function.cc
+++ b/gcc/function.cc
@@ -7009,6 +7009,115 @@ match_asm_constraints_1 (rtx_insn *insn, rtx *p_sets, int noutputs)
     df_insn_rescan (insn);
 }
 
+/* It is expected and desired that optimizations coalesce multiple pseudos into
+   one whenever possible.  However, in case of hard register constraints we may
+   have to undo this and introduce copies since otherwise we could constraint a
+   single pseudo to different hard registers.  For example, during register
+   allocation the following insn would be unsatisfiable since pseudo 60 is
+   constrained to hard register r5 and r6 at the same time.
+
+   (insn 7 5 0 2 (asm_operands/v ("foo") ("") 0 [
+	       (reg:DI 60) repeated x2
+	   ]
+	    [
+	       (asm_input:DI ("{r5}") t.c:4)
+	       (asm_input:DI ("{r6}") t.c:4)
+	   ]
+	    [] t.c:4) "t.c":4:3 -1
+	(expr_list:REG_DEAD (reg:DI 60)
+	   (nil)))
+
+   Therefore, introduce a copy of pseudo 60 and transform it into
+
+   (insn 10 5 7 2 (set (reg:DI 62)
+	   (reg:DI 60)) "t.c":4:3 1503 {*movdi_64}
+	(nil))
+   (insn 7 10 11 2 (asm_operands/v ("foo") ("") 0 [
+	       (reg:DI 60)
+	       (reg:DI 62)
+	   ]
+	    [
+	       (asm_input:DI ("{r5}") t.c:4)
+	       (asm_input:DI ("{r6}") t.c:4)
+	   ]
+	    [] t.c:4) "t.c":4:3 -1
+	(expr_list:REG_DEAD (reg:DI 62)
+	   (expr_list:REG_DEAD (reg:DI 60)
+	       (nil))))
+
+   Now, LRA can assign pseudo 60 to r5, and pseudo 62 to r6.
+
+   TODO: The current implementation is conservative and we could do a bit
+   better in case of alternatives.  For example
+
+   (insn 7 5 0 2 (asm_operands/v ("foo") ("") 0 [
+	       (reg:DI 60) repeated x2
+	   ]
+	    [
+	       (asm_input:DI ("r,{r5}") t.c:4)
+	       (asm_input:DI ("{r6},r") t.c:4)
+	   ]
+	    [] t.c:4) "t.c":4:3 -1
+	(expr_list:REG_DEAD (reg:DI 60)
+	   (nil)))
+
+   For this insn we wouldn't need to come up with a copy of pseudo 60 since in
+   each alternative pseudo 60 is constrained exactly one time.  */
+
+static void
+match_asm_constraints_2 (rtx_insn *insn, rtx pat)
+{
+  rtx op;
+  if (GET_CODE (pat) == SET && GET_CODE (SET_SRC (pat)) == ASM_OPERANDS)
+    op = SET_SRC (pat);
+  else if (GET_CODE (pat) == ASM_OPERANDS)
+    op = pat;
+  else
+    return;
+  int ninputs = ASM_OPERANDS_INPUT_LENGTH (op);
+  rtvec inputs = ASM_OPERANDS_INPUT_VEC (op);
+  bool changed = false;
+  auto_bitmap constrained_regs;
+
+  for (int i = 0; i < ninputs; ++i)
+    {
+      rtx input = RTVEC_ELT (inputs, i);
+      const char *constraint = ASM_OPERANDS_INPUT_CONSTRAINT (op, i);
+      if ((!REG_P (input) && !SUBREG_P (input))
+	  || (REG_P (input) && HARD_REGISTER_P (input))
+	  || strchr (constraint, '{') == nullptr)
+	continue;
+      int regno;
+      if (SUBREG_P (input))
+	{
+	  if (REG_P (SUBREG_REG (input)))
+	    regno = REGNO (SUBREG_REG (input));
+	  else
+	    continue;
+	}
+      else
+	regno = REGNO (input);
+      /* Keep the first usage of a constrained pseudo as is and only
+	 introduce copies for subsequent usages.  */
+      if (! bitmap_bit_p (constrained_regs, regno))
+	{
+	  bitmap_set_bit (constrained_regs, regno);
+	  continue;
+	}
+      rtx tmp = gen_reg_rtx (GET_MODE (input));
+      start_sequence ();
+      emit_move_insn (tmp, input);
+      rtx_insn *insns = get_insns ();
+      end_sequence ();
+      emit_insn_before (insns, insn);
+      RTVEC_ELT (inputs, i) = tmp;
+      changed = true;
+    }
+
+  if (changed)
+    df_insn_rescan (insn);
+}
+
 /* Add the decl D to the local_decls list of FUN.  */
 
 void
@@ -7065,6 +7174,13 @@ pass_match_asm_constraints::execute (function *fun)
 	    continue;
 
 	  pat = PATTERN (insn);
+
+	  if (GET_CODE (pat) == PARALLEL)
+	    for (int i = XVECLEN (pat, 0) - 1; i >= 0; --i)
+	      match_asm_constraints_2 (insn, XVECEXP (pat, 0, i));
+	  else
+	    match_asm_constraints_2 (insn, pat);
+
 	  if (GET_CODE (pat) == PARALLEL)
 	    p_sets = &XVECEXP (pat, 0, 0), noutputs = XVECLEN (pat, 0);
 	  else if (GET_CODE (pat) == SET)
diff --git a/gcc/diagnostic-spec.cc b/gcc/gcc-diagnostic-spec.cc
index df78579..99ca6a0 100644
--- a/gcc/diagnostic-spec.cc
+++ b/gcc/gcc-diagnostic-spec.cc
@@ -27,7 +27,7 @@
 #include "tree.h"
 #include "cgraph.h"
 #include "hash-map.h"
-#include "diagnostic-spec.h"
+#include "gcc-diagnostic-spec.h"
 #include "pretty-print.h"
 #include "options.h"
 
@@ -72,9 +72,9 @@ nowarn_spec_t::nowarn_spec_t (opt_code opt)
     case OPT_Wstrict_aliasing:
     case OPT_Wunused:
     case OPT_Wunused_function:
-    case OPT_Wunused_but_set_variable:
+    case OPT_Wunused_but_set_variable_:
     case OPT_Wunused_variable:
-    case OPT_Wunused_but_set_parameter:
+    case OPT_Wunused_but_set_parameter_:
       m_bits = NW_LEXICAL;
       break;
 
diff --git a/gcc/diagnostic-spec.h b/gcc/gcc-diagnostic-spec.h
index 13455ce..2b5e784 100644
--- a/gcc/diagnostic-spec.h
+++ b/gcc/gcc-diagnostic-spec.h
@@ -19,8 +19,8 @@
    along with GCC; see the file COPYING3.  If not see
    <http://www.gnu.org/licenses/>.  */
 
-#ifndef DIAGNOSTIC_SPEC_H_INCLUDED
-#define DIAGNOSTIC_SPEC_H_INCLUDED
+#ifndef GCC_DIAGNOSTIC_SPEC_H_INCLUDED
+#define GCC_DIAGNOSTIC_SPEC_H_INCLUDED
 
 #include "hash-map.h"
 
@@ -146,4 +146,4 @@ typedef hash_map<location_hash, nowarn_spec_t> nowarn_map_t;
 /* A mapping from a 'location_t' to the warning spec set for it.  */
 extern GTY(()) nowarn_map_t *nowarn_map;
 
-#endif // DIAGNOSTIC_SPEC_H_INCLUDED
+#endif // GCC_DIAGNOSTIC_SPEC_H_INCLUDED
diff --git a/gcc/gcc-rich-location.cc b/gcc/gcc-rich-location.cc
index 2095987..beb7b6c 100644
--- a/gcc/gcc-rich-location.cc
+++ b/gcc/gcc-rich-location.cc
@@ -37,6 +37,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "intl.h"
 #include "cpplib.h"
 #include "diagnostic.h"
+#include "diagnostics/file-cache.h"
 
 /* Add a range to the rich_location, covering expression EXPR,
    using LABEL if non-NULL. */
@@ -79,11 +80,11 @@ gcc_rich_location::add_fixit_misspelled_id (location_t misspelled_token_loc,
 /* Return true if there is nothing on LOC's line before LOC.  */
 
 static bool
-blank_line_before_p (file_cache &fc,
+blank_line_before_p (diagnostics::file_cache &fc,
 		     location_t loc)
 {
   expanded_location exploc = expand_location (loc);
-  char_span line = fc.get_source_line (exploc.file, exploc.line);
+  diagnostics::char_span line = fc.get_source_line (exploc.file, exploc.line);
   if (!line)
     return false;
   if (line.length () < (size_t)exploc.column)
@@ -101,7 +102,7 @@ blank_line_before_p (file_cache &fc,
    If true is returned then *OUT_START_OF_LINE is written to.  */
 
 static bool
-use_new_line (file_cache &fc,
+use_new_line (diagnostics::file_cache &fc,
 	      location_t insertion_point, location_t indent,
 	      location_t *out_start_of_line)
 {
diff --git a/gcc/gcc-rich-location.h b/gcc/gcc-rich-location.h
index cbf59b7..69cf074 100644
--- a/gcc/gcc-rich-location.h
+++ b/gcc/gcc-rich-location.h
@@ -22,7 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "rich-location.h"
 
-class diagnostic_source_print_policy;
+namespace diagnostics { class source_print_policy; }
 
 /* A gcc_rich_location is libcpp's rich_location with additional
    helper methods for working with gcc's types.  The class is not
@@ -79,14 +79,14 @@ class gcc_rich_location : public rich_location
 	if (!added secondary)
 	  inform (secondary_loc, "message for secondary");
 
-     Implemented in diagnostic-show-locus.cc.  */
+     Implemented in diagnostics/source-printing.cc.  */
 
-  bool add_location_if_nearby (const diagnostic_source_print_policy &policy,
+  bool add_location_if_nearby (const diagnostics::source_print_policy &policy,
 			       location_t loc,
 			       bool restrict_to_current_line_spans = true,
 			       const range_label *label = NULL);
 
-  bool add_location_if_nearby (const diagnostic_context &dc,
+  bool add_location_if_nearby (const diagnostics::context &dc,
 			       location_t loc,
 			       bool restrict_to_current_line_spans = true,
 			       const range_label *label = NULL);
diff --git a/gcc/gcc.cc b/gcc/gcc.cc
index 235fe80..bfa588e 100644
--- a/gcc/gcc.cc
+++ b/gcc/gcc.cc
@@ -43,7 +43,7 @@ compilation is specified by a string called a "spec".  */
 #include "opt-suggestions.h"
 #include "gcc.h"
 #include "diagnostic.h"
-#include "diagnostic-format.h"
+#include "diagnostics/sink.h"
 #include "pretty-print-urlifier.h"
 #include "flags.h"
 #include "opts.h"
@@ -4195,7 +4195,7 @@ driver_handle_option (struct gcc_options *opts,
 		      unsigned int lang_mask ATTRIBUTE_UNUSED, int kind,
 		      location_t loc,
 		      const struct cl_option_handlers *handlers ATTRIBUTE_UNUSED,
-		      diagnostic_context *dc,
+		      diagnostics::context *dc,
 		      void (*) (void))
 {
   size_t opt_index = decoded->opt_index;
@@ -4207,7 +4207,8 @@ driver_handle_option (struct gcc_options *opts,
 
   gcc_assert (opts == &global_options);
   gcc_assert (opts_set == &global_options_set);
-  gcc_assert (kind == DK_UNSPECIFIED);
+  gcc_assert (static_cast<diagnostics::kind> (kind)
+	      == diagnostics::kind::unspecified);
   gcc_assert (loc == UNKNOWN_LOCATION);
   gcc_assert (dc == global_dc);
 
@@ -4367,10 +4368,10 @@ driver_handle_option (struct gcc_options *opts,
 	  const char *basename = (opts->x_dump_base_name ? opts->x_dump_base_name
 				  : opts->x_main_input_basename);
 	  gcc_assert (dc);
-	  diagnostic_output_format_init (*dc,
-					 opts->x_main_input_filename, basename,
-					 (enum diagnostics_output_format)value,
-					 opts->x_flag_diagnostics_json_formatting);
+	  diagnostics::output_format_init (*dc,
+					   opts->x_main_input_filename, basename,
+					   (enum diagnostics_output_format)value,
+					   opts->x_flag_diagnostics_json_formatting);
 	  break;
 	}
 
diff --git a/gcc/gcov-io.cc b/gcc/gcov-io.cc
index f39b4bd..dd3fc88 100644
--- a/gcc/gcov-io.cc
+++ b/gcc/gcov-io.cc
@@ -69,7 +69,7 @@ gcov_position (void)
 
 /* Return nonzero if the error flag is set.  */
 /* We need to expose this function when compiling for gcov-tool.  */
-#ifndef IN_GCOV_TOOL
+#if !defined (IN_GCOV_TOOL) && !defined (IN_GCC)
 static inline
 #endif
 int
diff --git a/gcc/gcov-io.h b/gcc/gcov-io.h
index d48291c..313c15c 100644
--- a/gcc/gcov-io.h
+++ b/gcc/gcov-io.h
@@ -349,6 +349,11 @@ struct gcov_summary
 {
   gcov_unsigned_t runs;		/* Number of program runs.  */
   gcov_type sum_max;    	/* Sum of individual run max values.  */
+  gcov_type cutoff;		/* Values smaller than this value are not
+				   reliable (0 may mean non-zero).
+				   For read profile cutoff is typically 1
+				   however when we scale up or use auto-fdo
+				   it may become bigger value.  */
 };
 
 #if !defined(inhibit_libc)
@@ -382,6 +387,7 @@ char *mangle_path (char const *base);
 /* Available outside gcov */
 GCOV_LINKAGE void gcov_write (const void *, unsigned) ATTRIBUTE_HIDDEN;
 GCOV_LINKAGE void gcov_write_unsigned (gcov_unsigned_t) ATTRIBUTE_HIDDEN;
+GCOV_LINKAGE int gcov_is_error (void);
 #endif
 
 #if !IN_GCOV && !IN_LIBGCOV
diff --git a/gcc/gcse.cc b/gcc/gcse.cc
index 96aae0e..ea57c55 100644
--- a/gcc/gcse.cc
+++ b/gcc/gcse.cc
@@ -4238,8 +4238,15 @@ execute_hardreg_pre (void)
     {
       int changed;
       current_hardreg_regno = regnos[i];
+      if (!df_regs_ever_live_p (current_hardreg_regno))
+	{
+	  if (dump_file)
+	    fprintf (dump_file, "Skipping hardreg PRE for regno %d, which is never live\n",
+		     current_hardreg_regno);
+	  continue;
+	}
       if (dump_file)
-	fprintf(dump_file, "Entering hardreg PRE for regno %d\n",
+	fprintf (dump_file, "Entering hardreg PRE for regno %d\n",
 		current_hardreg_regno);
       delete_unreachable_blocks ();
       df_analyze ();
diff --git a/gcc/gdbinit.in b/gcc/gdbinit.in
index 417571f..1ee3d92 100644
--- a/gcc/gdbinit.in
+++ b/gcc/gdbinit.in
@@ -293,12 +293,12 @@ See also 'help-gcc-hooks'.
 end
 
 define break-on-diagnostic
-break diagnostic_context::maybe_show_locus
+break diagnostics::context::maybe_show_locus
 end
 
 document break-on-diagnostic
-Put a breakpoint on diagnostic_show_locus, called whenever a diagnostic
-is emitted (as opposed to those warnings that are suppressed by
+Put a breakpoint on diagnostics::context::show_locus, called whenever
+a diagnostic is emitted (as opposed to those warnings that are suppressed by
 command-line options).
 end
 
diff --git a/gcc/gengtype.cc b/gcc/gengtype.cc
index 6a3621c..84c7636 100644
--- a/gcc/gengtype.cc
+++ b/gcc/gengtype.cc
@@ -1722,7 +1722,7 @@ open_base_files (void)
       "target-globals.h", "ipa-ref.h", "cgraph.h", "symbol-summary.h",
       "sreal.h", "ipa-cp.h", "ipa-prop.h", "ipa-fnsummary.h", "dwarf2out.h",
       "omp-general.h", "omp-offload.h", "ipa-modref-tree.h", "ipa-modref.h",
-      "symtab-thunks.h", "symtab-clones.h", "diagnostic-spec.h", "ctfc.h",
+      "symtab-thunks.h", "symtab-clones.h", "gcc-diagnostic-spec.h", "ctfc.h",
       NULL
     };
     const char *const *ifp;
diff --git a/gcc/genoutput.cc b/gcc/genoutput.cc
index 25d0b8b..183ffc1 100644
--- a/gcc/genoutput.cc
+++ b/gcc/genoutput.cc
@@ -200,6 +200,8 @@ static const char indep_constraints[] = ",=+%*?!^$#&g";
 static class constraint_data *
 constraints_by_letter_table[1 << CHAR_BIT];
 
+static hash_set<free_string_hash> used_reg_names;
+
 static int mdep_constraint_len (const char *, file_location, int);
 static void note_constraint (md_rtx_info *);
 
@@ -1156,6 +1158,45 @@ main (int argc, const char **argv)
   output_insn_data ();
   output_get_insn_name ();
 
+  /* Since genoutput has no information about hard register names we cannot
+     statically verify hard register names in constraints of the machine
+     description.  Therefore, we have to do it at runtime.  Although
+     verification shouldn't be too expensive, restrict it to checking builds.
+   */
+  printf ("\n\n#if CHECKING_P\n");
+  if (used_reg_names.is_empty ())
+    printf ("void verify_reg_names_in_constraints () { }\n");
+  else
+    {
+      size_t max_len = 0;
+      for (auto it = used_reg_names.begin (); it != used_reg_names.end (); ++it)
+	{
+	  size_t len = strlen (*it);
+	  if (len > max_len)
+	    max_len = len;
+	}
+      printf ("void\nverify_reg_names_in_constraints ()\n{\n");
+      printf ("  static const char hregnames[%zu][%zu] = {\n",
+	      used_reg_names.elements (), max_len + 1);
+      auto it = used_reg_names.begin ();
+      while (it != used_reg_names.end ())
+	{
+	  printf ("    \"%s\"", *it);
+	  ++it;
+	  if (it != used_reg_names.end ())
+	    printf (",");
+	  printf ("\n");
+	}
+      printf ("  };\n");
+      printf ("  for (size_t i = 0; i < %zu; ++i)\n",
+	      used_reg_names.elements ());
+      printf ("    if (decode_reg_name (hregnames[i]) < 0)\n");
+      printf ("      internal_error (\"invalid register %%qs used in "
+	      "constraint of machine description\", hregnames[i]);\n");
+      printf ("}\n");
+    }
+  printf ("#endif\n");
+
   fflush (stdout);
   return (ferror (stdout) != 0 || have_error
 	? FATAL_EXIT_CODE : SUCCESS_EXIT_CODE);
@@ -1284,6 +1325,25 @@ mdep_constraint_len (const char *s, file_location loc, int opno)
       if (!strncmp (s, p->name, p->namelen))
 	return p->namelen;
 
+  if (*s == '{')
+    {
+      const char *end = s + 1;
+      while (*end != '}' && *end != '"' && *end != '\0')
+	++end;
+      /* Similarly as in decode_hreg_constraint(), consider any hard register
+	 name longer than a few characters as an error.  */
+      ptrdiff_t len = end - s;
+      if (*end == '}' && len > 1 && len < 31)
+	{
+	  char *regname = new char[len];
+	  memcpy (regname, s + 1, len - 1);
+	  regname[len - 1] = '\0';
+	  if (used_reg_names.add (regname))
+	    delete[] regname;
+	  return len + 1;
+	}
+    }
+
   error_at (loc, "error: undefined machine-specific constraint "
 	    "at this point: \"%s\"", s);
   message_at (loc, "note:  in operand %d", opno);
diff --git a/gcc/genpreds.cc b/gcc/genpreds.cc
index d09d769..7ee68c9 100644
--- a/gcc/genpreds.cc
+++ b/gcc/genpreds.cc
@@ -719,7 +719,6 @@ static const char const_dbl_constraints[] = "GH";
 
 /* Summary data used to decide whether to output various functions and
    macro definitions.  */
-static unsigned int constraint_max_namelen;
 static bool have_register_constraints;
 static bool have_memory_constraints;
 static bool have_special_memory_constraints;
@@ -942,7 +941,6 @@ add_constraint (const char *name, const char *regclass,
   *last_constraint_ptr = c;
   last_constraint_ptr = &c->next_textual;
 
-  constraint_max_namelen = MAX (constraint_max_namelen, strlen (name));
   have_register_constraints |= c->is_register;
   have_const_int_constraints |= c->is_const_int;
   have_extra_constraints |= c->is_extra;
@@ -1150,7 +1148,7 @@ write_insn_constraint_len (void)
   unsigned int i;
 
   puts ("static inline size_t\n"
-	"insn_constraint_len (char fc, const char *str ATTRIBUTE_UNUSED)\n"
+	"insn_constraint_len (char fc, const char *str)\n"
 	"{\n"
 	"  switch (fc)\n"
 	"    {");
@@ -1183,6 +1181,13 @@ write_insn_constraint_len (void)
 
   puts ("    default: break;\n"
 	"    }\n"
+	"  if (str[0] == '{')\n"
+	"    {\n"
+	"      size_t len = 1;\n"
+	"      while (str[len] != '}' && str[len] != '\\0')\n"
+	"        ++len;\n"
+	"      return len + 1;\n"
+	"    }\n"
 	"  return 1;\n"
 	"}\n");
 }
@@ -1556,133 +1561,124 @@ write_tm_preds_h (void)
 	  "#endif\n"
 	  "\n");
 
-  if (constraint_max_namelen > 0)
-    {
-      write_enum_constraint_num ();
-      puts ("extern enum constraint_num lookup_constraint_1 (const char *);\n"
-	    "extern const unsigned char lookup_constraint_array[];\n"
+  write_enum_constraint_num ();
+  puts ("extern enum constraint_num lookup_constraint_1 (const char *);\n"
+	"extern const unsigned char lookup_constraint_array[];\n"
+	"\n"
+	"/* Return the constraint at the beginning of P, or"
+	" CONSTRAINT__UNKNOWN if it\n"
+	"   isn't recognized.  */\n"
+	"\n"
+	"static inline enum constraint_num\n"
+	"lookup_constraint (const char *p)\n"
+	"{\n"
+	"  unsigned int index = lookup_constraint_array"
+	"[(unsigned char) *p];\n"
+	"  return (index == UCHAR_MAX\n"
+	"          ? lookup_constraint_1 (p)\n"
+	"          : (enum constraint_num) index);\n"
+	"}\n");
+  if (satisfied_start == num_constraints)
+    puts ("/* Return true if X satisfies constraint C.  */\n"
+	  "\n"
+	  "static inline bool\n"
+	  "constraint_satisfied_p (rtx, enum constraint_num)\n"
+	  "{\n"
+	  "  return false;\n"
+	  "}\n");
+  else
+    printf ("extern bool (*constraint_satisfied_p_array[]) (rtx);\n"
 	    "\n"
-	    "/* Return the constraint at the beginning of P, or"
-	    " CONSTRAINT__UNKNOWN if it\n"
-	    "   isn't recognized.  */\n"
+	    "/* Return true if X satisfies constraint C.  */\n"
 	    "\n"
-	    "static inline enum constraint_num\n"
-	    "lookup_constraint (const char *p)\n"
-	    "{\n"
-	    "  unsigned int index = lookup_constraint_array"
-	    "[(unsigned char) *p];\n"
-	    "  return (index == UCHAR_MAX\n"
-	    "          ? lookup_constraint_1 (p)\n"
-	    "          : (enum constraint_num) index);\n"
-	    "}\n");
-      if (satisfied_start == num_constraints)
-	puts ("/* Return true if X satisfies constraint C.  */\n"
-	      "\n"
-	      "static inline bool\n"
-	      "constraint_satisfied_p (rtx, enum constraint_num)\n"
-	      "{\n"
-	      "  return false;\n"
-	      "}\n");
-      else
-	printf ("extern bool (*constraint_satisfied_p_array[]) (rtx);\n"
-		"\n"
-		"/* Return true if X satisfies constraint C.  */\n"
-		"\n"
-		"static inline bool\n"
-		"constraint_satisfied_p (rtx x, enum constraint_num c)\n"
-		"{\n"
-		"  int i = (int) c - (int) CONSTRAINT_%s;\n"
-		"  return i >= 0 && constraint_satisfied_p_array[i] (x);\n"
-		"}\n"
-		"\n",
-		enum_order[satisfied_start]->name);
-
-      write_range_function ("insn_extra_register_constraint",
-			    register_start, register_end);
-      write_range_function ("insn_extra_memory_constraint",
-			    memory_start, memory_end);
-      write_range_function ("insn_extra_special_memory_constraint",
-			    special_memory_start, special_memory_end);
-      write_range_function ("insn_extra_relaxed_memory_constraint",
-			    relaxed_memory_start, relaxed_memory_end);
-      write_range_function ("insn_extra_address_constraint",
-			    address_start, address_end);
-      write_allows_reg_mem_function ();
-
-      if (constraint_max_namelen > 1)
-        {
-	  write_insn_constraint_len ();
-	  puts ("#define CONSTRAINT_LEN(c_,s_) "
-		"insn_constraint_len (c_,s_)\n");
-	}
-      else
-	puts ("#define CONSTRAINT_LEN(c_,s_) 1\n");
-      if (have_register_constraints)
-	puts ("extern enum reg_class reg_class_for_constraint_1 "
-	      "(enum constraint_num);\n"
-	      "\n"
-	      "static inline enum reg_class\n"
-	      "reg_class_for_constraint (enum constraint_num c)\n"
-	      "{\n"
-	      "  if (insn_extra_register_constraint (c))\n"
-	      "    return reg_class_for_constraint_1 (c);\n"
-	      "  return NO_REGS;\n"
-	      "}\n");
-      else
-	puts ("static inline enum reg_class\n"
-	      "reg_class_for_constraint (enum constraint_num)\n"
-	      "{\n"
-	      "  return NO_REGS;\n"
-	      "}\n");
-      if (have_const_int_constraints)
-	puts ("extern bool insn_const_int_ok_for_constraint "
-	      "(HOST_WIDE_INT, enum constraint_num);\n"
-	      "#define CONST_OK_FOR_CONSTRAINT_P(v_,c_,s_) \\\n"
-	      "    insn_const_int_ok_for_constraint (v_, "
-	      "lookup_constraint (s_))\n");
-      else
-	puts ("static inline bool\n"
-	      "insn_const_int_ok_for_constraint (HOST_WIDE_INT,"
-	      " enum constraint_num)\n"
-	      "{\n"
-	      "  return false;\n"
-	      "}\n");
-
-      puts ("enum constraint_type\n"
+	    "static inline bool\n"
+	    "constraint_satisfied_p (rtx x, enum constraint_num c)\n"
 	    "{\n"
-	    "  CT_REGISTER,\n"
-	    "  CT_CONST_INT,\n"
-	    "  CT_MEMORY,\n"
-	    "  CT_SPECIAL_MEMORY,\n"
-	    "  CT_RELAXED_MEMORY,\n"
-	    "  CT_ADDRESS,\n"
-	    "  CT_FIXED_FORM\n"
-	    "};\n"
-	    "\n"
-	    "static inline enum constraint_type\n"
-	    "get_constraint_type (enum constraint_num c)\n"
-	    "{");
-      auto_vec <std::pair <unsigned int, const char *>, 4> values;
-      if (const_int_start != const_int_end)
-	values.safe_push (std::make_pair (const_int_start, "CT_CONST_INT"));
-      if (memory_start != memory_end)
-	values.safe_push (std::make_pair (memory_start, "CT_MEMORY"));
-      if (special_memory_start != special_memory_end)
-	values.safe_push (std::make_pair (special_memory_start,
-					  "CT_SPECIAL_MEMORY"));
-      if (relaxed_memory_start != relaxed_memory_end)
-	values.safe_push (std::make_pair (relaxed_memory_start,
-					  "CT_RELAXED_MEMORY"));
-      if (address_start != address_end)
-	values.safe_push (std::make_pair (address_start, "CT_ADDRESS"));
-      if (address_end != num_constraints)
-	values.safe_push (std::make_pair (address_end, "CT_FIXED_FORM"));
-      print_type_tree (values, 0, values.length (), "CT_REGISTER", 2);
-      puts ("}");
-
-      write_get_register_filter ();
-      write_get_register_filter_id ();
-    }
+	    "  int i = (int) c - (int) CONSTRAINT_%s;\n"
+	    "  return i >= 0 && constraint_satisfied_p_array[i] (x);\n"
+	    "}\n"
+	    "\n",
+	    enum_order[satisfied_start]->name);
+
+  write_range_function ("insn_extra_register_constraint",
+			register_start, register_end);
+  write_range_function ("insn_extra_memory_constraint",
+			memory_start, memory_end);
+  write_range_function ("insn_extra_special_memory_constraint",
+			special_memory_start, special_memory_end);
+  write_range_function ("insn_extra_relaxed_memory_constraint",
+			relaxed_memory_start, relaxed_memory_end);
+  write_range_function ("insn_extra_address_constraint",
+			address_start, address_end);
+  write_allows_reg_mem_function ();
+
+  write_insn_constraint_len ();
+  puts ("#define CONSTRAINT_LEN(c_,s_) insn_constraint_len (c_,s_)\n");
+  if (have_register_constraints)
+    puts ("extern enum reg_class reg_class_for_constraint_1 "
+	  "(enum constraint_num);\n"
+	  "\n"
+	  "static inline enum reg_class\n"
+	  "reg_class_for_constraint (enum constraint_num c)\n"
+	  "{\n"
+	  "  if (insn_extra_register_constraint (c))\n"
+	  "    return reg_class_for_constraint_1 (c);\n"
+	  "  return NO_REGS;\n"
+	  "}\n");
+  else
+    puts ("static inline enum reg_class\n"
+	  "reg_class_for_constraint (enum constraint_num)\n"
+	  "{\n"
+	  "  return NO_REGS;\n"
+	  "}\n");
+  if (have_const_int_constraints)
+    puts ("extern bool insn_const_int_ok_for_constraint "
+	  "(HOST_WIDE_INT, enum constraint_num);\n"
+	  "#define CONST_OK_FOR_CONSTRAINT_P(v_,c_,s_) \\\n"
+	  "    insn_const_int_ok_for_constraint (v_, "
+	  "lookup_constraint (s_))\n");
+  else
+    puts ("static inline bool\n"
+	  "insn_const_int_ok_for_constraint (HOST_WIDE_INT,"
+	  " enum constraint_num)\n"
+	  "{\n"
+	  "  return false;\n"
+	  "}\n");
+
+  puts ("enum constraint_type\n"
+	"{\n"
+	"  CT_REGISTER,\n"
+	"  CT_CONST_INT,\n"
+	"  CT_MEMORY,\n"
+	"  CT_SPECIAL_MEMORY,\n"
+	"  CT_RELAXED_MEMORY,\n"
+	"  CT_ADDRESS,\n"
+	"  CT_FIXED_FORM\n"
+	"};\n"
+	"\n"
+	"static inline enum constraint_type\n"
+	"get_constraint_type (enum constraint_num c)\n"
+	"{");
+  auto_vec <std::pair <unsigned int, const char *>, 4> values;
+  if (const_int_start != const_int_end)
+    values.safe_push (std::make_pair (const_int_start, "CT_CONST_INT"));
+  if (memory_start != memory_end)
+    values.safe_push (std::make_pair (memory_start, "CT_MEMORY"));
+  if (special_memory_start != special_memory_end)
+    values.safe_push (std::make_pair (special_memory_start,
+				      "CT_SPECIAL_MEMORY"));
+  if (relaxed_memory_start != relaxed_memory_end)
+    values.safe_push (std::make_pair (relaxed_memory_start,
+				      "CT_RELAXED_MEMORY"));
+  if (address_start != address_end)
+    values.safe_push (std::make_pair (address_start, "CT_ADDRESS"));
+  if (address_end != num_constraints)
+    values.safe_push (std::make_pair (address_end, "CT_FIXED_FORM"));
+  print_type_tree (values, 0, values.length (), "CT_REGISTER", 2);
+  puts ("}");
+
+  write_get_register_filter ();
+  write_get_register_filter_id ();
 
   puts ("#endif /* tm-preds.h */");
 }
@@ -1743,17 +1739,14 @@ write_insn_preds_c (void)
   FOR_ALL_PREDICATES (p)
     write_one_predicate_function (p);
 
-  if (constraint_max_namelen > 0)
-    {
-      write_lookup_constraint_1 ();
-      write_lookup_constraint_array ();
-      if (have_register_constraints)
-	write_reg_class_for_constraint_1 ();
-      write_constraint_satisfied_p_array ();
-
-      if (have_const_int_constraints)
-	write_insn_const_int_ok_for_constraint ();
-    }
+  write_lookup_constraint_1 ();
+  write_lookup_constraint_array ();
+  if (have_register_constraints)
+    write_reg_class_for_constraint_1 ();
+  write_constraint_satisfied_p_array ();
+
+  if (have_const_int_constraints)
+    write_insn_const_int_ok_for_constraint ();
 
   write_init_reg_class_start_regs ();
 }
diff --git a/gcc/gimple-fold.cc b/gcc/gimple-fold.cc
index c5c8e22..85319b3 100644
--- a/gcc/gimple-fold.cc
+++ b/gcc/gimple-fold.cc
@@ -6762,10 +6762,10 @@ fold_stmt (gimple_stmt_iterator *gsi, tree (*valueize) (tree), bitmap dce_bitmap
    which can produce *&x = 0.  */
 
 bool
-fold_stmt_inplace (gimple_stmt_iterator *gsi)
+fold_stmt_inplace (gimple_stmt_iterator *gsi, tree (*valueize) (tree))
 {
   gimple *stmt = gsi_stmt (*gsi);
-  bool changed = fold_stmt_1 (gsi, true, no_follow_ssa_edges);
+  bool changed = fold_stmt_1 (gsi, true, valueize);
   gcc_assert (gsi_stmt (*gsi) == stmt);
   return changed;
 }
@@ -9916,10 +9916,17 @@ fold_nonarray_ctor_reference (tree type, tree ctor,
 	    {
 	      if (BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
 		return NULL_TREE;
-	      const unsigned int encoding_size
-		= GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (TREE_TYPE (cfield)));
 	      if (BYTES_BIG_ENDIAN)
-		inner_offset += encoding_size - wi::to_offset (field_size);
+		{
+		  tree ctype = TREE_TYPE (cfield);
+		  unsigned int encoding_size;
+		  if (TYPE_MODE (ctype) != BLKmode)
+		    encoding_size
+		      = GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (ctype));
+		  else
+		    encoding_size = TREE_INT_CST_LOW (TYPE_SIZE (ctype));
+		  inner_offset += encoding_size - wi::to_offset (field_size);
+		}
 	    }
 
 	  return fold_ctor_reference (type, cval,
diff --git a/gcc/gimple-fold.h b/gcc/gimple-fold.h
index e3cf1f6..b678502 100644
--- a/gcc/gimple-fold.h
+++ b/gcc/gimple-fold.h
@@ -28,9 +28,12 @@ struct c_strlen_data;
 extern bool get_range_strlen (tree, c_strlen_data *, unsigned eltsize);
 extern void gimplify_and_update_call_from_tree (gimple_stmt_iterator *, tree);
 extern bool update_gimple_call (gimple_stmt_iterator *, tree, int, ...);
+extern tree no_follow_ssa_edges (tree);
+extern tree follow_single_use_edges (tree);
+extern tree follow_all_ssa_edges (tree);
 extern bool fold_stmt (gimple_stmt_iterator *, bitmap = nullptr);
 extern bool fold_stmt (gimple_stmt_iterator *, tree (*) (tree), bitmap = nullptr);
-extern bool fold_stmt_inplace (gimple_stmt_iterator *);
+extern bool fold_stmt_inplace (gimple_stmt_iterator *, tree (*) (tree) = no_follow_ssa_edges);
 extern tree maybe_fold_and_comparisons (tree, enum tree_code, tree, tree,
 					enum tree_code, tree, tree,
 					basic_block = nullptr);
@@ -39,9 +42,6 @@ extern tree maybe_fold_or_comparisons (tree, enum tree_code, tree, tree,
 				       basic_block = nullptr);
 extern bool optimize_atomic_compare_exchange_p (gimple *);
 extern void fold_builtin_atomic_compare_exchange (gimple_stmt_iterator *);
-extern tree no_follow_ssa_edges (tree);
-extern tree follow_single_use_edges (tree);
-extern tree follow_all_ssa_edges (tree);
 extern tree gimple_fold_stmt_to_constant_1 (gimple *, tree (*) (tree),
 					    tree (*) (tree) = no_follow_ssa_edges);
 extern tree gimple_fold_stmt_to_constant (gimple *, tree (*) (tree));
diff --git a/gcc/gimple-ssa-sccopy.cc b/gcc/gimple-ssa-sccopy.cc
index 341bae4..e65f532 100644
--- a/gcc/gimple-ssa-sccopy.cc
+++ b/gcc/gimple-ssa-sccopy.cc
@@ -78,7 +78,7 @@ along with GCC; see the file COPYING3.  If not see
 
    To find all three types of copy statements we use an algorithm based on
    strongly-connected components (SCCs) in dataflow graph.  The algorithm was
-   introduced in an article from 2013[1]. We describe the algorithm bellow.
+   introduced in an article from 2013[1]. We describe the algorithm below.
 
    To identify SCCs we implement the Robert Tarjan's SCC algorithm.  For the
    SCC computation we wrap potential copy statements in the 'vertex' struct.
diff --git a/gcc/gimple-ssa-store-merging.cc b/gcc/gimple-ssa-store-merging.cc
index 90f449c..d8075ca 100644
--- a/gcc/gimple-ssa-store-merging.cc
+++ b/gcc/gimple-ssa-store-merging.cc
@@ -644,9 +644,7 @@ find_bswap_or_nop_1 (gimple *stmt, struct symbolic_number *n, int limit)
 
 	  /* Mask.  */
 	  uint64_t mask = 0;
-	  uint64_t tmp = (1 << BITS_PER_UNIT) - 1;
-	  for (unsigned i = 0; i < bitsize / BITS_PER_UNIT;
-	       i++, tmp <<= BITS_PER_UNIT)
+	  for (unsigned i = 0; i < bitsize / BITS_PER_UNIT; i++)
 	    mask |= (uint64_t) MARKER_MASK << (i * BITS_PER_MARKER);
 	  n->n &= mask;
 
@@ -1035,7 +1033,7 @@ find_bswap_or_nop (gimple *stmt, struct symbolic_number *n, bool *bswap,
 	 source when rsize < range.  */
       if (n->range == orig_range
 	  /* There're case like 0x300000200 for uint32->uint64 cast,
-	     Don't hanlde this.  */
+	     Don't handle this.  */
 	  && n->range == TYPE_PRECISION (n->type)
 	  && ((orig_range == 32
 	       && optab_handler (rotl_optab, SImode) != CODE_FOR_nothing)
@@ -1045,7 +1043,7 @@ find_bswap_or_nop (gimple *stmt, struct symbolic_number *n, bool *bswap,
 	{
 	  uint64_t range = (orig_range / BITS_PER_UNIT) * BITS_PER_MARKER;
 	  uint64_t count = (tmp_n & MARKER_MASK) * BITS_PER_MARKER;
-	  /* .i.e. hanlde 0x203040506070800 when lower byte is zero.  */
+	  /* .i.e. handle 0x203040506070800 when lower byte is zero.  */
 	  if (!count)
 	    {
 	      for (uint64_t i = 1; i != range / BITS_PER_MARKER; i++)
@@ -1057,6 +1055,8 @@ find_bswap_or_nop (gimple *stmt, struct symbolic_number *n, bool *bswap,
 		      if (count <= range / BITS_PER_MARKER)
 			{
 			  count = (count + i) * BITS_PER_MARKER % range;
+			  if (!count)
+			    return NULL;
 			  break;
 			}
 		      else
diff --git a/gcc/gimple-walk.cc b/gcc/gimple-walk.cc
index eceda41..b8b1077 100644
--- a/gcc/gimple-walk.cc
+++ b/gcc/gimple-walk.cc
@@ -118,7 +118,7 @@ walk_gimple_asm (gasm *stmt, walk_tree_fn callback_op,
       if (wi)
 	{
 	  if (parse_output_constraint (&constraint, i, 0, 0, &allows_mem,
-				       &allows_reg, &is_inout))
+				       &allows_reg, &is_inout, nullptr))
 	    wi->val_only = (allows_reg || !allows_mem);
 	}
       if (wi)
@@ -137,7 +137,8 @@ walk_gimple_asm (gasm *stmt, walk_tree_fn callback_op,
       if (wi)
 	{
 	  if (parse_input_constraint (&constraint, 0, 0, noutputs, 0,
-				      oconstraints, &allows_mem, &allows_reg))
+				      oconstraints, &allows_mem, &allows_reg,
+				      nullptr))
 	    {
 	      wi->val_only = (allows_reg || !allows_mem);
 	      /* Although input "m" is not really a LHS, we need a lvalue.  */
@@ -897,7 +898,7 @@ walk_stmt_load_store_addr_ops (gimple *stmt, void *data,
 		    (TREE_VALUE (TREE_PURPOSE (link)));
 		oconstraints[i] = constraint;
 		parse_output_constraint (&constraint, i, 0, 0, &allows_mem,
-					 &allows_reg, &is_inout);
+					 &allows_reg, &is_inout, nullptr);
 		if (op && !allows_reg && allows_mem)
 		  ret |= visit_addr (stmt, op, TREE_VALUE (link), data);
 	      }
@@ -922,8 +923,8 @@ walk_stmt_load_store_addr_ops (gimple *stmt, void *data,
 			constraint = TREE_STRING_POINTER
 			    (TREE_VALUE (TREE_PURPOSE (link)));
 			parse_input_constraint (&constraint, 0, 0, noutputs,
-						0, oconstraints,
-						&allows_mem, &allows_reg);
+						0, oconstraints, &allows_mem,
+						&allows_reg, nullptr);
 			if (!allows_reg && allows_mem)
 			  ret |= visit_addr (stmt, op, TREE_VALUE (link),
 					     data);
diff --git a/gcc/gimple.h b/gcc/gimple.h
index 2688846..5c970ce 100644
--- a/gcc/gimple.h
+++ b/gcc/gimple.h
@@ -865,7 +865,7 @@ struct GTY((tag("GSS_ASSUME")))
    tell the runtime that it should begin the transaction in
    serial-irrevocable mode.  */
 #define GTMA_DOES_GO_IRREVOCABLE	(1u << 6)
-/* The transaction contains no instrumentation code whatsover, most
+/* The transaction contains no instrumentation code whatsoever, most
    likely because it is guaranteed to go irrevocable upon entry.  */
 #define GTMA_HAS_NO_INSTRUMENTATION	(1u << 7)
 
diff --git a/gcc/gimplify-me.cc b/gcc/gimplify-me.cc
index 7407822..5b4b987 100644
--- a/gcc/gimplify-me.cc
+++ b/gcc/gimplify-me.cc
@@ -194,7 +194,7 @@ gimple_regimplify_operands (gimple *stmt, gimple_stmt_iterator *gsi_p)
 	    constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (op)));
 	    oconstraints[i] = constraint;
 	    parse_output_constraint (&constraint, i, 0, 0, &allows_mem,
-				     &allows_reg, &is_inout);
+				     &allows_reg, &is_inout, nullptr);
 	    gimplify_expr (&TREE_VALUE (op), &pre, NULL,
 			   is_inout ? is_gimple_min_lval : is_gimple_lvalue,
 			   fb_lvalue | fb_mayfail);
@@ -204,7 +204,8 @@ gimple_regimplify_operands (gimple *stmt, gimple_stmt_iterator *gsi_p)
 	    tree op = gimple_asm_input_op (asm_stmt, i);
 	    constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (op)));
 	    parse_input_constraint (&constraint, 0, 0, noutputs, 0,
-				    oconstraints, &allows_mem, &allows_reg);
+				    oconstraints, &allows_mem, &allows_reg,
+				    nullptr);
 	    if (TREE_ADDRESSABLE (TREE_TYPE (TREE_VALUE (op))) && allows_mem)
 	      allows_reg = 0;
 	    if (!allows_reg && allows_mem)
diff --git a/gcc/gimplify.cc b/gcc/gimplify.cc
index 9f9ff92..fbf47dd 100644
--- a/gcc/gimplify.cc
+++ b/gcc/gimplify.cc
@@ -71,6 +71,10 @@ along with GCC; see the file COPYING3.  If not see
 #include "context.h"
 #include "tree-nested.h"
 #include "gcc-urlifier.h"
+#include "insn-config.h"
+#include "recog.h"
+#include "output.h"
+#include "gimplify_reg_info.h"
 
 /* Identifier for a basic condition, mapping it to other basic conditions of
    its Boolean expression.  Basic conditions given the same uid (in the same
@@ -7305,6 +7309,7 @@ gimplify_modify_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p,
       && !(TREE_ADDRESSABLE (TREE_TYPE (*from_p))
 	   && TREE_CODE (*from_p) == CALL_EXPR))
     {
+      suppress_warning (*from_p, OPT_Wunused_result);
       gimplify_stmt (from_p, pre_p);
       gimplify_stmt (to_p, pre_p);
       *expr_p = NULL_TREE;
@@ -7822,6 +7827,42 @@ gimplify_addr_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
   return ret;
 }
 
+/* Return the number of times character C occurs in string S.  */
+
+static int
+num_occurrences (int c, const char *s)
+{
+  int n = 0;
+  while (*s)
+    n += (*s++ == c);
+  return n;
+}
+
+/* A subroutine of gimplify_asm_expr.  Check that all operands have
+   the same number of alternatives.  Return -1 if this is violated.  Otherwise
+   return the number of alternatives.  */
+
+static int
+num_alternatives (const_tree link)
+{
+  if (link == nullptr)
+    return 0;
+
+  const char *constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
+  int num = num_occurrences (',', constraint);
+
+  if (num + 1 > MAX_RECOG_ALTERNATIVES)
+    return -1;
+
+  for (link = TREE_CHAIN (link); link; link = TREE_CHAIN (link))
+    {
+      constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
+      if (num_occurrences (',', constraint) != num)
+	return -1;
+    }
+  return num + 1;
+}
+
 /* Gimplify the operands of an ASM_EXPR.  Input operands should be a gimple
    value; output operands should be a gimple lvalue.  */
 
@@ -7852,6 +7893,36 @@ gimplify_asm_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
   clobbers = NULL;
   labels = NULL;
 
+  int num_alternatives_out = num_alternatives (ASM_OUTPUTS (expr));
+  int num_alternatives_in = num_alternatives (ASM_INPUTS (expr));
+  if (num_alternatives_out == -1 || num_alternatives_in == -1
+      || (num_alternatives_out > 0 && num_alternatives_in > 0
+	  && num_alternatives_out != num_alternatives_in))
+    {
+      error ("operand constraints for %<asm%> differ "
+	     "in number of alternatives");
+      return GS_ERROR;
+    }
+  int num_alternatives = MAX (num_alternatives_out, num_alternatives_in);
+
+  gimplify_reg_info reg_info (num_alternatives, noutputs);
+
+  link_next = NULL_TREE;
+  for (link = ASM_CLOBBERS (expr); link; link = link_next)
+    {
+      /* The clobber entry could also be an error marker.  */
+      if (TREE_CODE (TREE_VALUE (link)) == STRING_CST)
+	{
+	  const char *regname= TREE_STRING_POINTER (TREE_VALUE (link));
+	  int regno = decode_reg_name (regname);
+	  if (regno >= 0)
+	    reg_info.set_clobbered (regno);
+	}
+      link_next = TREE_CHAIN (link);
+      TREE_CHAIN (link) = NULL_TREE;
+      vec_safe_push (clobbers, link);
+    }
+
   ret = GS_ALL_DONE;
   link_next = NULL_TREE;
   for (i = 0, link = ASM_OUTPUTS (expr); link; ++i, link = link_next)
@@ -7868,8 +7939,9 @@ gimplify_asm_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
       if (constraint_len == 0)
         continue;
 
-      ok = parse_output_constraint (&constraint, i, 0, 0,
-				    &allows_mem, &allows_reg, &is_inout);
+      reg_info.operand = TREE_VALUE (link);
+      ok = parse_output_constraint (&constraint, i, 0, 0, &allows_mem,
+				    &allows_reg, &is_inout, &reg_info);
       if (!ok)
 	{
 	  ret = GS_ERROR;
@@ -8000,8 +8072,8 @@ gimplify_asm_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
 			*end = '\0';
 		      beg[-1] = '=';
 		      tem = beg - 1;
-		      parse_output_constraint (&tem, i, 0, 0,
-					       &mem_p, &reg_p, &inout_p);
+		      parse_output_constraint (&tem, i, 0, 0, &mem_p, &reg_p,
+					       &inout_p, nullptr);
 		      if (dst != str)
 			*dst++ = ',';
 		      if (reg_p)
@@ -8040,13 +8112,60 @@ gimplify_asm_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
 	}
     }
 
+  /* After all output operands have been gimplified, verify that each output
+     operand is used at most once in case of hard register constraints.  Thus,
+     error out in cases like
+       asm ("" : "={0}" (x), "={1}" (x));
+     or even for
+       asm ("" : "=r" (x), "={1}" (x));
+
+     FIXME: Ideally we would also error out for cases like
+       int x;
+       asm ("" : "=r" (x), "=r" (x));
+     However, since code like that was previously accepted, erroring out now might
+     break existing code.  On the other hand, we already error out for register
+     asm like
+       register int x asm ("0");
+       asm ("" : "=r" (x), "=r" (x));
+     Thus, maybe it wouldn't be too bad to also error out in the former
+     non-register-asm case.
+  */
+  for (unsigned i = 0; i < vec_safe_length (outputs); ++i)
+    {
+      tree link = (*outputs)[i];
+      tree op1 = TREE_VALUE (link);
+      const char *constraint
+	= TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
+      if (strchr (constraint, '{') != nullptr)
+	for (unsigned j = 0; j < vec_safe_length (outputs); ++j)
+	  {
+	    if (i == j)
+	      continue;
+	    tree link2 = (*outputs)[j];
+	    tree op2 = TREE_VALUE (link2);
+	    if (op1 == op2)
+	      {
+		error ("multiple outputs to lvalue %qE", op2);
+		return GS_ERROR;
+	      }
+	  }
+    }
+
   link_next = NULL_TREE;
-  for (link = ASM_INPUTS (expr); link; ++i, link = link_next)
+  int input_num = 0;
+  for (link = ASM_INPUTS (expr); link; ++input_num, ++i, link = link_next)
     {
       link_next = TREE_CHAIN (link);
       constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
-      parse_input_constraint (&constraint, 0, 0, noutputs, 0,
-			      oconstraints, &allows_mem, &allows_reg);
+      reg_info.operand = TREE_VALUE (link);
+      bool ok = parse_input_constraint (&constraint, input_num, 0, noutputs, 0,
+					oconstraints, &allows_mem, &allows_reg,
+					&reg_info);
+      if (!ok)
+	{
+	  ret = GS_ERROR;
+	  is_inout = false;
+	}
 
       /* If we can't make copies, we can only accept memory.  */
       tree intype = TREE_TYPE (TREE_VALUE (link));
@@ -8128,15 +8247,7 @@ gimplify_asm_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
     }
 
   link_next = NULL_TREE;
-  for (link = ASM_CLOBBERS (expr); link; ++i, link = link_next)
-    {
-      link_next = TREE_CHAIN (link);
-      TREE_CHAIN (link) = NULL_TREE;
-      vec_safe_push (clobbers, link);
-    }
-
-  link_next = NULL_TREE;
-  for (link = ASM_LABELS (expr); link; ++i, link = link_next)
+  for (link = ASM_LABELS (expr); link; link = link_next)
     {
       link_next = TREE_CHAIN (link);
       TREE_CHAIN (link) = NULL_TREE;
@@ -9526,9 +9637,7 @@ gimplify_omp_affinity (tree *list_p, gimple_seq *pre_p)
     if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_AFFINITY)
       {
 	tree t = OMP_CLAUSE_DECL (c);
-	if (TREE_CODE (t) == TREE_LIST
-		    && TREE_PURPOSE (t)
-		    && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	if (OMP_ITERATOR_DECL_P (t))
 	  {
 	    if (TREE_VALUE (t) == null_pointer_node)
 	      continue;
@@ -9633,6 +9742,155 @@ gimplify_omp_affinity (tree *list_p, gimple_seq *pre_p)
   return;
 }
 
+/* Returns a tree expression containing the total iteration count of the
+   OpenMP iterator IT.  */
+
+static tree
+compute_omp_iterator_count (tree it, gimple_seq *pre_p)
+{
+  tree tcnt = size_one_node;
+  for (; it; it = TREE_CHAIN (it))
+    {
+      if (gimplify_expr (&TREE_VEC_ELT (it, 1), pre_p, NULL,
+			 is_gimple_val, fb_rvalue) == GS_ERROR
+	  || gimplify_expr (&TREE_VEC_ELT (it, 2), pre_p, NULL,
+			    is_gimple_val, fb_rvalue) == GS_ERROR
+	  || gimplify_expr (&TREE_VEC_ELT (it, 3), pre_p, NULL,
+			    is_gimple_val, fb_rvalue) == GS_ERROR
+	  || (gimplify_expr (&TREE_VEC_ELT (it, 4), pre_p, NULL,
+			     is_gimple_val, fb_rvalue) == GS_ERROR))
+	return NULL_TREE;
+      tree var = TREE_VEC_ELT (it, 0);
+      tree begin = TREE_VEC_ELT (it, 1);
+      tree end = TREE_VEC_ELT (it, 2);
+      tree step = TREE_VEC_ELT (it, 3);
+      tree orig_step = TREE_VEC_ELT (it, 4);
+      tree type = TREE_TYPE (var);
+      tree stype = TREE_TYPE (step);
+      location_t loc = DECL_SOURCE_LOCATION (var);
+      tree endmbegin;
+      /* Compute count for this iterator as
+	 orig_step > 0
+	 ? (begin < end ? (end - begin + (step - 1)) / step : 0)
+	 : (begin > end ? (end - begin + (step + 1)) / step : 0)
+	 and compute product of those for the entire clause.  */
+      if (POINTER_TYPE_P (type))
+	endmbegin = fold_build2_loc (loc, POINTER_DIFF_EXPR, stype, end, begin);
+      else
+	endmbegin = fold_build2_loc (loc, MINUS_EXPR, type, end, begin);
+      tree stepm1 = fold_build2_loc (loc, MINUS_EXPR, stype, step,
+				     build_int_cst (stype, 1));
+      tree stepp1 = fold_build2_loc (loc, PLUS_EXPR, stype, step,
+				     build_int_cst (stype, 1));
+      tree pos = fold_build2_loc (loc, PLUS_EXPR, stype,
+				  unshare_expr (endmbegin), stepm1);
+      pos = fold_build2_loc (loc, TRUNC_DIV_EXPR, stype, pos, step);
+      tree neg = fold_build2_loc (loc, PLUS_EXPR, stype, endmbegin, stepp1);
+      if (TYPE_UNSIGNED (stype))
+	{
+	  neg = fold_build1_loc (loc, NEGATE_EXPR, stype, neg);
+	  step = fold_build1_loc (loc, NEGATE_EXPR, stype, step);
+	}
+      neg = fold_build2_loc (loc, TRUNC_DIV_EXPR, stype, neg, step);
+      step = NULL_TREE;
+      tree cond = fold_build2_loc (loc, LT_EXPR, boolean_type_node, begin, end);
+      pos = fold_build3_loc (loc, COND_EXPR, stype, cond, pos,
+			     build_int_cst (stype, 0));
+      cond = fold_build2_loc (loc, LT_EXPR, boolean_type_node, end, begin);
+      neg = fold_build3_loc (loc, COND_EXPR, stype, cond, neg,
+			     build_int_cst (stype, 0));
+      tree osteptype = TREE_TYPE (orig_step);
+      cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node, orig_step,
+			      build_int_cst (osteptype, 0));
+      tree cnt = fold_build3_loc (loc, COND_EXPR, stype, cond, pos, neg);
+      cnt = fold_convert_loc (loc, sizetype, cnt);
+      if (gimplify_expr (&cnt, pre_p, NULL, is_gimple_val,
+			 fb_rvalue) == GS_ERROR)
+	return NULL_TREE;
+      tcnt = size_binop_loc (loc, MULT_EXPR, tcnt, cnt);
+    }
+  if (gimplify_expr (&tcnt, pre_p, NULL, is_gimple_val, fb_rvalue) == GS_ERROR)
+    return NULL_TREE;
+
+  return tcnt;
+}
+
+/* Build loops iterating over the space defined by the OpenMP iterator IT.
+   Returns a pointer to the BIND_EXPR_BODY in the innermost loop body.
+   LAST_BIND is set to point to the BIND_EXPR containing the whole loop.  */
+
+static tree *
+build_omp_iterator_loop (tree it, gimple_seq *pre_p, tree *last_bind)
+{
+  if (*last_bind)
+    gimplify_and_add (*last_bind, pre_p);
+  tree block = TREE_VEC_ELT (it, 5);
+  *last_bind = build3 (BIND_EXPR, void_type_node,
+		       BLOCK_VARS (block), NULL, block);
+  TREE_SIDE_EFFECTS (*last_bind) = 1;
+  tree *p = &BIND_EXPR_BODY (*last_bind);
+  for (; it; it = TREE_CHAIN (it))
+    {
+      tree var = TREE_VEC_ELT (it, 0);
+      tree begin = TREE_VEC_ELT (it, 1);
+      tree end = TREE_VEC_ELT (it, 2);
+      tree step = TREE_VEC_ELT (it, 3);
+      tree orig_step = TREE_VEC_ELT (it, 4);
+      tree type = TREE_TYPE (var);
+      location_t loc = DECL_SOURCE_LOCATION (var);
+      /* Emit:
+	 var = begin;
+	 goto cond_label;
+	 beg_label:
+	 ...
+	 var = var + step;
+	 cond_label:
+	 if (orig_step > 0) {
+	   if (var < end) goto beg_label;
+	 } else {
+	   if (var > end) goto beg_label;
+	 }
+	 for each iterator, with inner iterators added to
+	 the ... above.  */
+      tree beg_label = create_artificial_label (loc);
+      tree cond_label = NULL_TREE;
+      tree tem = build2_loc (loc, MODIFY_EXPR, void_type_node, var, begin);
+      append_to_statement_list_force (tem, p);
+      tem = build_and_jump (&cond_label);
+      append_to_statement_list_force (tem, p);
+      tem = build1 (LABEL_EXPR, void_type_node, beg_label);
+      append_to_statement_list (tem, p);
+      tree bind = build3 (BIND_EXPR, void_type_node, NULL_TREE,
+			  NULL_TREE, NULL_TREE);
+      TREE_SIDE_EFFECTS (bind) = 1;
+      SET_EXPR_LOCATION (bind, loc);
+      append_to_statement_list_force (bind, p);
+      if (POINTER_TYPE_P (type))
+	tem = build2_loc (loc, POINTER_PLUS_EXPR, type,
+			  var, fold_convert_loc (loc, sizetype, step));
+      else
+	tem = build2_loc (loc, PLUS_EXPR, type, var, step);
+      tem = build2_loc (loc, MODIFY_EXPR, void_type_node, var, tem);
+      append_to_statement_list_force (tem, p);
+      tem = build1 (LABEL_EXPR, void_type_node, cond_label);
+      append_to_statement_list (tem, p);
+      tree cond = fold_build2_loc (loc, LT_EXPR, boolean_type_node, var, end);
+      tree pos = fold_build3_loc (loc, COND_EXPR, void_type_node, cond,
+				  build_and_jump (&beg_label), void_node);
+      cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node, var, end);
+      tree neg = fold_build3_loc (loc, COND_EXPR, void_type_node, cond,
+				  build_and_jump (&beg_label), void_node);
+      tree osteptype = TREE_TYPE (orig_step);
+      cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node, orig_step,
+			      build_int_cst (osteptype, 0));
+      tem = fold_build3_loc (loc, COND_EXPR, void_type_node, cond, pos, neg);
+      append_to_statement_list_force (tem, p);
+      p = &BIND_EXPR_BODY (bind);
+    }
+
+  return p;
+}
+
 /* If *LIST_P contains any OpenMP depend clauses with iterators,
    lower all the depend clauses by populating corresponding depend
    array.  Returns 0 if there are no such depend clauses, or
@@ -9677,89 +9935,13 @@ gimplify_omp_depend (tree *list_p, gimple_seq *pre_p)
 	tree t = OMP_CLAUSE_DECL (c);
 	if (first_loc == UNKNOWN_LOCATION)
 	  first_loc = OMP_CLAUSE_LOCATION (c);
-	if (TREE_CODE (t) == TREE_LIST
-	    && TREE_PURPOSE (t)
-	    && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	if (OMP_ITERATOR_DECL_P (t))
 	  {
 	    if (TREE_PURPOSE (t) != last_iter)
 	      {
-		tree tcnt = size_one_node;
-		for (tree it = TREE_PURPOSE (t); it; it = TREE_CHAIN (it))
-		  {
-		    if (gimplify_expr (&TREE_VEC_ELT (it, 1), pre_p, NULL,
-				       is_gimple_val, fb_rvalue) == GS_ERROR
-			|| gimplify_expr (&TREE_VEC_ELT (it, 2), pre_p, NULL,
-					  is_gimple_val, fb_rvalue) == GS_ERROR
-			|| gimplify_expr (&TREE_VEC_ELT (it, 3), pre_p, NULL,
-					  is_gimple_val, fb_rvalue) == GS_ERROR
-			|| (gimplify_expr (&TREE_VEC_ELT (it, 4), pre_p, NULL,
-					   is_gimple_val, fb_rvalue)
-			    == GS_ERROR))
-		      return 2;
-		    tree var = TREE_VEC_ELT (it, 0);
-		    tree begin = TREE_VEC_ELT (it, 1);
-		    tree end = TREE_VEC_ELT (it, 2);
-		    tree step = TREE_VEC_ELT (it, 3);
-		    tree orig_step = TREE_VEC_ELT (it, 4);
-		    tree type = TREE_TYPE (var);
-		    tree stype = TREE_TYPE (step);
-		    location_t loc = DECL_SOURCE_LOCATION (var);
-		    tree endmbegin;
-		    /* Compute count for this iterator as
-		       orig_step > 0
-		       ? (begin < end ? (end - begin + (step - 1)) / step : 0)
-		       : (begin > end ? (end - begin + (step + 1)) / step : 0)
-		       and compute product of those for the entire depend
-		       clause.  */
-		    if (POINTER_TYPE_P (type))
-		      endmbegin = fold_build2_loc (loc, POINTER_DIFF_EXPR,
-						   stype, end, begin);
-		    else
-		      endmbegin = fold_build2_loc (loc, MINUS_EXPR, type,
-						   end, begin);
-		    tree stepm1 = fold_build2_loc (loc, MINUS_EXPR, stype,
-						   step,
-						   build_int_cst (stype, 1));
-		    tree stepp1 = fold_build2_loc (loc, PLUS_EXPR, stype, step,
-						   build_int_cst (stype, 1));
-		    tree pos = fold_build2_loc (loc, PLUS_EXPR, stype,
-						unshare_expr (endmbegin),
-						stepm1);
-		    pos = fold_build2_loc (loc, TRUNC_DIV_EXPR, stype,
-					   pos, step);
-		    tree neg = fold_build2_loc (loc, PLUS_EXPR, stype,
-						endmbegin, stepp1);
-		    if (TYPE_UNSIGNED (stype))
-		      {
-			neg = fold_build1_loc (loc, NEGATE_EXPR, stype, neg);
-			step = fold_build1_loc (loc, NEGATE_EXPR, stype, step);
-		      }
-		    neg = fold_build2_loc (loc, TRUNC_DIV_EXPR, stype,
-					   neg, step);
-		    step = NULL_TREE;
-		    tree cond = fold_build2_loc (loc, LT_EXPR,
-						 boolean_type_node,
-						 begin, end);
-		    pos = fold_build3_loc (loc, COND_EXPR, stype, cond, pos,
-					   build_int_cst (stype, 0));
-		    cond = fold_build2_loc (loc, LT_EXPR, boolean_type_node,
-					    end, begin);
-		    neg = fold_build3_loc (loc, COND_EXPR, stype, cond, neg,
-					   build_int_cst (stype, 0));
-		    tree osteptype = TREE_TYPE (orig_step);
-		    cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node,
-					    orig_step,
-					    build_int_cst (osteptype, 0));
-		    tree cnt = fold_build3_loc (loc, COND_EXPR, stype,
-						cond, pos, neg);
-		    cnt = fold_convert_loc (loc, sizetype, cnt);
-		    if (gimplify_expr (&cnt, pre_p, NULL, is_gimple_val,
-				       fb_rvalue) == GS_ERROR)
-		      return 2;
-		    tcnt = size_binop_loc (loc, MULT_EXPR, tcnt, cnt);
-		  }
-		if (gimplify_expr (&tcnt, pre_p, NULL, is_gimple_val,
-				   fb_rvalue) == GS_ERROR)
+		tree tcnt = compute_omp_iterator_count (TREE_PURPOSE (t),
+							pre_p);
+		if (!tcnt)
 		  return 2;
 		last_iter = TREE_PURPOSE (t);
 		last_count = tcnt;
@@ -9913,91 +10095,13 @@ gimplify_omp_depend (tree *list_p, gimple_seq *pre_p)
 	    gcc_unreachable ();
 	  }
 	tree t = OMP_CLAUSE_DECL (c);
-	if (TREE_CODE (t) == TREE_LIST
-	    && TREE_PURPOSE (t)
-	    && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	if (OMP_ITERATOR_DECL_P (t))
 	  {
 	    if (TREE_PURPOSE (t) != last_iter)
 	      {
-		if (last_bind)
-		  gimplify_and_add (last_bind, pre_p);
-		tree block = TREE_VEC_ELT (TREE_PURPOSE (t), 5);
-		last_bind = build3 (BIND_EXPR, void_type_node,
-				    BLOCK_VARS (block), NULL, block);
-		TREE_SIDE_EFFECTS (last_bind) = 1;
+		last_body = build_omp_iterator_loop (TREE_PURPOSE (t), pre_p,
+						     &last_bind);
 		SET_EXPR_LOCATION (last_bind, OMP_CLAUSE_LOCATION (c));
-		tree *p = &BIND_EXPR_BODY (last_bind);
-		for (tree it = TREE_PURPOSE (t); it; it = TREE_CHAIN (it))
-		  {
-		    tree var = TREE_VEC_ELT (it, 0);
-		    tree begin = TREE_VEC_ELT (it, 1);
-		    tree end = TREE_VEC_ELT (it, 2);
-		    tree step = TREE_VEC_ELT (it, 3);
-		    tree orig_step = TREE_VEC_ELT (it, 4);
-		    tree type = TREE_TYPE (var);
-		    location_t loc = DECL_SOURCE_LOCATION (var);
-		    /* Emit:
-		       var = begin;
-		       goto cond_label;
-		       beg_label:
-		       ...
-		       var = var + step;
-		       cond_label:
-		       if (orig_step > 0) {
-			 if (var < end) goto beg_label;
-		       } else {
-			 if (var > end) goto beg_label;
-		       }
-		       for each iterator, with inner iterators added to
-		       the ... above.  */
-		    tree beg_label = create_artificial_label (loc);
-		    tree cond_label = NULL_TREE;
-		    tem = build2_loc (loc, MODIFY_EXPR, void_type_node,
-				      var, begin);
-		    append_to_statement_list_force (tem, p);
-		    tem = build_and_jump (&cond_label);
-		    append_to_statement_list_force (tem, p);
-		    tem = build1 (LABEL_EXPR, void_type_node, beg_label);
-		    append_to_statement_list (tem, p);
-		    tree bind = build3 (BIND_EXPR, void_type_node, NULL_TREE,
-					NULL_TREE, NULL_TREE);
-		    TREE_SIDE_EFFECTS (bind) = 1;
-		    SET_EXPR_LOCATION (bind, loc);
-		    append_to_statement_list_force (bind, p);
-		    if (POINTER_TYPE_P (type))
-		      tem = build2_loc (loc, POINTER_PLUS_EXPR, type,
-					var, fold_convert_loc (loc, sizetype,
-							       step));
-		    else
-		      tem = build2_loc (loc, PLUS_EXPR, type, var, step);
-		    tem = build2_loc (loc, MODIFY_EXPR, void_type_node,
-				      var, tem);
-		    append_to_statement_list_force (tem, p);
-		    tem = build1 (LABEL_EXPR, void_type_node, cond_label);
-		    append_to_statement_list (tem, p);
-		    tree cond = fold_build2_loc (loc, LT_EXPR,
-						 boolean_type_node,
-						 var, end);
-		    tree pos
-		      = fold_build3_loc (loc, COND_EXPR, void_type_node,
-					 cond, build_and_jump (&beg_label),
-					 void_node);
-		    cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node,
-					    var, end);
-		    tree neg
-		      = fold_build3_loc (loc, COND_EXPR, void_type_node,
-					 cond, build_and_jump (&beg_label),
-					 void_node);
-		    tree osteptype = TREE_TYPE (orig_step);
-		    cond = fold_build2_loc (loc, GT_EXPR, boolean_type_node,
-					    orig_step,
-					    build_int_cst (osteptype, 0));
-		    tem = fold_build3_loc (loc, COND_EXPR, void_type_node,
-					   cond, pos, neg);
-		    append_to_statement_list_force (tem, p);
-		    p = &BIND_EXPR_BODY (bind);
-		  }
-		last_body = p;
 	      }
 	    last_iter = TREE_PURPOSE (t);
 	    if (TREE_CODE (TREE_VALUE (t)) == COMPOUND_EXPR)
diff --git a/gcc/gimplify_reg_info.h b/gcc/gimplify_reg_info.h
new file mode 100644
index 0000000..b56b225
--- /dev/null
+++ b/gcc/gimplify_reg_info.h
@@ -0,0 +1,182 @@
+/* gimplify_reg_info is used during gimplification while walking over
+   operands and their corresponding constraints of asm statements in order to
+   detect errors.
+
+   m_alt_output is a mapping describing which registers are potentially used in
+   which alternative over all outputs.  Similarly for m_alt_input but over all
+   inputs.
+
+   m_early_clobbered_alt is a mapping describing which register is early
+   clobbered in which alternative over all outputs.
+
+   m_early_clobbered_output is the counter part to the prior one, i.e., it
+   is a mapping describing which register is early clobbered in which operand
+   over all alternatives.
+
+   m_reg_asm_output is the set of registers (including register pairs) used for
+   register asm output operands.
+
+   m_reg_asm_input similar as m_reg_asm_output but for inputs.  */
+
+#include "regs.h"
+
+class gimplify_reg_info
+{
+  HARD_REG_SET *m_buf;
+  HARD_REG_SET *m_alt_output;
+  HARD_REG_SET *m_alt_input;
+  HARD_REG_SET *m_early_clobbered_alt;
+  HARD_REG_SET *m_early_clobbered_output;
+  HARD_REG_SET m_reg_asm_output;
+  HARD_REG_SET m_reg_asm_input;
+  const unsigned m_num_alternatives;
+  const unsigned m_num_outputs;
+  /* Member m_clobbered describes all the registers marked as clobbered in an
+     asm statement, i.e., this is the clobbers list of an extended asm
+
+     asm asm-qualifiers ( AssemblerTemplate
+		 : OutputOperands
+		 [ : InputOperands
+		 [ : Clobbers ] ])
+
+     and is not to be confused with the early clobbers sets.  */
+  HARD_REG_SET m_clobbered;
+
+  /* Return the first overlapping register of REGS and REGNO:MODE or -1.  */
+  int test (const HARD_REG_SET &regs, int regno) const
+  {
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+
+    if (TEST_HARD_REG_BIT (regs, regno))
+      return regno;
+
+    int end_regno = end_hard_regno (mode, regno);
+    while (++regno < end_regno)
+      if (TEST_HARD_REG_BIT (regs, regno))
+	return regno;
+
+    return -1;
+  }
+
+public:
+  tree operand;
+
+  gimplify_reg_info (unsigned num_alternatives,
+		     unsigned num_outputs)
+    : m_num_alternatives{num_alternatives}
+    , m_num_outputs{num_outputs}
+  {
+    CLEAR_HARD_REG_SET (m_reg_asm_output);
+    CLEAR_HARD_REG_SET (m_reg_asm_input);
+    CLEAR_HARD_REG_SET (m_clobbered);
+
+    /* If there are no alternatives, then there are no outputs/inputs and there
+       is nothing to do on our end.  Thus, we are dealing most likely with a
+       basic asm.  */
+    if (num_alternatives == 0)
+      return;
+
+    unsigned buf_size = num_alternatives * 3 + num_outputs;
+    m_buf = new HARD_REG_SET[buf_size];
+    for (unsigned i = 0; i < buf_size; ++i)
+      CLEAR_HARD_REG_SET (m_buf[i]);
+    m_alt_output = &m_buf[0];
+    m_alt_input = &m_buf[num_alternatives];
+    m_early_clobbered_alt = &m_buf[num_alternatives * 2];
+    if (num_outputs > 0)
+      m_early_clobbered_output = &m_buf[num_alternatives * 3];
+    else
+      m_early_clobbered_output = nullptr;
+  }
+
+  ~gimplify_reg_info ()
+  {
+    if (m_num_alternatives > 0)
+      delete[] m_buf;
+  }
+
+  void set_output (unsigned alt, int regno)
+  {
+    gcc_checking_assert (alt < m_num_alternatives);
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+    add_to_hard_reg_set (&m_alt_output[alt], mode, regno);
+  }
+
+  void set_input (unsigned alt, int regno)
+  {
+    gcc_checking_assert (alt < m_num_alternatives);
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+    add_to_hard_reg_set (&m_alt_input[alt], mode, regno);
+  }
+
+  int test_alt_output (unsigned alt, int regno) const
+  {
+    gcc_checking_assert (alt < m_num_alternatives);
+    return test (m_alt_output[alt], regno);
+  }
+
+  int test_alt_input (unsigned alt, int regno) const
+  {
+    gcc_checking_assert (alt < m_num_alternatives);
+    return test (m_alt_input[alt], regno);
+  }
+
+  void set_reg_asm_output (int regno)
+  {
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+    add_to_hard_reg_set (&m_reg_asm_output, mode, regno);
+  }
+
+  int test_reg_asm_output (int regno) const
+  {
+    return test (m_reg_asm_output, regno);
+  }
+
+  void set_reg_asm_input (int regno)
+  {
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+    add_to_hard_reg_set (&m_reg_asm_input, mode, regno);
+  }
+
+  int test_reg_asm_input (int regno) const
+  {
+    return test (m_reg_asm_input, regno);
+  }
+
+  void set_early_clobbered (unsigned alt, unsigned output, int regno)
+  {
+    gcc_checking_assert (alt < m_num_alternatives);
+    gcc_checking_assert (output < m_num_outputs);
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+    add_to_hard_reg_set (&m_early_clobbered_alt[alt], mode, regno);
+    add_to_hard_reg_set (&m_early_clobbered_output[output], mode, regno);
+  }
+
+  bool test_early_clobbered_alt (unsigned alt, int regno) const
+  {
+    gcc_checking_assert (alt < m_num_alternatives);
+    return TEST_HARD_REG_BIT (m_early_clobbered_alt[alt], regno);
+  }
+
+  bool is_early_clobbered_in_any_output_unequal (unsigned operand,
+						 int regno) const
+  {
+    gcc_checking_assert (operand < m_num_outputs);
+    for (unsigned op = 0; op < m_num_outputs; ++op)
+      if (op != operand
+	  && TEST_HARD_REG_BIT (m_early_clobbered_output[op], regno))
+	return true;
+    return false;
+  }
+
+  void set_clobbered (int regno)
+  {
+    SET_HARD_REG_BIT (m_clobbered, regno);
+  }
+
+  bool is_clobbered (int regno) const
+  {
+    machine_mode mode = TYPE_MODE (TREE_TYPE (operand));
+    return overlaps_hard_reg_set_p (m_clobbered, mode, regno);
+  }
+};
diff --git a/gcc/go/ChangeLog b/gcc/go/ChangeLog
index 83829c5..ed189f9 100644
--- a/gcc/go/ChangeLog
+++ b/gcc/go/ChangeLog
@@ -1,3 +1,10 @@
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* gofrontend/gogo.cc (Function::export_func_with_type): Remove
+	unused but set variable i.
+
 2025-02-05  Ian Lance Taylor  <iant@golang.org>
 
 	PR go/118746
diff --git a/gcc/go/gofrontend/MERGE b/gcc/go/gofrontend/MERGE
index b6fdf72..99bf24c 100644
--- a/gcc/go/gofrontend/MERGE
+++ b/gcc/go/gofrontend/MERGE
@@ -1,4 +1,4 @@
-f5c453aa726ebb509e7b8cb20df7734f0e411404
+c4d7bfb9895efc196b04f18f5da77fd99b39212a
 
 The first line of this file holds the git revision number of the last
 merge done from the gofrontend repository.
diff --git a/gcc/go/gofrontend/gogo.cc b/gcc/go/gofrontend/gogo.cc
index 3aad419..566eca8 100644
--- a/gcc/go/gofrontend/gogo.cc
+++ b/gcc/go/gofrontend/gogo.cc
@@ -5987,12 +5987,11 @@ Function::export_func_with_type(Export* exp, const Named_object* no,
   const Typed_identifier_list* parameters = fntype->parameters();
   if (parameters != NULL)
     {
-      size_t i = 0;
       bool is_varargs = fntype->is_varargs();
       bool first = true;
       for (Typed_identifier_list::const_iterator p = parameters->begin();
 	   p != parameters->end();
-	   ++p, ++i)
+	   ++p)
 	{
 	  if (first)
 	    first = false;
diff --git a/gcc/graphite.h b/gcc/graphite.h
index 66d4db6..9cb3513 100644
--- a/gcc/graphite.h
+++ b/gcc/graphite.h
@@ -155,7 +155,7 @@ struct poly_dr
      The OpenScop access function is printed as follows:
 
      | 1  # The number of disjunct components in a union of access functions.
-     | R C O I L P  # Described bellow.
+     | R C O I L P  # Described below.
      | a  s0  s1   i   j   k   1
      | 1   0   0   0   0   0  -5     =  0
      | 0   1   0  -1   0   0   0     =  0
diff --git a/gcc/graphviz.h b/gcc/graphviz.h
index 5943589..9a0fe6f 100644
--- a/gcc/graphviz.h
+++ b/gcc/graphviz.h
@@ -311,6 +311,16 @@ struct node_id : public ast_node
       m_port = std::make_unique<port> (*other.m_port);
   }
 
+  node_id &operator= (const node_id &other)
+  {
+    m_id = other.m_id;
+    if (other.m_port)
+      m_port = std::make_unique<port> (*other.m_port);
+    else
+      m_port = nullptr;
+    return *this;
+  }
+
   void print (writer &w) const final override;
 
   id m_id;
diff --git a/gcc/hooks.cc b/gcc/hooks.cc
index 951825d..76cb5931 100644
--- a/gcc/hooks.cc
+++ b/gcc/hooks.cc
@@ -117,6 +117,13 @@ hook_bool_mode_const_rtx_true (machine_mode, const_rtx)
   return true;
 }
 
+/* Generic hook that takes (machine_mode, int, unsigned) and returns false.  */
+bool
+hook_bool_mode_int_unsigned_false (machine_mode, int, unsigned)
+{
+  return false;
+}
+
 /* Generic hook that takes (machine_mode, rtx) and returns false.  */
 bool
 hook_bool_mode_rtx_false (machine_mode, rtx)
diff --git a/gcc/hooks.h b/gcc/hooks.h
index c0663bf..e95bd11 100644
--- a/gcc/hooks.h
+++ b/gcc/hooks.h
@@ -36,6 +36,7 @@ extern bool hook_bool_mode_true (machine_mode);
 extern bool hook_bool_mode_mode_true (machine_mode, machine_mode);
 extern bool hook_bool_mode_const_rtx_false (machine_mode, const_rtx);
 extern bool hook_bool_mode_const_rtx_true (machine_mode, const_rtx);
+extern bool hook_bool_mode_int_unsigned_false (machine_mode, int, unsigned);
 extern bool hook_bool_mode_rtx_false (machine_mode, rtx);
 extern bool hook_bool_mode_rtx_true (machine_mode, rtx);
 extern bool hook_bool_const_rtx_insn_const_rtx_insn_true (const rtx_insn *,
diff --git a/gcc/input.cc b/gcc/input.cc
index fabfbfb..b9a5539 100644
--- a/gcc/input.cc
+++ b/gcc/input.cc
@@ -22,6 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "coretypes.h"
 #include "intl.h"
 #include "diagnostic.h"
+#include "diagnostics/file-cache.h"
 #include "selftest.h"
 #include "cpplib.h"
 
@@ -35,188 +36,6 @@ special_fname_builtin ()
   return _("<built-in>");
 }
 
-/* Input charset configuration.  */
-static const char *default_charset_callback (const char *)
-{
-  return nullptr;
-}
-
-void
-file_cache::initialize_input_context (diagnostic_input_charset_callback ccb,
-				      bool should_skip_bom)
-{
-  m_input_context.ccb = (ccb ? ccb : default_charset_callback);
-  m_input_context.should_skip_bom = should_skip_bom;
-}
-
-/* This is a cache used by get_next_line to store the content of a
-   file to be searched for file lines.  */
-class file_cache_slot
-{
-public:
-  file_cache_slot ();
-  ~file_cache_slot ();
-
-  void dump (FILE *out, int indent) const;
-  void DEBUG_FUNCTION dump () const { dump (stderr, 0); }
-
-  bool read_line_num (size_t line_num,
-		      char ** line, ssize_t *line_len);
-
-  /* Accessors.  */
-  const char *get_file_path () const { return m_file_path; }
-  unsigned get_use_count () const { return m_use_count; }
-  bool missing_trailing_newline_p () const
-  {
-    return m_missing_trailing_newline;
-  }
-  char_span get_full_file_content ();
-
-  void inc_use_count () { m_use_count++; }
-
-  bool create (const file_cache::input_context &in_context,
-	       const char *file_path, FILE *fp, unsigned highest_use_count);
-  void evict ();
-  void set_content (const char *buf, size_t sz);
-
-  static size_t tune (size_t line_record_size_)
-  {
-    size_t ret = line_record_size;
-    line_record_size = line_record_size_;
-    return ret;
-  }
-
- private:
-  /* These are information used to store a line boundary.  */
-  class line_info
-  {
-  public:
-    /* The line number.  It starts from 1.  */
-    size_t line_num;
-
-    /* The position (byte count) of the beginning of the line,
-       relative to the file data pointer.  This starts at zero.  */
-    size_t start_pos;
-
-    /* The position (byte count) of the last byte of the line.  This
-       normally points to the '\n' character, or to one byte after the
-       last byte of the file, if the file doesn't contain a '\n'
-       character.  */
-    size_t end_pos;
-
-    line_info (size_t l, size_t s, size_t e)
-      : line_num (l), start_pos (s), end_pos (e)
-    {}
-
-    line_info ()
-      :line_num (0), start_pos (0), end_pos (0)
-    {}
-
-    static bool less_than(const line_info &a, const line_info &b)
-    {
-      return a.line_num < b.line_num;
-    }
-  };
-
-  bool needs_read_p () const;
-  bool needs_grow_p () const;
-  void maybe_grow ();
-  bool read_data ();
-  bool maybe_read_data ();
-  bool get_next_line (char **line, ssize_t *line_len);
-  bool read_next_line (char ** line, ssize_t *line_len);
-  bool goto_next_line ();
-
-  static const size_t buffer_size = 4 * 1024;
-  static size_t line_record_size;
-  static size_t recent_cached_lines_shift;
-
-  /* The number of time this file has been accessed.  This is used
-     to designate which file cache to evict from the cache
-     array.  */
-  unsigned m_use_count;
-
-  /* The file_path is the key for identifying a particular file in
-     the cache.  This copy is owned by the slot.  */
-  char *m_file_path;
-
-  FILE *m_fp;
-
-  /* True when an read error happened.  */
-  bool m_error;
-
-  /* This points to the content of the file that we've read so
-     far.  */
-  char *m_data;
-
-  /* The allocated buffer to be freed may start a little earlier than DATA,
-     e.g. if a UTF8 BOM was skipped at the beginning.  */
-  int m_alloc_offset;
-
-  /*  The size of the DATA array above.*/
-  size_t m_size;
-
-  /* The number of bytes read from the underlying file so far.  This
-     must be less (or equal) than SIZE above.  */
-  size_t m_nb_read;
-
-  /* The index of the beginning of the current line.  */
-  size_t m_line_start_idx;
-
-  /* The number of the previous line read.  This starts at 1.  Zero
-     means we've read no line so far.  */
-  size_t m_line_num;
-
-  /* Could this file be missing a trailing newline on its final line?
-     Initially true (to cope with empty files), set to true/false
-     as each line is read.  */
-  bool m_missing_trailing_newline;
-
-  /* This is a record of the beginning and end of the lines we've seen
-     while reading the file.  This is useful to avoid walking the data
-     from the beginning when we are asked to read a line that is
-     before LINE_START_IDX above.  When the lines exceed line_record_size
-     this is scaled down dynamically, with the line_info becoming anchors.  */
-  vec<line_info, va_heap> m_line_record;
-
-  /* A cache of the recently seen lines. This is maintained as a ring
-     buffer. */
-  vec<line_info, va_heap> m_line_recent;
-
-  /* First and last valid entry in m_line_recent.  */
-  size_t m_line_recent_last, m_line_recent_first;
-
-  void offset_buffer (int offset)
-  {
-    gcc_assert (offset < 0 ? m_alloc_offset + offset >= 0
-		: (size_t) offset <= m_size);
-    gcc_assert (m_data);
-    m_alloc_offset += offset;
-    m_data += offset;
-    m_size -= offset;
-  }
-
-};
-
-size_t file_cache_slot::line_record_size = 0;
-size_t file_cache_slot::recent_cached_lines_shift = 8;
-
-/* Tune file_cache.  */
-void
-file_cache::tune (size_t num_file_slots, size_t lines)
-{
-  if (file_cache_slot::tune (lines) != lines
-      || m_num_file_slots != num_file_slots)
-    {
-      delete[] m_file_slots;
-      m_file_slots = new file_cache_slot[num_file_slots];
-    }
-  m_num_file_slots = num_file_slots;
-}
-
-static const char *
-find_end_of_line (const char *s, size_t len);
-
 /* Current position in real source file.  */
 
 location_t input_location = UNKNOWN_LOCATION;
@@ -320,738 +139,13 @@ expand_location_1 (const line_maps *set,
   return xloc;
 }
 
-/* Lookup the cache used for the content of a given file accessed by
-   caret diagnostic.  Return the found cached file, or NULL if no
-   cached file was found.  */
-
-file_cache_slot *
-file_cache::lookup_file (const char *file_path)
-{
-  gcc_assert (file_path);
-
-  /* This will contain the found cached file.  */
-  file_cache_slot *r = NULL;
-  for (unsigned i = 0; i < m_num_file_slots; ++i)
-    {
-      file_cache_slot *c = &m_file_slots[i];
-      if (c->get_file_path () && !strcmp (c->get_file_path (), file_path))
-	{
-	  c->inc_use_count ();
-	  r = c;
-	}
-    }
-
-  if (r)
-    r->inc_use_count ();
-
-  return r;
-}
-
-/* Purge any mention of FILENAME from the cache of files used for
-   printing source code.  For use in selftests when working
-   with tempfiles.  */
-
-void
-file_cache::forcibly_evict_file (const char *file_path)
-{
-  gcc_assert (file_path);
-
-  file_cache_slot *r = lookup_file (file_path);
-  if (!r)
-    /* Not found.  */
-    return;
-
-  r->evict ();
-}
-
-/* Determine if FILE_PATH missing a trailing newline on its final line.
-   Only valid to call once all of the file has been loaded, by
-   requesting a line number beyond the end of the file.  */
-
-bool
-file_cache::missing_trailing_newline_p (const char *file_path)
-{
-  gcc_assert (file_path);
-
-  file_cache_slot *r = lookup_or_add_file (file_path);
-  return r->missing_trailing_newline_p ();
-}
-
-void
-file_cache::add_buffered_content (const char *file_path,
-				  const char *buffer,
-				  size_t sz)
-{
-  gcc_assert (file_path);
-
-  file_cache_slot *r = lookup_file (file_path);
-  if (!r)
-    {
-      unsigned highest_use_count = 0;
-      r = evicted_cache_tab_entry (&highest_use_count);
-      if (!r->create (m_input_context, file_path, nullptr, highest_use_count))
-	return;
-    }
-
-  r->set_content (buffer, sz);
-}
-
-void
-file_cache_slot::evict ()
-{
-  free (m_file_path);
-  m_file_path = NULL;
-  if (m_fp)
-    fclose (m_fp);
-  m_error = false;
-  m_fp = NULL;
-  m_nb_read = 0;
-  m_line_start_idx = 0;
-  m_line_num = 0;
-  m_line_record.truncate (0);
-  m_line_recent_first = 0;
-  m_line_recent_last = 0;
-  m_use_count = 0;
-  m_missing_trailing_newline = true;
-}
-
-/* Return the file cache that has been less used, recently, or the
-   first empty one.  If HIGHEST_USE_COUNT is non-null,
-   *HIGHEST_USE_COUNT is set to the highest use count of the entries
-   in the cache table.  */
-
-file_cache_slot*
-file_cache::evicted_cache_tab_entry (unsigned *highest_use_count)
-{
-  file_cache_slot *to_evict = &m_file_slots[0];
-  unsigned huc = to_evict->get_use_count ();
-  for (unsigned i = 1; i < m_num_file_slots; ++i)
-    {
-      file_cache_slot *c = &m_file_slots[i];
-      bool c_is_empty = (c->get_file_path () == NULL);
-
-      if (c->get_use_count () < to_evict->get_use_count ()
-	  || (to_evict->get_file_path () && c_is_empty))
-	/* We evict C because it's either an entry with a lower use
-	   count or one that is empty.  */
-	to_evict = c;
-
-      if (huc < c->get_use_count ())
-	huc = c->get_use_count ();
-
-      if (c_is_empty)
-	/* We've reached the end of the cache; subsequent elements are
-	   all empty.  */
-	break;
-    }
-
-  if (highest_use_count)
-    *highest_use_count = huc;
-
-  return to_evict;
-}
-
-/* Create the cache used for the content of a given file to be
-   accessed by caret diagnostic.  This cache is added to an array of
-   cache and can be retrieved by lookup_file_in_cache_tab.  This
-   function returns the created cache.  Note that only the last
-   m_num_file_slots files are cached.
-
-   This can return nullptr if the FILE_PATH can't be opened for
-   reading, or if the content can't be converted to the input_charset.  */
-
-file_cache_slot*
-file_cache::add_file (const char *file_path)
-{
-
-  FILE *fp = fopen (file_path, "r");
-  if (fp == NULL)
-    return NULL;
-
-  unsigned highest_use_count = 0;
-  file_cache_slot *r = evicted_cache_tab_entry (&highest_use_count);
-  if (!r->create (m_input_context, file_path, fp, highest_use_count))
-    return NULL;
-  return r;
-}
-
-/* Get a borrowed char_span to the full content of this file
-   as decoded according to the input charset, encoded as UTF-8.  */
-
-char_span
-file_cache_slot::get_full_file_content ()
-{
-  char *line;
-  ssize_t line_len;
-  while (get_next_line (&line, &line_len))
-    {
-    }
-  return char_span (m_data, m_nb_read);
-}
-
-/* Populate this slot for use on FILE_PATH and FP, dropping any
-   existing cached content within it.  */
-
-bool
-file_cache_slot::create (const file_cache::input_context &in_context,
-			 const char *file_path, FILE *fp,
-			 unsigned highest_use_count)
-{
-  m_file_path = file_path ? xstrdup (file_path) : nullptr;
-  if (m_fp)
-    fclose (m_fp);
-  m_error = false;
-  m_fp = fp;
-  if (m_alloc_offset)
-    offset_buffer (-m_alloc_offset);
-  m_nb_read = 0;
-  m_line_start_idx = 0;
-  m_line_num = 0;
-  m_line_recent_first = 0;
-  m_line_recent_last = 0;
-  m_line_record.truncate (0);
-  /* Ensure that this cache entry doesn't get evicted next time
-     add_file_to_cache_tab is called.  */
-  m_use_count = ++highest_use_count;
-  m_missing_trailing_newline = true;
-
-
-  /* Check the input configuration to determine if we need to do any
-     transformations, such as charset conversion or BOM skipping.  */
-  if (const char *input_charset = in_context.ccb (file_path))
-    {
-      /* Need a full-blown conversion of the input charset.  */
-      fclose (m_fp);
-      m_fp = NULL;
-      const cpp_converted_source cs
-	= cpp_get_converted_source (file_path, input_charset);
-      if (!cs.data)
-	return false;
-      if (m_data)
-	XDELETEVEC (m_data);
-      m_data = cs.data;
-      m_nb_read = m_size = cs.len;
-      m_alloc_offset = cs.data - cs.to_free;
-    }
-  else if (in_context.should_skip_bom)
-    {
-      if (read_data ())
-	{
-	  const int offset = cpp_check_utf8_bom (m_data, m_nb_read);
-	  offset_buffer (offset);
-	  m_nb_read -= offset;
-	}
-    }
-
-  return true;
-}
-
-void
-file_cache_slot::set_content (const char *buf, size_t sz)
-{
-  m_data = (char *)xmalloc (sz);
-  memcpy (m_data, buf, sz);
-  m_nb_read = m_size = sz;
-  m_alloc_offset = 0;
-
-  if (m_fp)
-    {
-      fclose (m_fp);
-      m_fp = nullptr;
-    }
-}
-
-/* file_cache's ctor.  */
-
-file_cache::file_cache ()
-: m_num_file_slots (16), m_file_slots (new file_cache_slot[m_num_file_slots])
-{
-  initialize_input_context (nullptr, false);
-}
-
-/* file_cache's dtor.  */
-
-file_cache::~file_cache ()
-{
-  delete[] m_file_slots;
-}
-
-void
-file_cache::dump (FILE *out, int indent) const
-{
-  for (size_t i = 0; i < m_num_file_slots; ++i)
-    {
-      fprintf (out, "%*sslot[%i]:\n", indent, "", (int)i);
-      m_file_slots[i].dump (out, indent + 2);
-    }
-}
-
-void
-file_cache::dump () const
-{
-  dump (stderr, 0);
-}
-
-/* Lookup the cache used for the content of a given file accessed by
-   caret diagnostic.  If no cached file was found, create a new cache
-   for this file, add it to the array of cached file and return
-   it.
-
-   This can return nullptr on a cache miss if FILE_PATH can't be opened for
-   reading, or if the content can't be converted to the input_charset.  */
-
-file_cache_slot*
-file_cache::lookup_or_add_file (const char *file_path)
-{
-  file_cache_slot *r = lookup_file (file_path);
-  if (r == NULL)
-    r = add_file (file_path);
-  return r;
-}
-
-/* Default constructor for a cache of file used by caret
-   diagnostic.  */
-
-file_cache_slot::file_cache_slot ()
-: m_use_count (0), m_file_path (NULL), m_fp (NULL), m_error (false), m_data (0),
-  m_alloc_offset (0), m_size (0), m_nb_read (0), m_line_start_idx (0),
-  m_line_num (0), m_missing_trailing_newline (true),
-  m_line_recent_last (0), m_line_recent_first (0)
-{
-  m_line_record.create (0);
-  m_line_recent.create (1U << recent_cached_lines_shift);
-  for (int i = 0; i < 1 << recent_cached_lines_shift; i++)
-    m_line_recent.quick_push (file_cache_slot::line_info (0, 0, 0));
-}
-
-/* Destructor for a cache of file used by caret diagnostic.  */
-
-file_cache_slot::~file_cache_slot ()
-{
-  free (m_file_path);
-  if (m_fp)
-    {
-      fclose (m_fp);
-      m_fp = NULL;
-    }
-  if (m_data)
-    {
-      offset_buffer (-m_alloc_offset);
-      XDELETEVEC (m_data);
-      m_data = 0;
-    }
-  m_line_record.release ();
-  m_line_recent.release ();
-}
-
-void
-file_cache_slot::dump (FILE *out, int indent) const
-{
-  if (!m_file_path)
-    {
-      fprintf (out, "%*s(unused)\n", indent, "");
-      return;
-    }
-  fprintf (out, "%*sfile_path: %s\n", indent, "", m_file_path);
-  fprintf (out, "%*sfp: %p\n", indent, "", (void *)m_fp);
-  fprintf (out, "%*sneeds_read_p: %i\n", indent, "", (int)needs_read_p ());
-  fprintf (out, "%*sneeds_grow_p: %i\n", indent, "", (int)needs_grow_p ());
-  fprintf (out, "%*suse_count: %i\n", indent, "", m_use_count);
-  fprintf (out, "%*ssize: %zi\n", indent, "", m_size);
-  fprintf (out, "%*snb_read: %zi\n", indent, "", m_nb_read);
-  fprintf (out, "%*sstart_line_idx: %zi\n", indent, "", m_line_start_idx);
-  fprintf (out, "%*sline_num: %zi\n", indent, "", m_line_num);
-  fprintf (out, "%*smissing_trailing_newline: %i\n",
-	   indent, "", (int)m_missing_trailing_newline);
-  fprintf (out, "%*sline records (%i):\n",
-	   indent, "", m_line_record.length ());
-  int idx = 0;
-  for (auto &line : m_line_record)
-    fprintf (out, "%*s[%i]: line %zi: byte offsets: %zi-%zi\n",
-	     indent + 2, "",
-	     idx++, line.line_num, line.start_pos, line.end_pos);
-}
-
-/* Returns TRUE iff the cache would need to be filled with data coming
-   from the file.  That is, either the cache is empty or full or the
-   current line is empty.  Note that if the cache is full, it would
-   need to be extended and filled again.  */
-
-bool
-file_cache_slot::needs_read_p () const
-{
-  return m_fp && (m_nb_read == 0
-	  || m_nb_read == m_size
-	  || (m_line_start_idx >= m_nb_read - 1));
-}
-
-/*  Return TRUE iff the cache is full and thus needs to be
-    extended.  */
-
-bool
-file_cache_slot::needs_grow_p () const
-{
-  return m_nb_read == m_size;
-}
-
-/* Grow the cache if it needs to be extended.  */
-
-void
-file_cache_slot::maybe_grow ()
-{
-  if (!needs_grow_p ())
-    return;
-
-  if (!m_data)
-    {
-      gcc_assert (m_size == 0 && m_alloc_offset == 0);
-      m_size = buffer_size;
-      m_data = XNEWVEC (char, m_size);
-    }
-  else
-    {
-      const int offset = m_alloc_offset;
-      offset_buffer (-offset);
-      m_size *= 2;
-      m_data = XRESIZEVEC (char, m_data, m_size);
-      offset_buffer (offset);
-    }
-}
-
-/*  Read more data into the cache.  Extends the cache if need be.
-    Returns TRUE iff new data could be read.  */
-
-bool
-file_cache_slot::read_data ()
-{
-  if (feof (m_fp) || ferror (m_fp))
-    return false;
-
-  maybe_grow ();
-
-  char * from = m_data + m_nb_read;
-  size_t to_read = m_size - m_nb_read;
-  size_t nb_read = fread (from, 1, to_read, m_fp);
-
-  if (ferror (m_fp))
-    {
-      m_error = true;
-      return false;
-    }
-
-  m_nb_read += nb_read;
-  return !!nb_read;
-}
-
-/* Read new data iff the cache needs to be filled with more data
-   coming from the file FP.  Return TRUE iff the cache was filled with
-   mode data.  */
-
-bool
-file_cache_slot::maybe_read_data ()
-{
-  if (!needs_read_p ())
-    return false;
-  return read_data ();
-}
-
-/* Helper function for file_cache_slot::get_next_line (), to find the end of
-   the next line.  Returns with the memchr convention, i.e. nullptr if a line
-   terminator was not found.  We need to determine line endings in the same
-   manner that libcpp does: any of \n, \r\n, or \r is a line ending.  */
-
-static const char *
-find_end_of_line (const char *s, size_t len)
-{
-  for (const auto end = s + len; s != end; ++s)
-    {
-      if (*s == '\n')
-	return s;
-      if (*s == '\r')
-	{
-	  const auto next = s + 1;
-	  if (next == end)
-	    {
-	      /* Don't find the line ending if \r is the very last character
-		 in the buffer; we do not know if it's the end of the file or
-		 just the end of what has been read so far, and we wouldn't
-		 want to break in the middle of what's actually a \r\n
-		 sequence.  Instead, we will handle the case of a file ending
-		 in a \r later.  */
-	      break;
-	    }
-	  return (*next == '\n' ? next : s);
-	}
-    }
-  return nullptr;
-}
-
-/* Read a new line from file FP, using C as a cache for the data
-   coming from the file.  Upon successful completion, *LINE is set to
-   the beginning of the line found.  *LINE points directly in the
-   line cache and is only valid until the next call of get_next_line.
-   *LINE_LEN is set to the length of the line.  Note that the line
-   does not contain any terminal delimiter.  This function returns
-   true if some data was read or process from the cache, false
-   otherwise.  Note that subsequent calls to get_next_line might
-   make the content of *LINE invalid.  */
-
-bool
-file_cache_slot::get_next_line (char **line, ssize_t *line_len)
-{
-  /* Fill the cache with data to process.  */
-  maybe_read_data ();
-
-  size_t remaining_size = m_nb_read - m_line_start_idx;
-  if (remaining_size == 0)
-    /* There is no more data to process.  */
-    return false;
-
-  const char *line_start = m_data + m_line_start_idx;
-
-  const char *next_line_start = NULL;
-  size_t len = 0;
-  const char *line_end = find_end_of_line (line_start, remaining_size);
-  if (line_end == NULL)
-    {
-      /* We haven't found an end-of-line delimiter in the cache.
-	 Fill the cache with more data from the file and look again.  */
-      while (maybe_read_data ())
-	{
-	  line_start = m_data + m_line_start_idx;
-	  remaining_size = m_nb_read - m_line_start_idx;
-	  line_end = find_end_of_line (line_start, remaining_size);
-	  if (line_end != NULL)
-	    {
-	      next_line_start = line_end + 1;
-	      break;
-	    }
-	}
-      if (line_end == NULL)
-	{
-	  /* We've loaded all the file into the cache and still no
-	     terminator.  Let's say the line ends up at one byte past the
-	     end of the file.  This is to stay consistent with the case
-	     of when the line ends up with a terminator and line_end points to
-	     that.  That consistency is useful below in the len calculation.
-
-	     If the file ends in a \r, we didn't identify it as a line
-	     terminator above, so do that now instead.  */
-	  line_end = m_data + m_nb_read;
-	  if (m_nb_read && line_end[-1] == '\r')
-	    {
-	      --line_end;
-	      m_missing_trailing_newline = false;
-	    }
-	  else
-	    m_missing_trailing_newline = true;
-	}
-      else
-	m_missing_trailing_newline = false;
-    }
-  else
-    {
-      next_line_start = line_end + 1;
-      m_missing_trailing_newline = false;
-    }
-
-  if (m_error)
-    return false;
-
-  /* At this point, we've found the end of the of line.  It either points to
-     the line terminator or to one byte after the last byte of the file.  */
-  gcc_assert (line_end != NULL);
-
-  len = line_end - line_start;
-
-  if (m_line_start_idx < m_nb_read)
-    *line = const_cast<char *> (line_start);
-
-  ++m_line_num;
-
-  /* Now update our line record so that re-reading lines from the
-     before m_line_start_idx is faster.  */
-  size_t rlen = m_line_record.length ();
-  /* Only update when beyond the previously cached region.  */
-  if (rlen == 0 || m_line_record[rlen - 1].line_num < m_line_num)
-    {
-      size_t spacing
-	= (rlen >= 2
-	   ? (m_line_record[rlen - 1].line_num
-	      - m_line_record[rlen - 2].line_num) : 1);
-      size_t delta
-	= rlen >= 1 ? m_line_num - m_line_record[rlen - 1].line_num : 1;
-
-      size_t max_size = line_record_size;
-      /* One anchor per hundred input lines.  */
-      if (max_size == 0)
-	max_size = m_line_num / 100;
-
-      /* If we're too far beyond drop half of the lines to rebalance.  */
-      if (rlen == max_size && delta >= spacing * 2)
-	{
-	  size_t j = 0;
-	  for (size_t i = 1; i < rlen; i += 2)
-	    m_line_record[j++] = m_line_record[i];
-	  m_line_record.truncate (j);
-	  rlen = j;
-	  spacing *= 2;
-	}
-
-      if (rlen < max_size && delta >= spacing)
-	{
-	  file_cache_slot::line_info li (m_line_num, m_line_start_idx,
-					 line_end - m_data);
-	  m_line_record.safe_push (li);
-	}
-    }
-
-  /* Cache recent tail lines separately for fast access. This assumes
-     most accesses do not skip backwards.  */
-  if (m_line_recent_last == m_line_recent_first
-      || m_line_recent[m_line_recent_last].line_num == m_line_num - 1)
-    {
-      size_t mask = ((size_t) 1 << recent_cached_lines_shift) - 1;
-      m_line_recent_last = (m_line_recent_last + 1) & mask;
-      if (m_line_recent_last == m_line_recent_first)
-	m_line_recent_first = (m_line_recent_first + 1) & mask;
-      m_line_recent[m_line_recent_last]
-	= file_cache_slot::line_info (m_line_num, m_line_start_idx,
-				      line_end - m_data);
-    }
-
-  /* Update m_line_start_idx so that it points to the next line to be
-     read.  */
-  if (next_line_start)
-    m_line_start_idx = next_line_start - m_data;
-  else
-    /* We didn't find any terminal '\n'.  Let's consider that the end
-       of line is the end of the data in the cache.  The next
-       invocation of get_next_line will either read more data from the
-       underlying file or return false early because we've reached the
-       end of the file.  */
-    m_line_start_idx = m_nb_read;
-
-  *line_len = len;
-
-  return true;
-}
-
-/* Consume the next bytes coming from the cache (or from its
-   underlying file if there are remaining unread bytes in the file)
-   until we reach the next end-of-line (or end-of-file).  There is no
-   copying from the cache involved.  Return TRUE upon successful
-   completion.  */
-
-bool
-file_cache_slot::goto_next_line ()
-{
-  char *l;
-  ssize_t len;
-
-  return get_next_line (&l, &len);
-}
-
-/* Read an arbitrary line number LINE_NUM from the file cached in C.
-   If the line was read successfully, *LINE points to the beginning
-   of the line in the file cache and *LINE_LEN is the length of the
-   line.  *LINE is not nul-terminated, but may contain zero bytes.
-   *LINE is only valid until the next call of read_line_num.
-   This function returns bool if a line was read.  */
-
-bool
-file_cache_slot::read_line_num (size_t line_num,
-				char ** line, ssize_t *line_len)
-{
-  gcc_assert (line_num > 0);
-
-  /* Is the line in the recent line cache?
-     This assumes the main file processing is only using
-     a single contiguous cursor with only temporary excursions.  */
-  if (m_line_recent_first != m_line_recent_last
-	&& m_line_recent[m_line_recent_first].line_num <= line_num
-	&& m_line_recent[m_line_recent_last].line_num >= line_num)
-    {
-      line_info &last = m_line_recent[m_line_recent_last];
-      size_t mask = (1U << recent_cached_lines_shift) - 1;
-      size_t idx = (m_line_recent_last - (last.line_num - line_num)) & mask;
-      line_info &recent = m_line_recent[idx];
-      gcc_assert (recent.line_num == line_num);
-      *line = m_data + recent.start_pos;
-      *line_len = recent.end_pos - recent.start_pos;
-      return true;
-    }
-
-  if (line_num <= m_line_num)
-    {
-      line_info l (line_num, 0, 0);
-      int i = m_line_record.lower_bound (l, line_info::less_than);
-      if (i == 0)
-	{
-	  m_line_start_idx = 0;
-	  m_line_num = 0;
-	}
-      else if (m_line_record[i - 1].line_num == line_num)
-	{
-	  /* We have the start/end of the line.  */
-	  *line = m_data + m_line_record[i - 1].start_pos;
-	  *line_len = m_line_record[i - 1].end_pos - m_line_record[i - 1].start_pos;
-	  return true;
-	}
-      else
-       {
-	 gcc_assert (m_line_record[i - 1].line_num < m_line_num);
-	 m_line_start_idx = m_line_record[i - 1].start_pos;
-	 m_line_num = m_line_record[i - 1].line_num - 1;
-       }
-    }
-
-  /*  Let's walk from line m_line_num up to line_num - 1, without
-      copying any line.  */
-  while (m_line_num < line_num - 1)
-    if (!goto_next_line ())
-      return false;
-
-  /* The line we want is the next one.  Let's read it.  */
-  return get_next_line (line, line_len);
-}
-
-/* Return the physical source line that corresponds to FILE_PATH/LINE.
-   The line is not nul-terminated.  The returned pointer is only
-   valid until the next call of location_get_source_line.
-   Note that the line can contain several null characters,
-   so the returned value's length has the actual length of the line.
-   If the function fails, a NULL char_span is returned.  */
-
-char_span
-file_cache::get_source_line (const char *file_path, int line)
-{
-  char *buffer = NULL;
-  ssize_t len;
-
-  if (line == 0)
-    return char_span (NULL, 0);
-
-  if (file_path == NULL)
-    return char_span (NULL, 0);
-
-  file_cache_slot *c = lookup_or_add_file (file_path);
-  if (c == NULL)
-    return char_span (NULL, 0);
-
-  bool read = c->read_line_num (line, &buffer, &len);
-  if (!read)
-    return char_span (NULL, 0);
-
-  return char_span (buffer, len);
-}
-
 /* Return a NUL-terminated copy of the source text between two locations, or
    NULL if the arguments are invalid.  The caller is responsible for freeing
    the return value.  */
 
 char *
-get_source_text_between (file_cache &fc, location_t start, location_t end)
+get_source_text_between (diagnostics::file_cache &fc,
+			 location_t start, location_t end)
 {
   expanded_location expstart
     = expand_location_to_spelling_point (start, LOCATION_ASPECT_START);
@@ -1076,7 +170,8 @@ get_source_text_between (file_cache &fc, location_t start, location_t end)
   /* For a single line we need to trim both edges.  */
   if (expstart.line == expend.line)
     {
-      char_span line = fc.get_source_line (expstart.file, expstart.line);
+      diagnostics::char_span line
+	= fc.get_source_line (expstart.file, expstart.line);
       if (line.length () < 1)
 	return NULL;
       int s = expstart.column - 1;
@@ -1093,7 +188,7 @@ get_source_text_between (file_cache &fc, location_t start, location_t end)
      parts of the start and end lines off depending on column values.  */
   for (int lnum = expstart.line; lnum <= expend.line; ++lnum)
     {
-      char_span line = fc.get_source_line (expstart.file, lnum);
+      diagnostics::char_span line = fc.get_source_line (expstart.file, lnum);
       if (line.length () < 1 && (lnum != expstart.line && lnum != expend.line))
 	continue;
 
@@ -1138,16 +233,6 @@ get_source_text_between (file_cache &fc, location_t start, location_t end)
   return xstrdup (buf);
 }
 
-
-char_span
-file_cache::get_source_file_content (const char *file_path)
-{
-  file_cache_slot *c = lookup_or_add_file (file_path);
-  if (c == nullptr)
-    return char_span (nullptr, 0);
-  return c->get_full_file_content ();
-}
-
 /* Test if the location originates from the spelling location of a
    builtin-tokens.  That is, return TRUE if LOC is a (possibly
    virtual) location of a built-in token that appears in the expansion
@@ -1280,13 +365,13 @@ make_location (location_t caret, source_range src_range)
    source line in order to calculate the display width.  If that cannot be done
    for any reason, then returns the byte column as a fallback.  */
 int
-location_compute_display_column (file_cache &fc,
+location_compute_display_column (diagnostics::file_cache &fc,
 				 expanded_location exploc,
 				 const cpp_char_column_policy &policy)
 {
   if (!(exploc.file && *exploc.file && exploc.line && exploc.column))
     return exploc.column;
-  char_span line = fc.get_source_line (exploc.file, exploc.line);
+  diagnostics::char_span line = fc.get_source_line (exploc.file, exploc.line);
   /* If line is NULL, this function returns exploc.column which is the
      desired fallback.  */
   return cpp_byte_column_to_display_column (line.get_buffer (), line.length (),
@@ -1431,7 +516,7 @@ dump_labelled_location_range (FILE *stream,
 void
 dump_location_info (FILE *stream)
 {
-  file_cache fc;
+  diagnostics::file_cache fc;
 
   /* Visualize the reserved locations.  */
   dump_labelled_location_range (stream, "RESERVED LOCATIONS",
@@ -1506,8 +591,8 @@ dump_location_info (FILE *stream)
 	    {
 	      /* Beginning of a new source line: draw the line.  */
 
-	      char_span line_text = fc.get_source_line (exploc.file,
-							exploc.line);
+	      diagnostics::char_span line_text
+		= fc.get_source_line (exploc.file, exploc.line);
 	      if (!line_text)
 		break;
 	      fprintf (stream,
@@ -1524,10 +609,10 @@ dump_location_info (FILE *stream)
 	      if (max_col > line_text.length ())
 		max_col = line_text.length () + 1;
 
-	      int len_lnum = num_digits (exploc.line);
+	      int len_lnum = diagnostics::num_digits (exploc.line);
 	      if (len_lnum < 3)
 		len_lnum = 3;
-	      int len_loc = num_digits (loc);
+	      int len_loc = diagnostics::num_digits (loc);
 	      if (len_loc < 5)
 		len_loc = 5;
 
@@ -1757,7 +842,7 @@ class auto_cpp_string_vec :  public auto_vec <cpp_string>
 
 static const char *
 get_substring_ranges_for_loc (cpp_reader *pfile,
-			      file_cache &fc,
+			      diagnostics::file_cache &fc,
 			      string_concat_db *concats,
 			      location_t strloc,
 			      enum cpp_ttype type,
@@ -1837,7 +922,7 @@ get_substring_ranges_for_loc (cpp_reader *pfile,
       if (start.column > finish.column)
 	return "range endpoints are reversed";
 
-      char_span line = fc.get_source_line (start.file, start.line);
+      diagnostics::char_span line = fc.get_source_line (start.file, start.line);
       if (!line)
 	return "unable to read source line";
 
@@ -1852,7 +937,8 @@ get_substring_ranges_for_loc (cpp_reader *pfile,
       if (line.length () < (start.column - 1 + literal_length))
 	return "line is not wide enough";
 
-      char_span literal = line.subspan (start.column - 1, literal_length);
+      diagnostics::char_span literal
+	= line.subspan (start.column - 1, literal_length);
 
       cpp_string from;
       from.len = literal_length;
@@ -1925,7 +1011,7 @@ get_substring_ranges_for_loc (cpp_reader *pfile,
 
 const char *
 get_location_within_string (cpp_reader *pfile,
-			    file_cache &fc,
+			    diagnostics::file_cache &fc,
 			    string_concat_db *concats,
 			    location_t strloc,
 			    enum cpp_ttype type,
@@ -2008,7 +1094,7 @@ namespace selftest {
 
 static const char *
 get_source_range_for_char (cpp_reader *pfile,
-			   file_cache &fc,
+			   diagnostics::file_cache &fc,
 			   string_concat_db *concats,
 			   location_t strloc,
 			   enum cpp_ttype type,
@@ -2036,7 +1122,7 @@ get_source_range_for_char (cpp_reader *pfile,
 
 static const char *
 get_num_source_ranges_for_substring (cpp_reader *pfile,
-				     file_cache &fc,
+				     diagnostics::file_cache &fc,
 				     string_concat_db *concats,
 				     location_t strloc,
 				     enum cpp_ttype type,
@@ -2359,119 +1445,6 @@ test_make_location_nonpure_range_endpoints (const line_table_case &case_)
   ASSERT_FALSE (IS_ADHOC_LOC (get_finish (not_aaa_eq_bbb)));
 }
 
-/* Verify reading of a specific line LINENUM in TMP, FC.  */
-
-static void
-check_line (temp_source_file &tmp, file_cache &fc, int linenum)
-{
-  char_span line = fc.get_source_line (tmp.get_filename (), linenum);
-  int n;
-  const char *b = line.get_buffer ();
-  size_t l = line.length ();
-  char buf[5];
-  ASSERT_LT (l, 5);
-  memcpy (buf, b, l);
-  buf[l] = '\0';
-  ASSERT_TRUE (sscanf (buf, "%d", &n) == 1);
-  ASSERT_EQ (n, linenum);
-}
-
-/* Test file cache replacement.  */
-
-static void
-test_replacement ()
-{
-  const int maxline = 1000;
-
-  char *vec = XNEWVEC (char, maxline * 5);
-  char *p = vec;
-  int i;
-  for (i = 1; i <= maxline; i++)
-    p += sprintf (p, "%d\n", i);
-
-  temp_source_file tmp (SELFTEST_LOCATION, ".txt", vec);
-  free (vec);
-  file_cache fc;
-
-  for (i = 2; i <= maxline; i++)
-    {
-      check_line (tmp, fc, i);
-      check_line (tmp, fc, i - 1);
-      if (i >= 10)
-	check_line (tmp, fc, i - 9);
-      if (i >= 350) /* Exceed the look behind cache.  */
-	check_line (tmp, fc, i - 300);
-    }
-  for (i = 5; i <= maxline; i += 100)
-    check_line (tmp, fc, i);
-  for (i = 1; i <= maxline; i++)
-    check_line (tmp, fc, i);
-}
-
-/* Verify reading of input files (e.g. for caret-based diagnostics).  */
-
-static void
-test_reading_source_line ()
-{
-  /* Create a tempfile and write some text to it.  */
-  temp_source_file tmp (SELFTEST_LOCATION, ".txt",
-			"01234567890123456789\n"
-			"This is the test text\n"
-			"This is the 3rd line");
-  file_cache fc;
-
-  /* Read back a specific line from the tempfile.  */
-  char_span source_line = fc.get_source_line (tmp.get_filename (), 3);
-  ASSERT_TRUE (source_line);
-  ASSERT_TRUE (source_line.get_buffer () != NULL);
-  ASSERT_EQ (20, source_line.length ());
-  ASSERT_TRUE (!strncmp ("This is the 3rd line",
-			 source_line.get_buffer (), source_line.length ()));
-
-  source_line = fc.get_source_line (tmp.get_filename (), 2);
-  ASSERT_TRUE (source_line);
-  ASSERT_TRUE (source_line.get_buffer () != NULL);
-  ASSERT_EQ (21, source_line.length ());
-  ASSERT_TRUE (!strncmp ("This is the test text",
-			 source_line.get_buffer (), source_line.length ()));
-
-  source_line = fc.get_source_line (tmp.get_filename (), 4);
-  ASSERT_FALSE (source_line);
-  ASSERT_TRUE (source_line.get_buffer () == NULL);
-}
-
-/* Verify reading from buffers (e.g. for sarif-replay).  */
-
-static void
-test_reading_source_buffer ()
-{
-  const char *text = ("01234567890123456789\n"
-		      "This is the test text\n"
-		      "This is the 3rd line");
-  const char *filename = "foo.txt";
-  file_cache fc;
-  fc.add_buffered_content (filename, text, strlen (text));
-
-  /* Read back a specific line from the tempfile.  */
-  char_span source_line = fc.get_source_line (filename, 3);
-  ASSERT_TRUE (source_line);
-  ASSERT_TRUE (source_line.get_buffer () != NULL);
-  ASSERT_EQ (20, source_line.length ());
-  ASSERT_TRUE (!strncmp ("This is the 3rd line",
-			 source_line.get_buffer (), source_line.length ()));
-
-  source_line = fc.get_source_line (filename, 2);
-  ASSERT_TRUE (source_line);
-  ASSERT_TRUE (source_line.get_buffer () != NULL);
-  ASSERT_EQ (21, source_line.length ());
-  ASSERT_TRUE (!strncmp ("This is the test text",
-			 source_line.get_buffer (), source_line.length ()));
-
-  source_line = fc.get_source_line (filename, 4);
-  ASSERT_FALSE (source_line);
-  ASSERT_TRUE (source_line.get_buffer () == NULL);
-}
-
 /* Tests of lexing.  */
 
 /* Verify that token TOK from PARSER has cpp_token_as_text
@@ -2628,7 +1601,7 @@ public:
   line_table_test m_ltt;
   cpp_reader_ptr m_parser;
   temp_source_file m_tempfile;
-  file_cache m_file_cache;
+  diagnostics::file_cache m_file_cache;
   string_concat_db m_concats;
   bool m_implicitly_expect_EOF;
 };
@@ -4356,10 +3329,6 @@ input_cc_tests ()
   for_each_line_table_case (test_lexer_string_locations_raw_string_unterminated);
   for_each_line_table_case (test_lexer_char_constants);
 
-  test_reading_source_line ();
-  test_reading_source_buffer ();
-  test_replacement ();
-
   test_line_offset_overflow ();
 
   test_cpp_utf8 ();
diff --git a/gcc/input.h b/gcc/input.h
index b0a1ca0..eeef290 100644
--- a/gcc/input.h
+++ b/gcc/input.h
@@ -23,7 +23,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "line-map.h"
 
-class file_cache;
+namespace diagnostics { class file_cache; }
 
 extern GTY(()) class line_maps *line_table;
 extern GTY(()) class line_maps *saved_line_table;
@@ -69,110 +69,12 @@ extern expanded_location expand_location (location_t);
 class cpp_char_column_policy;
 
 extern int
-location_compute_display_column (file_cache &fc,
+location_compute_display_column (diagnostics::file_cache &fc,
 				 expanded_location exploc,
 				 const cpp_char_column_policy &policy);
 
-/* A class capturing the bounds of a buffer, to allow for run-time
-   bounds-checking in a checked build.  */
-
-class char_span
-{
- public:
-  char_span (const char *ptr, size_t n_elts) : m_ptr (ptr), m_n_elts (n_elts) {}
-
-  /* Test for a non-NULL pointer.  */
-  operator bool() const { return m_ptr; }
-
-  /* Get length, not including any 0-terminator (which may not be,
-     in fact, present).  */
-  size_t length () const { return m_n_elts; }
-
-  const char *get_buffer () const { return m_ptr; }
-
-  char operator[] (int idx) const
-  {
-    gcc_assert (idx >= 0);
-    gcc_assert ((size_t)idx < m_n_elts);
-    return m_ptr[idx];
-  }
-
-  char_span subspan (int offset, int n_elts) const
-  {
-    gcc_assert (offset >= 0);
-    gcc_assert (offset < (int)m_n_elts);
-    gcc_assert (n_elts >= 0);
-    gcc_assert (offset + n_elts <= (int)m_n_elts);
-    return char_span (m_ptr + offset, n_elts);
-  }
-
-  char *xstrdup () const
-  {
-    return ::xstrndup (m_ptr, m_n_elts);
-  }
-
- private:
-  const char *m_ptr;
-  size_t m_n_elts;
-};
-
 extern char *
-get_source_text_between (file_cache &, location_t, location_t);
-
-/* Forward decl of slot within file_cache, so that the definition doesn't
-   need to be in this header.  */
-class file_cache_slot;
-
-/* A cache of source files for use when emitting diagnostics
-   (and in a few places in the C/C++ frontends).
-
-   Results are only valid until the next call to the cache, as
-   slots can be evicted.
-
-   Filenames are stored by pointer, and so must outlive the cache
-   instance.  */
-
-class file_cache
-{
- public:
-  file_cache ();
-  ~file_cache ();
-
-  void dump (FILE *out, int indent) const;
-  void DEBUG_FUNCTION dump () const;
-
-  file_cache_slot *lookup_or_add_file (const char *file_path);
-  void forcibly_evict_file (const char *file_path);
-
-  /* See comments in diagnostic.h about the input conversion context.  */
-  struct input_context
-  {
-    diagnostic_input_charset_callback ccb;
-    bool should_skip_bom;
-  };
-  void initialize_input_context (diagnostic_input_charset_callback ccb,
-				 bool should_skip_bom);
-
-  char_span get_source_file_content (const char *file_path);
-  char_span get_source_line (const char *file_path, int line);
-  bool missing_trailing_newline_p (const char *file_path);
-
-  void add_buffered_content (const char *file_path,
-			     const char *buffer,
-			     size_t sz);
-
-  void tune (size_t num_file_slots, size_t lines);
-
- private:
-  file_cache_slot *evicted_cache_tab_entry (unsigned *highest_use_count);
-  file_cache_slot *add_file (const char *file_path);
-  file_cache_slot *lookup_file (const char *file_path);
-
- private:
-  size_t m_num_file_slots;
-  file_cache_slot *m_file_slots;
-  input_context m_input_context;
-};
+get_source_text_between (diagnostics::file_cache &, location_t, location_t);
 
 extern expanded_location
 expand_location_to_spelling_point (location_t,
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index 3f4ac93..bf2fac8 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -3442,25 +3442,23 @@ expand_DEFERRED_INIT (internal_fn, gcall *stmt)
 }
 
 /* Expand the IFN_ACCESS_WITH_SIZE function:
-   ACCESS_WITH_SIZE (REF_TO_OBJ, REF_TO_SIZE, CLASS_OF_SIZE,
-		     TYPE_OF_SIZE, ACCESS_MODE)
+   ACCESS_WITH_SIZE (REF_TO_OBJ, REF_TO_SIZE,
+		     TYPE_OF_SIZE + ACCESS_MODE, TYPE_SIZE_UNIT for element)
    which returns the REF_TO_OBJ same as the 1st argument;
 
    1st argument REF_TO_OBJ: The reference to the object;
    2nd argument REF_TO_SIZE: The reference to the size of the object,
-   3rd argument CLASS_OF_SIZE: The size referenced by the REF_TO_SIZE represents
-     0: the number of bytes.
-     1: the number of the elements of the object type;
-   4th argument TYPE_OF_SIZE: A constant 0 with its TYPE being the same as the TYPE
-    of the object referenced by REF_TO_SIZE
-   5th argument ACCESS_MODE:
-    -1: Unknown access semantics
-     0: none
-     1: read_only
-     2: write_only
-     3: read_write
-   6th argument: A constant 0 with the pointer TYPE to the original flexible
-     array type.
+   3rd argument TYPE_OF_SIZE + ACCESS_MODE: An integer constant with a pointer
+     TYPE.
+     The pointee TYPE of the pointer TYPE is the TYPE of the object referenced
+	by REF_TO_SIZE.
+     The integer constant value represents the ACCESS_MODE:
+	0: none
+	1: read_only
+	2: write_only
+	3: read_write
+
+   4th argument: The TYPE_SIZE_UNIT of the element TYPE of the array.
 
    Both the return type and the type of the first argument of this
    function have been converted from the incomplete array type to
@@ -3654,8 +3652,8 @@ expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
   internal_fn ifn = gimple_call_internal_fn (stmt);
   int rhs_index = internal_fn_stored_value_index (ifn);
   tree base = gimple_call_arg (stmt, 0);
-  tree offset = gimple_call_arg (stmt, 1);
-  tree scale = gimple_call_arg (stmt, 2);
+  tree offset = gimple_call_arg (stmt, internal_fn_offset_index (ifn));
+  tree scale = gimple_call_arg (stmt, internal_fn_scale_index (ifn));
   tree rhs = gimple_call_arg (stmt, rhs_index);
 
   rtx base_rtx = expand_normal (base);
@@ -3680,12 +3678,12 @@ expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB.  */
 
 static void
-expand_gather_load_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
+expand_gather_load_optab_fn (internal_fn ifn, gcall *stmt, direct_optab optab)
 {
   tree lhs = gimple_call_lhs (stmt);
   tree base = gimple_call_arg (stmt, 0);
-  tree offset = gimple_call_arg (stmt, 1);
-  tree scale = gimple_call_arg (stmt, 2);
+  tree offset = gimple_call_arg (stmt, internal_fn_offset_index (ifn));
+  tree scale = gimple_call_arg (stmt, internal_fn_scale_index (ifn));
 
   rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
   rtx base_rtx = expand_normal (base);
@@ -4031,9 +4029,14 @@ expand_crc_optab_fn (internal_fn fn, gcall *stmt, convert_optab optab)
   rtx dest = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
   rtx crc = expand_normal (rhs1);
   rtx data = expand_normal (rhs2);
-  gcc_assert (TREE_CODE (rhs3) == INTEGER_CST);
-  rtx polynomial = gen_rtx_CONST_INT (TYPE_MODE (result_type),
-				      TREE_INT_CST_LOW (rhs3));
+  rtx polynomial;
+  if (TREE_CODE (rhs3) != INTEGER_CST)
+    {
+      error ("third argument to %<crc%> builtins must be a constant");
+      polynomial = const0_rtx;
+    }
+  else
+    polynomial = convert_to_mode (TYPE_MODE (result_type), expand_normal (rhs3), 0);
 
   /* Use target specific expansion if it exists.
      Otherwise, generate table-based CRC.  */
@@ -4423,6 +4426,7 @@ commutative_binary_fn_p (internal_fn fn)
     case IFN_ADD_OVERFLOW:
     case IFN_MUL_OVERFLOW:
     case IFN_SAT_ADD:
+    case IFN_SAT_MUL:
     case IFN_VEC_WIDEN_PLUS:
     case IFN_VEC_WIDEN_PLUS_LO:
     case IFN_VEC_WIDEN_PLUS_HI:
@@ -4544,6 +4548,33 @@ widening_fn_p (code_helper code)
     }
 }
 
+/* Return true if this CODE describes an internal_fn that returns a vector with
+   elements twice as wide as the element size of the input vectors and operates
+   on even/odd parts of the input.  */
+
+bool
+widening_evenodd_fn_p (code_helper code)
+{
+  if (!code.is_fn_code ())
+    return false;
+
+  if (!internal_fn_p ((combined_fn) code))
+    return false;
+
+  internal_fn fn = as_internal_fn ((combined_fn) code);
+  switch (fn)
+    {
+    #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
+    case IFN_##NAME##_EVEN:					  \
+    case IFN_##NAME##_ODD:					  \
+      return true;
+    #include "internal-fn.def"
+
+    default:
+      return false;
+    }
+}
+
 /* Return true if IFN_SET_EDOM is supported.  */
 
 bool
@@ -4936,11 +4967,13 @@ internal_fn_len_index (internal_fn fn)
       return 2;
 
     case IFN_MASK_LEN_SCATTER_STORE:
+      return 6;
+
     case IFN_MASK_LEN_STRIDED_LOAD:
       return 5;
 
     case IFN_MASK_LEN_GATHER_LOAD:
-      return 6;
+      return 7;
 
     case IFN_COND_LEN_FMA:
     case IFN_COND_LEN_FMS:
@@ -5044,7 +5077,7 @@ internal_fn_else_index (internal_fn fn)
 
     case IFN_MASK_GATHER_LOAD:
     case IFN_MASK_LEN_GATHER_LOAD:
-      return 5;
+      return 6;
 
     default:
       return -1;
@@ -5079,7 +5112,7 @@ internal_fn_mask_index (internal_fn fn)
     case IFN_MASK_SCATTER_STORE:
     case IFN_MASK_LEN_GATHER_LOAD:
     case IFN_MASK_LEN_SCATTER_STORE:
-      return 4;
+      return 5;
 
     case IFN_VCOND_MASK:
     case IFN_VCOND_MASK_LEN:
@@ -5104,10 +5137,11 @@ internal_fn_stored_value_index (internal_fn fn)
 
     case IFN_MASK_STORE:
     case IFN_MASK_STORE_LANES:
+      return 3;
     case IFN_SCATTER_STORE:
     case IFN_MASK_SCATTER_STORE:
     case IFN_MASK_LEN_SCATTER_STORE:
-      return 3;
+      return 4;
 
     case IFN_LEN_STORE:
       return 4;
@@ -5121,6 +5155,75 @@ internal_fn_stored_value_index (internal_fn fn)
     }
 }
 
+/* If FN has an alias pointer return its index, otherwise return -1.  */
+
+int
+internal_fn_alias_ptr_index (internal_fn fn)
+{
+  switch (fn)
+    {
+    case IFN_MASK_LOAD:
+    case IFN_MASK_LEN_LOAD:
+    case IFN_GATHER_LOAD:
+    case IFN_MASK_GATHER_LOAD:
+    case IFN_MASK_LEN_GATHER_LOAD:
+    case IFN_SCATTER_STORE:
+    case IFN_MASK_SCATTER_STORE:
+    case IFN_MASK_LEN_SCATTER_STORE:
+      return 1;
+
+    default:
+      return -1;
+    }
+}
+
+/* If FN is a gather/scatter return the index of its offset argument,
+   otherwise return -1.  */
+
+int
+internal_fn_offset_index (internal_fn fn)
+{
+  if (!internal_gather_scatter_fn_p (fn))
+    return -1;
+
+  switch (fn)
+    {
+    case IFN_GATHER_LOAD:
+    case IFN_MASK_GATHER_LOAD:
+    case IFN_MASK_LEN_GATHER_LOAD:
+    case IFN_SCATTER_STORE:
+    case IFN_MASK_SCATTER_STORE:
+    case IFN_MASK_LEN_SCATTER_STORE:
+      return 2;
+
+    default:
+      return -1;
+    }
+}
+
+/* If FN is a gather/scatter return the index of its scale argument,
+   otherwise return -1.  */
+
+int
+internal_fn_scale_index (internal_fn fn)
+{
+  if (!internal_gather_scatter_fn_p (fn))
+    return -1;
+
+  switch (fn)
+    {
+    case IFN_GATHER_LOAD:
+    case IFN_MASK_GATHER_LOAD:
+    case IFN_MASK_LEN_GATHER_LOAD:
+    case IFN_SCATTER_STORE:
+    case IFN_MASK_SCATTER_STORE:
+    case IFN_MASK_LEN_SCATTER_STORE:
+      return 3;
+
+    default:
+      return -1;
+    }
+}
 
 /* Store all supported else values for the optab referred to by ICODE
    in ELSE_VALS.  The index of the else operand must be specified in
@@ -5199,13 +5302,9 @@ internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
     && insn_operand_matches (icode, 2 + output_ops, GEN_INT (unsigned_p))
     && insn_operand_matches (icode, 3 + output_ops, GEN_INT (scale));
 
-  /* For gather the optab's operand indices do not match the IFN's because
-     the latter does not have the extension operand (operand 3).  It is
-     implicitly added during expansion so we use the IFN's else index + 1.
-     */
   if (ok && elsvals)
     get_supported_else_vals
-      (icode, internal_fn_else_index (IFN_MASK_GATHER_LOAD) + 1, *elsvals);
+      (icode, internal_fn_else_index (IFN_MASK_GATHER_LOAD), *elsvals);
 
   return ok;
 }
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index 8edfa35..d2480a1 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -282,6 +282,7 @@ DEF_INTERNAL_SIGNED_OPTAB_FN (MULHRS, ECF_CONST | ECF_NOTHROW, first,
 
 DEF_INTERNAL_SIGNED_OPTAB_FN (SAT_ADD, ECF_CONST, first, ssadd, usadd, binary)
 DEF_INTERNAL_SIGNED_OPTAB_FN (SAT_SUB, ECF_CONST, first, sssub, ussub, binary)
+DEF_INTERNAL_SIGNED_OPTAB_FN (SAT_MUL, ECF_CONST, first, ssmul, usmul, binary)
 
 DEF_INTERNAL_SIGNED_OPTAB_FN (SAT_TRUNC, ECF_CONST, first, sstrunc, ustrunc, unary_convert)
 
@@ -524,7 +525,7 @@ DEF_INTERNAL_FN (DEFERRED_INIT, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
 
 /* A function to associate the access size and access mode information
    with the corresponding reference to an object.  It only reads from the
-   2nd argument.  */
+   2nd and the 4th arguments.  */
 DEF_INTERNAL_FN (ACCESS_WITH_SIZE, ECF_PURE | ECF_LEAF | ECF_NOTHROW, NULL)
 
 /* DIM_SIZE and DIM_POS return the size of a particular compute
diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h
index afd4f8e..fd21694 100644
--- a/gcc/internal-fn.h
+++ b/gcc/internal-fn.h
@@ -219,6 +219,7 @@ extern bool commutative_ternary_fn_p (internal_fn);
 extern int first_commutative_argument (internal_fn);
 extern bool associative_binary_fn_p (internal_fn);
 extern bool widening_fn_p (code_helper);
+extern bool widening_evenodd_fn_p (code_helper);
 
 extern bool set_edom_supported_p (void);
 
@@ -239,6 +240,9 @@ extern int internal_fn_mask_index (internal_fn);
 extern int internal_fn_len_index (internal_fn);
 extern int internal_fn_else_index (internal_fn);
 extern int internal_fn_stored_value_index (internal_fn);
+extern int internal_fn_offset_index (internal_fn fn);
+extern int internal_fn_scale_index (internal_fn fn);
+extern int internal_fn_alias_ptr_index (internal_fn fn);
 extern bool internal_gather_scatter_fn_supported_p (internal_fn, tree,
 						    tree, tree, int,
 						    vec<int> * = nullptr);
diff --git a/gcc/ipa-devirt.cc b/gcc/ipa-devirt.cc
index 532e25e..18cb5a8 100644
--- a/gcc/ipa-devirt.cc
+++ b/gcc/ipa-devirt.cc
@@ -1763,7 +1763,7 @@ add_type_duplicate (odr_type val, tree type)
 	      }
 	    /* One base is polymorphic and the other not.
 	       This ought to be diagnosed earlier, but do not ICE in the
-	       checking bellow.  */
+	       checking below.  */
 	    else if (TYPE_BINFO (type1)
 		     && polymorphic_type_binfo_p (TYPE_BINFO (type1))
 		        != polymorphic_type_binfo_p (TYPE_BINFO (type2)))
diff --git a/gcc/ipa-fnsummary.cc b/gcc/ipa-fnsummary.cc
index 48343a7..924a54b 100644
--- a/gcc/ipa-fnsummary.cc
+++ b/gcc/ipa-fnsummary.cc
@@ -3421,6 +3421,21 @@ compute_fn_summary (struct cgraph_node *node, bool early)
 	 info->inlinable = tree_inlinable_function_p (node->decl);
 
        bool no_signature = false;
+
+       /* Don't allow signature changes for functions which have
+	  [[gnu::musttail]] or [[clang::musttail]] calls.  Sometimes
+	  (more often on targets which pass everything on the stack)
+	  signature changes can result in tail calls being impossible
+	  even when without the signature changes they would be ok.
+	  See PR121023.  */
+       if (cfun->has_musttail)
+	 {
+	   if (dump_file)
+	    fprintf (dump_file, "No signature change:"
+		     " function has calls with musttail attribute.\n");
+	   no_signature = true;
+	 }
+
        /* Type attributes can use parameter indices to describe them.
 	  Special case fn spec since we can safely preserve them in
 	  modref summaries.  */
diff --git a/gcc/ipa-inline-transform.cc b/gcc/ipa-inline-transform.cc
index 07a1024..9d759d2 100644
--- a/gcc/ipa-inline-transform.cc
+++ b/gcc/ipa-inline-transform.cc
@@ -331,7 +331,7 @@ inline_call (struct cgraph_edge *e, bool update_original,
   int estimated_growth = 0;
   if (! update_overall_summary)
     estimated_growth = estimate_edge_growth (e);
-  /* This is used only for assert bellow.  */
+  /* This is used only for assert below.  */
 #if 0
   bool predicated = inline_edge_summary (e)->predicate != NULL;
 #endif
diff --git a/gcc/ipa-inline.cc b/gcc/ipa-inline.cc
index ca605b0..0cf97a80 100644
--- a/gcc/ipa-inline.cc
+++ b/gcc/ipa-inline.cc
@@ -2222,6 +2222,7 @@ inline_small_functions (void)
 
   gcc_assert (in_lto_p
 	      || !(max_count > 0)
+	      || flag_auto_profile
 	      || (profile_info && flag_branch_probabilities));
 
   while (!edge_heap.empty ())
diff --git a/gcc/ipa-polymorphic-call.cc b/gcc/ipa-polymorphic-call.cc
index 7b105f1..09316de 100644
--- a/gcc/ipa-polymorphic-call.cc
+++ b/gcc/ipa-polymorphic-call.cc
@@ -1353,7 +1353,7 @@ record_known_type (struct type_change_info *tci, tree type, HOST_WIDE_INT offset
 
   /* If we found a constructor of type that is not polymorphic or
      that may contain the type in question as a field (not as base),
-     restrict to the inner class first to make type matching bellow
+     restrict to the inner class first to make type matching below
      happier.  */
   if (type
       && (offset
diff --git a/gcc/ipa-pure-const.cc b/gcc/ipa-pure-const.cc
index 93297ed..2519fb8 100644
--- a/gcc/ipa-pure-const.cc
+++ b/gcc/ipa-pure-const.cc
@@ -200,7 +200,7 @@ function_always_visible_to_compiler_p (tree decl)
    by the function.  */
 
 static hash_set<tree> *
-suggest_attribute (diagnostic_option_id option, tree decl, bool known_finite,
+suggest_attribute (diagnostics::option_id option, tree decl, bool known_finite,
 		   hash_set<tree> *warned_about,
 		   const char * attrib_name)
 {
diff --git a/gcc/ipa-reference.cc b/gcc/ipa-reference.cc
index 2d8e62f..975341c 100644
--- a/gcc/ipa-reference.cc
+++ b/gcc/ipa-reference.cc
@@ -732,7 +732,7 @@ get_read_write_all_from_node (struct cgraph_node *node,
 /* Skip edges from and to nodes without ipa_reference enabled.
    Ignore not available symbols.  This leave
    them out of strongly connected components and makes them easy to skip in the
-   propagation loop bellow.  */
+   propagation loop below.  */
 
 static bool
 ignore_edge_p (cgraph_edge *e)
diff --git a/gcc/ipa-strub.cc b/gcc/ipa-strub.cc
index d7c0a92..86270af 100644
--- a/gcc/ipa-strub.cc
+++ b/gcc/ipa-strub.cc
@@ -3059,6 +3059,8 @@ pass_ipa_strub::execute (function *)
 			 TYPE_ATTRIBUTES (TREE_TYPE (nnode->decl)));
 	}
     }
+#else
+    (void) named_args;
 #endif
 
     {
diff --git a/gcc/ira.cc b/gcc/ira.cc
index 979a034..4eebc9c 100644
--- a/gcc/ira.cc
+++ b/gcc/ira.cc
@@ -2113,6 +2113,87 @@ ira_get_dup_out_num (int op_num, alternative_mask alts,
 
 
 
+/* Return true if a replacement of SRC by DEST does not lead to unsatisfiable
+   asm.  A replacement is valid if SRC or DEST are not constrained in asm
+   inputs of a single asm statement.  See match_asm_constraints_2() for more
+   details.  TODO: As in match_asm_constraints_2() consider alternatives more
+   precisely.  */
+
+static bool
+valid_replacement_for_asm_input_p_1 (const_rtx asmops, const_rtx src, const_rtx dest)
+{
+  int ninputs = ASM_OPERANDS_INPUT_LENGTH (asmops);
+  rtvec inputs = ASM_OPERANDS_INPUT_VEC (asmops);
+  for (int i = 0; i < ninputs; ++i)
+    {
+      rtx input_src = RTVEC_ELT (inputs, i);
+      const char *constraint_src
+	= ASM_OPERANDS_INPUT_CONSTRAINT (asmops, i);
+      if (rtx_equal_p (input_src, src)
+	  && strchr (constraint_src, '{') != nullptr)
+	for (int j = 0; j < ninputs; ++j)
+	  {
+	    rtx input_dest = RTVEC_ELT (inputs, j);
+	    const char *constraint_dest
+	      = ASM_OPERANDS_INPUT_CONSTRAINT (asmops, j);
+	    if (rtx_equal_p (input_dest, dest)
+		&& strchr (constraint_dest, '{') != nullptr)
+	      return false;
+	  }
+    }
+  return true;
+}
+
+/* Return true if a replacement of SRC by DEST does not lead to unsatisfiable
+   asm.  A replacement is valid if SRC or DEST are not constrained in asm
+   inputs of a single asm statement.  The final check is done in function
+   valid_replacement_for_asm_input_p_1.  */
+
+static bool
+valid_replacement_for_asm_input_p (const_rtx src, const_rtx dest)
+{
+  /* Bail out early if there is no asm statement.  */
+  if (!crtl->has_asm_statement)
+    return true;
+  for (df_ref use = DF_REG_USE_CHAIN (REGNO (src));
+       use;
+       use = DF_REF_NEXT_REG (use))
+    {
+      struct df_insn_info *use_info = DF_REF_INSN_INFO (use);
+      /* Only check real uses, not artificial ones.  */
+      if (use_info)
+	{
+	  rtx_insn *insn = DF_REF_INSN (use);
+	  rtx pat = PATTERN (insn);
+	  if (asm_noperands (pat) <= 0)
+	    continue;
+	  if (GET_CODE (pat) == SET)
+	    {
+	      if (!valid_replacement_for_asm_input_p_1 (SET_SRC (pat), src, dest))
+		return false;
+	    }
+	  else if (GET_CODE (pat) == PARALLEL)
+	    for (int i = 0, len = XVECLEN (pat, 0); i < len; ++i)
+	      {
+		rtx asmops = XVECEXP (pat, 0, i);
+		if (GET_CODE (asmops) == SET)
+		  asmops = SET_SRC (asmops);
+		if (GET_CODE (asmops) == ASM_OPERANDS
+		    && !valid_replacement_for_asm_input_p_1 (asmops, src, dest))
+		  return false;
+	      }
+	  else if (GET_CODE (pat) == ASM_OPERANDS)
+	    {
+	      if (!valid_replacement_for_asm_input_p_1 (pat, src, dest))
+		return false;
+	    }
+	  else
+	    gcc_unreachable ();
+	}
+    }
+  return true;
+}
+
 /* Search forward to see if the source register of a copy insn dies
    before either it or the destination register is modified, but don't
    scan past the end of the basic block.  If so, we can replace the
@@ -2162,7 +2243,8 @@ decrease_live_ranges_number (void)
 	       auto-inc memory reference, so we must disallow this
 	       optimization on them.  */
 	    || sregno == STACK_POINTER_REGNUM
-	    || dregno == STACK_POINTER_REGNUM)
+	    || dregno == STACK_POINTER_REGNUM
+	    || !valid_replacement_for_asm_input_p (src, dest))
 	  continue;
 
 	dest_death = NULL_RTX;
diff --git a/gcc/jit/ChangeLog b/gcc/jit/ChangeLog
index adee3de..da88cc6 100644
--- a/gcc/jit/ChangeLog
+++ b/gcc/jit/ChangeLog
@@ -1,3 +1,30 @@
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* dummy-frontend.cc: Update usage of "diagnostic_info" to
+	explicitly refer to "diagnostics::diagnostic_info".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* dummy-frontend.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* dummy-frontend.cc: Update for diagnostic_context becoming
+	diagnostics::context.
+	* jit-playback.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* jit-playback.cc: Update for diagnostic_info moving into
+	namespace diagnostics.
+	* jit-playback.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* dummy-frontend.cc: Update for move of diagnostics output formats
+	into namespace "diagnostics" as "sinks".
+
 2025-04-28  David Malcolm  <dmalcolm@redhat.com>
 
 	* dummy-frontend.cc: Drop include of "make-unique.h".
diff --git a/gcc/jit/dummy-frontend.cc b/gcc/jit/dummy-frontend.cc
index bf31a9d..3ffca92 100644
--- a/gcc/jit/dummy-frontend.cc
+++ b/gcc/jit/dummy-frontend.cc
@@ -32,7 +32,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "attribs.h"
 #include "cgraph.h"
 #include "target.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 #include "print-tree.h"
 
 #include <mpfr.h>
@@ -993,16 +993,16 @@ struct ggc_root_tab jit_root_tab[] =
     LAST_GGC_ROOT_TAB
   };
 
-/* Subclass of diagnostic_output_format for libgccjit: like text
+/* Subclass of diagnostics::text_sink for libgccjit: like text
    output, but capture the message and call add_diagnostic with it
    on the active playback context.  */
 
-class jit_diagnostic_listener : public diagnostic_text_output_format
+class jit_diagnostic_listener : public diagnostics::text_sink
 {
 public:
-  jit_diagnostic_listener (diagnostic_context &dc,
+  jit_diagnostic_listener (diagnostics::context &dc,
 			   gcc::jit::playback::context &playback_ctxt)
-  : diagnostic_text_output_format (dc),
+  : diagnostics::text_sink (dc),
     m_playback_ctxt (playback_ctxt)
   {
   }
@@ -1015,13 +1015,13 @@ public:
 	     (void *)&m_playback_ctxt);
   }
 
-  void on_report_diagnostic (const diagnostic_info &info,
-			     diagnostic_t orig_diag_kind) final override
+  void on_report_diagnostic (const diagnostics::diagnostic_info &info,
+			     enum diagnostics::kind orig_diag_kind) final override
   {
     JIT_LOG_SCOPE (gcc::jit::active_playback_ctxt->get_logger ());
 
     /* Let the text output format do most of the work.  */
-    diagnostic_text_output_format::on_report_diagnostic (info, orig_diag_kind);
+    diagnostics::text_sink::on_report_diagnostic (info, orig_diag_kind);
 
     const char *text = pp_formatted_text (get_printer ());
 
@@ -1041,8 +1041,8 @@ private:
 /* Implementation of "begin_diagnostic".  */
 
 static void
-jit_begin_diagnostic (diagnostic_text_output_format &,
-		      const diagnostic_info */*diagnostic*/)
+jit_begin_diagnostic (diagnostics::text_sink &,
+		      const diagnostics::diagnostic_info */*diagnostic*/)
 {
   gcc_assert (gcc::jit::active_playback_ctxt);
   JIT_LOG_SCOPE (gcc::jit::active_playback_ctxt->get_logger ());
@@ -1054,9 +1054,9 @@ jit_begin_diagnostic (diagnostic_text_output_format &,
 /* Implementation of "end_diagnostic".  */
 
 static void
-jit_end_diagnostic (diagnostic_text_output_format &,
-		    const diagnostic_info *,
-		    diagnostic_t)
+jit_end_diagnostic (diagnostics::text_sink &,
+		    const diagnostics::diagnostic_info *,
+		    enum diagnostics::kind)
 {
   gcc_assert (gcc::jit::active_playback_ctxt);
   JIT_LOG_SCOPE (gcc::jit::active_playback_ctxt->get_logger ());
@@ -1081,13 +1081,13 @@ jit_langhook_init (void)
     }
 
   gcc_assert (global_dc);
-  diagnostic_text_starter (global_dc) = jit_begin_diagnostic;
-  diagnostic_text_finalizer (global_dc) = jit_end_diagnostic;
+  diagnostics::text_starter (global_dc) = jit_begin_diagnostic;
+  diagnostics::text_finalizer (global_dc) = jit_end_diagnostic;
   auto sink
     = std::make_unique<jit_diagnostic_listener>
 	(*global_dc,
 	 *gcc::jit::active_playback_ctxt);
-  global_dc->set_output_format (std::move (sink));
+  global_dc->set_sink (std::move (sink));
 
   build_common_tree_nodes (flag_signed_char);
 
diff --git a/gcc/jit/jit-playback.cc b/gcc/jit/jit-playback.cc
index 6946f10..4473d8b 100644
--- a/gcc/jit/jit-playback.cc
+++ b/gcc/jit/jit-playback.cc
@@ -3892,7 +3892,7 @@ add_error_va (location *loc, const char *fmt, va_list ap)
 void
 playback::context::
 add_diagnostic (const char *text,
-		const diagnostic_info &diagnostic)
+		const diagnostics::diagnostic_info &diagnostic)
 {
   /* Get location information (if any) from the diagnostic.
      The recording::context::add_error[_va] methods require a
diff --git a/gcc/jit/jit-playback.h b/gcc/jit/jit-playback.h
index e9a5c38..b0625dc 100644
--- a/gcc/jit/jit-playback.h
+++ b/gcc/jit/jit-playback.h
@@ -30,8 +30,11 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "jit-recording.h"
 
-class diagnostic_context;
-struct diagnostic_info;
+namespace diagnostics
+{
+  class context;
+  struct diagnostic_info;
+}
 
 namespace gcc {
 
@@ -298,7 +301,7 @@ public:
 
   void
   add_diagnostic (const char *text,
-		  const diagnostic_info &diagnostic);
+		  const diagnostics::diagnostic_info &diagnostic);
 
   void
   set_tree_location (tree t, location *loc);
diff --git a/gcc/json.cc b/gcc/json.cc
index f3f3645..7153f08 100644
--- a/gcc/json.cc
+++ b/gcc/json.cc
@@ -289,6 +289,30 @@ object::print (pretty_printer *pp, bool formatted) const
   pp_character (pp, '}');
 }
 
+std::unique_ptr<value>
+object::clone () const
+{
+  return clone_as_object ();
+}
+
+std::unique_ptr<object>
+object::clone_as_object () const
+{
+  auto result = std::make_unique<object> ();
+
+  /* Iterate in the order that the keys were inserted.  */
+  unsigned i;
+  const char *key;
+  FOR_EACH_VEC_ELT (m_keys, i, key)
+    {
+      map_t &mut_map = const_cast<map_t &> (m_map);
+      value *value = *mut_map.get (key);
+      result->set (key, value->clone ());
+    }
+
+  return result;
+}
+
 /* Set the json::value * for KEY, taking ownership of V
    (and taking a copy of KEY if necessary).  */
 
@@ -443,6 +467,17 @@ array::print (pretty_printer *pp, bool formatted) const
   pp_character (pp, ']');
 }
 
+std::unique_ptr<value>
+array::clone () const
+{
+  auto result = std::make_unique<array> ();
+  unsigned i;
+  value *v;
+  FOR_EACH_VEC_ELT (m_elements, i, v)
+    result->append (v->clone ());
+  return result;
+}
+
 /* Append non-NULL value V to a json::array, taking ownership of V.  */
 
 void
@@ -473,6 +508,12 @@ float_number::print (pretty_printer *pp,
   pp_string (pp, tmp);
 }
 
+std::unique_ptr<value>
+float_number::clone () const
+{
+  return std::make_unique<float_number> (m_value);
+}
+
 /* class json::integer_number, a subclass of json::value, wrapping a long.  */
 
 /* Implementation of json::value::print for json::integer_number.  */
@@ -486,6 +527,11 @@ integer_number::print (pretty_printer *pp,
   pp_string (pp, tmp);
 }
 
+std::unique_ptr<value>
+integer_number::clone () const
+{
+  return std::make_unique<integer_number> (m_value);
+}
 
 /* class json::string, a subclass of json::value.  */
 
@@ -501,9 +547,10 @@ string::string (const char *utf8)
 string::string (const char *utf8, size_t len)
 {
   gcc_assert (utf8);
-  m_utf8 = XNEWVEC (char, len);
+  m_utf8 = XNEWVEC (char, len + 1);
   m_len = len;
   memcpy (m_utf8, utf8, len);
+  m_utf8[len] = '\0';
 }
 
 /* Implementation of json::value::print for json::string.  */
@@ -515,6 +562,12 @@ string::print (pretty_printer *pp,
   print_escaped_json_string (pp, m_utf8, m_len);
 }
 
+std::unique_ptr<value>
+string::clone () const
+{
+  return std::make_unique<string> (m_utf8, m_len);
+}
+
 /* class json::literal, a subclass of json::value.  */
 
 /* Implementation of json::value::print for json::literal.  */
@@ -539,6 +592,12 @@ literal::print (pretty_printer *pp,
     }
 }
 
+std::unique_ptr<value>
+literal::clone () const
+{
+  return std::make_unique<literal> (m_kind);
+}
+
 
 #if CHECKING_P
 
@@ -914,6 +973,65 @@ test_comparisons ()
   ASSERT_JSON_NE (arr_1, arr_2);
 }
 
+/* Ensure that json::string's get_string is usable as a C-style string.  */
+
+static void
+test_strcmp ()
+{
+  string str ("foobar", 3);
+  ASSERT_EQ (strcmp (str.get_string (), "foo"), 0);
+}
+
+static void
+test_cloning ()
+{
+  // Objects
+  {
+    object obj;
+    obj.set_string ("foo", "bar");
+
+    auto obj_clone = obj.clone ();
+    ASSERT_JSON_EQ (obj, *obj_clone);
+  }
+
+  // Arrays
+  {
+    array arr;
+    arr.append (std::make_unique<string> ("foo"));
+
+    auto arr_clone = arr.clone ();
+    ASSERT_JSON_EQ (arr, *arr_clone);
+  }
+
+  // float_number
+  {
+    float_number f_one (1.0);
+    auto f_clone = f_one.clone ();
+    ASSERT_JSON_EQ (f_one, *f_clone);
+  }
+
+  // integer_number
+  {
+    integer_number num (42);
+    auto num_clone = num.clone ();
+    ASSERT_JSON_EQ (num, *num_clone);
+  }
+
+  // string
+  {
+    string str ("foo");
+    auto str_clone = str.clone ();
+    ASSERT_JSON_EQ (str, *str_clone);
+  }
+
+  // literal
+  {
+    literal lit (JSON_TRUE);
+    auto lit_clone = lit.clone ();
+    ASSERT_JSON_EQ (lit, *lit_clone);
+  }
+}
+
 /* Run all of the selftests within this file.  */
 
 void
@@ -928,6 +1046,8 @@ json_cc_tests ()
   test_writing_literals ();
   test_formatting ();
   test_comparisons ();
+  test_strcmp ();
+  test_cloning ();
 }
 
 } // namespace selftest
diff --git a/gcc/json.h b/gcc/json.h
index da4da85..c706f2a 100644
--- a/gcc/json.h
+++ b/gcc/json.h
@@ -124,11 +124,13 @@ class value
   virtual ~value () {}
   virtual enum kind get_kind () const = 0;
   virtual void print (pretty_printer *pp, bool formatted) const = 0;
+  virtual std::unique_ptr<value> clone () const = 0;
 
   void dump (FILE *, bool formatted) const;
   void DEBUG_FUNCTION dump () const;
 
   virtual object *dyn_cast_object () { return nullptr; }
+  virtual string *dyn_cast_string () { return nullptr; }
 
   static int compare (const json::value &val_a, const json::value &val_b);
 
@@ -150,6 +152,7 @@ class object : public value
 
   enum kind get_kind () const final override { return JSON_OBJECT; }
   void print (pretty_printer *pp, bool formatted) const final override;
+  std::unique_ptr<value> clone () const final override;
 
   object *dyn_cast_object () final override { return this; }
 
@@ -182,6 +185,11 @@ class object : public value
 
   static int compare (const json::object &obj_a, const json::object &obj_b);
 
+  size_t get_num_keys () const { return m_keys.length (); }
+  const char *get_key (size_t i) const { return m_keys[i]; }
+
+  std::unique_ptr<object> clone_as_object () const;
+
  private:
   typedef hash_map <char *, value *,
     simple_hashmap_traits<nofree_string_hash, value *> > map_t;
@@ -200,6 +208,7 @@ class array : public value
 
   enum kind get_kind () const final override { return JSON_ARRAY; }
   void print (pretty_printer *pp, bool formatted) const final override;
+  std::unique_ptr<value> clone () const final override;
 
   void append (value *v);
   void append_string (const char *utf8_value);
@@ -241,6 +250,7 @@ class float_number : public value
 
   enum kind get_kind () const final override { return JSON_FLOAT; }
   void print (pretty_printer *pp, bool formatted) const final override;
+  std::unique_ptr<value> clone () const final override;
 
   double get () const { return m_value; }
 
@@ -257,6 +267,7 @@ class integer_number : public value
 
   enum kind get_kind () const final override { return JSON_INTEGER; }
   void print (pretty_printer *pp, bool formatted) const final override;
+  std::unique_ptr<value> clone () const final override;
 
   long get () const { return m_value; }
 
@@ -276,6 +287,8 @@ class string : public value
 
   enum kind get_kind () const final override { return JSON_STRING; }
   void print (pretty_printer *pp, bool formatted) const final override;
+  std::unique_ptr<value> clone () const final override;
+  string *dyn_cast_string () final override { return this; }
 
   const char *get_string () const { return m_utf8; }
   size_t get_length () const { return m_len; }
@@ -298,6 +311,7 @@ class literal : public value
 
   enum kind get_kind () const final override { return m_kind; }
   void print (pretty_printer *pp, bool formatted) const final override;
+  std::unique_ptr<value> clone () const final override;
 
  private:
   enum kind m_kind;
diff --git a/gcc/langhooks-def.h b/gcc/langhooks-def.h
index 010aaed..518509a 100644
--- a/gcc/langhooks-def.h
+++ b/gcc/langhooks-def.h
@@ -23,7 +23,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "hooks.h"
 
-struct diagnostic_info;
+namespace diagnostics { struct diagnostic_info; }
 class substring_loc;
 
 /* Note to creators of new hooks:
@@ -49,9 +49,9 @@ extern void lhd_print_tree_nothing (FILE *, tree, int);
 extern const char *lhd_decl_printable_name (tree, int);
 extern const char *lhd_dwarf_name (tree, int);
 extern int lhd_types_compatible_p (tree, tree);
-extern void lhd_print_error_function (diagnostic_text_output_format &,
+extern void lhd_print_error_function (diagnostics::text_sink &,
 				      const char *,
-				      const struct diagnostic_info *);
+				      const diagnostics::diagnostic_info *);
 extern void lhd_set_decl_assembler_name (tree decl);
 extern void lhd_overwrite_decl_assembler_name (tree decl, tree name);
 extern bool lhd_warn_unused_global_decl (const_tree);
@@ -71,7 +71,7 @@ extern tree lhd_builtin_function (tree);
 extern tree lhd_enum_underlying_base_type (const_tree);
 
 /* Declarations of default tree inlining hooks.  */
-extern void lhd_initialize_diagnostics (diagnostic_context *);
+extern void lhd_initialize_diagnostics (diagnostics::context *);
 extern void lhd_init_options (unsigned int,
 			      struct cl_decoded_option *);
 extern bool lhd_complain_wrong_lang_p (const struct cl_option *);
diff --git a/gcc/langhooks.cc b/gcc/langhooks.cc
index 194701f..20d27a6 100644
--- a/gcc/langhooks.cc
+++ b/gcc/langhooks.cc
@@ -38,7 +38,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "stor-layout.h"
 #include "cgraph.h"
 #include "debug.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 
 /* Do nothing; in many cases the default hook.  */
 
@@ -330,7 +330,7 @@ global_decl_processing (void)
 
 /* Called to perform language-specific initialization of CTX.  */
 void
-lhd_initialize_diagnostics (diagnostic_context *ctx ATTRIBUTE_UNUSED)
+lhd_initialize_diagnostics (diagnostics::context *ctx ATTRIBUTE_UNUSED)
 {
 }
 
@@ -369,11 +369,11 @@ lhd_handle_option (size_t code ATTRIBUTE_UNUSED,
 /* The default function to print out name of current function that caused
    an error.  */
 void
-lhd_print_error_function (diagnostic_text_output_format &text_output,
+lhd_print_error_function (diagnostics::text_sink &text_output,
 			  const char *file,
-			  const diagnostic_info *diagnostic)
+			  const diagnostics::diagnostic_info *diagnostic)
 {
-  diagnostic_context *const context = &text_output.get_context ();
+  diagnostics::context *const context = &text_output.get_context ();
   if (diagnostic_last_function_changed (context, diagnostic))
     {
       pretty_printer *const pp = text_output.get_printer ();
diff --git a/gcc/langhooks.h b/gcc/langhooks.h
index cb03c83..3bf4152 100644
--- a/gcc/langhooks.h
+++ b/gcc/langhooks.h
@@ -22,7 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 
 /* FIXME: This file should be #include-d after tree.h (for enum tree_code).  */
 
-struct diagnostic_info;
+namespace diagnostics { struct diagnostic_info; }
 
 struct gimplify_omp_ctx;
 
@@ -421,7 +421,7 @@ struct lang_hooks
 
   /* Callback used to perform language-specific initialization for the
      global diagnostic context structure.  */
-  void (*initialize_diagnostics) (diagnostic_context *);
+  void (*initialize_diagnostics) (diagnostics::context *);
 
   /* Beginning the main source file.  */
   void (*preprocess_main_file) (cpp_reader *, line_maps *,
@@ -547,9 +547,9 @@ struct lang_hooks
 
   /* Called by diagnostic_report_current_function to print out function name
      for textual diagnostic output.  */
-  void (*print_error_function) (diagnostic_text_output_format &,
+  void (*print_error_function) (diagnostics::text_sink &,
 				const char *,
-				const struct diagnostic_info *);
+				const diagnostics::diagnostic_info *);
 
   /* Convert a character from the host's to the target's character
      set.  The character should be in what C calls the "basic source
diff --git a/gcc/lazily-created.h b/gcc/lazily-created.h
new file mode 100644
index 0000000..a6010ca
--- /dev/null
+++ b/gcc/lazily-created.h
@@ -0,0 +1,51 @@
+/* Template for deferring object creation until the object is needed.
+   Copyright (C) 2024-2025 Free Software Foundation, Inc.
+   Contributed by David Malcolm <dmalcolm@redhat.com>
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef GCC_LAZILY_CREATED_H
+#define GCC_LAZILY_CREATED_H
+
+/* A template for deferring object creation for a type T until the object
+   is needed, to avoid potentially expensive creation of T unless it's
+   actually needed (e.g. for directed graphs associated with a diagnostic,
+   which are ignored by the default "text" sink).  */
+
+template <typename T>
+class lazily_created
+{
+ public:
+  virtual ~lazily_created () {}
+
+  const T &
+  get_or_create () const
+  {
+    if (!m_object)
+      m_object = create_object ();
+    gcc_assert (m_object);
+    return *m_object;
+  }
+
+private:
+  virtual std::unique_ptr<T>
+  create_object () const = 0;
+
+  mutable std::unique_ptr<T> m_object;
+};
+
+#endif /* ! GCC_LAZILY_CREATED_H */
diff --git a/gcc/libgdiagnostics++.h b/gcc/libgdiagnostics++.h
index 93b8f90..c955d56 100644
--- a/gcc/libgdiagnostics++.h
+++ b/gcc/libgdiagnostics++.h
@@ -34,6 +34,10 @@ class execution_path;
 class group;
 class manager;
 class diagnostic;
+class graph;
+class node;
+class edge;
+class message_buffer;
 
 /* Wrapper around a borrowed diagnostic_text_sink *.  */
 
@@ -131,6 +135,65 @@ public:
   const diagnostic_logical_location *m_inner;
 };
 
+/* Wrapper around a diagnostic_message_buffer *, with ownership.  */
+
+class message_buffer
+{
+public:
+  message_buffer () : m_inner (nullptr) {}
+  message_buffer (diagnostic_message_buffer *inner) : m_inner (inner) {}
+  ~message_buffer ()
+  {
+    if (m_inner)
+      diagnostic_message_buffer_release (m_inner);
+  }
+  message_buffer (const message_buffer &) = delete;
+  message_buffer (message_buffer &&other)
+  {
+    m_inner = other.m_inner;
+    other.m_inner = nullptr;
+  }
+  message_buffer& operator= (const message_buffer &) = delete;
+  message_buffer& operator= (message_buffer &&other)
+  {
+    if (m_inner)
+      diagnostic_message_buffer_release (m_inner);
+    m_inner = other.m_inner;
+    other.m_inner = nullptr;
+    return *this;
+  }
+
+  message_buffer&
+  operator+= (const char *str)
+  {
+    diagnostic_message_buffer_append_str (m_inner, str);
+    return *this;
+  }
+
+  message_buffer&
+  operator+= (char ch)
+  {
+    diagnostic_message_buffer_append_byte (m_inner, ch);
+    return *this;
+  }
+
+  message_buffer &
+  begin_url (const char *url)
+  {
+    diagnostic_message_buffer_begin_url (m_inner, url);
+    return *this;
+  }
+
+  message_buffer &
+  end_url ()
+  {
+    diagnostic_message_buffer_end_url (m_inner);
+    return *this;
+  }
+
+  diagnostic_message_buffer *m_inner;
+};
+
 /* RAII class around a diagnostic_execution_path *.  */
 
 class execution_path
@@ -188,6 +251,12 @@ public:
 		va_list *args)
     LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING (5, 0);
 
+  diagnostic_event_id
+  add_event_via_msg_buf (physical_location physical_loc,
+			 logical_location logical_loc,
+			 unsigned stack_depth,
+			 message_buffer &&msg_buf);
+
   diagnostic_execution_path *m_inner;
   bool m_owned;
 };
@@ -228,6 +297,10 @@ public:
 			   const char *text);
 
   void
+  add_location_with_label (physical_location loc,
+			   message_buffer &&text);
+
+  void
   set_logical_location (logical_location loc);
 
   void
@@ -246,6 +319,9 @@ public:
   take_execution_path (execution_path path);
 
   void
+  take_graph (graph g);
+
+  void
   finish (const char *fmt, ...)
     LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2)
     LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING (2, 3);
@@ -255,6 +331,9 @@ public:
     LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2)
     LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING (2, 0);
 
+  void
+  finish_via_msg_buf (message_buffer &&msg_buf);
+
   ::diagnostic * const m_inner;
 };
 
@@ -395,10 +474,105 @@ public:
   void
   set_analysis_target (file f);
 
+  void
+  take_global_graph (graph g);
+
   diagnostic_manager *m_inner;
   bool m_owned;
 };
 
+class graph
+{
+public:
+  graph () : m_inner (nullptr), m_owned (false) {}
+
+  graph (diagnostic_graph *graph)
+  : m_inner (graph), m_owned (true)
+  {}
+
+  graph (const diagnostic_graph *graph)
+  : m_inner (const_cast<diagnostic_graph *> (graph)),
+    m_owned (false)
+  {}
+
+  graph (const graph &other) = delete;
+  graph &operator= (const graph &other) = delete;
+
+  graph (graph &&other)
+  : m_inner (other.m_inner),
+    m_owned (other.m_owned)
+  {
+    other.m_inner = nullptr;
+    other.m_owned = false;
+  }
+
+  graph &operator= (graph &&other)
+  {
+    m_inner = other.m_inner;
+    m_owned = other.m_owned;
+    other.m_inner = nullptr;
+    other.m_owned = false;
+    return *this;
+  }
+
+  ~graph ()
+  {
+    if (m_owned)
+      diagnostic_graph_release (m_inner);
+  }
+
+  void
+  set_description (const char *);
+  void
+  set_description (message_buffer &&);
+
+  node
+  get_node_by_id (const char *id) const;
+
+  edge
+  get_edge_by_id (const char *id) const;
+
+  edge
+  add_edge (const char *id, node src_node, node dst_node, const char *label);
+  edge
+  add_edge (const char *id, node src_node, node dst_node, message_buffer &&label);
+
+  diagnostic_graph *m_inner;
+  bool m_owned;
+};
+
+// Borrowed pointer to a diagnostic_node.
+
+class node
+{
+public:
+  node () : m_inner (nullptr) {}
+  node (diagnostic_node *node_) : m_inner (node_) {}
+
+  void
+  set_label (const char *);
+  void
+  set_label (message_buffer &&);
+
+  void
+  set_location (physical_location loc);
+
+  void
+  set_logical_location (logical_location loc);
+
+  diagnostic_node *m_inner;
+};
+
+// Borrowed edge to a diagnostic_edge.
+
+class edge
+{
+public:
+  edge (diagnostic_edge *edge_) : m_inner (edge_) {}
+
+  diagnostic_edge *m_inner;
+};
+
 // Implementation
 
 // class file
@@ -496,6 +670,21 @@ execution_path::add_event_va (physical_location physical_loc,
 						 args);
 }
 
+inline diagnostic_event_id
+execution_path::add_event_via_msg_buf (physical_location physical_loc,
+				       logical_location logical_loc,
+				       unsigned stack_depth,
+				       message_buffer &&msg_buf)
+{
+  diagnostic_message_buffer *inner_msg_buf = msg_buf.m_inner;
+  msg_buf.m_inner = nullptr;
+  return diagnostic_execution_path_add_event_via_msg_buf (m_inner,
+							  physical_loc.m_inner,
+							  logical_loc.m_inner,
+							  stack_depth,
+							  inner_msg_buf);
+}
+
 // class group
 
 inline
@@ -539,6 +728,17 @@ diagnostic::add_location_with_label (physical_location loc,
 }
 
 inline void
+diagnostic::add_location_with_label (physical_location loc,
+				     message_buffer &&msg_buf)
+{
+  diagnostic_message_buffer *inner_msg_buf = msg_buf.m_inner;
+  msg_buf.m_inner = nullptr;
+  diagnostic_add_location_with_label_via_msg_buf (m_inner,
+						  loc.m_inner,
+						  inner_msg_buf);
+}
+
+inline void
 diagnostic::add_location (physical_location loc)
 {
   diagnostic_add_location (m_inner, loc.m_inner);
@@ -593,6 +793,14 @@ diagnostic::take_execution_path (execution_path path)
 }
 
 inline void
+diagnostic::take_graph (graph g)
+{
+  diagnostic_take_graph (m_inner,
+			 g.m_inner);
+  g.m_owned = false;
+}
+
+inline void
 diagnostic::finish (const char *fmt, ...)
 {
   va_list ap;
@@ -607,6 +815,14 @@ diagnostic::finish_va (const char *fmt, va_list *args)
   diagnostic_finish_va (m_inner, fmt, args);
 }
 
+inline void
+diagnostic::finish_via_msg_buf (message_buffer &&msg_buf)
+{
+  diagnostic_message_buffer *inner_msg_buf = msg_buf.m_inner;
+  msg_buf.m_inner = nullptr;
+  diagnostic_finish_via_msg_buf (m_inner, inner_msg_buf);
+}
+
 // class manager
 
 inline file
@@ -702,6 +918,96 @@ manager::set_analysis_target (file f)
   diagnostic_manager_set_analysis_target (m_inner, f.m_inner);
 }
 
+inline void
+manager::take_global_graph (graph g)
+{
+  diagnostic_manager_take_global_graph (m_inner,
+					g.m_inner);
+  g.m_owned = false;
+}
+
+// class graph
+
+inline void
+graph::set_description (const char *desc)
+{
+  diagnostic_graph_set_description (m_inner, desc);
+}
+
+inline void
+graph::set_description (message_buffer &&msg_buf)
+{
+  diagnostic_message_buffer *inner_msg_buf = msg_buf.m_inner;
+  msg_buf.m_inner = nullptr;
+  diagnostic_graph_set_description_via_msg_buf (m_inner, inner_msg_buf);
+}
+
+inline node
+graph::get_node_by_id (const char *id) const
+{
+  return node (diagnostic_graph_get_node_by_id (m_inner, id));
+}
+
+inline edge
+graph::get_edge_by_id (const char *id) const
+{
+  return edge (diagnostic_graph_get_edge_by_id (m_inner, id));
+}
+
+inline edge
+graph::add_edge (const char *id,
+		 node src_node, node dst_node,
+		 const char *label)
+{
+  return edge (diagnostic_graph_add_edge (m_inner,
+					  id,
+					  src_node.m_inner,
+					  dst_node.m_inner,
+					  label));
+}
+
+inline edge
+graph::add_edge (const char *id,
+		 node src_node, node dst_node,
+		 message_buffer &&label)
+{
+  diagnostic_message_buffer *inner_label = label.m_inner;
+  label.m_inner = nullptr;
+  return edge (diagnostic_graph_add_edge_via_msg_buf (m_inner,
+						      id,
+						      src_node.m_inner,
+						      dst_node.m_inner,
+						      inner_label));
+}
+
+// class node
+
+inline void
+node::set_label (const char *label)
+{
+  diagnostic_node_set_label (m_inner, label);
+}
+
+inline void
+node::set_label (message_buffer &&label)
+{
+  diagnostic_message_buffer *inner_label = label.m_inner;
+  label.m_inner = nullptr;
+  diagnostic_node_set_label_via_msg_buf (m_inner, inner_label);
+}
+
+inline void
+node::set_location (physical_location loc)
+{
+  diagnostic_node_set_location (m_inner, loc.m_inner);
+}
+
+inline void
+node::set_logical_location (logical_location loc)
+{
+  diagnostic_node_set_logical_location (m_inner, loc.m_inner);
+}
+
 } // namespace libgdiagnostics
 
 #endif // #ifndef LIBGDIAGNOSTICSPP_H
diff --git a/gcc/libgdiagnostics-private.h b/gcc/libgdiagnostics-private.h
new file mode 100644
index 0000000..0e90f87
--- /dev/null
+++ b/gcc/libgdiagnostics-private.h
@@ -0,0 +1,63 @@
+/* Private API entrypoints to libgdiagnostics purely for use by sarif-replay.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.  */
+
+#ifndef LIBGDIAGNOSTICS_PRIVATE_H
+#define LIBGDIAGNOSTICS_PRIVATE_H
+
+#include "libgdiagnostics.h"
+
+namespace json { class object; }
+
+extern "C" {
+
+/* Private entrypoints, for use only by sarif-replay.
+   These are subject to removal without notice.  */
+
+/* Entrypoints added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+private_diagnostic_graph_set_property_bag (diagnostic_graph &graph,
+					   std::unique_ptr<json::object> properties);
+
+extern void
+private_diagnostic_node_set_property_bag (diagnostic_node &node,
+					  std::unique_ptr<json::object> properties);
+
+extern void
+private_diagnostic_edge_set_property_bag (diagnostic_edge &edge,
+					  std::unique_ptr<json::object> properties);
+
+/* Entrypoint added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern diagnostic_event_id
+private_diagnostic_execution_path_add_event_3 (diagnostic_execution_path *path,
+					       const diagnostic_physical_location *physical_loc,
+					       const diagnostic_logical_location *logical_loc,
+					       unsigned stack_depth,
+					       diagnostic_graph *state_graph,
+					       diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (3)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (5)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (6);
+
+} // extern "C"
+
+#endif  /* LIBGDIAGNOSTICS_PRIVATE_H  */
diff --git a/gcc/libgdiagnostics.cc b/gcc/libgdiagnostics.cc
index 74814c7..7351d336 100644
--- a/gcc/libgdiagnostics.cc
+++ b/gcc/libgdiagnostics.cc
@@ -19,22 +19,30 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "config.h"
 #define INCLUDE_MAP
+#define INCLUDE_STRING
 #define INCLUDE_VECTOR
 #include "system.h"
 #include "coretypes.h"
 #include "intl.h"
 #include "diagnostic.h"
-#include "diagnostic-color.h"
-#include "diagnostic-url.h"
-#include "diagnostic-metadata.h"
-#include "diagnostic-path.h"
-#include "diagnostic-client-data-hooks.h"
-#include "diagnostic-format-sarif.h"
-#include "diagnostic-format-text.h"
-#include "diagnostic-output-spec.h"
-#include "logical-location.h"
-#include "edit-context.h"
+#include "diagnostics/color.h"
+#include "diagnostics/file-cache.h"
+#include "diagnostics/url.h"
+#include "diagnostics/metadata.h"
+#include "diagnostics/paths.h"
+#include "diagnostics/client-data-hooks.h"
+#include "diagnostics/sarif-sink.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/output-spec.h"
+#include "diagnostics/digraphs.h"
+#include "diagnostics/state-graphs.h"
+#include "diagnostics/logical-locations.h"
+#include "diagnostics/changes.h"
 #include "libgdiagnostics.h"
+#include "libgdiagnostics-private.h"
+#include "pretty-print-format-impl.h"
+#include "pretty-print-markup.h"
+#include "auto-obstack.h"
 
 class owned_nullable_string
 {
@@ -203,7 +211,7 @@ struct diagnostic_logical_location
 };
 
 static diagnostic_event_id
-as_diagnostic_event_id (diagnostic_event_id_t id)
+as_diagnostic_event_id (diagnostics::paths::event_id_t id)
 {
   return id.zero_based ();
 }
@@ -225,7 +233,7 @@ public:
 			FILE *dst_stream,
 			enum diagnostic_colorize colorize);
 
-  diagnostic_source_printing_options &get_source_printing_options ()
+  diagnostics::source_printing_options &get_source_printing_options ()
   {
     return m_source_printing;
   }
@@ -234,12 +242,103 @@ public:
   set_colorize (enum diagnostic_colorize colorize);
 
   static void
-  text_starter (diagnostic_text_output_format &text_output,
-		const diagnostic_info *diagnostic);
+  text_starter (diagnostics::text_sink &text_output,
+		const diagnostics::diagnostic_info *diagnostic);
 
 private:
-  diagnostic_text_output_format *m_inner_sink; // borrowed from dc
-  diagnostic_source_printing_options m_source_printing;
+  diagnostics::text_sink *m_inner_sink; // borrowed from dc
+  diagnostics::source_printing_options m_source_printing;
+};
+
+/* A token_printer that makes a deep copy of the pp_token_list
+   into another obstack.  */
+
+class copying_token_printer : public token_printer
+{
+public:
+  copying_token_printer (obstack &dst_obstack,
+			 pp_token_list &dst_token_list)
+  : m_dst_obstack (dst_obstack),
+    m_dst_token_list (dst_token_list)
+  {
+  }
+
+  void
+  print_tokens (pretty_printer *,
+		const pp_token_list &tokens) final override
+  {
+    for (auto iter = tokens.m_first; iter; iter = iter->m_next)
+      switch (iter->m_kind)
+	{
+	default:
+	  gcc_unreachable ();
+
+	case pp_token::kind::text:
+	  {
+	    const pp_token_text *sub = as_a <const pp_token_text *> (iter);
+	    /* Copy the text, with null terminator.  */
+	    obstack_grow (&m_dst_obstack, sub->m_value.get (),
+			  strlen (sub->m_value.get ()) + 1);
+	    m_dst_token_list.push_back_text
+	      (label_text::borrow (XOBFINISH (&m_dst_obstack,
+					      const char *)));
+	  }
+	  break;
+
+	case pp_token::kind::begin_color:
+	  {
+	    pp_token_begin_color *sub = as_a <pp_token_begin_color *> (iter);
+	    /* Copy the color, with null terminator.  */
+	    obstack_grow (&m_dst_obstack, sub->m_value.get (),
+			  strlen (sub->m_value.get ()) + 1);
+	    m_dst_token_list.push_back<pp_token_begin_color>
+	      (label_text::borrow (XOBFINISH (&m_dst_obstack,
+					      const char *)));
+	  }
+	  break;
+	case pp_token::kind::end_color:
+	  m_dst_token_list.push_back<pp_token_end_color> ();
+	  break;
+
+	case pp_token::kind::begin_quote:
+	  m_dst_token_list.push_back<pp_token_begin_quote> ();
+	  break;
+	case pp_token::kind::end_quote:
+	  m_dst_token_list.push_back<pp_token_end_quote> ();
+	  break;
+
+	case pp_token::kind::begin_url:
+	  {
+	    pp_token_begin_url *sub = as_a <pp_token_begin_url *> (iter);
+	    /* Copy the URL, with null terminator.  */
+	    obstack_grow (&m_dst_obstack, sub->m_value.get (),
+			  strlen (sub->m_value.get ()) + 1);
+	    m_dst_token_list.push_back<pp_token_begin_url>
+	      (label_text::borrow (XOBFINISH (&m_dst_obstack,
+					      const char *)));
+	  }
+	  break;
+	case pp_token::kind::end_url:
+	  m_dst_token_list.push_back<pp_token_end_url> ();
+	  break;
+
+	case pp_token::kind::event_id:
+	  {
+	    pp_token_event_id *sub = as_a <pp_token_event_id *> (iter);
+	    m_dst_token_list.push_back<pp_token_event_id> (sub->m_event_id);
+	  }
+	  break;
+
+	case pp_token::kind::custom_data:
+	  /* These should have been eliminated by replace_custom_tokens.  */
+	  gcc_unreachable ();
+	  break;
+	}
+  }
+
+private:
+  obstack &m_dst_obstack;
+  pp_token_list &m_dst_token_list;
 };
 
 class sarif_sink : public sink
@@ -248,7 +347,108 @@ public:
   sarif_sink (diagnostic_manager &mgr,
 	      FILE *dst_stream,
 	      const diagnostic_file *main_input_file,
-	      const sarif_generation_options &sarif_gen_opts);
+	      const diagnostics::sarif_generation_options &sarif_gen_opts);
+};
+
+struct diagnostic_message_buffer
+{
+  diagnostic_message_buffer ()
+  : m_tokens (m_obstack)
+  {
+  }
+
+  diagnostic_message_buffer (const char *gmsgid,
+			     va_list *args)
+  : m_tokens (m_obstack)
+  {
+    text_info text (gmsgid, args, errno);
+    pretty_printer pp;
+    pp.set_output_stream (nullptr);
+    copying_token_printer tok_printer (m_obstack, m_tokens);
+    pp.set_token_printer (&tok_printer);
+    pp_format (&pp, &text);
+    pp_output_formatted_text (&pp, nullptr);
+  }
+
+
+  std::string to_string () const;
+
+  auto_obstack m_obstack;
+  pp_token_list m_tokens;
+};
+
+/* A pp_element subclass that replays the saved tokens in a
+   diagnostic_message_buffer.  */
+
+class pp_element_message_buffer : public pp_element
+{
+public:
+  pp_element_message_buffer (diagnostic_message_buffer &msg_buf)
+    : m_msg_buf (msg_buf)
+  {
+  }
+
+  void add_to_phase_2 (pp_markup::context &ctxt) final override
+  {
+    /* Convert to text, possibly with colorization, URLs, etc.  */
+    for (auto iter = m_msg_buf.m_tokens.m_first; iter; iter = iter->m_next)
+      switch (iter->m_kind)
+	{
+	default:
+	  gcc_unreachable ();
+
+	case pp_token::kind::text:
+	  {
+	    pp_token_text *sub = as_a <pp_token_text *> (iter);
+	    pp_string (&ctxt.m_pp, sub->m_value.get ());
+	    ctxt.push_back_any_text ();
+	  }
+	  break;
+
+	case pp_token::kind::begin_color:
+	  {
+	    pp_token_begin_color *sub = as_a <pp_token_begin_color *> (iter);
+	    ctxt.begin_highlight_color (sub->m_value.get ());
+	  }
+	  break;
+	case pp_token::kind::end_color:
+	  ctxt.end_highlight_color ();
+	  break;
+
+	case pp_token::kind::begin_quote:
+	  ctxt.begin_quote ();
+	  break;
+	case pp_token::kind::end_quote:
+	  ctxt.end_quote ();
+	  break;
+
+	case pp_token::kind::begin_url:
+	  {
+	    pp_token_begin_url *sub = as_a <pp_token_begin_url *> (iter);
+	    ctxt.begin_url (sub->m_value.get ());
+	  }
+	  break;
+	case pp_token::kind::end_url:
+	  ctxt.end_url ();
+	  break;
+
+	case pp_token::kind::event_id:
+	  {
+	    pp_token_event_id *sub = as_a <pp_token_event_id *> (iter);
+	    gcc_assert (sub->m_event_id.known_p ());
+	    ctxt.add_event_id (sub->m_event_id);
+	  }
+	  break;
+
+	case pp_token::kind::custom_data:
+	  /* We don't have a way of handling custom_data tokens here.  */
+	  gcc_unreachable ();
+	  break;
+	}
+  }
+
+private:
+  diagnostic_message_buffer &m_msg_buf;
 };
 
 /* Helper for the linemap code.  */
@@ -259,19 +459,23 @@ round_alloc_size (size_t s)
   return s;
 }
 
-class impl_logical_location_manager : public logical_location_manager
+class impl_logical_location_manager
+  : public diagnostics::logical_locations::manager
 {
 public:
+  using key = diagnostics::logical_locations::key;
+  using kind = diagnostics::logical_locations::kind;
+
   static const diagnostic_logical_location *
-  ptr_from_key (logical_location k)
+  ptr_from_key (key k)
   {
     return k.cast_to<const diagnostic_logical_location *> ();
   }
 
-  static logical_location
+  static key
   key_from_ptr (const diagnostic_logical_location *ptr)
   {
-    return logical_location::from_ptr (ptr);
+    return key::from_ptr (ptr);
   }
 
   const char *get_short_name (key k) const final override
@@ -298,7 +502,7 @@ public:
       return nullptr;
   }
 
-  enum logical_location_kind get_kind (key k) const final override
+  kind get_kind (key k) const final override
   {
     auto loc = ptr_from_key (k);
     gcc_assert (loc);
@@ -308,45 +512,45 @@ public:
 	gcc_unreachable ();
 
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_FUNCTION:
-	return logical_location_kind::function;
+	return kind::function;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_MEMBER:
-	return logical_location_kind::member;
+	return kind::member;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_MODULE:
-	return logical_location_kind::module_;
+	return kind::module_;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_NAMESPACE:
-	return logical_location_kind::namespace_;
+	return kind::namespace_;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_TYPE:
-	return logical_location_kind::type;
+	return kind::type;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_RETURN_TYPE:
-	return logical_location_kind::return_type;
+	return kind::return_type;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_PARAMETER:
-	return logical_location_kind::parameter;
+	return kind::parameter;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_VARIABLE:
-	return logical_location_kind::variable;
+	return kind::variable;
 
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_ELEMENT:
-	return logical_location_kind::element;
+	return kind::element;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_ATTRIBUTE:
-	return logical_location_kind::attribute;
+	return kind::attribute;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_TEXT:
-	return logical_location_kind::text;
+	return kind::text;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_COMMENT:
-	return logical_location_kind::comment;
+	return kind::comment;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_PROCESSING_INSTRUCTION:
-	return logical_location_kind::processing_instruction;
+	return kind::processing_instruction;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_DTD:
-	return logical_location_kind::dtd;
+	return kind::dtd;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_DECLARATION:
-	return logical_location_kind::declaration;
+	return kind::declaration;
 
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_OBJECT:
-	  return logical_location_kind::object;
+	  return kind::object;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_ARRAY:
-	return logical_location_kind::array;
+	return kind::array;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_PROPERTY:
-	return logical_location_kind::property;
+	return kind::property;
       case DIAGNOSTIC_LOGICAL_LOCATION_KIND_VALUE:
-	return logical_location_kind::value;
+	return kind::value;
       }
   }
 
@@ -365,25 +569,28 @@ public:
   }
 };
 
-class impl_diagnostic_client_data_hooks : public diagnostic_client_data_hooks
+class impl_diagnostic_client_data_hooks : public diagnostics::client_data_hooks
 {
 public:
   impl_diagnostic_client_data_hooks (diagnostic_manager &mgr)
   : m_mgr (mgr)
   {}
 
-  const client_version_info *get_any_version_info () const final override;
+  const diagnostics::client_version_info *
+  get_any_version_info () const final override;
 
-  const logical_location_manager *
+  const diagnostics::logical_locations::manager *
   get_logical_location_manager () const final override
   {
     return &m_logical_location_manager;
   }
-  logical_location get_current_logical_location () const final override;
+  diagnostics::logical_locations::key
+  get_current_logical_location () const final override;
 
   const char * maybe_get_sarif_source_language (const char *filename)
     const final override;
-  void add_sarif_invocation_properties (sarif_object &invocation_obj)
+  void
+  add_sarif_invocation_properties (diagnostics::sarif_object &invocation_obj)
     const final override;
 
 private:
@@ -391,7 +598,7 @@ private:
   impl_logical_location_manager m_logical_location_manager;
 };
 
-class impl_client_version_info : public client_version_info
+class impl_client_version_info : public diagnostics::client_version_info
 {
 public:
   const char *get_tool_name () const final override
@@ -450,23 +657,26 @@ public:
     m_dc.set_show_cwe (true);
     m_dc.set_show_rules (true);
     m_dc.m_show_column = true;
-    m_dc.m_source_printing.enabled = true;
-    m_dc.m_source_printing.colorize_source_p = true;
+    auto &source_printing_opts = m_dc.get_source_printing_options ();
+    source_printing_opts.enabled = true;
+    source_printing_opts.colorize_source_p = true;
 
     /* We don't currently expose a way for clients to manipulate the
        following.  */
-    m_dc.m_source_printing.show_labels_p = true;
-    m_dc.m_source_printing.show_line_numbers_p = true;
-    m_dc.m_source_printing.min_margin_width = 6;
+    source_printing_opts.show_labels_p = true;
+    source_printing_opts.show_line_numbers_p = true;
+    source_printing_opts.min_margin_width = 6;
     m_dc.set_path_format (DPF_INLINE_EVENTS);
 
     m_dc.m_client_aux_data = this;
     m_dc.set_client_data_hooks
       (std::make_unique<impl_diagnostic_client_data_hooks> (*this));
 
-    diagnostic_text_starter (&m_dc) = diagnostic_text_sink::text_starter;
+    diagnostics::text_starter (&m_dc) = diagnostic_text_sink::text_starter;
 
-    m_edit_context = std::make_unique <edit_context> (m_dc.get_file_cache ());
+    m_change_set
+      = std::make_unique <diagnostics::changes::change_set>
+	  (m_dc.get_file_cache ());
   }
 
   ~diagnostic_manager ()
@@ -487,9 +697,9 @@ public:
   }
 
   line_maps *get_line_table () { return &m_line_table; }
-  diagnostic_context &get_dc () { return m_dc; }
+  diagnostics::context &get_dc () { return m_dc; }
 
-  const logical_location_manager &
+  const diagnostics::logical_locations::manager &
   get_logical_location_manager () const
   {
     auto mgr = m_dc.get_logical_location_manager ();
@@ -504,9 +714,15 @@ public:
     m_sinks.push_back (std::move (sink));
   }
 
-  void emit (diagnostic &diag, const char *msgid, va_list *args)
+  void emit_va (diagnostic &diag, const char *msgid, va_list *args)
     LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING(3, 0);
 
+  void emit (diagnostic &diag, const char *msgid, ...)
+    LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING(3, 4);
+
+  void emit_msg_buf (diagnostic &diag,
+		     diagnostic_message_buffer &msg_buf);
+
   diagnostic_file *
   new_file (const char *name,
 	    const char *sarif_source_language)
@@ -613,7 +829,8 @@ public:
 
   const diagnostic *get_current_diag () { return m_current_diag; }
 
-  const client_version_info *get_client_version_info () const
+  const diagnostics::client_version_info *
+  get_client_version_info () const
   {
     return &m_client_version_info;
   }
@@ -644,6 +861,9 @@ public:
     return m_prev_diag_logical_loc;
   }
 
+  void
+  take_global_graph (std::unique_ptr<diagnostic_graph> graph);
+
 private:
   void
   ensure_linemap_for_file_and_line (const diagnostic_file *file,
@@ -676,7 +896,7 @@ private:
     return phys_loc;
   }
 
-  diagnostic_context m_dc;
+  diagnostics::context m_dc;
   line_maps m_line_table;
   impl_client_version_info m_client_version_info;
   std::vector<std::unique_ptr<sink>> m_sinks;
@@ -688,7 +908,7 @@ private:
   logical_locs_map_t m_logical_locs;
   const diagnostic *m_current_diag;
   const diagnostic_logical_location *m_prev_diag_logical_loc;
-  std::unique_ptr<edit_context> m_edit_context;
+  std::unique_ptr<diagnostics::changes::change_set> m_change_set;
 };
 
 class impl_rich_location : public rich_location
@@ -717,7 +937,7 @@ private:
   char *m_text;
 };
 
-class impl_rule : public diagnostic_metadata::rule
+class impl_rule : public diagnostics::metadata::rule
 {
 public:
   impl_rule (const char *title, const char *url)
@@ -743,23 +963,58 @@ private:
   owned_nullable_string m_url;
 };
 
-class libgdiagnostics_path_event : public diagnostic_event
+struct diagnostic_graph : public diagnostics::digraphs::digraph
+{
+  diagnostic_graph (diagnostic_manager &) {}
+
+  diagnostic_node *
+  add_node_with_id (std::string node_id,
+		    diagnostic_node *parent_node);
+  diagnostic_edge *
+  add_edge_with_label (const char *edge_id,
+		       diagnostic_node &src_node,
+		       diagnostic_node &dst_node,
+		       const char *label);
+};
+
+struct diagnostic_node : public diagnostics::digraphs::node
+{
+  diagnostic_node (diagnostic_graph &g,
+		   std::string id)
+  : node (g, std::move (id))
+  {
+  }
+};
+
+struct diagnostic_edge : public diagnostics::digraphs::edge
+{
+  diagnostic_edge (diagnostic_graph &g,
+		   const char *id,
+		   diagnostic_node &src_node,
+		   diagnostic_node &dst_node)
+  : edge (g, id, src_node, dst_node)
+  {
+  }
+};
+
+class libgdiagnostics_path_event : public diagnostics::paths::event
 {
 public:
   libgdiagnostics_path_event (const diagnostic_physical_location *physical_loc,
 			      const diagnostic_logical_location *logical_loc,
 			      unsigned stack_depth,
-			      const char *gmsgid,
-			      va_list *args)
+			      std::unique_ptr<diagnostic_graph> state_graph,
+			      std::unique_ptr<diagnostic_message_buffer> msg_buf)
   : m_physical_loc (physical_loc),
     m_logical_loc (logical_loc),
-    m_stack_depth (stack_depth)
+    m_stack_depth (stack_depth),
+    m_state_graph (std::move (state_graph)),
+    m_msg_buf (std::move (msg_buf))
   {
-    m_desc_uncolored = make_desc (gmsgid, args, false);
-    m_desc_colored = make_desc (gmsgid, args, true);
+    gcc_assert (m_msg_buf);
   }
 
-  /* diagnostic_event vfunc implementations.  */
+  /* diagnostics::paths::event vfunc implementations.  */
 
   location_t get_location () const final override
   {
@@ -773,13 +1028,15 @@ public:
 
   void print_desc (pretty_printer &pp) const final override
   {
-    if (pp_show_color (&pp))
-      pp_string (&pp, m_desc_colored.get ());
-    else
-      pp_string (&pp, m_desc_uncolored.get ());
+    if (m_msg_buf)
+      {
+	pp_element_message_buffer e_msg_buf (*m_msg_buf);
+	pp_printf (&pp, "%e", &e_msg_buf);
+      }
   }
 
-  logical_location get_logical_location () const final override
+  diagnostics::logical_locations::key
+  get_logical_location () const final override
   {
     return impl_logical_location_manager::key_from_ptr (m_logical_loc);
   }
@@ -794,11 +1051,20 @@ public:
     return false; // TODO
   }
 
-  diagnostic_thread_id_t get_thread_id () const final override
+  diagnostics::paths::thread_id_t get_thread_id () const final override
   {
     return 0;
   }
 
+  std::unique_ptr<diagnostics::digraphs::digraph>
+  maybe_make_diagnostic_state_graph (bool) const final override
+  {
+    if (!m_state_graph)
+      return nullptr;
+
+    return m_state_graph->clone ();
+  }
+
 private:
   static label_text make_desc (const char *gmsgid,
 			       va_list *args,
@@ -824,11 +1090,11 @@ private:
   const diagnostic_physical_location *m_physical_loc;
   const diagnostic_logical_location *m_logical_loc;
   unsigned m_stack_depth;
-  label_text m_desc_uncolored;
-  label_text m_desc_colored;
+  std::unique_ptr<diagnostic_graph> m_state_graph;
+  std::unique_ptr<diagnostic_message_buffer> m_msg_buf;
 };
 
-class libgdiagnostics_path_thread : public diagnostic_thread
+class libgdiagnostics_path_thread : public diagnostics::paths::thread
 {
 public:
   libgdiagnostics_path_thread (const char *name) : m_name (name) {}
@@ -843,43 +1109,62 @@ private:
 
 /* This has to be a "struct" as it is exposed in the C API.  */
 
-struct diagnostic_execution_path : public diagnostic_path
+struct diagnostic_execution_path : public diagnostics::paths::path
 {
-  diagnostic_execution_path (const logical_location_manager &logical_loc_mgr)
-  : diagnostic_path (logical_loc_mgr),
+  diagnostic_execution_path (const diagnostics::logical_locations::manager &logical_loc_mgr)
+  : diagnostics::paths::path (logical_loc_mgr),
     m_thread ("")
   {
   }
 
-  diagnostic_event_id_t
+  diagnostics::paths::event_id_t
   add_event_va (const diagnostic_physical_location *physical_loc,
 		const diagnostic_logical_location *logical_loc,
 		unsigned stack_depth,
+		std::unique_ptr<diagnostic_graph> state_graph,
 		const char *gmsgid,
 		va_list *args)
   {
+    auto msg_buf = std::make_unique<diagnostic_message_buffer> (gmsgid, args);
+
     m_events.push_back
       (std::make_unique<libgdiagnostics_path_event> (physical_loc,
 						     logical_loc,
 						     stack_depth,
-						     gmsgid,
-						     args));
+						     std::move (state_graph),
+						     std::move (msg_buf)));
     return m_events.size () - 1;
   }
 
-  /* diagnostic_path vfunc implementations.  */
+  diagnostic_event_id_t
+  add_event_via_msg_buf (const diagnostic_physical_location *physical_loc,
+			 const diagnostic_logical_location *logical_loc,
+			 unsigned stack_depth,
+			 std::unique_ptr<diagnostic_graph> state_graph,
+			 std::unique_ptr<diagnostic_message_buffer> msg_buf)
+  {
+    m_events.push_back
+      (std::make_unique<libgdiagnostics_path_event> (physical_loc,
+						     logical_loc,
+						     stack_depth,
+						     std::move (state_graph),
+						     std::move (msg_buf)));
+    return m_events.size () - 1;
+  }
+
+  /* diagnostics::paths::path vfunc implementations.  */
 
   unsigned num_events () const final override
   {
     return m_events.size ();
   }
-  const diagnostic_event & get_event (int idx) const final override
+  const diagnostics::paths::event & get_event (int idx) const final override
   {
     return *m_events[idx];
   }
   unsigned num_threads () const final override { return 1; }
-  const diagnostic_thread &
-  get_thread (diagnostic_thread_id_t) const final override
+  const diagnostics::paths::thread &
+  get_thread (diagnostics::paths::thread_id_t) const final override
   {
     return m_thread;
   }
@@ -888,9 +1173,10 @@ struct diagnostic_execution_path : public diagnostic_path
   same_function_p (int event_idx_a,
 		   int event_idx_b) const final override
   {
+    using logical_location = diagnostics::logical_locations::key;
     logical_location logical_loc_a
       = m_events[event_idx_a]->get_logical_location ();
-    logical_location logical_loc_b
+    logical_location  logical_loc_b
       = m_events[event_idx_b]->get_logical_location ();
 
     /* Pointer equality, as we uniqify logical location instances.  */
@@ -902,6 +1188,28 @@ private:
   std::vector<std::unique_ptr<libgdiagnostics_path_event>> m_events;
 };
 
+class prebuilt_digraphs
+  : public lazily_created<std::vector<std::unique_ptr<diagnostics::digraphs::digraph>>>
+{
+public:
+  using digraph = diagnostics::digraphs::digraph;
+
+  std::unique_ptr<std::vector<std::unique_ptr<digraph>>>
+  create_object () const final override
+  {
+    return std::make_unique<std::vector<std::unique_ptr<digraph>>> (std::move (m_digraphs));
+  }
+
+  void
+  take_graph (std::unique_ptr<diagnostic_graph> graph)
+  {
+    m_digraphs.push_back (std::move (graph));
+  }
+
+private:
+  mutable std::vector<std::unique_ptr<digraph>> m_digraphs;
+};
+
 /* This has to be a "struct" as it is exposed in the C API.  */
 
 struct diagnostic
@@ -914,7 +1222,9 @@ public:
     m_rich_loc (diag_mgr.get_line_table ()),
     m_logical_loc (nullptr),
     m_path (nullptr)
-  {}
+  {
+    m_metadata.set_lazy_digraphs (&m_graphs);
+  }
 
   diagnostic_manager &get_manager () const
   {
@@ -924,7 +1234,7 @@ public:
   enum diagnostic_level get_level () const { return m_level; }
 
   rich_location *get_rich_location () { return &m_rich_loc; }
-  const diagnostic_metadata *get_metadata () { return &m_metadata; }
+  const diagnostics::metadata *get_metadata () { return &m_metadata; }
 
   void set_cwe (unsigned cwe_id)
   {
@@ -964,6 +1274,19 @@ public:
   }
 
   void
+  add_location_with_label (const diagnostic_physical_location *loc,
+			   std::unique_ptr<diagnostic_message_buffer> msg_buf)
+  {
+    std::string str = msg_buf->to_string ();
+    std::unique_ptr<range_label> label
+      = std::make_unique <impl_range_label> (str.c_str ());
+    m_rich_loc.add_range (as_location_t (loc),
+			  SHOW_RANGE_WITHOUT_CARET,
+			  label.get ());
+    m_labels.push_back (std::move (label));
+  }
+
+  void
   set_logical_location (const diagnostic_logical_location *logical_loc)
   {
     m_logical_loc = logical_loc;
@@ -990,32 +1313,45 @@ public:
     m_rich_loc.set_path (path);
   }
 
+  void
+  take_graph (std::unique_ptr<diagnostic_graph> graph)
+  {
+    m_graphs.take_graph (std::move (graph));
+  }
+
+  const prebuilt_digraphs &
+  get_graphs () const
+  {
+    return m_graphs;
+  }
+
 private:
   diagnostic_manager &m_diag_mgr;
   enum diagnostic_level m_level;
   impl_rich_location m_rich_loc;
   const diagnostic_logical_location *m_logical_loc;
-  diagnostic_metadata m_metadata;
+  diagnostics::metadata m_metadata;
+  prebuilt_digraphs m_graphs;
   std::vector<std::unique_ptr<range_label>> m_labels;
   std::vector<std::unique_ptr<impl_rule>> m_rules;
   std::unique_ptr<diagnostic_execution_path> m_path;
 };
 
-static diagnostic_t
-diagnostic_t_from_diagnostic_level (enum diagnostic_level level)
+static enum diagnostics::kind
+diagnostics_kind_from_diagnostic_level (enum diagnostic_level level)
 {
   switch (level)
     {
     default:
       gcc_unreachable ();
     case DIAGNOSTIC_LEVEL_ERROR:
-      return DK_ERROR;
+      return diagnostics::kind::error;
     case DIAGNOSTIC_LEVEL_WARNING:
-      return DK_WARNING;
+      return diagnostics::kind::warning;
     case DIAGNOSTIC_LEVEL_NOTE:
-      return DK_NOTE;
+      return diagnostics::kind::note;
     case DIAGNOSTIC_LEVEL_SORRY:
-      return DK_SORRY;
+      return diagnostics::kind::sorry;
     }
 }
 
@@ -1025,7 +1361,7 @@ diagnostic_file::set_buffered_content (const char *buf, size_t sz)
   m_content = std::make_unique<content_buffer> (buf, sz);
 
   // Populate file_cache:
-  file_cache &fc = m_mgr.get_dc ().get_file_cache ();
+  diagnostics::file_cache &fc = m_mgr.get_dc ().get_file_cache ();
   fc.add_buffered_content (m_name.get_str (), buf, sz);
 }
 
@@ -1043,13 +1379,13 @@ diagnostic_physical_location::get_file () const
 
 /* class impl_diagnostic_client_data_hooks.  */
 
-const client_version_info *
+const diagnostics::client_version_info *
 impl_diagnostic_client_data_hooks::get_any_version_info () const
 {
   return m_mgr.get_client_version_info ();
 }
 
-logical_location
+diagnostics::logical_locations::key
 impl_diagnostic_client_data_hooks::get_current_logical_location () const
 {
   gcc_assert (m_mgr.get_current_diag ());
@@ -1067,7 +1403,7 @@ maybe_get_sarif_source_language (const char *filename) const
 
 void
 impl_diagnostic_client_data_hooks::
-add_sarif_invocation_properties (sarif_object &) const
+add_sarif_invocation_properties (diagnostics::sarif_object &) const
 {
   // No-op.
 }
@@ -1078,10 +1414,10 @@ diagnostic_text_sink::diagnostic_text_sink (diagnostic_manager &mgr,
 					    FILE *dst_stream,
 					    enum diagnostic_colorize colorize)
 : sink (mgr),
-  m_source_printing (mgr.get_dc ().m_source_printing)
+  m_source_printing (mgr.get_dc ().get_source_printing_options ())
 {
   auto inner_sink
-    = std::make_unique<diagnostic_text_output_format> (mgr.get_dc (),
+    = std::make_unique<diagnostics::text_sink> (mgr.get_dc (),
 						       &m_source_printing);
   inner_sink->get_printer ()->set_output_stream (dst_stream);
   m_inner_sink = inner_sink.get ();
@@ -1111,11 +1447,11 @@ diagnostic_text_sink::set_colorize (enum diagnostic_colorize colorize)
 }
 
 void
-diagnostic_text_sink::text_starter (diagnostic_text_output_format &text_output,
-				    const diagnostic_info *info)
+diagnostic_text_sink::text_starter (diagnostics::text_sink &text_output,
+				    const diagnostics::diagnostic_info *info)
 {
-  gcc_assert (info->x_data);
-  const diagnostic &diag = *static_cast<const diagnostic *> (info->x_data);
+  gcc_assert (info->m_x_data);
+  const diagnostic &diag = *static_cast<const diagnostic *> (info->m_x_data);
   pretty_printer *pp = text_output.get_printer ();
   const diagnostic_logical_location *diag_logical_loc
     = diag.get_logical_location ();
@@ -1181,15 +1517,17 @@ diagnostic_text_sink::text_starter (diagnostic_text_output_format &text_output,
 
 /* class sarif_sink : public sink.  */
 
-sarif_sink::sarif_sink (diagnostic_manager &mgr,
-			FILE *dst_stream,
-			const diagnostic_file *main_input_file,
-			const sarif_generation_options &sarif_gen_opts)
+sarif_sink::
+sarif_sink (diagnostic_manager &mgr,
+	    FILE *dst_stream,
+	    const diagnostic_file *main_input_file,
+	    const diagnostics::sarif_generation_options &sarif_gen_opts)
 : sink (mgr)
 {
-  diagnostic_output_file output_file (dst_stream, false,
-				      label_text::borrow ("sarif_sink"));
-  auto serialization = std::make_unique<sarif_serialization_format_json> (true);
+  diagnostics::output_file output_file (dst_stream, false,
+					label_text::borrow ("sarif_sink"));
+  auto serialization
+    = std::make_unique<diagnostics::sarif_serialization_format_json> (true);
   auto inner_sink = make_sarif_sink (mgr.get_dc (),
 				     *mgr.get_line_table (),
 				     std::move (serialization),
@@ -1199,6 +1537,64 @@ sarif_sink::sarif_sink (diagnostic_manager &mgr,
   mgr.get_dc ().add_sink (std::move (inner_sink));
 }
 
+// struct diagnostic_message_buffer
+
+std::string
+diagnostic_message_buffer::to_string () const
+{
+  std::string result;
+
+  /* Convert to text, dropping colorization, URLs, etc.  */
+  for (auto iter = m_tokens.m_first; iter; iter = iter->m_next)
+    switch (iter->m_kind)
+      {
+      default:
+	gcc_unreachable ();
+
+      case pp_token::kind::text:
+	{
+	  pp_token_text *sub = as_a <pp_token_text *> (iter);
+	  result += sub->m_value.get ();
+	}
+	break;
+
+      case pp_token::kind::begin_color:
+      case pp_token::kind::end_color:
+	// Skip
+	break;
+
+      case pp_token::kind::begin_quote:
+	result += open_quote;
+	break;
+
+      case pp_token::kind::end_quote:
+	result += close_quote;
+	break;
+
+      case pp_token::kind::begin_url:
+      case pp_token::kind::end_url:
+	// Skip
+	break;
+
+      case pp_token::kind::event_id:
+	{
+	  pp_token_event_id *sub = as_a <pp_token_event_id *> (iter);
+	  gcc_assert (sub->m_event_id.known_p ());
+	  result += '(';
+	  result += std::to_string (sub->m_event_id.one_based ());
+	  result += ')';
+	}
+	break;
+
+      case pp_token::kind::custom_data:
+	/* We don't have a way of handling custom_data tokens here.  */
+	gcc_unreachable ();
+	break;
+      }
+
+  return result;
+}
+
 /* struct diagnostic_manager.  */
 
 void
@@ -1206,12 +1602,12 @@ diagnostic_manager::write_patch (FILE *dst_stream)
 {
   pretty_printer pp;
   pp.set_output_stream (dst_stream);
-  m_edit_context->print_diff (&pp, true);
+  m_change_set->print_diff (&pp, true);
   pp_flush (&pp);
 }
 
 void
-diagnostic_manager::emit (diagnostic &diag, const char *msgid, va_list *args)
+diagnostic_manager::emit_va (diagnostic &diag, const char *msgid, va_list *args)
 {
   set_line_table_global ();
 
@@ -1220,13 +1616,14 @@ diagnostic_manager::emit (diagnostic &diag, const char *msgid, va_list *args)
   {
     m_dc.begin_group ();
 
-    diagnostic_info info;
+    diagnostics::diagnostic_info info;
 GCC_DIAGNOSTIC_PUSH_IGNORED(-Wsuggest-attribute=format)
     diagnostic_set_info (&info, msgid, args, diag.get_rich_location (),
-			 diagnostic_t_from_diagnostic_level (diag.get_level ()));
+			 diagnostics_kind_from_diagnostic_level
+			   (diag.get_level ()));
 GCC_DIAGNOSTIC_POP
-    info.metadata = diag.get_metadata ();
-    info.x_data = &diag;
+    info.m_metadata = diag.get_metadata ();
+    info.m_x_data = &diag;
     diagnostic_report_diagnostic (&m_dc, &info);
 
     m_dc.end_group ();
@@ -1234,12 +1631,30 @@ GCC_DIAGNOSTIC_POP
 
   rich_location *rich_loc = diag.get_rich_location ();
   if (rich_loc->fixits_can_be_auto_applied_p ())
-    m_edit_context->add_fixits (rich_loc);
+    m_change_set->add_fixits (rich_loc);
 
   m_prev_diag_logical_loc = diag.get_logical_location ();
   m_current_diag = nullptr;
 }
 
+void
+diagnostic_manager::emit (diagnostic &diag, const char *msgid, ...)
+{
+  va_list args;
+  va_start (args, msgid);
+  emit_va (diag, msgid, &args);
+  va_end (args);
+}
+
+void
+diagnostic_manager::emit_msg_buf (diagnostic &diag,
+				  diagnostic_message_buffer &msg_buf)
+{
+
+  pp_element_message_buffer e_msg_buf (msg_buf);
+  emit (diag, "%e", &e_msg_buf);
+}
+
 diagnostic_execution_path *
 diagnostic_manager::new_execution_path ()
 {
@@ -1248,13 +1663,36 @@ diagnostic_manager::new_execution_path ()
   return new diagnostic_execution_path (*mgr);
 }
 
+void
+diagnostic_manager::take_global_graph (std::unique_ptr<diagnostic_graph> graph)
+{
+  class prebuilt_lazy_digraph : public lazily_created<diagnostics::digraphs::digraph>
+  {
+  public:
+    prebuilt_lazy_digraph (std::unique_ptr<diagnostic_graph> graph)
+      : m_graph (std::move (graph))
+    {
+    }
+
+    std::unique_ptr<diagnostics::digraphs::digraph>
+    create_object () const final override
+    {
+      return std::move (m_graph);
+    }
+
+  private:
+    mutable std::unique_ptr<diagnostic_graph> m_graph;
+  };
+
+  m_dc.report_global_digraph (prebuilt_lazy_digraph (std::move (graph)));
+}
 /* Error-checking at the API boundary.  */
 
 #define FAIL_IF_NULL(PTR_ARG) \
-  do {						    \
-    volatile const void *p = (PTR_ARG);		    \
-    if (!p) {					    \
-      fprintf (stderr, "%s: %s must be non-NULL\n",   \
+  do {							    \
+    volatile const void *ptr_arg = (PTR_ARG);		    \
+    if (!ptr_arg) {					    \
+      fprintf (stderr, "%s: %s must be non-NULL\n",	    \
 	       __func__, #PTR_ARG);		      \
       abort ();					      \
     }						    \
@@ -1388,7 +1826,7 @@ diagnostic_manager_add_sarif_sink (diagnostic_manager *diag_mgr,
   FAIL_IF_NULL (dst_stream);
   FAIL_IF_NULL (main_input_file);
 
-  sarif_generation_options sarif_gen_opts;
+  diagnostics::sarif_generation_options sarif_gen_opts;
   switch (version)
     {
     default:
@@ -1396,10 +1834,11 @@ diagnostic_manager_add_sarif_sink (diagnostic_manager *diag_mgr,
 	       __func__, (int)version);
       abort ();
     case DIAGNOSTIC_SARIF_VERSION_2_1_0:
-      sarif_gen_opts.m_version = sarif_version::v2_1_0;
+      sarif_gen_opts.m_version = diagnostics::sarif_version::v2_1_0;
       break;
     case DIAGNOSTIC_SARIF_VERSION_2_2_PRERELEASE:
-      sarif_gen_opts.m_version = sarif_version::v2_2_prerelease_2024_08_08;
+      sarif_gen_opts.m_version
+	= diagnostics::sarif_version::v2_2_prerelease_2024_08_08;
       break;
     }
 
@@ -1529,12 +1968,12 @@ diagnostic_manager_debug_dump_location (const diagnostic_manager *diag_mgr,
       diag_mgr->set_line_table_global ();
       const expanded_location exp_loc (expand_location (cpplib_loc));
 
-      diagnostic_context dc;
+      diagnostics::context dc;
       diagnostic_initialize (&dc, 0);
       dc.m_show_column = true;
 
-      diagnostic_text_output_format text_format (dc);
-      label_text loc_text = text_format.get_location_text (exp_loc);
+      diagnostics::text_sink text_output (dc);
+      label_text loc_text = text_output.get_location_text (exp_loc);
       fprintf (out, "%s", loc_text.get ());
 
       diagnostic_finish (&dc);
@@ -1873,10 +2312,12 @@ diagnostic_execution_path_add_event (diagnostic_execution_path *path,
 
   va_list args;
   va_start (args, gmsgid);
-  diagnostic_event_id_t result = path->add_event_va (physical_loc,
-						     logical_loc,
-						     stack_depth,
-						     gmsgid, &args);
+  diagnostics::paths::event_id_t result
+    = path->add_event_va (physical_loc,
+			  logical_loc,
+			  stack_depth,
+			  nullptr,
+			  gmsgid, &args);
   va_end (args);
 
   return as_diagnostic_event_id (result);
@@ -1895,10 +2336,12 @@ diagnostic_execution_path_add_event_va (diagnostic_execution_path *path,
   FAIL_IF_NULL (path);
   FAIL_IF_NULL (gmsgid);
 
-  diagnostic_event_id_t result = path->add_event_va (physical_loc,
-						     logical_loc,
-						     stack_depth,
-						     gmsgid, args);
+  diagnostics::paths::event_id_t result
+    = path->add_event_va (physical_loc,
+			  logical_loc,
+			  stack_depth,
+			  nullptr,
+			  gmsgid, args);
   return as_diagnostic_event_id (result);
 }
 
@@ -1928,7 +2371,7 @@ diagnostic_finish_va (diagnostic *diag, const char *gmsgid, va_list *args)
   else
     progname = "progname";
   auto_diagnostic_group d;
-  diag->get_manager ().emit (*diag, gmsgid, args);
+  diag->get_manager ().emit_va (*diag, gmsgid, args);
   delete diag;
 }
 
@@ -1987,7 +2430,7 @@ diagnostic_logical_location_get_decorated_name (const diagnostic_logical_locatio
 
 namespace {
 
-struct spec_context : public diagnostics_output_spec::context
+struct spec_context : public diagnostics::output_spec::context
 {
 public:
   spec_context (const char *option_name,
@@ -2048,3 +2491,484 @@ diagnostic_manager_set_analysis_target (diagnostic_manager *mgr,
 
   mgr->get_dc ().set_main_input_filename (file->get_name ());
 }
+
+/* Public entrypoint.  */
+
+diagnostic_node *
+diagnostic_graph::add_node_with_id (std::string node_id,
+				    diagnostic_node *parent_node)
+{
+  auto node_up
+    = std::make_unique<diagnostic_node> (*this, std::move (node_id));
+  diagnostic_node *new_node = node_up.get ();
+  if (parent_node)
+    parent_node->add_child (std::move (node_up));
+  else
+    add_node (std::move (node_up));
+  return new_node;
+}
+
+/* Public entrypoint.  */
+
+diagnostic_edge *
+diagnostic_graph::add_edge_with_label (const char *edge_id,
+				       diagnostic_node &src_node,
+				       diagnostic_node &dst_node,
+				       const char *label)
+{
+  auto edge_up
+    = std::make_unique<diagnostic_edge> (*this, edge_id,
+					 src_node, dst_node);
+  diagnostic_edge *new_edge = edge_up.get ();
+  if (label)
+    new_edge->set_label (label);
+  add_edge (std::move (edge_up));
+  return new_edge;
+}
+
+/* Public entrypoint.  */
+
+diagnostic_graph *
+diagnostic_manager_new_graph (diagnostic_manager *manager)
+{
+  FAIL_IF_NULL (manager);
+
+  return new diagnostic_graph (*manager);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_manager_take_global_graph (diagnostic_manager *manager,
+				      diagnostic_graph *graph)
+{
+  FAIL_IF_NULL (manager);
+  FAIL_IF_NULL (graph);
+
+  manager->take_global_graph (std::unique_ptr<diagnostic_graph> (graph));
+}
+
+void
+diagnostic_take_graph (diagnostic *diag,
+		       diagnostic_graph *graph)
+{
+  FAIL_IF_NULL (diag);
+  FAIL_IF_NULL (graph);
+
+  diag->take_graph (std::unique_ptr<diagnostic_graph> (graph));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_graph_release (diagnostic_graph *graph)
+{
+  delete graph;
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_graph_set_description (diagnostic_graph *graph,
+				  const char *desc)
+{
+  FAIL_IF_NULL (graph);
+
+  graph->set_description (desc);
+}
+
+/* Public entrypoint.  */
+
+diagnostic_node *
+diagnostic_graph_add_node (diagnostic_graph *graph,
+			   const char *node_id,
+			   diagnostic_node *parent_node)
+{
+  FAIL_IF_NULL (graph);
+  FAIL_IF_NULL (node_id);
+
+  return graph->add_node_with_id (node_id, parent_node);
+}
+
+/* Public entrypoint.  */
+
+diagnostic_edge *
+diagnostic_graph_add_edge (diagnostic_graph *graph,
+			   const char *edge_id,
+			   diagnostic_node *src_node,
+			   diagnostic_node *dst_node,
+			   const char *label)
+{
+  FAIL_IF_NULL (graph);
+  FAIL_IF_NULL (src_node);
+  FAIL_IF_NULL (dst_node);
+
+  return graph->add_edge_with_label (edge_id, *src_node, *dst_node, label);
+}
+
+/* Public entrypoint.  */
+
+diagnostic_node *
+diagnostic_graph_get_node_by_id (diagnostic_graph *graph,
+				 const char *node_id)
+{
+  FAIL_IF_NULL (graph);
+  FAIL_IF_NULL (node_id);
+
+  return static_cast<diagnostic_node *> (graph->get_node_by_id (node_id));
+}
+
+/* Public entrypoint.  */
+
+diagnostic_edge *
+diagnostic_graph_get_edge_by_id (diagnostic_graph *graph,
+				 const char *edge_id)
+{
+  FAIL_IF_NULL (graph);
+  FAIL_IF_NULL (edge_id);
+
+  return static_cast<diagnostic_edge *> (graph->get_edge_by_id (edge_id));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_node_set_location (diagnostic_node *node,
+			      const diagnostic_physical_location *loc)
+{
+  FAIL_IF_NULL (node);
+
+  node->set_physical_loc (loc ? loc->m_inner : UNKNOWN_LOCATION);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_node_set_label (diagnostic_node *node,
+			   const char *label)
+{
+  FAIL_IF_NULL (node);
+
+  node->set_label (label);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_node_set_logical_location (diagnostic_node *node,
+				      const diagnostic_logical_location *logical_loc)
+{
+  FAIL_IF_NULL (node);
+
+  node->set_logical_loc
+    (impl_logical_location_manager::key_from_ptr (logical_loc));
+}
+
+/* Private entrypoint.  */
+
+void
+private_diagnostic_graph_set_property_bag (diagnostic_graph &graph,
+					  std::unique_ptr<json::object> properties)
+{
+  graph.set_property_bag (std::move (properties));
+}
+
+/* Private entrypoint.  */
+
+void
+private_diagnostic_node_set_property_bag (diagnostic_node &node,
+					  std::unique_ptr<json::object> properties)
+{
+  node.set_property_bag (std::move (properties));
+}
+
+/* Private entrypoint.  */
+
+void
+private_diagnostic_edge_set_property_bag (diagnostic_edge &edge,
+					  std::unique_ptr<json::object> properties)
+{
+  edge.set_property_bag (std::move (properties));
+}
+
+/* Public entrypoint.  */
+
+diagnostic_message_buffer *
+diagnostic_message_buffer_new ()
+{
+  return new diagnostic_message_buffer ();
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_release (diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (msg_buf);
+  delete msg_buf;
+}
+
+void
+diagnostic_message_buffer_append_str (diagnostic_message_buffer *msg_buf,
+				      const char *p)
+{
+  FAIL_IF_NULL (msg_buf);
+  FAIL_IF_NULL (p);
+  msg_buf->m_tokens.push_back_text (label_text::take (xstrdup (p)));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_append_text (diagnostic_message_buffer *msg_buf,
+				       const char *p,
+				       size_t len)
+{
+  FAIL_IF_NULL (msg_buf);
+  FAIL_IF_NULL (p);
+  msg_buf->m_tokens.push_back_text (label_text::take (xstrndup (p, len)));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_append_byte (diagnostic_message_buffer *msg_buf,
+				       char ch)
+{
+  FAIL_IF_NULL (msg_buf);
+  msg_buf->m_tokens.push_back_byte (ch);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_append_printf (diagnostic_message_buffer *msg_buf,
+					 const char *fmt, ...)
+{
+  FAIL_IF_NULL (msg_buf);
+  FAIL_IF_NULL (fmt);
+
+  va_list args;
+  va_start (args, fmt);
+
+  char *formatted_buf = xvasprintf (fmt, args);
+
+  va_end (args);
+
+  msg_buf->m_tokens.push_back_text (label_text::take (formatted_buf));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_append_event_id (diagnostic_message_buffer *msg_buf,
+					   diagnostic_event_id event_id)
+{
+  FAIL_IF_NULL (msg_buf);
+  msg_buf->m_tokens.push_back<pp_token_event_id> (event_id);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_begin_url (diagnostic_message_buffer *msg_buf,
+				     const char *url)
+{
+  FAIL_IF_NULL (msg_buf);
+  FAIL_IF_NULL (url);
+  msg_buf->m_tokens.push_back<pp_token_begin_url>
+    (label_text::take (xstrdup (url)));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_end_url (diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (msg_buf);
+  msg_buf->m_tokens.push_back<pp_token_end_url> ();
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_begin_quote (diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (msg_buf);
+  msg_buf->m_tokens.push_back<pp_token_begin_quote> ();
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_end_quote (diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (msg_buf);
+  msg_buf->m_tokens.push_back<pp_token_end_quote> ();
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_begin_color (diagnostic_message_buffer *msg_buf,
+				       const char *color)
+{
+  FAIL_IF_NULL (msg_buf);
+  FAIL_IF_NULL (color);
+  msg_buf->m_tokens.push_back<pp_token_begin_color>
+    (label_text::take (xstrdup (color)));
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_end_color (diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (msg_buf);
+  msg_buf->m_tokens.push_back<pp_token_end_color> ();
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_message_buffer_dump (const diagnostic_message_buffer *msg_buf,
+				FILE *outf)
+{
+  FAIL_IF_NULL (msg_buf);
+  FAIL_IF_NULL (outf);
+
+  msg_buf->m_tokens.dump (outf);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_finish_via_msg_buf (diagnostic *diag,
+			       diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (diag);
+  FAIL_IF_NULL (msg_buf);
+
+  if (const char *tool_name
+      = diag->get_manager ().get_client_version_info ()->m_name.get_str ())
+    progname = tool_name;
+  else
+    progname = "progname";
+  auto_diagnostic_group d;
+  diag->get_manager ().emit_msg_buf (*diag, *msg_buf);
+  delete diag;
+  delete msg_buf;
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_add_location_with_label_via_msg_buf (diagnostic *diag,
+						const diagnostic_physical_location *loc,
+						diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (diag);
+  diag->get_manager ().assert_valid_diagnostic_physical_location (loc);
+  FAIL_IF_NULL (msg_buf);
+
+  std::unique_ptr<diagnostic_message_buffer> msg_buf_up (msg_buf);
+  diag->add_location_with_label (loc, std::move (msg_buf_up));
+}
+
+/* Public entrypoint.  */
+
+diagnostic_event_id
+diagnostic_execution_path_add_event_via_msg_buf (diagnostic_execution_path *path,
+						 const diagnostic_physical_location *physical_loc,
+						 const diagnostic_logical_location *logical_loc,
+						 unsigned stack_depth,
+						 diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (path);
+  FAIL_IF_NULL (msg_buf);
+
+  std::unique_ptr<diagnostic_message_buffer> msg_buf_up (msg_buf);
+  diagnostic_event_id_t result
+    = path->add_event_via_msg_buf (physical_loc,
+				   logical_loc,
+				   stack_depth,
+				   nullptr,
+				   std::move (msg_buf_up));
+  return as_diagnostic_event_id (result);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_graph_set_description_via_msg_buf (diagnostic_graph *graph,
+					      diagnostic_message_buffer *desc)
+{
+  FAIL_IF_NULL (graph);
+
+  if (desc)
+    graph->set_description (desc->to_string ());
+  else
+    graph->set_description (nullptr);
+}
+
+/* Public entrypoint.  */
+
+diagnostic_edge *
+diagnostic_graph_add_edge_via_msg_buf (diagnostic_graph *graph,
+				       const char *edge_id,
+				       diagnostic_node *src_node,
+				       diagnostic_node *dst_node,
+				       diagnostic_message_buffer *label)
+{
+  FAIL_IF_NULL (graph);
+  FAIL_IF_NULL (src_node);
+  FAIL_IF_NULL (dst_node);
+
+  if (label)
+    {
+      std::string label_str (label->to_string ());
+      return graph->add_edge_with_label (edge_id, *src_node, *dst_node,
+					 label_str.c_str ());
+    }
+  else
+    return graph->add_edge_with_label (edge_id, *src_node, *dst_node,
+				       nullptr);
+}
+
+/* Public entrypoint.  */
+
+void
+diagnostic_node_set_label_via_msg_buf (diagnostic_node *node,
+				       diagnostic_message_buffer *label)
+{
+  FAIL_IF_NULL (node);
+
+  if (label)
+    node->set_label (label->to_string ());
+  else
+    node->set_label (nullptr);
+}
+
+/* Private entrypoint.  */
+
+diagnostic_event_id
+private_diagnostic_execution_path_add_event_3 (diagnostic_execution_path *path,
+					       const diagnostic_physical_location *physical_loc,
+					       const diagnostic_logical_location *logical_loc,
+					       unsigned stack_depth,
+					       diagnostic_graph *state_graph,
+					       diagnostic_message_buffer *msg_buf)
+{
+  FAIL_IF_NULL (path);
+  FAIL_IF_NULL (msg_buf);
+
+  diagnostic_event_id_t result
+    = path->add_event_via_msg_buf
+	(physical_loc,
+	 logical_loc,
+	 stack_depth,
+	 std::unique_ptr <diagnostic_graph> (state_graph),
+	 std::unique_ptr <diagnostic_message_buffer> (msg_buf));
+
+  return as_diagnostic_event_id (result);
+}
diff --git a/gcc/libgdiagnostics.h b/gcc/libgdiagnostics.h
index 9af2747..c202feb 100644
--- a/gcc/libgdiagnostics.h
+++ b/gcc/libgdiagnostics.h
@@ -50,6 +50,13 @@ extern "C" {
 #define LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL(ARG_NUM)
   /* empty; for the human reader */
 
+# if (LIBGDIAGNOSTICS_GCC_VERSION >= 4001)
+#  define LIBGDIAGNOSTICS_PARAM_FORMAT_STRING(FMT_KIND, FMT_ARG_NUM, ARGS_ARG_NUM) \
+     __attribute__ ((__format__ (FMT_KIND, FMT_ARG_NUM, ARGS_ARG_NUM)))
+# else
+#  define LIBGDIAGNOSTICS_PARAM_FORMAT_STRING(FMT_KIND, FMT_ARG_NUM, ARGS_ARG_NUM)
+# endif /* GNUC >= 4.1 */
+
 #define LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING(FMT_ARG_NUM, ARGS_ARG_NUM) \
   LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (FMT_ARG_NUM)
   /* In theory we'd also add
@@ -59,6 +66,10 @@ extern "C" {
      of -Wall but undocumented, and much fussier than I'd want to inflict
      on users of libgdiagnostics.  */
 
+#define LIBGDIAGNOSTICS_PARAM_PRINTF_FORMAT_STRING(FMT_ARG_NUM, ARGS_ARG_NUM) \
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (FMT_ARG_NUM) \
+  LIBGDIAGNOSTICS_PARAM_FORMAT_STRING(gnu_printf, FMT_ARG_NUM, ARGS_ARG_NUM)
+
 /**********************************************************************
  Data structures and types.
  All structs within the API are opaque.
@@ -230,6 +241,8 @@ enum diagnostic_level
 typedef struct diagnostic_execution_path diagnostic_execution_path;
 typedef int diagnostic_event_id;
 
+typedef struct diagnostic_message_buffer diagnostic_message_buffer;
+
 /**********************************************************************
  API entrypoints.
  **********************************************************************/
@@ -765,6 +778,343 @@ diagnostic_manager_set_analysis_target (diagnostic_manager *mgr,
   LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
   LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
 
+/* Directed graphs.  */
+
+typedef struct diagnostic_graph diagnostic_graph;
+typedef struct diagnostic_node diagnostic_node;
+typedef struct diagnostic_edge diagnostic_edge;
+
+/* Create a new graph.
+   This is owned by the caller and must have one of
+   diagnostic_manager_take_global_graph, diagnostic_take_graph,
+   or diagnostic_graph_release called on it.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern diagnostic_graph *
+diagnostic_manager_new_graph (diagnostic_manager *manager)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Report this graph "globally", taking ownership of it.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_manager_take_global_graph (diagnostic_manager *manager,
+				      diagnostic_graph *graph)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* Add this graph to DIAG, transferring ownership to it.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_take_graph (diagnostic *diag,
+		      diagnostic_graph *graph)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* Release this graph.  Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_graph_release (diagnostic_graph *graph)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (1);
+
+/* Set the description of GRAPH for use
+   in the value of the SARIF "description" property
+   (SARIF v2.1.0 section 3.39.2).
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_graph_set_description (diagnostic_graph *graph,
+				 const char *description)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2);
+
+/* Create and add a new node within GRAPH.
+   NODE_ID must be unique within nodes in GRAPH.
+   The new node is owned by GRAPH.
+   PARENT_NODE can be NULL (for a top-level node in the graph),
+   or non-null for a child node.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern diagnostic_node *
+diagnostic_graph_add_node (diagnostic_graph *graph,
+			   const char *node_id,
+			   diagnostic_node *parent_node)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (3);
+
+/* Create and add a new edge within GRAPH.
+
+   If non-null, then EDGE_ID must be unique within edges in GRAPH;
+   if EDGE_ID is null then a unique id of the form "edge0", "edge1", etc
+   will be used automatically.
+
+   The new edge is owned by GRAPH.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern diagnostic_edge *
+diagnostic_graph_add_edge (diagnostic_graph *graph,
+			   const char *edge_id,
+			   diagnostic_node *src_node,
+			   diagnostic_node *dst_node,
+			   const char *label)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (3)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (4)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (5);
+
+/* Get the node in GRAPH with the given id, or null.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern diagnostic_node *
+diagnostic_graph_get_node_by_id (diagnostic_graph *graph,
+				 const char *node_id)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* Get the edge in GRAPH with the given id, or null.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern diagnostic_edge *
+diagnostic_graph_get_edge_by_id (diagnostic_graph *graph,
+				 const char *edge_id)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* Set the label of NODE for use
+   in the value of the SARIF "label" property
+   (SARIF v2.1.0 section 3.40.3).
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_node_set_label (diagnostic_node *node,
+			   const char *label)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2);
+
+/* Set the physical location of NODE.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_node_set_location (diagnostic_node *node,
+			      const diagnostic_physical_location *loc)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2);
+
+/* Set the logical location of NODE.
+   Added in LIBGDIAGNOSTICS_ABI_3.  */
+
+extern void
+diagnostic_node_set_logical_location (diagnostic_node *node,
+				      const diagnostic_logical_location *logical_loc)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2);
+
+/* Message buffers.  */
+
+#define LIBDIAGNOSTICS_HAVE_diagnostic_message_buffer
+
+/* Create a new diagnostic_message_buffer.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern diagnostic_message_buffer *
+diagnostic_message_buffer_new (void);
+
+/* Release a diagnostic_message_buffer that hasn't been used.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_release (diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Append a UTF-8 encoded null-terminated string to the buffer.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_append_str (diagnostic_message_buffer *msg_buf,
+				      const char *p)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* Append a UTF-8 encoded run of bytes to the buffer.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_append_text (diagnostic_message_buffer *msg_buf,
+				       const char *p,
+				       size_t len)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* Append a byte to to the buffer.  This should be either
+   ASCII, or part of UTF-8 encoded text.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_append_byte (diagnostic_message_buffer *msg_buf,
+				       char ch)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Append a formatted string to the buffer, using the formatting rules
+   for "printf".
+   The string is assumed to be UTF-8 encoded.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_append_printf (diagnostic_message_buffer *msg_buf,
+					 const char *fmt, ...)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_PRINTF_FORMAT_STRING (2, 3);
+
+/* Append a diagnostic_event_id to the buffer in the form "(1)".
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_append_event_id (diagnostic_message_buffer *msg_buf,
+					   diagnostic_event_id event_id)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Begin a run of text associated with the given URL.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_begin_url (diagnostic_message_buffer *msg_buf,
+				     const char *url)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* End a run of text started with diagnostic_message_buffer_begin_url.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_end_url (diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Begin a run of text to be printed in quotes.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_begin_quote (diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* End a run of text started with diagnostic_message_buffer_begin_quote.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_end_quote (diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Begin a run of text to be printed with color.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_begin_color (diagnostic_message_buffer *msg_buf,
+				       const char *color)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* End a run of text started with diagnostic_message_buffer_begin_color.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_end_color (diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1);
+
+/* Write a debugging representation of MSG_BUG to OUTF.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_message_buffer_dump (const diagnostic_message_buffer *msg_buf,
+				FILE *outf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* As diagnostic_finish, but takes ownership of MSG_BUF.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_finish_via_msg_buf (diagnostic *diag,
+			       diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (2);
+
+/* As diagnostic_add_location_with_label but takes ownership of MSG_BUF.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_add_location_with_label_via_msg_buf (diagnostic *diag,
+						const diagnostic_physical_location *loc,
+						diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (3);
+
+/* As diagnostic_execution_path_add_event but takes ownership of MSG_BUF.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern diagnostic_event_id
+diagnostic_execution_path_add_event_via_msg_buf (diagnostic_execution_path *path,
+						 const diagnostic_physical_location *physical_loc,
+						 const diagnostic_logical_location *logical_loc,
+						 unsigned stack_depth,
+						 diagnostic_message_buffer *msg_buf)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (3)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (5);
+
+/* Set the description of GRAPH for use
+   in the value of the SARIF "description" property
+   (SARIF v2.1.0 section 3.39.2).
+
+   Takes ownership of DESC, if non-null.
+
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_graph_set_description_via_msg_buf (diagnostic_graph *graph,
+					      diagnostic_message_buffer *desc)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2);
+
+/* Create and add a new edge within GRAPH.
+
+   If non-null, then EDGE_ID must be unique within edges in GRAPH;
+   if EDGE_ID is null then a unique id of the form "edge0", "edge1", etc
+   will be used automatically.
+
+   Takes ownership of LABEL, if non-null.
+
+   The new edge is owned by GRAPH.
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern diagnostic_edge *
+diagnostic_graph_add_edge_via_msg_buf (diagnostic_graph *graph,
+				       const char *edge_id,
+				       diagnostic_node *src_node,
+				       diagnostic_node *dst_node,
+				       diagnostic_message_buffer *label)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (3)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (4)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (5);
+
+/* Set the label of NODE for use
+   in the value of the SARIF "label" property
+   (SARIF v2.1.0 section 3.40.3).
+
+   Takes ownership of LABEL, if non-null.
+
+   Added in LIBGDIAGNOSTICS_ABI_4.  */
+
+extern void
+diagnostic_node_set_label_via_msg_buf (diagnostic_node *node,
+				       diagnostic_message_buffer *label)
+  LIBGDIAGNOSTICS_PARAM_MUST_BE_NON_NULL (1)
+  LIBGDIAGNOSTICS_PARAM_CAN_BE_NULL (2);
+
 /* DEFERRED:
    - thread-safety
    - plural forms
@@ -772,6 +1122,7 @@ diagnostic_manager_set_analysis_target (diagnostic_manager *mgr,
    - locations within binary files
    - options and URLs for warnings
    - enable/disable of warnings by kind
+   - command-line arguments
    - plugin metadata.  */
 
 #ifdef __cplusplus
diff --git a/gcc/libgdiagnostics.map b/gcc/libgdiagnostics.map
index 49cabca..91f3951 100644
--- a/gcc/libgdiagnostics.map
+++ b/gcc/libgdiagnostics.map
@@ -69,6 +69,7 @@ LIBGDIAGNOSTICS_ABI_0
     diagnostic_finish;
     diagnostic_finish_va;
 
+
     diagnostic_physical_location_get_file;
 
   local: *;
@@ -90,3 +91,53 @@ LIBGDIAGNOSTICS_ABI_2 {
     diagnostic_manager_add_sink_from_spec;
     diagnostic_manager_set_analysis_target;
 } LIBGDIAGNOSTICS_ABI_1;
+
+# Add hooks needed for diagnostic_graph support.
+LIBGDIAGNOSTICS_ABI_3 {
+  global:
+    diagnostic_manager_new_graph;
+    diagnostic_manager_take_global_graph;
+    diagnostic_take_graph;
+    diagnostic_graph_release;
+    diagnostic_graph_set_description;
+    diagnostic_graph_add_node;
+    diagnostic_graph_add_edge;
+    diagnostic_graph_get_node_by_id;
+    diagnostic_graph_get_edge_by_id;
+    diagnostic_node_set_label;
+    diagnostic_node_set_location;
+    diagnostic_node_set_logical_location;
+
+    # Private hooks used by sarif-replay
+    private_diagnostic_graph_set_property_bag;
+    private_diagnostic_node_set_property_bag;
+    private_diagnostic_edge_set_property_bag;
+} LIBGDIAGNOSTICS_ABI_2;
+
+# Add diagnostic_message_buffer
+LIBGDIAGNOSTICS_ABI_4 {
+  global:
+    diagnostic_message_buffer_new;
+    diagnostic_message_buffer_release;
+    diagnostic_message_buffer_append_str;
+    diagnostic_message_buffer_append_text;
+    diagnostic_message_buffer_append_byte;
+    diagnostic_message_buffer_append_printf;
+    diagnostic_message_buffer_append_event_id;
+    diagnostic_message_buffer_begin_url;
+    diagnostic_message_buffer_end_url;
+    diagnostic_message_buffer_begin_quote;
+    diagnostic_message_buffer_end_quote;
+    diagnostic_message_buffer_begin_color;
+    diagnostic_message_buffer_end_color;
+    diagnostic_message_buffer_dump;
+    diagnostic_finish_via_msg_buf;
+    diagnostic_add_location_with_label_via_msg_buf;
+    diagnostic_execution_path_add_event_via_msg_buf;
+    diagnostic_graph_set_description_via_msg_buf;
+    diagnostic_graph_add_edge_via_msg_buf;
+    diagnostic_node_set_label_via_msg_buf;
+
+    # Private hook used by sarif-replay
+    private_diagnostic_execution_path_add_event_3;
+} LIBGDIAGNOSTICS_ABI_3;
diff --git a/gcc/libsarifreplay.cc b/gcc/libsarifreplay.cc
index 2d6c394..1e4a74f 100644
--- a/gcc/libsarifreplay.cc
+++ b/gcc/libsarifreplay.cc
@@ -26,6 +26,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "libgdiagnostics++.h"
+#include "libgdiagnostics-private.h"
 #include "json-parsing.h"
 #include "intl.h"
 #include "sarif-spec-urls.def"
@@ -281,16 +282,18 @@ class annotation
 {
 public:
   annotation (libgdiagnostics::physical_location phys_loc,
-	      label_text label)
+	      libgdiagnostics::message_buffer label)
   : m_phys_loc (phys_loc),
     m_label (std::move (label))
   {
   }
 
   libgdiagnostics::physical_location m_phys_loc;
-  label_text m_label;
+  libgdiagnostics::message_buffer m_label;
 };
 
+using id_map = std::map<std::string, const json::string *>;
+
 class sarif_replayer
 {
 public:
@@ -330,7 +333,7 @@ private:
 
   enum status emit_sarif_as_diagnostics (const json::value &jv);
 
-  label_text
+  libgdiagnostics::message_buffer
   make_plain_text_within_result_message (const json::object *tool_component_obj,
 					 const json::object &message_obj,
 					 const json::object *rule_obj);
@@ -413,6 +416,30 @@ private:
 				      const json::object &run_obj,
 				      libgdiagnostics::execution_path &out);
 
+  // "graph" object (§3.39)
+  enum status
+  handle_graph_object (const json::object &graph_obj,
+		       const json::object &run_obj,
+		       libgdiagnostics::graph &out);
+  // "node" object (§3.40)
+  libgdiagnostics::node
+  handle_node_object (const json::object &node_obj,
+		      const json::object &run_obj,
+		      libgdiagnostics::graph &graph,
+		      libgdiagnostics::node parent_node,
+		      id_map &node_id_map);
+
+  // "edge" object (§3.41)
+  libgdiagnostics::edge
+  handle_edge_object (const json::object &edge_obj,
+		      libgdiagnostics::graph &graph,
+		      id_map &edge_id_map);
+
+  libgdiagnostics::node
+  get_graph_node_by_id_property (const json::object &edge_json_object,
+				 const property_spec_ref &id_prop,
+				 libgdiagnostics::graph &graph);
+
   // reportingDescriptor lookup (§3.52.3)
   const json::object *
   lookup_rule_by_id_in_tool (const char *rule_id,
@@ -446,7 +473,7 @@ private:
   {
     va_list ap;
     va_start (ap, gmsgid);
-    report_problem (jv, ref, gmsgid, &ap, DIAGNOSTIC_LEVEL_ERROR);
+    report_problem (jv, &ref, gmsgid, &ap, DIAGNOSTIC_LEVEL_ERROR);
     va_end (ap);
     return status::err_invalid_sarif;
   }
@@ -462,14 +489,25 @@ private:
   {
     va_list ap;
     va_start (ap, gmsgid);
-    report_problem (jv, ref, gmsgid, &ap, DIAGNOSTIC_LEVEL_SORRY);
+    report_problem (jv, &ref, gmsgid, &ap, DIAGNOSTIC_LEVEL_SORRY);
     va_end (ap);
     return status::err_unhandled_sarif;
   }
 
   void
+  report_note (const json::value &jv,
+	       const char *gmsgid, ...)
+    LIBGDIAGNOSTICS_PARAM_GCC_FORMAT_STRING (3, 4)
+  {
+    va_list ap;
+    va_start (ap, gmsgid);
+    report_problem (jv, nullptr, gmsgid, &ap, DIAGNOSTIC_LEVEL_NOTE);
+    va_end (ap);
+  }
+
+  void
   report_problem (const json::value &jv,
-		  const spec_ref &ref,
+		  const spec_ref *ref,
 		  const char *gmsgid,
 		  va_list *args,
 		  enum diagnostic_level level)
@@ -481,11 +519,14 @@ private:
        There doesn't seem to be a systematic mapping from spec sections to
        HTML anchors, so we can't provide URLs
        (filed as https://github.com/oasis-tcs/sarif-spec/issues/533 ).  */
-    char *ref_desc = ref.make_description ();
-    char *ref_url = ref.make_url ();
-    diag.add_rule (ref_desc, ref_url);
-    free (ref_desc);
-    free (ref_url);
+    if (ref)
+      {
+	char *ref_desc = ref->make_description ();
+	char *ref_url = ref->make_url ();
+	diag.add_rule (ref_desc, ref_url);
+	free (ref_desc);
+	free (ref_url);
+      }
 
     auto loc_range
       = make_physical_location (m_control_mgr,
@@ -651,6 +692,38 @@ private:
 			      const string_property_value<ValueType> *value_arr,
 			      size_t num_values);
 
+  const json::object *
+  maybe_get_property_bag (const json::object &obj)
+  {
+    const property_spec_ref properties
+      ("object", "properties", "3.8.1");
+    return get_optional_property<json::object> (obj, properties);
+  }
+
+  /* Look for a property bag within OBJ.
+     If found, look for a property within it named PROPERTY_NAME
+     of the given type.
+     If successful, return the property's value.
+     Otherwise, return nullptr without complaining (unless the property bag
+     is itself not an object).  */
+  template <typename JsonType>
+  const JsonType *
+  maybe_get_property_bag_value (const json::object &obj,
+				const char *property_name)
+  {
+    auto property_bag_obj = maybe_get_property_bag (obj);
+    if (!property_bag_obj)
+      return nullptr;
+    const json::value *property_val = property_bag_obj->get (property_name);
+    if (!property_val)
+      return nullptr;
+    const JsonType *sub = dyn_cast<const JsonType *> (property_val);
+    if (!sub)
+      /* Property is wrong kind of value.  Don't treat this as an error.  */
+      return nullptr;
+    return sub;
+  }
+
   /* The manager to replay the SARIF files to.  */
   libgdiagnostics::manager m_output_mgr;
 
@@ -929,6 +1002,31 @@ sarif_replayer::handle_run_obj (const json::object &run_obj)
 	break;
       }
 
+  // §3.14.20 "graphs"
+  const property_spec_ref prop_graphs ("run", "graphs", "3.14.20");
+  if (const json::array *graphs
+      = get_optional_property<json::array> (run_obj,
+					    prop_graphs))
+    {
+      for (auto element : *graphs)
+	{
+	  if (const json::object *graph_json_obj
+	      = require_object_for_element (*element, prop_graphs))
+	    {
+	      libgdiagnostics::graph graph;
+	      enum status s = handle_graph_object (*graph_json_obj,
+						   run_obj,
+						   graph);
+	      if (s != status::ok)
+		return s;
+
+	      m_output_mgr.take_global_graph (std::move (graph));
+	    }
+	  else
+	    return status::err_invalid_sarif;
+      }
+    }
+
   return status::ok;
 }
 
@@ -1083,12 +1181,12 @@ sarif_replayer::get_level_from_level_str (const json::string &level_str)
 
 static void
 add_any_annotations (libgdiagnostics::diagnostic &diag,
-		     const std::vector<annotation> &annotations)
+		     std::vector<annotation> &annotations)
 {
   for (auto &annotation : annotations)
-    if (annotation.m_label.get ())
+    if (annotation.m_label.m_inner)
       diag.add_location_with_label (annotation.m_phys_loc,
-				    annotation.m_label.get ());
+				    std::move (annotation.m_label));
     else
       diag.add_location (annotation.m_phys_loc);
 }
@@ -1151,13 +1249,13 @@ sarif_replayer::handle_result_obj (const json::object &result_obj,
     }
 
   // §3.27.11 "message" property
-  label_text text;
+  libgdiagnostics::message_buffer msg_buf;
   if (auto message_obj
 	= get_optional_property<json::object> (result_obj, PROP_result_message))
-    text = make_plain_text_within_result_message (nullptr, // TODO: tool_component_obj,
-						  *message_obj,
-						  rule_obj);
-  if (!text.get ())
+    msg_buf = make_plain_text_within_result_message (nullptr, // TODO: tool_component_obj,
+						     *message_obj,
+						     rule_obj);
+  if (!msg_buf.m_inner)
     return status::err_invalid_sarif;
 
   // §3.27.12 "locations" property
@@ -1244,8 +1342,34 @@ sarif_replayer::handle_result_obj (const json::object &result_obj,
   if (path.m_inner)
     err.take_execution_path (std::move (path));
 
+  // §3.27.19 "graphs" property
+  const property_spec_ref prop_graphs ("result", "graphs", "3.27.19");
+  if (const json::array *graphs
+      = get_optional_property<json::array> (result_obj,
+					    prop_graphs))
+    {
+      for (auto element : *graphs)
+	{
+	  if (const json::object *graph_json_obj
+	      = require_object_for_element (*element, prop_graphs))
+	    {
+	      libgdiagnostics::graph graph;
+	      enum status s = handle_graph_object (*graph_json_obj,
+						   run_obj,
+						   graph);
+	      if (s != status::ok)
+		return s;
+
+	      err.take_graph (std::move (graph));
+	    }
+	  else
+	    return status::err_invalid_sarif;
+      }
+    }
+
   // §3.27.22 relatedLocations property
-  std::vector<std::pair<libgdiagnostics::diagnostic, label_text>> notes;
+  std::vector<std::pair<libgdiagnostics::diagnostic,
+			libgdiagnostics::message_buffer>> notes;
   const property_spec_ref prop_related_locations
     ("result", "relatedLocations", "3.27.22");
   if (auto related_locations_arr
@@ -1277,18 +1401,18 @@ sarif_replayer::handle_result_obj (const json::object &result_obj,
 						     prop_message))
 	    {
 	      /* Treat related locations with a message as a "note".  */
-	      label_text text
+	      libgdiagnostics::message_buffer msg_buf
 		(make_plain_text_within_result_message
 		 (tool_component_obj,
 		  *message_obj,
 		  rule_obj));
-	      if (!text.get ())
+	      if (!msg_buf.m_inner)
 		return status::err_invalid_sarif;
 	      auto note (m_output_mgr.begin_diagnostic (DIAGNOSTIC_LEVEL_NOTE));
 	      note.set_location (physical_loc);
 	      note.set_logical_location (logical_loc);
 	      add_any_annotations (note, annotations);
-	      notes.push_back ({std::move (note), std::move (text)});
+	      notes.push_back ({std::move (note), std::move (msg_buf)});
 	    }
 	  else
 	    {
@@ -1311,14 +1435,14 @@ sarif_replayer::handle_result_obj (const json::object &result_obj,
 	  handle_fix_object (err, *fix_obj);
     }
 
-  err.finish ("%s", text.get ());
+  err.finish_via_msg_buf (std::move (msg_buf));
 
   // Flush any notes
   for (auto &iter : notes)
     {
       auto &note = iter.first;
-      auto &text = iter.second;
-      note.finish ("%s", text.get ());
+      auto &msg_buf = iter.second;
+      note.finish_via_msg_buf (std::move (msg_buf));
     }
 
   return status::ok;
@@ -1446,13 +1570,10 @@ maybe_consume_embedded_link (const char *&iter_src)
    and substitute for any placeholders (§3.11.5) and handle any
    embedded links (§3.11.6).
 
-   Limitations:
-   - we don't preserve destinations within embedded links
-
    MESSAGE_OBJ is "theMessage"
    RULE_OBJ is "theRule".  */
 
-label_text
+libgdiagnostics::message_buffer
 sarif_replayer::
 make_plain_text_within_result_message (const json::object *tool_component_obj,
 				       const json::object &message_obj,
@@ -1465,7 +1586,7 @@ make_plain_text_within_result_message (const json::object *tool_component_obj,
 					       rule_obj,
 					       js_str);
   if (!original_text)
-    return label_text::borrow (nullptr);
+    return libgdiagnostics::message_buffer ();
 
   gcc_assert (js_str);
 
@@ -1475,7 +1596,7 @@ make_plain_text_within_result_message (const json::object *tool_component_obj,
     = get_optional_property<json::array> (message_obj, arguments_prop);
 
   /* Duplicate original_text, substituting any placeholders.  */
-  std::string accum;
+  libgdiagnostics::message_buffer result (diagnostic_message_buffer_new ());
 
   const char *iter_src = original_text;
   while (char ch = *iter_src)
@@ -1491,7 +1612,7 @@ make_plain_text_within_result_message (const json::object *tool_component_obj,
 		 " but message object has no %qs property",
 		 (int)arg_idx,
 		 arguments_prop.get_property_name ());
-	      return label_text::borrow (nullptr);
+	      return libgdiagnostics::message_buffer ();
 	    }
 	  if (arg_idx >= arguments->length ())
 	    {
@@ -1502,20 +1623,20 @@ make_plain_text_within_result_message (const json::object *tool_component_obj,
 		 arguments_prop.get_property_name (),
 		 (int)arg_idx);
 	      // TODO: might be nice to add a note showing the args
-	      return label_text::borrow (nullptr);
+	      return libgdiagnostics::message_buffer ();
 	    }
 	  auto replacement_jstr
 	    = require_string (*arguments->get (arg_idx), arguments_prop);
 	  if (!replacement_jstr)
-	    return label_text::borrow (nullptr);
-	  accum += replacement_jstr->get_string ();
+	    return libgdiagnostics::message_buffer ();
+	  result += replacement_jstr->get_string ();
 	}
       else if (ch == '{' || ch == '}')
 	{
 	  /* '{' and '}' are escaped by repeating them.  */
 	  if (iter_src[1] == ch)
 	    {
-	      accum += ch;
+	      result += ch;
 	      iter_src += 2;
 	    }
 	  else
@@ -1525,24 +1646,25 @@ make_plain_text_within_result_message (const json::object *tool_component_obj,
 	      report_invalid_sarif (*js_str, msgs_with_placeholders,
 				    "unescaped '%c' within message string",
 				    ch);
-	      return label_text::borrow (nullptr);
+	      return libgdiagnostics::message_buffer ();
 	    }
 	}
       else if (auto link = maybe_consume_embedded_link (iter_src))
 	{
-	  accum += link->text;
-	  /* TODO: use the destination.  */
+	  result.begin_url (link->destination.c_str ());
+	  result += link->text.c_str ();
+	  result.end_url ();
 	  /* TODO: potentially could try to convert
 	     intra-sarif links into event ids.  */
 	}
       else
 	{
-	  accum += ch;
+	  result += ch;
 	  iter_src++;
 	}
     }
 
-  return label_text::take (xstrdup (accum.c_str ()));
+  return result;
 }
 
 /* Handle a value that should be a multiformatMessageString object (§3.12).
@@ -1676,7 +1798,7 @@ handle_thread_flow_location_object (const json::object &tflow_loc_obj,
 {
   libgdiagnostics::physical_location physical_loc;
   libgdiagnostics::logical_location logical_loc;
-  label_text message;
+  libgdiagnostics::message_buffer msg_buf;
   int stack_depth = 0;
 
   const property_spec_ref location_prop
@@ -1698,7 +1820,7 @@ handle_thread_flow_location_object (const json::object &tflow_loc_obj,
 	  = get_optional_property<json::object> (*location_obj,
 						 location_message))
 	{
-	  message = make_plain_text_within_result_message
+	  msg_buf = make_plain_text_within_result_message
 	    (nullptr,
 	     *message_obj,
 	     nullptr/* TODO.  */);
@@ -1740,16 +1862,29 @@ handle_thread_flow_location_object (const json::object &tflow_loc_obj,
 	}
     }
 
-  if (message.get ())
-    path.add_event (physical_loc,
-		    logical_loc,
-		    stack_depth,
-		    "%s", message.get ());
-  else
-    path.add_event (physical_loc,
-		    logical_loc,
-		    stack_depth,
-		    "");
+  libgdiagnostics::graph state_graph;
+  if (auto sarif_state_graph
+	= maybe_get_property_bag_value<json::object> (tflow_loc_obj,
+						      "gcc/diagnostics/paths/event/state_graph"))
+    {
+      enum status s
+	= handle_graph_object (*sarif_state_graph, run_obj, state_graph);
+      if (s != status::ok)
+	return s;
+    }
+
+  if (!msg_buf.m_inner)
+    msg_buf.m_inner = diagnostic_message_buffer_new ();
+
+  private_diagnostic_execution_path_add_event_3 (path.m_inner,
+						 physical_loc.m_inner,
+						 logical_loc.m_inner,
+						 stack_depth,
+						 state_graph.m_inner,
+						 msg_buf.m_inner);
+
+  state_graph.m_owned = false;
+  msg_buf.m_inner = nullptr;
 
   return status::ok;
 }
@@ -1828,7 +1963,7 @@ handle_location_object (const json::object &location_obj,
 	  if (s != status::ok)
 	    return s;
 
-	  label_text label;
+	  libgdiagnostics::message_buffer label;
 
 	  // §3.30.14 message property
 	  {
@@ -2142,6 +2277,270 @@ handle_logical_location_object (const json::object &logical_loc_obj,
   return status::ok;
 }
 
+// "graph" object (§3.39)
+
+enum status
+sarif_replayer::handle_graph_object (const json::object &graph_json_obj,
+				     const json::object &run_obj,
+				     libgdiagnostics::graph &out_graph)
+{
+  out_graph = libgdiagnostics::graph
+		(diagnostic_manager_new_graph (m_output_mgr.m_inner));
+
+  id_map node_id_map;
+  id_map edge_id_map;
+
+  if (auto properties = maybe_get_property_bag (graph_json_obj))
+    private_diagnostic_graph_set_property_bag (*out_graph.m_inner,
+					       properties->clone_as_object ());
+
+  // §3.39.2: MAY contain a "description" property
+  const property_spec_ref description_prop
+    ("graph", "description", "3.39.2");
+  if (auto description_obj
+      = get_optional_property<json::object> (graph_json_obj, description_prop))
+    {
+      auto msg_buf
+	= make_plain_text_within_result_message (&run_obj,
+						 *description_obj,
+						 nullptr);
+      if (!msg_buf.m_inner)
+	return status::err_invalid_sarif;
+      out_graph.set_description (std::move (msg_buf));
+    }
+
+  // §3.39.3: MAY contain a "nodes" property
+  const property_spec_ref nodes_prop
+    ("graph", "nodes", "3.39.3");
+  if (auto nodes_arr
+      = get_optional_property<json::array> (graph_json_obj, nodes_prop))
+    {
+      for (auto element : *nodes_arr)
+	{
+	  const json::object *node_json_obj
+	    = require_object_for_element (*element, nodes_prop);
+	  if (!node_json_obj)
+	    return status::err_invalid_sarif;
+	  libgdiagnostics::node node
+	    = handle_node_object (*node_json_obj, run_obj, out_graph,
+				  nullptr, node_id_map);
+	  if (node.m_inner == nullptr)
+	    return status::err_invalid_sarif;
+	}
+    }
+  else
+    // If we have no nodes, we can't handle the edges.
+    return status::ok;
+
+  // §3.39.4 edges property
+  const property_spec_ref edges_prop
+    ("graph", "edges", "3.39.4");
+  if (auto edges_arr
+      = get_optional_property<json::array> (graph_json_obj, edges_prop))
+    {
+      for (auto element : *edges_arr)
+	{
+	  const json::object *edge_json_obj
+	    = require_object_for_element (*element, edges_prop);
+	  if (!edge_json_obj)
+	    return status::err_invalid_sarif;
+	  libgdiagnostics::edge edge
+	    = handle_edge_object (*edge_json_obj, out_graph, edge_id_map);
+	  if (edge.m_inner == nullptr)
+	    return status::err_invalid_sarif;
+	}
+    }
+
+  return status::ok;
+}
+
+// "node" object (§3.40)
+
+libgdiagnostics::node
+sarif_replayer::handle_node_object (const json::object &node_json_obj,
+				    const json::object &run_obj,
+				    libgdiagnostics::graph &graph,
+				    libgdiagnostics::node parent_node,
+				    id_map &node_id_map)
+{
+  // §3.40.2 "id" property
+  const property_spec_ref id_prop ("node", "id", "3.40.2");
+  auto id_str = get_required_property<json::string> (node_json_obj, id_prop);
+  if (!id_str)
+    return nullptr;
+  const char *id = id_str->get_string ();
+  if (diagnostic_graph_get_node_by_id (graph.m_inner, id))
+    {
+      // Duplicate id; fail:
+      libgdiagnostics::group g (m_control_mgr);
+      report_invalid_sarif (*id_str,
+			    id_prop,
+			    "duplicate node id %qs within graph",
+			    id);
+      gcc_assert (node_id_map[id]);
+      report_note (*node_id_map[id],
+		   "%qs already used as node id within graph here",
+		   id);
+      return nullptr;
+    }
+  node_id_map[id] = id_str;
+
+  libgdiagnostics::node new_node
+    = libgdiagnostics::node (diagnostic_graph_add_node (graph.m_inner,
+							id,
+							parent_node.m_inner));
+  if (auto properties = maybe_get_property_bag (node_json_obj))
+    private_diagnostic_node_set_property_bag (*new_node.m_inner,
+					      properties->clone_as_object ());
+
+  // §3.40.3 "label" property
+  const property_spec_ref label_prop
+    ("node", "label", "3.40.3");
+  if (auto label_obj
+      = get_optional_property<json::object> (node_json_obj, label_prop))
+    {
+      auto msg_buf
+	= make_plain_text_within_result_message (&run_obj,
+						 *label_obj,
+						 nullptr);
+      if (!msg_buf.m_inner)
+	return nullptr;
+      new_node.set_label (std::move (msg_buf));
+    }
+
+  // §3.40.4 "location" property
+  const property_spec_ref location_prop ("node", "location", "3.40.4");
+  if (auto location_json_obj
+      = get_optional_property<json::object> (node_json_obj, location_prop))
+    {
+      libgdiagnostics::physical_location physical_loc;
+      libgdiagnostics::logical_location logical_loc;
+      enum status s = handle_location_object (*location_json_obj,
+					      run_obj,
+					      physical_loc,
+					      logical_loc,
+					      nullptr); // annotations
+      if (s != status::ok)
+	return nullptr;
+
+      new_node.set_location (physical_loc);
+      new_node.set_logical_location (logical_loc);
+    }
+
+  // §3.40.5: MAY contain a "children" property
+  const property_spec_ref children_prop
+    ("graph", "children", "3.40.5");
+  if (auto children_json_arr
+      = get_optional_property<json::array> (node_json_obj, children_prop))
+    {
+      for (auto json_element : *children_json_arr)
+	{
+	  const json::object *child_json_obj
+	    = require_object_for_element (*json_element, children_prop);
+	  if (!child_json_obj)
+	    return nullptr;
+	  libgdiagnostics::node child_node
+	    = handle_node_object (*child_json_obj, run_obj, graph,
+				  new_node, node_id_map);
+	  if (child_node.m_inner == nullptr)
+	    return nullptr;
+	}
+    }
+
+  return new_node;
+}
+
+// "edge" object (§3.41)
+
+libgdiagnostics::edge
+sarif_replayer::handle_edge_object (const json::object &edge_json_obj,
+				    libgdiagnostics::graph &graph,
+				    id_map &edge_id_map)
+{
+  // §3.41.2 "id" property
+  const property_spec_ref id_prop ("edge", "id", "3.41.2");
+  auto id_str = get_required_property<json::string> (edge_json_obj, id_prop);
+  if (!id_str)
+    return nullptr;
+  const char *id = id_str->get_string ();
+  if (diagnostic_graph_get_edge_by_id (graph.m_inner, id))
+    {
+      // Duplicate id; fail:
+      libgdiagnostics::group g (m_control_mgr);
+      report_invalid_sarif (*id_str,
+			    id_prop,
+			    "duplicate edge id %qs within graph",
+			    id);
+      gcc_assert (edge_id_map[id]);
+      report_note (*edge_id_map[id],
+		   "%qs already used as edge id within graph here",
+		   id);
+      return nullptr;
+    }
+  edge_id_map[id] = id_str;
+
+  // §3.41.3 "label" property
+  libgdiagnostics::message_buffer label;
+  const property_spec_ref label_prop
+    ("edge", "label", "3.41.3");
+  if (auto label_obj
+      = get_optional_property<json::object> (edge_json_obj, label_prop))
+    {
+      label = make_plain_text_within_result_message (nullptr,
+						     *label_obj,
+						     nullptr);
+      if (!label.m_inner)
+	return nullptr;
+    }
+
+  // §3.41.4 "sourceNodeId" property
+  const property_spec_ref src_id_prop ("edge", "sourceNodeId", "3.41.4");
+  auto src_node = get_graph_node_by_id_property (edge_json_obj,
+						 src_id_prop,
+						 graph);
+  if (!src_node.m_inner)
+    return nullptr;
+
+  // §3.41.5 "targetNodeId" property
+  const property_spec_ref dst_id_prop ("edge", "targetNodeId", "3.41.5");
+  auto dst_node = get_graph_node_by_id_property (edge_json_obj,
+						 dst_id_prop,
+						 graph);
+  if (!dst_node.m_inner)
+    return nullptr;
+
+  auto result = graph.add_edge (id, src_node, dst_node, std::move (label));
+
+  if (auto properties = maybe_get_property_bag (edge_json_obj))
+    private_diagnostic_edge_set_property_bag (*result.m_inner,
+					      properties->clone_as_object ());
+
+  return result;
+}
+
+libgdiagnostics::node
+sarif_replayer::
+get_graph_node_by_id_property (const json::object &edge_json_obj,
+			       const property_spec_ref &id_prop,
+			       libgdiagnostics::graph &graph)
+{
+  auto id_str = get_required_property<json::string> (edge_json_obj, id_prop);
+  if (!id_str)
+    return nullptr;
+  const char *id = id_str->get_string ();
+  auto node = graph.get_node_by_id (id);
+  if (!node.m_inner)
+    {
+      // id not found; complain:
+      report_invalid_sarif (*id_str,
+			    id_prop,
+			    "no node with id %qs in graph",
+			    id);
+      return nullptr;
+    }
+  return node;
+}
+
 // 3.52.3 reportingDescriptor lookup
 // "For an example of the interaction between ruleId and rule.id, see §3.52.4."
 
diff --git a/gcc/loop-unroll.cc b/gcc/loop-unroll.cc
index 6149cec..c80a6cb 100644
--- a/gcc/loop-unroll.cc
+++ b/gcc/loop-unroll.cc
@@ -1185,7 +1185,7 @@ decide_unroll_stupid (class loop *loop, int flags)
 
   /* Do not unroll loops with branches inside -- it increases number
      of mispredicts.
-     TODO: this heuristic needs tunning; call inside the loop body
+     TODO: this heuristic needs tuning; call inside the loop body
      is also relatively good reason to not unroll.  */
   if (num_loop_branches (loop) > 1)
     {
diff --git a/gcc/lra-constraints.cc b/gcc/lra-constraints.cc
index 68aaf86..83f8fda 100644
--- a/gcc/lra-constraints.cc
+++ b/gcc/lra-constraints.cc
@@ -114,6 +114,7 @@
 #include "target.h"
 #include "rtl.h"
 #include "tree.h"
+#include "stmt.h"
 #include "predict.h"
 #include "df.h"
 #include "memmodel.h"
@@ -2168,6 +2169,7 @@ process_alt_operands (int only_alternative)
   bool costly_p;
   enum reg_class cl;
   const HARD_REG_SET *cl_filter;
+  HARD_REG_SET hard_reg_constraint;
 
   /* Calculate some data common for all alternatives to speed up the
      function.	*/
@@ -2416,14 +2418,15 @@ process_alt_operands (int only_alternative)
 			if (curr_static_id->operand[nop].type == OP_INOUT
 			    || curr_static_id->operand[m].type == OP_INOUT)
 			  break;
-			/* Operands don't match.  If the operands are
-			   different user defined explicit hard
+			/* Operands don't match.  For asm if the operands
+			   are different user defined explicit hard
 			   registers, then we cannot make them match
 			   when one is early clobber operand.  */
 			if ((REG_P (*curr_id->operand_loc[nop])
 			     || SUBREG_P (*curr_id->operand_loc[nop]))
 			    && (REG_P (*curr_id->operand_loc[m])
-				|| SUBREG_P (*curr_id->operand_loc[m])))
+				|| SUBREG_P (*curr_id->operand_loc[m]))
+			    && INSN_CODE (curr_insn) < 0)
 			  {
 			    rtx nop_reg = *curr_id->operand_loc[nop];
 			    if (SUBREG_P (nop_reg))
@@ -2544,6 +2547,17 @@ process_alt_operands (int only_alternative)
 		  cl_filter = nullptr;
 		  goto reg;
 
+		case '{':
+		    {
+		      int regno = decode_hard_reg_constraint (p);
+		      gcc_assert (regno >= 0);
+		      cl = REGNO_REG_CLASS (regno);
+		      CLEAR_HARD_REG_SET (hard_reg_constraint);
+		      SET_HARD_REG_BIT (hard_reg_constraint, regno);
+		      cl_filter = &hard_reg_constraint;
+		      goto reg;
+		    }
+
 		default:
 		  cn = lookup_constraint (p);
 		  switch (get_constraint_type (cn))
@@ -3328,19 +3342,15 @@ process_alt_operands (int only_alternative)
 		  first_conflict_j = j;
 		last_conflict_j = j;
 		/* Both the earlyclobber operand and conflicting operand
-		   cannot both be user defined hard registers.  */
+		   cannot both be user defined hard registers for asm.
+		   Let curr_insn_transform diagnose it.  */
 		if (HARD_REGISTER_P (operand_reg[i])
 		    && REG_USERVAR_P (operand_reg[i])
 		    && operand_reg[j] != NULL_RTX
 		    && HARD_REGISTER_P (operand_reg[j])
-		    && REG_USERVAR_P (operand_reg[j]))
-		  {
-		    /* For asm, let curr_insn_transform diagnose it.  */
-		    if (INSN_CODE (curr_insn) < 0)
+		    && REG_USERVAR_P (operand_reg[j])
+		    && INSN_CODE (curr_insn) < 0)
 		      return false;
-		    fatal_insn ("unable to generate reloads for "
-				"impossible constraints:", curr_insn);
-		  }
 	      }
 	  if (last_conflict_j < 0)
 	    continue;
@@ -3930,6 +3940,16 @@ process_address_1 (int nop, bool check_only_p,
       enum reg_class cl;
       rtx set;
       rtx_insn *insns, *last_insn;
+
+      cl = base_reg_class (ad.mode, ad.as, ad.base_outer_code,
+			   get_index_code (&ad), curr_insn);
+
+      if (REG_P (*ad.base_term)
+	  && ira_class_subset_p[get_reg_class (REGNO (*ad.base_term))][cl])
+	/* It seems base reg is already in the base reg class and changing it
+	   does not make a progress.  So reload the whole inner address.  */
+	goto reload_inner_addr;
+
       /* Try to reload base into register only if the base is invalid
          for the address but with valid offset, case (4) above.  */
       start_sequence ();
@@ -3975,8 +3995,6 @@ process_address_1 (int nop, bool check_only_p,
 	    {
 	      *ad.base_term = XEXP (SET_SRC (set), 0);
 	      *ad.disp_term = XEXP (SET_SRC (set), 1);
-	      cl = base_reg_class (ad.mode, ad.as, ad.base_outer_code,
-				   get_index_code (&ad), curr_insn);
 	      regno = REGNO (*ad.base_term);
 	      if (regno >= FIRST_PSEUDO_REGISTER
 		  && cl != lra_get_allocno_class (regno))
@@ -4019,11 +4037,11 @@ process_address_1 (int nop, bool check_only_p,
     }
   else
     {
-      enum reg_class cl = base_reg_class (ad.mode, ad.as,
-					  SCRATCH, SCRATCH,
-					  curr_insn);
-      rtx addr = *ad.inner;
-
+      enum reg_class cl;
+      rtx addr;
+    reload_inner_addr:
+      cl = base_reg_class (ad.mode, ad.as, SCRATCH, SCRATCH, curr_insn);
+      addr = *ad.inner;
       new_reg = lra_create_new_reg (Pmode, NULL_RTX, cl, NULL, "addr");
       /* addr => new_base.  */
       lra_emit_move (new_reg, addr);
@@ -4044,14 +4062,21 @@ process_address (int nop, bool check_only_p,
 		 rtx_insn **before, rtx_insn **after)
 {
   bool res = false;
-
-  while (process_address_1 (nop, check_only_p, before, after))
+  /* Use enough iterations to process all address parts:  */
+  for (int i = 0; i < 10; i++)
     {
-      if (check_only_p)
-	return true;
-      res = true;
+      if (!process_address_1 (nop, check_only_p, before, after))
+	{
+	  return res;
+	}
+      else
+	{
+	  if (check_only_p)
+	    return true;
+	  res = true;
+	}
     }
-  return res;
+  fatal_insn ("unable to reload address in ", curr_insn);
 }
 
 /* Override the generic address_reload_context in order to
diff --git a/gcc/lto-cgraph.cc b/gcc/lto-cgraph.cc
index ec34f65..0af2e88 100644
--- a/gcc/lto-cgraph.cc
+++ b/gcc/lto-cgraph.cc
@@ -718,11 +718,12 @@ output_profile_summary (struct lto_simple_output_block *ob)
 {
   if (profile_info)
     {
-      /* We do not output num and run_max, they are not used by
-         GCC profile feedback and they are difficult to merge from multiple
-         units.  */
       unsigned runs = (profile_info->runs);
       streamer_write_uhwi_stream (ob->main_stream, runs);
+      streamer_write_gcov_count_stream (ob->main_stream,
+					profile_info->sum_max);
+      streamer_write_gcov_count_stream (ob->main_stream,
+					profile_info->cutoff);
 
       /* IPA-profile computes hot bb threshold based on cumulated
 	 whole program profile.  We need to stream it down to ltrans.  */
@@ -1678,6 +1679,8 @@ input_profile_summary (class lto_input_block *ib,
   if (runs)
     {
       file_data->profile_info.runs = runs;
+      file_data->profile_info.sum_max = streamer_read_gcov_count (ib);
+      file_data->profile_info.cutoff = streamer_read_gcov_count (ib);
 
       /* IPA-profile computes hot bb threshold based on cumulated
 	 whole program profile.  We need to stream it down to ltrans.  */
@@ -1719,6 +1722,8 @@ merge_profile_summaries (struct lto_file_decl_data **file_data_vec)
 
   profile_info = XCNEW (gcov_summary);
   profile_info->runs = max_runs;
+  profile_info->sum_max = 0;
+  profile_info->cutoff = 0;
 
   /* If merging already happent at WPA time, we are done.  */
   if (flag_ltrans)
@@ -1735,6 +1740,14 @@ merge_profile_summaries (struct lto_file_decl_data **file_data_vec)
 
 	scale = RDIV (node->count_materialization_scale * max_runs,
                       node->lto_file_data->profile_info.runs);
+	gcov_type sum_max = RDIV (node->lto_file_data->profile_info.sum_max * max_runs,
+				  node->lto_file_data->profile_info.runs);
+	gcov_type cutoff = RDIV (node->lto_file_data->profile_info.cutoff * max_runs,
+				 node->lto_file_data->profile_info.runs);
+	if (sum_max > profile_info->sum_max)
+	  profile_info->sum_max = sum_max;
+	if (cutoff > profile_info->cutoff)
+	  profile_info->cutoff = cutoff;
 	node->count_materialization_scale = scale;
 	if (scale < 0)
 	  fatal_error (input_location, "Profile information in %s corrupted",
diff --git a/gcc/lto-wrapper.cc b/gcc/lto-wrapper.cc
index 27405e8..cb293f2 100644
--- a/gcc/lto-wrapper.cc
+++ b/gcc/lto-wrapper.cc
@@ -2274,13 +2274,13 @@ public:
   : gcc_diagnostic_option_manager (0 /* lang_mask */)
   {
   }
-  int option_enabled_p (diagnostic_option_id) const final override
+  int option_enabled_p (diagnostics::option_id) const final override
   {
     return true;
   }
-  char *make_option_name (diagnostic_option_id,
-			  diagnostic_t,
-			  diagnostic_t) const final override
+  char *make_option_name (diagnostics::option_id,
+			  enum diagnostics::kind,
+			  enum diagnostics::kind) const final override
   {
     return nullptr;
   }
diff --git a/gcc/m2/ChangeLog b/gcc/m2/ChangeLog
index f7254f9..ad10605 100644
--- a/gcc/m2/ChangeLog
+++ b/gcc/m2/ChangeLog
@@ -1,3 +1,135 @@
+2025-08-01  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121354
+	* gm2-compiler/M2GenGCC.mod (FoldHigh): Rewrite.
+	(IsUnboundedArray): New procedure function.
+
+2025-07-31  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121314
+	* mc-boot/GFormatStrings.cc (PerformFormatString): Rebuilt.
+	* mc-boot/GM2EXCEPTION.cc (M2EXCEPTION_M2Exception): Rebuilt.
+	* mc-boot/GSFIO.cc (SFIO_GetFileName): Rebuilt.
+	* mc-boot/GSFIO.h (SFIO_GetFileName): Rebuilt.
+	* mc-boot/Gdecl.cc: Rebuilt.
+	* mc-boot/GmcFileName.h: Rebuilt.
+	* mc/decl.mod (getStringChar): New procedure function.
+	(getStringContents): Call getStringChar.
+	(addQuotes): New procedure function.
+	(foldBinary): Call addQuotes to add delimiting quotes
+	to the new string.
+
+2025-07-29  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	* gm2-compiler/M2GenGCC.mod (FoldBecomes): Remove all
+	local variables.
+	(CodeIndrX): Remove length.
+	Remove newstr.
+	* gm2-compiler/M2Range.mod (FoldTypeIndrX): Remove desType.
+
+2025-07-29  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121289
+	* gm2-compiler/M2Students.def (CheckVariableAgainstKeyword): New
+	parameter tok.
+	* gm2-compiler/M2Students.mod (CheckVariableAgainstKeyword): New
+	parameter tok.
+	Pass tok to PerformVariableKeywordCheck.
+	(PerformVariableKeywordCheck): New parameter tok.
+	Pass tok to MetaErrorStringT0.
+	* gm2-compiler/P2SymBuild.mod (BuildVariable): Pass tok to
+	CheckVariableAgainstKeyword.
+	* gm2-libs-iso/LowLong.mod (except): Replace with ...
+	(exceptSrc): ... this.
+	* gm2-libs-iso/LowReal.mod (except): Replace with ...
+	(exceptSrc): ... this.
+	* gm2-libs-iso/LowShort.mod (except): Replace with ...
+	(exceptSrc): ... this.
+	* gm2-libs-iso/Processes.mod (Wait): Replace from with fromCor.
+	* gm2-libs-iso/RndFile.mod (EndPos): Replace end with endP.
+	* gm2-libs/SCmdArgs.mod (GetArg): Replace start with startPos.
+	Replace end with endPos.
+	(NArg): Replace start with startPos.
+	Replace end with endPos.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gm2-gcc/m2linemap.cc: Update usage of "diagnostic_info" to
+	explicitly refer to "diagnostics::diagnostic_info".
+	* gm2-gcc/rtegraph.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gm2-gcc/m2linemap.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* gm2-gcc/rtegraph.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gm2-gcc/m2color.cc: Update for move of diagnostic-color.h to
+	diagnostics/color.h.
+
+2025-07-23  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	* gm2-libs/M2EXCEPTION.mod (M2Exception): Add return
+	exException in case Raise completes.
+
+2025-07-23  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121164
+	* gm2-compiler/P1SymBuild.mod: Remove PutProcTypeParam.
+	Remove PutProcTypeParam.
+	(CheckFileName): Remove.
+	(P1EndBuildDefinitionModule): Correct spelling.
+	(P1EndBuildImplementationModule): Ditto.
+	(P1EndBuildProgramModule): Ditto.
+	(EndBuildInnerModule): Ditto.
+	* gm2-compiler/P2SymBuild.mod (P2EndBuildDefModule): Correct
+	spelling.
+	(P2EndBuildImplementationModule): Ditto.
+	(P2EndBuildProgramModule): Ditto.
+	(EndBuildInnerModule): Ditto.
+	(CheckFormalParameterSection): Ditto.
+	* gm2-compiler/P3SymBuild.mod (P3EndBuildDefModule): Ditto.
+	* gm2-compiler/PCSymBuild.mod (PCEndBuildDefModule): Ditto.
+	(fixupProcedureType): Pass tok to PutProcTypeVarParam.
+	Pass tok to PutProcTypeParam.
+	* gm2-compiler/SymbolTable.def (PutProcTypeParam): Add tok
+	parameter.
+	(PutProcTypeVarParam): Ditto.
+	* gm2-compiler/SymbolTable.mod (SymParam): At change type to
+	CARDINAL.
+	New field FullTok.
+	New field Scope.
+	(SymVarParam): At change type to CARDINAL.
+	New field FullTok.
+	New field Scope.
+	(GetVarDeclTok): Check ShadowVar for NulSym and return At.
+	(PutParam): Initialize FullTok.
+	Initialize At.
+	Initialize Scope.
+	(PutVarParam): Initialize FullTok.
+	Assign At.
+	Initialize Scope.
+	(AddProcedureProcTypeParam): Add tok parameter.
+	(GetScope): Add ParamSym and VarParamSym clause.
+	(PutProcTypeVarParam): Add tok parameter.
+	Initialize At.
+	Initialize FullTok.
+	(GetDeclaredDefinition): Clause ParamSym return At.
+	Clause VarParamSym return At.
+	(GetDeclaredModule): Ditto.
+	(PutDeclaredDefinition): Remove clause ParamSym.
+	Remove clause VarParamSym.
+	(PutDeclaredModule): Remove clause ParamSym.
+	Remove clause VarParamSym.
+
+2025-07-18  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121164
+	* gm2-libs/ARRAYOFCHAR.mod (Write): Rename 2nd parameter
+	name a to str.
+
 2025-07-01  Gaius Mulley  <gaiusmod2@gmail.com>
 
 	PR modula2/120912
diff --git a/gcc/m2/gm2-compiler/M2GenGCC.mod b/gcc/m2/gm2-compiler/M2GenGCC.mod
index 4a9ced3..2440b2a 100644
--- a/gcc/m2/gm2-compiler/M2GenGCC.mod
+++ b/gcc/m2/gm2-compiler/M2GenGCC.mod
@@ -2903,9 +2903,6 @@ END CheckStop ;
 *)
 
 PROCEDURE FoldBecomes (p: WalkAction; bb: BasicBlock; quad: CARDINAL) ;
-VAR
-   op            : QuadOperator ;
-   des, op2, expr: CARDINAL ;
 BEGIN
    IF DeclaredOperandsBecomes (p, quad)
    THEN
@@ -6442,37 +6439,52 @@ END ResolveHigh ;
 
 
 (*
+   IsUnboundedArray - return TRUE if symbol is an unbounded array.
+*)
+
+PROCEDURE IsUnboundedArray (sym: CARDINAL) : BOOLEAN ;
+BEGIN
+   IF IsParameter (sym) OR IsVar (sym)
+   THEN
+      RETURN IsUnbounded (GetType (sym))
+   END ;
+   RETURN FALSE
+END IsUnboundedArray ;
+
+
+(*
    FoldHigh - if the array is not dynamic then we should be able to
               remove the HighOp quadruple and assign op1 with
-              the known compile time HIGH(op3).
+              the known compile time HIGH(array).
 *)
 
 PROCEDURE FoldHigh (tokenno: CARDINAL; p: WalkAction;
-                    quad: CARDINAL; op1, dim, op3: CARDINAL) ;
+                    quad: CARDINAL; op1, dim, array: CARDINAL) ;
 VAR
    t       : tree ;
    location: location_t ;
 BEGIN
-   (* firstly ensure that any constant literal is declared *)
-   TryDeclareConstant(tokenno, op3) ;
-   location := TokenToLocation(tokenno) ;
-   IF GccKnowsAbout(op3) AND CompletelyResolved(op3)
+   (* Firstly ensure that any constant literal is declared.  *)
+   TryDeclareConstant (tokenno, array) ;
+   location := TokenToLocation (tokenno) ;
+   IF (NOT IsUnboundedArray (array)) AND
+      GccKnowsAbout (array) AND CompletelyResolved (array)
    THEN
-      t := ResolveHigh(tokenno, dim, op3) ;
-      (* fine, we can take advantage of this and fold constants *)
-      IF IsConst(op1) AND (t#tree(NIL))
+      t := ResolveHigh (tokenno, dim, array) ;
+      (* We can take advantage of this and fold constants.  *)
+      IF IsConst (op1) AND (t # tree (NIL))
       THEN
-         PutConst(op1, Cardinal) ;
-         AddModGcc(op1,
-                   DeclareKnownConstant(location, GetCardinalType(),
-                                        ToCardinal(location, t))) ;
-         p(op1) ;
+         PutConst (op1, Cardinal) ;
+         AddModGcc (op1,
+                    DeclareKnownConstant (location, GetCardinalType (),
+                                          ToCardinal (location, t))) ;
+         p (op1) ;
          NoChange := FALSE ;
-         SubQuad(quad)
+         SubQuad (quad)
       ELSE
-         (* we can still fold the expression, but not the assignment, however, we will
-            not do this here but in CodeHigh
-         *)
+         (* We can still fold the expression but not the assignment,
+            we will not do this here but in CodeHigh when the result
+            can be stored.  *)
       END
    END
 END FoldHigh ;
@@ -8154,8 +8166,6 @@ VAR
    rightpos,
    typepos,
    indrxpos        : CARDINAL ;
-   length,
-   newstr          : tree ;
    location        : location_t ;
 BEGIN
    GetQuadOtok (quad, indrxpos, op, left, type, right,
diff --git a/gcc/m2/gm2-compiler/M2Range.mod b/gcc/m2/gm2-compiler/M2Range.mod
index dcac2ba..f1516d3 100644
--- a/gcc/m2/gm2-compiler/M2Range.mod
+++ b/gcc/m2/gm2-compiler/M2Range.mod
@@ -1869,14 +1869,12 @@ END FoldTypeAssign ;
 
 PROCEDURE FoldTypeIndrX (q: CARDINAL; tokenNo: CARDINAL; des, expr: CARDINAL; r: CARDINAL) ;
 VAR
-   desType,
    exprType: CARDINAL ;
 BEGIN
    (* Need to skip over a variable or temporary in des and expr so
       long as expr is not a procedure.  In the case of des = *expr,
       both expr and des will be variables due to the property of
       indirection.  *)
-   desType := GetType (des) ;
    IF IsProcedure (expr)
    THEN
       (* Must not GetType for a procedure as it gives the return type.  *)
diff --git a/gcc/m2/gm2-compiler/M2Students.def b/gcc/m2/gm2-compiler/M2Students.def
index 7d67a0a..a3ecdcd 100644
--- a/gcc/m2/gm2-compiler/M2Students.def
+++ b/gcc/m2/gm2-compiler/M2Students.def
@@ -39,7 +39,7 @@ EXPORT QUALIFIED StudentVariableCheck, CheckVariableAgainstKeyword ;
                                  as a keyword except for its case.
 *)
 
-PROCEDURE CheckVariableAgainstKeyword (name: Name) ;
+PROCEDURE CheckVariableAgainstKeyword (tok: CARDINAL; name: Name) ;
 
 
 (*
diff --git a/gcc/m2/gm2-compiler/M2Students.mod b/gcc/m2/gm2-compiler/M2Students.mod
index e539eb0..3df160a 100644
--- a/gcc/m2/gm2-compiler/M2Students.mod
+++ b/gcc/m2/gm2-compiler/M2Students.mod
@@ -25,7 +25,7 @@ IMPLEMENTATION MODULE M2Students ;
 FROM SymbolTable IMPORT FinalSymbol, IsVar, IsProcedure, IsModule,
                         GetMainModule, IsType, NulSym, IsRecord, GetSymName, GetNth, GetNthProcedure, GetDeclaredMod, NoOfParam ;
 FROM NameKey IMPORT GetKey, WriteKey, MakeKey, IsSameExcludingCase, NulName, makekey, KeyToCharStar ;
-FROM M2MetaError IMPORT MetaErrorString0, MetaError2 ;
+FROM M2MetaError IMPORT MetaErrorStringT0, MetaError2 ;
 FROM Lists IMPORT List, InitList, IsItemInList, IncludeItemIntoList ;
 FROM M2Reserved IMPORT IsReserved, toktype ;
 FROM DynamicStrings IMPORT String, InitString, KillString, ToUpper, InitStringCharStar, string, Mark, ToUpper, Dup ;
@@ -78,11 +78,11 @@ END IsNotADuplicateName ;
                                  as a keyword except for its case.
 *)
 
-PROCEDURE CheckVariableAgainstKeyword (name: Name) ;
+PROCEDURE CheckVariableAgainstKeyword (tok: CARDINAL; name: Name) ;
 BEGIN
    IF StyleChecking
    THEN
-      PerformVariableKeywordCheck (name)
+      PerformVariableKeywordCheck (tok, name)
    END
 END CheckVariableAgainstKeyword ;
 
@@ -91,7 +91,7 @@ END CheckVariableAgainstKeyword ;
    PerformVariableKeywordCheck - performs the check and constructs the metaerror notes if appropriate.
 *)
 
-PROCEDURE PerformVariableKeywordCheck (name: Name) ;
+PROCEDURE PerformVariableKeywordCheck (tok: CARDINAL; name: Name) ;
 VAR
    upper : Name ;
    token : toktype ;
@@ -105,9 +105,11 @@ BEGIN
    THEN
       IF IsNotADuplicateName (name)
       THEN
-         MetaErrorString0 (Sprintf2 (Mark (InitString ('either the identifier has the same name as a keyword or alternatively a keyword has the wrong case ({%%K%s} and {!%%O:{%%K%s}})')),
-                                     upperS, orig)) ;
-         MetaErrorString0 (Sprintf1 (Mark (InitString ('the symbol name {!%%O:{%%K%s}} is legal as an identifier, however as such it might cause confusion and is considered bad programming practice')), orig))
+         MetaErrorStringT0 (tok,
+                            Sprintf2 (Mark (InitString ('either the identifier has the same name as a keyword or alternatively a keyword has the wrong case ({%%K%s} and {!%%O:{%%K%s}})')),
+                                      upperS, orig)) ;
+         MetaErrorStringT0 (tok,
+                            Sprintf1 (Mark (InitString ('the symbol name {!%%O:{%%K%s}} is legal as an identifier, however as such it might cause confusion and is considered bad programming practice')), orig))
       END
    END ;
    upperS := KillString (upperS) ;
diff --git a/gcc/m2/gm2-compiler/P1SymBuild.mod b/gcc/m2/gm2-compiler/P1SymBuild.mod
index 02d4ac7..08a0fc3 100644
--- a/gcc/m2/gm2-compiler/P1SymBuild.mod
+++ b/gcc/m2/gm2-compiler/P1SymBuild.mod
@@ -79,7 +79,6 @@ FROM SymbolTable IMPORT NulSym,
                         MakeSubscript, PutSubscript,
                         PutArray, GetType, IsArray,
                         IsProcType, MakeProcType,
-                        PutProcTypeVarParam, PutProcTypeParam,
                         PutProcedureBuiltin, PutProcedureInline,
                         GetSymName,
                         ResolveImports, PutDeclared,
@@ -109,42 +108,6 @@ VAR
 
 
 (*
-   CheckFileName - checks to see that the module name matches the file name.
-*)
-
-(*
-PROCEDURE CheckFileName (tok: CARDINAL; name: Name; ModuleType: ARRAY OF CHAR) ;
-VAR
-   ext,
-   basename: INTEGER ;
-   s,
-   FileName: String ;
-BEGIN
-   FileName := GetFileName() ;
-   basename := RIndex(FileName, '/', 0) ;
-   IF basename=-1
-   THEN
-      basename := 0
-   END ;
-   ext := RIndex(FileName, '.', 0) ;
-   IF ext=-1
-   THEN
-      ext := 0
-   END ;
-   FileName := Slice(FileName, basename, ext) ;
-   IF EqualCharStar(FileName, KeyToCharStar(name))
-   THEN
-      FileName := KillString(FileName)
-   ELSE
-      s := ConCat (InitString (ModuleType),
-                   Mark (InitString (" module name {%1Ea} is inconsistant with the filename {%F{%2a}}"))) ;
-      MetaErrorString2 (s, MakeError (tok, name), MakeErrorS (tok, FileName))
-   END
-END CheckFileName ;
-*)
-
-
-(*
    StartBuildDefinitionModule - Creates a definition module and starts
                                 a new scope.
 
@@ -227,7 +190,7 @@ BEGIN
    END ;
    IF NameStart#NameEnd
    THEN
-      MetaError1 ('inconsistant definition module name {%1Wa}', MakeError (start, NameStart))
+      MetaError1 ('inconsistent definition module name {%1Wa}', MakeError (start, NameStart))
    END ;
    LeaveBlock
 END P1EndBuildDefinitionModule ;
@@ -301,7 +264,7 @@ BEGIN
    IF NameStart#NameEnd
    THEN
       MetaErrorT1 (end,
-                   'inconsistant implementation module name {%1Wa}', MakeError (start, NameStart))
+                   'inconsistent implementation module name {%1Wa}', MakeError (start, NameStart))
    END ;
    LeaveBlock
 END P1EndBuildImplementationModule ;
@@ -381,7 +344,7 @@ BEGIN
    IF NameStart#NameEnd
    THEN
       MetaErrorT1 (end,
-                   'inconsistant program module name {%1Wa}', MakeError (start, NameStart))
+                   'inconsistent program module name {%1Wa}', MakeError (start, NameStart))
    END ;
    LeaveBlock
 END P1EndBuildProgramModule ;
@@ -446,7 +409,7 @@ BEGIN
    IF NameStart#NameEnd
    THEN
       MetaErrorT1 (end,
-                   'inconsistant inner module name {%1Wa}', MakeError (start, NameStart))
+                   'inconsistent inner module name {%1Wa}', MakeError (start, NameStart))
    END ;
    LeaveBlock
 END EndBuildInnerModule ;
diff --git a/gcc/m2/gm2-compiler/P2SymBuild.mod b/gcc/m2/gm2-compiler/P2SymBuild.mod
index 5c82ec8..54e624f 100644
--- a/gcc/m2/gm2-compiler/P2SymBuild.mod
+++ b/gcc/m2/gm2-compiler/P2SymBuild.mod
@@ -356,7 +356,7 @@ BEGIN
    END ;
    IF NameStart#NameEnd
    THEN
-      WriteFormat2('inconsistant definition module name, module began as (%a) and ended with (%a)', NameStart, NameEnd)
+      WriteFormat2('inconsistent definition module name, module began as (%a) and ended with (%a)', NameStart, NameEnd)
    END ;
    M2Error.LeaveErrorScope
 END P2EndBuildDefModule ;
@@ -425,7 +425,7 @@ BEGIN
    PopT (NameEnd) ;
    IF NameStart#NameEnd
    THEN
-      WriteFormat1('inconsistant implementation module name %a', NameStart)
+      WriteFormat1('inconsistent implementation module name %a', NameStart)
    END ;
    M2Error.LeaveErrorScope
 END P2EndBuildImplementationModule ;
@@ -499,7 +499,7 @@ BEGIN
    END ;
    IF NameStart#NameEnd
    THEN
-      WriteFormat2('inconsistant program module name %a does not match %a', NameStart, NameEnd)
+      WriteFormat2('inconsistent program module name %a does not match %a', NameStart, NameEnd)
    END ;
    M2Error.LeaveErrorScope
 END P2EndBuildProgramModule ;
@@ -564,7 +564,7 @@ BEGIN
    PopT(NameEnd) ;
    IF NameStart#NameEnd
    THEN
-      WriteFormat2('inconsistant inner module name %a does not match %a',
+      WriteFormat2('inconsistent inner module name %a does not match %a',
                    NameStart, NameEnd)
    END ;
    M2Error.LeaveErrorScope
@@ -1179,8 +1179,8 @@ BEGIN
    PopT (n) ;
    i := 1 ;
    WHILE i <= n DO
-      CheckVariableAgainstKeyword (OperandT (n+1-i)) ;
       tok := OperandTok (n+1-i) ;
+      CheckVariableAgainstKeyword (tok, OperandT (n+1-i)) ;
       Var := MakeVar (tok, OperandT (n+1-i)) ;
       AtAddress := OperandA (n+1-i) ;
       IF AtAddress # NulSym
@@ -1835,13 +1835,13 @@ BEGIN
          (* WarnStringAt (InitString ('parampos?'), OperandTok (pi)) ;  *)
          IF Unbounded AND (NOT IsUnboundedParam (ProcSym, prevkind, ParamTotal+i))
          THEN
-            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistant, %s',
+            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistent, %s',
                             'the parameter {%3EHa} was not declared as an ARRAY OF type',
                             'the parameter {%3EVa} was declared as an ARRAY OF type',
                             ParamTotal+i, ProcSym, curkind, prevkind)
          ELSIF (NOT Unbounded) AND IsUnboundedParam (ProcSym, prevkind, ParamTotal+i)
          THEN
-            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistant, %s',
+            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistent, %s',
                             'the parameter {%3EHa} was declared as an ARRAY OF type',
                             'the parameter {%3EVa} was not declared as an ARRAY OF type',
                             ParamTotal+i, ProcSym, curkind, prevkind)
@@ -1850,7 +1850,7 @@ BEGIN
          THEN
             IF GetDimension (GetNthParam (ProcSym, prevkind, ParamTotal+1)) # ndim
             THEN
-               ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistant, %s',
+               ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistent, %s',
                                'the dynamic array parameter {%3EHa} was declared with a different of dimensions',
                                'the dynamic array parameter {%3EVa} was declared with a different of dimensions',
                                ParamTotal+i, ProcSym, curkind, prevkind)
@@ -1859,14 +1859,14 @@ BEGIN
          IF isVarParam AND (NOT IsVarParam (ProcSym, prevkind, ParamTotal+i))
          THEN
             (* Expecting non VAR parameter.  *)
-            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistant, %s',
+            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistent, %s',
                             '{%3EHa} was not declared as a {%kVAR} parameter',
                             '{%3EVa} was declared as a {%kVAR} parameter',
                             ParamTotal+i, ProcSym, curkind, prevkind)
          ELSIF (NOT isVarParam) AND IsVarParam (ProcSym, prevkind, ParamTotal+i)
          THEN
             (* Expecting VAR pamarater.  *)
-            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistant, %s',
+            ParameterError ('declaration of procedure {%%1a} in the %s differs from the %s, {%%2N} parameter is inconsistent, %s',
                             '{%3EHa} was declared as a {%kVAR} parameter',
                             '{%3EVa} was not declared as a {%kVAR} parameter',
                             ParamTotal+i, ProcSym, curkind, prevkind)
@@ -1877,7 +1877,7 @@ BEGIN
             IF GetSymName (ParamI) # OperandT (pi)
             THEN
                (* Different parameter names.  *)
-               ParameterError ('procedure {%%1a} in the %s differs from the %s, {%%2N} parameter name is inconsistant, %s',
+               ParameterError ('procedure {%%1a} in the %s differs from the %s, {%%2N} parameter name is inconsistent, %s',
                             'named as {%3EVa}',
                             'named as {%3EVa}',
                             ParamTotal+i, ProcSym, curkind, prevkind)
@@ -1897,7 +1897,7 @@ BEGIN
             (NOT IsUnknown(SkipType(ParamIType)))
          THEN
             (* Different parameter types.  *)
-            ParameterError ('declaration in the %s differs from the %s, {%%2N} parameter is inconsistant, %s',
+            ParameterError ('declaration in the %s differs from the %s, {%%2N} parameter is inconsistent, %s',
                             'the parameter {%3EHa} was declared with a different type',
                             'the parameter {%3EVa} was declared with a different type',
                             ParamTotal+i, ProcSym, curkind, prevkind)
@@ -3072,10 +3072,10 @@ BEGIN
    IF Var=VarTok
    THEN
       (* VAR parameter *)
-      PutProcTypeVarParam(ProcTypeSym, TypeSym, IsUnbounded(TypeSym))
+      PutProcTypeVarParam (tok, ProcTypeSym, TypeSym, IsUnbounded (TypeSym))
    ELSE
       (* Non VAR parameter *)
-      PutProcTypeParam(ProcTypeSym, TypeSym, IsUnbounded(TypeSym))
+      PutProcTypeParam (tok, ProcTypeSym, TypeSym, IsUnbounded (TypeSym))
    END ;
    PushT(ProcTypeSym) ;
    Annotate("%1s(%1d)||proc type")
diff --git a/gcc/m2/gm2-compiler/P3SymBuild.mod b/gcc/m2/gm2-compiler/P3SymBuild.mod
index c4877f7..096057e 100644
--- a/gcc/m2/gm2-compiler/P3SymBuild.mod
+++ b/gcc/m2/gm2-compiler/P3SymBuild.mod
@@ -126,7 +126,7 @@ BEGIN
    PopT(NameStart) ;
    IF NameStart#NameEnd
    THEN
-      WriteFormat2('inconsistant definition module was named (%a) and concluded as (%a)',
+      WriteFormat2('inconsistent definition module was named (%a) and concluded as (%a)',
                    NameStart, NameEnd)
    END ;
    M2Error.LeaveErrorScope
diff --git a/gcc/m2/gm2-compiler/PCSymBuild.mod b/gcc/m2/gm2-compiler/PCSymBuild.mod
index 3bffe86..4db2730 100644
--- a/gcc/m2/gm2-compiler/PCSymBuild.mod
+++ b/gcc/m2/gm2-compiler/PCSymBuild.mod
@@ -269,7 +269,7 @@ BEGIN
    PopT(NameStart) ;
    IF NameStart#NameEnd
    THEN
-      WriteFormat2('inconsistant definition module was named (%a) and concluded as (%a)',
+      WriteFormat2('inconsistent definition module was named (%a) and concluded as (%a)',
                    NameStart, NameEnd)
    END ;
    M2Error.LeaveErrorScope
@@ -1168,9 +1168,9 @@ BEGIN
          par := GetParam (p, i) ;
          IF IsParameterVar (par)
          THEN
-            PutProcTypeVarParam (t, GetType (par), IsParameterUnbounded (par))
+            PutProcTypeVarParam (tok, t, GetType (par), IsParameterUnbounded (par))
          ELSE
-            PutProcTypeParam (t, GetType (par), IsParameterUnbounded (par))
+            PutProcTypeParam (tok, t, GetType (par), IsParameterUnbounded (par))
          END ;
          INC(i)
       END ;
diff --git a/gcc/m2/gm2-compiler/SymbolTable.def b/gcc/m2/gm2-compiler/SymbolTable.def
index 2a9865a..5b93f29 100644
--- a/gcc/m2/gm2-compiler/SymbolTable.def
+++ b/gcc/m2/gm2-compiler/SymbolTable.def
@@ -2650,7 +2650,8 @@ PROCEDURE MakeProcType (tok: CARDINAL; ProcTypeName: Name) : CARDINAL ;
                       ParamType into ProcType Sym.
 *)
 
-PROCEDURE PutProcTypeParam (Sym: CARDINAL;
+PROCEDURE PutProcTypeParam (tok: CARDINAL;
+                            Sym: CARDINAL;
                             ParamType: CARDINAL; isUnbounded: BOOLEAN) ;
 
 
@@ -2659,7 +2660,8 @@ PROCEDURE PutProcTypeParam (Sym: CARDINAL;
                          ParamType into ProcType Sym.
 *)
 
-PROCEDURE PutProcTypeVarParam (Sym: CARDINAL;
+PROCEDURE PutProcTypeVarParam (tok: CARDINAL;
+                               Sym: CARDINAL;
                                ParamType: CARDINAL; isUnbounded: BOOLEAN) ;
 
 
diff --git a/gcc/m2/gm2-compiler/SymbolTable.mod b/gcc/m2/gm2-compiler/SymbolTable.mod
index ff661dc..00946e5 100644
--- a/gcc/m2/gm2-compiler/SymbolTable.mod
+++ b/gcc/m2/gm2-compiler/SymbolTable.mod
@@ -463,9 +463,11 @@ TYPE
                                               (* of param.                   *)
                  Type          : CARDINAL ;   (* Index to the type of param. *)
                  IsUnbounded   : BOOLEAN ;    (* Is it an ARRAY OF Type?     *)
+                 Scope         : CARDINAL ;   (* Procedure declaration.      *)
                  ShadowVar     : CARDINAL ;   (* The local variable used to  *)
                                               (* shadow this parameter.      *)
-                 At            : Where ;      (* Where was sym declared/used *)
+                 FullTok,                     (* name: type virtual token.   *)
+                 At            : CARDINAL ;   (* Where was sym declared.     *)
               END ;
 
    SymVarParam = RECORD
@@ -476,9 +478,11 @@ TYPE
                     HeapVar       : CARDINAL ;(* The pointer value on heap.  *)
                                               (* Only used by static         *)
                                               (* analysis.                   *)
+                    Scope         : CARDINAL ;(* Procedure declaration.      *)
                     ShadowVar     : CARDINAL ;(* The local variable used to  *)
                                               (* shadow this parameter.      *)
-                    At            : Where ;   (* Where was sym declared/used *)
+                    FullTok,                  (* name: type virtual token.   *)
+                    At            : CARDINAL ;(* Where was sym declared.     *)
                  END ;
 
    ConstStringVariant = (m2str, cstr, m2nulstr, cnulstr) ;
@@ -4456,9 +4460,19 @@ BEGIN
       pSym := GetPsym (sym) ;
       IF IsParameterVar (sym)
       THEN
-         RETURN GetVarDeclTok (pSym^.VarParam.ShadowVar)
+         IF pSym^.VarParam.ShadowVar = NulSym
+         THEN
+            RETURN pSym^.VarParam.At
+         ELSE
+            RETURN GetVarDeclTok (pSym^.VarParam.ShadowVar)
+         END
       ELSE
-         RETURN GetVarDeclTok (pSym^.Param.ShadowVar)
+         IF pSym^.Param.ShadowVar = NulSym
+         THEN
+            RETURN pSym^.Param.At
+         ELSE
+            RETURN GetVarDeclTok (pSym^.Param.ShadowVar)
+         END
       END
    ELSIF IsVar (sym)
    THEN
@@ -4549,9 +4563,9 @@ BEGIN
    THEN
       IF IsParameterVar (sym)
       THEN
-         RETURN GetVarDeclFullTok (pSym^.VarParam.ShadowVar)
+         RETURN pSym^.VarParam.FullTok
       ELSE
-         RETURN GetVarDeclFullTok (pSym^.Param.ShadowVar)
+         RETURN pSym^.Param.FullTok
       END
    ELSIF IsVar (sym)
    THEN
@@ -10651,8 +10665,10 @@ BEGIN
             name := ParamName ;
             Type := ParamType ;
             IsUnbounded := isUnbounded ;
+            Scope := Sym ;
             ShadowVar := NulSym ;
-            InitWhereDeclaredTok(tok, At)
+            FullTok := MakeVirtual2Tok (tok, typetok) ;
+            At := tok
          END
       END ;
       AddParameter (Sym, kind, ParSym) ;
@@ -10671,7 +10687,7 @@ BEGIN
             pSym^.Param.ShadowVar := VariableSym
          END
       END ;
-      AddProcedureProcTypeParam (Sym, ParamType, isUnbounded, FALSE)
+      AddProcedureProcTypeParam (tok, Sym, ParamType, isUnbounded, FALSE)
    END ;
    RETURN( TRUE )
 END PutParam ;
@@ -10708,9 +10724,11 @@ BEGIN
             name := ParamName ;
             Type := ParamType ;
             IsUnbounded := isUnbounded ;
+            Scope := Sym ;
             ShadowVar := NulSym ;
             HeapVar := NulSym ;  (* Will contain a pointer value.  *)
-            InitWhereDeclaredTok(tok, At)
+            FullTok := MakeVirtual2Tok (tok, typetok) ;
+            At := tok
          END
       END ;
       AddParameter (Sym, kind, ParSym) ;
@@ -10729,7 +10747,7 @@ BEGIN
             pSym^.VarParam.ShadowVar := VariableSym
          END
       END ;
-      AddProcedureProcTypeParam (Sym, ParamType, isUnbounded, TRUE)
+      AddProcedureProcTypeParam (tok, Sym, ParamType, isUnbounded, TRUE)
    END ;
    RETURN( TRUE )
 END PutVarParam ;
@@ -10816,7 +10834,8 @@ END AddParameter ;
                                associated with procedure Sym.
 *)
 
-PROCEDURE AddProcedureProcTypeParam (Sym, ParamType: CARDINAL;
+PROCEDURE AddProcedureProcTypeParam (tok: CARDINAL;
+                                     Sym, ParamType: CARDINAL;
                                      isUnbounded, isVarParam: BOOLEAN) ;
 VAR
    pSym: PtrToSymbol ;
@@ -10829,10 +10848,12 @@ BEGIN
                     THEN
                        IF isVarParam
                        THEN
-                          PutProcTypeVarParam (Procedure.ProcedureType,
+                          PutProcTypeVarParam (tok,
+                                               Procedure.ProcedureType,
                                                ParamType, isUnbounded)
                        ELSE
-                          PutProcTypeParam (Procedure.ProcedureType,
+                          PutProcTypeParam (tok,
+                                            Procedure.ProcedureType,
                                             ParamType, isUnbounded)
                        END
                     END
@@ -13027,18 +13048,8 @@ BEGIN
       ConstLitSym        : RETURN( ConstLit.Scope ) |
       ConstStringSym     : RETURN( ConstString.Scope ) |
       ConstVarSym        : RETURN( ConstVar.Scope ) |
-      ParamSym           : IF Param.ShadowVar = NulSym
-                           THEN
-                              RETURN NulSym
-                           ELSE
-                              RETURN( GetScope (Param.ShadowVar) )
-                           END |
-      VarParamSym        : IF VarParam.ShadowVar = NulSym
-                           THEN
-                              RETURN NulSym
-                           ELSE
-                              RETURN( GetScope (VarParam.ShadowVar) )
-                           END |
+      ParamSym           : RETURN( Param.Scope ) |
+      VarParamSym        : RETURN( VarParam.Scope ) |
       UndefinedSym       : RETURN( NulSym ) |
       PartialUnboundedSym: InternalError ('should not be requesting the scope of a PartialUnbounded symbol')
 
@@ -13186,7 +13197,8 @@ END MakeProcType ;
                       ParamType into ProcType Sym.
 *)
 
-PROCEDURE PutProcTypeParam (Sym: CARDINAL;
+PROCEDURE PutProcTypeParam (tok: CARDINAL;
+                            Sym: CARDINAL;
                             ParamType: CARDINAL; isUnbounded: BOOLEAN) ;
 VAR
    pSym  : PtrToSymbol ;
@@ -13201,7 +13213,8 @@ BEGIN
          Type := ParamType ;
          IsUnbounded := isUnbounded ;
          ShadowVar := NulSym ;
-         InitWhereDeclared(At)
+         FullTok := tok ;
+         At := tok
       END
    END ;
    AddParameter (Sym, ProperProcedure, ParSym)
@@ -13213,7 +13226,8 @@ END PutProcTypeParam ;
                          ParamType into ProcType Sym.
 *)
 
-PROCEDURE PutProcTypeVarParam (Sym: CARDINAL;
+PROCEDURE PutProcTypeVarParam (tok: CARDINAL;
+                               Sym: CARDINAL;
                                ParamType: CARDINAL; isUnbounded: BOOLEAN) ;
 VAR
    pSym  : PtrToSymbol ;
@@ -13228,7 +13242,8 @@ BEGIN
          Type := ParamType ;
          IsUnbounded := isUnbounded ;
          ShadowVar := NulSym ;
-         InitWhereDeclared(At)
+         FullTok := tok ;
+         At := tok
       END
    END ;
    AddParameter (Sym, ProperProcedure, ParSym)
@@ -13918,8 +13933,8 @@ BEGIN
       UnboundedSym       : RETURN( Unbounded.At.DefDeclared ) |
       ProcedureSym       : RETURN( Procedure.At.DefDeclared ) |
       ProcTypeSym        : RETURN( ProcType.At.DefDeclared ) |
-      ParamSym           : RETURN( Param.At.DefDeclared ) |
-      VarParamSym        : RETURN( VarParam.At.DefDeclared ) |
+      ParamSym           : RETURN( Param.At ) |
+      VarParamSym        : RETURN( VarParam.At ) |
       ConstStringSym     : RETURN( ConstString.At.DefDeclared ) |
       ConstLitSym        : RETURN( ConstLit.At.DefDeclared ) |
       ConstVarSym        : RETURN( ConstVar.At.DefDeclared ) |
@@ -13968,8 +13983,8 @@ BEGIN
       UnboundedSym       : RETURN( Unbounded.At.ModDeclared ) |
       ProcedureSym       : RETURN( Procedure.At.ModDeclared ) |
       ProcTypeSym        : RETURN( ProcType.At.ModDeclared ) |
-      ParamSym           : RETURN( Param.At.ModDeclared ) |
-      VarParamSym        : RETURN( VarParam.At.ModDeclared ) |
+      ParamSym           : RETURN( Param.At ) |
+      VarParamSym        : RETURN( VarParam.At ) |
       ConstStringSym     : RETURN( ConstString.At.ModDeclared ) |
       ConstLitSym        : RETURN( ConstLit.At.ModDeclared ) |
       ConstVarSym        : RETURN( ConstVar.At.ModDeclared ) |
@@ -14019,8 +14034,6 @@ BEGIN
       UnboundedSym       : Unbounded.At.DefDeclared := tok |
       ProcedureSym       : Procedure.At.DefDeclared := tok |
       ProcTypeSym        : ProcType.At.DefDeclared := tok |
-      ParamSym           : Param.At.DefDeclared := tok |
-      VarParamSym        : VarParam.At.DefDeclared := tok |
       ConstStringSym     : ConstString.At.DefDeclared := tok |
       ConstLitSym        : ConstLit.At.DefDeclared := tok |
       ConstVarSym        : ConstVar.At.DefDeclared := tok |
@@ -14067,8 +14080,6 @@ BEGIN
       UnboundedSym       : Unbounded.At.ModDeclared := tok |
       ProcedureSym       : Procedure.At.ModDeclared := tok |
       ProcTypeSym        : ProcType.At.ModDeclared := tok |
-      ParamSym           : Param.At.ModDeclared := tok |
-      VarParamSym        : VarParam.At.ModDeclared := tok |
       ConstStringSym     : ConstString.At.ModDeclared := tok |
       ConstLitSym        : ConstLit.At.ModDeclared := tok |
       ConstVarSym        : ConstVar.At.ModDeclared := tok |
@@ -14323,8 +14334,10 @@ BEGIN
       UnboundedSym       : RETURN( Unbounded.At.FirstUsed ) |
       ProcedureSym       : RETURN( Procedure.At.FirstUsed ) |
       ProcTypeSym        : RETURN( ProcType.At.FirstUsed ) |
+      (*
       ParamSym           : RETURN( Param.At.FirstUsed ) |
       VarParamSym        : RETURN( VarParam.At.FirstUsed ) |
+      *)
       ConstStringSym     : RETURN( ConstString.At.FirstUsed ) |
       ConstLitSym        : RETURN( ConstLit.At.FirstUsed ) |
       ConstVarSym        : RETURN( ConstVar.At.FirstUsed ) |
diff --git a/gcc/m2/gm2-gcc/m2color.cc b/gcc/m2/gm2-gcc/m2color.cc
index fd30f46..a638d7c 100644
--- a/gcc/m2/gm2-gcc/m2color.cc
+++ b/gcc/m2/gm2-gcc/m2color.cc
@@ -23,7 +23,7 @@ along with GNU Modula-2; see the file COPYING3.  If not see
 #include "m2color.h"
 
 #include "gcc-consolidation.h"
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
 
 
 char *
diff --git a/gcc/m2/gm2-gcc/m2linemap.cc b/gcc/m2/gm2-gcc/m2linemap.cc
index feb62b3..89a1080 100644
--- a/gcc/m2/gm2-gcc/m2linemap.cc
+++ b/gcc/m2/gm2-gcc/m2linemap.cc
@@ -188,12 +188,13 @@ m2linemap_ErrorAt (location_t location, char *message)
 static void
 m2linemap_ErrorAtf_1 (location_t location, const char *message, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   rich_location richloc (line_table, location);
 
   va_start (ap, message);
-  diagnostic_set_info (&diagnostic, message, &ap, &richloc, DK_ERROR);
+  diagnostic_set_info (&diagnostic, message, &ap, &richloc,
+		       diagnostics::kind::error);
   diagnostic_report_diagnostic (global_dc, &diagnostic);
   va_end (ap);
 }
@@ -209,12 +210,13 @@ m2linemap_ErrorAtf (location_t location, const char *message)
 static void
 m2linemap_WarningAtf_1 (location_t location, const char *message, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   rich_location richloc (line_table, location);
 
   va_start (ap, message);
-  diagnostic_set_info (&diagnostic, message, &ap, &richloc, DK_WARNING);
+  diagnostic_set_info (&diagnostic, message, &ap, &richloc,
+		       diagnostics::kind::warning);
   diagnostic_report_diagnostic (global_dc, &diagnostic);
   va_end (ap);
 }
@@ -230,12 +232,13 @@ m2linemap_WarningAtf (location_t location, const char *message)
 static void
 m2linemap_NoteAtf_1 (location_t location, const char *message, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   rich_location richloc (line_table, location);
 
   va_start (ap, message);
-  diagnostic_set_info (&diagnostic, message, &ap, &richloc, DK_NOTE);
+  diagnostic_set_info (&diagnostic, message, &ap, &richloc,
+		       diagnostics::kind::note);
   diagnostic_report_diagnostic (global_dc, &diagnostic);
   va_end (ap);
 }
diff --git a/gcc/m2/gm2-gcc/rtegraph.cc b/gcc/m2/gm2-gcc/rtegraph.cc
index cd12582..f7b3065 100644
--- a/gcc/m2/gm2-gcc/rtegraph.cc
+++ b/gcc/m2/gm2-gcc/rtegraph.cc
@@ -283,9 +283,10 @@ rtegraph_lookup (gimple *g, tree fndecl, bool is_call)
 /* rte_error_at - wraps up an error message.  */
 
 static void
-rte_error_at (location_t location, diagnostic_t kind, const char *message, ...)
+rte_error_at (location_t location, diagnostics::kind kind,
+	      const char *message, ...)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   va_list ap;
   rich_location richloc (line_table, location);
 
@@ -338,7 +339,7 @@ access_string (tree t, const char **value)
    give the function context which might be misleading if this is inlined.  */
 
 static void
-generate_report (gimple *stmt, const char *report, diagnostic_t kind)
+generate_report (gimple *stmt, const char *report, diagnostics::kind kind)
 {
   if (gimple_call_num_args (stmt) == 5)
     {
@@ -393,15 +394,16 @@ const char *rtenode::create_message (const char *with_name, const char *without_
   return message;
 }
 
-/* error_message issue an DK_ERROR from grtenode.  */
+/* error_message issue an diagnostics::kind::error from grtenode.  */
 
 void rtenode::error_message (void)
 {
   if (grtenode != NULL)
-    generate_report (grtenode, "runtime error will occur", DK_ERROR);
+    generate_report (grtenode, "runtime error will occur",
+		     diagnostics::kind::error);
 }
 
-/* warning_message issue an DK_WARNING from grtenode.  */
+/* warning_message issue an diagnostics::kind::warning from grtenode.  */
 
 void rtenode::warning_message (void)
 {
@@ -409,15 +411,16 @@ void rtenode::warning_message (void)
     ("runtime error will occur if an exported procedure is called from %s",
      "runtime error will occur if an exported procedure is called");
   if (grtenode != NULL)
-    generate_report (grtenode, message, DK_WARNING);
+    generate_report (grtenode, message, diagnostics::kind::warning);
 }
 
-/* note_message issue an DK_NOTE from grtenode.  */
+/* note_message issue an diagnostics::kind::note from grtenode.  */
 
 void rtenode::note_message (void)
 {
   if (grtenode != NULL)
-    generate_report (grtenode, "runtime will occur if this procedure is called", DK_NOTE);
+    generate_report (grtenode, "runtime will occur if this procedure is called",
+		     diagnostics::kind::note);
 }
 
 /* dump_vec display contents of vector array list.  */
diff --git a/gcc/m2/gm2-libs-iso/LowLong.mod b/gcc/m2/gm2-libs-iso/LowLong.mod
index 92c7d91..f611923 100644
--- a/gcc/m2/gm2-libs-iso/LowLong.mod
+++ b/gcc/m2/gm2-libs-iso/LowLong.mod
@@ -182,7 +182,7 @@ BEGIN
    IF n<0
    THEN
       (* exception raised *)
-      RAISE(except, ORD(badparam),
+      RAISE(exceptSrc, ORD(badparam),
             'LowLong.trunc: cannot truncate to a negative number of digits') ;
       RETURN x
    ELSE
@@ -230,7 +230,7 @@ BEGIN
    IF n<0
    THEN
       (* exception raised *)
-      RAISE(except, ORD(badparam),
+      RAISE(exceptSrc, ORD(badparam),
             'LowLong.round: cannot round to a negative number of digits') ;
       RETURN x
    ELSE
@@ -287,12 +287,12 @@ END currentMode ;
 
 PROCEDURE IsLowException () : BOOLEAN ;
 BEGIN
-   RETURN( IsExceptionalExecution() AND IsCurrentSource(except) )
+   RETURN( IsExceptionalExecution () AND IsCurrentSource (exceptSrc) )
 END IsLowException ;
 
 
 VAR
-   except: ExceptionSource ;
+   exceptSrc: ExceptionSource ;
 BEGIN
-   AllocateSource(except)
+   AllocateSource (exceptSrc)
 END LowLong.
diff --git a/gcc/m2/gm2-libs-iso/LowReal.mod b/gcc/m2/gm2-libs-iso/LowReal.mod
index 580f36b..6d9ea00 100644
--- a/gcc/m2/gm2-libs-iso/LowReal.mod
+++ b/gcc/m2/gm2-libs-iso/LowReal.mod
@@ -183,8 +183,8 @@ BEGIN
    IF n<0
    THEN
       (* exception raised *)
-      RAISE(except, ORD(badparam),
-            'LowReal.trunc: cannot truncate to a negative number of digits') ;
+      RAISE (exceptSrc, ORD(badparam),
+             'LowReal.trunc: cannot truncate to a negative number of digits') ;
       RETURN x
    ELSE
       r := dtoa(x, maxsignificant, 100, point, sign) ;
@@ -231,8 +231,8 @@ BEGIN
    IF n<0
    THEN
       (* exception raised *)
-      RAISE(except, ORD(badparam),
-            'LowReal.round: cannot round to a negative number of digits') ;
+      RAISE (exceptSrc, ORD(badparam),
+             'LowReal.round: cannot round to a negative number of digits') ;
       RETURN x
    ELSE
       s := RealToFloatString(x, n) ;
@@ -288,12 +288,12 @@ END currentMode ;
 
 PROCEDURE IsLowException () : BOOLEAN ;
 BEGIN
-   RETURN( IsExceptionalExecution() AND IsCurrentSource(except) )
+   RETURN( IsExceptionalExecution () AND IsCurrentSource (exceptSrc) )
 END IsLowException ;
 
 
 VAR
-   except: ExceptionSource ;
+   exceptSrc: ExceptionSource ;
 BEGIN
-   AllocateSource(except)
+   AllocateSource (exceptSrc)
 END LowReal.
diff --git a/gcc/m2/gm2-libs-iso/LowShort.mod b/gcc/m2/gm2-libs-iso/LowShort.mod
index 8531a88..62e4887 100644
--- a/gcc/m2/gm2-libs-iso/LowShort.mod
+++ b/gcc/m2/gm2-libs-iso/LowShort.mod
@@ -183,8 +183,8 @@ BEGIN
    IF n<0
    THEN
       (* exception raised *)
-      RAISE(except, ORD(badparam),
-            'LowLong.trunc: cannot truncate to a negative number of digits') ;
+      RAISE (exceptSrc, ORD(badparam),
+             'LowLong.trunc: cannot truncate to a negative number of digits') ;
       RETURN x
    ELSE
       r := dtoa(x, maxsignificant, 100, point, sign) ;
@@ -231,8 +231,8 @@ BEGIN
    IF n<0
    THEN
       (* exception raised *)
-      RAISE(except, ORD(badparam),
-            'LowLong.round: cannot round to a negative number of digits') ;
+      RAISE (exceptSrc, ORD(badparam),
+             'LowLong.round: cannot round to a negative number of digits') ;
       RETURN x
    ELSE
       s := RealToFloatString(x, n) ;
@@ -288,12 +288,12 @@ END currentMode ;
 
 PROCEDURE IsLowException () : BOOLEAN ;
 BEGIN
-   RETURN( IsExceptionalExecution() AND IsCurrentSource(except) )
+   RETURN( IsExceptionalExecution () AND IsCurrentSource (exceptSrc) )
 END IsLowException ;
 
 
 VAR
-   except: ExceptionSource ;
+   exceptSrc: ExceptionSource ;
 BEGIN
-   AllocateSource(except)
+   AllocateSource (exceptSrc)
 END LowShort.
diff --git a/gcc/m2/gm2-libs-iso/Processes.mod b/gcc/m2/gm2-libs-iso/Processes.mod
index 8ef22c0..b0c1b69 100644
--- a/gcc/m2/gm2-libs-iso/Processes.mod
+++ b/gcc/m2/gm2-libs-iso/Processes.mod
@@ -441,7 +441,7 @@ PROCEDURE Wait ;
 VAR
    calling,
    best   : ProcessId ;
-   from   : COROUTINE ;
+   fromCor: COROUTINE ;
 BEGIN
    IF debugging
    THEN
@@ -451,17 +451,17 @@ BEGIN
    OnWaitingQueue (calling) ;
    best := chooseProcess () ;
    currentId := best ;
-   from := calling^.context ;
+   fromCor := calling^.context ;
    IF debugging
    THEN
       displayProcesses ("Wait about to perform IOTRANSFER")
    END ;
-   IOTRANSFER (from, currentId^.context) ;
+   IOTRANSFER (fromCor, currentId^.context) ;
    IF debugging
    THEN
       displayProcesses ("Wait after IOTRANSFER")
    END ;
-   currentId^.context := from ;
+   currentId^.context := fromCor ;
    currentId := calling ;
    OnReadyQueue (calling) ;
    IF debugging
diff --git a/gcc/m2/gm2-libs-iso/RndFile.mod b/gcc/m2/gm2-libs-iso/RndFile.mod
index e04cd8f..0a2264a 100644
--- a/gcc/m2/gm2-libs-iso/RndFile.mod
+++ b/gcc/m2/gm2-libs-iso/RndFile.mod
@@ -398,9 +398,9 @@ PROCEDURE EndPos (cid: ChanId): FilePos;
      position after which there have been no writes.
   *)
 VAR
-   d  : DeviceTablePtr ;
-   end,
-   old: FilePos ;
+   d   : DeviceTablePtr ;
+   endP,
+   old : FilePos ;
 BEGIN
    IF IsRndFile(cid)
    THEN
@@ -410,9 +410,9 @@ BEGIN
          old := CurrentPos(cid) ;
          FIO.SetPositionFromEnd(RTio.GetFile(cid), 0) ;
          checkErrno(dev, d) ;
-         end := CurrentPos(cid) ;
+         endP := CurrentPos(cid) ;
          FIO.SetPositionFromBeginning(RTio.GetFile(cid), old) ;
-         RETURN( end )
+         RETURN( endP )
       END
    ELSE
       RAISEdevException(cid, did, IOChan.wrongDevice,
diff --git a/gcc/m2/gm2-libs/ARRAYOFCHAR.mod b/gcc/m2/gm2-libs/ARRAYOFCHAR.mod
index f27378a..41aaf68 100644
--- a/gcc/m2/gm2-libs/ARRAYOFCHAR.mod
+++ b/gcc/m2/gm2-libs/ARRAYOFCHAR.mod
@@ -34,14 +34,14 @@ IMPORT StrLib ;
    Write - writes a string to file f.
 *)
 
-PROCEDURE Write (f: File; a: ARRAY OF CHAR) ;
+PROCEDURE Write (f: File; str: ARRAY OF CHAR) ;
 VAR
    len, i: CARDINAL ;
 BEGIN
-   len := StrLib.StrLen (a) ;
+   len := StrLib.StrLen (str) ;
    i := 0 ;
    WHILE i < len DO
-      WriteChar (f, a[i]) ;
+      WriteChar (f, str[i]) ;
       INC (i)
    END
 END Write ;
diff --git a/gcc/m2/gm2-libs/M2EXCEPTION.mod b/gcc/m2/gm2-libs/M2EXCEPTION.mod
index a17310f..e92b16d 100644
--- a/gcc/m2/gm2-libs/M2EXCEPTION.mod
+++ b/gcc/m2/gm2-libs/M2EXCEPTION.mod
@@ -33,7 +33,7 @@ FROM RTExceptions IMPORT EHBlock, GetExceptionBlock, GetNumber, Raise,
 
 (* If the program or coroutine is in the exception state then return the enumeration
    value representing the exception cause.  If it is not in the exception state then
-   raises and exception (exException).  *)
+   raises an exException exception.  *)
 
 PROCEDURE M2Exception () : M2Exceptions;
 VAR
@@ -45,7 +45,8 @@ BEGIN
    IF n = MAX (CARDINAL)
    THEN
       Raise (ORD (exException), ADR (__FILE__), __LINE__, __COLUMN__, ADR (__FUNCTION__),
-             ADR ('current coroutine is not in the exceptional execution state'))
+             ADR ('current coroutine is not in the exceptional execution state')) ;
+      RETURN exException
    ELSE
       RETURN VAL (M2Exceptions, n)
    END
diff --git a/gcc/m2/gm2-libs/SCmdArgs.mod b/gcc/m2/gm2-libs/SCmdArgs.mod
index ed76fc4..8443d5f 100644
--- a/gcc/m2/gm2-libs/SCmdArgs.mod
+++ b/gcc/m2/gm2-libs/SCmdArgs.mod
@@ -132,26 +132,27 @@ PROCEDURE GetArg (CmdLine: String;
 VAR
    i         : CARDINAL ;
    sn,
-   start, end: INTEGER ;
+   startPos,
+   endPos    : INTEGER ;
    ch        : CHAR ;
 BEGIN
    i := 0 ;
-   start := 0 ;
-   end := Length(CmdLine) ;
+   startPos := 0 ;
+   endPos := Length(CmdLine) ;
    WHILE i<n DO
-      start := skipWhite(CmdLine, start, end) ;
-      sn := skipNextArg(CmdLine, start, end) ;
-      IF sn<end
+      startPos := skipWhite(CmdLine, startPos, endPos) ;
+      sn := skipNextArg(CmdLine, startPos, endPos) ;
+      IF sn<endPos
       THEN
-         start := sn ;
+         startPos := sn ;
          INC(i)
       ELSE
          RETURN( FALSE )
       END
    END ;
-   start := skipWhite(CmdLine, start, end) ;
-   sn := skipNextArg(CmdLine, start, end) ;
-   Argi := Slice(CmdLine, start, sn) ;
+   startPos := skipWhite(CmdLine, startPos, endPos) ;
+   sn := skipNextArg(CmdLine, startPos, endPos) ;
+   Argi := Slice(CmdLine, startPos, sn) ;
    RETURN( TRUE )
 END GetArg ;
 
@@ -165,17 +166,18 @@ PROCEDURE Narg (CmdLine: String) : CARDINAL ;
 VAR
    n         : CARDINAL ;
    s,
-   start, end: INTEGER ;
+   startPos,
+   endPos    : INTEGER ;
 BEGIN
    n := 0 ;
-   start := 0 ;
-   end := Length(CmdLine) ;
+   startPos := 0 ;
+   endPos := Length(CmdLine) ;
    LOOP
-      start := skipWhite(CmdLine, start, end) ;
-      s := skipNextArg(CmdLine, start, end) ;
-      IF s<end
+      startPos := skipWhite(CmdLine, startPos, endPos) ;
+      s := skipNextArg(CmdLine, startPos, endPos) ;
+      IF s<endPos
       THEN
-         start := s ;
+         startPos := s ;
          INC(n)
       ELSE
          RETURN( n )
diff --git a/gcc/m2/mc-boot/GFormatStrings.cc b/gcc/m2/mc-boot/GFormatStrings.cc
index f4c4fd6..ad7e7d8 100644
--- a/gcc/m2/mc-boot/GFormatStrings.cc
+++ b/gcc/m2/mc-boot/GFormatStrings.cc
@@ -464,7 +464,7 @@ static DynamicStrings_String PerformFormatString (DynamicStrings_String fmt, int
               /* avoid dangling else.  */
               afterperc += 1;
               Cast ((unsigned char *) &u, (sizeof (u)-1), (const unsigned char *) w, _w_high);
-              in = DynamicStrings_ConCat (in, DynamicStrings_Slice (fmt, (*startpos), nextperc));
+              in = Copy (fmt, in, (*startpos), nextperc);
               in = DynamicStrings_ConCat (in, StringConvert_CardinalToString (u, static_cast<unsigned int> (width), leader, 16, true));
               (*startpos) = afterperc;
               DSdbExit (static_cast<DynamicStrings_String> (NULL));
@@ -475,7 +475,7 @@ static DynamicStrings_String PerformFormatString (DynamicStrings_String fmt, int
               /* avoid dangling else.  */
               afterperc += 1;
               Cast ((unsigned char *) &u, (sizeof (u)-1), (const unsigned char *) w, _w_high);
-              in = DynamicStrings_ConCat (in, DynamicStrings_Slice (fmt, (*startpos), nextperc));
+              in = Copy (fmt, in, (*startpos), nextperc);
               in = DynamicStrings_ConCat (in, StringConvert_CardinalToString (u, static_cast<unsigned int> (width), leader, 10, false));
               (*startpos) = afterperc;
               DSdbExit (static_cast<DynamicStrings_String> (NULL));
diff --git a/gcc/m2/mc-boot/GM2EXCEPTION.cc b/gcc/m2/mc-boot/GM2EXCEPTION.cc
index 62d47f0..6baff3c 100644
--- a/gcc/m2/mc-boot/GM2EXCEPTION.cc
+++ b/gcc/m2/mc-boot/GM2EXCEPTION.cc
@@ -34,7 +34,6 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
        typedef struct { PROC_t proc; } PROC;
 #   endif
 
-#   include "Gmcrts.h"
 #define _M2EXCEPTION_C
 
 #include "GM2EXCEPTION.h"
@@ -51,18 +50,19 @@ extern "C" M2EXCEPTION_M2Exceptions M2EXCEPTION_M2Exception (void)
 
   /* If the program or coroutine is in the exception state then return the enumeration
    value representing the exception cause.  If it is not in the exception state then
-   raises and exception (exException).  */
+   raises an exException exception.  */
   e = RTExceptions_GetExceptionBlock ();
   n = RTExceptions_GetNumber (e);
   if (n == (UINT_MAX))
     {
       RTExceptions_Raise ( ((unsigned int) (M2EXCEPTION_exException)), const_cast<void*> (static_cast<const void*>("../../gcc/m2/gm2-libs/M2EXCEPTION.mod")), 47, 6, const_cast<void*> (static_cast<const void*>("M2Exception")), const_cast<void*> (static_cast<const void*>("current coroutine is not in the exceptional execution state")));
+      return M2EXCEPTION_exException;
     }
   else
     {
       return (M2EXCEPTION_M2Exceptions) (n);
     }
-  ReturnException ("../../gcc/m2/gm2-libs/M2EXCEPTION.def", 25, 1);
+  /* static analysis guarentees a RETURN statement will be used before here.  */
   __builtin_unreachable ();
 }
 
diff --git a/gcc/m2/mc-boot/GSFIO.cc b/gcc/m2/mc-boot/GSFIO.cc
index 6ae0d5e..f8c13d3 100644
--- a/gcc/m2/mc-boot/GSFIO.cc
+++ b/gcc/m2/mc-boot/GSFIO.cc
@@ -99,6 +99,13 @@ extern "C" DynamicStrings_String SFIO_WriteS (FIO_File file, DynamicStrings_Stri
 
 extern "C" DynamicStrings_String SFIO_ReadS (FIO_File file);
 
+/*
+   GetFileName - return a new string containing the name of the file.
+                 The string should be killed by the caller.
+*/
+
+extern "C" DynamicStrings_String SFIO_GetFileName (FIO_File file);
+
 
 /*
    Exists - returns TRUE if a file named, fname exists for reading.
@@ -207,6 +214,19 @@ extern "C" DynamicStrings_String SFIO_ReadS (FIO_File file)
   __builtin_unreachable ();
 }
 
+
+/*
+   GetFileName - return a new string containing the name of the file.
+                 The string should be killed by the caller.
+*/
+
+extern "C" DynamicStrings_String SFIO_GetFileName (FIO_File file)
+{
+  return DynamicStrings_InitStringCharStar (FIO_getFileName (file));
+  /* static analysis guarentees a RETURN statement will be used before here.  */
+  __builtin_unreachable ();
+}
+
 extern "C" void _M2_SFIO_init (__attribute__((unused)) int argc, __attribute__((unused)) char *argv[], __attribute__((unused)) char *envp[])
 {
 }
diff --git a/gcc/m2/mc-boot/GSFIO.h b/gcc/m2/mc-boot/GSFIO.h
index 42ffc48..93c8099 100644
--- a/gcc/m2/mc-boot/GSFIO.h
+++ b/gcc/m2/mc-boot/GSFIO.h
@@ -103,6 +103,13 @@ EXTERN DynamicStrings_String SFIO_WriteS (FIO_File file, DynamicStrings_String s
 */
 
 EXTERN DynamicStrings_String SFIO_ReadS (FIO_File file);
+
+/*
+   GetFileName - return a new string containing the name of the file.
+                 The string should be killed by the caller.
+*/
+
+EXTERN DynamicStrings_String SFIO_GetFileName (FIO_File file);
 #   ifdef __cplusplus
 }
 #   endif
diff --git a/gcc/m2/mc-boot/Gdecl.cc b/gcc/m2/mc-boot/Gdecl.cc
index ae03483..94ea098 100644
--- a/gcc/m2/mc-boot/Gdecl.cc
+++ b/gcc/m2/mc-boot/Gdecl.cc
@@ -2550,6 +2550,14 @@ static bool isLeafString (decl_node__opaque n);
 static DynamicStrings_String getLiteralStringContents (decl_node__opaque n);
 
 /*
+   getStringChar - if the string is delimited by single
+                   or double quotes then strip both
+                   quotes from the string.
+*/
+
+static DynamicStrings_String getStringChar (decl_node__opaque n);
+
+/*
    getStringContents - return the string contents of a constant, literal,
                        string or a constexp node.
 */
@@ -2569,7 +2577,13 @@ static nameKey_Name addNames (decl_node__opaque a, decl_node__opaque b);
 static decl_node__opaque resolveString (decl_node__opaque n);
 
 /*
-   foldBinary -
+   addQuotes - adds delimiter quote char to string.
+*/
+
+static DynamicStrings_String addQuotes (DynamicStrings_String s, char quote);
+
+/*
+   foldBinary - attempt to fold binary + for string constants.
 */
 
 static decl_node__opaque foldBinary (decl_nodeT k, decl_node__opaque l, decl_node__opaque r, decl_node__opaque res);
@@ -7590,6 +7604,32 @@ static DynamicStrings_String getLiteralStringContents (decl_node__opaque n)
 
 
 /*
+   getStringChar - if the string is delimited by single
+                   or double quotes then strip both
+                   quotes from the string.
+*/
+
+static DynamicStrings_String getStringChar (decl_node__opaque n)
+{
+  DynamicStrings_String s;
+
+  s = getString (n);
+  if (((DynamicStrings_char (s, 0)) == '\'') && ((DynamicStrings_char (s, -1)) == '\''))
+    {
+      s = DynamicStrings_Slice (s, 1, -1);
+    }
+  else if (((DynamicStrings_char (s, 0)) == '"') && ((DynamicStrings_char (s, -1)) == '"'))
+    {
+      /* avoid dangling else.  */
+      s = DynamicStrings_Slice (s, 1, -1);
+    }
+  return s;
+  /* static analysis guarentees a RETURN statement will be used before here.  */
+  __builtin_unreachable ();
+}
+
+
+/*
    getStringContents - return the string contents of a constant, literal,
                        string or a constexp node.
 */
@@ -7608,7 +7648,7 @@ static DynamicStrings_String getStringContents (decl_node__opaque n)
   else if (isString (n))
     {
       /* avoid dangling else.  */
-      return getString (n);
+      return getStringChar (n);
     }
   else if (isConstExp (n))
     {
@@ -7672,11 +7712,29 @@ static decl_node__opaque resolveString (decl_node__opaque n)
 
 
 /*
-   foldBinary -
+   addQuotes - adds delimiter quote char to string.
+*/
+
+static DynamicStrings_String addQuotes (DynamicStrings_String s, char quote)
+{
+  DynamicStrings_String qs;
+
+  s = DynamicStrings_ConCatChar (s, quote);
+  qs = DynamicStrings_InitStringChar (quote);
+  qs = DynamicStrings_ConCat (qs, DynamicStrings_Mark (s));
+  return qs;
+  /* static analysis guarentees a RETURN statement will be used before here.  */
+  __builtin_unreachable ();
+}
+
+
+/*
+   foldBinary - attempt to fold binary + for string constants.
 */
 
 static decl_node__opaque foldBinary (decl_nodeT k, decl_node__opaque l, decl_node__opaque r, decl_node__opaque res)
 {
+  char qc;
   decl_node__opaque n;
   DynamicStrings_String ls;
   DynamicStrings_String rs;
@@ -7686,7 +7744,12 @@ static decl_node__opaque foldBinary (decl_nodeT k, decl_node__opaque l, decl_nod
     {
       ls = getStringContents (l);
       rs = getStringContents (r);
+      qc = '\'';
+      /* Add unquoted contents.  */
       ls = DynamicStrings_Add (ls, rs);
+      /* Add quote.  */
+      ls = addQuotes (ls, qc);
+      /* Build new string.  */
       n = static_cast<decl_node__opaque> (decl_makeString (nameKey_makekey (DynamicStrings_string (ls))));
       ls = DynamicStrings_KillString (ls);
       rs = DynamicStrings_KillString (rs);
@@ -22789,7 +22852,7 @@ static decl_node__opaque doDupExpr (decl_node__opaque n)
         break;
 
       case decl_length:
-        M2RTS_HALT (-1);
+        M2RTS_HALT (-1);  /* length should have been converted into unary.  */
         __builtin_unreachable ();
         break;
 
diff --git a/gcc/m2/mc-boot/GmcFileName.h b/gcc/m2/mc-boot/GmcFileName.h
index 11f1512..6c7ec75 100644
--- a/gcc/m2/mc-boot/GmcFileName.h
+++ b/gcc/m2/mc-boot/GmcFileName.h
@@ -50,7 +50,7 @@ extern "C" {
                        given a module and an extension. This file name
                        length will be operating system specific.
                        String, Extension, is concatenated onto
-                       Module and thus it is safe to `Mark' the extension
+                       Module and thus it is safe to Mark the extension
                        for garbage collection.
 */
 
diff --git a/gcc/m2/mc/decl.mod b/gcc/m2/mc/decl.mod
index 342487e..197ca5e 100644
--- a/gcc/m2/mc/decl.mod
+++ b/gcc/m2/mc/decl.mod
@@ -4643,6 +4643,28 @@ END getLiteralStringContents ;
 
 
 (*
+   getStringChar - if the string is delimited by single
+                   or double quotes then strip both
+                   quotes from the string.
+*)
+
+PROCEDURE getStringChar (n: node) : String ;
+VAR
+   s: String ;
+BEGIN
+   s := getString (n) ;
+   IF (DynamicStrings.char (s, 0) = "'") AND (DynamicStrings.char (s, -1) = "'")
+   THEN
+      s := DynamicStrings.Slice (s, 1, -1)
+   ELSIF (DynamicStrings.char (s, 0) = '"') AND (DynamicStrings.char (s, -1) = '"')
+   THEN
+      s := DynamicStrings.Slice (s, 1, -1)
+   END ;
+   RETURN s
+END getStringChar ;
+
+
+(*
    getStringContents - return the string contents of a constant, literal,
                        string or a constexp node.
 *)
@@ -4657,7 +4679,7 @@ BEGIN
       RETURN getLiteralStringContents (n)
    ELSIF isString (n)
    THEN
-      RETURN getString (n)
+      RETURN getStringChar (n)
    ELSIF isConstExp (n)
    THEN
       RETURN getStringContents (n^.unaryF.arg)
@@ -4709,11 +4731,27 @@ END resolveString ;
 
 
 (*
-   foldBinary -
+   addQuotes - adds delimiter quote char to string.
+*)
+
+PROCEDURE addQuotes (s: String; quote: CHAR) : String ;
+VAR
+   qs: String ;
+BEGIN
+   s := DynamicStrings.ConCatChar (s, quote) ;
+   qs := DynamicStrings.InitStringChar (quote) ;
+   qs := DynamicStrings.ConCat (qs, DynamicStrings.Mark (s)) ;
+   RETURN qs
+END addQuotes ;
+
+
+(*
+   foldBinary - attempt to fold binary + for string constants.
 *)
 
 PROCEDURE foldBinary (k: nodeT; l, r: node; res: node) : node ;
 VAR
+   qc: CHAR ;
    n : node ;
    ls,
    rs: String ;
@@ -4723,7 +4761,12 @@ BEGIN
    THEN
       ls := getStringContents (l) ;
       rs := getStringContents (r) ;
+      qc := "'" ;
+      (* Add unquoted contents.  *)
       ls := DynamicStrings.Add (ls, rs) ;
+      (* Add quote.  *)
+      ls := addQuotes (ls, qc) ;
+      (* Build new string.  *)
       n := makeString (makekey (DynamicStrings.string (ls))) ;
       ls := DynamicStrings.KillString (ls) ;
       rs := DynamicStrings.KillString (rs)
diff --git a/gcc/machmode.h b/gcc/machmode.h
index 467681d9..2f2d349 100644
--- a/gcc/machmode.h
+++ b/gcc/machmode.h
@@ -958,7 +958,8 @@ private:
 
 /* Find the best mode to use to access a bit field.  */
 
-extern bool get_best_mode (int, int, poly_uint64, poly_uint64, unsigned int,
+extern bool get_best_mode (HOST_WIDE_INT, HOST_WIDE_INT,
+			   poly_uint64, poly_uint64, unsigned int,
 			   unsigned HOST_WIDE_INT, bool, scalar_int_mode *);
 
 /* Determine alignment, 1<=result<=BIGGEST_ALIGNMENT.  */
diff --git a/gcc/match.pd b/gcc/match.pd
index f4416d9..82e6e29 100644
--- a/gcc/match.pd
+++ b/gcc/match.pd
@@ -3500,7 +3500,18 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
      wide_int c2 = wi::to_wide (@2);
      wide_int sum = wi::add (c1, c2);
     }
-    (if (wi::eq_p (sum, wi::max_value (precision, SIGNED)))))))
+    (if (wi::eq_p (sum, wi::max_value (precision, SIGNED))))))
+
+(match (signed_integer_sat_add @0 @1)
+  /* T SUM = (T)((UT)X + (UT)IMM)
+     SAT_S_ADD = (X ^ SUM) < 0 && (X ^ IMM) >= 0 ? (-(T)(X < 0) ^ MAX) : SUM  */
+   (cond^ (ge (bit_ior:c (bit_xor:c @0 INTEGER_CST@1)
+                       (bit_not (bit_xor:c @0 (nop_convert@2 (plus (nop_convert @0)
+                       INTEGER_CST@3)))))
+                   integer_zerop)
+           (signed_integer_sat_val @0)
+           @2)
+   (if (wi::eq_p (wi::to_wide (@1), wi::to_wide (@3))))))
 
 /* Saturation sub for signed integer.  */
 (if (INTEGRAL_TYPE_P (type) && !TYPE_UNSIGNED (type))
@@ -3572,6 +3583,48 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
 	 || (wi::eq_p (int_cst_1, itype_max) && wi::eq_p (int_cst_2, limit_1)))
 	 && wi::eq_p (int_cst_3, otype_max)))))))
 
+/* Saturation mult for unsigned integer.  */
+(if (INTEGRAL_TYPE_P (type) && TYPE_UNSIGNED (type))
+  (match (unsigned_integer_sat_mul @0 @1)
+   /* SAT_U_MUL (X, Y) = {
+	WT x = (WT)a * (WT)b;
+	T max = -1;
+	if (x > (WT)(max))
+	  return max;
+	else
+	  return (T)x;
+      }
+      while WT is uint128_t, T is uint8_t, uint16_t, uint32_t or uint64_t.  */
+   (convert (min (widen_mult:c@3 (convert@4 (convert @0))
+				 (convert@5 (convert @1)))
+		   INTEGER_CST@2))
+   (if (types_match (type, @0, @1))
+    (with
+     {
+      unsigned prec = TYPE_PRECISION (type);
+      unsigned widen_prec = TYPE_PRECISION (TREE_TYPE (@3));
+      unsigned cvt4_prec = TYPE_PRECISION (TREE_TYPE (@4));
+      unsigned cvt5_prec = TYPE_PRECISION (TREE_TYPE (@5));
+      wide_int c2 = wi::to_wide (@2);
+      wide_int max = wi::mask (prec, false, widen_prec);
+      bool c2_is_max_p = wi::eq_p (c2, max);
+      bool widen_mult_p = cvt4_prec == cvt5_prec && widen_prec == cvt5_prec * 2;
+     }
+     (if (widen_prec > prec && c2_is_max_p && widen_mult_p)))))
+  (match (unsigned_integer_sat_mul @0 @1)
+   (convert (min (mult:c@3 (convert @0) (convert @1)) INTEGER_CST@2))
+   (if (types_match (type, @0, @1))
+    (with
+     {
+      unsigned prec = TYPE_PRECISION (type);
+      unsigned widen_prec = TYPE_PRECISION (TREE_TYPE (@3));
+      wide_int c2 = wi::to_wide (@2);
+      wide_int max = wi::mask (prec, false, widen_prec);
+      bool c2_is_max_p = wi::eq_p (c2, max);
+     }
+     (if (widen_prec > prec && c2_is_max_p)))))
+)
+
 /* The boundary condition for case 10: IMM = 1:
    SAT_U_SUB = X >= IMM ? (X - IMM) : 0.
    simplify (X != 0 ? X + ~0 : 0) to X - (X != 0).  */
@@ -3674,6 +3727,8 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
    (if ((TREE_CODE (@1) == INTEGER_CST
 	 && TREE_CODE (@2) == INTEGER_CST)
 	|| ((INTEGRAL_TYPE_P (TREE_TYPE (@1))
+   || (VECTOR_TYPE_P (TREE_TYPE (@1))
+   && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, code2))
 	     || POINTER_TYPE_P (TREE_TYPE (@1)))
 	    && bitwise_equal_p (@1, @2)))
     (with
@@ -3712,27 +3767,39 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
       (if (code1 == EQ_EXPR && val) @3)
       (if (code1 == EQ_EXPR && !val) { constant_boolean_node (false, type); })
       (if (code1 == NE_EXPR && !val && allbits) @4)
-      (if (code1 == NE_EXPR
+      (if ((code1 == NE_EXPR
            && code2 == GE_EXPR
 	   && cmp == 0
 	   && allbits)
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, GT_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (gt @c0 (convert @1)))
-      (if (code1 == NE_EXPR
+      (if ((code1 == NE_EXPR
            && code2 == LE_EXPR
 	   && cmp == 0
 	   && allbits)
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, LT_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (lt @c0 (convert @1)))
       /* (a != (b+1)) & (a > b) -> a > (b+1) */
-      (if (code1 == NE_EXPR
+      (if ((code1 == NE_EXPR
            && code2 == GT_EXPR
 	   && one_after
 	   && allbits)
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, GT_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (gt @c0 (convert @1)))
       /* (a != (b-1)) & (a < b) -> a < (b-1) */
-      (if (code1 == NE_EXPR
+      (if ((code1 == NE_EXPR
            && code2 == LT_EXPR
 	   && one_before
 	   && allbits)
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, LT_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (lt @c0 (convert @1)))
      )
     )
@@ -3751,6 +3818,8 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
   (if ((TREE_CODE (@1) == INTEGER_CST
 	&& TREE_CODE (@2) == INTEGER_CST)
        || ((INTEGRAL_TYPE_P (TREE_TYPE (@1))
+       || (VECTOR_TYPE_P (TREE_TYPE (@1))
+       && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, code2))
 	    || POINTER_TYPE_P (TREE_TYPE (@1)))
 	   && operand_equal_p (@1, @2)))
    (with
@@ -3801,6 +3870,7 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
    (if ((TREE_CODE (@1) == INTEGER_CST
 	 && TREE_CODE (@2) == INTEGER_CST)
 	|| ((INTEGRAL_TYPE_P (TREE_TYPE (@1))
+   || (VECTOR_TYPE_P (TREE_TYPE (@1)))
 	    || POINTER_TYPE_P (TREE_TYPE (@1)))
 	    && bitwise_equal_p (@1, @2)))
     (with
@@ -3842,24 +3912,36 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
       (if (code1 == EQ_EXPR
            && code2 == GT_EXPR
 	   && cmp == 0
-	   && allbits)
+	   && allbits
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, GE_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (ge @c0 @2))
       (if (code1 == EQ_EXPR
            && code2 == LT_EXPR
 	   && cmp == 0
-	   && allbits)
+	   && allbits
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, LE_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (le @c0 @2))
       /* (a == (b-1)) | (a >= b) -> a >= (b-1) */
       (if (code1 == EQ_EXPR
            && code2 == GE_EXPR
 	   && one_before
-	   && allbits)
+	   && allbits
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, GE_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (ge @c0 (convert @1)))
       /* (a == (b+1)) | (a <= b) -> a <= (b-1) */
       (if (code1 == EQ_EXPR
            && code2 == LE_EXPR
 	   && one_after
-	   && allbits)
+	   && allbits
+      && ((VECTOR_BOOLEAN_TYPE_P (type)
+      && expand_vec_cmp_expr_p (TREE_TYPE (@1), type, LE_EXPR))
+      || !VECTOR_TYPE_P (TREE_TYPE (@1))))
        (le @c0 (convert @1)))
      )
     )
@@ -3924,7 +4006,10 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
       rcmp (ne le gt ne lt ge)
   (simplify
    (op:c (cmp1:c @0 @1) (cmp2 @0 @1))
-   (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)) || POINTER_TYPE_P (TREE_TYPE (@0)))
+   (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
+      || POINTER_TYPE_P (TREE_TYPE (@0))
+      || (VECTOR_TYPE_P (TREE_TYPE (@1))
+      && expand_vec_cmp_expr_p (TREE_TYPE (@0), type, rcmp)))
     (rcmp @0 @1)))))
 
 /* Optimize (a CMP b) == (a CMP b)  */
@@ -3933,7 +4018,10 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
      rcmp (eq gt le eq ge lt)
  (simplify
   (eq:c (cmp1:c @0 @1) (cmp2 @0 @1))
-  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0)) || POINTER_TYPE_P (TREE_TYPE (@0)))
+  (if (INTEGRAL_TYPE_P (TREE_TYPE (@0))
+		|| POINTER_TYPE_P (TREE_TYPE (@0))
+      || (VECTOR_TYPE_P (TREE_TYPE (@0))
+      && expand_vec_cmp_expr_p (TREE_TYPE (@0), type,  rcmp)))
     (rcmp @0 @1))))
 
 /* (type)([0,1]@a != 0) -> (type)a
@@ -5939,6 +6027,15 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
 	       && !expand_vec_cond_expr_p (TREE_TYPE (@1), TREE_TYPE (@0)))))
    (vec_cond @0 (op! @1 @3) (op! @2 @4))))
 
+/* (@0 ? @2 : @3) lop (@1 ? @2 : @3)  -->  (@0 lop @1) ? @2 : @3.  */
+(for lop (bit_and bit_ior bit_xor)
+   (simplify
+   (lop
+      (vec_cond @0 integer_minus_onep@2 integer_zerop@3)
+      (vec_cond @1 @2 @3))
+      (if (target_supports_op_p (TREE_TYPE (@0), lop, optab_vector))
+   (vec_cond (lop @0 @1) @2 @3))))
+
 /* (c ? a : b) op d  -->  c ? (a op d) : (b op d) */
  (simplify
   (op (vec_cond:s @0 @1 @2) @3)
@@ -6950,6 +7047,22 @@ DEFINE_INT_AND_FLOAT_ROUND_FN (RINT)
     (icmp @0 @1)
     (if (ic == ncmp)
      (ncmp @0 @1)))))
+ /* ((cast)cmp) - 1 -> -(cast)icmp . */
+ (simplify
+  (plus (convert? (cmp@2 @0 @1)) integer_minus_onep)
+  (if (TYPE_PRECISION (type) > 1
+       && INTEGRAL_TYPE_P (TREE_TYPE (@2)) && TYPE_PRECISION (TREE_TYPE (@2)) == 1)
+   /* Comparison inversion may be impossible for trapping math,
+      invert_tree_comparison will tell us.  But we can't use
+      a computed operator in the replacement tree thus we have
+      to play the trick below.  */
+   (with { enum tree_code ic = invert_tree_comparison
+             (cmp, HONOR_NANS (@0));
+           tree cmptype = TREE_TYPE (@2); }
+    (if (ic == icmp)
+     (negate (convert (icmp:cmptype @0 @1)))
+     (if (ic == ncmp)
+      (negate (convert (ncmp:cmptype @0 @1))))))))
  /* The following bits are handled by fold_binary_op_with_conditional_arg.  */
  (simplify
   (ne (cmp@2 @0 @1) integer_zerop)
diff --git a/gcc/optabs-query.cc b/gcc/optabs-query.cc
index f5ca98d..5335d0d 100644
--- a/gcc/optabs-query.cc
+++ b/gcc/optabs-query.cc
@@ -719,13 +719,9 @@ supports_vec_gather_load_p (machine_mode mode, vec<int> *elsvals)
 	= (icode != CODE_FOR_nothing) ? 1 : -1;
     }
 
-  /* For gather the optab's operand indices do not match the IFN's because
-     the latter does not have the extension operand (operand 3).  It is
-     implicitly added during expansion so we use the IFN's else index + 1.
-     */
   if (elsvals && icode != CODE_FOR_nothing)
     get_supported_else_vals
-      (icode, internal_fn_else_index (IFN_MASK_GATHER_LOAD) + 1, *elsvals);
+      (icode, internal_fn_else_index (IFN_MASK_GATHER_LOAD), *elsvals);
 
   return this_fn_optabs->supports_vec_gather_load[mode] > 0;
 }
diff --git a/gcc/optabs-tree.cc b/gcc/optabs-tree.cc
index 6dfe8ee..0de74c7 100644
--- a/gcc/optabs-tree.cc
+++ b/gcc/optabs-tree.cc
@@ -82,6 +82,9 @@ optab_for_tree_code (enum tree_code code, const_tree type,
 	return unknown_optab;
       /* FALLTHRU */
     case RDIV_EXPR:
+      gcc_assert (FLOAT_TYPE_P (type)
+		  || ALL_FIXED_POINT_MODE_P (TYPE_MODE (type)));
+      /* FALLTHRU */
     case TRUNC_DIV_EXPR:
     case EXACT_DIV_EXPR:
       if (TYPE_SATURATING (type))
diff --git a/gcc/optabs.def b/gcc/optabs.def
index 0c1435d..87a8b85 100644
--- a/gcc/optabs.def
+++ b/gcc/optabs.def
@@ -134,8 +134,8 @@ OPTAB_NX(smul_optab, "mul$P$a3")
 OPTAB_NX(smul_optab, "mul$F$a3")
 OPTAB_VL(smulv_optab, "mulv$I$a3", MULT, "mul", '3', gen_intv_fp_libfunc)
 OPTAB_VX(smulv_optab, "mul$F$a3")
-OPTAB_NL(ssmul_optab, "ssmul$Q$a3", SS_MULT, "ssmul", '3', gen_signed_fixed_libfunc)
-OPTAB_NL(usmul_optab, "usmul$Q$a3", US_MULT, "usmul", '3', gen_unsigned_fixed_libfunc)
+OPTAB_NL(ssmul_optab, "ssmul$a3", SS_MULT, "ssmul", '3', gen_signed_fixed_libfunc)
+OPTAB_NL(usmul_optab, "usmul$a3", US_MULT, "usmul", '3', gen_unsigned_fixed_libfunc)
 OPTAB_NL(sdiv_optab, "div$a3", DIV, "div", '3', gen_int_fp_signed_fixed_libfunc)
 OPTAB_VL(sdivv_optab, "divv$I$a3", DIV, "divv", '3', gen_int_libfunc)
 OPTAB_VX(sdivv_optab, "div$F$a3")
diff --git a/gcc/optc-gen.awk b/gcc/optc-gen.awk
index 3d074e4..4e9b538 100644
--- a/gcc/optc-gen.awk
+++ b/gcc/optc-gen.awk
@@ -497,7 +497,7 @@ print "                           const struct cl_decoded_option *decoded,   "
 print "                           unsigned int lang_mask, int kind,          "
 print "                           location_t loc,                            "
 print "                           const struct cl_option_handlers *handlers, "
-print "                           diagnostic_context *dc)                    "
+print "                           diagnostics::context *dc)                  "
 print "{                                                                     "
 print "  size_t scode = decoded->opt_index;                                  "
 print "  HOST_WIDE_INT value = decoded->value;                               "
@@ -554,7 +554,7 @@ for (i = 0; i < n_langs; i++) {
     print "                           unsigned int lang_mask" mark_unused ", int kind" mark_unused ",          "
     print "                           location_t loc" mark_unused ",                            "
     print "                           const struct cl_option_handlers *handlers" mark_unused ", "
-    print "                           diagnostic_context *dc" mark_unused ")                    "
+    print "                           diagnostics::context *dc" mark_unused ")                  "
     print "{                                                                     "
     print "  enum opt_code code = (enum opt_code) scode;                         "
     print "                                                                      "
diff --git a/gcc/optc-save-gen.awk b/gcc/optc-save-gen.awk
index a3d7e5a..31756ec 100644
--- a/gcc/optc-save-gen.awk
+++ b/gcc/optc-save-gen.awk
@@ -1313,6 +1313,12 @@ for (i = 0; i < n_opts; i++) {
 		# offloading is enabled.
 		if (flag_set_p("Target", flags[i]))
 			var_target_opt[n_opt_val] = 1;
+
+		# These options should not be passed from host to target, but
+		# are not actually target specific.
+		if (flag_set_p("NoOffload", flags[i]))
+			var_target_opt[n_opt_val] = 2;
+
 		n_opt_val++;
 	}
 }
@@ -1393,7 +1399,7 @@ for (i = 0; i < n_opt_val; i++) {
 		# Do not stream out target-specific opts if offloading is
 		# enabled.
 		if (var_target_opt[i])
-			print "  if (!lto_stream_offload_p)"
+			print "  if (!lto_stream_offload_p) {"
 		# If applicable, encode the streamed value.
 		if (var_opt_optimize_init[i]) {
 			print "  if (" var_opt_optimize_init[i] " > (" var_opt_val_type[i] ") 10)";
@@ -1403,6 +1409,8 @@ for (i = 0; i < n_opt_val; i++) {
 		} else {
 			print "  bp_pack_var_len_" sgn " (bp, ptr->" name");";
 		}
+		if (var_target_opt[i])
+			print "}"
 	}
 }
 print "  for (size_t i = 0; i < ARRAY_SIZE (ptr->explicit_mask); i++)";
@@ -1418,10 +1426,14 @@ print "                           struct cl_optimization *ptr ATTRIBUTE_UNUSED)"
 print "{";
 for (i = 0; i < n_opt_val; i++) {
 	name = var_opt_val[i]
-        if (var_target_opt[i]) {
+        if (var_target_opt[i] == 1) {
 		print "#ifdef ACCEL_COMPILER"
 		print "#error accel compiler cannot define Optimization attribute for target-specific option " name;
 		print "#else"
+	} else if (var_target_opt[i] == 2) {
+		print "#ifdef ACCEL_COMPILER"
+		print "  ptr->" name " = global_options." name ";"
+		print "#else"
 	}
 	otype = var_opt_val_type[i];
 	if (otype ~ "^const char \\**$") {
@@ -1489,6 +1501,9 @@ for (i = 0; i < n_opts; i++) {
 	if (flag_set_p("Warning", flags[i]))
 		continue;
 
+	if (flag_set_p("NoOffload", flags[i]))
+		continue;
+
 	if (name in checked_options)
 		continue;
 	checked_options[name]++
diff --git a/gcc/opth-gen.awk b/gcc/opth-gen.awk
index b6613d3..91fcf56 100644
--- a/gcc/opth-gen.awk
+++ b/gcc/opth-gen.awk
@@ -357,7 +357,7 @@ print "                           const struct cl_decoded_option *decoded,   "
 print "                           unsigned int lang_mask, int kind,          "
 print "                           location_t loc,                            "
 print "                           const struct cl_option_handlers *handlers, "
-print "                           diagnostic_context *dc);                   "
+print "                           diagnostics::context *dc);                   "
 for (i = 0; i < n_langs; i++) {
     lang_name = lang_sanitized_name(langs[i]);
     print "bool"
@@ -368,7 +368,7 @@ for (i = 0; i < n_langs; i++) {
     print "                           unsigned int lang_mask, int kind,"
     print "                           location_t loc,"
     print "                           const struct cl_option_handlers *handlers,"
-    print "                           diagnostic_context *dc);"
+    print "                           diagnostics::context *dc);"
 }
 print "void cpp_handle_option_auto (const struct gcc_options * opts, size_t scode,"
 print "                             struct cpp_options * cpp_opts);"
diff --git a/gcc/opts-common.cc b/gcc/opts-common.cc
index 03b1a79..91cad49 100644
--- a/gcc/opts-common.cc
+++ b/gcc/opts-common.cc
@@ -1282,9 +1282,9 @@ keep:
 
 /* Handle option DECODED for the language indicated by LANG_MASK,
    using the handlers in HANDLERS and setting fields in OPTS and
-   OPTS_SET.  KIND is the diagnostic_t if this is a diagnostics
-   option, DK_UNSPECIFIED otherwise, and LOC is the location of the
-   option for options from the source file, UNKNOWN_LOCATION
+   OPTS_SET.  KIND is the enum diagnostics::kind if this is a diagnostics
+   option, diagnostics::kind::unspecified otherwise, and LOC is the location
+   of the option for options from the source file, UNKNOWN_LOCATION
    otherwise.  GENERATED_P is true for an option generated as part of
    processing another option or otherwise generated internally, false
    for one explicitly passed by the user.  control_warning_option
@@ -1298,7 +1298,7 @@ handle_option (struct gcc_options *opts,
 	       const struct cl_decoded_option *decoded,
 	       unsigned int lang_mask, int kind, location_t loc,
 	       const struct cl_option_handlers *handlers,
-	       bool generated_p, diagnostic_context *dc)
+	       bool generated_p, diagnostics::context *dc)
 {
   size_t opt_index = decoded->opt_index;
   const char *arg = decoded->arg;
@@ -1336,7 +1336,7 @@ handle_generated_option (struct gcc_options *opts,
 			 size_t opt_index, const char *arg, HOST_WIDE_INT value,
 			 unsigned int lang_mask, int kind, location_t loc,
 			 const struct cl_option_handlers *handlers,
-			 bool generated_p, diagnostic_context *dc)
+			 bool generated_p, diagnostics::context *dc)
 {
   struct cl_decoded_option decoded;
 
@@ -1604,7 +1604,7 @@ read_cmdline_option (struct gcc_options *opts,
 		     location_t loc,
 		     unsigned int lang_mask,
 		     const struct cl_option_handlers *handlers,
-		     diagnostic_context *dc)
+		     diagnostics::context *dc)
 {
   const struct cl_option *option;
   const char *opt = decoded->orig_option_with_args_text;
@@ -1645,7 +1645,8 @@ read_cmdline_option (struct gcc_options *opts,
 
   gcc_assert (!decoded->errors);
 
-  if (!handle_option (opts, opts_set, decoded, lang_mask, DK_UNSPECIFIED,
+  if (!handle_option (opts, opts_set, decoded, lang_mask,
+		      static_cast<int> (diagnostics::kind::unspecified),
 		      loc, handlers, false, dc))
     error_at (loc, "unrecognized command-line option %qs", opt);
 }
@@ -1658,7 +1659,7 @@ read_cmdline_option (struct gcc_options *opts,
 void
 set_option (struct gcc_options *opts, struct gcc_options *opts_set,
 	    size_t opt_index, HOST_WIDE_INT value, const char *arg,
-	    int kind, location_t loc, diagnostic_context *dc,
+	    int kind, location_t loc, diagnostics::context *dc,
 	    HOST_WIDE_INT mask /* = 0 */)
 {
   const struct cl_option *option = &cl_options[opt_index];
@@ -1668,8 +1669,10 @@ set_option (struct gcc_options *opts, struct gcc_options *opts_set,
   if (!flag_var)
     return;
 
-  if ((diagnostic_t) kind != DK_UNSPECIFIED && dc != NULL)
-    diagnostic_classify_diagnostic (dc, opt_index, (diagnostic_t) kind, loc);
+  if ((enum diagnostics::kind) kind != diagnostics::kind::unspecified
+      && dc != nullptr)
+    diagnostic_classify_diagnostic (dc, opt_index,
+				    (enum diagnostics::kind) kind, loc);
 
   if (opts_set != NULL)
     set_flag_var = option_flag_var (opt_index, opts_set);
@@ -1874,7 +1877,7 @@ option_enabled (int opt_idx, unsigned lang_mask, void *opts)
 
 int
 compiler_diagnostic_option_manager::
-option_enabled_p (diagnostic_option_id opt_id) const
+option_enabled_p (diagnostics::option_id opt_id) const
 {
   return option_enabled (opt_id.m_idx, m_lang_mask, m_opts);
 }
@@ -1941,7 +1944,7 @@ control_warning_option (unsigned int opt_index, int kind, const char *arg,
 			const struct cl_option_handlers *handlers,
 			struct gcc_options *opts,
 			struct gcc_options *opts_set,
-			diagnostic_context *dc)
+			diagnostics::context *dc)
 {
   if (cl_options[opt_index].alias_target != N_OPTS)
     {
@@ -1954,7 +1957,9 @@ control_warning_option (unsigned int opt_index, int kind, const char *arg,
   if (opt_index == OPT_SPECIAL_ignore || opt_index == OPT_SPECIAL_warn_removed)
     return;
   if (dc)
-    diagnostic_classify_diagnostic (dc, opt_index, (diagnostic_t) kind, loc);
+    diagnostic_classify_diagnostic (dc, opt_index,
+				    static_cast<enum diagnostics::kind> (kind),
+				    loc);
   if (imply)
     {
       /* -Werror=foo implies -Wfoo.  */
diff --git a/gcc/opts-diagnostic.cc b/gcc/opts-diagnostic.cc
index d2c3a93..70d7d74 100644
--- a/gcc/opts-diagnostic.cc
+++ b/gcc/opts-diagnostic.cc
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 
 /* This file implements the options -fdiagnostics-add-output=,
    -fdiagnostics-set-output=.  Most of the work is done
-   by diagnostic-output-spec.cc so it can be shared by libgdiagnostics.  */
+   by diagnostics/output-spec.cc so it can be shared by libgdiagnostics.  */
 
 #include "config.h"
 #define INCLUDE_ARRAY
@@ -31,7 +31,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "version.h"
 #include "intl.h"
 #include "diagnostic.h"
-#include "diagnostic-output-spec.h"
+#include "diagnostics/output-spec.h"
 #include "opts.h"
 #include "options.h"
 
@@ -39,19 +39,19 @@ along with GCC; see the file COPYING3.  If not see
 
 namespace {
 
-struct opt_spec_context : public diagnostics_output_spec::gcc_spec_context
+struct opt_spec_context : public diagnostics::output_spec::dc_spec_context
 {
 public:
   opt_spec_context (const gcc_options &opts,
-		    diagnostic_context &dc,
+		    diagnostics::context &dc,
 		    line_maps *location_mgr,
 		    location_t loc,
 		    const char *option_name)
-  : gcc_spec_context (dc,
-		      location_mgr,
-		      location_mgr,
-		      loc,
-		      option_name),
+  : dc_spec_context (dc,
+		     location_mgr,
+		     location_mgr,
+		     loc,
+		     option_name),
     m_opts (opts)
   {}
 
@@ -70,7 +70,7 @@ public:
 
 void
 handle_OPT_fdiagnostics_add_output_ (const gcc_options &opts,
-				     diagnostic_context &dc,
+				     diagnostics::context &dc,
 				     const char *arg,
 				     location_t loc)
 {
@@ -89,7 +89,7 @@ handle_OPT_fdiagnostics_add_output_ (const gcc_options &opts,
 
 void
 handle_OPT_fdiagnostics_set_output_ (const gcc_options &opts,
-				     diagnostic_context &dc,
+				     diagnostics::context &dc,
 				     const char *arg,
 				     location_t loc)
 {
@@ -103,5 +103,5 @@ handle_OPT_fdiagnostics_set_output_ (const gcc_options &opts,
     return;
 
   sink->set_main_input_filename (opts.x_main_input_filename);
-  dc.set_output_format (std::move (sink));
+  dc.set_sink (std::move (sink));
 }
diff --git a/gcc/opts-diagnostic.h b/gcc/opts-diagnostic.h
index 384fbb3..4fa4ea8 100644
--- a/gcc/opts-diagnostic.h
+++ b/gcc/opts-diagnostic.h
@@ -20,12 +20,12 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_OPTS_DIAGNOSTIC_H
 #define GCC_OPTS_DIAGNOSTIC_H
 
-/* Abstract subclass of diagnostic_option_manager for gcc options.  */
+/* Abstract subclass of diagnostics::option_manager for gcc options.  */
 
-class gcc_diagnostic_option_manager : public diagnostic_option_manager
+class gcc_diagnostic_option_manager : public diagnostics::option_manager
 {
 public:
-  char *make_option_url (diagnostic_option_id option_id) const final override;
+  char *make_option_url (diagnostics::option_id option_id) const final override;
 
 protected:
   gcc_diagnostic_option_manager (unsigned lang_mask)
@@ -40,7 +40,7 @@ protected:
 class compiler_diagnostic_option_manager : public gcc_diagnostic_option_manager
 {
 public:
-  compiler_diagnostic_option_manager (const diagnostic_context &context,
+  compiler_diagnostic_option_manager (const diagnostics::context &context,
 				      unsigned lang_mask,
 				      void *opts)
   : gcc_diagnostic_option_manager (lang_mask),
@@ -49,25 +49,26 @@ public:
   {
   }
 
-  int option_enabled_p (diagnostic_option_id option_id) const final override;
-  char *make_option_name (diagnostic_option_id option_id,
-			  diagnostic_t orig_diag_kind,
-			  diagnostic_t diag_kind) const final override;
+  int option_enabled_p (diagnostics::option_id option_id) const final override;
+  char *
+  make_option_name (diagnostics::option_id option_id,
+		    enum diagnostics::kind orig_diag_kind,
+		    enum diagnostics::kind diag_kind) const final override;
 
 private:
-  const diagnostic_context &m_context;
+  const diagnostics::context &m_context;
   void *m_opts;
 };
 
 extern void
 handle_OPT_fdiagnostics_add_output_ (const gcc_options &opts,
-				     diagnostic_context &dc,
+				     diagnostics::context &dc,
 				     const char *arg,
 				     location_t loc);
 
 extern void
 handle_OPT_fdiagnostics_set_output_ (const gcc_options &opts,
-				     diagnostic_context &dc,
+				     diagnostics::context &dc,
 				     const char *arg,
 				     location_t loc);
 #endif
diff --git a/gcc/opts-global.cc b/gcc/opts-global.cc
index b9b42d3..701a6bb 100644
--- a/gcc/opts-global.cc
+++ b/gcc/opts-global.cc
@@ -181,7 +181,7 @@ lang_handle_option (struct gcc_options *opts,
 		    unsigned int lang_mask ATTRIBUTE_UNUSED, int kind,
 		    location_t loc,
 		    const struct cl_option_handlers *handlers,
-		    diagnostic_context *dc,
+		    diagnostics::context *dc,
 		    void (*) (void))
 {
   gcc_assert (opts == &global_options);
@@ -215,7 +215,7 @@ read_cmdline_options (struct gcc_options *opts, struct gcc_options *opts_set,
 		      location_t loc,
 		      unsigned int lang_mask,
 		      const struct cl_option_handlers *handlers,
-		      diagnostic_context *dc)
+		      diagnostics::context *dc)
 {
   unsigned int i;
 
@@ -312,7 +312,7 @@ void
 decode_options (struct gcc_options *opts, struct gcc_options *opts_set,
 		struct cl_decoded_option *decoded_options,
 		unsigned int decoded_options_count,
-		location_t loc, diagnostic_context *dc,
+		location_t loc, diagnostics::context *dc,
 		void (*target_option_override_hook) (void))
 {
   struct cl_option_handlers handlers;
diff --git a/gcc/opts.cc b/gcc/opts.cc
index 6ca1ec7..c21e66b 100644
--- a/gcc/opts.cc
+++ b/gcc/opts.cc
@@ -31,8 +31,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "common/common-target.h"
 #include "spellcheck.h"
 #include "opt-suggestions.h"
-#include "diagnostic-color.h"
-#include "diagnostic-format.h"
+#include "diagnostics/color.h"
+#include "diagnostics/sink.h"
 #include "version.h"
 #include "selftest.h"
 #include "file-prefix-map.h"
@@ -347,7 +347,7 @@ static void set_debug_level (uint32_t dinfo, int extended,
 			     location_t loc);
 static void set_fast_math_flags (struct gcc_options *opts, int set);
 static void decode_d_option (const char *arg, struct gcc_options *opts,
-			     location_t loc, diagnostic_context *dc);
+			     location_t loc, diagnostics::context *dc);
 static void set_unsafe_math_optimizations_flags (struct gcc_options *opts,
 						 int set);
 static void enable_warning_as_error (const char *arg, int value,
@@ -356,7 +356,7 @@ static void enable_warning_as_error (const char *arg, int value,
 				     struct gcc_options *opts,
 				     struct gcc_options *opts_set,
 				     location_t loc,
-				     diagnostic_context *dc);
+				     diagnostics::context *dc);
 
 /* Handle a back-end option; arguments and return value as for
    handle_option.  */
@@ -368,10 +368,11 @@ target_handle_option (struct gcc_options *opts,
 		      unsigned int lang_mask ATTRIBUTE_UNUSED, int kind,
 		      location_t loc,
 		      const struct cl_option_handlers *handlers ATTRIBUTE_UNUSED,
-		      diagnostic_context *dc, void (*) (void))
+		      diagnostics::context *dc, void (*) (void))
 {
   gcc_assert (dc == global_dc);
-  gcc_assert (kind == DK_UNSPECIFIED);
+  gcc_assert (static_cast<diagnostics::kind> (kind)
+	      == diagnostics::kind::unspecified);
   return targetm_common.handle_option (opts, opts_set, decoded, loc);
 }
 
@@ -469,7 +470,7 @@ maybe_default_option (struct gcc_options *opts,
 		      unsigned int lang_mask,
 		      const struct cl_option_handlers *handlers,
 		      location_t loc,
-		      diagnostic_context *dc)
+		      diagnostics::context *dc)
 {
   const struct cl_option *option = &cl_options[default_opt->opt_index];
   bool enabled;
@@ -535,14 +536,18 @@ maybe_default_option (struct gcc_options *opts,
   if (enabled)
     handle_generated_option (opts, opts_set, default_opt->opt_index,
 			     default_opt->arg, default_opt->value,
-			     lang_mask, DK_UNSPECIFIED, loc,
+			     lang_mask,
+			     static_cast<int> (diagnostics::kind::unspecified),
+			     loc,
 			     handlers, true, dc);
   else if (default_opt->arg == NULL
 	   && !option->cl_reject_negative
 	   && !(option->flags & CL_PARAMS))
     handle_generated_option (opts, opts_set, default_opt->opt_index,
 			     default_opt->arg, !default_opt->value,
-			     lang_mask, DK_UNSPECIFIED, loc,
+			     lang_mask,
+			     static_cast<int> (diagnostics::kind::unspecified),
+			     loc,
 			     handlers, true, dc);
 }
 
@@ -559,7 +564,7 @@ maybe_default_options (struct gcc_options *opts,
 		       unsigned int lang_mask,
 		       const struct cl_option_handlers *handlers,
 		       location_t loc,
-		       diagnostic_context *dc)
+		       diagnostics::context *dc)
 {
   size_t i;
 
@@ -727,7 +732,7 @@ default_options_optimization (struct gcc_options *opts,
 			      location_t loc,
 			      unsigned int lang_mask,
 			      const struct cl_option_handlers *handlers,
-			      diagnostic_context *dc)
+			      diagnostics::context *dc)
 {
   unsigned int i;
   int opt2;
@@ -1411,11 +1416,14 @@ finish_options (struct gcc_options *opts, struct gcc_options *opts_set,
 	opts->x_debug_info_level = DINFO_LEVEL_NONE;
     }
 
+  /* Also enable markers with -fauto-profile even when debug info is disabled,
+     so we assign same discriminators and can read back the profile info.  */
   if (!opts_set->x_debug_nonbind_markers_p)
     opts->x_debug_nonbind_markers_p
       = (opts->x_optimize
-	 && opts->x_debug_info_level >= DINFO_LEVEL_NORMAL
-	 && (dwarf_debuginfo_p (opts) || codeview_debuginfo_p ())
+	 && ((opts->x_debug_info_level >= DINFO_LEVEL_NORMAL
+	      && (dwarf_debuginfo_p (opts) || codeview_debuginfo_p ()))
+	     || opts->x_flag_auto_profile)
 	 && !(opts->x_flag_selective_scheduling
 	      || opts->x_flag_selective_scheduling2));
 
@@ -2700,7 +2708,7 @@ common_handle_option (struct gcc_options *opts,
 		      unsigned int lang_mask, int kind ATTRIBUTE_UNUSED,
 		      location_t loc,
 		      const struct cl_option_handlers *handlers,
-		      diagnostic_context *dc,
+		      diagnostics::context *dc,
 		      void (*target_option_override_hook) (void))
 {
   size_t scode = decoded->opt_index;
@@ -2954,19 +2962,19 @@ common_handle_option (struct gcc_options *opts,
       break;
 
     case OPT_fdiagnostics_show_caret:
-      dc->m_source_printing.enabled = value;
+      dc->get_source_printing_options ().enabled = value;
       break;
 
     case OPT_fdiagnostics_show_event_links:
-      dc->m_source_printing.show_event_links_p = value;
+      dc->get_source_printing_options ().show_event_links_p = value;
       break;
 
     case OPT_fdiagnostics_show_labels:
-      dc->m_source_printing.show_labels_p = value;
+      dc->get_source_printing_options ().show_labels_p = value;
       break;
 
     case OPT_fdiagnostics_show_line_numbers:
-      dc->m_source_printing.show_line_numbers_p = value;
+      dc->get_source_printing_options ().show_line_numbers_p = value;
       break;
 
     case OPT_fdiagnostics_color_:
@@ -2982,10 +2990,10 @@ common_handle_option (struct gcc_options *opts,
 	  const char *basename = (opts->x_dump_base_name ? opts->x_dump_base_name
 				  : opts->x_main_input_basename);
 	  gcc_assert (dc);
-	  diagnostic_output_format_init (*dc,
-					 opts->x_main_input_filename, basename,
-					 (enum diagnostics_output_format)value,
-					 opts->x_flag_diagnostics_json_formatting);
+	  diagnostics::output_format_init (*dc,
+					   opts->x_main_input_filename, basename,
+					   (enum diagnostics_output_format)value,
+					   opts->x_flag_diagnostics_json_formatting);
 	  break;
 	}
 
@@ -3044,7 +3052,7 @@ common_handle_option (struct gcc_options *opts,
       break;
 
     case OPT_fdiagnostics_minimum_margin_width_:
-      dc->m_source_printing.min_margin_width = value;
+      dc->get_source_printing_options ().min_margin_width = value;
       break;
 
     case OPT_fdump_:
@@ -3082,7 +3090,7 @@ common_handle_option (struct gcc_options *opts,
 
     case OPT_fmessage_length_:
       pp_set_line_maximum_length (dc->get_reference_printer (), value);
-      diagnostic_set_caret_max_width (dc, value);
+      dc->set_caret_max_width (value);
       break;
 
     case OPT_fopt_info:
@@ -3314,7 +3322,9 @@ common_handle_option (struct gcc_options *opts,
 
     case OPT_pedantic_errors:
       dc->m_pedantic_errors = 1;
-      control_warning_option (OPT_Wpedantic, DK_ERROR, NULL, value,
+      control_warning_option (OPT_Wpedantic,
+			      static_cast<int> (diagnostics::kind::error),
+			      NULL, value,
 			      loc, lang_mask,
 			      handlers, opts, opts_set,
                               dc);
@@ -3615,7 +3625,7 @@ set_debug_level (uint32_t dinfo, int extended, const char *arg,
    abort ().)  */
 
 static void
-setup_core_dumping (diagnostic_context *dc)
+setup_core_dumping (diagnostics::context *dc)
 {
 #ifdef SIGABRT
   signal (SIGABRT, SIG_DFL);
@@ -3639,7 +3649,7 @@ setup_core_dumping (diagnostic_context *dc)
 
 static void
 decode_d_option (const char *arg, struct gcc_options *opts,
-		 location_t loc, diagnostic_context *dc)
+		 location_t loc, diagnostics::context *dc)
 {
   int c;
 
@@ -3688,7 +3698,7 @@ enable_warning_as_error (const char *arg, int value, unsigned int lang_mask,
 			 const struct cl_option_handlers *handlers,
 			 struct gcc_options *opts,
 			 struct gcc_options *opts_set,
-			 location_t loc, diagnostic_context *dc)
+			 location_t loc, diagnostics::context *dc)
 {
   char *new_option;
   int option_index;
@@ -3714,7 +3724,9 @@ enable_warning_as_error (const char *arg, int value, unsigned int lang_mask,
 	      "controls warnings", arg, new_option);
   else
     {
-      const diagnostic_t kind = value ? DK_ERROR : DK_WARNING;
+      const enum diagnostics::kind kind = (value
+					   ? diagnostics::kind::error
+					   : diagnostics::kind::warning);
       const char *arg = NULL;
 
       if (cl_options[option_index].flags & CL_JOINED)
@@ -3733,15 +3745,16 @@ enable_warning_as_error (const char *arg, int value, unsigned int lang_mask,
 
 char *
 compiler_diagnostic_option_manager::
-make_option_name (diagnostic_option_id option_id,
-		  diagnostic_t orig_diag_kind,
-		  diagnostic_t diag_kind) const
+make_option_name (diagnostics::option_id option_id,
+		  enum diagnostics::kind orig_diag_kind,
+		  enum diagnostics::kind diag_kind) const
 {
   if (option_id.m_idx)
     {
       /* A warning classified as an error.  */
-      if ((orig_diag_kind == DK_WARNING || orig_diag_kind == DK_PEDWARN)
-	  && diag_kind == DK_ERROR)
+      if ((orig_diag_kind == diagnostics::kind::warning
+	   || orig_diag_kind == diagnostics::kind::pedwarn)
+	  && diag_kind == diagnostics::kind::error)
 	return concat (cl_options[OPT_Werror_].opt_text,
 		       /* Skip over "-W".  */
 		       cl_options[option_id.m_idx].opt_text + 2,
@@ -3751,8 +3764,9 @@ make_option_name (diagnostic_option_id option_id,
 	return xstrdup (cl_options[option_id.m_idx].opt_text);
     }
   /* A warning without option classified as an error.  */
-  else if ((orig_diag_kind == DK_WARNING || orig_diag_kind == DK_PEDWARN
-	    || diag_kind == DK_WARNING)
+  else if ((orig_diag_kind == diagnostics::kind::warning
+	    || orig_diag_kind == diagnostics::kind::pedwarn
+	    || diag_kind == diagnostics::kind::warning)
 	   && m_context.warning_as_error_requested_p ())
     return xstrdup (cl_options[OPT_Werror].opt_text);
   else
@@ -3810,7 +3824,7 @@ get_option_url_suffix (int option_index, unsigned lang_mask)
 
 char *
 gcc_diagnostic_option_manager::
-make_option_url (diagnostic_option_id option_id) const
+make_option_url (diagnostics::option_id option_id) const
 {
   if (option_id.m_idx)
     {
diff --git a/gcc/opts.h b/gcc/opts.h
index 382faab..ea92c49 100644
--- a/gcc/opts.h
+++ b/gcc/opts.h
@@ -313,7 +313,7 @@ struct cl_option_handler_func
 		   const struct cl_decoded_option *decoded,
 		   unsigned int lang_mask, int kind, location_t loc,
 		   const struct cl_option_handlers *handlers,
-		   diagnostic_context *dc,
+		   diagnostics::context *dc,
 		   void (*target_option_override_hook) (void));
 
   /* The mask that must have some bit in common with the flags for the
@@ -390,7 +390,7 @@ extern void decode_options (struct gcc_options *opts,
 			    struct cl_decoded_option *decoded_options,
 			    unsigned int decoded_options_count,
 			    location_t loc,
-			    diagnostic_context *dc,
+			    diagnostics::context *dc,
 			    void (*target_option_override_hook) (void));
 extern int option_enabled (int opt_idx, unsigned lang_mask, void *opts);
 
@@ -399,7 +399,7 @@ extern bool get_option_state (struct gcc_options *, int,
 extern void set_option (struct gcc_options *opts,
 			struct gcc_options *opts_set,
 			size_t opt_index, HOST_WIDE_INT value, const char *arg,
-			int kind, location_t loc, diagnostic_context *dc,
+			int kind, location_t loc, diagnostics::context *dc,
 			HOST_WIDE_INT = 0);
 extern void *option_flag_var (int opt_index, struct gcc_options *opts);
 bool handle_generated_option (struct gcc_options *opts,
@@ -408,7 +408,7 @@ bool handle_generated_option (struct gcc_options *opts,
 			      HOST_WIDE_INT value,
 			      unsigned int lang_mask, int kind, location_t loc,
 			      const struct cl_option_handlers *handlers,
-			      bool generated_p, diagnostic_context *dc);
+			      bool generated_p, diagnostics::context *dc);
 void generate_option (size_t opt_index, const char *arg, HOST_WIDE_INT value,
 		      unsigned int lang_mask,
 		      struct cl_decoded_option *decoded);
@@ -420,14 +420,14 @@ extern void read_cmdline_option (struct gcc_options *opts,
 				 location_t loc,
 				 unsigned int lang_mask,
 				 const struct cl_option_handlers *handlers,
-				 diagnostic_context *dc);
+				 diagnostics::context *dc);
 extern void control_warning_option (unsigned int opt_index, int kind,
 				    const char *arg, bool imply, location_t loc,
 				    unsigned int lang_mask,
 				    const struct cl_option_handlers *handlers,
 				    struct gcc_options *opts,
 				    struct gcc_options *opts_set,
-				    diagnostic_context *dc);
+				    diagnostics::context *dc);
 extern char *write_langs (unsigned int mask);
 extern void print_ignored_options (void);
 extern void handle_common_deferred_options (void);
@@ -442,7 +442,7 @@ extern bool common_handle_option (struct gcc_options *opts,
 				  unsigned int lang_mask, int kind,
 				  location_t loc,
 				  const struct cl_option_handlers *handlers,
-				  diagnostic_context *dc,
+				  diagnostics::context *dc,
 				  void (*target_option_override_hook) (void));
 extern bool target_handle_option (struct gcc_options *opts,
 				  struct gcc_options *opts_set,
@@ -450,7 +450,7 @@ extern bool target_handle_option (struct gcc_options *opts,
 				  unsigned int lang_mask, int kind,
 				  location_t loc,
 				  const struct cl_option_handlers *handlers,
-				  diagnostic_context *dc,
+				  diagnostics::context *dc,
 				  void (*target_option_override_hook) (void));
 extern void finish_options (struct gcc_options *opts,
 			    struct gcc_options *opts_set,
@@ -466,7 +466,7 @@ extern void default_options_optimization (struct gcc_options *opts,
 					  location_t loc,
 					  unsigned int lang_mask,
 					  const struct cl_option_handlers *handlers,
-					  diagnostic_context *dc);
+					  diagnostics::context *dc);
 extern void set_struct_debug_option (struct gcc_options *opts,
 				     location_t loc,
 				     const char *value);
diff --git a/gcc/output.h b/gcc/output.h
index 372d63c..51c2d36 100644
--- a/gcc/output.h
+++ b/gcc/output.h
@@ -545,6 +545,9 @@ extern GTY(()) section *bss_noswitch_section;
 extern GTY(()) section *in_section;
 extern GTY(()) bool in_cold_section_p;
 
+/* MAX bit alignment for mergable sections. */
+#define MAX_ALIGN_MERGABLE 256
+
 extern section *get_unnamed_section (unsigned int, void (*) (const char *),
 				     const char *);
 extern section *get_section (const char *, unsigned int, tree,
@@ -557,6 +560,9 @@ extern rtx get_section_anchor (struct object_block *, HOST_WIDE_INT,
 extern section *mergeable_constant_section (machine_mode,
 					    unsigned HOST_WIDE_INT,
 					    unsigned int);
+extern section *mergeable_constant_section (unsigned HOST_WIDE_INT,
+					    unsigned HOST_WIDE_INT,
+					    unsigned int);
 extern section *function_section (tree);
 extern section *unlikely_text_section (void);
 extern section *current_function_section (void);
@@ -639,4 +645,6 @@ extern int default_address_cost (rtx, machine_mode, addr_space_t, bool);
 /* Stack usage.  */
 extern void output_stack_usage (void);
 
+extern void verify_reg_names_in_constraints ();
+
 #endif /* ! GCC_OUTPUT_H */
diff --git a/gcc/params.opt b/gcc/params.opt
index 31aa0bd..ac1b2c7 100644
--- a/gcc/params.opt
+++ b/gcc/params.opt
@@ -1177,7 +1177,7 @@ Common Joined UInteger Var(param_uninit_control_dep_attempts) Init(1000) Integer
 Maximum number of nested calls to search for control dependencies during uninitialized variable analysis.
 
 -param=uninit-max-chain-len=
-Common Joined UInteger Var(param_uninit_max_chain_len) Init(8) IntegerRange(1, 128) Param Optimization
+Common Joined UInteger Var(param_uninit_max_chain_len) Init(12) IntegerRange(1, 128) Param Optimization
 Maximum number of predicates anded for each predicate ored in the normalized
 predicate chain.
 
@@ -1226,7 +1226,7 @@ Common Joined UInteger Var(param_use_canonical_types) Init(1) IntegerRange(0, 1)
 Whether to use canonical types.
 
 -param=vect-epilogues-nomask=
-Common Joined UInteger Var(param_vect_epilogues_nomask) Init(1) IntegerRange(0, 1) Param Optimization
+Common Joined UInteger Var(param_vect_epilogues_nomask) Init(1) IntegerRange(0, 1) Param Optimization NoOffload
 Enable loop epilogue vectorization using smaller vector size.
 
 -param=vect-max-layout-candidates=
@@ -1246,11 +1246,11 @@ Common Joined UInteger Var(param_vect_max_version_for_alignment_checks) Init(6)
 Bound on number of runtime checks inserted by the vectorizer's loop versioning for alignment check.
 
 -param=vect-partial-vector-usage=
-Common Joined UInteger Var(param_vect_partial_vector_usage) Init(2) IntegerRange(0, 2) Param Optimization
+Common Joined UInteger Var(param_vect_partial_vector_usage) Init(2) IntegerRange(0, 2) Param Optimization NoOffload
 Controls how loop vectorizer uses partial vectors.  0 means never, 1 means only for loops whose need to iterate can be removed, 2 means for all loops.  The default value is 2.
 
 -param=vect-inner-loop-cost-factor=
-Common Joined UInteger Var(param_vect_inner_loop_cost_factor) Init(50) IntegerRange(1, 10000) Param Optimization
+Common Joined UInteger Var(param_vect_inner_loop_cost_factor) Init(50) IntegerRange(1, 10000) Param Optimization NoOffload
 The maximum factor which the loop vectorizer applies to the cost of statements in an inner loop relative to the loop being vectorized.
 
 -param=vect-induction-float=
diff --git a/gcc/predict.cc b/gcc/predict.cc
index 872f54d..5639d81 100644
--- a/gcc/predict.cc
+++ b/gcc/predict.cc
@@ -245,7 +245,10 @@ unlikely_executed_edge_p (edge e)
 {
   return (e->src->count == profile_count::zero ()
 	  || e->probability == profile_probability::never ())
-	 || (e->flags & (EDGE_EH | EDGE_FAKE));
+	 || (e->flags & EDGE_FAKE)
+	 /* If we read profile and know EH edge is executed, trust it.
+	    Otherwise we consider EH edges never executed.  */
+	 || ((e->flags & EDGE_EH) && !e->probability.reliable_p ());
 }
 
 /* Return true if edge E of function FUN is probably never executed.  */
@@ -830,6 +833,26 @@ unlikely_executed_stmt_p (gimple *stmt)
 {
   if (!is_gimple_call (stmt))
     return false;
+
+  /* Those calls are inserted by optimizers when code is known to be
+     unreachable or undefined.  */
+  if (gimple_call_builtin_p (stmt, BUILT_IN_UNREACHABLE)
+      || gimple_call_builtin_p (stmt, BUILT_IN_UNREACHABLE_TRAP)
+      || gimple_call_builtin_p (stmt, BUILT_IN_TRAP))
+    return false;
+
+  /* Checks below do not need to be fully reliable.  Cold attribute may be
+     misplaced by user and in the presence of comdat we may result in call to
+     function with 0 profile having non-zero profile.
+
+     We later detect that profile is lost and will drop the profile of the
+     comdat.
+
+     So if we think profile count is reliable, do not try to apply these
+     heuristics.  */
+  if (gimple_bb (stmt)->count.reliable_p ()
+      && gimple_bb (stmt)->count.nonzero_p ())
+    return gimple_bb (stmt)->count == profile_count::zero ();
   /* NORETURN attribute alone is not strong enough: exit() may be quite
      likely executed once during program run.  */
   if (gimple_call_fntype (stmt)
@@ -3269,7 +3292,8 @@ tree_estimate_probability (bool dry_run)
   calculate_dominance_info (CDI_POST_DOMINATORS);
   /* Decide which edges are known to be unlikely.  This improves later
      branch prediction. */
-  determine_unlikely_bbs ();
+  if (!dry_run)
+    determine_unlikely_bbs ();
 
   bb_predictions = new hash_map<const_basic_block, edge_prediction *>;
   ssa_expected_value = new hash_map<int_hash<unsigned, 0>, expected_value>;
diff --git a/gcc/pretty-print-format-impl.h b/gcc/pretty-print-format-impl.h
index cbbd21f..2aded64 100644
--- a/gcc/pretty-print-format-impl.h
+++ b/gcc/pretty-print-format-impl.h
@@ -22,7 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 #define GCC_PRETTY_PRINT_FORMAT_IMPL_H
 
 #include "pretty-print.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 
 /* A struct representing a pending item to be printed within
    pp_format.
@@ -224,14 +224,14 @@ struct pp_token_end_url : public pp_token
 
 struct pp_token_event_id : public pp_token
 {
-  pp_token_event_id (diagnostic_event_id_t event_id)
+  pp_token_event_id (diagnostics::paths::event_id_t event_id)
   : pp_token (kind::event_id),
     m_event_id (event_id)
   {
     gcc_assert (event_id.known_p ());
   }
 
-  diagnostic_event_id_t m_event_id;
+  diagnostics::paths::event_id_t m_event_id;
 };
 
 template <>
@@ -334,6 +334,7 @@ public:
     push_back (std::move (tok));
   }
   void push_back_text (label_text &&text);
+  void push_back_byte (char ch);
   void push_back (std::unique_ptr<pp_token> tok);
   void push_back_list (pp_token_list &&list);
 
diff --git a/gcc/pretty-print-markup.h b/gcc/pretty-print-markup.h
index 18e298c..e4346f1 100644
--- a/gcc/pretty-print-markup.h
+++ b/gcc/pretty-print-markup.h
@@ -20,7 +20,7 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_PRETTY_PRINT_MARKUP_H
 #define GCC_PRETTY_PRINT_MARKUP_H
 
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
 
 class pp_token_list;
 
@@ -45,6 +45,11 @@ public:
   void begin_highlight_color (const char *color_name);
   void end_highlight_color ();
 
+  void begin_url (const char *url);
+  void end_url ();
+
+  void add_event_id (diagnostic_event_id_t event_id);
+
   void push_back_any_text ();
 
   pretty_printer &m_pp;
diff --git a/gcc/pretty-print.cc b/gcc/pretty-print.cc
index 6ecfcb2..77d40ec 100644
--- a/gcc/pretty-print.cc
+++ b/gcc/pretty-print.cc
@@ -27,9 +27,10 @@ along with GCC; see the file COPYING3.  If not see
 #include "pretty-print-format-impl.h"
 #include "pretty-print-markup.h"
 #include "pretty-print-urlifier.h"
-#include "diagnostic-color.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/color.h"
+#include "diagnostics/event-id.h"
 #include "diagnostic-highlight-colors.h"
+#include "auto-obstack.h"
 #include "selftest.h"
 
 #if HAVE_ICONV
@@ -714,7 +715,7 @@ static int
 decode_utf8_char (const unsigned char *, size_t len, unsigned int *);
 static void pp_quoted_string (pretty_printer *, const char *, size_t = -1);
 
-static void
+extern void
 default_token_printer (pretty_printer *pp,
 		       const pp_token_list &tokens);
 
@@ -1327,6 +1328,15 @@ pp_token_list::push_back_text (label_text &&text)
 }
 
 void
+pp_token_list::push_back_byte (char ch)
+{
+  char buf[2];
+  buf[0] = ch;
+  buf[1] = '\0';
+  push_back_text (label_text::take (xstrdup (buf)));
+}
+
+void
 pp_token_list::push_back (std::unique_ptr<pp_token> tok)
 {
   if (!m_first)
@@ -2034,6 +2044,16 @@ format_phase_2 (pretty_printer *pp,
 	    pp_string (pp, va_arg (*text.m_args_ptr, const char *));
 	  break;
 
+	case 'B':
+	  {
+	    string_slice s = *va_arg (*text.m_args_ptr, string_slice *);
+	    if (quote)
+	      pp_quoted_string (pp, s.begin (), s.size ());
+	    else
+	      pp_string_n (pp, s.begin (), s.size ());
+	    break;
+	  }
+
 	case 'p':
 	  pp_pointer (pp, va_arg (*text.m_args_ptr, void *));
 	  break;
@@ -2177,38 +2197,6 @@ format_phase_2 (pretty_printer *pp,
       gcc_assert (!formatters[argno]);
 }
 
-struct auto_obstack
-{
-  auto_obstack ()
-  {
-    obstack_init (&m_obstack);
-  }
-
-  ~auto_obstack ()
-  {
-    obstack_free (&m_obstack, NULL);
-  }
-
-  operator obstack & () { return m_obstack; }
-
-  void grow (const void *src, size_t length)
-  {
-    obstack_grow (&m_obstack, src, length);
-  }
-
-  void *object_base () const
-  {
-    return m_obstack.object_base;
-  }
-
-  size_t object_size () const
-  {
-    return obstack_object_size (&m_obstack);
-  }
-
-  obstack m_obstack;
-};
-
 /* Phase 3 of formatting a message (phases 1 and 2 done by pp_format).
 
    Pop a pp_formatted_chunks from chunk_obstack, collecting all the tokens from
@@ -2261,7 +2249,7 @@ pp_output_formatted_text (pretty_printer *pp,
 
 /* Default implementation of token printing.  */
 
-static void
+void
 default_token_printer (pretty_printer *pp,
 		       const pp_token_list &tokens)
 {
@@ -3189,6 +3177,29 @@ pp_markup::context::end_highlight_color ()
 }
 
 void
+pp_markup::context::begin_url (const char *url)
+{
+  push_back_any_text ();
+  m_formatted_token_list->push_back<pp_token_begin_url>
+    (label_text::take (xstrdup (url)));
+}
+
+void
+pp_markup::context::end_url ()
+{
+  push_back_any_text ();
+  m_formatted_token_list->push_back<pp_token_end_url> ();
+}
+
+void
+pp_markup::context::add_event_id (diagnostic_event_id_t event_id)
+{
+  gcc_assert (event_id.known_p ());
+  push_back_any_text ();
+  m_formatted_token_list->push_back<pp_token_event_id> (event_id);
+}
+
+void
 pp_markup::context::push_back_any_text ()
 {
   obstack *cur_obstack = m_buf.m_obstack;
@@ -3402,8 +3413,8 @@ test_pp_format ()
 			    "foo", 0x12345678);
   /* Verify "%@".  */
   {
-    diagnostic_event_id_t first (2);
-    diagnostic_event_id_t second (7);
+    diagnostics::paths::event_id_t first (2);
+    diagnostics::paths::event_id_t second (7);
 
     ASSERT_PP_FORMAT_2 ("first `free' at (3); second `free' at (8)",
 			"first %<free%> at %@; second %<free%> at %@",
diff --git a/gcc/pretty-print.h b/gcc/pretty-print.h
index 6cd9150..ced2e3c 100644
--- a/gcc/pretty-print.h
+++ b/gcc/pretty-print.h
@@ -23,7 +23,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "obstack.h"
 #include "rich-location.h"
-#include "diagnostic-url.h"
+#include "diagnostics/url.h"
 
 /* Maximum number of format string arguments.  */
 #define PP_NL_ARGMAX   30
diff --git a/gcc/profile-count.cc b/gcc/profile-count.cc
index 190bbeb..8f05a79 100644
--- a/gcc/profile-count.cc
+++ b/gcc/profile-count.cc
@@ -32,6 +32,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "cgraph.h"
 #include "wide-int.h"
 #include "sreal.h"
+#include "profile.h"
 
 /* Names from profile_quality enum values.  */
 
@@ -557,7 +558,7 @@ profile_count::operator* (const sreal &num) const
   sreal scaled = num * m_val;
   gcc_checking_assert (scaled >= 0);
   profile_count ret;
-  if (m_val > max_count)
+  if (scaled > max_count)
     ret.m_val = max_count;
   else
     ret.m_val = scaled.to_nearest_int ();
@@ -570,3 +571,27 @@ profile_count::operator*= (const sreal &num)
 {
   return *this * num;
 }
+
+/* Make counter forcibly nonzero.  */
+profile_count
+profile_count::force_nonzero () const
+{
+  if (!initialized_p ())
+    return *this;
+  profile_count ret = *this;
+  /* Generally values are forced non-zero to handle inconsistent profile 
+     where count 0 needs to be scaled up to non-zero.
+
+     Use cutoff value here to avoid situation where profile has large
+     cutoff and we perform count = count * num / den where num is non-zero
+     and den is 0.   If profile was scaled by large factor, forcing value
+     to 1 would lead to large scale factor.  */
+  gcov_unsigned_t small = profile_info ? profile_info->cutoff / 2 + 1
+			  : 1;
+  if (ret.m_val < small)
+    {
+      ret.m_val = small;
+      ret.m_quality = MIN (m_quality, ADJUSTED);
+    }
+  return ret;
+}
diff --git a/gcc/profile-count.h b/gcc/profile-count.h
index 2160540..c893aec 100644
--- a/gcc/profile-count.h
+++ b/gcc/profile-count.h
@@ -597,7 +597,7 @@ public:
      There are two exceptions - edges leading to noreturn edges and edges
      predicted by number of iterations heuristics are predicted well.  This macro
      should be able to distinguish those, but at the moment it simply check for
-     noreturn heuristic that is only one giving probability over 99% or bellow
+     noreturn heuristic that is only one giving probability over 99% or below
      1%.  In future we might want to propagate reliability information across the
      CFG if we find this information useful on multiple places.   */
   bool probably_reliable_p () const
@@ -1112,18 +1112,7 @@ public:
     }
 
   /* Make counter forcibly nonzero.  */
-  profile_count force_nonzero () const
-    {
-      if (!initialized_p ())
-	return *this;
-      profile_count ret = *this;
-      if (ret.m_val == 0)
-	{
-	  ret.m_val = 1;
-	  ret.m_quality = MIN (m_quality, ADJUSTED);
-	}
-      return ret;
-    }
+  profile_count force_nonzero () const;
 
   profile_count max (profile_count other) const
     {
diff --git a/gcc/pta-andersen.cc b/gcc/pta-andersen.cc
new file mode 100644
index 0000000..0253f05
--- /dev/null
+++ b/gcc/pta-andersen.cc
@@ -0,0 +1,2565 @@
+/* Andersen-style solver for tree based points-to analysis
+   Copyright (C) 2005-2025 Free Software Foundation, Inc.
+   Contributed by Daniel Berlin <dberlin@dberlin.org>
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   GCC is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "backend.h"
+
+#include "tree-ssa-structalias.h"
+#include "pta-andersen.h"
+
+/* During variable substitution and the offline version of indirect
+   cycle finding, we create nodes to represent dereferences and
+   address taken constraints.  These represent where these start and
+   end.  */
+#define FIRST_REF_NODE (varmap).length ()
+#define LAST_REF_NODE (FIRST_REF_NODE + (FIRST_REF_NODE - 1))
+
+#define EXECUTE_IF_IN_NONNULL_BITMAP(a, b, c, d)	\
+  if (a)						\
+    EXECUTE_IF_SET_IN_BITMAP (a, b, c, d)
+
+using namespace pointer_analysis;
+
+/* Used for predecessor bitmaps.  */
+static bitmap_obstack predbitmap_obstack;
+
+/* Used for per-solver-iteration bitmaps.  */
+static bitmap_obstack iteration_obstack;
+
+typedef struct constraint_graph *constraint_graph_t;
+
+/* The constraint graph is represented as an array of bitmaps
+   containing successor nodes.  */
+
+struct constraint_graph
+{
+  /* Size of this graph, which may be different than the number of
+     nodes in the variable map.  */
+  unsigned int size;
+
+  /* Explicit successors of each node.  */
+  bitmap *succs;
+
+  /* Implicit predecessors of each node (Used for variable
+     substitution). */
+  bitmap *implicit_preds;
+
+  /* Explicit predecessors of each node (Used for variable substitution).  */
+  bitmap *preds;
+
+  /* Indirect cycle representatives, or -1 if the node has no indirect
+     cycles.  */
+  int *indirect_cycles;
+
+  /* Representative node for a node.  rep[a] == a unless the node has
+     been unified.  */
+  unsigned int *rep;
+
+  /* Equivalence class representative for a label.  This is used for
+     variable substitution.  */
+  int *eq_rep;
+
+  /* Pointer equivalence label for a node.  All nodes with the same
+     pointer equivalence label can be unified together at some point
+     (either during constraint optimization or after the constraint
+     graph is built).  */
+  unsigned int *pe;
+
+  /* Pointer equivalence representative for a label.  This is used to
+     handle nodes that are pointer equivalent but not location
+     equivalent.  We can unite these once the addressof constraints
+     are transformed into initial points-to sets.  */
+  int *pe_rep;
+
+  /* Pointer equivalence label for each node, used during variable
+     substitution.  */
+  unsigned int *pointer_label;
+
+  /* Location equivalence label for each node, used during location
+     equivalence finding.  */
+  unsigned int *loc_label;
+
+  /* Pointed-by set for each node, used during location equivalence
+     finding.  This is pointed-by rather than pointed-to, because it
+     is constructed using the predecessor graph.  */
+  bitmap *pointed_by;
+
+  /* Points to sets for pointer equivalence.  This is *not* the actual
+     points-to sets for nodes.  */
+  bitmap *points_to;
+
+  /* Bitmap of nodes where the bit is set if the node is a direct
+     node.  Used for variable substitution.  */
+  sbitmap direct_nodes;
+
+  /* Bitmap of nodes where the bit is set if the node is address
+     taken.  Used for variable substitution.  */
+  bitmap address_taken;
+
+  /* Vector of complex constraints for each graph node.  Complex
+     constraints are those involving dereferences or offsets that are
+     not 0.  */
+  vec<constraint_t> *complex;
+};
+
+static constraint_graph_t graph;
+
+static void unify_nodes (constraint_graph_t, unsigned int, unsigned int, bool);
+
+
+/* Return the representative node for NODE, if NODE has been unioned
+   with another NODE.
+   This function performs path compression along the way to finding
+   the representative.  */
+
+static unsigned int
+find (unsigned int node)
+{
+  gcc_checking_assert (node < graph->size);
+  if (graph->rep[node] != node)
+    return graph->rep[node] = find (graph->rep[node]);
+  return node;
+}
+
+/* Union the TO and FROM nodes to the TO nodes.
+   Note that at some point in the future, we may want to do
+   union-by-rank, in which case we are going to have to return the
+   node we unified to.  */
+
+static bool
+unite (unsigned int to, unsigned int from)
+{
+  gcc_checking_assert (to < graph->size && from < graph->size);
+  if (to != from && graph->rep[from] != to)
+    {
+      graph->rep[from] = to;
+      return true;
+    }
+  return false;
+}
+
+/* Perform path compression for all nodes in the node representatives
+   union-find structure.  */
+
+static void
+union_find_compress_all (void)
+{
+  unsigned int i;
+  for (i = 0; i < graph->size; i++)
+    find (i);
+}
+
+/* Print the constraint graph in dot format.  */
+
+static void
+dump_constraint_graph (FILE *file)
+{
+  unsigned int i;
+
+  /* Only print the graph if it has already been initialized:  */
+  if (!graph)
+    return;
+
+  /* Prints the header of the dot file:  */
+  fprintf (file, "strict digraph {\n");
+  fprintf (file, "  node [\n    shape = box\n  ]\n");
+  fprintf (file, "  edge [\n    fontsize = \"12\"\n  ]\n");
+  fprintf (file, "\n  // List of nodes and complex constraints in "
+	   "the constraint graph:\n");
+
+  /* The next lines print the nodes in the graph together with the
+     complex constraints attached to them.  */
+  for (i = 1; i < graph->size; i++)
+    {
+      if (i == FIRST_REF_NODE)
+	continue;
+      if (find (i) != i)
+	continue;
+      if (i < FIRST_REF_NODE)
+	fprintf (file, "\"%s\"", get_varinfo (i)->name);
+      else
+	fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
+      if (graph->complex[i].exists ())
+	{
+	  unsigned j;
+	  constraint_t c;
+	  fprintf (file, " [label=\"\\N\\n");
+	  for (j = 0; graph->complex[i].iterate (j, &c); ++j)
+	    {
+	      dump_constraint (file, c);
+	      fprintf (file, "\\l");
+	    }
+	  fprintf (file, "\"]");
+	}
+      fprintf (file, ";\n");
+    }
+
+  /* Go over the edges.  */
+  fprintf (file, "\n  // Edges in the constraint graph:\n");
+  for (i = 1; i < graph->size; i++)
+    {
+      unsigned j;
+      bitmap_iterator bi;
+      if (find (i) != i)
+	continue;
+      EXECUTE_IF_IN_NONNULL_BITMAP (graph->succs[i], 0, j, bi)
+	{
+	  unsigned to = find (j);
+	  if (i == to)
+	    continue;
+	  if (i < FIRST_REF_NODE)
+	    fprintf (file, "\"%s\"", get_varinfo (i)->name);
+	  else
+	    fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
+	  fprintf (file, " -> ");
+	  if (to < FIRST_REF_NODE)
+	    fprintf (file, "\"%s\"", get_varinfo (to)->name);
+	  else
+	    fprintf (file, "\"*%s\"", get_varinfo (to - FIRST_REF_NODE)->name);
+	  fprintf (file, ";\n");
+	}
+    }
+
+  /* Prints the tail of the dot file.  */
+  fprintf (file, "}\n");
+}
+
+/* Print out the constraint graph to stderr.  */
+
+DEBUG_FUNCTION void
+debug_constraint_graph (void)
+{
+  dump_constraint_graph (stderr);
+}
+
+
+/* SOLVER FUNCTIONS
+
+   The solver is a simple worklist solver, that works on the following
+   algorithm:
+
+   sbitmap changed_nodes = all zeroes;
+   changed_count = 0;
+   For each node that is not already collapsed:
+       changed_count++;
+       set bit in changed nodes
+
+   while (changed_count > 0)
+   {
+     compute topological ordering for constraint graph
+
+     find and collapse cycles in the constraint graph (updating
+     changed if necessary)
+
+     for each node (n) in the graph in topological order:
+       changed_count--;
+
+       Process each complex constraint associated with the node,
+       updating changed if necessary.
+
+       For each outgoing edge from n, propagate the solution from n to
+       the destination of the edge, updating changed as necessary.
+
+   }  */
+
+/* Return true if two constraint expressions A and B are equal.  */
+
+static bool
+constraint_expr_equal (struct constraint_expr a, struct constraint_expr b)
+{
+  return a.type == b.type && a.var == b.var && a.offset == b.offset;
+}
+
+/* Return true if constraint expression A is less than constraint expression
+   B.  This is just arbitrary, but consistent, in order to give them an
+   ordering.  */
+
+static bool
+constraint_expr_less (struct constraint_expr a, struct constraint_expr b)
+{
+  if (a.type == b.type)
+    {
+      if (a.var == b.var)
+	return a.offset < b.offset;
+      else
+	return a.var < b.var;
+    }
+  else
+    return a.type < b.type;
+}
+
+/* Return true if constraint A is less than constraint B.  This is just
+   arbitrary, but consistent, in order to give them an ordering.  */
+
+static bool
+constraint_less (const constraint_t &a, const constraint_t &b)
+{
+  if (constraint_expr_less (a->lhs, b->lhs))
+    return true;
+  else if (constraint_expr_less (b->lhs, a->lhs))
+    return false;
+  else
+    return constraint_expr_less (a->rhs, b->rhs);
+}
+
+/* Return true if two constraints A and B are equal.  */
+
+static bool
+constraint_equal (const constraint &a, const constraint &b)
+{
+  return constraint_expr_equal (a.lhs, b.lhs)
+    && constraint_expr_equal (a.rhs, b.rhs);
+}
+
+/* Find a constraint LOOKFOR in the sorted constraint vector VEC.  */
+
+static constraint_t
+constraint_vec_find (vec<constraint_t> vec,
+		     constraint &lookfor)
+{
+  unsigned int place;
+  constraint_t found;
+
+  if (!vec.exists ())
+    return NULL;
+
+  place = vec.lower_bound (&lookfor, constraint_less);
+  if (place >= vec.length ())
+    return NULL;
+  found = vec[place];
+  if (!constraint_equal (*found, lookfor))
+    return NULL;
+  return found;
+}
+
+/* Union two constraint vectors, TO and FROM.  Put the result in TO.
+   Returns true of TO set is changed.  */
+
+static bool
+constraint_set_union (vec<constraint_t> *to,
+		      vec<constraint_t> *from)
+{
+  int i;
+  constraint_t c;
+  bool any_change = false;
+
+  FOR_EACH_VEC_ELT (*from, i, c)
+    {
+      if (constraint_vec_find (*to, *c) == NULL)
+	{
+	  unsigned int place = to->lower_bound (c, constraint_less);
+	  to->safe_insert (place, c);
+	  any_change = true;
+	}
+    }
+  return any_change;
+}
+
+/* Expands the solution in SET to all sub-fields of variables included.  */
+
+static bitmap
+solution_set_expand (bitmap set, bitmap *expanded)
+{
+  bitmap_iterator bi;
+  unsigned j;
+
+  if (*expanded)
+    return *expanded;
+
+  *expanded = BITMAP_ALLOC (&iteration_obstack);
+
+  /* In a first pass expand variables, once for each head to avoid
+     quadratic behavior, to include all sub-fields.  */
+  unsigned prev_head = 0;
+  EXECUTE_IF_SET_IN_BITMAP (set, 0, j, bi)
+    {
+      varinfo_t v = get_varinfo (j);
+      if (v->is_artificial_var
+	  || v->is_full_var)
+	continue;
+      if (v->head != prev_head)
+	{
+	  varinfo_t head = get_varinfo (v->head);
+	  unsigned num = 1;
+	  for (varinfo_t n = vi_next (head); n != NULL; n = vi_next (n))
+	    {
+	      if (n->id != head->id + num)
+		{
+		  /* Usually sub variables are adjacent but since we
+		     create pointed-to restrict representatives there
+		     can be gaps as well.  */
+		  bitmap_set_range (*expanded, head->id, num);
+		  head = n;
+		  num = 1;
+		}
+	      else
+		num++;
+	    }
+
+	  bitmap_set_range (*expanded, head->id, num);
+	  prev_head = v->head;
+	}
+    }
+
+  /* And finally set the rest of the bits from SET in an efficient way.  */
+  bitmap_ior_into (*expanded, set);
+
+  return *expanded;
+}
+
+/* Union solution sets TO and DELTA, and add INC to each member of DELTA in the
+   process.  */
+
+static bool
+set_union_with_increment (bitmap to, bitmap delta, HOST_WIDE_INT inc,
+			  bitmap *expanded_delta)
+{
+  bool changed = false;
+  bitmap_iterator bi;
+  unsigned int i;
+
+  /* If the solution of DELTA contains anything it is good enough to transfer
+     this to TO.  */
+  if (bitmap_bit_p (delta, anything_id))
+    return bitmap_set_bit (to, anything_id);
+
+  /* If the offset is unknown we have to expand the solution to
+     all subfields.  */
+  if (inc == UNKNOWN_OFFSET)
+    {
+      delta = solution_set_expand (delta, expanded_delta);
+      changed |= bitmap_ior_into (to, delta);
+      return changed;
+    }
+
+  /* For non-zero offset union the offsetted solution into the destination.  */
+  EXECUTE_IF_SET_IN_BITMAP (delta, 0, i, bi)
+    {
+      varinfo_t vi = get_varinfo (i);
+
+      /* If this is a variable with just one field just set its bit
+	 in the result.  */
+      if (vi->is_artificial_var
+	  || vi->is_unknown_size_var
+	  || vi->is_full_var)
+	changed |= bitmap_set_bit (to, i);
+      else
+	{
+	  HOST_WIDE_INT fieldoffset = vi->offset + inc;
+	  unsigned HOST_WIDE_INT size = vi->size;
+
+	  /* If the offset makes the pointer point to before the
+	     variable use offset zero for the field lookup.  */
+	  if (fieldoffset < 0)
+	    vi = get_varinfo (vi->head);
+	  else
+	    vi = first_or_preceding_vi_for_offset (vi, fieldoffset);
+
+	  do
+	    {
+	      changed |= bitmap_set_bit (to, vi->id);
+	      if (vi->is_full_var
+		  || vi->next == 0)
+		break;
+
+	      /* We have to include all fields that overlap the current field
+		 shifted by inc.  */
+	      vi = vi_next (vi);
+	    }
+	  while (vi->offset < fieldoffset + size);
+	}
+    }
+
+  return changed;
+}
+
+/* Insert constraint C into the list of complex constraints for graph
+   node VAR.  */
+
+static void
+insert_into_complex (constraint_graph_t graph,
+		     unsigned int var, constraint_t c)
+{
+  vec<constraint_t> complex = graph->complex[var];
+  unsigned int place = complex.lower_bound (c, constraint_less);
+
+  /* Only insert constraints that do not already exist.  */
+  if (place >= complex.length ()
+      || !constraint_equal (*c, *complex[place]))
+    graph->complex[var].safe_insert (place, c);
+}
+
+
+/* Condense two variable nodes into a single variable node, by moving
+   all associated info from FROM to TO.  Returns true if TO node's
+   constraint set changes after the merge.  */
+
+static bool
+merge_node_constraints (constraint_graph_t graph, unsigned int to,
+			unsigned int from)
+{
+  unsigned int i;
+  constraint_t c;
+  bool any_change = false;
+
+  gcc_checking_assert (find (from) == to);
+
+  /* Move all complex constraints from src node into to node.  */
+  FOR_EACH_VEC_ELT (graph->complex[from], i, c)
+    {
+      /* In complex constraints for node FROM, we may have either
+	 a = *FROM, and *FROM = a, or an offseted constraint which are
+	 always added to the rhs node's constraints.  */
+
+      if (c->rhs.type == DEREF)
+	c->rhs.var = to;
+      else if (c->lhs.type == DEREF)
+	c->lhs.var = to;
+      else
+	c->rhs.var = to;
+
+    }
+  any_change = constraint_set_union (&graph->complex[to],
+				     &graph->complex[from]);
+  graph->complex[from].release ();
+  return any_change;
+}
+
+/* Remove edges involving NODE from GRAPH.  */
+
+static void
+clear_edges_for_node (constraint_graph_t graph, unsigned int node)
+{
+  if (graph->succs[node])
+    BITMAP_FREE (graph->succs[node]);
+}
+
+/* Merge GRAPH nodes FROM and TO into node TO.  */
+
+static void
+merge_graph_nodes (constraint_graph_t graph, unsigned int to,
+		   unsigned int from)
+{
+  if (graph->indirect_cycles[from] != -1)
+    {
+      /* If we have indirect cycles with the from node, and we have
+	 none on the to node, the to node has indirect cycles from the
+	 from node now that they are unified.
+	 If indirect cycles exist on both, unify the nodes that they
+	 are in a cycle with, since we know they are in a cycle with
+	 each other.  */
+      if (graph->indirect_cycles[to] == -1)
+	graph->indirect_cycles[to] = graph->indirect_cycles[from];
+    }
+
+  /* Merge all the successor edges.  */
+  if (graph->succs[from])
+    {
+      if (!graph->succs[to])
+	graph->succs[to] = BITMAP_ALLOC (&pta_obstack);
+      bitmap_ior_into (graph->succs[to],
+		       graph->succs[from]);
+    }
+
+  clear_edges_for_node (graph, from);
+}
+
+
+/* Add an indirect graph edge to GRAPH, going from TO to FROM if
+   it doesn't exist in the graph already.  */
+
+static void
+add_implicit_graph_edge (constraint_graph_t graph, unsigned int to,
+			 unsigned int from)
+{
+  if (to == from)
+    return;
+
+  if (!graph->implicit_preds[to])
+    graph->implicit_preds[to] = BITMAP_ALLOC (&predbitmap_obstack);
+
+  if (bitmap_set_bit (graph->implicit_preds[to], from))
+    stats.num_implicit_edges++;
+}
+
+/* Add a predecessor graph edge to GRAPH, going from TO to FROM if
+   it doesn't exist in the graph already.
+   Return false if the edge already existed, true otherwise.  */
+
+static void
+add_pred_graph_edge (constraint_graph_t graph, unsigned int to,
+		     unsigned int from)
+{
+  if (!graph->preds[to])
+    graph->preds[to] = BITMAP_ALLOC (&predbitmap_obstack);
+  bitmap_set_bit (graph->preds[to], from);
+}
+
+/* Add a graph edge to GRAPH, going from FROM to TO if
+   it doesn't exist in the graph already.
+   Return false if the edge already existed, true otherwise.  */
+
+static bool
+add_graph_edge (constraint_graph_t graph, unsigned int to,
+		unsigned int from)
+{
+  if (to == from)
+    {
+      return false;
+    }
+  else
+    {
+      bool r = false;
+
+      if (!graph->succs[from])
+	graph->succs[from] = BITMAP_ALLOC (&pta_obstack);
+
+      /* The graph solving process does not avoid "triangles", thus
+	 there can be multiple paths from a node to another involving
+	 intermediate other nodes.  That causes extra copying which is
+	 most difficult to avoid when the intermediate node is ESCAPED
+	 because there are no edges added from ESCAPED.  Avoid
+	 adding the direct edge FROM -> TO when we have FROM -> ESCAPED
+	 and TO contains ESCAPED.
+	 ???  Note this is only a heuristic, it does not prevent the
+	 situation from occuring.  The heuristic helps PR38474 and
+	 PR99912 significantly.  */
+      if (to < FIRST_REF_NODE
+	  && bitmap_bit_p (graph->succs[from], find (escaped_id))
+	  && bitmap_bit_p (get_varinfo (find (to))->solution, escaped_id))
+	{
+	  stats.num_avoided_edges++;
+	  return false;
+	}
+
+      if (bitmap_set_bit (graph->succs[from], to))
+	{
+	  r = true;
+	  if (to < FIRST_REF_NODE && from < FIRST_REF_NODE)
+	    stats.num_edges++;
+	}
+      return r;
+    }
+}
+
+/* Initialize the constraint graph structure to contain SIZE nodes.  */
+
+static void
+init_graph (unsigned int size)
+{
+  unsigned int j;
+
+  bitmap_obstack_initialize (&predbitmap_obstack);
+
+  graph = XCNEW (struct constraint_graph);
+  graph->size = size;
+  graph->succs = XCNEWVEC (bitmap, graph->size);
+  graph->indirect_cycles = XNEWVEC (int, graph->size);
+  graph->rep = XNEWVEC (unsigned int, graph->size);
+  /* ??? Macros do not support template types with multiple arguments,
+     so we use a typedef to work around it.  */
+  typedef vec<constraint_t> vec_constraint_t_heap;
+  graph->complex = XCNEWVEC (vec_constraint_t_heap, size);
+  graph->pe = XCNEWVEC (unsigned int, graph->size);
+  graph->pe_rep = XNEWVEC (int, graph->size);
+
+  for (j = 0; j < graph->size; j++)
+    {
+      graph->rep[j] = j;
+      graph->pe_rep[j] = -1;
+      graph->indirect_cycles[j] = -1;
+    }
+}
+
+/* Build the constraint graph, adding only predecessor edges right now.  */
+
+static void
+build_pred_graph (void)
+{
+  int i;
+  constraint_t c;
+  unsigned int j;
+
+  graph->implicit_preds = XCNEWVEC (bitmap, graph->size);
+  graph->preds = XCNEWVEC (bitmap, graph->size);
+  graph->pointer_label = XCNEWVEC (unsigned int, graph->size);
+  graph->loc_label = XCNEWVEC (unsigned int, graph->size);
+  graph->pointed_by = XCNEWVEC (bitmap, graph->size);
+  graph->points_to = XCNEWVEC (bitmap, graph->size);
+  graph->eq_rep = XNEWVEC (int, graph->size);
+  graph->direct_nodes = sbitmap_alloc (graph->size);
+  graph->address_taken = BITMAP_ALLOC (&predbitmap_obstack);
+  bitmap_clear (graph->direct_nodes);
+
+  for (j = 1; j < FIRST_REF_NODE; j++)
+    {
+      if (!get_varinfo (j)->is_special_var)
+	bitmap_set_bit (graph->direct_nodes, j);
+    }
+
+  for (j = 0; j < graph->size; j++)
+    graph->eq_rep[j] = -1;
+
+  for (j = 0; j < varmap.length (); j++)
+    graph->indirect_cycles[j] = -1;
+
+  FOR_EACH_VEC_ELT (constraints, i, c)
+    {
+      struct constraint_expr lhs = c->lhs;
+      struct constraint_expr rhs = c->rhs;
+      unsigned int lhsvar = lhs.var;
+      unsigned int rhsvar = rhs.var;
+
+      if (lhs.type == DEREF)
+	{
+	  /* *x = y.  */
+	  if (rhs.offset == 0 && lhs.offset == 0 && rhs.type == SCALAR)
+	    {
+	      if (lhs.var == anything_id)
+		add_pred_graph_edge (graph, storedanything_id, rhsvar);
+	      else
+		add_pred_graph_edge (graph, FIRST_REF_NODE + lhsvar, rhsvar);
+	    }
+	}
+      else if (rhs.type == DEREF)
+	{
+	  /* x = *y */
+	  if (rhs.offset == 0 && lhs.offset == 0 && lhs.type == SCALAR)
+	    add_pred_graph_edge (graph, lhsvar, FIRST_REF_NODE + rhsvar);
+	  else
+	    bitmap_clear_bit (graph->direct_nodes, lhsvar);
+	}
+      else if (rhs.type == ADDRESSOF)
+	{
+	  varinfo_t v;
+
+	  /* x = &y */
+	  if (graph->points_to[lhsvar] == NULL)
+	    graph->points_to[lhsvar] = BITMAP_ALLOC (&predbitmap_obstack);
+	  bitmap_set_bit (graph->points_to[lhsvar], rhsvar);
+
+	  if (graph->pointed_by[rhsvar] == NULL)
+	    graph->pointed_by[rhsvar] = BITMAP_ALLOC (&predbitmap_obstack);
+	  bitmap_set_bit (graph->pointed_by[rhsvar], lhsvar);
+
+	  /* Implicitly, *x = y */
+	  add_implicit_graph_edge (graph, FIRST_REF_NODE + lhsvar, rhsvar);
+
+	  /* All related variables are no longer direct nodes.  */
+	  bitmap_clear_bit (graph->direct_nodes, rhsvar);
+	  v = get_varinfo (rhsvar);
+	  if (!v->is_full_var)
+	    {
+	      v = get_varinfo (v->head);
+	      do
+		{
+		  bitmap_clear_bit (graph->direct_nodes, v->id);
+		  v = vi_next (v);
+		}
+	      while (v != NULL);
+	    }
+	  bitmap_set_bit (graph->address_taken, rhsvar);
+	}
+      else if (lhsvar > anything_id
+	       && lhsvar != rhsvar && lhs.offset == 0 && rhs.offset == 0)
+	{
+	  /* x = y */
+	  add_pred_graph_edge (graph, lhsvar, rhsvar);
+	  /* Implicitly, *x = *y */
+	  add_implicit_graph_edge (graph, FIRST_REF_NODE + lhsvar,
+				   FIRST_REF_NODE + rhsvar);
+	}
+      else if (lhs.offset != 0 || rhs.offset != 0)
+	{
+	  if (rhs.offset != 0)
+	    bitmap_clear_bit (graph->direct_nodes, lhs.var);
+	  else if (lhs.offset != 0)
+	    bitmap_clear_bit (graph->direct_nodes, rhs.var);
+	}
+    }
+}
+
+/* Build the constraint graph, adding successor edges.  */
+
+static void
+build_succ_graph (void)
+{
+  unsigned i, t;
+  constraint_t c;
+
+  FOR_EACH_VEC_ELT (constraints, i, c)
+    {
+      struct constraint_expr lhs;
+      struct constraint_expr rhs;
+      unsigned int lhsvar;
+      unsigned int rhsvar;
+
+      if (!c)
+	continue;
+
+      lhs = c->lhs;
+      rhs = c->rhs;
+      lhsvar = find (lhs.var);
+      rhsvar = find (rhs.var);
+
+      if (lhs.type == DEREF)
+	{
+	  if (rhs.offset == 0 && lhs.offset == 0 && rhs.type == SCALAR)
+	    {
+	      if (lhs.var == anything_id)
+		add_graph_edge (graph, storedanything_id, rhsvar);
+	      else
+		add_graph_edge (graph, FIRST_REF_NODE + lhsvar, rhsvar);
+	    }
+	}
+      else if (rhs.type == DEREF)
+	{
+	  if (rhs.offset == 0 && lhs.offset == 0 && lhs.type == SCALAR)
+	    add_graph_edge (graph, lhsvar, FIRST_REF_NODE + rhsvar);
+	}
+      else if (rhs.type == ADDRESSOF)
+	{
+	  /* x = &y */
+	  gcc_checking_assert (find (rhs.var) == rhs.var);
+	  bitmap_set_bit (get_varinfo (lhsvar)->solution, rhsvar);
+	}
+      else if (lhsvar > anything_id
+	       && lhsvar != rhsvar && lhs.offset == 0 && rhs.offset == 0)
+	{
+	  add_graph_edge (graph, lhsvar, rhsvar);
+	}
+    }
+
+  /* Add edges from STOREDANYTHING to all nodes that can receive pointers.  */
+  t = find (storedanything_id);
+  for (i = integer_id + 1; i < FIRST_REF_NODE; ++i)
+    {
+      if (get_varinfo (i)->may_have_pointers)
+	add_graph_edge (graph, find (i), t);
+    }
+
+  /* Everything stored to ANYTHING also potentially escapes.  */
+  add_graph_edge (graph, find (escaped_id), t);
+}
+
+
+/* Changed variables on the last iteration.  */
+static bitmap changed;
+
+/* Strongly Connected Component visitation info.  */
+
+class scc_info
+{
+public:
+  scc_info (size_t size);
+  ~scc_info ();
+
+  auto_sbitmap visited;
+  auto_sbitmap deleted;
+  unsigned int *dfs;
+  unsigned int *node_mapping;
+  int current_index;
+  auto_vec<unsigned> scc_stack;
+};
+
+
+/* Recursive routine to find strongly connected components in GRAPH.
+   SI is the SCC info to store the information in, and N is the id of current
+   graph node we are processing.
+
+   This is Tarjan's strongly connected component finding algorithm, as
+   modified by Nuutila to keep only non-root nodes on the stack.
+   The algorithm can be found in "On finding the strongly connected
+   connected components in a directed graph" by Esko Nuutila and Eljas
+   Soisalon-Soininen, in Information Processing Letters volume 49,
+   number 1, pages 9-14.  */
+
+static void
+scc_visit (constraint_graph_t graph, class scc_info *si, unsigned int n)
+{
+  unsigned int i;
+  bitmap_iterator bi;
+  unsigned int my_dfs;
+
+  bitmap_set_bit (si->visited, n);
+  si->dfs[n] = si->current_index ++;
+  my_dfs = si->dfs[n];
+
+  /* Visit all the successors.  */
+  EXECUTE_IF_IN_NONNULL_BITMAP (graph->succs[n], 0, i, bi)
+    {
+      unsigned int w;
+
+      if (i > LAST_REF_NODE)
+	break;
+
+      w = find (i);
+      if (bitmap_bit_p (si->deleted, w))
+	continue;
+
+      if (!bitmap_bit_p (si->visited, w))
+	scc_visit (graph, si, w);
+
+      unsigned int t = find (w);
+      gcc_checking_assert (find (n) == n);
+      if (si->dfs[t] < si->dfs[n])
+	si->dfs[n] = si->dfs[t];
+    }
+
+  /* See if any components have been identified.  */
+  if (si->dfs[n] == my_dfs)
+    {
+      if (si->scc_stack.length () > 0
+	  && si->dfs[si->scc_stack.last ()] >= my_dfs)
+	{
+	  bitmap scc = BITMAP_ALLOC (NULL);
+	  unsigned int lowest_node;
+	  bitmap_iterator bi;
+
+	  bitmap_set_bit (scc, n);
+
+	  while (si->scc_stack.length () != 0
+		 && si->dfs[si->scc_stack.last ()] >= my_dfs)
+	    {
+	      unsigned int w = si->scc_stack.pop ();
+
+	      bitmap_set_bit (scc, w);
+	    }
+
+	  lowest_node = bitmap_first_set_bit (scc);
+	  gcc_assert (lowest_node < FIRST_REF_NODE);
+
+	  /* Collapse the SCC nodes into a single node, and mark the
+	     indirect cycles.  */
+	  EXECUTE_IF_SET_IN_BITMAP (scc, 0, i, bi)
+	    {
+	      if (i < FIRST_REF_NODE)
+		{
+		  if (unite (lowest_node, i))
+		    unify_nodes (graph, lowest_node, i, false);
+		}
+	      else
+		{
+		  unite (lowest_node, i);
+		  graph->indirect_cycles[i - FIRST_REF_NODE] = lowest_node;
+		}
+	    }
+	  bitmap_set_bit (si->deleted, lowest_node);
+	}
+      else
+	bitmap_set_bit (si->deleted, n);
+    }
+  else
+    si->scc_stack.safe_push (n);
+}
+
+/* Unify node FROM into node TO, updating the changed count if
+   necessary when UPDATE_CHANGED is true.  */
+
+static void
+unify_nodes (constraint_graph_t graph, unsigned int to, unsigned int from,
+	     bool update_changed)
+{
+  gcc_checking_assert (to != from && find (to) == to);
+
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    fprintf (dump_file, "Unifying %s to %s\n",
+	     get_varinfo (from)->name,
+	     get_varinfo (to)->name);
+
+  if (update_changed)
+    stats.unified_vars_dynamic++;
+  else
+    stats.unified_vars_static++;
+
+  merge_graph_nodes (graph, to, from);
+  if (merge_node_constraints (graph, to, from))
+    {
+      if (update_changed)
+	bitmap_set_bit (changed, to);
+    }
+
+  /* Mark TO as changed if FROM was changed.  If TO was already marked
+     as changed, decrease the changed count.  */
+
+  if (update_changed
+      && bitmap_clear_bit (changed, from))
+    bitmap_set_bit (changed, to);
+  varinfo_t fromvi = get_varinfo (from);
+  if (fromvi->solution)
+    {
+      /* If the solution changes because of the merging, we need to mark
+	 the variable as changed.  */
+      varinfo_t tovi = get_varinfo (to);
+      if (bitmap_ior_into (tovi->solution, fromvi->solution))
+	{
+	  if (update_changed)
+	    bitmap_set_bit (changed, to);
+	}
+
+      BITMAP_FREE (fromvi->solution);
+      if (fromvi->oldsolution)
+	BITMAP_FREE (fromvi->oldsolution);
+
+      if (stats.iterations > 0
+	  && tovi->oldsolution)
+	BITMAP_FREE (tovi->oldsolution);
+    }
+  if (graph->succs[to])
+    bitmap_clear_bit (graph->succs[to], to);
+}
+
+/* Add a copy edge FROM -> TO, optimizing special cases.  Returns TRUE
+   if the solution of TO changed.  */
+
+static bool
+solve_add_graph_edge (constraint_graph_t graph, unsigned int to,
+		      unsigned int from)
+{
+  /* Adding edges from the special vars is pointless.
+     They don't have sets that can change.  */
+  if (get_varinfo (from)->is_special_var)
+    return bitmap_ior_into (get_varinfo (to)->solution,
+			    get_varinfo (from)->solution);
+  /* Merging the solution from ESCAPED needlessly increases
+     the set.  Use ESCAPED as representative instead.  */
+  else if (from == find (escaped_id))
+    return bitmap_set_bit (get_varinfo (to)->solution, escaped_id);
+  else if (get_varinfo (from)->may_have_pointers
+	   && add_graph_edge (graph, to, from))
+    return bitmap_ior_into (get_varinfo (to)->solution,
+			    get_varinfo (from)->solution);
+  return false;
+}
+
+/* Process a constraint C that represents x = *(y + off), using DELTA as the
+   starting solution for y.  */
+
+static void
+do_sd_constraint (constraint_graph_t graph, constraint_t c,
+		  bitmap delta, bitmap *expanded_delta)
+{
+  unsigned int lhs = c->lhs.var;
+  bool flag = false;
+  bitmap sol = get_varinfo (lhs)->solution;
+  unsigned int j;
+  bitmap_iterator bi;
+  HOST_WIDE_INT roffset = c->rhs.offset;
+
+  /* Our IL does not allow this.  */
+  gcc_checking_assert (c->lhs.offset == 0);
+
+  /* If the solution of Y contains anything it is good enough to transfer
+     this to the LHS.  */
+  if (bitmap_bit_p (delta, anything_id))
+    {
+      flag |= bitmap_set_bit (sol, anything_id);
+      goto done;
+    }
+
+  /* If we do not know at with offset the rhs is dereferenced compute
+     the reachability set of DELTA, conservatively assuming it is
+     dereferenced at all valid offsets.  */
+  if (roffset == UNKNOWN_OFFSET)
+    {
+      delta = solution_set_expand (delta, expanded_delta);
+      /* No further offset processing is necessary.  */
+      roffset = 0;
+    }
+
+  /* For each variable j in delta (Sol(y)), add
+     an edge in the graph from j to x, and union Sol(j) into Sol(x).  */
+  EXECUTE_IF_SET_IN_BITMAP (delta, 0, j, bi)
+    {
+      varinfo_t v = get_varinfo (j);
+      HOST_WIDE_INT fieldoffset = v->offset + roffset;
+      unsigned HOST_WIDE_INT size = v->size;
+      unsigned int t;
+
+      if (v->is_full_var)
+	;
+      else if (roffset != 0)
+	{
+	  if (fieldoffset < 0)
+	    v = get_varinfo (v->head);
+	  else
+	    v = first_or_preceding_vi_for_offset (v, fieldoffset);
+	}
+
+      /* We have to include all fields that overlap the current field
+	 shifted by roffset.  */
+      do
+	{
+	  t = find (v->id);
+
+	  flag |= solve_add_graph_edge (graph, lhs, t);
+
+	  if (v->is_full_var
+	      || v->next == 0)
+	    break;
+
+	  v = vi_next (v);
+	}
+      while (v->offset < fieldoffset + size);
+    }
+
+done:
+  /* If the LHS solution changed, mark the var as changed.  */
+  if (flag)
+    bitmap_set_bit (changed, lhs);
+}
+
+/* Process a constraint C that represents *(x + off) = y using DELTA
+   as the starting solution for x.  */
+
+static void
+do_ds_constraint (constraint_t c, bitmap delta, bitmap *expanded_delta)
+{
+  unsigned int rhs = c->rhs.var;
+  bitmap sol = get_varinfo (rhs)->solution;
+  unsigned int j;
+  bitmap_iterator bi;
+  HOST_WIDE_INT loff = c->lhs.offset;
+  bool escaped_p = false;
+
+  /* Our IL does not allow this.  */
+  gcc_checking_assert (c->rhs.offset == 0);
+
+  /* If the solution of y contains ANYTHING simply use the ANYTHING
+     solution.  This avoids needlessly increasing the points-to sets.  */
+  if (bitmap_bit_p (sol, anything_id))
+    sol = get_varinfo (find (anything_id))->solution;
+
+  /* If the solution for x contains ANYTHING we have to merge the
+     solution of y into all pointer variables which we do via
+     STOREDANYTHING.  */
+  if (bitmap_bit_p (delta, anything_id))
+    {
+      unsigned t = find (storedanything_id);
+      if (solve_add_graph_edge (graph, t, rhs))
+	bitmap_set_bit (changed, t);
+      return;
+    }
+
+  /* If we do not know at with offset the rhs is dereferenced compute
+     the reachability set of DELTA, conservatively assuming it is
+     dereferenced at all valid offsets.  */
+  if (loff == UNKNOWN_OFFSET)
+    {
+      delta = solution_set_expand (delta, expanded_delta);
+      loff = 0;
+    }
+
+  /* For each member j of delta (Sol(x)), add an edge from y to j and
+     union Sol(y) into Sol(j) */
+  EXECUTE_IF_SET_IN_BITMAP (delta, 0, j, bi)
+    {
+      varinfo_t v = get_varinfo (j);
+      unsigned int t;
+      HOST_WIDE_INT fieldoffset = v->offset + loff;
+      unsigned HOST_WIDE_INT size = v->size;
+
+      if (v->is_full_var)
+	;
+      else if (loff != 0)
+	{
+	  if (fieldoffset < 0)
+	    v = get_varinfo (v->head);
+	  else
+	    v = first_or_preceding_vi_for_offset (v, fieldoffset);
+	}
+
+      /* We have to include all fields that overlap the current field
+	 shifted by loff.  */
+      do
+	{
+	  if (v->may_have_pointers)
+	    {
+	      /* If v is a global variable then this is an escape point.  */
+	      if (v->is_global_var
+		  && !escaped_p)
+		{
+		  t = find (escaped_id);
+		  if (add_graph_edge (graph, t, rhs)
+		      && bitmap_ior_into (get_varinfo (t)->solution, sol))
+		    bitmap_set_bit (changed, t);
+		  /* Enough to let rhs escape once.  */
+		  escaped_p = true;
+		}
+
+	      if (v->is_special_var)
+		break;
+
+	      t = find (v->id);
+
+	      if (solve_add_graph_edge (graph, t, rhs))
+		bitmap_set_bit (changed, t);
+	    }
+
+	  if (v->is_full_var
+	      || v->next == 0)
+	    break;
+
+	  v = vi_next (v);
+	}
+      while (v->offset < fieldoffset + size);
+    }
+}
+
+/* Handle a non-simple (simple meaning requires no iteration),
+   constraint (IE *x = &y, x = *y, *x = y, and x = y with offsets involved).  */
+
+static void
+do_complex_constraint (constraint_graph_t graph, constraint_t c, bitmap delta,
+		       bitmap *expanded_delta)
+{
+  if (c->lhs.type == DEREF)
+    {
+      if (c->rhs.type == ADDRESSOF)
+	{
+	  gcc_unreachable ();
+	}
+      else
+	{
+	  /* *x = y */
+	  do_ds_constraint (c, delta, expanded_delta);
+	}
+    }
+  else if (c->rhs.type == DEREF)
+    {
+      /* x = *y */
+      if (!(get_varinfo (c->lhs.var)->is_special_var))
+	do_sd_constraint (graph, c, delta, expanded_delta);
+    }
+  else
+    {
+      bitmap tmp;
+      bool flag = false;
+
+      gcc_checking_assert (c->rhs.type == SCALAR && c->lhs.type == SCALAR
+			   && c->rhs.offset != 0 && c->lhs.offset == 0);
+      tmp = get_varinfo (c->lhs.var)->solution;
+
+      flag = set_union_with_increment (tmp, delta, c->rhs.offset,
+				       expanded_delta);
+
+      if (flag)
+	bitmap_set_bit (changed, c->lhs.var);
+    }
+}
+
+/* Initialize and return a new SCC info structure.  */
+
+scc_info::scc_info (size_t size) :
+  visited (size), deleted (size), current_index (0), scc_stack (1)
+{
+  bitmap_clear (visited);
+  bitmap_clear (deleted);
+  node_mapping = XNEWVEC (unsigned int, size);
+  dfs = XCNEWVEC (unsigned int, size);
+
+  for (size_t i = 0; i < size; i++)
+    node_mapping[i] = i;
+}
+
+/* Free an SCC info structure pointed to by SI.  */
+
+scc_info::~scc_info ()
+{
+  free (node_mapping);
+  free (dfs);
+}
+
+
+/* Find indirect cycles in GRAPH that occur, using strongly connected
+   components, and note them in the indirect cycles map.
+
+   This technique comes from Ben Hardekopf and Calvin Lin,
+   "It Pays to be Lazy: Fast and Accurate Pointer Analysis for Millions of
+   Lines of Code", submitted to PLDI 2007.  */
+
+static void
+find_indirect_cycles (constraint_graph_t graph)
+{
+  unsigned int i;
+  unsigned int size = graph->size;
+  scc_info si (size);
+
+  for (i = 0; i < MIN (LAST_REF_NODE, size); i++)
+    if (!bitmap_bit_p (si.visited, i) && find (i) == i)
+      scc_visit (graph, &si, i);
+}
+
+/* Visit the graph in topological order starting at node N, and store the
+   order in TOPO_ORDER using VISITED to indicate visited nodes.  */
+
+static void
+topo_visit (constraint_graph_t graph, vec<unsigned> &topo_order,
+	    sbitmap visited, unsigned int n)
+{
+  bitmap_iterator bi;
+  unsigned int j;
+
+  bitmap_set_bit (visited, n);
+
+  if (graph->succs[n])
+    EXECUTE_IF_SET_IN_BITMAP (graph->succs[n], 0, j, bi)
+      {
+	unsigned k = find (j);
+	if (!bitmap_bit_p (visited, k))
+	  topo_visit (graph, topo_order, visited, k);
+      }
+
+  /* Also consider copy with offset complex constraints as implicit edges.  */
+  for (auto c : graph->complex[n])
+    {
+      /* Constraints are ordered so that SCALAR = SCALAR appear first.  */
+      if (c->lhs.type != SCALAR || c->rhs.type != SCALAR)
+	break;
+      gcc_checking_assert (c->rhs.var == n);
+      unsigned k = find (c->lhs.var);
+      if (!bitmap_bit_p (visited, k))
+	topo_visit (graph, topo_order, visited, k);
+    }
+
+  topo_order.quick_push (n);
+}
+
+/* Compute a topological ordering for GRAPH, and return the result.  */
+
+static auto_vec<unsigned>
+compute_topo_order (constraint_graph_t graph)
+{
+  unsigned int i;
+  unsigned int size = graph->size;
+
+  auto_sbitmap visited (size);
+  bitmap_clear (visited);
+
+  /* For the heuristic in add_graph_edge to work optimally make sure to
+     first visit the connected component of the graph containing
+     ESCAPED.  Do this by extracting the connected component
+     with ESCAPED and append that to all other components as solve_graph
+     pops from the order.  */
+  auto_vec<unsigned> tail (size);
+  topo_visit (graph, tail, visited, find (escaped_id));
+
+  auto_vec<unsigned> topo_order (size);
+
+  for (i = 0; i != size; ++i)
+    if (!bitmap_bit_p (visited, i) && find (i) == i)
+      topo_visit (graph, topo_order, visited, i);
+
+  topo_order.splice (tail);
+  return topo_order;
+}
+
+/* Structure used to for hash value numbering of pointer equivalence
+   classes.  */
+
+typedef struct equiv_class_label
+{
+  hashval_t hashcode;
+  unsigned int equivalence_class;
+  bitmap labels;
+} *equiv_class_label_t;
+typedef const struct equiv_class_label *const_equiv_class_label_t;
+
+/* Equiv_class_label hashtable helpers.  */
+
+struct equiv_class_hasher : nofree_ptr_hash <equiv_class_label>
+{
+  static inline hashval_t hash (const equiv_class_label *);
+  static inline bool equal (const equiv_class_label *,
+			    const equiv_class_label *);
+};
+
+/* A hashtable for mapping a bitmap of labels->pointer equivalence
+   classes.  */
+static hash_table<equiv_class_hasher> *pointer_equiv_class_table;
+
+/* A hashtable for mapping a bitmap of labels->location equivalence
+   classes.  */
+static hash_table<equiv_class_hasher> *location_equiv_class_table;
+
+/* Hash function for a equiv_class_label_t.  */
+
+inline hashval_t
+equiv_class_hasher::hash (const equiv_class_label *ecl)
+{
+  return ecl->hashcode;
+}
+
+/* Equality function for two equiv_class_label_t's.  */
+
+inline bool
+equiv_class_hasher::equal (const equiv_class_label *eql1,
+			   const equiv_class_label *eql2)
+{
+  return (eql1->hashcode == eql2->hashcode
+	  && bitmap_equal_p (eql1->labels, eql2->labels));
+}
+
+struct obstack equiv_class_obstack;
+
+/* Lookup a equivalence class in TABLE by the bitmap of LABELS with
+   hash HAS it contains.  Sets *REF_LABELS to the bitmap LABELS
+   is equivalent to.  */
+
+static equiv_class_label *
+equiv_class_lookup_or_add (hash_table<equiv_class_hasher> *table,
+			   bitmap labels)
+{
+  equiv_class_label **slot;
+  equiv_class_label ecl;
+
+  ecl.labels = labels;
+  ecl.hashcode = bitmap_hash (labels);
+  slot = table->find_slot (&ecl, INSERT);
+  if (!*slot)
+    {
+      *slot = XOBNEW (&equiv_class_obstack, struct equiv_class_label);
+      (*slot)->labels = labels;
+      (*slot)->hashcode = ecl.hashcode;
+      (*slot)->equivalence_class = 0;
+    }
+
+  return *slot;
+}
+
+
+/* Perform offline variable substitution.
+
+   This is a worst case quadratic time way of identifying variables
+   that must have equivalent points-to sets, including those caused by
+   static cycles, and single entry subgraphs, in the constraint graph.
+
+   The technique is described in "Exploiting Pointer and Location
+   Equivalence to Optimize Pointer Analysis.  In the 14th International
+   Static Analysis Symposium (SAS), August 2007."  It is known as the
+   "HU" algorithm, and is equivalent to value numbering the collapsed
+   constraint graph including evaluating unions.
+
+   The general method of finding equivalence classes is as follows:
+   Add fake nodes (REF nodes) and edges for *a = b and a = *b constraints.
+   Initialize all non-REF nodes to be direct nodes.
+   For each constraint a = a U {b}, we set pts(a) = pts(a) u {fresh
+   variable}
+   For each constraint containing the dereference, we also do the same
+   thing.
+
+   We then compute SCC's in the graph and unify nodes in the same SCC,
+   including pts sets.
+
+   For each non-collapsed node x:
+    Visit all unvisited explicit incoming edges.
+    Ignoring all non-pointers, set pts(x) = Union of pts(a) for y
+    where y->x.
+    Lookup the equivalence class for pts(x).
+     If we found one, equivalence_class(x) = found class.
+     Otherwise, equivalence_class(x) = new class, and new_class is
+    added to the lookup table.
+
+   All direct nodes with the same equivalence class can be replaced
+   with a single representative node.
+   All unlabeled nodes (label == 0) are not pointers and all edges
+   involving them can be eliminated.
+   We perform these optimizations during rewrite_constraints
+
+   In addition to pointer equivalence class finding, we also perform
+   location equivalence class finding.  This is the set of variables
+   that always appear together in points-to sets.  We use this to
+   compress the size of the points-to sets.  */
+
+/* Current maximum pointer equivalence class id.  */
+static int pointer_equiv_class;
+
+/* Current maximum location equivalence class id.  */
+static int location_equiv_class;
+
+/* Recursive routine to find strongly connected components in GRAPH,
+   and label it's nodes with DFS numbers.  */
+
+static void
+condense_visit (constraint_graph_t graph, class scc_info *si, unsigned int n)
+{
+  unsigned int i;
+  bitmap_iterator bi;
+  unsigned int my_dfs;
+
+  gcc_checking_assert (si->node_mapping[n] == n);
+  bitmap_set_bit (si->visited, n);
+  si->dfs[n] = si->current_index ++;
+  my_dfs = si->dfs[n];
+
+  /* Visit all the successors.  */
+  EXECUTE_IF_IN_NONNULL_BITMAP (graph->preds[n], 0, i, bi)
+    {
+      unsigned int w = si->node_mapping[i];
+
+      if (bitmap_bit_p (si->deleted, w))
+	continue;
+
+      if (!bitmap_bit_p (si->visited, w))
+	condense_visit (graph, si, w);
+
+      unsigned int t = si->node_mapping[w];
+      gcc_checking_assert (si->node_mapping[n] == n);
+      if (si->dfs[t] < si->dfs[n])
+	si->dfs[n] = si->dfs[t];
+    }
+
+  /* Visit all the implicit predecessors.  */
+  EXECUTE_IF_IN_NONNULL_BITMAP (graph->implicit_preds[n], 0, i, bi)
+    {
+      unsigned int w = si->node_mapping[i];
+
+      if (bitmap_bit_p (si->deleted, w))
+	continue;
+
+      if (!bitmap_bit_p (si->visited, w))
+	condense_visit (graph, si, w);
+
+      unsigned int t = si->node_mapping[w];
+      gcc_assert (si->node_mapping[n] == n);
+      if (si->dfs[t] < si->dfs[n])
+	si->dfs[n] = si->dfs[t];
+    }
+
+  /* See if any components have been identified.  */
+  if (si->dfs[n] == my_dfs)
+    {
+      if (si->scc_stack.length () != 0
+	  && si->dfs[si->scc_stack.last ()] >= my_dfs)
+	{
+	  /* Find the first node of the SCC and do non-bitmap work.  */
+	  bool direct_p = true;
+	  unsigned first = si->scc_stack.length ();
+	  do
+	    {
+	      --first;
+	      unsigned int w = si->scc_stack[first];
+	      si->node_mapping[w] = n;
+	      if (!bitmap_bit_p (graph->direct_nodes, w))
+		direct_p = false;
+	    }
+	  while (first > 0
+		 && si->dfs[si->scc_stack[first - 1]] >= my_dfs);
+	  if (!direct_p)
+	    bitmap_clear_bit (graph->direct_nodes, n);
+
+	  /* Want to reduce to node n, push that first.  */
+	  si->scc_stack.reserve (1);
+	  si->scc_stack.quick_push (si->scc_stack[first]);
+	  si->scc_stack[first] = n;
+
+	  unsigned scc_size = si->scc_stack.length () - first;
+	  unsigned split = scc_size / 2;
+	  unsigned carry = scc_size - split * 2;
+	  while (split > 0)
+	    {
+	      for (unsigned i = 0; i < split; ++i)
+		{
+		  unsigned a = si->scc_stack[first + i];
+		  unsigned b = si->scc_stack[first + split + carry + i];
+
+		  /* Unify our nodes.  */
+		  if (graph->preds[b])
+		    {
+		      if (!graph->preds[a])
+			std::swap (graph->preds[a], graph->preds[b]);
+		      else
+			bitmap_ior_into_and_free (graph->preds[a],
+						  &graph->preds[b]);
+		    }
+		  if (graph->implicit_preds[b])
+		    {
+		      if (!graph->implicit_preds[a])
+			std::swap (graph->implicit_preds[a],
+				   graph->implicit_preds[b]);
+		      else
+			bitmap_ior_into_and_free (graph->implicit_preds[a],
+						  &graph->implicit_preds[b]);
+		    }
+		  if (graph->points_to[b])
+		    {
+		      if (!graph->points_to[a])
+			std::swap (graph->points_to[a], graph->points_to[b]);
+		      else
+			bitmap_ior_into_and_free (graph->points_to[a],
+						  &graph->points_to[b]);
+		    }
+		}
+	      unsigned remain = split + carry;
+	      split = remain / 2;
+	      carry = remain - split * 2;
+	    }
+	  /* Actually pop the SCC.  */
+	  si->scc_stack.truncate (first);
+	}
+      bitmap_set_bit (si->deleted, n);
+    }
+  else
+    si->scc_stack.safe_push (n);
+}
+
+/* Label pointer equivalences.
+
+   This performs a value numbering of the constraint graph to
+   discover which variables will always have the same points-to sets
+   under the current set of constraints.
+
+   The way it value numbers is to store the set of points-to bits
+   generated by the constraints and graph edges.  This is just used as a
+   hash and equality comparison.  The *actual set of points-to bits* is
+   completely irrelevant, in that we don't care about being able to
+   extract them later.
+
+   The equality values (currently bitmaps) just have to satisfy a few
+   constraints, the main ones being:
+   1. The combining operation must be order independent.
+   2. The end result of a given set of operations must be unique iff the
+      combination of input values is unique
+   3. Hashable.  */
+
+static void
+label_visit (constraint_graph_t graph, class scc_info *si, unsigned int n)
+{
+  unsigned int i, first_pred;
+  bitmap_iterator bi;
+
+  bitmap_set_bit (si->visited, n);
+
+  /* Label and union our incoming edges's points to sets.  */
+  first_pred = -1U;
+  EXECUTE_IF_IN_NONNULL_BITMAP (graph->preds[n], 0, i, bi)
+    {
+      unsigned int w = si->node_mapping[i];
+      if (!bitmap_bit_p (si->visited, w))
+	label_visit (graph, si, w);
+
+      /* Skip unused edges.  */
+      if (w == n || graph->pointer_label[w] == 0)
+	continue;
+
+      if (graph->points_to[w])
+	{
+	  if (!graph->points_to[n])
+	    {
+	      if (first_pred == -1U)
+		first_pred = w;
+	      else
+		{
+		  graph->points_to[n] = BITMAP_ALLOC (&predbitmap_obstack);
+		  bitmap_ior (graph->points_to[n],
+			      graph->points_to[first_pred],
+			      graph->points_to[w]);
+		}
+	    }
+	  else
+	    bitmap_ior_into (graph->points_to[n], graph->points_to[w]);
+	}
+    }
+
+  /* Indirect nodes get fresh variables and a new pointer equiv class.  */
+  if (!bitmap_bit_p (graph->direct_nodes, n))
+    {
+      if (!graph->points_to[n])
+	{
+	  graph->points_to[n] = BITMAP_ALLOC (&predbitmap_obstack);
+	  if (first_pred != -1U)
+	    bitmap_copy (graph->points_to[n], graph->points_to[first_pred]);
+	}
+      bitmap_set_bit (graph->points_to[n], FIRST_REF_NODE + n);
+      graph->pointer_label[n] = pointer_equiv_class++;
+      equiv_class_label_t ecl;
+      ecl = equiv_class_lookup_or_add (pointer_equiv_class_table,
+				       graph->points_to[n]);
+      ecl->equivalence_class = graph->pointer_label[n];
+      return;
+    }
+
+  /* If there was only a single non-empty predecessor the pointer equiv
+     class is the same.  */
+  if (!graph->points_to[n])
+    {
+      if (first_pred != -1U)
+	{
+	  graph->pointer_label[n] = graph->pointer_label[first_pred];
+	  graph->points_to[n] = graph->points_to[first_pred];
+	}
+      return;
+    }
+
+  if (!bitmap_empty_p (graph->points_to[n]))
+    {
+      equiv_class_label_t ecl;
+      ecl = equiv_class_lookup_or_add (pointer_equiv_class_table,
+				       graph->points_to[n]);
+      if (ecl->equivalence_class == 0)
+	ecl->equivalence_class = pointer_equiv_class++;
+      else
+	{
+	  BITMAP_FREE (graph->points_to[n]);
+	  graph->points_to[n] = ecl->labels;
+	}
+      graph->pointer_label[n] = ecl->equivalence_class;
+    }
+}
+
+/* Print the pred graph in dot format.  */
+
+static void
+dump_pred_graph (class scc_info *si, FILE *file)
+{
+  unsigned int i;
+
+  /* Only print the graph if it has already been initialized:  */
+  if (!graph)
+    return;
+
+  /* Prints the header of the dot file:  */
+  fprintf (file, "strict digraph {\n");
+  fprintf (file, "  node [\n    shape = box\n  ]\n");
+  fprintf (file, "  edge [\n    fontsize = \"12\"\n  ]\n");
+  fprintf (file, "\n  // List of nodes and complex constraints in "
+	   "the constraint graph:\n");
+
+  /* The next lines print the nodes in the graph together with the
+     complex constraints attached to them.  */
+  for (i = 1; i < graph->size; i++)
+    {
+      if (i == FIRST_REF_NODE)
+	continue;
+      if (si->node_mapping[i] != i)
+	continue;
+      if (i < FIRST_REF_NODE)
+	fprintf (file, "\"%s\"", get_varinfo (i)->name);
+      else
+	fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
+      if (graph->points_to[i]
+	  && !bitmap_empty_p (graph->points_to[i]))
+	{
+	  if (i < FIRST_REF_NODE)
+	    fprintf (file, "[label=\"%s = {", get_varinfo (i)->name);
+	  else
+	    fprintf (file, "[label=\"*%s = {",
+		     get_varinfo (i - FIRST_REF_NODE)->name);
+	  unsigned j;
+	  bitmap_iterator bi;
+	  EXECUTE_IF_SET_IN_BITMAP (graph->points_to[i], 0, j, bi)
+	    fprintf (file, " %d", j);
+	  fprintf (file, " }\"]");
+	}
+      fprintf (file, ";\n");
+    }
+
+  /* Go over the edges.  */
+  fprintf (file, "\n  // Edges in the constraint graph:\n");
+  for (i = 1; i < graph->size; i++)
+    {
+      unsigned j;
+      bitmap_iterator bi;
+      if (si->node_mapping[i] != i)
+	continue;
+      EXECUTE_IF_IN_NONNULL_BITMAP (graph->preds[i], 0, j, bi)
+	{
+	  unsigned from = si->node_mapping[j];
+	  if (from < FIRST_REF_NODE)
+	    fprintf (file, "\"%s\"", get_varinfo (from)->name);
+	  else
+	    fprintf (file, "\"*%s\"",
+		     get_varinfo (from - FIRST_REF_NODE)->name);
+	  fprintf (file, " -> ");
+	  if (i < FIRST_REF_NODE)
+	    fprintf (file, "\"%s\"", get_varinfo (i)->name);
+	  else
+	    fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
+	  fprintf (file, ";\n");
+	}
+    }
+
+  /* Prints the tail of the dot file.  */
+  fprintf (file, "}\n");
+}
+
+/* Perform offline variable substitution, discovering equivalence
+   classes, and eliminating non-pointer variables.  */
+
+static class scc_info *
+perform_var_substitution (constraint_graph_t graph)
+{
+  unsigned int i;
+  unsigned int size = graph->size;
+  scc_info *si = new scc_info (size);
+
+  bitmap_obstack_initialize (&iteration_obstack);
+  gcc_obstack_init (&equiv_class_obstack);
+  pointer_equiv_class_table = new hash_table<equiv_class_hasher> (511);
+  location_equiv_class_table
+    = new hash_table<equiv_class_hasher> (511);
+  pointer_equiv_class = 1;
+  location_equiv_class = 1;
+
+  /* Condense the nodes, which means to find SCC's, count incoming
+     predecessors, and unite nodes in SCC's.  */
+  for (i = 1; i < FIRST_REF_NODE; i++)
+    if (!bitmap_bit_p (si->visited, si->node_mapping[i]))
+      condense_visit (graph, si, si->node_mapping[i]);
+
+  if (dump_file && (dump_flags & TDF_GRAPH))
+    {
+      fprintf (dump_file, "\n\n// The constraint graph before var-substitution "
+	       "in dot format:\n");
+      dump_pred_graph (si, dump_file);
+      fprintf (dump_file, "\n\n");
+    }
+
+  bitmap_clear (si->visited);
+  /* Actually the label the nodes for pointer equivalences.  */
+  for (i = 1; i < FIRST_REF_NODE; i++)
+    if (!bitmap_bit_p (si->visited, si->node_mapping[i]))
+      label_visit (graph, si, si->node_mapping[i]);
+
+  /* Calculate location equivalence labels.  */
+  for (i = 1; i < FIRST_REF_NODE; i++)
+    {
+      bitmap pointed_by;
+      bitmap_iterator bi;
+      unsigned int j;
+
+      if (!graph->pointed_by[i])
+	continue;
+      pointed_by = BITMAP_ALLOC (&iteration_obstack);
+
+      /* Translate the pointed-by mapping for pointer equivalence
+	 labels.  */
+      EXECUTE_IF_SET_IN_BITMAP (graph->pointed_by[i], 0, j, bi)
+	{
+	  bitmap_set_bit (pointed_by,
+			  graph->pointer_label[si->node_mapping[j]]);
+	}
+      /* The original pointed_by is now dead.  */
+      BITMAP_FREE (graph->pointed_by[i]);
+
+      /* Look up the location equivalence label if one exists, or make
+	 one otherwise.  */
+      equiv_class_label_t ecl;
+      ecl = equiv_class_lookup_or_add (location_equiv_class_table, pointed_by);
+      if (ecl->equivalence_class == 0)
+	ecl->equivalence_class = location_equiv_class++;
+      else
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    fprintf (dump_file, "Found location equivalence for node %s\n",
+		     get_varinfo (i)->name);
+	  BITMAP_FREE (pointed_by);
+	}
+      graph->loc_label[i] = ecl->equivalence_class;
+
+    }
+
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    for (i = 1; i < FIRST_REF_NODE; i++)
+      {
+	unsigned j = si->node_mapping[i];
+	if (j != i)
+	  {
+	    fprintf (dump_file, "%s node id %d ",
+		     bitmap_bit_p (graph->direct_nodes, i)
+		     ? "Direct" : "Indirect", i);
+	    if (i < FIRST_REF_NODE)
+	      fprintf (dump_file, "\"%s\"", get_varinfo (i)->name);
+	    else
+	      fprintf (dump_file, "\"*%s\"",
+		       get_varinfo (i - FIRST_REF_NODE)->name);
+	    fprintf (dump_file, " mapped to SCC leader node id %d ", j);
+	    if (j < FIRST_REF_NODE)
+	      fprintf (dump_file, "\"%s\"\n", get_varinfo (j)->name);
+	    else
+	      fprintf (dump_file, "\"*%s\"\n",
+		       get_varinfo (j - FIRST_REF_NODE)->name);
+	  }
+	else
+	  {
+	    fprintf (dump_file,
+		     "Equivalence classes for %s node id %d ",
+		     bitmap_bit_p (graph->direct_nodes, i)
+		     ? "direct" : "indirect", i);
+	    if (i < FIRST_REF_NODE)
+	      fprintf (dump_file, "\"%s\"", get_varinfo (i)->name);
+	    else
+	      fprintf (dump_file, "\"*%s\"",
+		       get_varinfo (i - FIRST_REF_NODE)->name);
+	    fprintf (dump_file,
+		     ": pointer %d, location %d\n",
+		     graph->pointer_label[i], graph->loc_label[i]);
+	  }
+      }
+
+  /* Quickly eliminate our non-pointer variables.  */
+
+  for (i = 1; i < FIRST_REF_NODE; i++)
+    {
+      unsigned int node = si->node_mapping[i];
+
+      if (graph->pointer_label[node] == 0)
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    fprintf (dump_file,
+		     "%s is a non-pointer variable, eliminating edges.\n",
+		     get_varinfo (node)->name);
+	  stats.nonpointer_vars++;
+	  clear_edges_for_node (graph, node);
+	}
+    }
+
+  return si;
+}
+
+/* Free information that was only necessary for variable
+   substitution.  */
+
+static void
+free_var_substitution_info (class scc_info *si)
+{
+  delete si;
+  free (graph->pointer_label);
+  free (graph->loc_label);
+  free (graph->pointed_by);
+  free (graph->points_to);
+  free (graph->eq_rep);
+  sbitmap_free (graph->direct_nodes);
+  delete pointer_equiv_class_table;
+  pointer_equiv_class_table = NULL;
+  delete location_equiv_class_table;
+  location_equiv_class_table = NULL;
+  obstack_free (&equiv_class_obstack, NULL);
+  bitmap_obstack_release (&iteration_obstack);
+}
+
+/* Return an existing node that is equivalent to NODE, which has
+   equivalence class LABEL, if one exists.  Return NODE otherwise.  */
+
+static unsigned int
+find_equivalent_node (constraint_graph_t graph,
+		      unsigned int node, unsigned int label)
+{
+  /* If the address version of this variable is unused, we can
+     substitute it for anything else with the same label.
+     Otherwise, we know the pointers are equivalent, but not the
+     locations, and we can unite them later.  */
+
+  if (!bitmap_bit_p (graph->address_taken, node))
+    {
+      gcc_checking_assert (label < graph->size);
+
+      if (graph->eq_rep[label] != -1)
+	{
+	  /* Unify the two variables since we know they are equivalent.  */
+	  if (unite (graph->eq_rep[label], node))
+	    unify_nodes (graph, graph->eq_rep[label], node, false);
+	  return graph->eq_rep[label];
+	}
+      else
+	{
+	  graph->eq_rep[label] = node;
+	  graph->pe_rep[label] = node;
+	}
+    }
+  else
+    {
+      gcc_checking_assert (label < graph->size);
+      graph->pe[node] = label;
+      if (graph->pe_rep[label] == -1)
+	graph->pe_rep[label] = node;
+    }
+
+  return node;
+}
+
+/* Unite pointer equivalent but not location equivalent nodes in
+   GRAPH.  This may only be performed once variable substitution is
+   finished.  */
+
+static void
+unite_pointer_equivalences (constraint_graph_t graph)
+{
+  unsigned int i;
+
+  /* Go through the pointer equivalences and unite them to their
+     representative, if they aren't already.  */
+  for (i = 1; i < FIRST_REF_NODE; i++)
+    {
+      unsigned int label = graph->pe[i];
+      if (label)
+	{
+	  int label_rep = graph->pe_rep[label];
+
+	  if (label_rep == -1)
+	    continue;
+
+	  label_rep = find (label_rep);
+	  if (label_rep >= 0 && unite (label_rep, find (i)))
+	    unify_nodes (graph, label_rep, i, false);
+	}
+    }
+}
+
+/* Move complex constraints to the GRAPH nodes they belong to.  */
+
+static void
+move_complex_constraints (constraint_graph_t graph)
+{
+  int i;
+  constraint_t c;
+
+  FOR_EACH_VEC_ELT (constraints, i, c)
+    {
+      if (c)
+	{
+	  struct constraint_expr lhs = c->lhs;
+	  struct constraint_expr rhs = c->rhs;
+
+	  if (lhs.type == DEREF)
+	    {
+	      insert_into_complex (graph, lhs.var, c);
+	    }
+	  else if (rhs.type == DEREF)
+	    {
+	      if (!(get_varinfo (lhs.var)->is_special_var))
+		insert_into_complex (graph, rhs.var, c);
+	    }
+	  else if (rhs.type != ADDRESSOF && lhs.var > anything_id
+		   && (lhs.offset != 0 || rhs.offset != 0))
+	    {
+	      insert_into_complex (graph, rhs.var, c);
+	    }
+	}
+    }
+}
+
+/* Optimize and rewrite complex constraints while performing
+   collapsing of equivalent nodes.  SI is the SCC_INFO that is the
+   result of perform_variable_substitution.  */
+
+static void
+rewrite_constraints (constraint_graph_t graph,
+		     class scc_info *si)
+{
+  int i;
+  constraint_t c;
+
+  if (flag_checking)
+    {
+      for (unsigned int j = 0; j < graph->size; j++)
+	gcc_assert (find (j) == j);
+    }
+
+  FOR_EACH_VEC_ELT (constraints, i, c)
+    {
+      struct constraint_expr lhs = c->lhs;
+      struct constraint_expr rhs = c->rhs;
+      unsigned int lhsvar = find (lhs.var);
+      unsigned int rhsvar = find (rhs.var);
+      unsigned int lhsnode, rhsnode;
+      unsigned int lhslabel, rhslabel;
+
+      lhsnode = si->node_mapping[lhsvar];
+      rhsnode = si->node_mapping[rhsvar];
+      lhslabel = graph->pointer_label[lhsnode];
+      rhslabel = graph->pointer_label[rhsnode];
+
+      /* See if it is really a non-pointer variable, and if so, ignore
+	 the constraint.  */
+      if (lhslabel == 0)
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    {
+
+	      fprintf (dump_file, "%s is a non-pointer variable, "
+		       "ignoring constraint:",
+		       get_varinfo (lhs.var)->name);
+	      dump_constraint (dump_file, c);
+	      fprintf (dump_file, "\n");
+	    }
+	  constraints[i] = NULL;
+	  continue;
+	}
+
+      if (rhslabel == 0)
+	{
+	  if (dump_file && (dump_flags & TDF_DETAILS))
+	    {
+
+	      fprintf (dump_file, "%s is a non-pointer variable, "
+		       "ignoring constraint:",
+		       get_varinfo (rhs.var)->name);
+	      dump_constraint (dump_file, c);
+	      fprintf (dump_file, "\n");
+	    }
+	  constraints[i] = NULL;
+	  continue;
+	}
+
+      lhsvar = find_equivalent_node (graph, lhsvar, lhslabel);
+      rhsvar = find_equivalent_node (graph, rhsvar, rhslabel);
+      c->lhs.var = lhsvar;
+      c->rhs.var = rhsvar;
+    }
+}
+
+/* Eliminate indirect cycles involving NODE.  Return true if NODE was
+   part of an SCC, false otherwise.  */
+
+static bool
+eliminate_indirect_cycles (unsigned int node)
+{
+  if (graph->indirect_cycles[node] != -1
+      && !bitmap_empty_p (get_varinfo (node)->solution))
+    {
+      unsigned int i;
+      auto_vec<unsigned> queue;
+      int queuepos;
+      unsigned int to = find (graph->indirect_cycles[node]);
+      bitmap_iterator bi;
+
+      /* We can't touch the solution set and call unify_nodes
+	 at the same time, because unify_nodes is going to do
+	 bitmap unions into it.  */
+
+      EXECUTE_IF_SET_IN_BITMAP (get_varinfo (node)->solution, 0, i, bi)
+	{
+	  if (find (i) == i && i != to)
+	    {
+	      if (unite (to, i))
+		queue.safe_push (i);
+	    }
+	}
+
+      for (queuepos = 0;
+	   queue.iterate (queuepos, &i);
+	   queuepos++)
+	{
+	  unify_nodes (graph, to, i, true);
+	}
+      return true;
+    }
+  return false;
+}
+
+/* Solve the constraint graph GRAPH using our worklist solver.
+   This is based on the PW* family of solvers from the "Efficient Field
+   Sensitive Pointer Analysis for C" paper.
+   It works by iterating over all the graph nodes, processing the complex
+   constraints and propagating the copy constraints, until everything stops
+   changed.  This corresponds to steps 6-8 in the solving list given above.  */
+
+static void
+solve_graph (constraint_graph_t graph)
+{
+  unsigned int size = graph->size;
+  unsigned int i;
+  bitmap pts;
+
+  changed = BITMAP_ALLOC (NULL);
+
+  /* Mark all initial non-collapsed nodes as changed.  */
+  for (i = 1; i < size; i++)
+    {
+      varinfo_t ivi = get_varinfo (i);
+      if (find (i) == i && !bitmap_empty_p (ivi->solution)
+	  && ((graph->succs[i] && !bitmap_empty_p (graph->succs[i]))
+	      || graph->complex[i].length () > 0))
+	bitmap_set_bit (changed, i);
+    }
+
+  /* Allocate a bitmap to be used to store the changed bits.  */
+  pts = BITMAP_ALLOC (&pta_obstack);
+
+  while (!bitmap_empty_p (changed))
+    {
+      unsigned int i;
+      stats.iterations++;
+
+      bitmap_obstack_initialize (&iteration_obstack);
+
+      auto_vec<unsigned> topo_order = compute_topo_order (graph);
+      while (topo_order.length () != 0)
+	{
+	  i = topo_order.pop ();
+
+	  /* If this variable is not a representative, skip it.  */
+	  if (find (i) != i)
+	    continue;
+
+	  /* In certain indirect cycle cases, we may merge this
+	     variable to another.  */
+	  if (eliminate_indirect_cycles (i) && find (i) != i)
+	    continue;
+
+	  /* If the node has changed, we need to process the
+	     complex constraints and outgoing edges again.  For complex
+	     constraints that modify i itself, like the common group of
+	       callarg = callarg + UNKNOWN;
+	       callarg = *callarg + UNKNOWN;
+	       *callarg = callescape;
+	     make sure to iterate immediately because that maximizes
+	     cache reuse and expands the graph quickest, leading to
+	     better visitation order in the next iteration.  */
+	  while (bitmap_clear_bit (changed, i))
+	    {
+	      bitmap solution;
+	      vec<constraint_t> &complex = graph->complex[i];
+	      varinfo_t vi = get_varinfo (i);
+	      bool solution_empty;
+
+	      /* Compute the changed set of solution bits.  If anything
+		 is in the solution just propagate that.  */
+	      if (bitmap_bit_p (vi->solution, anything_id))
+		{
+		  /* If anything is also in the old solution there is
+		     nothing to do.
+		     ???  But we shouldn't ended up with "changed" set ...  */
+		  if (vi->oldsolution
+		      && bitmap_bit_p (vi->oldsolution, anything_id))
+		    break;
+		  bitmap_copy (pts, get_varinfo (find (anything_id))->solution);
+		}
+	      else if (vi->oldsolution)
+		bitmap_and_compl (pts, vi->solution, vi->oldsolution);
+	      else
+		bitmap_copy (pts, vi->solution);
+
+	      if (bitmap_empty_p (pts))
+		break;
+
+	      if (vi->oldsolution)
+		bitmap_ior_into (vi->oldsolution, pts);
+	      else
+		{
+		  vi->oldsolution = BITMAP_ALLOC (&oldpta_obstack);
+		  bitmap_copy (vi->oldsolution, pts);
+		}
+
+	      solution = vi->solution;
+	      solution_empty = bitmap_empty_p (solution);
+
+	      /* Process the complex constraints.  */
+	      hash_set<constraint_t> *cvisited = nullptr;
+	      if (flag_checking)
+		cvisited = new hash_set<constraint_t>;
+	      bitmap expanded_pts = NULL;
+	      for (unsigned j = 0; j < complex.length (); ++j)
+		{
+		  constraint_t c = complex[j];
+		  /* At unification time only the directly involved nodes
+		     will get their complex constraints updated.  Update
+		     our complex constraints now but keep the constraint
+		     vector sorted and clear of duplicates.  Also make
+		     sure to evaluate each prevailing constraint only once.  */
+		  unsigned int new_lhs = find (c->lhs.var);
+		  unsigned int new_rhs = find (c->rhs.var);
+		  if (c->lhs.var != new_lhs || c->rhs.var != new_rhs)
+		    {
+		      constraint tem = *c;
+		      tem.lhs.var = new_lhs;
+		      tem.rhs.var = new_rhs;
+		      unsigned int place
+			= complex.lower_bound (&tem, constraint_less);
+		      c->lhs.var = new_lhs;
+		      c->rhs.var = new_rhs;
+		      if (place != j)
+			{
+			  complex.ordered_remove (j);
+			  if (j < place)
+			    --place;
+			  if (place < complex.length ())
+			    {
+			      if (constraint_equal (*complex[place], *c))
+				{
+				  j--;
+				  continue;
+				}
+			      else
+				complex.safe_insert (place, c);
+			    }
+			  else
+			    complex.quick_push (c);
+			  if (place > j)
+			    {
+			      j--;
+			      continue;
+			    }
+			}
+		    }
+
+		  /* The only complex constraint that can change our
+		     solution to non-empty, given an empty solution,
+		     is a constraint where the lhs side is receiving
+		     some set from elsewhere.  */
+		  if (cvisited && cvisited->add (c))
+		    gcc_unreachable ();
+		  if (!solution_empty || c->lhs.type != DEREF)
+		    do_complex_constraint (graph, c, pts, &expanded_pts);
+		}
+	      if (cvisited)
+		{
+		  /* When checking, verify the order of constraints is
+		     maintained and each constraint is evaluated exactly
+		     once.  */
+		  for (unsigned j = 1; j < complex.length (); ++j)
+		    gcc_assert (constraint_less (complex[j-1], complex[j]));
+		  gcc_assert (cvisited->elements () == complex.length ());
+		  delete cvisited;
+		}
+	      BITMAP_FREE (expanded_pts);
+
+	      solution_empty = bitmap_empty_p (solution);
+
+	      if (!solution_empty)
+		{
+		  bitmap_iterator bi;
+		  unsigned eff_escaped_id = find (escaped_id);
+		  unsigned j;
+
+		  /* Propagate solution to all successors.  */
+		  unsigned to_remove = ~0U;
+		  EXECUTE_IF_IN_NONNULL_BITMAP (graph->succs[i],
+						0, j, bi)
+		    {
+		      if (to_remove != ~0U)
+			{
+			  bitmap_clear_bit (graph->succs[i], to_remove);
+			  to_remove = ~0U;
+			}
+		      unsigned int to = find (j);
+		      if (to != j)
+			{
+			  /* Update the succ graph, avoiding duplicate
+			     work.  */
+			  to_remove = j;
+			  if (! bitmap_set_bit (graph->succs[i], to))
+			    continue;
+			  /* We eventually end up processing 'to' twice
+			     as it is undefined whether bitmap iteration
+			     iterates over bits set during iteration.
+			     Play safe instead of doing tricks.  */
+			}
+		      /* Don't try to propagate to ourselves.  */
+		      if (to == i)
+			{
+			  to_remove = j;
+			  continue;
+			}
+		      /* Early node unification can lead to edges from
+			 escaped - remove them.  */
+		      if (i == eff_escaped_id)
+			{
+			  to_remove = j;
+			  if (bitmap_set_bit (get_varinfo (to)->solution,
+					      escaped_id))
+			    bitmap_set_bit (changed, to);
+			  continue;
+			}
+
+		      if (bitmap_ior_into (get_varinfo (to)->solution, pts))
+			bitmap_set_bit (changed, to);
+		    }
+		  if (to_remove != ~0U)
+		    bitmap_clear_bit (graph->succs[i], to_remove);
+		}
+	    }
+	}
+      bitmap_obstack_release (&iteration_obstack);
+    }
+
+  BITMAP_FREE (pts);
+  BITMAP_FREE (changed);
+  bitmap_obstack_release (&oldpta_obstack);
+}
+
+void
+delete_graph (void)
+{
+  unsigned int i;
+  for (i = 0; i < graph->size; i++)
+    graph->complex[i].release ();
+  free (graph->complex);
+
+  free (graph->succs);
+  free (graph->pe);
+  free (graph->pe_rep);
+  free (graph->indirect_cycles);
+  /* We are not doing free (graph->rep) since the representatives mapping is
+     needed outside of the solver too.  */
+  free (graph);
+}
+
+/* Remove the REF and ADDRESS edges from GRAPH, as well as all the
+   predecessor edges.  */
+
+static void
+remove_preds_and_fake_succs (constraint_graph_t graph)
+{
+  unsigned int i;
+
+  /* Clear the implicit ref and address nodes from the successor
+     lists.  */
+  for (i = 1; i < FIRST_REF_NODE; i++)
+    {
+      if (graph->succs[i])
+	bitmap_clear_range (graph->succs[i], FIRST_REF_NODE,
+			    FIRST_REF_NODE * 2);
+    }
+
+  /* Free the successor list for the non-ref nodes.  */
+  for (i = FIRST_REF_NODE + 1; i < graph->size; i++)
+    {
+      if (graph->succs[i])
+	BITMAP_FREE (graph->succs[i]);
+    }
+
+  /* Now reallocate the size of the successor list as, and blow away
+     the predecessor bitmaps.  */
+  graph->size = varmap.length ();
+  graph->succs = XRESIZEVEC (bitmap, graph->succs, graph->size);
+
+  free (graph->implicit_preds);
+  graph->implicit_preds = NULL;
+  free (graph->preds);
+  graph->preds = NULL;
+  bitmap_obstack_release (&predbitmap_obstack);
+}
+
+namespace pointer_analysis {
+
+/* Solve the constraint set.  The entry function of the solver.  */
+
+void
+solve_constraints (void)
+{
+  class scc_info *si;
+
+  /* Sort varinfos so that ones that cannot be pointed to are last.
+     This makes bitmaps more efficient.  */
+  unsigned int *map = XNEWVEC (unsigned int, varmap.length ());
+  for (unsigned i = 0; i < integer_id + 1; ++i)
+    map[i] = i;
+  /* Start with address-taken vars, followed by not address-taken vars
+     to move vars never appearing in the points-to solution bitmaps last.  */
+  unsigned j = integer_id + 1;
+  for (unsigned i = integer_id + 1; i < varmap.length (); ++i)
+    if (varmap[varmap[i]->head]->address_taken)
+      map[i] = j++;
+  for (unsigned i = integer_id + 1; i < varmap.length (); ++i)
+    if (! varmap[varmap[i]->head]->address_taken)
+      map[i] = j++;
+  /* Shuffle varmap according to map.  */
+  for (unsigned i = integer_id + 1; i < varmap.length (); ++i)
+    {
+      while (map[varmap[i]->id] != i)
+	std::swap (varmap[i], varmap[map[varmap[i]->id]]);
+      gcc_assert (bitmap_empty_p (varmap[i]->solution));
+      varmap[i]->id = i;
+      varmap[i]->next = map[varmap[i]->next];
+      varmap[i]->head = map[varmap[i]->head];
+    }
+  /* Finally rewrite constraints.  */
+  for (unsigned i = 0; i < constraints.length (); ++i)
+    {
+      constraints[i]->lhs.var = map[constraints[i]->lhs.var];
+      constraints[i]->rhs.var = map[constraints[i]->rhs.var];
+    }
+  free (map);
+
+  if (dump_file)
+    fprintf (dump_file,
+	     "\nCollapsing static cycles and doing variable "
+	     "substitution\n");
+
+  init_graph (varmap.length () * 2);
+
+  if (dump_file)
+    fprintf (dump_file, "Building predecessor graph\n");
+  build_pred_graph ();
+
+  if (dump_file)
+    fprintf (dump_file, "Detecting pointer and location "
+	     "equivalences\n");
+  si = perform_var_substitution (graph);
+
+  if (dump_file)
+    fprintf (dump_file, "Rewriting constraints and unifying "
+	     "variables\n");
+  rewrite_constraints (graph, si);
+
+  build_succ_graph ();
+
+  free_var_substitution_info (si);
+
+  /* Attach complex constraints to graph nodes.  */
+  move_complex_constraints (graph);
+
+  if (dump_file)
+    fprintf (dump_file, "Uniting pointer but not location equivalent "
+	     "variables\n");
+  unite_pointer_equivalences (graph);
+
+  if (dump_file)
+    fprintf (dump_file, "Finding indirect cycles\n");
+  find_indirect_cycles (graph);
+
+  /* Implicit nodes and predecessors are no longer necessary at this
+     point.  */
+  remove_preds_and_fake_succs (graph);
+
+  if (dump_file && (dump_flags & TDF_GRAPH))
+    {
+      fprintf (dump_file, "\n\n// The constraint graph before solve-graph "
+	       "in dot format:\n");
+      dump_constraint_graph (dump_file);
+      fprintf (dump_file, "\n\n");
+    }
+
+  if (dump_file)
+    fprintf (dump_file, "Solving graph\n");
+
+  solve_graph (graph);
+
+  if (dump_file && (dump_flags & TDF_GRAPH))
+    {
+      fprintf (dump_file, "\n\n// The constraint graph after solve-graph "
+	       "in dot format:\n");
+      dump_constraint_graph (dump_file);
+      fprintf (dump_file, "\n\n");
+    }
+
+  /* The mapping node -> representative is one of the outputs of the
+     solver.  */
+  union_find_compress_all ();
+  var_rep = graph->rep;
+
+  delete_graph ();
+}
+
+} // namespace pointer_analysis
diff --git a/gcc/pta-andersen.h b/gcc/pta-andersen.h
new file mode 100644
index 0000000..190a273
--- /dev/null
+++ b/gcc/pta-andersen.h
@@ -0,0 +1,31 @@
+/* Andersen-style solver for tree based points-to analysis
+   Copyright (C) 2005-2025 Free Software Foundation, Inc.
+   Contributed by Daniel Berlin <dberlin@dberlin.org>
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   GCC is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef PTA_ANDERSEN_H
+#define PTA_ANDERSEN_H
+
+namespace pointer_analysis {
+
+/* Solve the constraint set.  */
+void solve_constraints (void);
+
+} // namespace pointer_analysis
+
+#endif /* PTA_ANDERSEN_H  */
diff --git a/gcc/read-rtl-function.cc b/gcc/read-rtl-function.cc
index fb9c955..1f08c50 100644
--- a/gcc/read-rtl-function.cc
+++ b/gcc/read-rtl-function.cc
@@ -1065,7 +1065,10 @@ function_reader::read_rtx_operand_r (rtx x)
   if (regno == -1)
     fatal_at (loc, "unrecognized register: '%s'", name.string);
 
-  set_regno_raw (x, regno, 1);
+  int nregs = 1;
+  if (HARD_REGISTER_NUM_P (regno))
+    nregs = hard_regno_nregs (regno, GET_MODE (x));
+  set_regno_raw (x, regno, nregs);
 
   /* Consolidate singletons.  */
   x = consolidate_singletons (x);
diff --git a/gcc/recog.cc b/gcc/recog.cc
index edf22cb..67d7fa6 100644
--- a/gcc/recog.cc
+++ b/gcc/recog.cc
@@ -25,6 +25,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "target.h"
 #include "rtl.h"
 #include "tree.h"
+#include "stmt.h"
 #include "cfghooks.h"
 #include "df.h"
 #include "memmodel.h"
@@ -2363,7 +2364,8 @@ asm_operand_ok (rtx op, const char *constraint, const char **constraints)
 	    {
 	    case CT_REGISTER:
 	      if (!result
-		  && reg_class_for_constraint (cn) != NO_REGS
+		  && (reg_class_for_constraint (cn) != NO_REGS
+		      || constraint[0] == '{')
 		  && GET_MODE (op) != BLKmode
 		  && register_operand (op, VOIDmode))
 		result = 1;
@@ -3304,6 +3306,13 @@ constrain_operands (int strict, alternative_mask alternatives)
 		  win = true;
 		break;
 
+	      case '{':
+		if ((REG_P (op) && HARD_REGISTER_P (op)
+		     && (int) REGNO (op) == decode_hard_reg_constraint (p))
+		    || !reload_completed)
+		  win = true;
+		break;
+
 	      default:
 		{
 		  enum constraint_num cn = lookup_constraint (p);
diff --git a/gcc/rtl-error.cc b/gcc/rtl-error.cc
index 06d060c..7391a2f 100644
--- a/gcc/rtl-error.cc
+++ b/gcc/rtl-error.cc
@@ -28,7 +28,7 @@ along with GCC; see the file COPYING3.  If not see
 
 static location_t location_for_asm (const rtx_insn *);
 static void diagnostic_for_asm (const rtx_insn *, const char *, va_list *,
-				diagnostic_t) ATTRIBUTE_GCC_DIAG(2,0);
+				enum diagnostics::kind) ATTRIBUTE_GCC_DIAG(2,0);
 
 /* Figure the location of the given INSN.  */
 static location_t
@@ -64,9 +64,9 @@ location_for_asm (const rtx_insn *insn)
    and each ASM_OPERANDS records its own source file and line.  */
 static void
 diagnostic_for_asm (const rtx_insn *insn, const char *msg, va_list *args_ptr,
-		    diagnostic_t kind)
+		    enum diagnostics::kind kind)
 {
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   rich_location richloc (line_table, location_for_asm (insn));
 
   diagnostic_set_info (&diagnostic, msg, args_ptr,
@@ -80,7 +80,7 @@ error_for_asm (const rtx_insn *insn, const char *gmsgid, ...)
   va_list ap;
 
   va_start (ap, gmsgid);
-  diagnostic_for_asm (insn, gmsgid, &ap, DK_ERROR);
+  diagnostic_for_asm (insn, gmsgid, &ap, diagnostics::kind::error);
   va_end (ap);
 }
 
@@ -90,7 +90,7 @@ warning_for_asm (const rtx_insn *insn, const char *gmsgid, ...)
   va_list ap;
 
   va_start (ap, gmsgid);
-  diagnostic_for_asm (insn, gmsgid, &ap, DK_WARNING);
+  diagnostic_for_asm (insn, gmsgid, &ap, diagnostics::kind::warning);
   va_end (ap);
 }
 
diff --git a/gcc/rust/ChangeLog b/gcc/rust/ChangeLog
index 2a5219d..f103c7e 100644
--- a/gcc/rust/ChangeLog
+++ b/gcc/rust/ChangeLog
@@ -1,3 +1,37 @@
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* resolve/rust-ice-finalizer.cc: Update usage of "diagnostic_info"
+	to explicitly refer to "diagnostics::diagnostic_info".
+	* resolve/rust-ice-finalizer.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* backend/rust-tree.cc: Update for diagnostic_t becoming
+	enum class diagnostics::kind.
+	* backend/rust-tree.h: Likewise.
+	* resolve/rust-ast-resolve-expr.cc: Likewise.
+	* resolve/rust-ice-finalizer.cc: Likewise.
+	* resolve/rust-ice-finalizer.h: Likewise.
+	* resolve/rust-late-name-resolver-2.0.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* resolve/rust-ast-resolve-expr.cc: Update for diagnostic_text_finalizer
+	becoming diagnostics::text_finalizer.
+	* resolve/rust-late-name-resolver-2.0.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* resolve/rust-ice-finalizer.cc: Update for move of diagnostics
+	output formats into namespace "diagnostics" as "sinks".
+	* resolve/rust-ice-finalizer.h: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* rust-diagnostics.cc: Update #include for move of
+	"diagnostic-metadata.h" to "diagnostics/metadata.h", and update
+	for move of diagnostic_metadata to diagnostics::metadata.
+
 2025-06-26  Martin Jambor  <mjambor@suse.cz>
 
 	* checks/errors/borrowck/rust-borrow-checker-diagnostics.cc
diff --git a/gcc/rust/backend/rust-tree.cc b/gcc/rust/backend/rust-tree.cc
index 6cba04b..2302ffb 100644
--- a/gcc/rust/backend/rust-tree.cc
+++ b/gcc/rust/backend/rust-tree.cc
@@ -3848,16 +3848,18 @@ strip_top_quals (tree t)
 /* Print an error message for invalid use of an incomplete type.
    VALUE is the expression that was used (or 0 if that isn't known)
    and TYPE is the type that was invalid.  DIAG_KIND indicates the
-   type of diagnostic (see diagnostic.def).  */
+   type of diagnostic (see diagnostics/kinds.def).  */
 
 void
 cxx_incomplete_type_diagnostic (location_t loc, const_tree value,
-				const_tree type, diagnostic_t diag_kind)
+				const_tree type,
+				enum diagnostics::kind diag_kind)
 {
   //  bool is_decl = false, complained = false;
 
-  gcc_assert (diag_kind == DK_WARNING || diag_kind == DK_PEDWARN
-	      || diag_kind == DK_ERROR);
+  gcc_assert (diag_kind == diagnostics::kind::warning
+	      || diag_kind == diagnostics::kind::pedwarn
+	      || diag_kind == diagnostics::kind::error);
 
   /* Avoid duplicate error message.  */
   if (TREE_CODE (type) == ERROR_MARK)
@@ -5131,7 +5133,7 @@ complete_type_or_maybe_complain (tree type, tree value, tsubst_flags_t complain)
   else if (!COMPLETE_TYPE_P (type))
     {
       if (complain & tf_error)
-	cxx_incomplete_type_diagnostic (value, type, DK_ERROR);
+	cxx_incomplete_type_diagnostic (value, type, diagnostics::kind::error);
       note_failed_type_completion_for_satisfaction (type);
       return NULL_TREE;
     }
diff --git a/gcc/rust/backend/rust-tree.h b/gcc/rust/backend/rust-tree.h
index bb99684..3630b0e 100644
--- a/gcc/rust/backend/rust-tree.h
+++ b/gcc/rust/backend/rust-tree.h
@@ -3137,7 +3137,7 @@ extern bool reject_gcc_builtin (const_tree, location_t = UNKNOWN_LOCATION);
 extern tree resolve_nondeduced_context (tree, tsubst_flags_t);
 
 extern void cxx_incomplete_type_diagnostic (location_t, const_tree, const_tree,
-					    diagnostic_t);
+					    enum diagnostics::kind);
 
 extern void cxx_incomplete_type_error (location_t, const_tree, const_tree);
 
@@ -3430,7 +3430,7 @@ null_node_p (const_tree expr)
 
 inline void
 cxx_incomplete_type_diagnostic (const_tree value, const_tree type,
-				diagnostic_t diag_kind)
+				enum diagnostics::kind diag_kind)
 {
   cxx_incomplete_type_diagnostic (rs_expr_loc_or_input_loc (value), value, type,
 				  diag_kind);
@@ -3439,7 +3439,7 @@ cxx_incomplete_type_diagnostic (const_tree value, const_tree type,
 inline void
 cxx_incomplete_type_error (const_tree value, const_tree type)
 {
-  cxx_incomplete_type_diagnostic (value, type, DK_ERROR);
+  cxx_incomplete_type_diagnostic (value, type, diagnostics::kind::error);
 }
 
 extern location_t
diff --git a/gcc/rust/resolve/rust-ast-resolve-expr.cc b/gcc/rust/resolve/rust-ast-resolve-expr.cc
index 6242235..5524e18 100644
--- a/gcc/rust/resolve/rust-ast-resolve-expr.cc
+++ b/gcc/rust/resolve/rust-ast-resolve-expr.cc
@@ -129,8 +129,8 @@ ResolveExpr::visit (AST::IdentifierExpr &expr)
 	 resolve.  Emit a funny ICE.  We set the finalizer to our custom one,
 	 and use the lower-level emit_diagnostic () instead of the more common
 	 internal_error_no_backtrace () in order to pass our locus.  */
-      diagnostic_text_finalizer (global_dc) = funny_ice_text_finalizer;
-      emit_diagnostic (DK_ICE_NOBT, expr.get_locus (), -1,
+      diagnostics::text_finalizer (global_dc) = funny_ice_text_finalizer;
+      emit_diagnostic (diagnostics::kind::ice_nobt, expr.get_locus (), -1,
 		       "are you trying to break %s? how dare you?",
 		       expr.as_string ().c_str ());
     }
diff --git a/gcc/rust/resolve/rust-ice-finalizer.cc b/gcc/rust/resolve/rust-ice-finalizer.cc
index bd4763f..79a06c3 100644
--- a/gcc/rust/resolve/rust-ice-finalizer.cc
+++ b/gcc/rust/resolve/rust-ice-finalizer.cc
@@ -22,12 +22,12 @@ namespace Rust {
 namespace Resolver {
 
 void ATTRIBUTE_NORETURN
-funny_ice_text_finalizer (diagnostic_text_output_format &text_output,
-			  const diagnostic_info *diagnostic,
-			  diagnostic_t diag_kind)
+funny_ice_text_finalizer (diagnostics::text_sink &text_output,
+			  const diagnostics::diagnostic_info *diagnostic,
+			  enum diagnostics::kind diag_kind)
 {
-  gcc_assert (diag_kind == DK_ICE_NOBT);
-  default_diagnostic_text_finalizer (text_output, diagnostic, diag_kind);
+  gcc_assert (diag_kind == diagnostics::kind::ice_nobt);
+  diagnostics::default_text_finalizer (text_output, diagnostic, diag_kind);
   fnotice (stderr, "You have broken GCC Rust. This is a feature.\n");
   exit (ICE_EXIT_CODE);
 }
diff --git a/gcc/rust/resolve/rust-ice-finalizer.h b/gcc/rust/resolve/rust-ice-finalizer.h
index 85ab88f..5120176 100644
--- a/gcc/rust/resolve/rust-ice-finalizer.h
+++ b/gcc/rust/resolve/rust-ice-finalizer.h
@@ -55,9 +55,9 @@ namespace Resolver {
    the default bug reporting instructions, as there is no bug to report.  */
 
 void ATTRIBUTE_NORETURN
-funny_ice_text_finalizer (diagnostic_text_output_format &text_output,
-			  const diagnostic_info *diagnostic,
-			  diagnostic_t diag_kind);
+funny_ice_text_finalizer (diagnostics::text_sink &text_output,
+			  const diagnostics::diagnostic_info *diagnostic,
+			  enum diagnostics::kind diag_kind);
 
 } // namespace Resolver
 } // namespace Rust
diff --git a/gcc/rust/resolve/rust-late-name-resolver-2.0.cc b/gcc/rust/resolve/rust-late-name-resolver-2.0.cc
index 6ec0422..a15e17f 100644
--- a/gcc/rust/resolve/rust-late-name-resolver-2.0.cc
+++ b/gcc/rust/resolve/rust-late-name-resolver-2.0.cc
@@ -375,8 +375,8 @@ Late::visit (AST::IdentifierExpr &expr)
     }
   else if (funny_error)
     {
-      diagnostic_text_finalizer (global_dc) = Resolver::funny_ice_text_finalizer;
-      emit_diagnostic (DK_ICE_NOBT, expr.get_locus (), -1,
+      diagnostics::text_finalizer (global_dc) = Resolver::funny_ice_text_finalizer;
+      emit_diagnostic (diagnostics::kind::ice_nobt, expr.get_locus (), -1,
 		       "are you trying to break %s? how dare you?",
 		       expr.as_string ().c_str ());
     }
diff --git a/gcc/rust/rust-diagnostics.cc b/gcc/rust/rust-diagnostics.cc
index 0c0ef6e..702da71 100644
--- a/gcc/rust/rust-diagnostics.cc
+++ b/gcc/rust/rust-diagnostics.cc
@@ -22,7 +22,7 @@
 #include "rust-diagnostics.h"
 
 #include "options.h"
-#include "diagnostic-metadata.h"
+#include "diagnostics/metadata.h"
 
 static std::string
 mformat_value ()
@@ -192,7 +192,7 @@ rust_error_at (const location_t location, const char *fmt, ...)
   va_end (ap);
 }
 
-class rust_error_code_rule : public diagnostic_metadata::rule
+class rust_error_code_rule : public diagnostics::metadata::rule
 {
 public:
   rust_error_code_rule (const ErrorCode code) : m_code (code) {}
@@ -237,7 +237,7 @@ rust_be_error_at (const location_t location, const ErrorCode code,
 		  const std::string &errmsg)
 {
   rich_location gcc_loc (line_table, location);
-  diagnostic_metadata m;
+  diagnostics::metadata m;
   rust_error_code_rule rule (code);
   m.add_rule (rule);
   error_meta (&gcc_loc, m, "%s", errmsg.c_str ());
@@ -260,7 +260,7 @@ rust_be_error_at (const rich_location &location, const ErrorCode code,
 {
   /* TODO: 'error_at' would like a non-'const' 'rich_location *'.  */
   rich_location &gcc_loc = const_cast<rich_location &> (location);
-  diagnostic_metadata m;
+  diagnostics::metadata m;
   rust_error_code_rule rule (code);
   m.add_rule (rule);
   error_meta (&gcc_loc, m, "%s", errmsg.c_str ());
@@ -281,7 +281,7 @@ void
 rust_be_error_at (rich_location *richloc, const ErrorCode code,
 		  const std::string &errmsg)
 {
-  diagnostic_metadata m;
+  diagnostics::metadata m;
   rust_error_code_rule rule (code);
   m.add_rule (rule);
   error_meta (richloc, m, "%s", errmsg.c_str ());
diff --git a/gcc/selftest-run-tests.cc b/gcc/selftest-run-tests.cc
index 3245453..c68d3e7 100644
--- a/gcc/selftest-run-tests.cc
+++ b/gcc/selftest-run-tests.cc
@@ -28,6 +28,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "stringpool.h"
 #include "attribs.h"
 #include "analyzer/analyzer-selftests.h"
+#include "diagnostics/diagnostics-selftests.h"
 #include "text-art/selftests.h"
 
 /* This function needed to be split out from selftest.cc as it references
@@ -94,23 +95,17 @@ selftest::run_tests ()
   digraph_cc_tests ();
   tristate_cc_tests ();
   ipa_modref_tree_cc_tests ();
-  selftest_logical_location_cc_tests ();
+
+  /* Run the diagnostics selftests.  */
+  diagnostics::selftest::run_diagnostics_selftests ();
 
   /* Higher-level tests, or for components that other selftests don't
      rely on.  */
-  diagnostic_color_cc_tests ();
-  diagnostic_show_locus_cc_tests ();
-  diagnostic_format_html_cc_tests ();
-  diagnostic_format_sarif_cc_tests ();
-  diagnostic_output_spec_cc_tests ();
-  edit_context_cc_tests ();
   fold_const_cc_tests ();
   spellcheck_cc_tests ();
   spellcheck_tree_cc_tests ();
   tree_cfg_cc_tests ();
-  diagnostic_path_output_cc_tests ();
   simple_diagnostic_path_cc_tests ();
-  lazy_diagnostic_path_cc_tests ();
   attribs_cc_tests ();
   path_coverage_cc_tests ();
 
diff --git a/gcc/selftest.cc b/gcc/selftest.cc
index 4fdce07..dc90340 100644
--- a/gcc/selftest.cc
+++ b/gcc/selftest.cc
@@ -21,6 +21,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "system.h"
 #include "coretypes.h"
 #include "selftest.h"
+#include "diagnostics/file-cache.h"
 #include "intl.h"
 
 #if CHECKING_P
@@ -186,7 +187,7 @@ assert_str_startswith (const location &loc,
 /* Constructor.  Generate a name for the file.  */
 
 named_temp_file::named_temp_file (const char *suffix,
-				  file_cache *fc)
+				  diagnostics::file_cache *fc)
 {
   m_filename = make_temp_file (suffix);
   ASSERT_NE (m_filename, NULL);
@@ -210,7 +211,7 @@ named_temp_file::~named_temp_file ()
 temp_source_file::temp_source_file (const location &loc,
 				    const char *suffix,
 				    const char *content,
-				    file_cache *fc)
+				    diagnostics::file_cache *fc)
 : named_temp_file (suffix, fc)
 {
   FILE *out = fopen (get_filename (), "w");
diff --git a/gcc/selftest.h b/gcc/selftest.h
index eb9e604..4501d34 100644
--- a/gcc/selftest.h
+++ b/gcc/selftest.h
@@ -25,7 +25,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #if CHECKING_P
 
-class file_cache;
+namespace diagnostics { class file_cache; }
 
 namespace selftest {
 
@@ -100,13 +100,13 @@ extern void assert_str_startswith (const location &loc,
 class named_temp_file
 {
  public:
-  named_temp_file (const char *suffix, file_cache *fc = nullptr);
+  named_temp_file (const char *suffix, diagnostics::file_cache *fc = nullptr);
   ~named_temp_file ();
   const char *get_filename () const { return m_filename; }
 
  private:
   char *m_filename;
-  file_cache *m_file_cache;
+  diagnostics::file_cache *m_file_cache;
 };
 
 /* A class for writing out a temporary sourcefile for use in selftests
@@ -116,7 +116,7 @@ class temp_source_file : public named_temp_file
 {
  public:
   temp_source_file (const location &loc, const char *suffix,
-		    const char *content, file_cache *fc = nullptr);
+		    const char *content, diagnostics::file_cache *fc = nullptr);
   temp_source_file (const location &loc, const char *suffix,
 		    const char *content, size_t sz);
 };
@@ -221,15 +221,8 @@ extern void bitmap_cc_tests ();
 extern void cgraph_cc_tests ();
 extern void convert_cc_tests ();
 extern void dbgcnt_cc_tests ();
-extern void diagnostic_color_cc_tests ();
-extern void diagnostic_format_html_cc_tests ();
-extern void diagnostic_format_sarif_cc_tests ();
-extern void diagnostic_output_spec_cc_tests ();
-extern void diagnostic_path_output_cc_tests ();
-extern void diagnostic_show_locus_cc_tests ();
 extern void digraph_cc_tests ();
 extern void dumpfile_cc_tests ();
-extern void edit_context_cc_tests ();
 extern void et_forest_cc_tests ();
 extern void fibonacci_heap_cc_tests ();
 extern void fold_const_cc_tests ();
@@ -246,7 +239,6 @@ extern void input_cc_tests ();
 extern void ipa_modref_tree_cc_tests ();
 extern void json_cc_tests ();
 extern void json_parser_cc_tests ();
-extern void lazy_diagnostic_path_cc_tests ();
 extern void opt_suggestions_cc_tests ();
 extern void optinfo_emit_json_cc_tests ();
 extern void opts_cc_tests ();
@@ -261,7 +253,6 @@ extern void relation_tests ();
 extern void rtl_tests_cc_tests ();
 extern void sbitmap_cc_tests ();
 extern void selftest_cc_tests ();
-extern void selftest_logical_location_cc_tests ();
 extern void simple_diagnostic_path_cc_tests ();
 extern void simplify_rtx_cc_tests ();
 extern void spellcheck_cc_tests ();
diff --git a/gcc/simple-diagnostic-path.cc b/gcc/simple-diagnostic-path.cc
index d64f7b6..2e112b5 100644
--- a/gcc/simple-diagnostic-path.cc
+++ b/gcc/simple-diagnostic-path.cc
@@ -1,4 +1,4 @@
-/* Concrete classes for implementing diagnostic paths.
+/* Concrete classes for implementing diagnostic paths, using tree.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>
 
@@ -32,29 +32,31 @@ along with GCC; see the file COPYING3.  If not see
 #include "simple-diagnostic-path.h"
 #include "selftest.h"
 
-/* class simple_diagnostic_path : public diagnostic_path.  */
+using namespace diagnostics::paths;
+
+/* class simple_diagnostic_path : public diagnostics::paths::path.  */
 
 simple_diagnostic_path::
 simple_diagnostic_path (const tree_logical_location_manager &logical_loc_mgr,
 			pretty_printer *event_pp)
-: diagnostic_path (logical_loc_mgr),
+: path (logical_loc_mgr),
   m_event_pp (event_pp),
   m_localize_events (true)
 {
   add_thread ("main");
 }
 
-/* Implementation of diagnostic_path::get_event vfunc for
+/* Implementation of path::get_event vfunc for
    simple_diagnostic_path: simply return the event in the vec.  */
 
-const diagnostic_event &
+const event &
 simple_diagnostic_path::get_event (int idx) const
 {
   return *m_events[idx];
 }
 
-const diagnostic_thread &
-simple_diagnostic_path::get_thread (diagnostic_thread_id_t idx) const
+const thread &
+simple_diagnostic_path::get_thread (thread_id_t idx) const
 {
   return *m_threads[idx];
 }
@@ -67,7 +69,7 @@ simple_diagnostic_path::same_function_p (int event_idx_a,
 	  == m_events[event_idx_b]->get_fndecl ());
 }
 
-diagnostic_thread_id_t
+thread_id_t
 simple_diagnostic_path::add_thread (const char *name)
 {
   m_threads.safe_push (new simple_diagnostic_thread (name));
@@ -82,7 +84,7 @@ simple_diagnostic_path::add_thread (const char *name)
 
    Return the id of the new event.  */
 
-diagnostic_event_id_t
+event_id_t
 simple_diagnostic_path::add_event (location_t loc, tree fndecl, int depth,
 				   const char *fmt, ...)
 {
@@ -108,11 +110,11 @@ simple_diagnostic_path::add_event (location_t loc, tree fndecl, int depth,
 
   pp_clear_output_area (pp);
 
-  return diagnostic_event_id_t (m_events.length () - 1);
+  return event_id_t (m_events.length () - 1);
 }
 
-diagnostic_event_id_t
-simple_diagnostic_path::add_thread_event (diagnostic_thread_id_t thread_id,
+event_id_t
+simple_diagnostic_path::add_thread_event (thread_id_t thread_id,
 					  location_t loc,
 					  tree fndecl,
 					  int depth,
@@ -141,7 +143,7 @@ simple_diagnostic_path::add_thread_event (diagnostic_thread_id_t thread_id,
 
   pp_clear_output_area (pp);
 
-  return diagnostic_event_id_t (m_events.length () - 1);
+  return event_id_t (m_events.length () - 1);
 }
 
 /* Mark the most recent event on this path (which must exist) as being
@@ -163,7 +165,7 @@ simple_diagnostic_event (location_t loc,
 			 tree fndecl,
 			 int depth,
 			 const char *desc,
-			 diagnostic_thread_id_t thread_id)
+			 thread_id_t thread_id)
 : m_loc (loc), m_fndecl (fndecl),
   m_logical_loc (tree_logical_location_manager::key_from_tree (fndecl)),
   m_depth (depth), m_desc (xstrdup (desc)),
diff --git a/gcc/simple-diagnostic-path.h b/gcc/simple-diagnostic-path.h
index d2b366e..d549c24 100644
--- a/gcc/simple-diagnostic-path.h
+++ b/gcc/simple-diagnostic-path.h
@@ -1,4 +1,4 @@
-/* Concrete classes for implementing diagnostic paths.
+/* Concrete classes for implementing diagnostic paths, using tree.
    Copyright (C) 2019-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>
 
@@ -21,26 +21,30 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_SIMPLE_DIAGNOSTIC_PATH_H
 #define GCC_SIMPLE_DIAGNOSTIC_PATH_H
 
-#include "diagnostic-path.h"
+#include "diagnostics/paths.h"
 #include "tree-logical-location.h"
 
 /* Concrete subclasses of the abstract base classes
    declared in diagnostic-path.h.  */
 
-/* A simple implementation of diagnostic_event.  */
+/* A simple implementation of diagnostic event.
+   This uses "tree" and so is not in "namespace diagnostics".  */
 
-class simple_diagnostic_event : public diagnostic_event
+class simple_diagnostic_event : public diagnostics::paths::event
 {
  public:
+  using thread_id_t = diagnostics::paths::thread_id_t;
+
   simple_diagnostic_event (location_t loc, tree fndecl, int depth,
 			   const char *desc,
-			   diagnostic_thread_id_t thread_id = 0);
+			   thread_id_t thread_id = 0);
   ~simple_diagnostic_event ();
 
   location_t get_location () const final override { return m_loc; }
   int get_stack_depth () const final override { return m_depth; }
   void print_desc (pretty_printer &pp) const final override;
-  logical_location get_logical_location () const final override
+  diagnostics::logical_locations::key
+  get_logical_location () const final override
   {
     return tree_logical_location_manager::key_from_tree (m_fndecl);
   }
@@ -52,7 +56,7 @@ class simple_diagnostic_event : public diagnostic_event
   {
     return m_connected_to_next_event;
   }
-  diagnostic_thread_id_t get_thread_id () const final override
+  thread_id_t get_thread_id () const final override
   {
     return m_thread_id;
   }
@@ -67,16 +71,16 @@ class simple_diagnostic_event : public diagnostic_event
  private:
   location_t m_loc;
   tree m_fndecl;
-  logical_location m_logical_loc;
+  diagnostics::logical_locations::key m_logical_loc;
   int m_depth;
   char *m_desc; // has been i18n-ed and formatted
   bool m_connected_to_next_event;
-  diagnostic_thread_id_t m_thread_id;
+  thread_id_t m_thread_id;
 };
 
-/* A simple implementation of diagnostic_thread.  */
+/* A simple implementation of diagnostics::paths::thread.  */
 
-class simple_diagnostic_thread : public diagnostic_thread
+class simple_diagnostic_thread : public diagnostics::paths::thread
 {
 public:
   simple_diagnostic_thread (const char *name) : m_name (name) {}
@@ -92,28 +96,33 @@ private:
 /* A simple implementation of diagnostic_path, as a vector of
    simple_diagnostic_event instances.  */
 
-class simple_diagnostic_path : public diagnostic_path
+class simple_diagnostic_path : public diagnostics::paths::path
 {
  public:
+  using thread = diagnostics::paths::thread;
+  using thread_id_t = diagnostics::paths::thread_id_t;
+  using event = diagnostics::paths::event;
+  using event_id_t = diagnostics::paths::event_id_t;
+
   simple_diagnostic_path (const tree_logical_location_manager &logical_loc_mgr,
 			  pretty_printer *event_pp);
 
   unsigned num_events () const final override { return m_events.length (); }
-  const diagnostic_event & get_event (int idx) const final override;
+  const event & get_event (int idx) const final override;
   unsigned num_threads () const final override { return m_threads.length (); }
-  const diagnostic_thread &
-  get_thread (diagnostic_thread_id_t) const final override;
+  const thread &
+  get_thread (thread_id_t) const final override;
   bool
   same_function_p (int event_idx_a,
 		   int event_idx_b) const final override;
 
-  diagnostic_thread_id_t add_thread (const char *name);
+  thread_id_t add_thread (const char *name);
 
-  diagnostic_event_id_t add_event (location_t loc, tree fndecl, int depth,
-				   const char *fmt, ...)
+  event_id_t add_event (location_t loc, tree fndecl, int depth,
+			const char *fmt, ...)
     ATTRIBUTE_GCC_DIAG(5,6);
-  diagnostic_event_id_t
-  add_thread_event (diagnostic_thread_id_t thread_id,
+  event_id_t
+  add_thread_event (thread_id_t thread_id,
 		    location_t loc, tree fndecl, int depth,
 		    const char *fmt, ...)
     ATTRIBUTE_GCC_DIAG(6,7);
diff --git a/gcc/simplify-rtx.cc b/gcc/simplify-rtx.cc
index cbe61b4..c723a07 100644
--- a/gcc/simplify-rtx.cc
+++ b/gcc/simplify-rtx.cc
@@ -8344,6 +8344,15 @@ simplify_context::simplify_subreg (machine_mode outermode, rtx op,
 	return simplify_gen_binary (GET_CODE (op), outermode, op0, op1);
     }
 
+  /* Attempt to simplify WORD_MODE SUBREGs of unary bitwise expression.  */
+  if (outermode == word_mode && GET_CODE (op) == NOT
+      && SCALAR_INT_MODE_P (innermode))
+    {
+      rtx op0 = simplify_subreg (outermode, XEXP (op, 0), innermode, byte);
+      if (op0)
+	return simplify_gen_unary (GET_CODE (op), outermode, op0, outermode);
+    }
+
   scalar_int_mode int_outermode, int_innermode;
   if (is_a <scalar_int_mode> (outermode, &int_outermode)
       && is_a <scalar_int_mode> (innermode, &int_innermode)
@@ -8394,9 +8403,45 @@ simplify_context::simplify_subreg (machine_mode outermode, rtx op,
       && VECTOR_MODE_P (innermode)
       && known_eq (GET_MODE_NUNITS (outermode), GET_MODE_NUNITS (innermode))
       && known_eq (GET_MODE_UNIT_SIZE (outermode),
-		    GET_MODE_UNIT_SIZE (innermode)))
+		   GET_MODE_UNIT_SIZE (innermode)))
     return simplify_gen_relational (GET_CODE (op), outermode, innermode,
 				    XEXP (op, 0), XEXP (op, 1));
+
+  /* Distribute non-paradoxical subregs through logic ops in cases where one term
+     disappears.
+
+     (subreg:M1 (and:M2 X C1)) -> (subreg:M1 X)
+     (subreg:M1 (ior:M2 X C1)) -> (subreg:M1 C1)
+     (subreg:M1 (xor:M2 X C1)) -> (subreg:M1 (not:M2 X))
+
+     if M2 is no smaller than M1 and (subreg:M1 C1) is all-ones.
+
+     (subreg:M1 (and:M2 X C2)) -> (subreg:M1 C2)
+     (subreg:M1 (ior/xor:M2 X C2)) -> (subreg:M1 X)
+
+     if M2 is no smaller than M1 and (subreg:M1 C2) is zero.  */
+  if (known_ge (innersize, outersize)
+      && GET_MODE_CLASS (outermode) == GET_MODE_CLASS (innermode)
+      && (GET_CODE (op) == AND || GET_CODE (op) == IOR || GET_CODE (op) == XOR)
+      && CONSTANT_P (XEXP (op, 1)))
+    {
+      rtx op1_subreg = simplify_subreg (outermode, XEXP (op, 1), innermode, byte);
+      if (op1_subreg == CONSTM1_RTX (outermode))
+	{
+	  if (GET_CODE (op) == IOR)
+	    return op1_subreg;
+	  rtx op0 = XEXP (op, 0);
+	  if (GET_CODE (op) == XOR)
+	    op0 = simplify_gen_unary (NOT, innermode, op0, innermode);
+	  return simplify_gen_subreg (outermode, op0, innermode, byte);
+	}
+
+      if (op1_subreg == CONST0_RTX (outermode))
+	return (GET_CODE (op) == AND
+		? op1_subreg
+		: simplify_gen_subreg (outermode, XEXP (op, 0), innermode, byte));
+    }
+
   return NULL_RTX;
 }
 
@@ -8668,6 +8713,43 @@ test_scalar_int_ext_ops (machine_mode bmode, machine_mode smode)
 				     lowpart_subreg (bmode, sreg, smode),
 				     bmode),
 		 sreg);
+
+  /* Test extensions, followed by logic ops, followed by truncations.  */
+  rtx bsubreg = lowpart_subreg (bmode, sreg, smode);
+  rtx smask = gen_int_mode (GET_MODE_MASK (smode), bmode);
+  rtx inv_smask = gen_int_mode (~GET_MODE_MASK (smode), bmode);
+  ASSERT_RTX_EQ (lowpart_subreg (smode,
+				 simplify_gen_binary (AND, bmode,
+						      bsubreg, smask),
+				 bmode),
+		 sreg);
+  ASSERT_RTX_EQ (lowpart_subreg (smode,
+				 simplify_gen_binary (AND, bmode,
+						      bsubreg, inv_smask),
+				 bmode),
+		 const0_rtx);
+  ASSERT_RTX_EQ (lowpart_subreg (smode,
+				 simplify_gen_binary (IOR, bmode,
+						      bsubreg, smask),
+				 bmode),
+		 constm1_rtx);
+  ASSERT_RTX_EQ (lowpart_subreg (smode,
+				 simplify_gen_binary (IOR, bmode,
+						      bsubreg, inv_smask),
+				 bmode),
+		 sreg);
+  ASSERT_RTX_EQ (lowpart_subreg (smode,
+				 simplify_gen_binary (XOR, bmode,
+						      bsubreg, smask),
+				 bmode),
+		 lowpart_subreg (smode,
+				 gen_rtx_NOT (bmode, bsubreg),
+				 bmode));
+  ASSERT_RTX_EQ (lowpart_subreg (smode,
+				 simplify_gen_binary (XOR, bmode,
+						      bsubreg, inv_smask),
+				 bmode),
+		 sreg);
 }
 
 /* Verify more simplifications of integer extension/truncation.
diff --git a/gcc/spellcheck.cc b/gcc/spellcheck.cc
index 9a0a920..3fb0f50 100644
--- a/gcc/spellcheck.cc
+++ b/gcc/spellcheck.cc
@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
+#define INCLUDE_ALGORITHM
 #include "config.h"
 #include "system.h"
 #include "coretypes.h"
@@ -25,11 +26,15 @@ along with GCC; see the file COPYING3.  If not see
 #include "spellcheck.h"
 #include "selftest.h"
 
+namespace {
+
 /* Cost of a case transformation.  */
-#define CASE_COST 1
+const edit_distance_t case_cost = 1;
 
 /* Cost of another kind of edit.  */
-#define BASE_COST 2
+const edit_distance_t base_cost = 2;
+
+} // anonymous namespace
 
 /* Get the edit distance between the two strings: the minimal
    number of edits that are needed to change one string into another,
@@ -55,9 +60,9 @@ get_edit_distance (const char *s, int len_s,
     }
 
   if (len_s == 0)
-    return BASE_COST * len_t;
+    return base_cost * len_t;
   if (len_t == 0)
-    return BASE_COST * len_s;
+    return base_cost * len_s;
 
   /* We effectively build a matrix where each (i, j) contains the
      distance between the prefix strings s[0:j] and t[0:i].
@@ -75,7 +80,7 @@ get_edit_distance (const char *s, int len_s,
   /* The first row is for the case of an empty target string, which
      we can reach by deleting every character in the source string.  */
   for (int i = 0; i < len_s + 1; i++)
-    v_one_ago[i] = i * BASE_COST;
+    v_one_ago[i] = i * base_cost;
 
   /* Build successive rows.  */
   for (int i = 0; i < len_t; i++)
@@ -91,7 +96,7 @@ get_edit_distance (const char *s, int len_s,
       /* The initial column is for the case of an empty source string; we
 	 can reach prefixes of the target string of length i
 	 by inserting i characters.  */
-      v_next[0] = (i + 1) * BASE_COST;
+      v_next[0] = (i + 1) * base_cost;
 
       /* Build the rest of the row by considering neighbors to
 	 the north, west and northwest.  */
@@ -102,18 +107,18 @@ get_edit_distance (const char *s, int len_s,
 	  if (s[j] == t[i])
 	    cost = 0;
 	  else if (TOLOWER (s[j]) == TOLOWER (t[i]))
-	    cost = CASE_COST;
+	    cost = case_cost;
 	  else
-	    cost = BASE_COST;
-	  edit_distance_t deletion     = v_next[j] + BASE_COST;
-	  edit_distance_t insertion    = v_one_ago[j + 1] + BASE_COST;
+	    cost = base_cost;
+	  edit_distance_t deletion     = v_next[j] + base_cost;
+	  edit_distance_t insertion    = v_one_ago[j + 1] + base_cost;
 	  edit_distance_t substitution = v_one_ago[j] + cost;
-	  edit_distance_t cheapest = MIN (deletion, insertion);
-	  cheapest = MIN (cheapest, substitution);
+	  edit_distance_t cheapest = std::min (deletion, insertion);
+	  cheapest = std::min (cheapest, substitution);
 	  if (i > 0 && j > 0 && s[j] == t[i - 1] && s[j - 1] == t[i])
 	    {
-	      edit_distance_t transposition = v_two_ago[j - 1] + BASE_COST;
-	      cheapest = MIN (cheapest, transposition);
+	      edit_distance_t transposition = v_two_ago[j - 1] + base_cost;
+	      cheapest = std::min (cheapest, transposition);
 	    }
 	  v_next[j + 1] = cheapest;
 	}
@@ -187,8 +192,8 @@ find_closest_string (const char *target,
 edit_distance_t
 get_edit_distance_cutoff (size_t goal_len, size_t candidate_len)
 {
-  size_t max_length = MAX (goal_len, candidate_len);
-  size_t min_length = MIN (goal_len, candidate_len);
+  size_t max_length = std::max (goal_len, candidate_len);
+  size_t min_length = std::min (goal_len, candidate_len);
 
   gcc_assert (max_length >= min_length);
 
@@ -200,11 +205,11 @@ get_edit_distance_cutoff (size_t goal_len, size_t candidate_len)
   /* If the lengths are close, then round down.  */
   if (max_length - min_length <= 1)
     /* ...but allow an edit distance of at least 1.  */
-    return BASE_COST * MAX (max_length / 3, 1);
+    return base_cost * std::max (max_length / 3, size_t {1});
 
   /* Otherwise, round up (thus giving a little extra leeway to some cases
      involving insertions/deletions).  */
-  return BASE_COST * (max_length + 2) / 3;
+  return base_cost * (max_length + 2) / 3;
 }
 
 #if CHECKING_P
@@ -244,49 +249,49 @@ static void
 test_edit_distances ()
 {
   test_get_edit_distance_both_ways ("", "nonempty",
-				    BASE_COST * strlen ("nonempty"));
+				    base_cost * strlen ("nonempty"));
   test_get_edit_distance_both_ways ("saturday", "sunday",
-				    BASE_COST * 3);
-  test_get_edit_distance_both_ways ("foo", "m_foo", BASE_COST * 2);
+				    base_cost * 3);
+  test_get_edit_distance_both_ways ("foo", "m_foo", base_cost * 2);
   test_get_edit_distance_both_ways ("hello_world", "HelloWorld", 4);
   test_get_edit_distance_both_ways
-    ("the quick brown fox jumps over the lazy dog", "dog", BASE_COST * 40);
+    ("the quick brown fox jumps over the lazy dog", "dog", base_cost * 40);
   test_get_edit_distance_both_ways
     ("the quick brown fox jumps over the lazy dog",
      "the quick brown dog jumps over the lazy fox",
-     BASE_COST * 4);
+     base_cost * 4);
   test_get_edit_distance_both_ways
     ("Lorem ipsum dolor sit amet, consectetur adipiscing elit,",
      "All your base are belong to us",
-     BASE_COST * 44);
+     base_cost * 44);
   test_get_edit_distance_both_ways ("foo", "FOO", 3);
-  test_get_edit_distance_both_ways ("fee", "deed", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("coorzd1", "coordx1", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("assert", "sqrt", BASE_COST * 3);
-  test_get_edit_distance_both_ways ("PATH_MAX", "INT8_MAX", BASE_COST * 3);
-  test_get_edit_distance_both_ways ("time", "nice", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("bar", "carg", BASE_COST * 2);
+  test_get_edit_distance_both_ways ("fee", "deed", base_cost * 2);
+  test_get_edit_distance_both_ways ("coorzd1", "coordx1", base_cost * 2);
+  test_get_edit_distance_both_ways ("assert", "sqrt", base_cost * 3);
+  test_get_edit_distance_both_ways ("PATH_MAX", "INT8_MAX", base_cost * 3);
+  test_get_edit_distance_both_ways ("time", "nice", base_cost * 2);
+  test_get_edit_distance_both_ways ("bar", "carg", base_cost * 2);
   test_get_edit_distance_both_ways ("gtk_widget_show_all",
 				    "GtkWidgetShowAll", 10);
-  test_get_edit_distance_both_ways ("m_bar", "bar", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("MACRO", "MACRAME", BASE_COST * 3);
-  test_get_edit_distance_both_ways ("ab", "ac", BASE_COST * 1);
-  test_get_edit_distance_both_ways ("ab", "a", BASE_COST * 1);
-  test_get_edit_distance_both_ways ("a", "b", BASE_COST * 1);
-  test_get_edit_distance_both_ways ("nanl", "name", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("char", "bar", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("-optimize", "fsanitize", BASE_COST * 5);
-  test_get_edit_distance_both_ways ("__DATE__", "__i386__", BASE_COST * 4);
+  test_get_edit_distance_both_ways ("m_bar", "bar", base_cost * 2);
+  test_get_edit_distance_both_ways ("MACRO", "MACRAME", base_cost * 3);
+  test_get_edit_distance_both_ways ("ab", "ac", base_cost * 1);
+  test_get_edit_distance_both_ways ("ab", "a", base_cost * 1);
+  test_get_edit_distance_both_ways ("a", "b", base_cost * 1);
+  test_get_edit_distance_both_ways ("nanl", "name", base_cost * 2);
+  test_get_edit_distance_both_ways ("char", "bar", base_cost * 2);
+  test_get_edit_distance_both_ways ("-optimize", "fsanitize", base_cost * 5);
+  test_get_edit_distance_both_ways ("__DATE__", "__i386__", base_cost * 4);
 
   /* Examples where transposition helps.  */
-  test_get_edit_distance_both_ways ("ab", "ba", BASE_COST * 1);
-  test_get_edit_distance_both_ways ("ba", "abc", BASE_COST * 2);
-  test_get_edit_distance_both_ways ("coorzd1", "coordz1", BASE_COST * 1);
+  test_get_edit_distance_both_ways ("ab", "ba", base_cost * 1);
+  test_get_edit_distance_both_ways ("ba", "abc", base_cost * 2);
+  test_get_edit_distance_both_ways ("coorzd1", "coordz1", base_cost * 1);
   test_get_edit_distance_both_ways ("abcdefghijklmnopqrstuvwxyz",
 				    "bacdefghijklmnopqrstuvwxzy",
-				    BASE_COST * 2);
-  test_get_edit_distance_both_ways ("saturday", "sundya", BASE_COST * 4);
-  test_get_edit_distance_both_ways ("signed", "singed", BASE_COST * 1);
+				    base_cost * 2);
+  test_get_edit_distance_both_ways ("saturday", "sundya", base_cost * 4);
+  test_get_edit_distance_both_ways ("signed", "singed", base_cost * 1);
 }
 
 /* Subroutine of test_get_edit_distance_cutoff, for emulating the
@@ -295,7 +300,7 @@ test_edit_distances ()
 static edit_distance_t
 get_old_cutoff (size_t goal_len, size_t candidate_len)
 {
-  return BASE_COST * MAX (goal_len, candidate_len) / 2;
+  return base_cost * std::max (goal_len, candidate_len) / 2;
 }
 
 /* Verify that the cutoff for "meaningfulness" of suggestions is at least as
@@ -339,7 +344,7 @@ assert_not_suggested_for (const location &loc, const char *candidate,
 {
   auto_vec<const char *> candidates;
   candidates.safe_push (candidate);
-  ASSERT_EQ_AT (loc, NULL, find_closest_string (target, &candidates));
+  ASSERT_EQ_AT (loc, nullptr, find_closest_string (target, &candidates));
 }
 
 /* Assert that CANDIDATE is not offered as a suggestion for TARGET.  */
@@ -409,7 +414,7 @@ test_find_closest_string ()
   auto_vec<const char *> candidates;
 
   /* Verify that it can handle an empty vec.  */
-  ASSERT_EQ (NULL, find_closest_string ("", &candidates));
+  ASSERT_EQ (nullptr, find_closest_string ("", &candidates));
 
   /* Verify that it works sanely for non-empty vecs.  */
   candidates.safe_push ("apple");
@@ -419,7 +424,7 @@ test_find_closest_string ()
   ASSERT_STREQ ("apple", find_closest_string ("app", &candidates));
   ASSERT_STREQ ("banana", find_closest_string ("banyan", &candidates));
   ASSERT_STREQ ("cherry", find_closest_string ("berry", &candidates));
-  ASSERT_EQ (NULL, find_closest_string ("not like the others", &candidates));
+  ASSERT_EQ (nullptr, find_closest_string ("not like the others", &candidates));
 
   /* The order of the vec can matter, but it should not matter for these
      inputs.  */
@@ -430,12 +435,12 @@ test_find_closest_string ()
   ASSERT_STREQ ("apple", find_closest_string ("app", &candidates));
   ASSERT_STREQ ("banana", find_closest_string ("banyan", &candidates));
   ASSERT_STREQ ("cherry", find_closest_string ("berry", &candidates));
-  ASSERT_EQ (NULL, find_closest_string ("not like the others", &candidates));
+  ASSERT_EQ (nullptr, find_closest_string ("not like the others", &candidates));
 
   /* If the goal string somehow makes it into the candidate list, offering
      it as a suggestion will be nonsensical.  Verify that we don't offer such
      suggestions.  */
-  ASSERT_EQ (NULL, find_closest_string ("banana", &candidates));
+  ASSERT_EQ (nullptr, find_closest_string ("banana", &candidates));
 
   /* Example from PR 69968 where transposition helps.  */
   candidates.truncate (0);
diff --git a/gcc/spellcheck.h b/gcc/spellcheck.h
index b0c2c89..c5006ee 100644
--- a/gcc/spellcheck.h
+++ b/gcc/spellcheck.h
@@ -39,13 +39,13 @@ find_closest_string (const char *target,
    class best_match.
    Specializations should provide the implementations of the following:
 
-     static size_t get_length (TYPE);
-     static const char *get_string (TYPE);
+     static size_t get_length (StringlikeType);
+     static const char *get_string (StringlikeType);
 
    get_string should return a non-NULL ptr, which does not need to be
    0-terminated.  */
 
-template <typename TYPE>
+template <typename StringlikeType>
 struct edit_distance_traits {};
 
 /* Specialization of edit_distance_traits for C-style strings.  */
@@ -74,23 +74,23 @@ extern edit_distance_t get_edit_distance_cutoff (size_t goal_len,
    string-like types (const char *, frontend identifiers, and preprocessor
    macros).
 
-   This type accumulates the best possible match against GOAL_TYPE for
-   a sequence of elements of CANDIDATE_TYPE, whilst minimizing the
+   This type accumulates the best possible match against GoalType for
+   a sequence of elements of CandidateType, whilst minimizing the
    number of calls to get_edit_distance and to
    edit_distance_traits<T>::get_length.  */
 
-template <typename GOAL_TYPE, typename CANDIDATE_TYPE>
+template <typename GoalType, typename CandidateType>
 class best_match
 {
  public:
-  typedef GOAL_TYPE goal_t;
-  typedef CANDIDATE_TYPE candidate_t;
+  typedef GoalType goal_t;
+  typedef CandidateType candidate_t;
   typedef edit_distance_traits<goal_t> goal_traits;
   typedef edit_distance_traits<candidate_t> candidate_traits;
 
   /* Constructor.  */
 
-  best_match (GOAL_TYPE goal,
+  best_match (goal_t goal,
 	      edit_distance_t best_distance_so_far = MAX_EDIT_DISTANCE)
   : m_goal (goal_traits::get_string (goal)),
     m_goal_len (goal_traits::get_length (goal)),
@@ -163,7 +163,7 @@ class best_match
      m_best_candidate, update (without recomputing the edit distance to
      the goal).  */
 
-  void set_best_so_far (CANDIDATE_TYPE best_candidate,
+  void set_best_so_far (candidate_t best_candidate,
 			edit_distance_t best_distance,
 			size_t best_candidate_len)
   {
diff --git a/gcc/stmt.cc b/gcc/stmt.cc
index a510f8f..7942aa3 100644
--- a/gcc/stmt.cc
+++ b/gcc/stmt.cc
@@ -39,6 +39,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "emit-rtl.h"
 #include "pretty-print.h"
 #include "diagnostic-core.h"
+#include "output.h"
+#include "gimplify_reg_info.h"
 
 #include "fold-const.h"
 #include "varasm.h"
@@ -53,6 +55,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "dumpfile.h"
 #include "builtins.h"
 #include "cfgexpand.h"
+#include "output.h"
 
 
 /* Functions and data structures for expanding case statements.  */
@@ -175,6 +178,77 @@ expand_label (tree label)
     maybe_set_first_label_num (label_r);
 }
 
+/* Parse a hard register constraint and return its number or -1 in case of an
+   error.  BEGIN should point to a string of the form `{regname}`.  For the
+   sake of simplicity assume that a register name is not longer than 31
+   characters, if not error out.  */
+
+int
+decode_hard_reg_constraint (const char *begin)
+{
+  if (*begin != '{')
+    return -1;
+  ++begin;
+  const char *end = begin;
+  while (*end != '}' && *end != '\0')
+    ++end;
+  if (*end != '}' || end == begin)
+    return -1;
+  ptrdiff_t len = end - begin;
+  if (len >= 31)
+    return -1;
+  char regname[32];
+  memcpy (regname, begin, len);
+  regname[len] = '\0';
+  int regno = decode_reg_name (regname);
+  return regno;
+}
+
+static bool
+eliminable_regno_p (int regnum)
+{
+  static const struct
+  {
+    const int from;
+    const int to;
+  } eliminables[] = ELIMINABLE_REGS;
+  for (size_t i = 0; i < ARRAY_SIZE (eliminables); i++)
+    if (regnum == eliminables[i].from)
+      return true;
+  return false;
+}
+
+/* Perform a similar check as done in make_decl_rtl().  */
+
+static bool
+hardreg_ok_p (int reg_number, machine_mode mode, int operand_num)
+{
+  if (mode == BLKmode)
+    error ("data type isn%'t suitable for register %s of operand %i",
+	   reg_names[reg_number], operand_num);
+  else if (!in_hard_reg_set_p (accessible_reg_set, mode, reg_number))
+    error ("register %s for operand %i cannot be accessed"
+	   " by the current target", reg_names[reg_number], operand_num);
+  else if (!in_hard_reg_set_p (operand_reg_set, mode, reg_number))
+    error ("register %s for operand %i is not general enough"
+	   " to be used as a register variable", reg_names[reg_number], operand_num);
+  else if (!targetm.hard_regno_mode_ok (reg_number, mode))
+    error ("register %s for operand %i isn%'t suitable for data type",
+	   reg_names[reg_number], operand_num);
+  else if (reg_number != HARD_FRAME_POINTER_REGNUM
+	   && (reg_number == FRAME_POINTER_REGNUM
+#ifdef RETURN_ADDRESS_POINTER_REGNUM
+	       || reg_number == RETURN_ADDRESS_POINTER_REGNUM
+#endif
+	       || reg_number == ARG_POINTER_REGNUM)
+	   && eliminable_regno_p (reg_number))
+    error ("register for operand %i is an internal GCC "
+	   "implementation detail", operand_num);
+  else
+    return true;
+  return false;
+}
+
 /* Parse the output constraint pointed to by *CONSTRAINT_P.  It is the
    OPERAND_NUMth output operand, indexed from zero.  There are NINPUTS
    inputs and NOUTPUTS outputs to this extended-asm.  Upon return,
@@ -191,7 +265,8 @@ expand_label (tree label)
 bool
 parse_output_constraint (const char **constraint_p, int operand_num,
 			 int ninputs, int noutputs, bool *allows_mem,
-			 bool *allows_reg, bool *is_inout)
+			 bool *allows_reg, bool *is_inout,
+			 gimplify_reg_info *reg_info)
 {
   const char *constraint = *constraint_p;
   const char *p;
@@ -245,6 +320,9 @@ parse_output_constraint (const char **constraint_p, int operand_num,
       constraint = *constraint_p;
     }
 
+  unsigned int alt = 0;
+  bool early_clobbered = false;
+
   /* Loop through the constraint string.  */
   for (p = constraint + 1; *p; )
     {
@@ -264,12 +342,21 @@ parse_output_constraint (const char **constraint_p, int operand_num,
 	    }
 	  break;
 
-	case '?':  case '!':  case '*':  case '&':  case '#':
+	case '&':
+	  early_clobbered = true;
+	  break;
+
+	case '?':  case '!':  case '*':  case '#':
 	case '$':  case '^':
 	case 'E':  case 'F':  case 'G':  case 'H':
 	case 's':  case 'i':  case 'n':
 	case 'I':  case 'J':  case 'K':  case 'L':  case 'M':
-	case 'N':  case 'O':  case 'P':  case ',':  case '-':
+	case 'N':  case 'O':  case 'P':  case '-':
+	  break;
+
+	case ',':
+	  ++alt;
+	  early_clobbered = false;
 	  break;
 
 	case '0':  case '1':  case '2':  case '3':  case '4':
@@ -294,6 +381,60 @@ parse_output_constraint (const char **constraint_p, int operand_num,
 	  *allows_mem = true;
 	  break;
 
+	case '{':
+	  {
+	    if (!targetm.lra_p ())
+	      {
+		error ("hard register constraints are only supported while using LRA");
+		return false;
+	      }
+	    if (reg_info)
+	      {
+		int regno = decode_hard_reg_constraint (p);
+		if (regno < 0)
+		  {
+		    error ("invalid output constraint: %s", p);
+		    return false;
+		  }
+		int overlap_regno = reg_info->test_alt_output (alt, regno);
+		if (overlap_regno < 0)
+		  overlap_regno = reg_info->test_reg_asm_output (regno);
+		if (overlap_regno >= 0)
+		  {
+		    error ("multiple outputs to hard register: %s",
+			   reg_names[overlap_regno]);
+		    return false;
+		  }
+		reg_info->set_output (alt, regno);
+		if (early_clobbered)
+		  reg_info->set_early_clobbered (alt, operand_num, regno);
+		if (reg_info->is_clobbered (regno))
+		  {
+		    error ("hard register constraint for output %i conflicts "
+			   "with %<asm%> clobber list", operand_num);
+		    return false;
+		  }
+		if (VAR_P (reg_info->operand)
+		    && DECL_HARD_REGISTER (reg_info->operand))
+		  {
+		      tree id = DECL_ASSEMBLER_NAME (reg_info->operand);
+		      const char *asmspec = IDENTIFIER_POINTER (id) + 1;
+		      int regno_op = decode_reg_name (asmspec);
+		      if (regno != regno_op)
+			{
+			  error ("constraint and register %<asm%> for output "
+				 "operand %i are unsatisfiable", operand_num);
+			  return false;
+			}
+		  }
+		machine_mode mode = TYPE_MODE (TREE_TYPE (reg_info->operand));
+		if (!hardreg_ok_p (regno, mode, operand_num))
+		  return false;
+	      }
+	    *allows_reg = true;
+	    break;
+	  }
+
 	default:
 	  if (!ISALPHA (*p))
 	    break;
@@ -305,6 +446,31 @@ parse_output_constraint (const char **constraint_p, int operand_num,
 	    *allows_mem = true;
 	  else
 	    insn_extra_constraint_allows_reg_mem (cn, allows_reg, allows_mem);
+	  if (reg_info && *allows_reg
+	      && VAR_P (reg_info->operand)
+	      && DECL_HARD_REGISTER (reg_info->operand))
+	    {
+		tree id = DECL_ASSEMBLER_NAME (reg_info->operand);
+		const char *asmspec = IDENTIFIER_POINTER (id) + 1;
+		int regno = decode_reg_name (asmspec);
+		if (regno < 0)
+		  {
+		    location_t loc = DECL_SOURCE_LOCATION (reg_info->operand);
+		    error_at (loc, "invalid register name for %q+D",
+			      reg_info->operand);
+		    return false;
+		  }
+		int overlap_regno = reg_info->test_alt_output (alt, regno);
+		if (overlap_regno >= 0)
+		  {
+		    error ("multiple outputs to hard register: %s",
+			   reg_names[overlap_regno]);
+		    return false;
+		  }
+		reg_info->set_reg_asm_output (regno);
+		if (early_clobbered)
+		  reg_info->set_early_clobbered (alt, operand_num, regno);
+	    }
 	  break;
 	}
 
@@ -322,7 +488,8 @@ bool
 parse_input_constraint (const char **constraint_p, int input_num,
 			int ninputs, int noutputs, int ninout,
 			const char * const * constraints,
-			bool *allows_mem, bool *allows_reg)
+			bool *allows_mem, bool *allows_reg,
+			gimplify_reg_info *reg_info)
 {
   const char *constraint = *constraint_p;
   const char *orig_constraint = constraint;
@@ -338,6 +505,9 @@ parse_input_constraint (const char **constraint_p, int input_num,
 
   /* Make sure constraint has neither `=', `+', nor '&'.  */
 
+  unsigned int alt = 0;
+  unsigned long match = 0;
+
   for (j = 0; j < c_len; j += CONSTRAINT_LEN (constraint[j], constraint+j))
     switch (constraint[j])
       {
@@ -364,7 +534,7 @@ parse_input_constraint (const char **constraint_p, int input_num,
       case 'E':  case 'F':  case 'G':  case 'H':
       case 's':  case 'i':  case 'n':
       case 'I':  case 'J':  case 'K':  case 'L':  case 'M':
-      case 'N':  case 'O':  case 'P':  case ',':  case '-':
+      case 'N':  case 'O':  case 'P':  case '-':
 	break;
 
       case ':':
@@ -382,6 +552,10 @@ parse_input_constraint (const char **constraint_p, int input_num,
 	  }
 	break;
 
+      case ',':
+	++alt;
+	break;
+
 	/* Whether or not a numeric constraint allows a register is
 	   decided by the matching constraint, and so there is no need
 	   to do anything special with them.  We must handle them in
@@ -391,7 +565,6 @@ parse_input_constraint (const char **constraint_p, int input_num,
       case '5':  case '6':  case '7':  case '8':  case '9':
 	{
 	  char *end;
-	  unsigned long match;
 
 	  saw_match = true;
 
@@ -429,6 +602,65 @@ parse_input_constraint (const char **constraint_p, int input_num,
 	*allows_mem = true;
 	break;
 
+      case '{':
+	{
+	  if (!targetm.lra_p ())
+	    {
+	      error ("hard register constraints are only supported while using LRA");
+	      return false;
+	    }
+	  if (reg_info)
+	    {
+	      int regno = decode_hard_reg_constraint (constraint + j);
+	      if (regno < 0)
+		{
+		  error ("invalid input constraint: %s", constraint + j);
+		  return false;
+		}
+	      int overlap_regno = reg_info->test_alt_input (alt, regno);
+	      if (overlap_regno < 0)
+		overlap_regno = reg_info->test_reg_asm_input (regno);
+	      if (overlap_regno >= 0)
+		{
+		  error ("multiple inputs to hard register: %s",
+			    reg_names[overlap_regno]);
+		  return false;
+		}
+	      reg_info->set_input (alt, regno);
+	      if (reg_info->is_clobbered (regno))
+		{
+		  error ("hard register constraint for input %i conflicts "
+			 "with %<asm%> clobber list", input_num);
+		  return false;
+		}
+	      if (constraint == orig_constraint
+		  && reg_info->test_early_clobbered_alt (alt, regno))
+		{
+		  error ("invalid hard register usage between earlyclobber "
+			 "operand and input operand");
+		  return false;
+		}
+	      if (VAR_P (reg_info->operand)
+		  && DECL_HARD_REGISTER (reg_info->operand))
+		{
+		    tree id = DECL_ASSEMBLER_NAME (reg_info->operand);
+		    const char *asmspec = IDENTIFIER_POINTER (id) + 1;
+		    int regno_op = decode_reg_name (asmspec);
+		    if (regno != regno_op)
+		      {
+			error ("constraint and register %<asm%> for input "
+			       "operand %i are unsatisfiable", input_num);
+			return false;
+		      }
+		}
+	      machine_mode mode = TYPE_MODE (TREE_TYPE (reg_info->operand));
+	      if (!hardreg_ok_p (regno, mode, input_num))
+		return false;
+	    }
+	  *allows_reg = true;
+	  break;
+	}
+
       default:
 	if (! ISALPHA (constraint[j]))
 	  {
@@ -445,6 +677,39 @@ parse_input_constraint (const char **constraint_p, int input_num,
 	  *allows_mem = true;
 	else
 	  insn_extra_constraint_allows_reg_mem (cn, allows_reg, allows_mem);
+	if (reg_info && *allows_reg
+	    && VAR_P (reg_info->operand)
+	    && DECL_HARD_REGISTER (reg_info->operand))
+	  {
+	      tree id = DECL_ASSEMBLER_NAME (reg_info->operand);
+	      const char *asmspec = IDENTIFIER_POINTER (id) + 1;
+	      int regno = decode_reg_name (asmspec);
+	      if (regno < 0)
+		{
+		  location_t loc = DECL_SOURCE_LOCATION (reg_info->operand);
+		  error_at (loc, "invalid register name for %q+D",
+			    reg_info->operand);
+		  return false;
+		}
+	      int overlap_regno = reg_info->test_alt_input (alt, regno);
+	      if (overlap_regno >= 0)
+		{
+		  error ("multiple inputs to hard register: %s",
+			 reg_names[overlap_regno]);
+		  return false;
+		}
+	      reg_info->set_reg_asm_input (regno);
+	      if ((constraint == orig_constraint
+		   && reg_info->test_early_clobbered_alt (alt, regno))
+		  || (constraint != orig_constraint
+		      && reg_info->is_early_clobbered_in_any_output_unequal
+			  (match, regno)))
+		{
+		  error ("invalid hard register usage between earlyclobber "
+			 "operand and input operand");
+		  return false;
+		}
+	  }
 	break;
       }
 
diff --git a/gcc/stmt.h b/gcc/stmt.h
index 2beb4d5..c178036 100644
--- a/gcc/stmt.h
+++ b/gcc/stmt.h
@@ -20,11 +20,16 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_STMT_H
 #define GCC_STMT_H
 
+class gimplify_reg_info;
+
 extern void expand_label (tree);
 extern bool parse_output_constraint (const char **, int, int, int,
-				     bool *, bool *, bool *);
+				     bool *, bool *, bool *,
+				     gimplify_reg_info *);
 extern bool parse_input_constraint (const char **, int, int, int, int,
-				    const char * const *, bool *, bool *);
+				    const char * const *, bool *, bool *,
+				    gimplify_reg_info *);
+extern int decode_hard_reg_constraint (const char *);
 extern tree resolve_asm_operand_names (tree, tree, tree, tree);
 #ifdef HARD_CONST
 /* Silly ifdef to avoid having all includers depend on hard-reg-set.h.  */
diff --git a/gcc/stor-layout.cc b/gcc/stor-layout.cc
index 12071c9..63e830a 100644
--- a/gcc/stor-layout.cc
+++ b/gcc/stor-layout.cc
@@ -3167,7 +3167,7 @@ bit_field_mode_iterator::prefer_smaller_modes ()
    decide which of the above modes should be used.  */
 
 bool
-get_best_mode (int bitsize, int bitpos,
+get_best_mode (HOST_WIDE_INT bitsize, HOST_WIDE_INT bitpos,
 	       poly_uint64 bitregion_start, poly_uint64 bitregion_end,
 	       unsigned int align,
 	       unsigned HOST_WIDE_INT largest_mode_bitsize, bool volatilep,
diff --git a/gcc/substring-locations.cc b/gcc/substring-locations.cc
index 78ce722..2f920ca 100644
--- a/gcc/substring-locations.cc
+++ b/gcc/substring-locations.cc
@@ -151,7 +151,7 @@ format_string_diagnostic_t (const substring_loc &fmt_loc,
    Return true if a warning was emitted, false otherwise.  */
 
 bool
-format_string_diagnostic_t::emit_warning_n_va (diagnostic_option_id option_id,
+format_string_diagnostic_t::emit_warning_n_va (diagnostics::option_id option_id,
 					       unsigned HOST_WIDE_INT n,
 					       const char *singular_gmsgid,
 					       const char *plural_gmsgid,
@@ -203,7 +203,7 @@ format_string_diagnostic_t::emit_warning_n_va (diagnostic_option_id option_id,
   if (!err && m_corrected_substring && substring_within_range)
     richloc.add_fixit_replace (fmt_substring_range, m_corrected_substring);
 
-  diagnostic_info diagnostic;
+  diagnostics::diagnostic_info diagnostic;
   if (singular_gmsgid != plural_gmsgid)
     {
       unsigned long gtn;
@@ -218,12 +218,12 @@ format_string_diagnostic_t::emit_warning_n_va (diagnostic_option_id option_id,
 
       const char *text = ngettext (singular_gmsgid, plural_gmsgid, gtn);
       diagnostic_set_info_translated (&diagnostic, text, ap, &richloc,
-				      DK_WARNING);
+				      diagnostics::kind::warning);
     }
   else
     diagnostic_set_info (&diagnostic, singular_gmsgid, ap, &richloc,
-			 DK_WARNING);
-  diagnostic.option_id = option_id;
+			 diagnostics::kind::warning);
+  diagnostic.m_option_id = option_id;
   bool warned = diagnostic_report_diagnostic (global_dc, &diagnostic);
 
   if (!err && fmt_substring_loc && !substring_within_range)
@@ -248,7 +248,7 @@ format_string_diagnostic_t::emit_warning_n_va (diagnostic_option_id option_id,
 /* Singular-only version of the above.  */
 
 bool
-format_string_diagnostic_t::emit_warning_va (diagnostic_option_id option_id,
+format_string_diagnostic_t::emit_warning_va (diagnostics::option_id option_id,
 					     const char *gmsgid,
 					     va_list *ap) const
 {
@@ -258,7 +258,7 @@ format_string_diagnostic_t::emit_warning_va (diagnostic_option_id option_id,
 /* Variadic version of the above (singular only).  */
 
 bool
-format_string_diagnostic_t::emit_warning (diagnostic_option_id option_id,
+format_string_diagnostic_t::emit_warning (diagnostics::option_id option_id,
 					  const char *gmsgid,
 					  ...) const
 {
@@ -273,7 +273,7 @@ format_string_diagnostic_t::emit_warning (diagnostic_option_id option_id,
 /* Variadic version of the above (singular vs plural).  */
 
 bool
-format_string_diagnostic_t::emit_warning_n (diagnostic_option_id option_id,
+format_string_diagnostic_t::emit_warning_n (diagnostics::option_id option_id,
 					    unsigned HOST_WIDE_INT n,
 					    const char *singular_gmsgid,
 					    const char *plural_gmsgid,
diff --git a/gcc/substring-locations.h b/gcc/substring-locations.h
index fee7a89..94139e2 100644
--- a/gcc/substring-locations.h
+++ b/gcc/substring-locations.h
@@ -90,23 +90,23 @@ class format_string_diagnostic_t
 
   /* Functions for emitting a warning about a format string.  */
 
-  bool emit_warning_va (diagnostic_option_id option_id,
+  bool emit_warning_va (diagnostics::option_id option_id,
 			const char *gmsgid,
 			va_list *ap) const
     ATTRIBUTE_GCC_DIAG (3, 0);
 
-  bool emit_warning_n_va (diagnostic_option_id option_id,
+  bool emit_warning_n_va (diagnostics::option_id option_id,
 			  unsigned HOST_WIDE_INT n,
 			  const char *singular_gmsgid,
 			  const char *plural_gmsgid,
 			  va_list *ap) const
   ATTRIBUTE_GCC_DIAG (4, 0) ATTRIBUTE_GCC_DIAG (5, 0);
 
-  bool emit_warning (diagnostic_option_id option_id,
+  bool emit_warning (diagnostics::option_id option_id,
 		     const char *gmsgid, ...) const
     ATTRIBUTE_GCC_DIAG (3, 4);
 
-  bool emit_warning_n (diagnostic_option_id option_id,
+  bool emit_warning_n (diagnostics::option_id option_id,
 		       unsigned HOST_WIDE_INT n,
 		       const char *singular_gmsgid,
 		       const char *plural_gmsgid, ...) const
@@ -125,7 +125,7 @@ class format_string_diagnostic_t
    LANG_HOOKS_GET_SUBSTRING_LOCATION.  */
 
 extern const char *get_location_within_string (cpp_reader *pfile,
-					       file_cache &fc,
+					       diagnostics::file_cache &fc,
 					       string_concat_db *concats,
 					       location_t strloc,
 					       enum cpp_ttype type,
diff --git a/gcc/symtab.cc b/gcc/symtab.cc
index fc1155f..20dfe09 100644
--- a/gcc/symtab.cc
+++ b/gcc/symtab.cc
@@ -303,8 +303,14 @@ symbol_table::change_decl_assembler_name (tree decl, tree name)
 	warning (0, "%qD renamed after being referenced in assembly", decl);
 
       SET_DECL_ASSEMBLER_NAME (decl, name);
+      if (DECL_RTL_SET_P (decl))
+	{
+	  SET_DECL_RTL (decl, NULL);
+	  make_decl_rtl (decl);
+	}
       if (alias)
 	{
+	  gcc_assert (!IDENTIFIER_INTERNAL_P (name));
 	  IDENTIFIER_TRANSPARENT_ALIAS (name) = 1;
 	  TREE_CHAIN (name) = alias;
 	}
diff --git a/gcc/target.def b/gcc/target.def
index 38903eb..5dd8f25 100644
--- a/gcc/target.def
+++ b/gcc/target.def
@@ -1918,17 +1918,21 @@ implementation approaches itself.",
  NULL)
 
 /* Return true if the target supports misaligned store/load of a
-   specific factor denoted in the third parameter.  The last parameter
-   is true if the access is defined in a packed struct.  */
+   specific factor denoted in the third parameter.  The second to the last
+   parameter is true if the access is defined in a packed struct and
+   the last parameter is true if the access is a gather/scatter.  */
 DEFHOOK
 (support_vector_misalignment,
  "This hook should return true if the target supports misaligned vector\n\
 store/load of a specific factor denoted in the @var{misalignment}\n\
 parameter.  The vector store/load should be of machine mode @var{mode} and\n\
-the elements in the vectors should be of type @var{type}.  @var{is_packed}\n\
-parameter is true if the memory access is defined in a packed struct.",
+the elements in the vectors should be of type @var{type}.  The\n\
+@var{is_packed} parameter is true if the misalignment is unknown and the\n\
+memory access is defined in a packed struct.  @var{is_gather_scatter} is true\n\
+if the load/store is a gather or scatter.",
  bool,
- (machine_mode mode, const_tree type, int misalignment, bool is_packed),
+ (machine_mode mode, const_tree type, int misalignment, bool is_packed,
+  bool is_gather_scatter),
  default_builtin_support_vector_misalignment)
 
 /* Returns the preferred mode for SIMD operations for the specified
@@ -2056,6 +2060,20 @@ all zeros.  GCC can then try to branch around the instruction instead.",
  (unsigned ifn),
  default_empty_mask_is_expensive)
 
+/* Prefer gather/scatter loads/stores to e.g. elementwise accesses if\n\
+we cannot use a contiguous access.  */
+DEFHOOK
+(prefer_gather_scatter,
+ "This hook returns TRUE if gather loads or scatter stores are cheaper on\n\
+this target than a sequence of elementwise loads or stores.  The @var{mode}\n\
+and @var{scale} correspond to the @code{gather_load} and\n\
+@code{scatter_store} instruction patterns.  The @var{group_size} is the\n\
+number of scalar elements in each scalar loop iteration that are to be\n\
+combined into the vector.",
+ bool,
+ (machine_mode mode, int scale, unsigned int group_size),
+ hook_bool_mode_int_unsigned_false)
+
 /* Target builtin that implements vector gather operation.  */
 DEFHOOK
 (builtin_gather,
diff --git a/gcc/target.h b/gcc/target.h
index c9c7b52..47f1846 100644
--- a/gcc/target.h
+++ b/gcc/target.h
@@ -169,7 +169,6 @@ class predefined_function_abi;
 struct store_fwd_info;
 
 /* These are defined in tree-vect-stmts.cc.  */
-extern tree stmt_vectype (class _stmt_vec_info *);
 extern bool stmt_in_inner_loop_p (class vec_info *, class _stmt_vec_info *);
 
 /* Assembler instructions for creating various kinds of integer object.  */
diff --git a/gcc/targhooks.cc b/gcc/targhooks.cc
index c79458e..e723bbb 100644
--- a/gcc/targhooks.cc
+++ b/gcc/targhooks.cc
@@ -1556,6 +1556,8 @@ default_builtin_support_vector_misalignment (machine_mode mode,
 					     int misalignment
 					     ATTRIBUTE_UNUSED,
 					     bool is_packed
+					     ATTRIBUTE_UNUSED,
+					     bool is_gather_scatter
 					     ATTRIBUTE_UNUSED)
 {
   if (optab_handler (movmisalign_optab, mode) != CODE_FOR_nothing)
diff --git a/gcc/targhooks.h b/gcc/targhooks.h
index f16b587..3fa20af 100644
--- a/gcc/targhooks.h
+++ b/gcc/targhooks.h
@@ -114,7 +114,7 @@ extern bool default_builtin_vector_alignment_reachable (const_tree, bool);
 extern bool
 default_builtin_support_vector_misalignment (machine_mode mode,
 					     const_tree,
-					     int, bool);
+					     int, bool, bool);
 extern machine_mode default_preferred_simd_mode (scalar_mode mode);
 extern machine_mode default_split_reduction (machine_mode);
 extern unsigned int default_autovectorize_vector_modes (vector_modes *, bool);
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index ab15e88..db4d560 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,3016 @@
+2025-08-02  Martin Uecker  <uecker@tugraz.at>
+
+	* gcc.dg/Warray-parameter-11.c: Change Warray-parameter to
+	-Wvla-parameter as these are VLAs.
+	* gcc.dg/Warray-parameter.c: Remove xfail.
+
+2025-08-01  Artemiy Granat  <a.granat@ispras.ru>
+
+	* gcc.target/i386/attributes-error.c: Change incorrect
+	sseregparm,fastcall combination to cdecl,fastcall.
+
+2025-08-01  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/121322
+	* gcc.dg/pr121322.c: New test.
+
+2025-08-01  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/108080
+	* g++.dg/modules/pr108080.H: New test.
+
+2025-08-01  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/121238
+	* g++.dg/modules/merge-19.h: New test.
+	* g++.dg/modules/merge-19_a.H: New test.
+	* g++.dg/modules/merge-19_b.C: New test.
+
+2025-07-31  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121314
+	* gm2/errors/fail/badindrtype.mod: New test.
+	* gm2/errors/fail/badindrtype2.mod: New test.
+
+2025-07-31  Mikael Morin  <morin-mikael@orange.fr>
+
+	PR fortran/121342
+	* gfortran.dg/class_elemental_1.f90: New test.
+
+2025-07-31  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120800
+	* g++.dg/cpp0x/constexpr-array30.C: New test.
+
+2025-07-31  Marek Polacek  <polacek@redhat.com>
+
+	PR c++/120775
+	* g++.dg/cpp26/consteval-block1.C: New test.
+	* g++.dg/cpp26/consteval-block2.C: New test.
+	* g++.dg/cpp26/consteval-block3.C: New test.
+	* g++.dg/cpp26/consteval-block4.C: New test.
+	* g++.dg/cpp26/consteval-block5.C: New test.
+	* g++.dg/cpp26/consteval-block6.C: New test.
+	* g++.dg/cpp26/consteval-block7.C: New test.
+	* g++.dg/cpp26/consteval-block8.C: New test.
+
+2025-07-31  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c: Add asm check
+	for signed avg ceil.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h: Add test
+	helper macros.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h: Add
+	test data for run test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i8.c: New test.
+
+2025-07-31  Artemiy Granat  <a.granat@ispras.ru>
+
+	* gcc.target/i386/attributes-error.c: Add more attributes
+	combinations.
+
+2025-07-31  Artemiy Granat  <a.granat@ispras.ru>
+
+	* g++.dg/abi/regparm1.C: Require ia32 target.
+	* gcc.target/i386/20020224-1.c: Likewise.
+	* gcc.target/i386/pr103785.c: Use regparm attribute only if
+	not in 64-bit mode.
+	* gcc.target/i386/pr36533.c: Likewise.
+	* gcc.target/i386/pr59099.c: Likewise.
+	* gcc.target/i386/sibcall-8.c: Likewise.
+	* gcc.target/i386/sw-1.c: Likewise.
+	* gcc.target/i386/pr15184-2.c: Fix invalid comment.
+	* gcc.target/i386/attributes-ignore.c: New test.
+
+2025-07-31  Yury Khrustalev  <yury.khrustalev@arm.com>
+
+	* g++.target/aarch64/mv-cpu-features.C: new test.
+
+2025-07-31  Yury Khrustalev  <yury.khrustalev@arm.com>
+
+	* gcc.target/aarch64/ifunc-resolver.in: add core test functions.
+	* gcc.target/aarch64/ifunc-resolver-0.c: new test.
+	* gcc.target/aarch64/ifunc-resolver-1.c: ditto.
+	* gcc.target/aarch64/ifunc-resolver-2.c: ditto.
+	* gcc.target/aarch64/ifunc-resolver-3.c: ditto.
+	* gcc.target/aarch64/ifunc-resolver-4.c: as above.
+
+2025-07-31  Spencer Abson  <spencer.abson@arm.com>
+
+	PR target/121028
+	* gcc.target/aarch64/sme/call_sm_switch_1.c: Tell check-function
+	-bodies not to ignore .inst directives, and replace the test for
+	"smstart sm" with one for it's encoding.
+	* gcc.target/aarch64/sme/call_sm_switch_11.c: Likewise.
+	* gcc.target/aarch64/sme/pr121028.c: New test.
+
+2025-07-31  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/121264
+	* gcc.dg/tree-ssa/pr121264.c: New test.
+
+2025-07-31  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sme2/acle-asm/amax_f16_x2.c: Gate do-assemble on
+	assembler support for +faminmax and +sme2.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f16_x4.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f32_x2.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f32_x4.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f64_x2.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f64_x4.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f16_x2.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f16_x4.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f32_x2.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f32_x4.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f64_x2.c: Likewise.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f64_x4.c: Likewise.
+	* lib/target-supports.exp: Split the extensions that require SME into
+	a separate set, and use armv9-a as their baseline.
+
+2025-07-31  Jakub Jelinek  <jakub@redhat.com>
+
+	* gcc.target/i386/apx-1.c (apx_hanlder): Rename to ...
+	(apx_handler): ... this.
+	* gcc.target/i386/uintr-2.c (UINTR_hanlder): Rename to ...
+	(UINTR_handler): ... this.
+	* gcc.target/i386/uintr-5.c (UINTR_hanlder): Rename to ...
+	(UINTR_handler): ... this.
+
+2025-07-30  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/121291
+	* g++.dg/ext/is_invocable7.C: New test.
+	* g++.dg/ext/is_nothrow_convertible5.C: New test.
+
+2025-07-30  Jason Merrill  <jason@redhat.com>
+
+	* g++.dg/tc1/dr49.C: Adjust diagnostic.
+	* g++.dg/template/func2.C: Likewise.
+	* g++.dg/cpp1z/nontype8.C: New test.
+
+2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/121236
+	PR tree-optimization/121295
+	* gcc.dg/torture/pr121236-1.c: New test.
+	* gcc.dg/torture/pr121295-1.c: New test.
+
+2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	Revert:
+	2025-07-30  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/121236
+	* gcc.dg/torture/pr121236-1.c: New test.
+
+2025-07-30  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc.target/s390/spaceship-fp-1.c: New test.
+	* gcc.target/s390/spaceship-fp-2.c: New test.
+	* gcc.target/s390/spaceship-fp-3.c: New test.
+	* gcc.target/s390/spaceship-fp-4.c: New test.
+	* gcc.target/s390/spaceship-int-1.c: New test.
+	* gcc.target/s390/spaceship-int-2.c: New test.
+	* gcc.target/s390/spaceship-int-3.c: New test.
+
+2025-07-30  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120427
+	* gcc.target/i386/pr120427-5.c: New test.
+
+2025-07-30  Jan Hubicka  <jh@suse.cz>
+
+	* g++.dg/tree-prof/eh1.C: New test.
+
+2025-07-30  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121130
+	* gcc.dg/vect/vect-simd-pr121130.c: New testcase.
+
+2025-07-30  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/121133
+	* g++.dg/warn/pr121133-1.C: New test.
+	* g++.dg/warn/pr121133-2.C: New test.
+	* g++.dg/warn/pr121133-3.C: New test.
+	* g++.dg/warn/pr121133-4.C: New test.
+
+2025-07-30  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120778
+	* g++.dg/cpp/if-comma-1.C: New test.
+
+2025-07-30  Pengfei Li  <Pengfei.Li2@arm.com>
+
+	PR tree-optimization/121020
+	* gcc.dg/vect/vect-early-break_138-pr121020.c: New test.
+
+2025-07-30  Pengfei Li  <Pengfei.Li2@arm.com>
+
+	PR tree-optimization/121190
+	* gcc.dg/vect/vect-early-break_52.c: Update an unsafe test.
+	* gcc.dg/vect/vect-early-break_137-pr121190.c: New test.
+
+2025-07-30  Alfie Richards  <alfie.richards@arm.com>
+
+	PR target/121300
+	* gcc.target/aarch64/pr121300.c: New test.
+
+2025-07-30  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_cond_fmla_1.c: Add test cases
+	for merging with multiplcand.
+	* gcc.target/aarch64/sve/unpacked_cond_fmls_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fmla_2.c: New test.
+	* gcc.target/aarch64/sve/unpacked_cond_fmls_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fnmla_2.c: Likewise..
+	* gcc.target/aarch64/sve/unpacked_cond_fnmls_2.c: Likewise.
+	* g++.target/aarch64/sve/unpacked_cond_ternary_bf16_1.C: Likewise.
+	* g++.target/aarch64/sve/unpacked_cond_ternary_bf16_2.C: Likewise.
+
+2025-07-30  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_cond_fmla_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_cond_fmls_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c: Likewise.
+
+2025-07-30  Yuao Ma  <c8ef@outlook.com>
+
+	* gfortran.dg/split_1.f90: New test.
+	* gfortran.dg/split_2.f90: New test.
+	* gfortran.dg/split_3.f90: New test.
+	* gfortran.dg/split_4.f90: New test.
+
+2025-07-30  Spencer Abson  <spencer.abson@arm.com>
+
+	* g++.target/aarch64/sve/unpacked_ternary_bf16_1.C: New test.
+	* g++.target/aarch64/sve/unpacked_ternary_bf16_2.C: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmla_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmla_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmls_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmls_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fnmla_1.c: Likeiwse.
+	* gcc.target/aarch64/sve/unpacked_fnmla_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fnmls_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fnmls_2.c: Likewise.
+
+2025-07-30  liuhongt  <hongtao.liu@intel.com>
+
+	* gcc.target/i386/pr121274.c: New test.
+
+2025-07-30  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c: Add asm check
+	for unsigned avg ceil.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h: Add test
+	helper macros.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h: Add
+	test data.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u8.c: New test.
+
+2025-07-29  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR testsuite/121215
+	* lib/profopt.exp (profopt-execute): Call cleanup-after-saved-dg-test
+	if returning early for the -fauto-profile case failing case.
+
+2025-07-29  Spencer Abson  <spencer.abson@arm.com>
+
+	* g++.target/aarch64/sve/unpacked_cond_binary_bf16_2.C: New test.
+	* gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fadd_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fdiv_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fmaxnm_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fminnm_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fmul_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fsubr_2.c: Likewise.
+
+2025-07-29  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/121208
+	* gcc.target/i386/pr121208-1a.c (dg-options): Add -mno-80387.
+	* gcc.target/i386/pr121208-1b.c (dg-options): Likewise.
+
+2025-07-29  Juergen Christ  <jchrist@linux.ibm.com>
+
+	PR testsuite/121286
+	PR testsuite/121288
+	* gcc.dg/vect/pr112325.c: Adjust parameters for s390.
+	* gcc.dg/vect/pr117888-1.c: Ditto.
+
+2025-07-29  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/saturating_arithmetic_1.c: Allow w0 and w1
+	to be duplicated in either order.
+	* gcc.target/aarch64/saturating_arithmetic_2.c: Likewise.
+
+2025-07-29  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/cmpbr.c: Support both operand orders
+	for 8-bit and 16-bit comparisons.
+
+2025-07-29  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	PR rtl-optimization/120660
+	* gcc.dg/pr120660.c: New test.
+
+2025-07-29  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	PR rtl-optimization/119795
+	* gcc.target/i386/pr119795.c: New test.
+
+2025-07-29  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c: Add rv64
+	target for run.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-1-u16-from-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u8-from-u16.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u8-from-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-2-u16-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-2-u32-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-2-u8-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u16.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u32.c: New test.
+
+2025-07-29  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120687
+	* gcc.dg/vect/pr120687-3.c: New testcase.
+
+2025-07-29  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR testsuite/121285
+	* g++.dg/modules/class-11_a.H: Make static_asserts valid for
+	C++14.
+
+2025-07-29  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120687
+	* gcc.dg/vect/pr120687-1.c: New testcase.
+	* gcc.dg/vect/pr120687-2.c: Likewise.
+
+2025-07-29  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121289
+	* gm2/warnings/style/fail/badvarname.mod: New test.
+	* gm2/warnings/style/fail/warnings-style-fail.exp: New test.
+
+2025-07-29  Christophe Lyon  <christophe.lyon@linaro.org>
+
+	* gcc.dg/pr116906-1.c: Add 'dg-do run'.
+	* gcc.dg/pr116906-2.c: Likewise.
+	* gcc.dg/pr78185.c: Likewise.
+
+2025-07-29  Jakub Jelinek  <jakub@redhat.com>
+
+	PR middle-end/121159
+	* c-c++-common/pr121159.c: New test.
+	* gcc.dg/plugin/must-tail-call-2.c (test_5): Don't expect an error.
+
+2025-07-29  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/120523
+	* gcc.dg/tree-ssa/cswtch-7.c: New test.
+
+2025-07-28  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/121236
+	* gcc.dg/torture/pr121236-1.c: New test.
+
+2025-07-28  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/121208
+	* gcc.target/i386/pr121208-1a.c: New test.
+	* gcc.target/i386/pr121208-1b.c: Likewise.
+	* gcc.target/i386/pr121208-2a.c: Likewise.
+	* gcc.target/i386/pr121208-2b.c: Likewise.
+	* gcc.target/i386/pr121208-3a.c: Likewise.
+	* gcc.target/i386/pr121208-3b.c: Likewise.
+
+2025-07-28  Thomas Schwinge  <tschwinge@baylibre.com>
+
+	* gcc.target/nvptx/march-map=sm_100.c: New.
+	* gcc.target/nvptx/march-map=sm_100a.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_100f.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_101.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_101a.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_101f.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_103.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_103a.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_103f.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_120.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_120a.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_120f.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_121.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_121a.c: Likewise.
+	* gcc.target/nvptx/march-map=sm_121f.c: Likewise.
+
+2025-07-28  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121256
+	* gcc.dg/vect/vect-recurr-pr121256.c: New testcase.
+	* gcc.dg/vect/vect-recurr-pr121256-2.c: Likewise.
+
+2025-07-27  Mikael Morin  <mikael@gcc.gnu.org>
+
+	PR fortran/121185
+	* gfortran.dg/assign_14.f90: New test.
+
+2025-07-27  Mikael Morin  <mikael@gcc.gnu.org>
+
+	PR fortran/121185
+	* gfortran.dg/assign_13.f90: New test.
+
+2025-07-27  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i8.c: New test.
+
+2025-07-27  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c: Ditto.
+
+2025-07-27  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h: Add test helper
+	macros.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h: Add test
+	data for run test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i8.c: New test.
+
+2025-07-27  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c:
+	Add zvfh requirements and options.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c:
+	Ditto.
+
+2025-07-27  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	* g++.dg/modules/class-11_a.H: New test.
+	* g++.dg/modules/class-11_b.C: New test.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_test_graphs.cc: Eliminate
+	digraphs::lazy_digraph and digraphs::lazy_digraphs in favor of
+	lazily_created template.
+	* gcc.dg/plugin/diagnostic_plugin_test_metadata.cc: Define
+	INCLUDE_VECTOR since diagnostics/metadata.h now requires it.
+	* gcc.dg/plugin/diagnostic_plugin_test_paths.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_show_trees.cc: Make
+	diagnostics::context::m_source_printing private.
+	* gcc.dg/plugin/diagnostic_plugin_test_inlining.cc: Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_string_literals.cc:
+	Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_tree_expression_range.cc:
+	Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_paths.cc: Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Likewise.
+	* gcc.dg/plugin/expensive_selftests_plugin.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* g++.dg/plugin/show_template_tree_color_plugin.cc: Update usage
+	of "diagnostic_info" to explicitly refer to
+	"diagnostics::diagnostic_info".
+	* gcc.dg/plugin/diagnostic_group_plugin.cc: Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Likewise.
+	* gcc.dg/plugin/location_overflow_plugin.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Update for
+	file_cache and char_span moving from input.h to
+	diagnostics/file-cache.h and into the "diagnostics::" namespace.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Update for
+	diagnostic_t becoming enum class diagnostics::kind.
+	* gcc.dg/plugin/expensive_selftests_plugin.cc: Likewise.
+	* gcc.dg/plugin/location_overflow_plugin.cc: Likewise.
+	* lib/gcc-dg.exp: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/expensive_selftests_plugin.cc: Update for change
+	from edit-context.h to changes.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/expensive_selftests_plugin.cc: Update for move of
+	selftest::test_diagnostic_context to
+	diagnostics::selftest::test_context.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/analyzer_cpython_plugin.cc: Update for move of
+	diagnostic-color.h to diagnostics/color.h.
+	* gcc.dg/plugin/analyzer_kernel_plugin.cc: Likewise.
+	* gcc.dg/plugin/analyzer_known_fns_plugin.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/expensive_selftests_plugin.cc: Update for move of
+	selftest-diagnostic.h to diagnostics/selftest-context.h.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* g++.dg/plugin/show_template_tree_color_plugin.cc: Update for
+	moves to namespace diagnostics.
+	* gcc.dg/plugin/diagnostic_group_plugin.cc: Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Likewise.
+	* gcc.dg/plugin/location_overflow_plugin.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Update to
+	add "m_" prefix to fields of diagnostic_info throughout.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* g++.dg/plugin/show-template-tree-color-labels.C: Update for
+	moves to "source-printing".
+	* gcc.dg/plugin/diagnostic-test-show-locus.py: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_group_plugin.cc: Update for move of
+	diagnostics output formats into namespace "diagnostics" as
+	"sinks".
+	* gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc: Likewise.
+	* gcc.dg/plugin/location_overflow_plugin.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/analyzer_gil_plugin.cc: Update #include for
+	"diagnostic-path.h" moving to "diagnostics/paths.h",
+	diagnostic_thread_id_t to diagnostics::paths::thread_id_t,
+	diagnostic_event_id_t to diagnostics::paths::event_id_t,
+	diagnostic_path to diagnostics::paths::path, and
+	diagnostic_thread to diagnostics::paths::thread, and
+	diagnostic_event to diagnostics::paths::event.
+	* gcc.dg/plugin/diagnostic_plugin_test_paths.cc: Likewise.
+	* lib/sarif.py (get_state_graph): Update property prefix for
+	threadFlowLocations from "gcc/diagnostic_event/" to
+	"gcc/diagnostics/paths/event/".
+	* gcc.dg/sarif-output/include-chain-2.h: Update comment.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/expensive_selftests_plugin.cc: Update #include for
+	move of edit-context.h to diagnostics subdir.  Update
+	for move of edit_context to diagnostics::edit_context.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_test_text_art.cc: Update
+	#include for move of "diagnostic-diagram.h" to
+	"diagnostics/diagram.h". Update for move of diagnostic_diagram to
+	diagnostics::diagram.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/analyzer_cpython_plugin.cc: : Update #include for
+	move of "diagnostic-metadata.h" to "diagnostics/metadata.h"
+	* gcc.dg/plugin/analyzer_kernel_plugin.cc: Likewise.
+	* gcc.dg/plugin/analyzer_known_fns_plugin.cc: Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_graphs.cc: Likewise.  Also
+	update for move of diagnostic_metadata to diagnostics::metadata.
+	* gcc.dg/plugin/diagnostic_plugin_test_metadata.cc: Likewise.
+	* gcc.dg/plugin/diagnostic_plugin_test_paths.cc: Likewise.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/plugin/diagnostic_plugin_test_graphs.cc: Update #include
+	for move of "diagnostic-digraphs.h" to "diagnostics/digraphs.h".
+
+2025-07-25  Patrick Palka  <ppalka@redhat.com>
+
+	* g++.dg/lookup/operator-8.C: Remove XFAILs and properly
+	suppress all -Wunused-result warnings.
+
+2025-07-25  Iain Sandoe  <iain@sandoe.co.uk>
+
+	PR c++/121219
+	* g++.dg/coroutines/torture/pr121219.C: New test.
+
+2025-07-25  Christoph Müllner  <christoph.muellner@vrull.eu>
+
+	* gcc.target/riscv/interrupt-conflict-mode.c: Remove "user"
+	interrupts.
+	* gcc.target/riscv/xtheadint-push-pop.c: Likewise.
+	* gcc.target/riscv/interrupt-umode.c: Removed.
+
+2025-07-25  Christoph Müllner  <christoph.muellner@vrull.eu>
+
+	* gcc.target/riscv/interrupt-rnmi.c: New test.
+
+2025-07-24  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/117294
+	PR c++/113854
+	* g++.dg/cpp2a/concepts-traits3.C: Adjust diagnostics.
+	* g++.dg/cpp2a/concepts-traits4.C: New test.
+	* g++.dg/diagnostic/static_assert5.C: New test.
+	* g++.dg/ext/has_virtual_destructor2.C: New test.
+	* g++.dg/ext/is_assignable2.C: New test.
+	* g++.dg/ext/is_constructible9.C: New test.
+	* g++.dg/ext/is_convertible7.C: New test.
+	* g++.dg/ext/is_destructible3.C: New test.
+	* g++.dg/ext/is_invocable6.C: New test.
+	* g++.dg/ext/is_virtual_base_of_diagnostic2.C: New test.
+
+2025-07-24  Jason Merrill  <jason@redhat.com>
+
+	PR c++/114632
+	PR c++/101233
+	* g++.dg/cpp23/explicit-obj-lambda18.C: New test.
+	* g++.dg/cpp23/static-operator-call7.C: New test.
+
+2025-07-24  Robert Dubner  <rdubner@symas.com>
+
+	* cobol.dg/group2/_-static__compilation.cob: Modify for -static warning.
+	* cobol.dg/group2/_-static__compilation.out: Removed.
+
+2025-07-24  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* lib/target-supports.exp: Fix misalignment check.
+
+2025-07-24  Spencer Abson  <spencer.abson@arm.com>
+
+	* g++.target/aarch64/sve/unpacked_cond_binary_bf16_1.C: New test.
+	* gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fadd_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fdiv_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fmaxnm_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fminnm_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fmul_1.c: Likewise..
+	* gcc.target/aarch64/sve/unpacked_cond_fsubr_1.c: Likewise.
+
+2025-07-24  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_fdiv_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_fdiv_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fdiv_3.c: Likewise.
+
+2025-07-24  Spencer Abson  <spencer.abson@arm.com>
+
+	* g++.target/aarch64/sve/unpacked_binary_bf16_1.C: New test.
+	* g++.target/aarch64/sve/unpacked_binary_bf16_2.C: Likewise.
+	* gcc.target/aarch64/sve/unpacked_builtin_fmax_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_builtin_fmax_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_builtin_fmin_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_builtin_fmin_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fadd_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fadd_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmaxnm_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmaxnm_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fminnm_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fminnm_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmul_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fmul_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fsubr_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_fsubr_2.c: Likewise.
+
+2025-07-24  Jeevitha Palanisamy  <jeevitha@linux.ibm.com>
+
+	PR testsuite/119382
+	* gcc.target/powerpc/vsx-builtin-7.c: Add '-fno-ipa-icf' to dg-options.
+
+2025-07-24  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u8.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm.h: New test.
+
+2025-07-24  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/120412
+	* g++.dg/modules/internal-14_a.C: New test.
+	* g++.dg/modules/internal-14_b.C: New test.
+	* g++.dg/modules/internal-14_c.C: New test.
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_cond_fabs_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_cond_fneg_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frinta_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frinta_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frinti_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frintm_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frintp_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frintx_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_frintz_1.c: Likewise.
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_fabs_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_fneg_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frinta_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frinta_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frinti_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frinti_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintm_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintm_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintp_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintp_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintx_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintx_2.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintz_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_frintz_2.c: Likewise.
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_cond_cvtf_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_cond_fcvt_1.c: Likewise.
+	* gcc.target/aarch64/sve/unpacked_cond_fcvtz_1.c: Likewise.
+
+2025-07-23  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/121203
+	* gfortran.dg/function_charlen_4.f90: New test.
+
+2025-07-23  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/121073
+	* gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c: Adjust test
+	expectation.
+	* gcc.target/riscv/rvv/base/scalar_move-5.c: Ditto.
+	* gcc.target/riscv/rvv/base/scalar_move-6.c: Ditto.
+	* gcc.target/riscv/rvv/base/scalar_move-7.c: Ditto.
+	* gcc.target/riscv/rvv/base/scalar_move-8.c: Ditto.
+	* gcc.target/riscv/rvv/base/scalar_move-9.c: Ditto.
+	* gcc.target/riscv/rvv/pr121073.c: New test.
+
+2025-07-23  Robin Dapp  <rdapp@ventanamicro.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c:
+	Add zvfh requirements and options.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c:
+	Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c:
+	Ditto.
+	* lib/target-supports.exp: Add zvfh options.
+
+2025-07-23  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR target/120119
+	* g++.dg/torture/pr120119-1.C: New test.
+
+2025-07-23  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/121179
+	* g++.dg/lookup/operator-8.C: Strengthen test and remove one
+	XFAIL.
+
+2025-07-23  Patrick Palka  <ppalka@redhat.com>
+
+	PR c++/121055
+	* g++.dg/ext/is_invocable5.C: New test.
+
+2025-07-23  Spencer Abson  <spencer.abson@arm.com>
+
+	* lib/gcc-defs.exp (aarch64-arg-dg-options): Split add_tune into
+	add_tune and add_override, so that specifying -moverride does not
+	change the baseline tuning from the testuite's default (generic).
+
+2025-07-23  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121220
+	* gcc.dg/tree-ssa/ssa-sink-23.c: New testcase.
+
+2025-07-23  Martin Jambor  <mjambor@suse.cz>
+
+	PR tree-optimization/119085
+	* gcc.dg/tree-ssa/pr119085.c: New test.
+
+2025-07-23  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121164
+	* gm2/switches/pedantic-params/fail/arrayofchar.def: New test.
+	* gm2/switches/pedantic-params/fail/arrayofchar.mod: New test.
+
+2025-07-23  Richard Biener  <rguenther@suse.de>
+
+	PR middle-end/121216
+	* gcc.dg/pr121216.c: New testcase.
+
+2025-07-23  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR testsuite/120101
+	* gcc.dg/tree-ssa/pr81627.c (fn1): Mark as noinline.
+
+2025-07-23  Siddhesh Poyarekar  <siddhesh@gotplt.org>
+
+	* gcc.dg/vect/pr116125.c (mem_overlap): Expand A to 10 members.
+
+2025-07-22  Jason Merrill  <jason@redhat.com>
+
+	PR c++/121068
+	* g++.dg/cpp2a/constexpr-union6.C: Expect x5 to work.
+	* g++.dg/cpp26/constexpr-new4.C: New test.
+
+2025-07-22  Jason Merrill  <jason@redhat.com>
+
+	* g++.dg/warn/Wmismatched-new-delete-5.C: Fix allocation.
+
+2025-07-22  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/109267
+	* lib/target-supports.exp (check_effective_target_trap): New proc.
+	* g++.dg/missing-return.C: Update testcase for the !trap case.
+	* gcc.dg/pr109267-1.c: New test.
+	* gcc.dg/pr109267-2.c: New test.
+
+2025-07-22  Karl Meakin  <karl.meakin@arm.com>
+
+	* gcc.target/aarch64/sve/mask_load_2.c: Update tests.
+
+2025-07-22  Karl Meakin  <karl.meakin@arm.com>
+
+	* gcc.target/aarch64/sve/mask_load_2.c: New test.
+
+2025-07-22  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121202
+	* gcc.dg/pr121202.c: New testcase.
+
+2025-07-22  Richard Biener  <rguenther@suse.de>
+
+	* gcc.dg/vect/slp-28.c: Adjust.
+
+2025-07-21  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/110949
+	PR tree-optimization/95906
+	* gcc.dg/tree-ssa/cmp-2.c: New test.
+	* gcc.dg/tree-ssa/max-bitcmp-1.c: New test.
+
+2025-07-21  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u64.c: New test.
+
+2025-07-21  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c: Ditto.
+
+2025-07-21  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h: Add test
+	helper macros.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h: Add test
+	data for run test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u8.c: New test.
+
+2025-07-21  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/simd/mf8_data_1.c (test_set_lane4,
+	test_setq_lane4): Relax allowed assembly.
+	* gcc.target/aarch64/vec-set-zero.c: Use -Os in flags.
+	* gcc.target/aarch64/inszero_split_1.c: New test.
+
+2025-07-21  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121194
+	* gcc.dg/torture/pr121194.c: New testcase.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc.dg/pr87600-2.c: Split test into two files since errors for
+	functions test{0,1} are thrown during expand, and for
+	test{2,3} during gimplification.
+	* lib/scanasm.exp: On s390, skip lines beginning with #.
+	* gcc.dg/asm-hard-reg-error-1.c: New test.
+	* gcc.dg/asm-hard-reg-error-2.c: New test.
+	* gcc.dg/asm-hard-reg-error-3.c: New test.
+	* gcc.dg/asm-hard-reg-error-4.c: New test.
+	* gcc.dg/asm-hard-reg-error-5.c: New test.
+	* gcc.dg/pr87600-3.c: New test.
+	* gcc.target/aarch64/asm-hard-reg-2.c: New test.
+	* gcc.target/s390/asm-hard-reg-7.c: New test.
+
+2025-07-21  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc.dg/asm-hard-reg-1.c: New test.
+	* gcc.dg/asm-hard-reg-2.c: New test.
+	* gcc.dg/asm-hard-reg-3.c: New test.
+	* gcc.dg/asm-hard-reg-4.c: New test.
+	* gcc.dg/asm-hard-reg-5.c: New test.
+	* gcc.dg/asm-hard-reg-6.c: New test.
+	* gcc.dg/asm-hard-reg-7.c: New test.
+	* gcc.dg/asm-hard-reg-8.c: New test.
+	* gcc.target/aarch64/asm-hard-reg-1.c: New test.
+	* gcc.target/i386/asm-hard-reg-1.c: New test.
+	* gcc.target/i386/asm-hard-reg-2.c: New test.
+	* gcc.target/s390/asm-hard-reg-1.c: New test.
+	* gcc.target/s390/asm-hard-reg-2.c: New test.
+	* gcc.target/s390/asm-hard-reg-3.c: New test.
+	* gcc.target/s390/asm-hard-reg-4.c: New test.
+	* gcc.target/s390/asm-hard-reg-5.c: New test.
+	* gcc.target/s390/asm-hard-reg-6.c: New test.
+	* gcc.target/s390/asm-hard-reg-longdouble.h: New test.
+
+2025-07-21  Richard Biener  <rguenther@suse.de>
+
+	* gcc.dg/vect/vect-127.c: New testcase.
+
+2025-07-21  Andre Vehreschild  <vehre@gcc.gnu.org>
+
+	PR fortran/119106
+	* gfortran.dg/array_constructor_58.f90: New test.
+
+2025-07-21  panciyan  <panciyan@eswincomputing.com>
+
+	* gcc.target/riscv/sat/sat_arith.h: Unsigned testcase form8 form9.
+	* gcc.target/riscv/sat/sat_u_add-8-u16.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-8-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-8-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-8-u8.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-9-u16.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-9-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-9-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-9-u8.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-8-u16.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-8-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-8-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-8-u8.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-9-u16.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-9-u32.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-9-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_add-run-9-u8.c: New test.
+
+2025-07-20  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR testsuite/120859
+	* gcc.dg/tree-prof/afdo-crossmodule-1b.c: Add some dg-*
+	commands like what is in afdo-crossmodule-1.c
+
+2025-07-20  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i32.c:
+	Leverage DEF_AVG_0_WRAP to generate the correct func name.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i32-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i64-from-i128.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i32-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i64-from-i128.c: Ditto.
+
+2025-07-19  Dimitar Dimitrov  <dimitar@dinux.eu>
+
+	PR target/121124
+	* gcc.target/pru/pragma-ctable_entry-2.c: New test.
+
+2025-07-19  Paul-Antoine Arras  <parras@baylibre.com>
+
+	PR target/119100
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f16.c: Add vfwnmacc and
+	vfwnmsac.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f32.c: New test.
+
+2025-07-18  Harald Anlauf  <anlauf@gmx.de>
+
+	PR fortran/121145
+	* gfortran.dg/pointer_check_15.f90: New test.
+
+2025-07-18  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR testsuite/121153
+	* gcc.dg/vect/vect-reduc-cond-1.c: Require vect_condition.
+	* gcc.dg/vect/vect-reduc-cond-2.c: Likewise.
+
+2025-07-18  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/avg_data.h: Adjust the test data.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-1-i64-from-i128.c: New test.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i64-from-i128.c: New test.
+
+2025-07-18  Martin Jambor  <mjambor@suse.cz>
+
+	PR tree-optimization/117423
+	* gcc.dg/tree-ssa/pr117423.c: New test.
+
+2025-07-18  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121126
+	* gcc.dg/vect/pr121126.c: New testcase.
+
+2025-07-18  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120924
+	* gcc.dg/uninit-pr120924.c: New testcase.
+
+2025-07-18  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/121131
+	* gcc.dg/bitint-124.c: New test.
+
+2025-07-17  Jason Merrill  <jason@redhat.com>
+
+	PR c++/87097
+	* g++.dg/cpp0x/constexpr-array29.C: New test.
+
+2025-07-17  Richard Sandiford  <richard.sandiford@arm.com>
+	    Yury Khrustalev  <yury.khrustalev@arm.com>
+
+	* lib/target-supports.exp (add_options_for_aarch64_sme)
+	(check_effective_target_aarch64_sme_hw): New procedures.
+	* g++.target/aarch64/sme/sme_throw_1.C: New test.
+	* g++.target/aarch64/sme/sme_throw_2.C: Likewise.
+
+2025-07-17  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc.target/s390/isfinite-isinf-isnormal-signbit-2.c: Adapt
+	scan assembler directives.
+	* gcc.target/s390/isfinite-isinf-isnormal-signbit-3.c: Ditto.
+	* gcc.target/s390/signbit-1.c: New test.
+	* gcc.target/s390/signbit-2.c: New test.
+	* gcc.target/s390/signbit-3.c: New test.
+	* gcc.target/s390/signbit-4.c: New test.
+	* gcc.target/s390/signbit-5.c: New test.
+	* gcc.target/s390/signbit.h: New test.
+
+2025-07-17  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc.target/s390/vector/vlgv-zero-extend-1.c: New test.
+
+2025-07-17  Xi Ruoyao  <xry111@xry111.site>
+
+	PR target/121064
+	* gcc.target/loongarch/pr121064.c: New test.
+
+2025-07-17  Richard Biener  <rguenther@suse.de>
+
+	* gcc.dg/vect/bb-slp-39.c: Adjust.
+
+2025-07-17  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121035
+	* gcc.dg/pr121035.c: New testcase.
+
+2025-07-16  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/121062
+	* gcc.target/i386/pr121062-1.c: New test.
+	* gcc.target/i386/pr121062-2.c: Likewise.
+	* gcc.target/i386/pr121062-3a.c: Likewise.
+	* gcc.target/i386/pr121062-3b.c: Likewise.
+	* gcc.target/i386/pr121062-3c.c: Likewise.
+	* gcc.target/i386/pr121062-4.c: Likewise.
+	* gcc.target/i386/pr121062-5.c: Likewise.
+	* gcc.target/i386/pr121062-6.c: Likewise.
+	* gcc.target/i386/pr121062-7.c: Likewise.
+
+2025-07-16  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120881
+	PR testsuite/121078
+	* gcc.dg/20021014-1.c (dg-additional-options): Add -mfentry
+	-fno-pic only on gnu/x86 targets.
+	* gcc.dg/aru-2.c (dg-additional-options): Likewise.
+	* gcc.dg/nest.c (dg-additional-options): Likewise.
+	* gcc.dg/pr32450.c (dg-additional-options): Likewise.
+	* gcc.dg/pr43643.c (dg-additional-options): Likewise.
+	* gcc.target/i386/pr104447.c (dg-additional-options): Likewise.
+	* gcc.target/i386/pr113122-3.c(dg-additional-options): Likewise.
+	* gcc.target/i386/pr119386-1.c (dg-additional-options): Add
+	-mfentry only on gnu targets.
+	* gcc.target/i386/pr119386-2.c (dg-additional-options): Likewise.
+
+2025-07-16  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121049
+	* gcc.dg/vect/pr121049.c: New testcase.
+
+2025-07-16  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/119920
+	PR tree-optimization/112324
+	PR tree-optimization/110015
+	* gcc.dg/vect/vect-reduc-cond-1.c: New test.
+	* gcc.dg/vect/vect-reduc-cond-2.c: New test.
+	* gcc.dg/vect/vect-reduc-cond-3.c: New test.
+
+2025-07-16  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121116
+	* gcc.dg/torture/pr121116.c: New testcase.
+
+2025-07-16  Spencer Abson  <spencer.abson@arm.com>
+
+	PR target/117850
+	* gcc.target/aarch64/simd/vabal_combine.c: Removed.  This is
+	covered by fold_to_highpart_1.c
+	* gcc.target/aarch64/simd/fold_to_highpart_1.c: New test.
+	* gcc.target/aarch64/simd/fold_to_highpart_2.c: Likewise.
+	* gcc.target/aarch64/simd/fold_to_highpart_3.c: Likewise.
+	* gcc.target/aarch64/simd/fold_to_highpart_4.c: Likewise.
+	* gcc.target/aarch64/simd/fold_to_highpart_5.c: Likewise.
+	* gcc.target/aarch64/simd/fold_to_highpart_6.c: Likewise.
+
+2025-07-16  Alfie Richards  <alfie.richards@arm.com>
+
+	* g++.dg/warn/Wformat-gcc_diag-1.C: Add string_slice "%B" format tests.
+
+2025-07-16  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR middle-end/121065
+	* gcc.target/arm/pr121065.c: New test.
+
+2025-07-16  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/120297
+	* gcc.target/riscv/rvv/pr120297.c: New test.
+
+2025-07-16  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/sve2/eon_bsl2n.c: New test.
+
+2025-07-16  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/sve2/nbsl_nor_nand_neon.c: New test.
+
+2025-07-16  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/121060
+	* gfortran.dg/associate_75.f90: New test.
+
+2025-07-16  Steve Kargl  <sgk@troutmask.apl.washington.edu>
+
+	* gfortran.dg/import13.f90: New test.
+
+2025-07-16  Jeremy Rifkin  <jeremy@rifkin.dev>
+
+	PR c/82134
+	* c-c++-common/attr-warn-unused-result-2.c: New test.
+
+2025-07-16  Haochen Jiang  <haochen.jiang@intel.com>
+
+	* gcc.target/i386/amxavx512-cvtrowd2ps-2.c: Add -mavx512fp16 to
+	use FP16 related intrins for convert.
+	* gcc.target/i386/amxavx512-cvtrowps2bf16-2.c: Ditto.
+	* gcc.target/i386/amxavx512-cvtrowps2ph-2.c: Ditto.
+	* gcc.target/i386/amxavx512-movrow-2.c: Ditto.
+
+2025-07-16  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/sat/sat_s_add-1-i16.c: Remove function-body
+	check and add no jmp label asm check.
+	* gcc.target/riscv/sat/sat_s_add-1-i32.c:
+	* gcc.target/riscv/sat/sat_s_add-1-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-1-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-2-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-2-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-2-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-2-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-3-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-3-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-3-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-3-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-4-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-4-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-4-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-4-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-1-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-1-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-1-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-1-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-1-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-1-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-1-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-1-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-2-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-2-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-2-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-2-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-3-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-3-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-3-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-3-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-4-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-4-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-4-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-4-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-1-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-2-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-3-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-4-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-5-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-6-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-7-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-8-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-5-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-5-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-5-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-5-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-6-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-6-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-6-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-6-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-7-u16-from-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-7-u16-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-7-u32-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-7-u8-from-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-7-u8-from-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-7-u8-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-1-u16-from-u128.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-1-u32-from-u128.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-1-u64-from-u128.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-1-u8-from-u128.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-10-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-10-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-10-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-10-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-11-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-11-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-11-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-11-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-12-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-12-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-12-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-12-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-5-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-5-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-5-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-5-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-6-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-6-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-6-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-6-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-7-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-7-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-7-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-7-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-8-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-8-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-8-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-8-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-9-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-9-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-9-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-9-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u16-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u16-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u16-3.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u16-4.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u32-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u32-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u32-3.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u32-4.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u64-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u64-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u8-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u8-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u8-3.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u8-4.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u16-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u16-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u16-3.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u32-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u32-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u32-3.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u64-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u8-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u8-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u8-3.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u16-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u16-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u32-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u32-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u8-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u8-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u16-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u16-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u32-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u32-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u8-1.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u8-2.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-5-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-5-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-5-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-5-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-6-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-6-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-6-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-6-u8.c: Ditto.
+
+2025-07-16  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/avg.h: Add int128 type when
+	xlen == 64.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i32.c:
+	Suppress __int128 warning for run test.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i32-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_data.h: Fix one incorrect
+	test data.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i32-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/avg_floor-1-i64-from-i128.c: New test.
+	* gcc.target/riscv/rvv/autovec/avg_floor-run-1-i64-from-i128.c: New test.
+
+2025-07-15  David Malcolm  <dmalcolm@redhat.com>
+
+	PR sarif-replay/120792
+	* libgdiagnostics.dg/sarif.py: Delete duplicate script.
+	* libgdiagnostics.dg/test-message-buffer-c.py: New test script.
+	* libgdiagnostics.dg/test-message-buffer.c: New test.
+	* libgdiagnostics.dg/test-warning-with-path-c.py: Update expected
+	output to reflect that SARIF for event messages now contains JSON
+	pointers when referring to other events by ID.
+	* sarif-replay.dg/2.1.0-valid/3.11.6-embedded-links.sarif: Add
+	HTML and SARIF output, and call out to Python scripts to verify
+	the output.  Add example of a result with a link in its message.
+	* sarif-replay.dg/2.1.0-valid/embedded-links-check-html.py: New
+	test script.
+	* sarif-replay.dg/2.1.0-valid/embedded-links-check-sarif-roundtrip.py:
+	New test script.
+
+2025-07-15  Umesh Kalappa  <ukalappa.mips@gmail.com>
+
+	* gcc.target/riscv/mipscondmov.c: Test file for mips.ccmov insn.
+
+2025-07-15  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120577
+	* g++.dg/cpp2a/constexpr-union9.C: New test.
+
+2025-07-15  Jason Merrill  <jason@redhat.com>
+
+	PR c++/117784
+	* g++.dg/cpp26/decomp25.C: Add -fno-implicit-constexpr.
+
+2025-07-15  Kwok Cheung Yeung  <kcyeung@baylibre.com>
+
+	PR fortran/104428
+	* gfortran.dg/gomp/pr104428.f90: New.
+
+2025-07-15  Konstantinos Eleftheriou  <konstantinos.eleftheriou@vrull.eu>
+
+	* gcc.target/aarch64/avoid-store-forwarding-be.c: New test.
+
+2025-07-15  Soumya AR  <soumyaa@nvidia.com>
+
+	* gcc.target/aarch64/ldapr-sext.c: Update expected output to include
+	offsets.
+	* gcc.target/aarch64/ldapur.c: New test for LDAPUR.
+	* gcc.target/aarch64/ldapur_avoid.c: New test for AVOID_LDAPUR flag.
+
+2025-07-15  Richard Biener  <rguenther@suse.de>
+	    Richard Sandiford   <richard.sandiford@arm.com>
+
+	PR tree-optimization/121059
+	* gcc.dg/vect/pr121059.c: New testcase.
+
+2025-07-15  Jakub Jelinek  <jakub@redhat.com>
+	    Jason Merrill  <jason@redhat.com>
+
+	PR c/44677
+	* c-c++-common/Wunused-parm-1.c: New test.
+	* c-c++-common/Wunused-parm-2.c: New test.
+	* c-c++-common/Wunused-parm-3.c: New test.
+	* c-c++-common/Wunused-parm-4.c: New test.
+	* c-c++-common/Wunused-parm-5.c: New test.
+	* c-c++-common/Wunused-parm-6.c: New test.
+	* c-c++-common/Wunused-var-7.c (bar, baz): Expect warning on a.
+	* c-c++-common/Wunused-var-19.c: New test.
+	* c-c++-common/Wunused-var-20.c: New test.
+	* c-c++-common/Wunused-var-21.c: New test.
+	* c-c++-common/Wunused-var-22.c: New test.
+	* c-c++-common/Wunused-var-23.c: New test.
+	* c-c++-common/Wunused-var-24.c: New test.
+	* g++.dg/cpp26/name-independent-decl1.C (foo): Expect one
+	set but not used warning.
+	* g++.dg/warn/Wunused-parm-12.C: New test.
+	* g++.dg/warn/Wunused-parm-13.C: New test.
+	* g++.dg/warn/Wunused-var-2.C (f2): Expect set but not used warning
+	on parameter x and variable a.
+	* g++.dg/warn/Wunused-var-40.C: New test.
+	* g++.dg/warn/Wunused-var-41.C: New test.
+	* gcc.dg/memchr-3.c (test_find): Change return type from void to int,
+	and add return n; statement.
+	* gcc.dg/unused-9.c (g): Move dg-bogus to the correct line and expect
+	a warning on i.
+
+2025-07-15  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	Revert:
+	2025-07-15  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/simd/eor3_d.c: Add tests for DImode operands.
+
+2025-07-15  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_fcm_combines_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_fcm_combines_2.c: Likewise.
+
+2025-07-14  Richard Biener  <rguenther@suse.de>
+
+	Revert:
+	2025-07-14  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121059
+	* gcc.dg/vect/pr121059.c: New testcase.
+
+2025-07-14  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* lib/target-supports.exp: Add s390 to vect_logical_reduc targets.
+	* gcc.target/s390/vector/reduc-binops-1.c: New test.
+	* gcc.target/s390/vector/reduc-minmax-1.c: New test.
+	* gcc.target/s390/vector/reduc-plus-1.c: New test.
+
+2025-07-14  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121059
+	* gcc.dg/vect/pr121059.c: New testcase.
+
+2025-07-14  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/sat/sat_u_mul-1-u16-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u32-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u8-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c: New test.
+
+2025-07-14  Uros Bizjak  <ubizjak@gmail.com>
+
+	PR target/121015
+	* gcc.target/i386/pr121015.c: New test.
+
+2025-07-14  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120881
+	* gcc.dg/20021014-1.c: Add additional -mfentry -fno-pic options
+	for x86.
+	* gcc.dg/aru-2.c: Likewise.
+	* gcc.dg/nest.c: Likewise.
+	* gcc.dg/pr32450.c: Likewise.
+	* gcc.dg/pr43643.c: Likewise.
+	* gcc.target/i386/pr104447.c: Likewise.
+	* gcc.target/i386/pr113122-3.c: Likewise.
+	* gcc.target/i386/pr119386-1.c: Add additional -mfentry if not
+	ia32.
+	* gcc.target/i386/pr119386-2.c: Likewise.
+	* gcc.target/i386/pr120881-1a.c: New test.
+	* gcc.target/i386/pr120881-1b.c: Likewise.
+	* gcc.target/i386/pr120881-1c.c: Likewise.
+	* gcc.target/i386/pr120881-1d.c: Likewise.
+	* gcc.target/i386/pr120881-2a.c: Likewise.
+	* gcc.target/i386/pr120881-2b.c: Likewise.
+	* gcc.target/i386/pr82699-1.c: Add additional -mfentry.
+	* lib/target-supports.exp (check_effective_target_fentry): New.
+
+2025-07-14  François-Xavier Coudert  <fxcoudert@gcc.gnu.org>
+
+	* gcc.dg/darwin-minversion-link.c: Account for macOS 26.
+
+2025-07-14  Paul-Antoine Arras  <parras@baylibre.com>
+
+	PR target/119100
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f16.c: Add vfwmacc and
+	vfwmsac.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f16.c: Likewise. Also check
+	for fcvt and vfmv.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f16.c: Add vfwmacc and
+	vfwmsac.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f16.c: Likewise. Also check
+	for fcvt and vfmv.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f32.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop.h: Add support for
+	widening variants.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_widen_run.h: New test
+	helper.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f32.c: New test.
+
+2025-07-14  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* gnat.dg/deref4.adb: New test.
+	* gnat.dg/deref4_pkg.ads: New helper.
+
+2025-07-14  Alfie Richards  <alfie.richards@arm.com>
+
+	* gcc.target/aarch64/sme2/acle-asm/amax_f16_x2.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f16_x4.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f32_x2.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f32_x4.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f64_x2.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amax_f64_x4.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f16_x2.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f16_x4.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f32_x2.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f32_x4.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f64_x2.c: New test.
+	* gcc.target/aarch64/sme2/acle-asm/amin_f64_x4.c: New test.
+
+2025-07-14  panciyan  <panciyan@eswincomputing.com>
+
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_arith.h: Unsigned vector SAT_SUB form11 form12.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_data.h: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u16.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u32.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u64.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u8.c: Use ussub instead of usub.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u64.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u8.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u64.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u8.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u64.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u8.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u16.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u32.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u64.c: New test.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u8.c: New test.
+
+2025-07-12  Xi Ruoyao  <xry111@xry111.site>
+
+	PR rtl-optimization/87600
+	PR rtl-optimization/120983
+	* gcc.dg/pr87600.h [__loongarch__]: Define REG0 and REG1.
+	* gcc.dg/pr87600-1.c (dg-do): Add loongarch.
+	* gcc.dg/pr87600-2.c (dg-do): Likewise.
+
+2025-07-12  Tobias Burnus  <tburnus@baylibre.com>
+
+	* gfortran.dg/goacc/parameter.f95: Add -Wsurprising flag
+	and update expected diagnostic.
+	* gfortran.dg/goacc/parameter-3.f90: New test.
+	* gfortran.dg/goacc/parameter-4.f90: New test.
+
+2025-07-11  David Malcolm  <dmalcolm@redhat.com>
+
+	* gcc.dg/analyzer/state-diagram-1-sarif.py (test_xml_state):
+	Rename to...
+	(test_state_graph): ...this.  Port from XML to SARIF graphs.
+	* gcc.dg/analyzer/state-diagram-1.c: Update sink option
+	from "sarif:xml-state=yes" to "sarif:state-graphs=yes".
+	* gcc.dg/analyzer/state-diagram-5-sarif.c: Likewise.
+	* gcc.dg/analyzer/state-diagram-5-sarif.py: Drop import of ET.
+	(test_nested_types_in_xml_state): Rename to...
+	(test_nested_types_in_state_graph): ...this.  Port from XML to
+	SARIF graphs.
+	* gcc.dg/plugin/diagnostic-test-graphs-html.c: New test.
+	* gcc.dg/plugin/diagnostic-test-graphs-html.py: New test script.
+	* gcc.dg/plugin/diagnostic-test-graphs-sarif.c: New test.
+	* gcc.dg/plugin/diagnostic-test-graphs-sarif.py: New test script.
+	* gcc.dg/plugin/diagnostic-test-graphs.c: New test.
+	* gcc.dg/plugin/diagnostic_plugin_test_graphs.cc: New test plugin.
+	* gcc.dg/plugin/plugin.exp (plugin_test_list): Add the above.
+	* lib/sarif.py (get_xml_state): Delete.
+	(get_state_graph): New.
+	(def get_state_node_attr): New.
+	(get_state_node_kind): New.
+	(get_state_node_name): New.
+	(get_state_node_type): New.
+	(get_state_node_value): New.
+	* sarif-replay.dg/2.1.0-invalid/3.40.2-duplicate-node-id.sarif:
+	New test.
+	* sarif-replay.dg/2.1.0-invalid/3.41.4-unrecognized-node-id.sarif:
+	New test.
+	* sarif-replay.dg/2.1.0-valid/graphs-check-html.py: New test
+	script.
+	* sarif-replay.dg/2.1.0-valid/graphs-check-sarif-roundtrip.py: New
+	test script.
+	* sarif-replay.dg/2.1.0-valid/graphs.sarif: New test.
+
+2025-07-11  Vladimir N. Makarov  <vmakarov@redhat.com>
+
+	* gcc.target/powerpc/pr121007.c: New.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/119064
+	* g++.dg/cpp26/feat-cxx26.C: Add test for
+	__cpp_trivial_relocatability.
+	* g++.dg/cpp26/trivially-relocatable1.C: New test.
+	* g++.dg/cpp26/trivially-relocatable2.C: New test.
+	* g++.dg/cpp26/trivially-relocatable3.C: New test.
+	* g++.dg/cpp26/trivially-relocatable4.C: New test.
+	* g++.dg/cpp26/trivially-relocatable5.C: New test.
+	* g++.dg/cpp26/trivially-relocatable6.C: New test.
+	* g++.dg/cpp26/trivially-relocatable7.C: New test.
+	* g++.dg/cpp26/trivially-relocatable8.C: New test.
+	* g++.dg/cpp26/trivially-relocatable9.C: New test.
+	* g++.dg/cpp26/trivially-relocatable10.C: New test.
+	* g++.dg/cpp26/trivially-relocatable11.C: New test.
+
+2025-07-11  Richard Sandiford  <richard.sandiford@arm.com>
+
+	PR target/121027
+	* gcc.target/aarch64/sve/acle/general/perm_1.c: New test.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/simd/eor3_d.c: Add tests for DImode operands.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/simd/bcax_d.c: Add tests for DImode arguments.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/simd/eor3_d.c: New test.
+
+2025-07-11  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/simd/bcax_d.c: New test.
+
+2025-07-11  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/121034
+	* gcc.dg/vect/pr121034.c: New testcase.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120954
+	* c-c++-common/Warray-bounds-11.c: New test.
+
+2025-07-11  Jan Hubicka  <hubicka@ucw.cz>
+
+	* g++.dg/lto/pr114790_0.C: New test.
+	* g++.dg/lto/pr114790_1.C: New test.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+	    Martin Jambor  <mjambor@suse.cz>
+
+	PR ipa/121023
+	* c-c++-common/musttail32.c: New test.
+
+2025-07-11  Hu, Lin1  <lin1.hu@intel.com>
+
+	PR target/91384
+	* gcc.target/i386/pr91384-1.c: New test.
+
+2025-07-11  Paul Thomas  <pault@gcc.gnu.org>
+
+	PR fortran/106135
+	* gfortran.dg/import3.f90: Use -std=f2008 and comment on change
+	in error message texts with f2018.
+	* gfortran.dg/import12.f90: New test.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120628
+	* g++.dg/cpp0x/final1.C: New test.
+	* g++.dg/cpp0x/final2.C: New test.
+	* g++.dg/cpp0x/override6.C: New test.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120569
+	* g++.dg/cpp0x/override2.C: Expect different diagnostics with
+	override or duplicate final.
+	* g++.dg/cpp0x/override5.C: New test.
+	* g++.dg/cpp0x/duplicate1.C: Expect different diagnostics with
+	duplicate final.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/117785
+	* g++.dg/cpp0x/constexpr-ellipsis2.C: Expect different diagnostics for
+	C++26.
+	* g++.dg/cpp0x/constexpr-throw.C: Likewise.
+	* g++.dg/cpp1y/constexpr-84192.C: Expect different diagnostics.
+	* g++.dg/cpp1y/constexpr-throw.C: Expect different diagnostics for
+	C++26.
+	* g++.dg/cpp1z/constexpr-asm-5.C: Likewise.
+	* g++.dg/cpp26/constexpr-eh1.C: New test.
+	* g++.dg/cpp26/constexpr-eh2.C: New test.
+	* g++.dg/cpp26/constexpr-eh3.C: New test.
+	* g++.dg/cpp26/constexpr-eh4.C: New test.
+	* g++.dg/cpp26/constexpr-eh5.C: New test.
+	* g++.dg/cpp26/constexpr-eh6.C: New test.
+	* g++.dg/cpp26/constexpr-eh7.C: New test.
+	* g++.dg/cpp26/constexpr-eh8.C: New test.
+	* g++.dg/cpp26/constexpr-eh9.C: New test.
+	* g++.dg/cpp26/constexpr-eh10.C: New test.
+	* g++.dg/cpp26/constexpr-eh11.C: New test.
+	* g++.dg/cpp26/constexpr-eh12.C: New test.
+	* g++.dg/cpp26/constexpr-eh13.C: New test.
+	* g++.dg/cpp26/constexpr-eh14.C: New test.
+	* g++.dg/cpp26/constexpr-eh15.C: New test.
+	* g++.dg/cpp26/feat-cxx26.C: Change formatting in __cpp_pack_indexing
+	and __cpp_pp_embed test.  Add __cpp_constexpr_exceptions test.
+	* g++.dg/cpp26/static_assert1.C: Expect different diagnostics for
+	C++26.
+	* g++.dg/cpp2a/consteval34.C: Likewise.
+	* g++.dg/cpp2a/consteval-memfn1.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic4.C: For C++26 add std::exception and
+	std::bad_cast definitions and expect different diagnostics.
+	* g++.dg/cpp2a/constexpr-dynamic6.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic7.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic8.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic9.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic11.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic14.C: Likewise.
+	* g++.dg/cpp2a/constexpr-dynamic18.C: Likewise.
+	* g++.dg/cpp2a/constexpr-new27.C: New test.
+	* g++.dg/cpp2a/constexpr-typeid5.C: New test.
+
+2025-07-10  Qing Zhao  <qing.zhao@oracle.com>
+
+	PR middle-end/121000
+	* gcc.dg/flex-array-counted-by-pr121000.c: New test.
+
+2025-07-10  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* gfortran.dg/asan/array_constructor_1.f90: Allocate array
+	before using it.
+
+2025-07-10  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gcc.dg/tree-prof/afdo-inline.c: Add user symbol names.
+
+2025-07-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/sve/pfalse-binary.c: Add -funwind-tables.
+	* gcc.target/aarch64/sve/pfalse-binary_int_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-binary_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-binary_opt_single_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-binary_rotate.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-binary_uint64_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-binary_uint_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-binaryxn.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-clast.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-compare_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-compare_wide_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-count_pred.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-fold_left.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load_ext.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load_ext_gather_index.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load_ext_gather_offset.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load_gather_sv.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load_gather_vs.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-load_replicate.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-prefetch.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-prefetch_gather_index.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-prefetch_gather_offset.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-ptest.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-rdffr.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-reduction.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-reduction_wide.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-shift_right_imm.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-store.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-store_scatter_index.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-store_scatter_offset.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-storexn.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-ternary_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-ternary_rotate.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unary.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unary_convert_narrowt.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unary_convertxn.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unary_n.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unary_pred.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unary_to_uint.c: Likewise.
+	* gcc.target/aarch64/sve/pfalse-unaryxn.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_int_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_int_opt_single_n.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_opt_single_n.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_to_uint.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_uint_opt_n.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-binary_wide.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-compare.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-load_ext_gather_index_restricted.c,
+	* gcc.target/aarch64/sve2/pfalse-load_ext_gather_offset_restricted.c,
+	* gcc.target/aarch64/sve2/pfalse-load_gather_sv_restricted.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-load_gather_vs.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-shift_left_imm_to_uint.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-shift_right_imm.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-store_scatter_index_restricted.c,
+	* gcc.target/aarch64/sve2/pfalse-store_scatter_offset_restricted.c,
+	* gcc.target/aarch64/sve2/pfalse-unary.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-unary_convert.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-unary_convert_narrowt.c: Likewise.
+	* gcc.target/aarch64/sve2/pfalse-unary_to_int.c: Likewise.
+
+2025-07-10  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	PR target/120999
+	* gcc.target/aarch64/sve2/pr120999.c: New test.
+
+2025-07-10  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* lib/target-supports.exp (check_effective_target_aarch64_sve2p1_hw):
+	New proc.
+	* gcc.target/aarch64/sve2/dupq_1.c: Extend to big-endian.  Add
+	noipa attributes.
+	* gcc.target/aarch64/sve2/extq_1.c: Likewise.
+	* gcc.target/aarch64/sve2/uzpq_1.c: Likewise.
+	* gcc.target/aarch64/sve2/zipq_1.c: Likewise.
+	* gcc.target/aarch64/sve2/dupq_1_run.c: New test.
+	* gcc.target/aarch64/sve2/extq_1_run.c: Likewise.
+	* gcc.target/aarch64/sve2/uzpq_1_run.c: Likewise.
+	* gcc.target/aarch64/sve2/zipq_1_run.c: Likewise.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	* gfortran.dg/g77/980310-3.f: Comment spelling fix: bellow -> below.
+	* jit.dg/test-debuginfo.c: Likewise.
+
+2025-07-10  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/sat/sat_s_add-run-1-i16.c: Take rv32 || rv64
+	instead of riscv_v for scalar run test.
+	* gcc.target/riscv/sat/sat_s_add-run-1-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-1-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-1-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-2-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-2-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-2-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-2-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-3-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-3-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-3-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-3-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-4-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-4-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-4-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_add-run-4-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-1-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-1-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-1-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-1-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-2-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-2-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-2-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-2-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-3-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-3-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-3-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-3-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-4-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-4-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-4-i64.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_sub-run-4-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-1-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-2-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-3-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-4-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-5-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-6-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-7-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-8-i16-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i16.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i32.c: Ditto.
+	* gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-5-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-5-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-5-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-5-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-6-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-6-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-6-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-6-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-7-u32-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_add_imm-run-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-10-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-10-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-10-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-10-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-11-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-11-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-11-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-11-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-12-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-12-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-12-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-12-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-5-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-5-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-5-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-5-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-6-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-6-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-6-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-6-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-7-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-7-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-7-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-7-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-8-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-8-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-8-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-8-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-9-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-9-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-9-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub-run-9-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_sub_imm-run-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-1-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-1-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-1-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-1-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-2-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-2-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-2-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-2-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-3-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-3-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-3-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-3-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-4-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-4-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-4-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-4-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-5-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-5-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-5-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-5-u8.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-6-u16.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-6-u32.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-6-u64.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_trunc-run-6-u8.c: Ditto.
+
+2025-07-09  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120243
+	* g++.dg/coroutines/torture/pr120243-unhandled-1.C: New test.
+	* g++.dg/coroutines/torture/pr120243-unhandled-2.C: New test.
+
+2025-07-09  Jason Merrill  <jason@redhat.com>
+
+	PR c++/121012
+	PR c++/120917
+	* g++.dg/cpp2a/lambda-targ17.C: New test.
+
+2025-07-09  Jason Merrill  <jason@redhat.com>
+
+	PR c++/121008
+	PR c++/113563
+	* g++.dg/cpp2a/lambda-uneval28.C: New test.
+
+2025-07-09  Marek Polacek  <polacek@redhat.com>
+
+	PR c++/119838
+	* g++.dg/parse/template32.C: New test.
+
+2025-07-09  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.dg/rtl/aarch64/vec-series-1.c: New test.
+	* gcc.dg/rtl/aarch64/vec-series-2.c: Likewise.
+	* gcc.target/aarch64/sve/acle/general/dupq_2.c: Fix expected
+	output for this big-endian test.
+	* gcc.target/aarch64/sve/acle/general/dupq_4.c: Likewise.
+	* gcc.target/aarch64/sve/vec_init_3.c: Restrict to little-endian
+	targets and add more tests.
+	* gcc.target/aarch64/sve/vec_init_4.c: New big-endian version
+	of vec_init_3.c.
+
+2025-07-09  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c: Ditto.
+
+2025-07-09  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h: Add test
+	helper macros.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h: Add test
+	data for run test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i8.c: New test.
+
+2025-07-09  Paul-Antoine Arras  <parras@baylibre.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_run.h: Set
+	MAX_RELATIVE_DIFF depending on type.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c: Enable zvfh.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c: Likewise.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c: Likewise.
+
+2025-07-09  Ciyan Pan  <panciyan@eswincomputing.com>
+
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_arith.h: Add vec_sat_u_sub_fmt wrap define.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_data.h: Add vec_sat_u_sub test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u8.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u16.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u32.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u64.c: Remove test data.
+	* gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u8.c: Remove test data.
+
+2025-07-09  Richard Sandiford  <richard.sandiford@arm.com>
+
+	* gcc.target/aarch64/pr118348_1.c: Require fstack_protector.
+	* gcc.target/aarch64/pr118348_2.c: Likewise.
+
+2025-07-09  Icen Zeyada  <Icen.Zeyada2@arm.com>
+
+	* gcc.target/aarch64/vector-compare-5.c: Add new test for vector compare simplification.
+
+2025-07-09  Jeff Law  <jlaw@ventanamicro.com>
+
+	PR target/120642
+	* gcc.target/riscv/rvv/xtheadvector/pr120642.c: New test.
+
+2025-07-09  Richard Biener  <rguenther@suse.de>
+
+	PR testsuite/120093
+	* gcc.dg/vect/pr101145.c: Use noipa instead of noinline
+	attribute.
+
+2025-07-09  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* gcc.target/s390/vector/pattern-avg-1.c: Fix on -m31.
+	* gcc.target/s390/vector/pattern-mulh-1.c: Fix on -m31.
+	* gcc.target/s390/vector/pattern-mulh-2.c: Fix on -m31.
+
+2025-07-09  Thomas Schwinge  <tschwinge@baylibre.com>
+
+	* gcc.dg/builtin-dynamic-object-size-pr120780.c: Fix 'main' function.
+
+2025-07-09  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/120922
+	* gcc.dg/vect/pr120922.c: New test.
+
+2025-07-09  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/sat/sat_arith.h: Add xlen check for
+	uint128_t.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u128.c: Enable
+	run test for rv64 only.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u128.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u64-from-u128.c: Ditto.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u128.c: Ditto.
+
+2025-07-08  Marek Polacek  <polacek@redhat.com>
+	    Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR c++/83469
+	PR c++/93809
+	* g++.dg/template/error45.C: Adjust dg-error.
+	* g++.dg/warn/Wredundant-tags-3.C: Remove xfail.
+	* g++.dg/parse/union1.C: New test.
+	* g++.dg/parse/union2.C: New test.
+	* g++.dg/parse/union3.C: New test.
+	* g++.dg/parse/union4.C: New test.
+	* g++.dg/parse/union5.C: New test.
+	* g++.dg/parse/union6.C: New test.
+
+2025-07-08  Takayuki 'January June' Suwa  <jjsuwa_sys3175@yahoo.co.jp>
+
+	* gcc.target/xtensa/BGEUI-BLTUI-32k-64k.c: New.
+
+2025-07-08  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/117784
+	* g++.dg/cpp1z/decomp3.C (test): For constexpr structured binding
+	initialize from constexpr var instead of non-constexpr and expect
+	just a pedwarn for C++23 and older instead of error always.
+	* g++.dg/cpp26/decomp9.C (foo): Likewise.
+	* g++.dg/cpp26/decomp22.C: New test.
+	* g++.dg/cpp26/decomp23.C: New test.
+	* g++.dg/cpp26/decomp24.C: New test.
+	* g++.dg/cpp26/decomp25.C: New test.
+
+2025-07-08  Stefan Schulze Frielinghaus  <stefansf@gcc.gnu.org>
+
+	* gcc.target/s390/stack-protector-guard-tls-1.c: New test.
+
+2025-07-08  Richard Biener  <rguenther@suse.de>
+
+	* gcc.dg/guality/guality.h (guality_main): Declare noipa.
+	(guality_check): Likewise.
+
+2025-07-08  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/120461
+	* gcc.target/riscv/rvv/xtheadvector/pr120461.c: New test.
+
+2025-07-08  Robin Dapp  <rdapp@ventanamicro.com>
+
+	PR target/113829
+	* gcc.target/riscv/rvv/base/pr113829.c: New test.
+
+2025-07-08  Andreas Schwab  <schwab@suse.de>
+
+	PR target/120995
+	* gcc.target/riscv/amo/zabha-zacas-atomic-cas.c: New test.
+
+2025-07-08  Jason Merrill  <jason@redhat.com>
+
+	* g++.dg/cpp1y/lambda-generic-variadic.C: Change to 'compile'.
+
+2025-07-08  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* gcc.target/i386/memcpy-pr120683-1.c (dg-options): Add
+	-fasynchronous-unwind-tables -fdwarf2-cfi-asm.
+	* gcc.target/i386/memcpy-pr120683-2.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-3.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-4.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-5.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-6.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-7.c: Likewise.
+	* gcc.target/i386/memcpy-strategy-12.c: Likewise.
+	* gcc.target/i386/memset-pr120683-1.c: Likewise.
+	* gcc.target/i386/memset-pr120683-10.c: Likewise.
+	* gcc.target/i386/memset-pr120683-11.c: Likewise.
+	* gcc.target/i386/memset-pr120683-12.c: Likewise.
+	* gcc.target/i386/memset-pr120683-13.c: Likewise.
+	* gcc.target/i386/memset-pr120683-14.c: Likewise.
+	* gcc.target/i386/memset-pr120683-15.c: Likewise.
+	* gcc.target/i386/memset-pr120683-16.c: Likewise.
+	* gcc.target/i386/memset-pr120683-17.c: Likewise.
+	* gcc.target/i386/memset-pr120683-18.c: Likewise.
+	* gcc.target/i386/memset-pr120683-19.c: Likewise.
+	* gcc.target/i386/memset-pr120683-2.c: Likewise.
+	* gcc.target/i386/memset-pr120683-20.c: Likewise.
+	* gcc.target/i386/memset-pr120683-21.c: Likewise.
+	* gcc.target/i386/memset-pr120683-22.c: Likewise.
+	* gcc.target/i386/memset-pr120683-23.c: Likewise.
+	* gcc.target/i386/memset-pr120683-3.c: Likewise.
+	* gcc.target/i386/memset-pr120683-4.c: Likewise.
+	* gcc.target/i386/memset-pr120683-5.c: Likewise.
+	* gcc.target/i386/memset-pr120683-6.c: Likewise.
+	* gcc.target/i386/memset-pr120683-7.c: Likewise.
+	* gcc.target/i386/memset-pr120683-8.c: Likewise.
+	* gcc.target/i386/memset-pr120683-9.c: Likewise.
+
+2025-07-08  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* gcc.target/s390/vector/pattern-avg-1.c: Split test.
+	* gcc.target/s390/vector/pattern-mulh-1.c: Split test.
+	* gcc.target/s390/vector/pattern-avg-2.c: New test.
+	* gcc.target/s390/vector/pattern-mulh-2.c: New test.
+
+2025-07-08  Richard Biener  <rguenther@suse.de>
+
+	* gcc.target/i386/vect-mask-epilogue-1.c: New testcase.
+	* gcc.target/i386/vect-mask-epilogue-2.c: Likewise.
+	* gcc.target/i386/vect-epilogues-3.c: Adjust.
+
+2025-07-08  Andre Vehreschild  <vehre@gcc.gnu.org>
+
+	PR fortran/120637
+	* gfortran.dg/asan/finalize_1.f90: New test.
+
+2025-07-08  Jeff Law  <jlaw@ventanamicro.com>
+
+	* gcc.dg/torture/pr120654.c: Use __builtin variants of malloc and free.
+
+2025-07-08  Jeff Law  <jlaw@ventanamicro.com>
+
+	* gcc.target/riscv/amo/zalrsc-rvwmo-amo-add-int.c: Adjust expected
+	output.
+	* gcc.target/riscv/amo/zalrsc-ztso-amo-add-int.c: Likewise.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	* gcc.dg/flex-array-counted-by.c: Update test.
+	* gcc.dg/pointer-counted-by-1.c: New test.
+	* gcc.dg/pointer-counted-by-2.c: New test.
+	* gcc.dg/pointer-counted-by-3.c: New test.
+	* gcc.dg/pointer-counted-by.c: New test.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	* gcc.dg/pointer-counted-by-4-char.c: New test.
+	* gcc.dg/pointer-counted-by-4-float.c: New test.
+	* gcc.dg/pointer-counted-by-4-struct.c: New test.
+	* gcc.dg/pointer-counted-by-4-union.c: New test.
+	* gcc.dg/pointer-counted-by-4.c: New test.
+	* gcc.dg/pointer-counted-by-5.c: New test.
+	* gcc.dg/pointer-counted-by-6.c: New test.
+	* gcc.dg/pointer-counted-by-7.c: New test.
+
+2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	Revert:
+	2025-07-07  Qing Zhao  <qing.zhao@oracle.com>
+
+	* gcc.dg/ubsan/pointer-counted-by-bounds-2.c: New test.
+	* gcc.dg/ubsan/pointer-counted-by-bounds-3.c: New test.
+	* gcc.dg/ubsan/pointer-counted-by-bounds-4.c: New test.
+	* gcc.dg/ubsan/pointer-counted-by-bounds-5.c: New test.
+	* gcc.dg/ubsan/pointer-counted-by-bounds.c: New test.
+
+2025-07-07  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR testsuite/120881
+	* lib/scanasm.exp (check-function-bodies): Allow "^[0-9]+:".
+
+2025-07-07  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120888
+	* gcc.target/xtensa/pr120888-1.c: New test.
+	* gcc.target/xtensa/pr120888-2.c: Likewise.
+
+2025-07-07  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* gcc.target/s390/fminmax-1.c: New test.
+	* gcc.target/s390/fminmax-2.c: New test.
+
+2025-07-07  Tamar Christina  <tamar.christina@arm.com>
+
+	PR tree-optimization/120817
+	* gcc.dg/vect/pr120817.c: Add SVE HW check.
+
+2025-07-07  Alfie Richards  <alfie.richards@arm.com>
+
+	PR c++/119498
+	* g++.target/aarch64/pr119498.C: New test.
+
+2025-07-07  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120917
+	* g++.dg/concepts/auto7a.C: Add diagnostic.
+	* g++.dg/concepts/auto7b.C: New test.
+	* g++.dg/concepts/auto7c.C: New test.
+	* g++.dg/cpp1y/pr85076.C: Expect 'auto' error.
+	* g++.dg/concepts/pr67249.C: Likewise.
+	* g++.dg/cpp1y/lambda-generic-variadic.C: Likewise.
+	* g++.dg/cpp2a/concepts-pr67210.C: Likewise.
+	* g++.dg/concepts/pr67249a.C: New test.
+	* g++.dg/cpp1y/lambda-generic-variadic-a.C: New test.
+	* g++.dg/cpp2a/concepts-pr67210a.C: New test.
+
+2025-07-07  Kyrylo Tkachov  <ktkachov@nvidia.com>
+
+	* gcc.target/aarch64/popcnt9.c: Add +nosve to target pragma.
+	* gcc.target/aarch64/popcnt13.c: New test.
+
+2025-07-07  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120817
+	* gcc.dg/vect/pr120817.c: New testcase.
+
+2025-07-07  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/sat/sat_arith.h: Add test helper macros.
+	* gcc.target/riscv/sat/sat_arith_data.h: Add test data for
+	run test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u16-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u32-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u64-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-1-u8-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u64-from-u128.c: New test.
+	* gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u128.c: New test.
+
+2025-07-07  Eric Botcazou  <ebotcazou@adacore.com>
+
+	* ada/acats-3/tests/c9/c94001c.ada: Tweak delay statements.
+	* ada/acats-4/tests/c9/c94001c.ada: Likewise.
+	* ada/acats-4/tests/c9/c94006a.ada: Likewise.
+	* ada/acats-4/tests/c9/c94008c.ada: Likewise.
+	* ada/acats-4/tests/c9/c951002.a: Likewise.
+	* ada/acats-4/tests/c9/c954a01.a: Likewise.
+	* ada/acats-4/tests/c9/c940005.a: Tweak duration constants.
+	* ada/acats-4/tests/c9/c940007.a: Likewise.
+	* ada/acats-4/tests/c9/c96001a.ada: Likewise.
+
+2025-07-07  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* gcc.target/s390/vector/pattern-avg-1.c: New test.
+	* gcc.target/s390/vector/pattern-mulh-1.c: New test.
+
+2025-07-07  Spencer Abson  <spencer.abson@arm.com>
+
+	* gcc.target/aarch64/sve/unpacked_fcm_1.c: New test.
+	* gcc.target/aarch64/sve/unpacked_fcm_2.c: Likewise.
+
+2025-07-07  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120670
+	PR target/120683
+	* gcc.target/i386/auto-init-padding-9.c: Updated.
+	* gcc.target/i386/memcpy-strategy-12.c: Likewise.
+	* gcc.target/i386/memset-strategy-25.c: Likewise.
+	* gcc.target/i386/memset-strategy-29.c: Likewise.
+	* gcc.target/i386/memset-strategy-30.c: Likewise.
+	* gcc.target/i386/memset-strategy-31.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-1.c: New test.
+	* gcc.target/i386/memcpy-pr120683-2.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-3.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-4.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-5.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-6.c: Likewise.
+	* gcc.target/i386/memcpy-pr120683-7.c: Likewise.
+	* gcc.target/i386/memset-pr120683-1.c: Likewise.
+	* gcc.target/i386/memset-pr120683-2.c: Likewise.
+	* gcc.target/i386/memset-pr120683-3.c: Likewise.
+	* gcc.target/i386/memset-pr120683-4.c: Likewise.
+	* gcc.target/i386/memset-pr120683-5.c: Likewise.
+	* gcc.target/i386/memset-pr120683-6.c: Likewise.
+	* gcc.target/i386/memset-pr120683-7.c: Likewise.
+	* gcc.target/i386/memset-pr120683-8.c: Likewise.
+	* gcc.target/i386/memset-pr120683-9.c: Likewise.
+	* gcc.target/i386/memset-pr120683-10.c: Likewise.
+	* gcc.target/i386/memset-pr120683-11.c: Likewise.
+	* gcc.target/i386/memset-pr120683-12.c: Likewise.
+	* gcc.target/i386/memset-pr120683-13.c: Likewise.
+	* gcc.target/i386/memset-pr120683-14.c: Likewise.
+	* gcc.target/i386/memset-pr120683-15.c: Likewise.
+	* gcc.target/i386/memset-pr120683-16.c: Likewise.
+	* gcc.target/i386/memset-pr120683-17.c: Likewise.
+	* gcc.target/i386/memset-pr120683-18.c: Likewise.
+	* gcc.target/i386/memset-pr120683-19.c: Likewise.
+	* gcc.target/i386/memset-pr120683-20.c: Likewise.
+	* gcc.target/i386/memset-pr120683-21.c: Likewise.
+	* gcc.target/i386/memset-pr120683-22.c: Likewise.
+	* gcc.target/i386/memset-pr120683-23.c: Likewise.
+
+2025-07-07  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/84009
+	* g++.dg/cpp0x/range-for40.C: New test.
+	* g++.dg/cpp0x/range-for41.C: New test.
+	* g++.dg/cpp0x/range-for42.C: New test.
+	* g++.dg/cpp0x/range-for43.C: New test.
+
+2025-07-07  Mikael Morin  <mikael@gcc.gnu.org>
+
+	* gfortran.dg/move_alloc_20.f03: New test.
+
+2025-07-07  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR middle-end/120709
+	* gcc.dg/crc-non-cst-poly-1.c: New test.
+
+2025-07-06  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR tree-optimization/120951
+	* gcc.dg/torture/pr120951-1.c: New test.
+
+2025-07-06  Jan Hubicka  <hubicka@ucw.cz>
+
+	* gcc.dg/tree-prof/clone-merge-1.c:
+
+2025-07-04  Vineet Gupta  <vineetg@rivosinc.com>
+
+	PR target/118241
+	* gcc.target/riscv/pr118241-b.cc: New test.
+
+2025-07-04  Raphael Moreira Zinsly  <rzinsly@ventanamicro.com>
+
+	* gcc.target/sh/pr54236-2.c: Fix comments and expected output
+
+2025-07-04  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	PR c/118948
+	* gcc.dg/pr118948-1.c: New test.
+
+2025-07-04  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120575
+	PR c++/116064
+	* g++.dg/template/permissive-error3.C: New test.
+
+2025-07-04  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c: Ditto.
+
+2025-07-04  Pan Li  <pan2.li@intel.com>
+
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c: Add asm check.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c: Ditto.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h: Add test
+	helper macros.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h: Add test
+	data for run test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i16.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i32.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i64.c: New test.
+	* gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i8.c: New test.
+
+2025-07-04  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120944
+	* gcc.dg/torture/pr120944.c: New testcase.
+
+2025-07-04  Richard Biener  <rguenther@suse.de>
+
+	PR tree-optimization/120927
+	* gcc.dg/vect/vect-pr120927.c: New testcase.
+	* gcc.dg/vect/vect-pr120927-2.c: Likewise.
+
+2025-07-04  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c/120837
+	* gcc.dg/ubsan/pr120837.c: New test.
+
+2025-07-04  Xi Ruoyao  <xry111@xry111.site>
+
+	PR target/120807
+	* gcc.c-torture/compile/pr120708.c: Rename to ...
+	* gcc.c-torture/compile/pr120807.c: ... here.
+
+2025-07-04  Xi Ruoyao  <xry111@xry111.site>
+
+	* gcc.c-torture/compile/pr120708.c: New test.
+
+2025-07-04  panciyan  <panciyan@eswincomputing.com>
+
+	* gcc.target/riscv/sat/sat_arith.h: Add signed scalar SAT_ADD IMM form2.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i16.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i32.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i64.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-2-i8.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-run-2-i16.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-run-2-i32.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-run-2-i64.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm-run-2-i8.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i16.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i32.c: New test.
+	* gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i8.c: New test.
+
+2025-07-03  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120716
+	* g++.dg/cpp0x/lambda/lambda-constexpr3.C: New test.
+	* g++.dg/cpp0x/lambda/lambda-constexpr3a.C: New test.
+
+2025-07-03  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120748
+	* g++.dg/cpp2a/lambda-targ16.C: New test.
+	* g++.dg/cpp0x/this1.C: Adjust diagnostics.
+
+2025-07-03  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120940
+	* g++.dg/parse/pr120940.C: New test.
+	* g++.dg/warn/Wduplicated-branches9.C: New test.
+
+2025-07-03  Juergen Christ  <jchrist@linux.ibm.com>
+
+	* gcc.target/s390/vector/vec-perm-merge-1.c: New test.
+	* gcc.target/s390/vector/vec-perm-pack-1.c: New test.
+
+2025-07-03  Thomas Schwinge  <tschwinge@baylibre.com>
+
+	* c-c++-common/gomp/omp_get_num_devices_initial_device.c: Fix.
+	* gfortran.dg/gomp/omp_get_num_devices_initial_device.f90: Likewise.
+
+2025-07-03  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>
+
+	* gcc.dg/ipa/pr120295.c (glob): Rename to glob_.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* gcc.target/aarch64/cmpbr.c: Update tests.
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* gcc.target/aarch64/cmpbr.c:
+
+2025-07-03  Karl Meakin  <karl.meakin@arm.com>
+
+	* lib/target-supports.exp: Add `cmpbr` to the list of extensions.
+	* gcc.target/aarch64/cmpbr.c: New test.
+
+2025-07-03  Siddhesh Poyarekar  <siddhesh@gotplt.org>
+
+	PR tree-optimization/120780
+	* gcc.dg/builtin-dynamic-object-size-pr120780.c: New test case.
+
+2025-07-03  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120936
+	* gcc.target/i386/pr120936-1.c: New test
+	* gcc.target/i386/pr120936-2.c: Likewise.
+	* gcc.target/i386/pr120936-3.c: Likewise.
+	* gcc.target/i386/pr120936-4.c: Likewise.
+	* gcc.target/i386/pr120936-5.c: Likewise.
+	* gcc.target/i386/pr120936-6.c: Likewise.
+	* gcc.target/i386/pr120936-7.c: Likewise.
+	* gcc.target/i386/pr120936-8.c: Likewise.
+	* gcc.target/i386/pr120936-9.c: Likewise.
+	* gcc.target/i386/pr120936-10.c: Likewise.
+	* gcc.target/i386/pr120936-11.c: Likewise.
+	* gcc.target/i386/pr120936-12.c: Likewise.
+	* gcc.target/i386/pr93492-3.c: Updated.
+	* gcc.target/i386/pr93492-5.c: Likewise.
+
+2025-07-03  Andre Vehreschild  <vehre@gcc.gnu.org>
+
+	PR fortran/120843
+	* gfortran.dg/coarray/coindexed_6.f90: Enhance test to have
+	coarray components covered.
+
+2025-07-03  H.J. Lu  <hjl.tools@gmail.com>
+
+	PR target/120908
+	* gcc.target/i386/pr120908.c: New test.
+
+2025-07-03  Jason Merrill  <jason@redhat.com>
+
+	PR c++/120684
+	PR c++/118856
+	* g++.dg/cpp23/range-for10.C: New test.
+
 2025-07-02  Dimitar Dimitrov  <dimitar@dinux.eu>
 
 	* gcc.target/riscv/mcpu-xt-c908.c: Disable for E ABI variants.
diff --git a/gcc/testsuite/ada/acats-3/tests/c9/c94001c.ada b/gcc/testsuite/ada/acats-3/tests/c9/c94001c.ada
index 0cc14f4..df38f99 100644
--- a/gcc/testsuite/ada/acats-3/tests/c9/c94001c.ada
+++ b/gcc/testsuite/ada/acats-3/tests/c9/c94001c.ada
@@ -211,7 +211,7 @@ BEGIN
 
      BEGIN -- (E)
           WHILE NOT(OUT_TSK'TERMINATED) AND DELAY_COUNT < 60 LOOP
-               DELAY 1.0;
+               DELAY 1.0 * Impdef.One_Long_Second;
                DELAY_COUNT := DELAY_COUNT + 1;
           END LOOP;
           IF DELAY_COUNT = 60 THEN
@@ -254,7 +254,7 @@ BEGIN
 
      BEGIN
           WHILE NOT(OUT_TSK'TERMINATED) AND DELAY_COUNT < 60 LOOP
-               DELAY 1.0;
+               DELAY 1.0 * Impdef.One_Long_Second;
                DELAY_COUNT := DELAY_COUNT + 1;
           END LOOP;
           IF DELAY_COUNT = 60 THEN
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c940005.a b/gcc/testsuite/ada/acats-4/tests/c9/c940005.a
index adb58b1..47a97bf 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c940005.a
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c940005.a
@@ -85,7 +85,7 @@ begin
       -- In reality one would expect a time of 5 to 10 seconds.  In
       -- the interests of speeding up the test suite a shorter time
       -- is used
-      Pulse_Time_Delta : constant duration := ImpDef.Switch_To_New_Task;
+      Pulse_Time_Delta : constant duration := ImpDef.Long_Switch_To_New_Task;
 
       -- control over stopping tasks
       protected Control is
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c940007.a b/gcc/testsuite/ada/acats-4/tests/c9/c940007.a
index c678463..41e80f4 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c940007.a
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c940007.a
@@ -90,7 +90,7 @@ begin
       -- In reality one would expect a time of 5 to 10 seconds.  In
       -- the interests of speeding up the test suite a shorter time
       -- is used
-      Pulse_Time_Delta : constant duration := ImpDef.Switch_To_New_Task;
+      Pulse_Time_Delta : constant duration := ImpDef.Long_Switch_To_New_Task;
 
 
       -- control over stopping tasks
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c94001c.ada b/gcc/testsuite/ada/acats-4/tests/c9/c94001c.ada
index 0cc14f4..df38f99 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c94001c.ada
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c94001c.ada
@@ -211,7 +211,7 @@ BEGIN
 
      BEGIN -- (E)
           WHILE NOT(OUT_TSK'TERMINATED) AND DELAY_COUNT < 60 LOOP
-               DELAY 1.0;
+               DELAY 1.0 * Impdef.One_Long_Second;
                DELAY_COUNT := DELAY_COUNT + 1;
           END LOOP;
           IF DELAY_COUNT = 60 THEN
@@ -254,7 +254,7 @@ BEGIN
 
      BEGIN
           WHILE NOT(OUT_TSK'TERMINATED) AND DELAY_COUNT < 60 LOOP
-               DELAY 1.0;
+               DELAY 1.0 * Impdef.One_Long_Second;
                DELAY_COUNT := DELAY_COUNT + 1;
           END LOOP;
           IF DELAY_COUNT = 60 THEN
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c94006a.ada b/gcc/testsuite/ada/acats-4/tests/c9/c94006a.ada
index 6b9c85f..cac5fc6 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c94006a.ada
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c94006a.ada
@@ -28,6 +28,7 @@
 -- TBN  9/17/86
 -- PWN 01/31/95  REMOVED PRAGMA PRIORITY FOR ADA 9X.
 
+with Impdef;
 WITH REPORT; USE REPORT;
 WITH SYSTEM; USE SYSTEM;
 PROCEDURE C94006A IS
@@ -41,7 +42,7 @@ PROCEDURE C94006A IS
           SELECT
                ACCEPT E;
           OR
-               DELAY 30.0;
+               DELAY 30.0 * Impdef.One_Long_Second;
           END SELECT;
      END TT;
 
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c94008c.ada b/gcc/testsuite/ada/acats-4/tests/c9/c94008c.ada
index 6d10e25..fb2eee9 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c94008c.ada
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c94008c.ada
@@ -33,6 +33,7 @@
 -- JBG 8/29/86 ELIMINATED SHARED VARIABLES; ADDED GENERIC UNIT
 -- PWN 11/30/94 REMOVED PRAGMA PRIORITY INSTANCES FOR ADA 9X.
 
+with Impdef;
 WITH REPORT; USE REPORT;
 WITH SYSTEM; USE SYSTEM;
 PROCEDURE C94008C IS
@@ -198,10 +199,10 @@ BEGIN -- C94008C
                     OR WHEN ENTER_TERMINATE => TERMINATE;
                     END SELECT;
 
-                    DELAY 10.0;
+                    DELAY 10.0 * Impdef.One_Second;
 
                     IF TERMINATE_COUNT.GET /= 1 THEN
-                         DELAY 20.0;
+                         DELAY 20.0 * Impdef.One_Long_Second;
                     END IF;
 
                     IF TERMINATE_COUNT.GET /= 1 THEN
@@ -239,10 +240,10 @@ BEGIN -- C94008C
 
      BEGIN
 
-          DELAY 10.0; -- WAIT FOR T1, T2, AND T3 TO GET TO SELECT STMTS.
+          DELAY 10.0 * Impdef.One_Second; -- WAIT FOR T1, T2, AND T3 TO GET TO SELECT STMTS.
 
            IF TERMINATE_COUNT.GET /= 3 THEN
-                DELAY 20.0;
+                DELAY 20.0 * Impdef.One_Long_Second;
            END IF;
 
            IF TERMINATE_COUNT.GET /= 3 THEN
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c951002.a b/gcc/testsuite/ada/acats-4/tests/c9/c951002.a
index 8ccb2d0..65b696c 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c951002.a
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c951002.a
@@ -278,14 +278,14 @@ begin
       -- Wait until the message is queued on the entry before starting
       -- the Credit_Task
       while not Hold.TC_Message_is_Queued loop
-         delay ImpDef.Minimum_Task_Switch;   
+         delay ImpDef.Long_Minimum_Task_Switch;   
       end loop;
       --
       Credit_Task.TC_Start;
 
       -- Ensure the first part of the test is complete before continuing
       while not (Credit_Message'terminated and Credit_Task'terminated) loop
-         delay ImpDef.Minimum_Task_Switch;   
+         delay ImpDef.Long_Minimum_Task_Switch;   
       end loop;
 
       --======================================================
@@ -298,12 +298,12 @@ begin
       -- for it to reach the accept statement and call Hold.Set_DB_Overload
       -- before starting Debit_Message
       --
-      delay ImpDef.Switch_To_New_Task;
+      delay ImpDef.Long_Switch_To_New_Task;
 
       Debit_Message.TC_Start;
 
       while not Debit_Task'terminated loop
-         delay ImpDef.Minimum_Task_Switch;   
+         delay ImpDef.Long_Minimum_Task_Switch;   
       end loop;    
   
       Hold.Clear_DB_Overload;  -- Allow completion 
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c954a01.a b/gcc/testsuite/ada/acats-4/tests/c9/c954a01.a
index 34f48b2..3ea545a 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c954a01.a
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c954a01.a
@@ -148,7 +148,7 @@ package body C954A01_0 is  -- Printer server abstraction.
          end select;
 
          -- Allow other tasks to get control
-         delay ImpDef.Minimum_Task_Switch;
+         delay ImpDef.Long_Minimum_Task_Switch;
 
       end loop;
 
@@ -175,7 +175,7 @@ use  F954A00;
 
 procedure C954A01 is
 
-   Long_Enough : constant Duration := ImpDef.Switch_To_New_Task;
+   Long_Enough : constant Duration := ImpDef.Long_Switch_To_New_Task;
 
                --==============================================--
 
diff --git a/gcc/testsuite/ada/acats-4/tests/c9/c96001a.ada b/gcc/testsuite/ada/acats-4/tests/c9/c96001a.ada
index 74374b9..f958ea1 100644
--- a/gcc/testsuite/ada/acats-4/tests/c9/c96001a.ada
+++ b/gcc/testsuite/ada/acats-4/tests/c9/c96001a.ada
@@ -36,6 +36,7 @@
 --     RJW 11/13/87  ADDED CODE WHICH ALLOWS TEST TO REPORT "PASSED"
 --                   IF TICK > DURATION'SMALL.
 
+with Impdef;
 WITH CALENDAR;  USE CALENDAR;
 WITH SYSTEM;    USE SYSTEM;
 WITH REPORT;    USE REPORT;
@@ -50,7 +51,7 @@ BEGIN
      ---------------------------------------------
 
      DECLARE   -- (A)
-          X : DURATION := 5.0;
+          X : DURATION := 5.0 * Impdef.One_Second;
           OLD_TIME : TIME;
           LAPSE : DURATION;
      BEGIN     -- (A)
@@ -138,8 +139,8 @@ BEGIN
      ---------------------------------------------
 
      DECLARE   -- (E)
-          INC1 : DURATION := 2.0;
-          INC2 : DURATION := 3.0;
+          INC1 : DURATION := 2.0 * Impdef.One_Second;
+          INC2 : DURATION := 3.0 * Impdef.One_Second;
           OLD_TIME : TIME;
           LAPSE : DURATION;
      BEGIN     -- (E)
diff --git a/gcc/testsuite/c-c++-common/Warray-bounds-11.c b/gcc/testsuite/c-c++-common/Warray-bounds-11.c
new file mode 100644
index 0000000..686a94d
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Warray-bounds-11.c
@@ -0,0 +1,21 @@
+/* PR c++/120954 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Warray-bounds=2" } */
+
+static const int a[32] = { 11, 12, 13, 14, 15 };
+static const int b[32] = { 21, 22, 23, 24, 25 };
+static const int c[32] = { 31, 32, 33, 34, 35 };
+static const int d[32] = { 111, 112, 113, 114, 115 };
+static const int e[32] = { 121, 122, 123, 124, 125 };
+static const int f[32] = { 131, 132, 133, 134, 135 };
+
+int
+foo (int x, int y)
+{
+  int r = 0;
+  if (x >= 0 && x < 32)
+    r = (y >= 4 ? (y >= 0x65 ? a : b ) : c)[x];
+  else if (x >= 0x100 && x < 0x120)
+    r = (y >= 4 ? (y >= 0x65 ? d : e ) : f)[x - 0x100];
+  return r;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-parm-1.c b/gcc/testsuite/c-c++-common/Wunused-parm-1.c
new file mode 100644
index 0000000..355fa4a
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-parm-1.c
@@ -0,0 +1,50 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-parameter" } */
+
+void baz (int);
+
+void
+foo (int a,		/* { dg-warning "parameter 'a' set but not used" } */
+     int b,		/* { dg-warning "parameter 'b' set but not used" } */
+     int c,		/* { dg-warning "parameter 'c' set but not used" } */
+     int d,		/* { dg-warning "parameter 'd' set but not used" } */
+     int e,		/* { dg-warning "parameter 'e' set but not used" } */
+     int f,		/* { dg-warning "parameter 'f' set but not used" } */
+     int g,		/* { dg-warning "parameter 'g' set but not used" } */
+     int h,		/* { dg-warning "parameter 'h' set but not used" } */
+     int i,		/* { dg-warning "parameter 'i' set but not used" } */
+     int j,		/* { dg-warning "parameter 'j' set but not used" } */
+     int k,		/* { dg-warning "parameter 'k' set but not used" } */
+     int l,		/* { dg-warning "parameter 'l' set but not used" } */
+     int m)		/* { dg-warning "parameter 'm' set but not used" } */
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-parm-2.c b/gcc/testsuite/c-c++-common/Wunused-parm-2.c
new file mode 100644
index 0000000..2caea94
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-parm-2.c
@@ -0,0 +1,50 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused -Wextra" } */
+
+void baz (int);
+
+void
+foo (int a,		/* { dg-warning "parameter 'a' set but not used" } */
+     int b,		/* { dg-warning "parameter 'b' set but not used" } */
+     int c,		/* { dg-warning "parameter 'c' set but not used" } */
+     int d,		/* { dg-warning "parameter 'd' set but not used" } */
+     int e,		/* { dg-warning "parameter 'e' set but not used" } */
+     int f,		/* { dg-warning "parameter 'f' set but not used" } */
+     int g,		/* { dg-warning "parameter 'g' set but not used" } */
+     int h,		/* { dg-warning "parameter 'h' set but not used" } */
+     int i,		/* { dg-warning "parameter 'i' set but not used" } */
+     int j,		/* { dg-warning "parameter 'j' set but not used" } */
+     int k,		/* { dg-warning "parameter 'k' set but not used" } */
+     int l,		/* { dg-warning "parameter 'l' set but not used" } */
+     int m)		/* { dg-warning "parameter 'm' set but not used" } */
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-parm-3.c b/gcc/testsuite/c-c++-common/Wunused-parm-3.c
new file mode 100644
index 0000000..2978cd4
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-parm-3.c
@@ -0,0 +1,50 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-parameter=3" } */
+
+void baz (int);
+
+void
+foo (int a,		/* { dg-warning "parameter 'a' set but not used" } */
+     int b,		/* { dg-warning "parameter 'b' set but not used" } */
+     int c,		/* { dg-warning "parameter 'c' set but not used" } */
+     int d,		/* { dg-warning "parameter 'd' set but not used" } */
+     int e,		/* { dg-warning "parameter 'e' set but not used" } */
+     int f,		/* { dg-warning "parameter 'f' set but not used" } */
+     int g,		/* { dg-warning "parameter 'g' set but not used" } */
+     int h,		/* { dg-warning "parameter 'h' set but not used" } */
+     int i,		/* { dg-warning "parameter 'i' set but not used" } */
+     int j,		/* { dg-warning "parameter 'j' set but not used" } */
+     int k,		/* { dg-warning "parameter 'k' set but not used" } */
+     int l,		/* { dg-warning "parameter 'l' set but not used" } */
+     int m)		/* { dg-warning "parameter 'm' set but not used" } */
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-parm-4.c b/gcc/testsuite/c-c++-common/Wunused-parm-4.c
new file mode 100644
index 0000000..063b40f
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-parm-4.c
@@ -0,0 +1,50 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-parameter=2" } */
+
+void baz (int);
+
+void
+foo (int a,		/* { dg-warning "parameter 'a' set but not used" } */
+     int b,		/* { dg-warning "parameter 'b' set but not used" } */
+     int c,		/* { dg-warning "parameter 'c' set but not used" } */
+     int d,		/* { dg-warning "parameter 'd' set but not used" } */
+     int e,		/* { dg-warning "parameter 'e' set but not used" } */
+     int f,
+     int g,
+     int h,
+     int i,
+     int j,
+     int k,
+     int l,
+     int m)		/* { dg-warning "parameter 'm' set but not used" } */
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-parm-5.c b/gcc/testsuite/c-c++-common/Wunused-parm-5.c
new file mode 100644
index 0000000..1c80a82
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-parm-5.c
@@ -0,0 +1,50 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-parameter=1" } */
+
+void baz (int);
+
+void
+foo (int a,		/* { dg-warning "parameter 'a' set but not used" } */
+     int b,
+     int c,
+     int d,
+     int e,
+     int f,
+     int g,
+     int h,
+     int i,
+     int j,
+     int k,
+     int l,
+     int m)
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-parm-6.c b/gcc/testsuite/c-c++-common/Wunused-parm-6.c
new file mode 100644
index 0000000..ee328bd
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-parm-6.c
@@ -0,0 +1,50 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-parameter=0" } */
+
+void baz (int);
+
+void
+foo (int a,
+     int b,
+     int c,
+     int d,
+     int e,
+     int f,
+     int g,
+     int h,
+     int i,
+     int j,
+     int k,
+     int l,
+     int m)
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-19.c b/gcc/testsuite/c-c++-common/Wunused-var-19.c
new file mode 100644
index 0000000..32c47e6
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-var-19.c
@@ -0,0 +1,60 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-variable" } */
+
+void baz (int);
+
+void
+foo (void)
+{
+  int a = 0;		/* { dg-warning "variable 'a' set but not used" } */
+  a = 1;
+  int b = 0;		/* { dg-warning "variable 'b' set but not used" } */
+  ++b;
+  int c = 0;		/* { dg-warning "variable 'c' set but not used" } */
+  c++;
+  int d = 0;		/* { dg-warning "variable 'd' set but not used" } */
+  --d;
+  int e = 0;		/* { dg-warning "variable 'e' set but not used" } */
+  e--;
+  int f = 0;		/* { dg-warning "variable 'f' set but not used" } */
+  f += 2;
+  int g = 0;		/* { dg-warning "variable 'g' set but not used" } */
+  g |= 2;
+  int h = 0;		/* { dg-warning "variable 'h' set but not used" } */
+  h -= 2;
+  int i = 0;		/* { dg-warning "variable 'i' set but not used" } */
+  i &= 2;
+  int j = 0;		/* { dg-warning "variable 'j' set but not used" } */
+  j ^= 2;
+  int k = 0;		/* { dg-warning "variable 'k' set but not used" } */
+  k *= 2;
+  int l = 0;		/* { dg-warning "variable 'l' set but not used" } */
+  l %= 2;
+  int m = 0;		/* { dg-warning "variable 'm' set but not used" } */
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-20.c b/gcc/testsuite/c-c++-common/Wunused-var-20.c
new file mode 100644
index 0000000..e25b26b
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-var-20.c
@@ -0,0 +1,60 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused" } */
+
+void baz (int);
+
+void
+foo (void)
+{
+  int a = 0;		/* { dg-warning "variable 'a' set but not used" } */
+  a = 1;
+  int b = 0;		/* { dg-warning "variable 'b' set but not used" } */
+  ++b;
+  int c = 0;		/* { dg-warning "variable 'c' set but not used" } */
+  c++;
+  int d = 0;		/* { dg-warning "variable 'd' set but not used" } */
+  --d;
+  int e = 0;		/* { dg-warning "variable 'e' set but not used" } */
+  e--;
+  int f = 0;		/* { dg-warning "variable 'f' set but not used" } */
+  f += 2;
+  int g = 0;		/* { dg-warning "variable 'g' set but not used" } */
+  g |= 2;
+  int h = 0;		/* { dg-warning "variable 'h' set but not used" } */
+  h -= 2;
+  int i = 0;		/* { dg-warning "variable 'i' set but not used" } */
+  i &= 2;
+  int j = 0;		/* { dg-warning "variable 'j' set but not used" } */
+  j ^= 2;
+  int k = 0;		/* { dg-warning "variable 'k' set but not used" } */
+  k *= 2;
+  int l = 0;		/* { dg-warning "variable 'l' set but not used" } */
+  l %= 2;
+  int m = 0;		/* { dg-warning "variable 'm' set but not used" } */
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-21.c b/gcc/testsuite/c-c++-common/Wunused-var-21.c
new file mode 100644
index 0000000..0732d98
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-var-21.c
@@ -0,0 +1,60 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-variable=3" } */
+
+void baz (int);
+
+void
+foo (void)
+{
+  int a = 0;		/* { dg-warning "variable 'a' set but not used" } */
+  a = 1;
+  int b = 0;		/* { dg-warning "variable 'b' set but not used" } */
+  ++b;
+  int c = 0;		/* { dg-warning "variable 'c' set but not used" } */
+  c++;
+  int d = 0;		/* { dg-warning "variable 'd' set but not used" } */
+  --d;
+  int e = 0;		/* { dg-warning "variable 'e' set but not used" } */
+  e--;
+  int f = 0;		/* { dg-warning "variable 'f' set but not used" } */
+  f += 2;
+  int g = 0;		/* { dg-warning "variable 'g' set but not used" } */
+  g |= 2;
+  int h = 0;		/* { dg-warning "variable 'h' set but not used" } */
+  h -= 2;
+  int i = 0;		/* { dg-warning "variable 'i' set but not used" } */
+  i &= 2;
+  int j = 0;		/* { dg-warning "variable 'j' set but not used" } */
+  j ^= 2;
+  int k = 0;		/* { dg-warning "variable 'k' set but not used" } */
+  k *= 2;
+  int l = 0;		/* { dg-warning "variable 'l' set but not used" } */
+  l %= 2;
+  int m = 0;		/* { dg-warning "variable 'm' set but not used" } */
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-22.c b/gcc/testsuite/c-c++-common/Wunused-var-22.c
new file mode 100644
index 0000000..84f57c5
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-var-22.c
@@ -0,0 +1,60 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-variable=2" } */
+
+void baz (int);
+
+void
+foo (void)
+{
+  int a = 0;		/* { dg-warning "variable 'a' set but not used" } */
+  a = 1;
+  int b = 0;		/* { dg-warning "variable 'b' set but not used" } */
+  ++b;
+  int c = 0;		/* { dg-warning "variable 'c' set but not used" } */
+  c++;
+  int d = 0;		/* { dg-warning "variable 'd' set but not used" } */
+  --d;
+  int e = 0;		/* { dg-warning "variable 'e' set but not used" } */
+  e--;
+  int f = 0;
+  f += 2;
+  int g = 0;
+  g |= 2;
+  int h = 0;
+  h -= 2;
+  int i = 0;
+  i &= 2;
+  int j = 0;
+  j ^= 2;
+  int k = 0;
+  k *= 2;
+  int l = 0;
+  l %= 2;
+  int m = 0;		/* { dg-warning "variable 'm' set but not used" } */
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-23.c b/gcc/testsuite/c-c++-common/Wunused-var-23.c
new file mode 100644
index 0000000..b74c3f4
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-var-23.c
@@ -0,0 +1,60 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-variable=1" } */
+
+void baz (int);
+
+void
+foo (void)
+{
+  int a = 0;		/* { dg-warning "variable 'a' set but not used" } */
+  a = 1;
+  int b = 0;
+  ++b;
+  int c = 0;
+  c++;
+  int d = 0;
+  --d;
+  int e = 0;
+  e--;
+  int f = 0;
+  f += 2;
+  int g = 0;
+  g |= 2;
+  int h = 0;
+  h -= 2;
+  int i = 0;
+  i &= 2;
+  int j = 0;
+  j ^= 2;
+  int k = 0;
+  k *= 2;
+  int l = 0;
+  l %= 2;
+  int m = 0;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-24.c b/gcc/testsuite/c-c++-common/Wunused-var-24.c
new file mode 100644
index 0000000..a59f50a
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/Wunused-var-24.c
@@ -0,0 +1,60 @@
+/* PR c/44677 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wunused-but-set-variable=0" } */
+
+void baz (int);
+
+void
+foo (void)
+{
+  int a = 0;
+  a = 1;
+  int b = 0;
+  ++b;
+  int c = 0;
+  c++;
+  int d = 0;
+  --d;
+  int e = 0;
+  e--;
+  int f = 0;
+  f += 2;
+  int g = 0;
+  g |= 2;
+  int h = 0;
+  h -= 2;
+  int i = 0;
+  i &= 2;
+  int j = 0;
+  j ^= 2;
+  int k = 0;
+  k *= 2;
+  int l = 0;
+  l %= 2;
+  int m = 0;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
diff --git a/gcc/testsuite/c-c++-common/Wunused-var-7.c b/gcc/testsuite/c-c++-common/Wunused-var-7.c
index 7419643..ea6babd 100644
--- a/gcc/testsuite/c-c++-common/Wunused-var-7.c
+++ b/gcc/testsuite/c-c++-common/Wunused-var-7.c
@@ -24,7 +24,7 @@ foo (void)
 void
 bar (void)
 {
-  int a;
+  int a;		/* { dg-warning "set but not used" } */
   int b;
   int c;		/* { dg-warning "set but not used" } */
   a = 1;
@@ -36,7 +36,7 @@ bar (void)
 void
 baz (void)
 {
-  int a;
+  int a;		/* { dg-warning "set but not used" } */
   int b;
   int c;
   int d;
diff --git a/gcc/testsuite/c-c++-common/attr-warn-unused-result-2.c b/gcc/testsuite/c-c++-common/attr-warn-unused-result-2.c
new file mode 100644
index 0000000..f35198d
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/attr-warn-unused-result-2.c
@@ -0,0 +1,16 @@
+/* PR c/82134 */
+/* { dg-do compile } */
+/* { dg-additional-options -Wno-c++-compat { target c } } */
+
+struct S {};
+
+__attribute__((warn_unused_result)) struct S foo();
+
+void use_s(struct S);
+
+void
+test (void)
+{
+  struct S s = foo(); /* { dg-bogus "ignoring return value of" } */
+  use_s(foo()); /* { dg-bogus "ignoring return value of" } */
+}
diff --git a/gcc/testsuite/c-c++-common/gomp/omp_get_num_devices_initial_device.c b/gcc/testsuite/c-c++-common/gomp/omp_get_num_devices_initial_device.c
index 4b17143..6e2c1a8 100644
--- a/gcc/testsuite/c-c++-common/gomp/omp_get_num_devices_initial_device.c
+++ b/gcc/testsuite/c-c++-common/gomp/omp_get_num_devices_initial_device.c
@@ -25,8 +25,8 @@ int f()
 
 /* { dg-final { scan-tree-dump-not "abort" "optimized" } }  */
 
-/* { dg-final { scan-tree-dump-not "omp_get_num_devices;" "optimized" { target { ! offloading_enabled } } } }  */
+/* { dg-final { scan-tree-dump-not "omp_get_num_devices" "optimized" { target { ! offloading_enabled } } } }  */
 /* { dg-final { scan-tree-dump "return 0;" "optimized" { target { ! offloading_enabled } } } }  */
 
-/* { dg-final { scan-tree-dump-times "omp_get_num_devices;" 1 "optimized" { target offloading_enabled } } }  */
+/* { dg-final { scan-tree-dump-times "omp_get_num_devices" 1 "optimized" { target offloading_enabled } } }  */
 /* { dg-final { scan-tree-dump "_1 = __builtin_omp_get_num_devices \\(\\);\[\\r\\n\]+\[ \]+return _1;" "optimized" { target offloading_enabled } } }  */
diff --git a/gcc/testsuite/c-c++-common/musttail32.c b/gcc/testsuite/c-c++-common/musttail32.c
new file mode 100644
index 0000000..f1b7052
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/musttail32.c
@@ -0,0 +1,23 @@
+/* PR ipa/121023 */
+/* { dg-do compile { target musttail } } */
+/* { dg-options "-O2" } */
+
+struct S { int a, b; };
+
+[[gnu::noipa]] int
+foo (struct S x, int y, int z)
+{
+  return x.a + y + z;
+}
+
+[[gnu::noinline]] static int
+bar (struct S x, int y, int z)
+{
+  [[gnu::musttail]] return foo ((struct S) { x.a, 0 }, y, 1);
+}
+
+int
+baz (int x)
+{
+  return bar ((struct S) { 1, 2 }, x, 2) + bar ((struct S) { 2, 3 }, x + 1, 2);
+}
diff --git a/gcc/testsuite/c-c++-common/pr121159.c b/gcc/testsuite/c-c++-common/pr121159.c
new file mode 100644
index 0000000..c8c5d67
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/pr121159.c
@@ -0,0 +1,17 @@
+/* PR middle-end/121159 */
+/* { dg-do compile { target musttail } } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times "foo \\\(\[^\n\r]*\\\); \\\[tail call\\\] \\\[must tail call\\\]" 1 "optimized" } } */
+		
+[[noreturn, gnu::noipa]] void
+foo (void)
+{
+  for (;;)
+    ;
+}
+
+void
+bar (void)
+{
+  [[gnu::musttail]] return foo ();
+}
diff --git a/gcc/testsuite/cobol.dg/group2/_-static__compilation.cob b/gcc/testsuite/cobol.dg/group2/_-static__compilation.cob
index 7843d3d..f344a84 100644
--- a/gcc/testsuite/cobol.dg/group2/_-static__compilation.cob
+++ b/gcc/testsuite/cobol.dg/group2/_-static__compilation.cob
@@ -1,7 +1,7 @@
        *> { dg-do run }
        *> { dg-options "-static" }
-       *> { dg-output-file "group2/_-static__compilation.out" }
-
+       *> { dg-prune-output {warning} }
+       *> { dg-output {hello, world} }
        IDENTIFICATION DIVISION.
        PROGRAM-ID. prog.
        PROCEDURE DIVISION.
diff --git a/gcc/testsuite/cobol.dg/group2/_-static__compilation.out b/gcc/testsuite/cobol.dg/group2/_-static__compilation.out
deleted file mode 100644
index ae0e511..0000000
--- a/gcc/testsuite/cobol.dg/group2/_-static__compilation.out
+++ /dev/null
@@ -1,2 +0,0 @@
-hello, world
-
diff --git a/gcc/testsuite/g++.dg/abi/regparm1.C b/gcc/testsuite/g++.dg/abi/regparm1.C
index c471046..3aae3dd 100644
--- a/gcc/testsuite/g++.dg/abi/regparm1.C
+++ b/gcc/testsuite/g++.dg/abi/regparm1.C
@@ -1,5 +1,5 @@
 // PR c++/29911 (9381)
-// { dg-do run { target i?86-*-* x86_64-*-* } }
+// { dg-do run { target { { i?86-*-* x86_64-*-* } && ia32 } } }
 // { dg-require-effective-target c++11 }
 
 extern "C" int printf(const char *, ...);
diff --git a/gcc/testsuite/g++.dg/concepts/auto7a.C b/gcc/testsuite/g++.dg/concepts/auto7a.C
index 88868f4..f36038d 100644
--- a/gcc/testsuite/g++.dg/concepts/auto7a.C
+++ b/gcc/testsuite/g++.dg/concepts/auto7a.C
@@ -2,6 +2,7 @@
 
 template <class T> struct A { };
 void f(A<auto> a) { }		// { dg-error "auto. in template argument" }
+// { dg-message "in parameter declaration" "" { target c++17_down } .-1 }
 int main()
 {
   f(A<int>());
diff --git a/gcc/testsuite/g++.dg/concepts/auto7b.C b/gcc/testsuite/g++.dg/concepts/auto7b.C
new file mode 100644
index 0000000..874192c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/concepts/auto7b.C
@@ -0,0 +1,10 @@
+// PR c++/120917
+// { dg-do compile { target c++14 } }
+// { dg-additional-options "-fconcepts -Wno-abbreviated-auto-in-template-arg" }
+
+template <class T> struct A { };
+void f(A<auto> a) { }
+int main()
+{
+  f(A<int>());
+}
diff --git a/gcc/testsuite/g++.dg/concepts/auto7c.C b/gcc/testsuite/g++.dg/concepts/auto7c.C
new file mode 100644
index 0000000..5b16027
--- /dev/null
+++ b/gcc/testsuite/g++.dg/concepts/auto7c.C
@@ -0,0 +1,12 @@
+// { dg-do compile { target c++17 } }
+// { dg-additional-options -fconcepts }
+
+template <class T>
+concept True = true;
+
+template <class T> struct A { };
+void f(A<True auto> a) { } // { dg-error "use of .auto. in template argument" }
+int main()
+{
+  f(A<int>());
+}
diff --git a/gcc/testsuite/g++.dg/concepts/pr67249.C b/gcc/testsuite/g++.dg/concepts/pr67249.C
index e0f8d5a..e97183d 100644
--- a/gcc/testsuite/g++.dg/concepts/pr67249.C
+++ b/gcc/testsuite/g++.dg/concepts/pr67249.C
@@ -4,4 +4,4 @@
 template<class T> concept C1 = true;
 template<class A, class B> struct Pair {};
 // We used to test "Pair<auto, C1 >".
-void f(Pair<C1 auto, C1 auto>);
+void f(Pair<C1 auto, C1 auto>);	// { dg-error "auto" }
diff --git a/gcc/testsuite/g++.dg/concepts/pr67249a.C b/gcc/testsuite/g++.dg/concepts/pr67249a.C
new file mode 100644
index 0000000..cb5d90e
--- /dev/null
+++ b/gcc/testsuite/g++.dg/concepts/pr67249a.C
@@ -0,0 +1,7 @@
+// { dg-do compile { target c++17 } }
+// { dg-options "-fconcepts -Wno-abbreviated-auto-in-template-arg" }
+
+template<class T> concept C1 = true;
+template<class A, class B> struct Pair {};
+// We used to test "Pair<auto, C1 >".
+void f(Pair<C1 auto, C1 auto>);
diff --git a/gcc/testsuite/g++.dg/coroutines/torture/pr120243-unhandled-1.C b/gcc/testsuite/g++.dg/coroutines/torture/pr120243-unhandled-1.C
new file mode 100644
index 0000000..16bfef1
--- /dev/null
+++ b/gcc/testsuite/g++.dg/coroutines/torture/pr120243-unhandled-1.C
@@ -0,0 +1,33 @@
+// PR c++/120243
+// { dg-do run }
+
+#include <coroutine>
+
+struct coro {
+     struct promise_type {
+        promise_type() = default;
+
+        std::suspend_never initial_suspend() const noexcept { return {}; }
+        std::suspend_never final_suspend() const noexcept { return {}; }
+
+        void unhandled_exception() { throw; }
+
+        coro get_return_object() { return {}; }
+        void return_void() {}
+
+    };
+};
+
+int main() {
+    auto c = []() -> coro {
+        throw "hello";
+        __builtin_abort();
+        co_return;
+    };
+    try {
+        c();
+    }
+    catch(...) {
+        __builtin_printf("Caught!\n");
+    }
+}
diff --git a/gcc/testsuite/g++.dg/coroutines/torture/pr120243-unhandled-2.C b/gcc/testsuite/g++.dg/coroutines/torture/pr120243-unhandled-2.C
new file mode 100644
index 0000000..614c4ec
--- /dev/null
+++ b/gcc/testsuite/g++.dg/coroutines/torture/pr120243-unhandled-2.C
@@ -0,0 +1,34 @@
+// PR c++/120243
+// { dg-do run }
+
+#include <coroutine>
+
+struct coro {
+     struct promise_type {
+        promise_type() = default;
+
+        std::suspend_always initial_suspend() const noexcept { return {}; }
+        std::suspend_always final_suspend() const noexcept { return {}; }
+
+        void unhandled_exception() { throw; }
+
+        coro get_return_object() { return {std::coroutine_handle<promise_type>::from_promise(*this)}; }
+        void return_void() {}
+    };
+
+    std::coroutine_handle<promise_type> h;
+};
+
+int main() {
+    auto c = []() -> coro {
+        throw "hello";
+        __builtin_abort();
+        co_return;
+    };
+    try {
+        c().h.resume();
+    }
+    catch(...) {
+        __builtin_printf("Caught!\n");
+    }
+}
diff --git a/gcc/testsuite/g++.dg/coroutines/torture/pr121219.C b/gcc/testsuite/g++.dg/coroutines/torture/pr121219.C
new file mode 100644
index 0000000..d1e7cb1
--- /dev/null
+++ b/gcc/testsuite/g++.dg/coroutines/torture/pr121219.C
@@ -0,0 +1,149 @@
+// PR c++/121219
+// { dg-do run }
+
+#include <coroutine>
+#ifdef OUTPUT
+#include <iostream>
+#endif
+#include <stdexcept>
+
+struct Task {
+    struct promise_type;
+    using handle_type = std::coroutine_handle<promise_type>;
+
+    struct promise_type {
+        Task* task_;
+        int result_;
+
+        static void* operator new(std::size_t size) noexcept {
+            void* p = ::operator new(size, std::nothrow);
+#ifdef OUTPUT
+            std::cerr << "operator new (no arg) " << size << " -> " << p << std::endl;  
+#endif
+            return p;
+        }
+        static void operator delete(void* ptr) noexcept {
+            return ::operator delete(ptr, std::nothrow);
+        }
+#if 1 // change to 0 to fix crash
+        static Task get_return_object_on_allocation_failure() noexcept {
+#ifdef OUTPUT
+            std::cerr << "get_return_object_on_allocation_failure" << std::endl;
+#endif
+            return Task(nullptr);
+        }
+#endif
+
+        auto get_return_object() { 
+#ifdef OUTPUT
+            std::cerr << "get_return_object" << std::endl;
+#endif
+            return Task{handle_type::from_promise(*this)}; 
+        }
+
+        auto initial_suspend() { 
+#ifdef OUTPUT
+            std::cerr << "initial_suspend" << std::endl;
+#endif
+            return std::suspend_always{}; 
+        }
+
+        auto final_suspend() noexcept { 
+#ifdef OUTPUT
+            std::cerr << "final_suspend" << std::endl;
+#endif
+            return std::suspend_never{};  // Coroutine auto-destructs
+        }
+
+        ~promise_type() {
+            if (task_) {
+#ifdef OUTPUT
+                std::cerr << "promise_type destructor: Clearing Task handle" << std::endl;
+#endif
+                task_->h_ = nullptr;
+            }
+        }
+
+        void unhandled_exception() { 
+#ifdef OUTPUT
+            std::cerr << "unhandled_exception" << std::endl;
+#endif
+            std::terminate(); 
+        }
+
+        void return_value(int value) { 
+#ifdef OUTPUT
+            std::cerr << "return_value: " << value << std::endl;
+#endif
+            result_ = value;
+            if (task_) {
+                task_->result_ = value;
+                task_->completed_ = true;
+            }
+        }
+    };
+
+    handle_type h_;
+    int result_;
+    bool completed_ = false;
+
+    Task(handle_type h) : h_(h) {
+#ifdef OUTPUT
+        std::cerr << "Task constructor" << std::endl;
+#endif
+        if (h_) {
+            h_.promise().task_ = this;  // Link promise to Task
+        }
+    }
+
+    ~Task() { 
+#ifdef OUTPUT
+        std::cerr << "~Task destructor" << std::endl;
+#endif
+        // Only destroy handle if still valid (coroutine not completed)
+        if (h_) {
+#ifdef OUTPUT
+            std::cerr << "Destroying coroutine handle" << std::endl;
+#endif
+            h_.destroy(); 
+        }
+    }
+
+    bool done() const { 
+        return completed_ || !h_ || h_.done(); 
+    }
+
+    void resume() { 
+#ifdef OUTPUT
+        std::cerr << "Resuming task" << std::endl;
+#endif
+        if (h_) h_.resume(); 
+    }
+
+    int result() const {
+        if (!done()) throw std::runtime_error("Result not available");
+        return result_;
+    }
+};
+
+Task my_coroutine() {
+#ifdef OUTPUT
+    std::cerr << "Inside my_coroutine" << std::endl;
+#endif
+   co_return 42;
+}
+
+int main() {
+    auto task = my_coroutine();
+    while (!task.done()) {
+#ifdef OUTPUT
+        std::cerr << "Resuming task in main" << std::endl;
+#endif
+        task.resume();
+    }
+#ifdef OUTPUT
+    std::cerr << "Task completed in main, printing result" << std::endl;
+#endif
+    if (task.result() != 42)
+      __builtin_abort ();
+}
diff --git a/gcc/testsuite/g++.dg/cpp/if-comma-1.C b/gcc/testsuite/g++.dg/cpp/if-comma-1.C
new file mode 100644
index 0000000..0daaff9
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp/if-comma-1.C
@@ -0,0 +1,42 @@
+// PR c++/120778
+// { dg-do preprocess }
+// { dg-options "-pedantic-errors" }
+
+#if (1, 2)
+#define M1 1
+#else
+#error
+#endif
+#if 1 ? 2, 3 : 4
+#define M2 2
+#else
+#error
+#endif
+#if 0 ? 2, 0 : 1
+#define M3 3
+#else
+#error
+#endif
+#if 0 || (1, 2)
+#define M4 4
+#else
+#error
+#endif
+#if 1 || (1, 2)
+#define M5 5
+#else
+#error
+#endif
+#if (1, 2) && 1
+#define M6 6
+#else
+#error
+#endif
+#if 1 && (1, 2)
+#define M7 7
+#else
+#error
+#endif
+#if M1 + M2 + M3 + M4 + M5 + M6 + M7 != 28
+#error
+#endif
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-array29.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-array29.C
new file mode 100644
index 0000000..714d050
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-array29.C
@@ -0,0 +1,13 @@
+// PR c++/87097
+// { dg-do compile { target c++11 } }
+
+struct A {
+  constexpr A() : data() {}
+  struct X { int n; };
+  X data[2];
+};
+
+static_assert((A(), true), "");
+static_assert(A().data[0].n == 0, "");
+static_assert(A().data[1].n == 0, "");
+constexpr A x;
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-array30.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-array30.C
new file mode 100644
index 0000000..3f72407
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-array30.C
@@ -0,0 +1,22 @@
+// PR c++/120800
+// { dg-do compile { target c++11 } }
+
+template<typename T>
+struct Container
+{
+ T m_data[1] {};
+};
+
+class Element
+{
+private:
+ Element() = default;
+
+private:
+ bool m_bool1 { false };
+ bool m_bool2;
+
+ friend struct Container<Element>;
+};
+
+Container<Element> element;
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-ellipsis2.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-ellipsis2.C
index b6a5323..c473257 100644
--- a/gcc/testsuite/g++.dg/cpp0x/constexpr-ellipsis2.C
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-ellipsis2.C
@@ -9,4 +9,6 @@ struct A
 
 constexpr int ellipsis(...) { return 1; }
 
-static_assert(ellipsis(A().empty()), "Error"); // { dg-error "non-constant condition|empty" }
+static_assert(ellipsis(A().empty()), "Error"); // { dg-error "non-constant condition" }
+// { dg-error "call to non-'constexpr' function 'bool A::empty\\\(\\\)'" "" { target c++23_down } .-1 }
+// { dg-error "temporary of non-literal type 'A' in a constant expression" "" { target c++26 } .-2 }
diff --git a/gcc/testsuite/g++.dg/cpp0x/constexpr-throw.C b/gcc/testsuite/g++.dg/cpp0x/constexpr-throw.C
index 9c49fa4..9abf4a3 100644
--- a/gcc/testsuite/g++.dg/cpp0x/constexpr-throw.C
+++ b/gcc/testsuite/g++.dg/cpp0x/constexpr-throw.C
@@ -2,8 +2,8 @@
 // Explicit { dg-require-effective-target exceptions_enabled } to avoid verify compiler messages FAILs for '-fno-exceptions'.
 
 constexpr int may_throw(bool decide) {
-	return decide ? 42 : throw -1; // { dg-error "throw" }
+	return decide ? 42 : throw -1; // { dg-error "throw" "" { target c++23_down } }
 }
 
-constexpr int x = may_throw(false); // { dg-message "may_throw" }
-constexpr int y = may_throw(true);
+constexpr int x = may_throw(false); // { dg-message "may_throw" "" { target c++23_down } }
+constexpr int y = may_throw(true); // { dg-error "uncaught exception '-1'" "" { target c++26 } .-1 }
diff --git a/gcc/testsuite/g++.dg/cpp0x/duplicate1.C b/gcc/testsuite/g++.dg/cpp0x/duplicate1.C
index 1545e1c..4e85edc 100644
--- a/gcc/testsuite/g++.dg/cpp0x/duplicate1.C
+++ b/gcc/testsuite/g++.dg/cpp0x/duplicate1.C
@@ -6,7 +6,7 @@ struct A
   virtual void foo() const;
 };
 
-struct B final final : A  /* { dg-error "duplicate virt-specifier" }
+struct B final final : A  /* { dg-error "duplicate 'final' specifier" }
   { dg-begin-multiline-output "" }
  struct B final final : A
                 ^~~~~
diff --git a/gcc/testsuite/g++.dg/cpp0x/final1.C b/gcc/testsuite/g++.dg/cpp0x/final1.C
new file mode 100644
index 0000000..1d64095
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/final1.C
@@ -0,0 +1,11 @@
+// PR c++/120628
+// { dg-do compile { target c++98_only } }
+
+namespace A {
+  struct B {};
+  struct B final = {};
+}
+namespace C {
+  struct D { D (int, int); };
+  struct D final (42, 0);
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/final2.C b/gcc/testsuite/g++.dg/cpp0x/final2.C
new file mode 100644
index 0000000..d8d5866
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/final2.C
@@ -0,0 +1,26 @@
+// PR c++/120628
+// { dg-do compile }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++14 } }
+
+namespace U {
+  struct A {};
+  struct A final = {};
+}
+namespace V {
+  template <int N>
+  struct B {};
+  template <int N>
+  struct B<N> final = {};	// { dg-warning "variable templates only available with" "" { target c++11_down } }
+}
+struct C {
+  struct D {};
+  static D foo ();
+  struct D final = foo ();	// { dg-warning "non-static data member initializers only available with" "" { target c++98_only } }
+};
+namespace W {
+  struct E { struct F {}; };
+  struct E::F final = {};
+}
+template <int N>
+struct V::B<N> final = {};	// { dg-warning "variable templates only available with" "" { target c++11_down } }
diff --git a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-constexpr3.C b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-constexpr3.C
new file mode 100644
index 0000000..6525213
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-constexpr3.C
@@ -0,0 +1,13 @@
+// PR c++/120716
+// { dg-do compile { target c++11 } }
+
+struct A { int *r; };
+
+void
+foo ()
+{
+  static int i;
+  constexpr A a = { &i };
+  static_assert (a.r == &i, "");
+  [&] { static_assert (a.r != nullptr, ""); } ();
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-constexpr3a.C b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-constexpr3a.C
new file mode 100644
index 0000000..398e079
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/lambda/lambda-constexpr3a.C
@@ -0,0 +1,12 @@
+// PR c++/120716
+// { dg-do compile { target c++11 } }
+
+struct A { int *const &r; };
+
+void
+foo (int x)
+{
+  constexpr A a = { &x };	// { dg-error "constant" }
+  static_assert (a.r == &x, "");	// { dg-error "non-constant" }
+  [&] { static_assert (a.r != nullptr, ""); } (); // { dg-error "non-constant" }
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/override2.C b/gcc/testsuite/g++.dg/cpp0x/override2.C
index ab4dec4..d7f542e 100644
--- a/gcc/testsuite/g++.dg/cpp0x/override2.C
+++ b/gcc/testsuite/g++.dg/cpp0x/override2.C
@@ -23,9 +23,9 @@ struct D5 : B3<D5> {};
 
 struct D6 : B4<D6> {}; // { dg-error "cannot derive from 'final' base" }
 
-struct B6 final final {}; // { dg-error "duplicate virt-specifier" }
+struct B6 final final {}; // { dg-error "duplicate 'final' specifier" }
 
-struct B7 override {}; // { dg-error "cannot specify 'override' for a class" }
+struct B7 override {}; // { dg-error "variable 'B7 override' has initializer but incomplete type" }
 
 namespace N
 {
@@ -45,7 +45,7 @@ int main()
   struct B2 final; // { dg-error "redeclaration" }
   struct B2 override; // { dg-message "previously declared here" }
   struct B2 final {}; // { dg-error "redefinition" }
-  struct B2 override {}; // { dg-error "cannot specify 'override' for a class" }
+  struct B2 override {}; // { dg-error "redeclaration of 'main\\\(\\\)::B2 override'" }
   B2 override{}; // { dg-error "redeclaration" }
   struct foo final {}; // { dg-message "previous definition" }
   struct foo final {}; // { dg-error "redefinition" }
diff --git a/gcc/testsuite/g++.dg/cpp0x/override5.C b/gcc/testsuite/g++.dg/cpp0x/override5.C
new file mode 100644
index 0000000..3382c14
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/override5.C
@@ -0,0 +1,26 @@
+// PR c++/120569
+// { dg-do compile }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++14 } }
+
+namespace U {
+  struct A {};
+  struct A override {};		// { dg-warning "extended initializer lists only available with" "" { target c++98_only } }
+}
+namespace V {
+  template <int N>
+  struct B {};
+  template <int N>
+  struct B<N> override {};	// { dg-warning "extended initializer lists only available with" "" { target c++98_only } }
+}				// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+struct C {
+  struct D {};
+  struct D override {};		// { dg-warning "extended initializer lists only available with" "" { target c++98_only } }
+};				// { dg-warning "non-static data member initializers only available with" "" { target c++98_only } .-1 }
+namespace W {
+  struct E { struct F {}; };
+  struct E::F override {};	// { dg-warning "extended initializer lists only available with" "" { target c++98_only } }
+}
+template <int N>
+struct V::B<N> override {};	// { dg-warning "extended initializer lists only available with" "" { target c++98_only } }
+				// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
diff --git a/gcc/testsuite/g++.dg/cpp0x/override6.C b/gcc/testsuite/g++.dg/cpp0x/override6.C
new file mode 100644
index 0000000..601e91d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/override6.C
@@ -0,0 +1,26 @@
+// PR c++/120628
+// { dg-do compile }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++14 } }
+
+namespace U {
+  struct A {};
+  struct A override = {};
+}
+namespace V {
+  template <int N>
+  struct B {};
+  template <int N>
+  struct B<N> override = {};	// { dg-warning "variable templates only available with" "" { target c++11_down } }
+}
+struct C {
+  struct D {};
+  static D foo ();
+  struct D override = foo ();	// { dg-warning "non-static data member initializers only available with" "" { target c++98_only } }
+};
+namespace W {
+  struct E { struct F {}; };
+  struct E::F override = {};
+}
+template <int N>
+struct V::B<N> override = {};	// { dg-warning "variable templates only available with" "" { target c++11_down } }
diff --git a/gcc/testsuite/g++.dg/cpp0x/range-for40.C b/gcc/testsuite/g++.dg/cpp0x/range-for40.C
new file mode 100644
index 0000000..dea4a2a
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/range-for40.C
@@ -0,0 +1,41 @@
+// PR c++/84009
+// { dg-do compile { target c++11 } }
+
+int z[64];
+
+void
+foo ()
+{
+  for (static auto a : z)		// { dg-error "for-range-declaration cannot be 'static'" }
+    ;
+  for (thread_local auto a : z)		// { dg-error "for-range-declaration cannot be 'thread_local'" }
+    ;
+  for (__thread auto a : z)		// { dg-error "for-range-declaration cannot be '__thread'" }
+    ;					// { dg-error "function-scope 'a' implicitly auto and declared '__thread'" "" { target *-*-* } .-1 }
+  for (register auto a : z)		// { dg-error "for-range-declaration cannot be 'register'" }
+    ;					// { dg-error "does not allow 'register' storage class specifier" "" { target c++17 } .-1 }
+  for (extern auto a : z)		// { dg-error "for-range-declaration cannot be 'extern'" }
+    ;					// { dg-error "'a' has both 'extern' and initializer" "" { target *-*-* } .-1 }
+  for (mutable auto a : z)		// { dg-error "non-member 'a' cannot be declared 'mutable'" }
+    ;
+  for (virtual auto a : z)		// { dg-error "'virtual' outside class declaration" }
+    ;
+  for (explicit auto a : z)		// { dg-error "'explicit' outside class declaration" }
+    ;
+  for (friend auto a : z)		// { dg-error "'friend' used outside of class" }
+    ;
+  for (typedef auto a : z)		// { dg-error "typedef declared 'auto'" }
+    ;					// { dg-error "typedef 'a' is initialized \\\(use 'decltype' instead\\\)" "" { target *-*-* } .-1 }
+#if __cplusplus >= 202002L
+  for (consteval auto a : z)		// { dg-error "a variable cannot be declared 'consteval'" "" { target c++20 } }
+    ;
+  for (constinit auto a : z)		// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+    ;
+#endif
+  for (inline auto a : z)		// { dg-error "'inline' specifier invalid for variable 'a' declared at block scope" }
+    ;
+  for (struct S { int a; } a : z)	// { dg-error "types may not be defined in a for-range-declaration" }
+    ;					// { dg-error "conversion from 'int' to non-scalar type 'foo\\\(\\\)::S' requested" "" { target *-*-* } .-1 }
+  for (enum E { E0 } a : z)		// { dg-error "types may not be defined in a for-range-declaration" }
+    ;					// { dg-error "invalid conversion from 'int' to 'foo\\\(\\\)::E'" "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/range-for41.C b/gcc/testsuite/g++.dg/cpp0x/range-for41.C
new file mode 100644
index 0000000..d690365
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/range-for41.C
@@ -0,0 +1,42 @@
+// PR c++/84009
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+
+int z[64];
+
+void
+foo ()
+{
+  for (static auto a : z)		// { dg-warning "for-range-declaration cannot be 'static'" }
+    ;
+  for (thread_local auto a : z)		// { dg-warning "for-range-declaration cannot be 'thread_local'" }
+    ;
+  for (__thread auto a : z)		// { dg-warning "for-range-declaration cannot be '__thread'" }
+    ;					// { dg-warning "function-scope 'a' implicitly auto and declared '__thread'" "" { target *-*-* } .-1 }
+  for (register auto a : z)		// { dg-warning "for-range-declaration cannot be 'register'" }
+    ;					// { dg-warning "does not allow 'register' storage class specifier" "" { target c++17 } .-1 }
+  for (extern auto a : z)		// { dg-warning "for-range-declaration cannot be 'extern'" }
+    ;					// { dg-error "'a' has both 'extern' and initializer" "" { target *-*-* } .-1 }
+  for (mutable auto a : z)		// { dg-error "non-member 'a' cannot be declared 'mutable'" }
+    ;
+  for (virtual auto a : z)		// { dg-error "'virtual' outside class declaration" }
+    ;
+  for (explicit auto a : z)		// { dg-error "'explicit' outside class declaration" }
+    ;
+  for (friend auto a : z)		// { dg-error "'friend' used outside of class" }
+    ;
+  for (typedef auto a : z)		// { dg-error "typedef declared 'auto'" }
+    ;					// { dg-error "typedef 'a' is initialized \\\(use 'decltype' instead\\\)" "" { target *-*-* } .-1 }
+#if __cplusplus >= 202002L
+  for (consteval auto a : z)		// { dg-error "a variable cannot be declared 'consteval'" "" { target c++20 } }
+    ;
+  for (constinit auto a : z)		// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+    ;
+#endif
+  for (inline auto a : z)		// { dg-error "'inline' specifier invalid for variable 'a' declared at block scope" }
+    ;
+  for (struct S { int a; } a : z)	// { dg-error "types may not be defined in a for-range-declaration" }
+    ;					// { dg-error "conversion from 'int' to non-scalar type 'foo\\\(\\\)::S' requested" "" { target *-*-* } .-1 }
+  for (enum E { E0 } a : z)		// { dg-error "types may not be defined in a for-range-declaration" }
+    ;					// { dg-error "invalid conversion from 'int' to 'foo\\\(\\\)::E'" "" { target *-*-* } .-1 }
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/range-for42.C b/gcc/testsuite/g++.dg/cpp0x/range-for42.C
new file mode 100644
index 0000000..a5d94fc
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/range-for42.C
@@ -0,0 +1,41 @@
+// PR c++/84009
+// { dg-do compile { target c++11 } }
+
+struct S { int y; } z[64];
+
+void
+foo ()
+{
+  for (static auto [ a ] : z)		// { dg-error "for-range-declaration cannot be 'static'" }
+    ;					// { dg-error "structured binding declaration can be 'static' only in" "" { target c++17_down } .-1 }
+					// { dg-error "structured bindings only available with" "" { target c++14_down } .-2 }
+  for (thread_local auto [ a ] : z)	// { dg-error "for-range-declaration cannot be 'thread_local'" }
+    ;					// { dg-error "structured binding declaration can be 'thread_local' only in" "" { target c++17_down } .-1 }
+					// { dg-error "structured bindings only available with" "" { target c++14_down } .-2 }
+  for (__thread auto [ a ] : z)		// { dg-error "for-range-declaration cannot be '__thread'" }
+    ;					// { dg-error "function-scope 'structured binding' implicitly auto and declared '__thread'" "" { target *-*-* } .-1 }
+					// { dg-error "structured binding declaration can be '__thread' only in" "" { target c++17_down } .-2 }
+					// { dg-error "structured bindings only available with" "" { target c++14_down } .-3 }
+  for (register auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'register'" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (extern auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'extern'" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (mutable auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'mutable'" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (virtual auto [ a ] : z)		// { dg-error "'virtual' outside class declaration" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (explicit auto [ a ] : z)		// { dg-error "'explicit' outside class declaration" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (friend auto [ a ] : z)		// { dg-error "'friend' used outside of class" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (typedef auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'typedef'" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+#if __cplusplus >= 202002L
+  for (consteval auto [ a ] : z)	// { dg-error "structured binding declaration cannot be 'consteval'" "" { target c++20 } }
+    ;
+  for (constinit auto [ a ] : z)	// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+    ;
+#endif
+  for (inline auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'inline'" }
+    ;					// { dg-error "structured bindings only available with" "" { target c++14_down } .-1 }
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/range-for43.C b/gcc/testsuite/g++.dg/cpp0x/range-for43.C
new file mode 100644
index 0000000..77060e3
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp0x/range-for43.C
@@ -0,0 +1,42 @@
+// PR c++/84009
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+
+struct S { int y; } z[64];
+
+void
+foo ()
+{
+  for (static auto [ a ] : z)		// { dg-warning "for-range-declaration cannot be 'static'" }
+    ;					// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-1 }
+					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-2 }
+  for (thread_local auto [ a ] : z)	// { dg-warning "for-range-declaration cannot be 'thread_local'" }
+    ;					// { dg-warning "structured binding declaration can be 'thread_local' only in" "" { target c++17_down } .-1 }
+					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-2 }
+  for (__thread auto [ a ] : z)		// { dg-warning "for-range-declaration cannot be '__thread'" }
+    ;					// { dg-warning "function-scope 'structured binding' implicitly auto and declared '__thread'" "" { target *-*-* } .-1 }
+ 					// { dg-warning "structured binding declaration can be '__thread' only in" "" { target c++17_down } .-2 }
+					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-3 }
+  for (register auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'register'" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (extern auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'extern'" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (mutable auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'mutable'" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (virtual auto [ a ] : z)		// { dg-error "'virtual' outside class declaration" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (explicit auto [ a ] : z)		// { dg-error "'explicit' outside class declaration" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (friend auto [ a ] : z)		// { dg-error "'friend' used outside of class" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+  for (typedef auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'typedef'" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+#if __cplusplus >= 202002L
+  for (consteval auto [ a ] : z)	// { dg-error "structured binding declaration cannot be 'consteval'" "" { target c++20 } }
+    ;
+  for (constinit auto [ a ] : z)	// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+    ;
+#endif
+  for (inline auto [ a ] : z)		// { dg-error "structured binding declaration cannot be 'inline'" }
+    ;					// { dg-warning "structured bindings only available with" "" { target c++14_down } .-1 }
+}
diff --git a/gcc/testsuite/g++.dg/cpp0x/this1.C b/gcc/testsuite/g++.dg/cpp0x/this1.C
index 486e045..e70cd1a 100644
--- a/gcc/testsuite/g++.dg/cpp0x/this1.C
+++ b/gcc/testsuite/g++.dg/cpp0x/this1.C
@@ -8,10 +8,10 @@ struct S1 {
   void m3 () noexcept(noexcept(this->a)) { }
   void m4 () noexcept(noexcept(this)) { }
 
-  static auto m5 () -> decltype(this->a) { return 0; } // { dg-error ".this. may not be used in this context" }
-  static auto m6 () -> decltype(this) { return 0; } // { dg-error ".this. may not be used in this context" }
-  static void m7 () noexcept(noexcept(this->a)) { } // { dg-error ".this. may not be used in this context" }
-  static void m8 () noexcept(noexcept(this)) { } // { dg-error ".this. may not be used in this context" }
+  static auto m5 () -> decltype(this->a) { return 0; } // { dg-error ".this." }
+  static auto m6 () -> decltype(this) { return 0; } // { dg-error ".this." }
+  static void m7 () noexcept(noexcept(this->a)) { } // { dg-error ".this." }
+  static void m8 () noexcept(noexcept(this)) { } // { dg-error ".this." }
 };
 
 template <typename T>
@@ -41,6 +41,6 @@ test ()
 }
 
 struct S5 {
-  friend auto bar() -> decltype(this); // { dg-error ".this. may not be used in this context" }
+  friend auto bar() -> decltype(this); // { dg-error ".this." }
   auto bar2() -> decltype(this);
 };
diff --git a/gcc/testsuite/g++.dg/cpp1y/constexpr-84192.C b/gcc/testsuite/g++.dg/cpp1y/constexpr-84192.C
index f7439d8..18efd23 100644
--- a/gcc/testsuite/g++.dg/cpp1y/constexpr-84192.C
+++ b/gcc/testsuite/g++.dg/cpp1y/constexpr-84192.C
@@ -12,7 +12,7 @@ void
 f2 ()
 { 
   for (;;)
-    constexpr bool b = ({ break; false; }) && false;	// { dg-error "is not a constant expression" }
+    constexpr bool b = ({ break; false; }) && false;	// { dg-error "'break' outside of a loop or 'switch'" }
 }
 
 constexpr bool
diff --git a/gcc/testsuite/g++.dg/cpp1y/constexpr-throw.C b/gcc/testsuite/g++.dg/cpp1y/constexpr-throw.C
index b5fa6ca..75e2fb8 100644
--- a/gcc/testsuite/g++.dg/cpp1y/constexpr-throw.C
+++ b/gcc/testsuite/g++.dg/cpp1y/constexpr-throw.C
@@ -27,8 +27,8 @@ constexpr int fun(int n) {
   case 0:
     return 1;
   default:
-    throw; // { dg-error "not a constant expression" }
-  }
+    throw; // { dg-error "not a constant expression" "" { target c++23_down } }
+  } // { dg-error "'void __cxa_rethrow\\\(\\\)' called with no caught exceptions active" "" { target c++26 } .-1 }
 }
 
 static_assert(fun(0), "");
diff --git a/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic-a.C b/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic-a.C
new file mode 100644
index 0000000..557ecb9
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic-a.C
@@ -0,0 +1,15 @@
+// Basic generic lambda test
+// { dg-do run { target c++14 } }
+// { dg-additional-options -Wno-abbreviated-auto-in-template-arg }
+
+template <typename T, typename U> struct pair {};
+template <typename... T> struct tuple {};
+
+int main()
+{
+  auto a = [] (auto, pair<auto,auto> v) { return sizeof (v); };
+  auto b = [] (auto, pair<pair<auto,auto>,auto>... v) { return sizeof... (v); };
+
+  a(1, pair<int, float>());
+  b(2, pair<pair<short,char>, double>(), pair<pair<float,long>, int>());
+}
diff --git a/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic.C b/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic.C
index 6d2d250..6ef8652 100644
--- a/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic.C
+++ b/gcc/testsuite/g++.dg/cpp1y/lambda-generic-variadic.C
@@ -1,13 +1,13 @@
 // Basic generic lambda test
-// { dg-do run { target c++14 } }
+// { dg-do compile { target c++14 } }
 
 template <typename T, typename U> struct pair {};
 template <typename... T> struct tuple {};
 
 int main()
 {
-  auto a = [] (auto, pair<auto,auto> v) { return sizeof (v); };
-  auto b = [] (auto, pair<pair<auto,auto>,auto>... v) { return sizeof... (v); };
+  auto a = [] (auto, pair<auto,auto> v) { return sizeof (v); }; // { dg-error "auto" }
+  auto b = [] (auto, pair<pair<auto,auto>,auto>... v) { return sizeof... (v); }; // { dg-error "auto" }
 
   a(1, pair<int, float>());
   b(2, pair<pair<short,char>, double>(), pair<pair<float,long>, int>());
diff --git a/gcc/testsuite/g++.dg/cpp1y/pr85076.C b/gcc/testsuite/g++.dg/cpp1y/pr85076.C
index 6d54dea..b68143b 100644
--- a/gcc/testsuite/g++.dg/cpp1y/pr85076.C
+++ b/gcc/testsuite/g++.dg/cpp1y/pr85076.C
@@ -3,4 +3,4 @@
 
 template<typename> struct A*;	// { dg-error "expected unqualified-id before" }
 
-auto a = [](A<auto>) {};	// { dg-error "is not a template|has incomplete type" }
+auto a = [](A<auto>) {};	// { dg-error "is not a template|has incomplete type|auto" }
diff --git a/gcc/testsuite/g++.dg/cpp1z/constexpr-asm-5.C b/gcc/testsuite/g++.dg/cpp1z/constexpr-asm-5.C
index 35beb27..2c69b3b 100644
--- a/gcc/testsuite/g++.dg/cpp1z/constexpr-asm-5.C
+++ b/gcc/testsuite/g++.dg/cpp1z/constexpr-asm-5.C
@@ -18,7 +18,7 @@ struct H { short size () const { return 0; }
 	   constexpr const char *data () const { return ""; } };
 struct I { constexpr signed char size () const { return 0; }
 	   const char *data () const { return ""; } };
-struct J { constexpr int size () const { return j ? throw 1 : 0; }	// { dg-error "expression '<throw-expression>' is not a constant expression" }
+struct J { constexpr int size () const { return j ? throw 1 : 0; }	// { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23_down } }
 	   constexpr const char *data () const { return ""; };
 	   constexpr J (int x) : j (x) {}
 	   int j; };
@@ -114,6 +114,7 @@ foo ()
 
   asm ((J (0)));
   asm ("" :: (J (1)) (1));	// { dg-error "constexpr string 'size\\\(\\\)' must be a constant expression" }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
   asm ((M {}));
 #if __cpp_constexpr_dynamic_alloc >= 201907L
   asm ((N {}));			// { dg-error "constexpr string 'data\\\(\\\)\\\[0\\\]' must be a constant expression" "" { target c++20 } }
@@ -188,6 +189,7 @@ bar ()
 
   asm ((J (0)));
   asm ("" :: (J (1)) (1));	// { dg-error "constexpr string 'size\\\(\\\)' must be a constant expression" }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
   asm ((M {}));
 #if __cpp_constexpr_dynamic_alloc >= 201907L
   asm ((N {}));			// { dg-error "constexpr string 'data\\\(\\\)\\\[0\\\]' must be a constant expression" "" { target c++20 } }
@@ -272,7 +274,7 @@ namespace NN
 #if __cplusplus >= 201402L
   struct J {
     static constexpr int size () { return 0; }
-    static constexpr const char *data (int x = 0) { if (x) return nullptr; else throw 1; } }; // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++14 } }
+    static constexpr const char *data (int x = 0) { if (x) return nullptr; else throw 1; } }; // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target { c++14 && c++23_down } } }
 #endif
 #if __cpp_if_consteval >= 202106L
   struct K {
@@ -284,12 +286,12 @@ namespace NN
     static constexpr const char *data () { if consteval { return "test"; } else { throw 1; } }
   };
   struct M {
-    static constexpr int size () { if consteval { throw 1; } else { return 4; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23 } }
+    static constexpr int size () { if consteval { throw 1; } else { return 4; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23_only } }
     static constexpr const char *data () { return "test"; }
   };
   struct N {
     static constexpr int size () { return 4; }
-    static constexpr const char *data () { if consteval { throw 1; } else { return "test"; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23 } }
+    static constexpr const char *data () { if consteval { throw 1; } else { return "test"; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23_only } }
   };
 #endif
   struct O { constexpr int operator () () const { return 12; } };
@@ -318,12 +320,15 @@ namespace NN
     asm ((I {}));
 #if __cplusplus >= 201402L
     asm ((J {}));		// { dg-error "constexpr string 'data\\\(\\\)' must be a core constant expression" "" { target c++14 } }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
 #endif
 #if __cpp_if_consteval >= 202106L
     asm ((K {}));
     asm ((L {}));
     asm ((M {}));		// { dg-error "constexpr string 'size\\\(\\\)' must be a constant expression" "" { target c++23 } }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
     asm ((N {}));		// { dg-error "constexpr string 'data\\\(\\\)\\\[0\\\]' must be a constant expression" "" { target c++23 } }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
 #endif
     asm ((Q {}));
     asm ((R {}));
@@ -348,12 +353,15 @@ namespace NN
     asm ((I {}));
 #if __cplusplus >= 201402L
     asm ((J {}));		// { dg-error "constexpr string 'data\\\(\\\)' must be a core constant expression" "" { target c++14 } }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
 #endif
 #if __cpp_if_consteval >= 202106L
     asm ((K {}));
     asm ((L {}));
     asm ((M {}));		// { dg-error "constexpr string 'size\\\(\\\)' must be a constant expression" "" { target c++23 } }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
     asm ((N {}));		// { dg-error "constexpr string 'data\\\(\\\)\\\[0\\\]' must be a constant expression" "" { target c++23 } }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
 #endif
     asm ((Q {}));
     asm ((R {}));
diff --git a/gcc/testsuite/g++.dg/cpp1z/decomp3.C b/gcc/testsuite/g++.dg/cpp1z/decomp3.C
index a9b23b0..202edc3 100644
--- a/gcc/testsuite/g++.dg/cpp1z/decomp3.C
+++ b/gcc/testsuite/g++.dg/cpp1z/decomp3.C
@@ -19,7 +19,8 @@ test (A &b, B c)
 					// { dg-error "expected primary-expression before 'decltype'" "" { target c++11_down } .-2 }
   auto & & && & [ m, n, o ] = b;	// { dg-error "multiple ref-qualifiers" }
 					// { dg-warning "structured bindings only available with '-std=c..17' or '-std=gnu..17'" "" { target c++14_down } .-1 }
-  constexpr auto [ p ] = c;		// { dg-error "structured binding declaration cannot be 'constexpr'" }
+  constexpr B c2 = { 42 };
+  constexpr auto [ p ] = c2;		// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } }
 					// { dg-warning "structured bindings only available with '-std=c..17' or '-std=gnu..17'" "" { target c++14_down } .-1 }
   friend auto [ q ] = c;		// { dg-error "'friend' used outside of class" }
 					// { dg-warning "structured bindings only available with '-std=c..17' or '-std=gnu..17'" "" { target c++14_down } .-1 }
diff --git a/gcc/testsuite/g++.dg/cpp1z/nontype8.C b/gcc/testsuite/g++.dg/cpp1z/nontype8.C
new file mode 100644
index 0000000..b81e85b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp1z/nontype8.C
@@ -0,0 +1,12 @@
+// Test that the diagnostic mentions lack of constexpr
+// { dg-do compile { target c++17 } }
+
+template <auto f> void g() {}
+void x()
+{
+  using fp = void (*)();
+  fp f = nullptr;		// { dg-message "constexpr" }
+  g<f>();			// { dg-error "" }
+  int *p = nullptr;		// { dg-message "constexpr" }
+  g<p>();			// { dg-error "" }
+}
diff --git a/gcc/testsuite/g++.dg/cpp23/explicit-obj-lambda18.C b/gcc/testsuite/g++.dg/cpp23/explicit-obj-lambda18.C
new file mode 100644
index 0000000..d54a93d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp23/explicit-obj-lambda18.C
@@ -0,0 +1,12 @@
+// PR c++/114632
+// { dg-do compile { target c++23 } }
+
+struct S {};
+
+auto lambda = [](this auto& self, const int x) /* -> void */ {};
+
+int main()
+{
+    void (*func)(S&, int) = lambda; // { dg-error "" }
+    return 0;
+}
diff --git a/gcc/testsuite/g++.dg/cpp23/static-operator-call7.C b/gcc/testsuite/g++.dg/cpp23/static-operator-call7.C
new file mode 100644
index 0000000..7c381e6
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp23/static-operator-call7.C
@@ -0,0 +1,12 @@
+// PR c++/114632
+// { dg-do compile { target c++23 } }
+
+struct S {};
+
+auto lambda = [](auto, const int x) static /* -> void */ {};
+
+int main()
+{
+    void (*func)(int, int) = lambda;
+    return 0;
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block1.C b/gcc/testsuite/g++.dg/cpp26/consteval-block1.C
new file mode 100644
index 0000000..9e2cf22
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block1.C
@@ -0,0 +1,82 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+
+constexpr int fn () { return 42; }
+struct M {
+  static consteval void foo () {}
+};
+
+consteval { }
+consteval { fn (); }
+consteval { M::foo (); }
+consteval { auto x = fn (); return; }
+consteval {
+  [](int i) { return i; }(5);
+}
+auto lam = [] { };
+consteval { lam (); }
+
+struct S {
+  consteval { }
+};
+
+struct S2 {
+  consteval { fn(); }
+};
+
+class C {
+  consteval { }
+};
+
+class C2 {
+  consteval { M::foo (); }
+};
+
+union U {
+  consteval { }
+};
+
+template<typename>
+struct TS {
+  consteval { }
+};
+
+template<typename... Ts>
+struct TS2 {
+  consteval {
+    (Ts::foo (), ...);
+  }
+};
+
+TS2<M> ts2;
+
+void
+g ()
+{
+  consteval { }
+}
+
+template<typename>
+void
+tg ()
+{
+  consteval { }
+}
+
+void die ();
+constexpr int
+bar (int i)
+{
+  if (i != 42)
+    die ();
+  return 0;
+}
+
+void
+foo ()
+{
+  constexpr int r = 42;
+  consteval {
+    bar (r);
+  }
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block2.C b/gcc/testsuite/g++.dg/cpp26/consteval-block2.C
new file mode 100644
index 0000000..895fcb6
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block2.C
@@ -0,0 +1,49 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+
+void fn ();
+
+consteval { fn (); }	    // { dg-error "call to non-.constexpr. function" }
+consteval { return 42; }    // { dg-error "return-statement with a value" }
+
+struct S {
+  consteval {
+    fn ();		    // { dg-error "call to non-.constexpr. function" }
+  }
+  consteval {
+    return 42;		    // { dg-error "return-statement with a value" }
+  }
+};
+
+template<typename T>
+constexpr void foo (T t) { return t; }	// { dg-error "return-statement with a value" }
+
+template<int N>
+struct R {
+  consteval { foo (N); }
+};
+
+R<1> r;
+
+template<typename T>
+constexpr void foo2 (T t) { return t; }	// { dg-error "return-statement with a value" }
+
+template<int N>
+void
+f ()
+{
+  consteval { foo2 (1); }
+}
+
+constexpr int bar (int) { return 0; }
+
+void
+g ()
+{
+  f<1>();
+
+  int r = 42;
+  consteval {
+    bar (r);	      // { dg-error ".r. is not captured" }
+  }
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block3.C b/gcc/testsuite/g++.dg/cpp26/consteval-block3.C
new file mode 100644
index 0000000..c1221c3
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block3.C
@@ -0,0 +1,41 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+// Test that we actually evaluate the consteval block.
+
+void bar () { }
+
+template<int N>
+constexpr void
+fn ()
+{
+  if (N > 0)
+    bar ();			// { dg-error "call to non-.constexpr. function" }
+}
+
+template<int N>
+struct S {
+  consteval { fn<N>(); }	// { dg-error "called in a constant expression" }
+};
+
+S<1> s;
+
+template<int N>
+constexpr void
+fn2 ()
+{
+  if (N > 0)
+    bar ();			// { dg-error "call to non-.constexpr. function" }
+}
+
+template<int N>
+void
+g ()
+{
+  consteval { fn2<N>(); }	// { dg-error "called in a constant expression" }
+}
+
+void
+f ()
+{
+  g<1>();
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block4.C b/gcc/testsuite/g++.dg/cpp26/consteval-block4.C
new file mode 100644
index 0000000..be95e17
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block4.C
@@ -0,0 +1,41 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+// Test that we actually evaluate the consteval block.
+
+void bar () { }
+
+template<int N>
+constexpr void
+fn ()
+{
+  if (N > 0)
+    bar ();
+}
+
+template<int N>
+struct S {
+  consteval { fn<N>(); }
+};
+
+S<0> s;
+
+template<int N>
+constexpr void
+fn2 ()
+{
+  if (N > 0)
+    bar ();
+}
+
+template<int N>
+void
+g ()
+{
+  consteval { fn2<N>(); }
+}
+
+void
+f ()
+{
+  g<0>();
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block5.C b/gcc/testsuite/g++.dg/cpp26/consteval-block5.C
new file mode 100644
index 0000000..462cebe
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block5.C
@@ -0,0 +1,70 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+
+void bar () { }
+
+template<int N>
+constexpr void
+fn ()
+{
+  if (N > 0)
+    bar ();
+}
+
+template<typename>
+struct S {
+  consteval { fn<1>(); }
+};
+
+template<>
+struct S<int> {
+  consteval { fn<0>(); }
+};
+
+S<int> s1;
+
+template<typename T>
+struct S<T*> {
+  consteval { fn<0>(); }
+};
+
+S<int *> s2;
+
+template<typename T, int N>
+struct W {
+  consteval { T t; fn<N - 1>(); }
+};
+
+template<typename T>
+struct W<T, 0> {
+  consteval { T t; fn<0>(); }
+};
+
+template<>
+struct W<char, 0> {
+  consteval { fn<0>(); }
+};
+
+W<int, 0> w1;
+W<int, 1> w2;
+W<char, 0> w3;
+
+template<typename>
+void
+f ()
+{
+  consteval { fn<1>(); }
+}
+
+template<>
+void
+f<int> ()
+{
+  consteval { fn<0>(); }
+}
+
+void
+g ()
+{
+  f<int> ();
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block6.C b/gcc/testsuite/g++.dg/cpp26/consteval-block6.C
new file mode 100644
index 0000000..ca90b3e
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block6.C
@@ -0,0 +1,108 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+
+void die () {}
+
+template<int N>
+constexpr void
+fn ()
+{
+  if (N > 0)
+    die ();
+}
+
+template<int N>
+void
+fn2 ()
+{
+  struct S {
+    consteval {
+      fn<N>();
+    }
+  };
+}
+
+template<int N>
+struct A {
+  struct B {
+    consteval {
+      fn<N>();
+    }
+  };
+  template<int M>
+  struct C {
+    consteval {
+      fn<N + M>();
+    }
+  };
+};
+
+template<int N>
+struct D {
+  constexpr static int i = 0;
+  struct E {
+    consteval {
+      fn<i>();
+    }
+  };
+};
+
+A<0>::B b;
+A<0>::C<0> c;
+D<0>::E e;
+
+void
+f ()
+{
+  fn2<0>();
+}
+
+static constexpr int j = 0;
+const int x = 0;
+
+consteval {
+  fn<j>();
+  consteval {
+    fn<j + j>();
+    consteval {
+      fn<j + j + j>();
+      consteval {
+	fn<j + j + x>();
+	consteval {
+	  fn<j + x>();
+	}
+      }
+    }
+  }
+}
+
+struct R { constexpr R() {} };
+
+template<int N>
+constexpr auto X = N;
+
+consteval {
+  R{};
+  constexpr auto x = 0;
+  fn<x>();
+  fn<X<0>>();
+  if consteval
+    {
+      fn<j>();
+    }
+  else
+    {
+      die ();
+    }
+}
+
+template<typename T>
+struct G {
+  consteval {
+    using U = T[3];
+    U arr{};
+    int i = arr[2];
+  }
+};
+
+G<int> g;
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block7.C b/gcc/testsuite/g++.dg/cpp26/consteval-block7.C
new file mode 100644
index 0000000..231682f
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block7.C
@@ -0,0 +1,12 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+
+consteval {
+  template <class T> // { dg-error "template declaration cannot appear at block scope" }
+  struct X { };
+
+  template <class T> // { dg-error "template declaration cannot appear at block scope" }
+  concept C = true;
+
+  return; // OK
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/consteval-block8.C b/gcc/testsuite/g++.dg/cpp26/consteval-block8.C
new file mode 100644
index 0000000..ad164fd
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/consteval-block8.C
@@ -0,0 +1,38 @@
+// { dg-do compile { target c++26 } }
+// Test consteval blocks, as specified by P2996.
+
+/* __func__ won't be set.  Make sure we warn.  */
+consteval { __func__; }	  // { dg-error "outside of function scope" }
+consteval { { __func__; } }	  // { dg-error "outside of function scope" }
+consteval { []() mutable consteval -> void { __func__; } (); }	// { dg-bogus "outside of function scope" }
+consteval { []() mutable consteval -> void { consteval { __func__; } } (); } // { dg-bogus "outside of function scope" }
+
+auto l = []() -> void {
+  consteval { __func__; } // { dg-bogus "outside of function scope" }
+};
+
+struct F {
+  consteval { __func__; } // { dg-error "outside of function scope" }
+};
+template<typename>
+struct TF {
+  consteval { __func__; } // { dg-error "outside of function scope" }
+};
+
+void
+g ()
+{
+  consteval { __func__; } // { dg-bogus "outside of function scope" }
+  // Not a consteval-block-declaration.
+  []() mutable consteval -> void { __func__; } (); // { dg-bogus "outside of function scope" }
+}
+
+template<typename>
+void
+f ()
+{
+  consteval { __func__; } // { dg-bogus "outside of function scope" }
+  { consteval { __func__; } } // { dg-bogus "outside of function scope" }
+  __func__; // { dg-bogus "outside of function scope" }
+  []() mutable consteval -> void { __func__; } (); // { dg-bogus "outside of function scope" }
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh1.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh1.C
new file mode 100644
index 0000000..9eed3aa
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh1.C
@@ -0,0 +1,140 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct S {
+  constexpr S () : s (new int (0)) {}
+  constexpr S (int x) : s (new int (x)) {}
+  constexpr S (const S &x) : s (new int (*x.s)) {}
+  constexpr ~S () { delete s; }
+  int *s;
+};
+struct T : public S {
+  constexpr T () : S () {}
+  constexpr T (int x) : S (x) {}
+  constexpr T (const T &x) : S (*x.s) {}
+  constexpr ~T () {}
+};
+struct U : public T {
+  constexpr U () : T () {}
+  constexpr U (int x) : T (x) {}
+  constexpr U (const U &x) : T (*x.s) {}
+  constexpr ~U () {}
+};
+struct V : public T {
+  constexpr V () : T () {}
+  constexpr V (int x) : T (x) {}
+  constexpr V (const U &x) : T (*x.s) {}
+  constexpr ~V () {}
+};
+
+template <typename X>
+constexpr int
+foo (X x)
+{
+  try { throw x; }
+  catch (int &a) { return 42 + a; }
+  catch (const unsigned b) { return 43 + b; }
+  catch (const long &c) { return 44 + c; }
+  catch (bool d) { return 45 + d; }
+  catch (const U &e) { return 46 + *e.s; }
+  catch (const T &f) { return 47 + *f.s; }
+  catch (S g) { return 48 + *g.s; }
+  catch (int *const &h) { return 49; }
+  catch (long long *) { return 50; }
+  catch (const S *const &) { return 51; }
+  catch (...) { return 52; }
+}
+
+template <typename X>
+constexpr int
+bar (const X &x)
+{
+  throw x;
+}
+
+template <typename X>
+constexpr int
+baz (const X &x)
+{
+  try
+    {
+      try { bar (x); }
+      catch (int &a) { a += 80; throw; }
+      catch (long b) { b += 80; throw; }
+      catch (U &c) { c.s[0] += 82; throw; }
+      catch (V d) { d.s[0] += 83; throw; }
+    }
+  catch (int a) { return 42 + a; }
+  catch (const long &b) { return 43 + b; }
+  catch (S &c) { return 44 + c.s[0]; }
+  catch (long long d) { return 45 + d; }
+  catch (...) { return -1; }
+}
+
+constexpr int
+qux (int x, bool y = true)
+{
+  try
+    {
+      switch (x)
+	{
+	case 0: throw 42; break;
+	case 1: x = y ? throw 43 : 5; break;
+	case 2: x = -(throw 44, 6); break;
+	case 3: x = x + (throw 45, 7); break;
+	case 4: x = (throw 46, 8) + x; break;
+	case 5: x = (throw 47, y) ? 4 : 5; break;
+	case 6: x += (throw 48, y); break;
+	case 7: x = (double) (throw 49, y); break;
+	case 8: x = foo ((throw 50, x)); break;
+	default: break;
+	}
+    }
+  catch (int a) { return a; }
+  return -1;
+}
+
+constexpr int
+corge ()
+{
+  try { throw 0; }
+  catch (int *const &h) { return 49; }
+  catch (long long *) { return 50; }
+  catch (const S *const &) { return 51; }
+  catch (...) { return 52; }
+}
+
+static_assert (foo (12) == 54);
+static_assert (foo (12U) == 55);
+static_assert (foo (12L) == 56);
+static_assert (foo (false) == 45);
+static_assert (foo (true) == 46);
+static_assert (foo (U (12)) == 58);
+static_assert (foo (T (20)) == 67);
+static_assert (foo (S (30)) == 78);
+static_assert (foo (nullptr) == 49);
+static_assert (foo ((int *)nullptr) == 49);
+static_assert (foo ((long long *)nullptr) == 50);
+static_assert (foo ((const S *)nullptr) == 51);
+static_assert (foo ((const T *)nullptr) == 51);
+static_assert (foo ((const U *)nullptr) == 51);
+static_assert (foo (12ULL) == 52);
+static_assert (baz (5) == 127);
+static_assert (baz (6L) == 49);
+static_assert (baz (U (25)) == 151);
+static_assert (baz (V (26)) == 70);
+static_assert (baz (T (27)) == 71);
+static_assert (baz (S (28)) == 72);
+static_assert (baz (7LL) == 52);
+static_assert (baz (8ULL) == -1);
+static_assert (qux (0) == 42);
+static_assert (qux (1) == 43);
+static_assert (qux (2) == 44);
+static_assert (qux (3) == 45);
+static_assert (qux (4) == 46);
+static_assert (qux (5) == 47);
+static_assert (qux (6) == 48);
+static_assert (qux (7) == 49);
+static_assert (qux (8) == 50);
+static_assert (corge () == 52);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh10.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh10.C
new file mode 100644
index 0000000..a86cc4d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh10.C
@@ -0,0 +1,110 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct S {
+};
+struct T {
+  constexpr ~T () noexcept (false) { throw S {}; }
+};
+struct U {
+  int u;
+};
+struct V {
+  int v;
+  constexpr V (int x)
+  try : v { x }
+  {
+    if (v > 42)
+      throw U { 42 };
+  }
+  catch (U &u)
+  {
+    --u.u;
+  }
+};
+struct W {
+  constexpr ~W () { ++w; }
+  int &w;
+};
+struct X : public V {
+  constexpr X (int x)
+  try : V(x)
+  {
+  }
+  catch (U &u)
+  {
+    --u.u;
+  }
+};
+
+constexpr int
+foo (bool x)
+{
+  try
+    {
+      T t;				// { dg-error "'std::terminate' called after throwing an exception '42'" }
+      if (x)				// { dg-message "destructor exited with an exception" "" { target *-*-* } .-1 }
+	throw 42;
+      return 10;
+    }
+  catch (S)
+    {
+      return 11;
+    }
+}
+
+constexpr int
+bar ()
+{
+  V v { 42 };
+  try
+    {
+      V w { 43 };
+    }
+  catch (const U &u)
+    {
+      if (u.u == 41)
+	return 44;
+    }
+  return -1;
+}
+
+constexpr int
+baz ()
+{
+  int i = 42;
+  try
+    {
+      W w { i };
+      throw S ();
+    }
+  catch (...)
+    {
+      if (i == 43)
+	return 42;
+    }
+  return -1;
+}
+
+constexpr int
+qux ()
+{
+  X v { 42 };
+  try
+    {
+      X w { 43 };
+    }
+  catch (const U &u)
+    {
+      if (u.u == 40)
+	return 48;
+    }
+  return -1;
+}
+
+static_assert (foo (false) == 11);
+constexpr int a = foo (true);		// { dg-message "in 'constexpr' expansion of" }
+static_assert (bar () == 44);
+static_assert (baz () == 42);
+static_assert (qux () == 48);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh11.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh11.C
new file mode 100644
index 0000000..287e066
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh11.C
@@ -0,0 +1,69 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct A {
+  explicit constexpr A (int x) noexcept : a (x) {}
+  constexpr virtual int foo () const noexcept { return a; }
+  constexpr virtual ~A () {}
+  int a;
+};
+struct B : public A {
+  explicit constexpr B (int x) noexcept : A (x) {}
+  constexpr int foo () const noexcept override { return a | 0x10; }
+};
+struct C : public A {
+  explicit constexpr C (int x) noexcept : A (x) {}
+};
+struct D : public A {
+  explicit constexpr D (int x) noexcept : A (x) {}
+};
+struct E {
+  constexpr E () noexcept : e (0) {}
+  explicit constexpr E (int x) noexcept : e (x) {}
+  int e;
+};
+struct F : public E, public B {
+  explicit constexpr F (int x) noexcept : B (x) {}
+};
+struct G : public E, public C {
+  explicit constexpr G (int x) noexcept : C (x) {}
+};
+struct H : public E, public D {
+  explicit constexpr H (int x) noexcept : D (x) {}
+};
+
+consteval int
+bar (void (*fn) ())
+{
+  try
+    {
+      fn ();
+    }
+  catch (C &a)
+    {
+      return a.foo () | 0x20;
+    }
+  catch (const C &b)		// { dg-warning "exception of type 'C' will be caught by earlier handler" }
+    {
+      return b.foo () | 0x60;
+    }
+  catch (A &c)
+    {
+      return c.foo ();
+    }
+  catch (const A &d)		// { dg-warning "exception of type 'A' will be caught by earlier handler" }
+    {
+      return d.foo () | 0x40;
+    }
+  return -1;
+}
+
+static_assert (bar ([] { throw A { 1 }; }) == 1);
+static_assert (bar ([] { throw B { 2 }; }) == 0x12);
+static_assert (bar ([] { throw C { 3 }; }) == 0x23);
+static_assert (bar ([] { throw D { 4 }; }) == 4);
+constexpr int a = bar ([] { throw E { 5 }; });	// { dg-error "uncaught exception 'E\\\{5\\\}'" }
+static_assert (bar ([] { throw F { 6 }; }) == 0x16);
+static_assert (bar ([] { throw G { 7 }; }) == 0x27);
+static_assert (bar ([] { throw H { 8 }; }) == 8);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh12.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh12.C
new file mode 100644
index 0000000..4a85f15
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh12.C
@@ -0,0 +1,74 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct A {
+  explicit constexpr A (int x) noexcept : a (x) {}
+  constexpr virtual int foo () const noexcept { return a; }
+  constexpr virtual ~A () {}
+  int a;
+};
+struct B : public A {
+  explicit constexpr B (int x) noexcept : A (x) {}
+  constexpr int foo () const noexcept override { return a | 0x10; }
+};
+struct C : public A {
+  explicit constexpr C (int x) noexcept : A (x) {}
+};
+struct D : public A {
+  explicit constexpr D (int x) noexcept : A (x) {}
+};
+struct E {
+  constexpr E () noexcept : e (0) {}
+  explicit constexpr E (int x) noexcept : e (x) {}
+  int e;
+};
+struct F : public E, public B {
+  explicit constexpr F (int x) noexcept : B (x) {}
+};
+struct G : public E, public C {
+  explicit constexpr G (int x) noexcept : C (x) {}
+};
+struct H : public E, public D {
+  explicit constexpr H (int x) noexcept : D (x) {}
+};
+
+consteval int
+bar (void (*fn) ())
+{
+  int r = 0;
+  try
+    {
+      fn ();
+    }
+  catch (C *a)
+    {
+      r = a->foo () | 0x20;
+      delete a;
+    }
+  catch (const C *b)
+    {
+      r = b->foo () | 0x60;
+      delete b;
+    }
+  catch (A *c)
+    {
+      r = c->foo ();
+      delete c;
+    }
+  catch (const A *d)
+    {
+      r = d->foo () | 0x40;
+      delete d;
+    }
+  return r;
+}
+
+static_assert (bar ([] { throw new A { 1 }; }) == 1);
+static_assert (bar ([] { throw new B { 2 }; }) == 0x12);
+static_assert (bar ([] { throw new C { 3 }; }) == 0x23);
+static_assert (bar ([] { throw new D { 4 }; }) == 4);
+constexpr int a = bar ([] { throw new E { 5 }; });	// { dg-error "uncaught exception of type 'E\\\*'" }
+static_assert (bar ([] { throw new F { 6 }; }) == 0x16);
+static_assert (bar ([] { throw new G { 7 }; }) == 0x27);
+static_assert (bar ([] { throw new H { 8 }; }) == 8);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh13.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh13.C
new file mode 100644
index 0000000..d62771c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh13.C
@@ -0,0 +1,36 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+template <typename T>
+consteval T
+foo (T x)
+{
+  try
+    {
+      throw &x;
+    }
+  catch (void *ptr)		// { dg-message "for type 'void\\\*'" }
+    {
+      return *static_cast<T *> (ptr) | 0x10;
+    }
+  catch (const void *ptr)	// { dg-message "for type 'const void\\\*'" }
+    {
+      return *static_cast<const T *> (ptr) | 0x20;
+    }
+  catch (T *ptr)		// { dg-warning "exception of type 'T\\\*' will be caught by earlier handler" }
+    {				// { dg-warning "exception of type 'int\\\*' will be caught by earlier handler" "" { target *-*-* } .-1 }
+      return *ptr | 0x30;	// { dg-warning "exception of type 'long long unsigned int\\\*' will be caught by earlier handler" "" { target *-*-* } .-2 }
+    }
+  catch (const T *ptr)
+    {
+      return *ptr | 0x40;
+    }
+  catch (...)
+    {
+      return -1;
+    }
+}
+
+static_assert (foo (1) == 0x11);
+static_assert (foo (2ULL) == 0x12ULL);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh14.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh14.C
new file mode 100644
index 0000000..3e52f2b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh14.C
@@ -0,0 +1,42 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+template <typename T>
+constexpr T
+foo (T x, auto... y)
+{
+  const T z[] = { x, y... };
+  try
+    {
+      throw z;
+    }
+  catch (const T (&a)[4])
+    {
+      return T ();
+    }
+  catch (const T *b)
+    {
+      return b[0];
+    }
+  catch (...)
+    {
+      return T ();
+    }
+}
+
+void
+bar ()
+{
+}
+
+void
+baz ()
+{
+}
+
+static_assert (foo (42, 43, 44, 45, 46) == 42);
+static_assert (foo (43U, 44U, 45U, 46U) == 43U);
+static_assert (foo (44LL, 45LL) == 44LL);
+static_assert (foo (bar, baz, bar, baz) == bar);
+static_assert (foo (baz, bar) == baz);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh15.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh15.C
new file mode 100644
index 0000000..3dea461
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh15.C
@@ -0,0 +1,39 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+int
+foo (int x, int y)
+{
+  return x + y;
+}
+
+constexpr int
+bar (int x)
+{
+  if (x < 0)
+    throw x;
+  return x;
+}
+
+constexpr int
+baz (int x, int y)
+{
+  return foo (bar (x), bar (y));
+}
+
+constexpr int
+qux (int x, int y)
+{
+  try
+    {
+      return baz (x, y);
+    }
+  catch (int)
+    {
+      return 42;
+    }
+}
+
+static_assert (qux (12, -1) == 42);
+static_assert (qux (-7, 12) == 42);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh2.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh2.C
new file mode 100644
index 0000000..8bd6b6f
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh2.C
@@ -0,0 +1,112 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct S {
+  constexpr S () : s (0) {}
+  constexpr S (int x) : s (x) { if (x == 42) throw 42; }
+  constexpr S (const S &x) : s (x.s) {}
+  constexpr ~S () noexcept (false) { if (s == 41) throw 41; }
+  constexpr const char *what () const noexcept { return "S"; }
+  int s;
+};
+struct T : public S {
+  constexpr T () {}
+  constexpr T (int x) : S (x) {}
+  constexpr T (const T &x) : S (x.s) {}
+  constexpr ~T () {}
+  constexpr const char *what () const noexcept { return "T"; }
+};
+struct U {
+  constexpr U () : u (0) {}
+  constexpr U (int x) : u (x) {}
+  constexpr U (const S &x) : u (0) {}
+  constexpr U (const U &x) : u (x.u) { if (u == 42) throw 43; }
+  constexpr ~U () {}
+  constexpr const char *what () const noexcept { return "U"; }
+  int u;
+};
+
+constexpr int
+foo (int x)
+{
+  if (x == 1)
+    throw 43;
+  return x;
+}
+
+constexpr int
+bar (int x) noexcept	// { dg-error "'std::terminate' called" }
+{			// { dg-message "uncaught exception exited from 'noexcept' function 'constexpr int bar\\\(int\\\)'" "" { target *-*-* } .-1 }
+  return foo (x);
+}
+
+constexpr int
+baz (int x)
+{
+  switch (x)
+    {
+    case 0: throw 1; break;
+    case 1: try { x = bar (x); } catch (...) {} break;		// { dg-message "in 'constexpr' expansion of" }
+    case 2: throw S (2); break;
+    case 3: try { throw S (42); } catch (int a) { if (a != 42) throw -1; } break;
+    case 4: try { S s (41); throw 2; } catch (...) {} break;	// { dg-error "'std::terminate' called" }
+    case 5: return 5;						// { dg-message "destructor exited with an exception" "" { target *-*-* } .-1 }
+    case 6:
+      try
+	{
+	  throw S (5);
+	}
+      catch (const T &) {}
+      catch (int) {}
+      catch (const bool &) {}
+      catch (const T **const &) {}
+      break;
+    case 7: try { constexpr int y = foo (2); } catch (...) {} break;
+    case 8:
+      try
+	{
+	  try
+	    {
+	      throw U ();
+	    }
+	  catch (U &u)
+	    {
+	      u.u = 42;
+	      throw;
+	    }
+	}
+      catch (U u)					// { dg-error "'std::terminate' called" }
+	{						// { dg-message "constructor exited with another exception while entering handler" "" { target *-*-* } .-1 }
+	}
+      break;
+    case 9:
+      try
+	{
+	  throw U (S (41));				// { dg-error "'std::terminate' called" }
+	}						// { dg-message "destructor exited with an exception" "" { target *-*-* } .-1 }
+      catch (...)
+	{
+	}
+      break;
+    }
+  return -1;
+}
+
+constexpr int
+qux (int x)
+{
+  try { constexpr int y = foo (1); } catch (...) {}	// { dg-error "uncaught exception" }
+  return 0;
+}
+
+constexpr int a = baz (0);	// { dg-error "uncaught exception" }
+constexpr int b = baz (1);	// { dg-message "in 'constexpr' expansion of" }
+constexpr int c = baz (2);	// { dg-error "uncaught exception" }
+constexpr int d = baz (3);
+constexpr int e = baz (4);	// { dg-message "in 'constexpr' expansion of" }
+constexpr int f = baz (5);
+constexpr int g = baz (6);	// { dg-error "uncaught exception" }
+constexpr int h = baz (7);
+constexpr int i = baz (8);	// { dg-message "in 'constexpr' expansion of" }
+constexpr int j = baz (9);	// { dg-message "in 'constexpr' expansion of" }
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh3.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh3.C
new file mode 100644
index 0000000..f844d11
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh3.C
@@ -0,0 +1,442 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+#include <exception>
+#include <new>
+#include <typeinfo>
+
+constexpr std::exception a;
+constexpr const char *b = a.what ();
+constexpr std::bad_exception c;
+constexpr const char *d = c.what ();
+constexpr std::bad_alloc e;
+constexpr const char *f = e.what ();
+constexpr std::bad_array_new_length g;
+constexpr const char *h = g.what ();
+constexpr std::bad_cast i;
+constexpr const char *j = i.what ();
+constexpr std::bad_typeid k;
+constexpr const char *l = k.what ();
+constexpr std::exception_ptr m = nullptr;
+static_assert (m == nullptr);
+constexpr std::exception_ptr n = std::current_exception ();
+static_assert (n == nullptr);
+constexpr std::exception_ptr o;
+static_assert (o == nullptr);
+constexpr std::nested_exception p;
+static_assert (p.nested_ptr () == nullptr);
+
+struct A { virtual ~A () {} };
+struct B { virtual void b (); };
+struct C { virtual void c (); };
+struct D : private B { virtual void d (); };
+struct E { virtual void e (); };
+struct F : D, E, private C { virtual void f (); };
+struct G { constexpr G () { if (std::uncaught_exceptions () != 0) asm (""); } };
+struct H { constexpr H () : h (0) {} constexpr ~H () { if (std::uncaught_exceptions () != h) asm (""); } int h; };
+struct I : std::nested_exception { };
+struct J { virtual ~J () noexcept = default; };
+struct K final { };
+struct L : J, std::nested_exception { };
+struct M { };
+struct N : I, L { };
+struct O : private std::nested_exception { };
+
+constexpr int
+foo (int x)
+{
+  if (std::uncaught_exceptions () != 0)
+    return -1;
+  switch (x)
+    {
+    case 0:
+      try
+	{
+	  const std::type_info &s = typeid (*(A *) 0);
+	  return -1;
+	}
+      catch (const std::bad_typeid &x)
+	{
+	  if (std::uncaught_exceptions () != 0)
+	    return -1;
+	  const char *p = x.what ();
+	  return 1;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    case 1:
+      try
+	{
+	  static constexpr F f;
+	  D &d = dynamic_cast<D &>((B &) f);
+	  return -1;
+	}
+      catch (std::bad_cast x)
+	{
+	  const char *p = x.what ();
+	  return 2;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    case 2:
+      try
+	{
+	  H h;
+	  h.h = 1;
+	  if (std::current_exception () != nullptr)
+	    return -1;
+	  throw G ();
+	}
+      catch (const G &g)
+	{
+	  if (std::uncaught_exceptions () != 0)
+	    return -1;
+	  if (std::current_exception () == nullptr)
+	    return -1;
+	  return 3;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    case 3:
+      try
+	{
+	  decltype (sizeof 0) x = -64;
+	  char (*a)[2] = new char[x][2];
+	  delete[] a;
+	}
+      catch (std::bad_array_new_length x)
+	{
+	  return 4;
+	}
+      break;
+    case 4:
+      try
+	{
+	  int y = -1;
+	  int *a = new int[y];
+	  delete[] a;
+	}
+      catch (const std::bad_array_new_length &)
+	{
+	  return 5;
+	}
+      break;
+    case 5:
+      try
+	{
+	  int z = 1;
+	  int *a = new int[z]{1, 2, 3};
+	  delete[] a;
+	}
+      catch (std::bad_array_new_length &)
+	{
+	  return 6;
+	}
+      break;
+    case 6:
+      {
+	std::exception_ptr b, d;
+	if (b != nullptr || d != nullptr)
+	  return -1;
+	try
+	  {
+	    throw 1;
+	  }
+	catch (int a)
+	  {
+	    if (a != 1)
+	      return -1;
+	    b = std::current_exception ();
+	    if (b == nullptr)
+	      return -1;
+	    try
+	      {
+		throw 2L;
+	      }
+	    catch (long int c)
+	      {
+		if (c != 2L)
+		  return -1;
+		d = std::current_exception ();
+		if (d == nullptr || b == d)
+		  return -1;
+	      }
+	    if (std::current_exception () != b)
+	      return -1;
+	  }
+	if (std::current_exception () != nullptr)
+	  return -1;
+	try
+	  {
+	    std::rethrow_exception (d);
+	  }
+	catch (long int &e)
+	  {
+	    if (e != 2L)
+	      return -1;
+	    try
+	      {
+		std::rethrow_exception (b);
+	      }
+	    catch (const int &f)
+	      {
+		if (f != 1)
+		  return -1;
+		try
+		  {
+		    std::rethrow_exception (d);
+		  }
+		catch (const long int g)
+		  {
+		    if (g != 2L)
+		      return -1;
+		    try
+		      {
+			std::rethrow_exception (b);
+		      }
+		    catch (int h)
+		      {
+			if (h != 1)
+			  return -1;
+			std::exception_ptr i (b);
+			std::exception_ptr j;
+			if (j != nullptr || i == nullptr || i != b || bool (j))
+			  return -1;
+			j = i;
+			if (j != b || !bool (j))
+			  return -1;
+			j = nullptr;
+			std::swap (i, j);
+			if (j == nullptr || j != b || i != nullptr)
+			  return -1;
+		      }
+		  }
+	      }
+	  }
+	return 7;
+      }
+    case 7:
+      {
+	std::exception_ptr a = std::make_exception_ptr (42);
+	std::exception_ptr b = std::make_exception_ptr (std::exception ());
+	std::exception_ptr c
+	  = std::make_exception_ptr (std::bad_array_new_length ());
+	try
+	  {
+	    std::rethrow_exception (a);
+	  }
+	catch (int d)
+	  {
+	    if (d != 42)
+	      return -1;
+	    try
+	      {
+		std::rethrow_exception (b);
+	      }
+	    catch (const std::exception &e)
+	      {
+		const char *f = e.what ();
+		try
+		  {
+		    std::rethrow_exception (c);
+		  }
+		catch (const std::bad_alloc &g)
+		  {
+		    try
+		      {
+			throw;
+		      }
+		    catch (const std::bad_array_new_length &h)
+		      {
+			const char *i = h.what ();
+			const char *j = g.what ();
+		      }
+		  }
+	      }
+	  }
+	return 8;
+      }
+    case 8:
+      {
+	std::nested_exception a;
+	if (a.nested_ptr () != nullptr)
+	  return -1;
+	try
+	  {
+	    std::nested_exception b;
+	    if (b.nested_ptr () != nullptr)
+	      return -1;
+	    throw 42;
+	  }
+	catch (...)
+	  {
+	    std::nested_exception c;
+	    if (c.nested_ptr () != std::current_exception ())
+	      return -1;
+	    std::nested_exception d = c;
+	    if (d.nested_ptr () != c.nested_ptr ())
+	      return -1;
+	    c = d;
+	    try
+	      {
+		c.rethrow_nested ();
+	      }
+	    catch (const int &e)
+	      {
+		if (e != 42)
+		  return -1;
+	      }
+	  }
+	return 9;
+      }
+    case 9:
+      try
+	{
+	  std::throw_with_nested (I ());
+	}
+      catch (const std::nested_exception &a)
+	{
+	  if (a.nested_ptr () != nullptr)
+	    return -1;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (const I &)
+	    {
+	      return 10;
+	    }
+	}
+      return -1;
+    case 10:
+      try
+	{
+	  std::throw_with_nested (J ());
+	}
+      catch (const std::nested_exception &a)
+	{
+	  if (a.nested_ptr () != nullptr)
+	    return -1;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (const J &)
+	    {
+	      return 11;
+	    }
+	}
+      return -1;
+    case 11:
+      try
+	{
+	  std::throw_with_nested (K ());
+	}
+      catch (const std::nested_exception &)
+	{
+	  return -1;
+	}
+      catch (const K &)
+	{
+	  return 12;
+	}
+      return -1;
+    case 12:
+      try
+	{
+	  throw 42;
+	}
+      catch (...)
+	{
+	  I a;
+	  try
+	    {
+	      std::rethrow_if_nested (a);
+	    }
+	  catch (const int &b)
+	    {
+	      if (b == 42)
+		return 13;
+	    }
+	}
+      return -1;
+    case 13:
+      try
+	{
+	  throw J ();
+	}
+      catch (const J &a)
+	{
+	  std::rethrow_if_nested (a);
+	  return 14;
+	}
+      return -1;
+    case 14:
+      try
+	{
+	  throw 42;
+	}
+      catch (...)
+	{
+	  try
+	    {
+	      throw L ();
+	    }
+	  catch (const J &a)
+	    {
+	      try
+		{
+		  std::rethrow_if_nested (a);
+		}
+	      catch (const int &b)
+		{
+		  if (b == 42)
+		    return 15;
+		}
+	    }
+	}
+      return -1;
+    case 15:
+      {
+	std::rethrow_if_nested (1);
+	M m;
+	std::rethrow_if_nested (m);
+	N n;
+	std::rethrow_if_nested (n);
+	O o;
+	std::rethrow_if_nested (o);
+	return 16;
+      }
+    default:
+      break;
+    }
+  return -1;
+}
+
+static_assert (foo (0) == 1);
+static_assert (foo (1) == 2);
+static_assert (foo (2) == 3);
+static_assert (foo (3) == 4);
+static_assert (foo (4) == 5);
+static_assert (foo (5) == 6);
+static_assert (foo (6) == 7);
+static_assert (foo (7) == 8);
+static_assert (foo (8) == 9);
+static_assert (foo (9) == 10);
+static_assert (foo (10) == 11);
+static_assert (foo (11) == 12);
+static_assert (foo (12) == 13);
+static_assert (foo (13) == 14);
+static_assert (foo (14) == 15);
+static_assert (foo (15) == 16);
+static_assert (std::uncaught_exceptions () == 0);
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh4.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh4.C
new file mode 100644
index 0000000..24118ca
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh4.C
@@ -0,0 +1,72 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct A { virtual ~A () {} };
+struct B { virtual void b (); };
+struct C { virtual void c (); };
+struct D : private B { virtual void d (); };
+struct E { virtual void e (); };
+struct F : D, E, private C { virtual void f (); };
+
+constexpr int
+foo (int x)
+{
+  switch (x)
+    {
+    case 1:
+      try
+	{
+	  static constexpr F f;
+	  D &d = dynamic_cast<D &>((B &) f);	// { dg-error "called without 'std::bad_cast' being defined" }
+	  return -1;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    case 3:
+      try
+	{
+	  decltype (sizeof 0) x = -64;
+	  char (*a)[2] = new char[x][2];	// { dg-error "called without 'std::bad_array_new_length' being defined" }
+	  delete[] a;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    case 4:
+      try
+	{
+	  int y = -1;
+	  int *a = new int[y];			// { dg-error "called without 'std::bad_array_new_length' being defined" }
+	  delete[] a;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    case 5:
+      try
+	{
+	  int z = 1;
+	  int *a = new int[z]{1, 2, 3};		// { dg-error "called without 'std::bad_array_new_length' being defined" }
+	  delete[] a;
+	}
+      catch (...)
+	{
+	  return -1;
+	}
+      break;
+    }
+  return -1;
+}
+
+constexpr int a = foo (1);			// { dg-message "in 'constexpr' expansion of" }
+constexpr int b = foo (3);			// { dg-message "in 'constexpr' expansion of" }
+constexpr int c = foo (4);			// { dg-message "in 'constexpr' expansion of" }
+constexpr int d = foo (5);			// { dg-message "in 'constexpr' expansion of" }
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh5.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh5.C
new file mode 100644
index 0000000..512aa34
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh5.C
@@ -0,0 +1,55 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+constexpr void
+foo ()
+{
+  throw 1;
+}
+
+void
+bar ()
+{
+}
+
+constexpr void
+baz ()
+{
+  foo ();
+  bar ();
+}
+
+constexpr void
+qux ()
+{
+  if consteval {
+    throw 2;
+  }
+  bar ();
+}
+
+constexpr bool
+corge ()
+{
+  try
+    {
+      baz ();
+    }
+  catch (int a)
+    {
+      if (a != 1)
+	return false;
+      try
+	{
+	  qux ();
+	}
+      catch (int b)
+	{
+	  return b == 2;
+	}
+    }
+  return false;
+}
+
+static_assert (corge ());
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh6.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh6.C
new file mode 100644
index 0000000..6fd9462
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh6.C
@@ -0,0 +1,134 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+struct S {
+  constexpr S () : s (0) {}
+  constexpr S (int x) : s (x) {}
+  constexpr S (const S &x) : s (x.s) {}
+  constexpr ~S () {}
+  int s;
+};
+struct T {
+  constexpr T () : t (0) {}
+  constexpr T (int x) : t (x) {}
+  constexpr T (const T &x) : t (x.t) {}
+  constexpr ~T () {}
+  int t;
+};
+struct U : public S, public T {
+  constexpr U () : S (0), T (0) {}
+  constexpr U (int x, int y) : S (x), T (y) {}
+  constexpr U (const U &x) : S (x.s), T (x.t) {}
+  constexpr ~U () {}
+};
+
+constexpr bool
+foo ()
+{
+  try
+    {
+      throw U (1, 2);
+    }
+  catch (const U &x)
+    {
+      if (x.s != 1 || x.t != 2)
+	return false;
+      try
+	{
+	  throw;
+	}
+      catch (const S &y)
+	{
+	  if (y.s != 1)
+	    return false;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (const T &z)
+	    {
+	      if (z.t != 2)
+		return false;
+	      return true;
+	    }
+	}
+    }
+  return false;
+}
+
+constexpr bool
+bar ()
+{
+  try
+    {
+      throw U (1, 2);
+    }
+  catch (U &x)
+    {
+      if (x.s != 1 || x.t != 2)
+	return false;
+      try
+	{
+	  x.s = 3;
+	  x.t = 4;
+	  throw;
+	}
+      catch (S &y)
+	{
+	  if (y.s != 3)
+	    return false;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (T &z)
+	    {
+	      if (z.t != 4)
+		return false;
+	      return true;
+	    }
+	}
+    }
+  return false;
+}
+
+constexpr bool
+baz ()
+{
+  try
+    {
+      throw U (1, 2);
+    }
+  catch (U x)
+    {
+      if (x.s != 1 || x.t != 2)
+	return false;
+      try
+	{
+	  x.s = 3;
+	  x.t = 4;
+	  throw;
+	}
+      catch (S y)
+	{
+	  if (y.s != 1)
+	    return false;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (T z)
+	    {
+	      if (z.t != 2)
+		return false;
+	      return true;
+	    }
+	}
+    }
+  return false;
+}
+
+static_assert (foo ());
+static_assert (bar ());
+static_assert (baz ());
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh7.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh7.C
new file mode 100644
index 0000000..6bdf0c3
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh7.C
@@ -0,0 +1,151 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+constexpr char p[] = "hello";
+constexpr const char *q[] = { &p[0], &p[3] };
+constexpr const char *const *r = &q[0];
+const char *s[] = { &p[0], &p[3] };
+constexpr const char **t = &s[0];
+
+constexpr bool
+foo ()
+{
+  try
+    {
+      throw t;
+    }
+  catch (const char **const &x)
+    {
+      if (x != t)
+	return false;
+      try
+	{
+	  throw;
+	}
+      catch (const char **&y)
+	{
+	  if (y != t)
+	    return false;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (const char **z)
+	    {
+	      if (z != t)
+		return false;
+	      try
+		{
+		  throw;
+		}
+	      catch (const char *const *const &v)
+		{
+		  if (v != (const char *const *) t)
+		    return false;
+		  try
+		    {
+		      throw;
+		    }
+		  catch (const char *const *w)
+		    {
+		      if (w != (const char *const *) t)
+			return false;
+		      return true;
+		    }
+		}
+	    }
+	}
+    }
+  return false;
+}
+
+constexpr bool
+bar ()
+{
+  try
+    {
+      throw nullptr;
+    }
+  catch (const char **const &x)
+    {
+      if (x != nullptr)
+	return false;
+      try
+	{
+	  throw;
+	}
+      catch (const char **&y)
+	{
+	  if (y != nullptr)
+	    return false;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (const char **z)
+	    {
+	      if (z != nullptr)
+		return false;
+	      try
+		{
+		  throw;
+		}
+	      catch (const char *const *const &v)
+		{
+		  if (v != nullptr)
+		    return false;
+		  try
+		    {
+		      throw;
+		    }
+		  catch (const char *const *w)
+		    {
+		      if (w != nullptr)
+			return false;
+		      return true;
+		    }
+		}
+	    }
+	}
+    }
+  return false;
+}
+
+constexpr bool
+baz ()
+{
+  try
+    {
+      throw r;
+    }
+  catch (const char *const *const &x)
+    {
+      if (x != r || **x != 'h')
+	return false;
+      try
+	{
+	  throw;
+	}
+      catch (const char *const *&y)
+	{
+	  if (y != r || **y != 'h')
+	    return false;
+	  try
+	    {
+	      throw;
+	    }
+	  catch (const char *const *z)
+	    {
+	      if (z != r || **z != 'h')
+		return false;
+	      return true;
+	    }
+	}
+    }
+  return false;
+}
+
+static_assert (foo ());
+static_assert (bar ());
+static_assert (baz ());
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh8.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh8.C
new file mode 100644
index 0000000..2560364
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh8.C
@@ -0,0 +1,36 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+#include <exception>
+
+constexpr std::exception_ptr
+foo ()
+{
+  try
+    {
+      throw 42;
+    }
+  catch (...)
+    {
+      return std::current_exception ();
+    }
+}
+
+constexpr bool
+bar ()
+{
+  try
+    {
+      std::rethrow_exception (foo ());
+    }
+  catch (const int &a)
+    {
+       return a == 42;
+    }
+  return false;
+}
+
+static_assert (bar ());
+constexpr std::exception_ptr a = foo ();				// { dg-error "is not a constant expression because it refers to exception object allocated with '__cxa_allocate_exception'" }
+constexpr std::exception_ptr b = std::make_exception_ptr (42ULL);	// { dg-error "is not a constant expression because it refers to exception object allocated with '__cxa_allocate_exception'" }
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-eh9.C b/gcc/testsuite/g++.dg/cpp26/constexpr-eh9.C
new file mode 100644
index 0000000..fd39fe0
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-eh9.C
@@ -0,0 +1,127 @@
+// C++26 P3068R5 - Allowing exception throwing in constant-evaluation
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target exceptions_enabled }
+
+namespace std
+{
+  struct exception
+  {
+    constexpr exception () noexcept { }
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr exception (exception &&) = default;
+    constexpr exception &operator= (exception &&) = default;
+    constexpr virtual const char *what () const noexcept
+    { return "std::exception"; }
+  };
+}
+
+struct S : public std::exception {
+  constexpr S () : s (0) {}
+  constexpr S (int x) : s (x) {}
+  constexpr S (const S &x) : s (x.s) {}
+  constexpr virtual ~S () {}
+  constexpr virtual const char *what () noexcept { return "this is S"; }
+  int s;
+};
+struct T : public std::exception {
+  constexpr T () : s (new char[1]), t (0) { s[0] = '\0'; }
+  constexpr T (const char *p, int q) : s (new char[q + 1]), t (q)
+  {
+    for (int i = 0; i <= t; ++i)
+      s[i] = p[i];
+  }
+  constexpr T (const T &x) : s (new char[x.t + 1]), t (x.t)
+  {
+    for (int i = 0; i <= t; ++i)
+      s[i] = x.s[i];
+  }
+  constexpr virtual ~T () { delete[] s; }
+  constexpr virtual const char *what () noexcept { return s; }
+  char *s;
+  int t;
+};
+struct U {
+  constexpr U () : x (0), y (0), z (0) {}
+  constexpr U (int a, long b, unsigned long long c) : x (a), y (b), z (c) {}
+  constexpr U (const U &u) = default;
+  int x;
+  long y;
+  unsigned long long z;
+};
+struct V {
+  constexpr V () : v (0) {}
+  constexpr V (int x) : v (x) {}
+  constexpr V (const V &x) : v (x.v) {}
+  constexpr virtual ~V () {}
+  constexpr virtual const char *what () noexcept { return "this is V"; }
+  int v;
+};
+
+constexpr int
+foo (int x)
+{
+  if (x == 1)
+    throw S (42);
+  else if (x == 2)
+    throw T ("hello, world", sizeof ("hello, world") - 1);
+  else if (x == 3)
+    throw U (1, -2L, 42ULL);
+  else if (x == 4)
+    throw 42;
+  else if (x == 5)
+    throw 1.0;
+  else if (x == 6)
+    throw V (42);
+  else
+    return 42;
+}
+
+constexpr int
+bar (int x) noexcept
+// { dg-error "'std::terminate' called after throwing an exception of type 'S'; 'what\\\(\\\)': 'this is S'" "" { target *-*-* } .-1 }
+// { dg-message "uncaught exception exited from 'noexcept' function 'constexpr int bar\\\(int\\\)'" "" { target *-*-* } .-2 }
+// { dg-error "'std::terminate' called after throwing an exception of type 'T'; 'what\\\(\\\)': 'hello, world'" "" { target *-*-* } .-3 }
+// { dg-error "'std::terminate' called after throwing an exception 'U\\\{1, -2, 42\\\}'" "" { target *-*-* } .-4 }
+// { dg-error "'std::terminate' called after throwing an exception '42'" "" { target *-*-* } .-5 }
+// { dg-error "'std::terminate' called after throwing an exception '1\\\.0e\\\+0'" "" { target *-*-* } .-6 }
+// { dg-error "'std::terminate' called after throwing an exception 'V\\\{\[^\n\r]*42\\\}" "" { target *-*-* } .-7 }
+{
+  return foo (x);
+}
+
+constexpr int
+baz (int x)
+{
+  try
+    {
+      return foo (x);
+    }
+  catch (...)
+    {
+      return -1;
+    }
+}
+
+static_assert (bar (0) == 42);
+constexpr int a = bar (1);		// { dg-message "in 'constexpr' expansion of" }
+constexpr int b = bar (2);		// { dg-message "in 'constexpr' expansion of" }
+constexpr int c = bar (3);		// { dg-message "in 'constexpr' expansion of" }
+constexpr int d = bar (4);		// { dg-message "in 'constexpr' expansion of" }
+constexpr int e = bar (5);		// { dg-message "in 'constexpr' expansion of" }
+constexpr int f = bar (6);		// { dg-message "in 'constexpr' expansion of" }
+static_assert (baz (0) == 42);
+static_assert (baz (1) == -1);
+static_assert (baz (2) == -1);
+static_assert (baz (3) == -1);
+static_assert (baz (4) == -1);
+static_assert (baz (5) == -1);
+static_assert (baz (6) == -1);
+static_assert (foo (0) == 42);
+constexpr int g = foo (1);		// { dg-error "uncaught exception of type 'S'; 'what\\\(\\\)': 'this is S'" }
+constexpr int h = foo (2);		// { dg-error "uncaught exception of type 'T'; 'what\\\(\\\)': 'hello, world'" }
+constexpr int i = foo (3);		// { dg-error "uncaught exception 'U\\\{1, -2, 42\\\}'" }
+constexpr int j = foo (4);		// { dg-error "uncaught exception '42'" }
+constexpr int k = foo (5);		// { dg-error "uncaught exception '1\\\.0e\\\+0'" }
+constexpr int l = foo (6);		// { dg-error "uncaught exception 'V\\\{\[^\n\r]*42\\\}'" }
diff --git a/gcc/testsuite/g++.dg/cpp26/constexpr-new4.C b/gcc/testsuite/g++.dg/cpp26/constexpr-new4.C
new file mode 100644
index 0000000..12d8a46
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/constexpr-new4.C
@@ -0,0 +1,21 @@
+// PR c++/121068
+// { dg-do compile { target c++26 } }
+
+constexpr void *operator new (__SIZE_TYPE__, void *p) { return p; }
+constexpr void *operator new[] (__SIZE_TYPE__, void *p) { return p; }
+
+consteval int
+foo()
+{
+    using T = int;
+    union { T arr[3]; };
+    new(arr) T[3]; // makes arr active
+    for (int i = 0; i < 3; ++i)
+      arr[i].~T();
+
+    new (arr + 2) T{10}; // A
+
+    return 1;
+};
+
+constexpr int g = foo();
diff --git a/gcc/testsuite/g++.dg/cpp26/decomp22.C b/gcc/testsuite/g++.dg/cpp26/decomp22.C
new file mode 100644
index 0000000..576a93b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/decomp22.C
@@ -0,0 +1,66 @@
+// C++26 P2686R4 - constexpr structured bindings
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+
+namespace std {
+  template <typename T> struct tuple_size;
+  template <int, typename> struct tuple_element;
+}
+
+struct A {
+  int i, j;
+  template <int I> constexpr const int &get () const { return I == 1 ? j : i; }
+};
+
+template <> struct std::tuple_size <const A> { static const int value = 3; };
+template <int I> struct std::tuple_element <I, const A> { using type = const int; };
+
+constexpr struct B {
+  int i, j;
+  long long k, l;
+} a[3] = { { 1, 2, 3, 4 }, { 5, 6, 7, 8 }, { 9, 10, 11, 12 } };
+
+constexpr auto [ b, c, d ] = a;				// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto [ e, f, g ] = a;				// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+constexpr auto [ h, i, j, k ] = a[1];			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto [ l, m, n, o ] = a[2];			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+constexpr auto & [ p, q, r ] = a;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto & [ s, t, u ] = a;			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+constexpr auto & [ v, w, x, y ] = a[1];			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto & [ aa, ab, ac, ad ] = a[2];		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+static_assert (b.i == 1 && b.l == 4 && c.j == 6 && c.k == 7 && d.i == 9 && d.k == 11, "");
+static_assert (h == 5 && i == 6 && j == 7 && k == 8, "");
+static_assert (p.i == 1 && p.l == 4 && q.j == 6 && q.k == 7 && r.i == 9 && r.k == 11, "");
+static_assert (&p.i == &a[0].i && &p.l == &a[0].l && &q.j == &a[1].j, "");
+static_assert (&q.k == &a[1].k && &r.i == &a[2].i && &r.k == &a[2].k, "");
+static_assert (v == 5 && w == 6 && x == 7 && y == 8, "");
+static_assert (&v == &a[1].i && &w == &a[1].j && &x == &a[1].k && &y == &a[1].l, "");
+
+constexpr A z = { 42, -42 };
+constexpr auto [ ae, af, ag ] = z;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit const auto [ ah, ai, aj ] = z;		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+constexpr auto & [ ak, al, am ] = z;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto & [ an, ao, ap ] = z;			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+static_assert (ae == 42 && af == -42 && ag == 42, "");
+static_assert (&af == &ae + 1 && &ag == &ae, "");
+static_assert (&ae != &z.i && &af != &z.j && &ag != &z.i, "");
+static_assert (ak == 42 && al == -42 && am == 42, "");
+static_assert (&ak == &z.i && &al == &z.j && &am == &z.i, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/decomp23.C b/gcc/testsuite/g++.dg/cpp26/decomp23.C
new file mode 100644
index 0000000..ad2f7e0
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/decomp23.C
@@ -0,0 +1,77 @@
+// C++26 P2686R4 - constexpr structured bindings
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+
+namespace std {
+  template <typename T> struct tuple_size;
+  template <int, typename> struct tuple_element;
+}
+
+struct A {
+  int i, j;
+  template <int I> constexpr const int &get () const { return I == 1 ? j : i; }
+};
+
+template <> struct std::tuple_size <const A> { static const int value = 3; };
+template <int I> struct std::tuple_element <I, const A> { using type = const int; };
+
+struct B {
+  int i, j;
+  long long k, l;
+};
+
+void
+foo ()
+{
+  static constexpr B a[3] = { { 1, 2, 3, 4 }, { 5, 6, 7, 8 }, { 9, 10, 11, 12 } };
+  static constexpr auto [ b, c, d ] = a;		// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
+#if __cpp_constinit >= 201907
+  static constinit auto [ e, f, g ] = a;		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+  static constexpr auto [ h, i, j, k ] = a[1];		// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
+#if __cpp_constinit >= 201907
+  static constinit auto [ l, m, n, o ] = a[2];		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+  static constexpr auto & [ p, q, r ] = a;		// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
+#if __cpp_constinit >= 201907
+  static constinit auto & [ s, t, u ] = a;		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+  static constexpr auto & [ v, w, x, y ] = a[1];	// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
+#if __cpp_constinit >= 201907
+  static constinit auto & [ aa, ab, ac, ad ] = a[2];	// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+  static_assert (b.i == 1 && b.l == 4 && c.j == 6 && c.k == 7 && d.i == 9 && d.k == 11, "");
+  static_assert (h == 5 && i == 6 && j == 7 && k == 8, "");
+  static_assert (p.i == 1 && p.l == 4 && q.j == 6 && q.k == 7 && r.i == 9 && r.k == 11, "");
+  static_assert (&p.i == &a[0].i && &p.l == &a[0].l && &q.j == &a[1].j, "");
+  static_assert (&q.k == &a[1].k && &r.i == &a[2].i && &r.k == &a[2].k, "");
+  static_assert (v == 5 && w == 6 && x == 7 && y == 8, "");
+  static_assert (&v == &a[1].i && &w == &a[1].j && &x == &a[1].k && &y == &a[1].l, "");
+
+  static constexpr A z = { 42, -42 };
+  static constexpr auto [ ae, af, ag ] = z;		// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
+#if __cpp_constinit >= 201907
+  static constinit const auto [ ah, ai, aj ] = z;	// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+  static constexpr auto & [ ak, al, am ] = z;		// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
+#if __cpp_constinit >= 201907
+  static constinit auto & [ an, ao, ap ] = z;		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+  static_assert (ae == 42 && af == -42 && ag == 42, "");
+  static_assert (&af == &ae + 1 && &ag == &ae, "");
+  static_assert (&ae != &z.i && &af != &z.j && &ag != &z.i, "");
+  static_assert (ak == 42 && al == -42 && am == 42, "");
+  static_assert (&ak == &z.i && &al == &z.j && &am == &z.i, "");
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/decomp24.C b/gcc/testsuite/g++.dg/cpp26/decomp24.C
new file mode 100644
index 0000000..5da1321
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/decomp24.C
@@ -0,0 +1,20 @@
+// C++26 P2686R4 - constexpr structured bindings
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+
+struct B {
+  int i, j;
+  long long k, l;
+};
+
+void
+foo ()
+{
+  constexpr B a[3] = { { 1, 2, 3, 4 }, { 5, 6, 7, 8 }, { 9, 10, 11, 12 } };
+  constexpr auto [ b, c, d ] = a;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+  constexpr auto [ h, i, j, k ] = a[1];			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+  static_assert (b.i == 1 && b.l == 4 && c.j == 6 && c.k == 7 && d.i == 9 && d.k == 11, "");
+  static_assert (h == 5 && i == 6 && j == 7 && k == 8, "");
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/decomp25.C b/gcc/testsuite/g++.dg/cpp26/decomp25.C
new file mode 100644
index 0000000..f395685
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/decomp25.C
@@ -0,0 +1,119 @@
+// C++26 P2686R4 - constexpr structured bindings
+// { dg-do compile { target c++11 } }
+// { dg-options "-fno-implicit-constexpr" }
+
+namespace std {
+  template <typename T> struct tuple_size;
+  template <int, typename> struct tuple_element;
+}
+
+struct A {
+  int i, j;
+  template <int I> int &get () { return I == 1 ? j : i; }
+};
+
+template <> struct std::tuple_size <A> { static const int value = 3; };
+template <int I> struct std::tuple_element <I, A> { using type = int; };
+template <> struct std::tuple_size <const A> { static const int value = 3; };
+template <int I> struct std::tuple_element <I, const A> { using type = int; };
+
+struct B {
+  int i, j;
+  long long k, l;
+} a[3] = { { 1, 2, 3, 4 }, { 5, 6, 7, 8 }, { 9, 10, 11, 12 } };	// { dg-message "'a' was not declared 'constexpr'" "" }
+
+struct C {
+  int i, j;
+  template <int I> const int &get () const { return I == 1 ? j : i; }
+};
+
+template <> struct std::tuple_size <const C> { static const int value = 3; };
+template <int I> struct std::tuple_element <I, const C> { using type = const int; };
+
+constexpr auto [ b, c, d ] = a;				// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-error "the value of 'a' is not usable in a constant expression" "" { target *-*-* } .-2 }
+#if __cpp_constinit >= 201907
+constinit auto [ e, f, g ] = a;				// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+							// { dg-error "'constinit' variable '<structured bindings>' does not have a constant initializer" "" { target c++20 } .-1 }
+							// { dg-error "the value of 'a' is not usable in a constant expression" "" { target c++20 } .-2 }
+#endif
+constexpr auto [ h, i, j, k ] = a[1];			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-error "the value of 'a' is not usable in a constant expression" "" { target *-*-* } .-2 }
+#if __cpp_constinit >= 201907
+constinit auto [ l, m, n, o ] = a[2];			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+							// { dg-error "'constinit' variable '<structured bindings>' does not have a constant initializer" "" { target c++20 } .-1 }
+							// { dg-error "the value of 'a' is not usable in a constant expression" "" { target c++20 } .-2 }
+#endif
+constexpr auto & [ p, q, r ] = a;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto & [ s, t, u ] = a;			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+constexpr auto & [ v, w, x, y ] = a[1];			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+#if __cpp_constinit >= 201907
+constinit auto & [ aa, ab, ac, ad ] = a[2];		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+#endif
+
+A z = { 42, -42 };					// { dg-message "'z' was not declared 'constexpr'" "" }
+constexpr auto [ ae, af, ag ] = z;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-error "the value of 'z' is not usable in a constant expression" "" { target *-*-* } .-2 }
+							// { dg-error "passing 'const A' as 'this' argument discards qualifiers" "" { target *-*-* } .-3 }
+							// { dg-error "call to non-'constexpr' function 'int\\\& A::get\\\(\\\)" "" { target *-*-* } .-4 }
+#if __cpp_constinit >= 201907
+constinit const auto [ ah, ai, aj ] = z;		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+							// { dg-error "'constinit' variable '<structured bindings>' does not have a constant initializer" "" { target c++20 } .-1 }
+							// { dg-error "the value of 'z' is not usable in a constant expression" "" { target c++20 } .-2 }
+							// { dg-error "passing 'const A' as 'this' argument discards qualifiers" "" { target c++20 } .-3 }
+							// { dg-error "call to non-'constexpr' function 'int\\\& A::get\\\(\\\)" "" { target c++20 } .-4 }
+							// { dg-error "'constinit' variable 'ah' does not have a constant initializer" "" { target c++20 } .-5 }
+							// { dg-error "'constinit' variable 'ai' does not have a constant initializer" "" { target c++20 } .-6 }
+							// { dg-error "'constinit' variable 'aj' does not have a constant initializer" "" { target c++20 } .-7 }
+#endif
+constexpr auto & [ ak, al, am ] = z;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-error "call to non-'constexpr' function 'int\\\& A::get\\\(\\\)" "" { target *-*-* } .-2 }
+#if __cpp_constinit >= 201907
+constinit auto & [ an, ao, ap ] = z;			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+							// { dg-error "'constinit' variable 'an' does not have a constant initializer" "" { target c++20 } .-1 }
+							// { dg-error "'constinit' variable 'ao' does not have a constant initializer" "" { target c++20 } .-2 }
+							// { dg-error "'constinit' variable 'ap' does not have a constant initializer" "" { target c++20 } .-3 }
+							// { dg-message "call to non-'constexpr' function 'int\\\& A::get\\\(\\\)" "" { target c++20 } .-4 }
+#endif
+
+constexpr C zz = { 42, -42 };
+constexpr auto [ aq, ar, as ] = zz;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-error "call to non-'constexpr' function 'const int\\\& C::get\\\(\\\) const" "" { target *-*-* } .-2 }
+#if __cpp_constinit >= 201907
+constinit const auto [ at, au, av ] = zz;		// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+							// { dg-error "call to non-'constexpr' function 'const int\\\& C::get\\\(\\\) const" "" { target c++20 } .-1 }
+							// { dg-error "'constinit' variable 'at' does not have a constant initializer" "" { target c++20 } .-2 }
+							// { dg-error "'constinit' variable 'au' does not have a constant initializer" "" { target c++20 } .-3 }
+							// { dg-error "'constinit' variable 'av' does not have a constant initializer" "" { target c++20 } .-4 }
+#endif
+constexpr auto & [ aw, ax, ay ] = zz;			// { dg-warning "structured bindings only available with" "" { target c++14_down } }
+							// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
+							// { dg-error "call to non-'constexpr' function 'const int\\\& C::get\\\(\\\) const" "" { target *-*-* } .-2 }
+#if __cpp_constinit >= 201907
+constinit auto & [ az, ba, bb ] = zz;			// { dg-warning "'constinit' can be applied to structured binding only with" "" { target { c++20 && c++23_down } } }
+							// { dg-error "'constinit' variable 'az' does not have a constant initializer" "" { target c++20 } .-1 }
+							// { dg-error "'constinit' variable 'ba' does not have a constant initializer" "" { target c++20 } .-2 }
+							// { dg-error "'constinit' variable 'bb' does not have a constant initializer" "" { target c++20 } .-3 }
+							// { dg-message "call to non-'constexpr' function 'const int\\\& C::get\\\(\\\) const" "" { target c++20 } .-4 }
+#endif
+
+void
+foo ()
+{
+#if __cpp_constinit >= 201907
+  constexpr B a[3] = { { 1, 2, 3, 4 }, { 5, 6, 7, 8 }, { 9, 10, 11, 12 } };
+  constinit auto [ b, c, d ] = a;			// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+  constinit auto & [ e, f, g ] = a;			// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+  constinit auto [ h, i, j, k ] = a[1];			// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+  constinit auto & [ l, m, n, o ] = a[2];		// { dg-error "'constinit' can only be applied to a variable with static or thread storage duration" "" { target c++20 } }
+#endif
+}
diff --git a/gcc/testsuite/g++.dg/cpp26/decomp9.C b/gcc/testsuite/g++.dg/cpp26/decomp9.C
index 5629c4c..ee18b60 100644
--- a/gcc/testsuite/g++.dg/cpp26/decomp9.C
+++ b/gcc/testsuite/g++.dg/cpp26/decomp9.C
@@ -63,6 +63,7 @@ foo (const S &&s)
   if (static auto [i, j, k] = t)		// { dg-warning "structured bindings in conditions only available with" "" { target c++23_down } }
     ;						// { dg-error "'static' invalid in condition" "" { target *-*-* } .-1 }
 						// { dg-warning "structured binding declaration can be 'static' only in" "" { target c++17_down } .-2 }
-  if (constexpr auto [i, j, k] = t)		// { dg-warning "structured bindings in conditions only available with" "" { target c++23_down } }
-    ;						// { dg-error "structured binding declaration cannot be 'constexpr'" "" { target *-*-* } .-1 }
+  constexpr T t2 = { 1, 2, 3 };
+  if (constexpr auto [i, j, k] = t2)		// { dg-warning "structured bindings in conditions only available with" "" { target c++23_down } }
+    ;						// { dg-warning "structured binding declaration can be 'constexpr' only with" "" { target c++23_down } .-1 }
 }
diff --git a/gcc/testsuite/g++.dg/cpp26/feat-cxx26.C b/gcc/testsuite/g++.dg/cpp26/feat-cxx26.C
index e4ffc35..cfc5f61 100644
--- a/gcc/testsuite/g++.dg/cpp26/feat-cxx26.C
+++ b/gcc/testsuite/g++.dg/cpp26/feat-cxx26.C
@@ -624,19 +624,31 @@
 #endif
 
 #ifndef __cpp_pack_indexing
-# error "__cpp_pack_indexing"
+#  error "__cpp_pack_indexing"
 #elif __cpp_pack_indexing != 202311
 #  error "__cpp_pack_indexing != 202311"
 #endif
 
 #ifndef __cpp_pp_embed
-# error "__cpp_pp_embed"
+#  error "__cpp_pp_embed"
 #elif __cpp_pp_embed != 202502
 #  error "__cpp_pp_embed != 202502"
 #endif
 
 #ifndef __cpp_constexpr_virtual_inheritance
-# error "__cpp_constexpr_virtual_inheritance"
+#  error "__cpp_constexpr_virtual_inheritance"
 #elif __cpp_constexpr_virtual_inheritance != 202506
 #  error "__cpp_constexpr_virtual_inheritance != 202506"
 #endif
+
+#ifndef __cpp_constexpr_exceptions
+#  error "__cpp_constexpr_exceptions"
+#elif __cpp_constexpr_exceptions != 202411
+#  error "__cpp_constexpr_exceptions != 202411"
+#endif
+
+#ifndef __cpp_trivial_relocatability
+#  error "__cpp_trivial_relocatability"
+#elif __cpp_trivial_relocatability != 202502
+#  error "__cpp_trivial_relocatability != 202502"
+#endif
diff --git a/gcc/testsuite/g++.dg/cpp26/name-independent-decl1.C b/gcc/testsuite/g++.dg/cpp26/name-independent-decl1.C
index 0830ce8..9b56e84 100644
--- a/gcc/testsuite/g++.dg/cpp26/name-independent-decl1.C
+++ b/gcc/testsuite/g++.dg/cpp26/name-independent-decl1.C
@@ -70,7 +70,7 @@ foo ()
       ++_;
     }
     {
-      static int _ = 3;
+      static int _ = 3;		// { dg-warning "variable '_' set but not used" }
       ++_;
     }
     {
diff --git a/gcc/testsuite/g++.dg/cpp26/static_assert1.C b/gcc/testsuite/g++.dg/cpp26/static_assert1.C
index 1d0e6f2..a6eab3c 100644
--- a/gcc/testsuite/g++.dg/cpp26/static_assert1.C
+++ b/gcc/testsuite/g++.dg/cpp26/static_assert1.C
@@ -51,7 +51,7 @@ static_assert (true, H {});	// { dg-warning "'static_assert' with non-string mes
 struct I { constexpr signed char size () const { return 0; }
 	   const char *data () const { return ""; } };
 static_assert (true, I {});	// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_down } }
-struct J { constexpr int size () const { return j ? throw 1 : 0; }	// { dg-error "expression '<throw-expression>' is not a constant expression" }
+struct J { constexpr int size () const { return j ? throw 1 : 0; }	// { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23_down } }
 	   constexpr const char *data () const { return ""; };
 	   constexpr J (int x) : j (x) {}
 	   int j; };
@@ -60,6 +60,7 @@ static_assert (false, J (0));	// { dg-warning "'static_assert' with non-string m
 				// { dg-error "static assertion failed" "" { target *-*-* } .-1 }
 static_assert (false, J (1));	// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_down } }
 				// { dg-error "constexpr string 'size\\\(\\\)' must be a constant expression" "" { target *-*-* } .-1 }
+				// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
 struct K { constexpr operator int () { return 4; } };
 struct L { constexpr operator const char * () { return "test"; } };
 struct M { constexpr K size () const { return {}; }
@@ -261,10 +262,11 @@ namespace NN
 #if __cplusplus >= 201402L
   struct J {
     static constexpr int size () { return 0; }
-    static constexpr const char *data (int x = 0) { if (x) return nullptr; else throw 1; } }; // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++14 } }
+    static constexpr const char *data (int x = 0) { if (x) return nullptr; else throw 1; } }; // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target { c++14 && c++23_down } } }
   static_assert (true, J{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target { c++14 && c++23_down } } }
   static_assert (false, J{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target { c++14 && c++23_down } } }
 					// { dg-error "constexpr string 'data\\\(\\\)' must be a core constant expression" "" { target c++14 } .-1 }
+					// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
 #endif
 #if __cpp_if_consteval >= 202106L
   struct K {
@@ -282,19 +284,21 @@ namespace NN
   static_assert (false, L{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_only } }
 					// { dg-error "static assertion failed: test" "" { target c++23 } .-1 }
   struct M {
-    static constexpr int size () { if consteval { throw 1; } else { return 4; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23 } }
+    static constexpr int size () { if consteval { throw 1; } else { return 4; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23_only } }
     static constexpr const char *data () { return "test"; }
   };
   static_assert (true, M{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_only } }
   static_assert (false, M{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_only } }
 					// { dg-error "constexpr string 'size\\\(\\\)' must be a constant expression" "" { target c++23 } .-1 }
+					// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
   struct N {
     static constexpr int size () { return 4; }
-    static constexpr const char *data () { if consteval { throw 1; } else { return "test"; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23 } }
+    static constexpr const char *data () { if consteval { throw 1; } else { return "test"; } } // { dg-error "expression '<throw-expression>' is not a constant expression" "" { target c++23_only } }
   };
   static_assert (true, N{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_only } }
   static_assert (false, N{});		// { dg-warning "'static_assert' with non-string message only available with" "" { target c++23_only } }
 					// { dg-error "constexpr string 'data\\\(\\\)\\\[0\\\]' must be a constant expression" "" { target c++23 } .-1 }
+					// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
 #endif
   struct O { constexpr int operator () () const { return 12; } };
   struct P { constexpr const char *operator () () const { return "another test"; } };
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable1.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable1.C
new file mode 100644
index 0000000..29ba907
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable1.C
@@ -0,0 +1,137 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+struct A {};
+
+static_assert (std::is_trivially_relocatable_v <A>, "");
+static_assert (std::is_replaceable_v <A>, "");
+
+struct B {
+  B ();
+  ~B ();
+  B (const B &);
+  B (B &&);
+  B &operator= (const B &);
+  B &operator= (B &&);
+};
+
+static_assert (!std::is_trivially_relocatable_v <B>, "");
+static_assert (!std::is_replaceable_v <B>, "");
+
+struct C {
+  C (C &&) = delete;
+  C &operator= (C &&) = delete;
+  C () = default;
+};
+
+// Note, P2786R13 says it is trivially relocatable, but I think
+// it isn't default-movable because overload resolution in both
+// cases selects a deleted special member fn.
+static_assert (!std::is_trivially_relocatable_v <C>, "");
+static_assert (!std::is_replaceable_v <C>, "");
+
+struct D : A {};
+
+static_assert (std::is_trivially_relocatable_v <D>, "");
+static_assert (std::is_replaceable_v <D>, "");
+
+struct E : virtual A {};
+
+static_assert (!std::is_trivially_relocatable_v <E>, "");
+static_assert (std::is_replaceable_v <E>, "");
+
+struct F trivially_relocatable_if_eligible : virtual A {};
+
+static_assert (!std::is_trivially_relocatable_v <F>, "");
+static_assert (std::is_replaceable_v <F>, "");
+
+struct G { B data; };
+
+static_assert (!std::is_trivially_relocatable_v <G>, "");
+static_assert (!std::is_replaceable_v <G>, "");
+
+struct H { ~H () = default; };
+
+static_assert (std::is_trivially_relocatable_v <H>, "");
+static_assert (std::is_replaceable_v <H>, "");
+
+struct I { ~I (); };
+I::~I () = default;
+
+static_assert (!std::is_trivially_relocatable_v <I>, "");
+static_assert (!std::is_replaceable_v <I>, "");
+
+struct J { virtual ~J () = default; };
+
+// Note, P2786R13 says otherwise for both, but that looks like
+// a bug in the paper, it otherwise says that polymorphic types
+// can be both trivially relocatable and replaceable.
+static_assert (std::is_trivially_relocatable_v <J>, "");
+static_assert (std::is_replaceable_v <J>, "");
+
+struct K { ~K () = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (!std::is_replaceable_v <K>, "");
+
+struct L { L (L &&) = default; };
+
+// Note, P2786R13 says otherwise for both, but that looks like
+// a bug in the paper to me.  While move ctor is trivial here,
+// copy assignment operator is implicitly declared as deleted
+// and move assignent operator is not declared.
+static_assert (!std::is_trivially_relocatable_v <L>, "");
+static_assert (!std::is_replaceable_v <L>, "");
+
+struct M { M (M &&); };
+M::M (M &&) = default;
+
+static_assert (!std::is_trivially_relocatable_v <M>, "");
+static_assert (!std::is_replaceable_v <M>, "");
+
+struct N { N (N &&) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <N>, "");
+static_assert (!std::is_replaceable_v <N>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable10.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable10.C
new file mode 100644
index 0000000..2ed1d5e
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable10.C
@@ -0,0 +1,135 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+struct A replaceable_if_eligible { A (A &&); A &operator= (A &&); ~A (); int a; };
+
+static_assert (std::is_replaceable_v <A>, "");
+static_assert (!std::is_replaceable_v <const A>, "");
+static_assert (!std::is_replaceable_v <A volatile>, "");
+static_assert (!std::is_replaceable_v <const A volatile>, "");
+
+struct B { B (B &&); B &operator= (B &&); ~B (); int a; };
+
+static_assert (!std::is_replaceable_v <B>, "");
+
+struct C replaceable_if_eligible : public A { C (C &&); C &operator= (C &&); ~C (); int a; };
+
+static_assert (std::is_replaceable_v <C>, "");
+
+struct D replaceable_if_eligible : public B { D (D &&); D &operator= (D &&); ~D (); int a; };
+
+static_assert (!std::is_replaceable_v <D>, "");
+
+struct E replaceable_if_eligible { E (E &&); E &operator= (E &&); ~E (); A a; };
+
+static_assert (std::is_replaceable_v <E>, "");
+
+struct F replaceable_if_eligible { F (F &&); F &operator= (F &&); ~F (); B a; };
+
+static_assert (!std::is_replaceable_v <F>, "");
+
+struct G replaceable_if_eligible { G (G &&); G &operator= (G &&); ~G () = delete; int a; };
+
+static_assert (!std::is_replaceable_v <G>, "");
+
+struct H replaceable_if_eligible : virtual A { H (H &&); H &operator= (H &&); ~H (); int a; };
+
+static_assert (std::is_replaceable_v <H>, "");
+
+struct I replaceable_if_eligible { I (I &&) = delete; I &operator= (I &&); ~I (); int a; };
+
+static_assert (!std::is_replaceable_v <I>, "");
+
+struct J replaceable_if_eligible { J (J &&); J &operator= (J &&) = delete; ~J (); int a; };
+
+static_assert (!std::is_replaceable_v <J>, "");
+
+struct K replaceable_if_eligible { K (const K &) = delete; K &operator= (K &&); ~K (); int a; };
+
+static_assert (!std::is_replaceable_v <K>, "");
+
+struct L replaceable_if_eligible { L (L &&); L &operator= (const L &) = delete; ~L (); int a; };
+
+static_assert (!std::is_replaceable_v <L>, "");
+
+struct M replaceable_if_eligible { M (); private: M (M &&); M &operator= (M &&); ~M (); int a; };
+
+static_assert (std::is_replaceable_v <M>, "");
+
+struct N replaceable_if_eligible { N (N &&); N &operator= (N &&); ~N (); const A a; };
+
+static_assert (!std::is_replaceable_v <N>, "");
+
+struct O replaceable_if_eligible { O (O &&); O &operator= (O &&); ~O (); volatile A a; };
+
+static_assert (!std::is_replaceable_v <O>, "");
+
+struct P replaceable_if_eligible { P (P &&); P &operator= (P &&); ~P (); const volatile A a; };
+
+static_assert (!std::is_replaceable_v <P>, "");
+
+struct Q replaceable_if_eligible { Q (Q &&); Q &operator= (Q &&); ~Q (); union { A a; int b; char c; }; };
+
+static_assert (!std::is_replaceable_v <Q>, "");
+
+struct R replaceable_if_eligible { R (R &&); R &operator= (R &&); ~R (); union { int a; B b; short c; }; };
+
+static_assert (!std::is_replaceable_v <R>, "");
+
+struct S replaceable_if_eligible { S (S &&); S &operator= (S &&); ~S (); union { int a; const int b; short c; }; };
+
+static_assert (!std::is_replaceable_v <S>, "");
+
+struct T replaceable_if_eligible { T (T &&); T &operator= (T &&); ~T () = default; int a; };
+
+static_assert (std::is_replaceable_v <T>, "");
+
+struct U replaceable_if_eligible { U (U &&); U &operator= (U &&); ~U (); union { T a; int b; char c; }; };
+
+static_assert (!std::is_replaceable_v <U>, "");
+
+struct V replaceable_if_eligible { V (V &&); V &operator= (V &&); ~V (); union { unsigned long long a; int b; char c; }; };
+
+static_assert (std::is_replaceable_v <V>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable11.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable11.C
new file mode 100644
index 0000000..71fc6f1
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable11.C
@@ -0,0 +1,134 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+struct A { A (A &&) = default; A &operator= (A &&) = default; ~A () = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <A>, "");
+static_assert (std::is_replaceable_v <A>, "");
+
+struct B { B (B &&); B &operator= (B &&) = default; ~B () = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <B>, "");
+static_assert (!std::is_replaceable_v <B>, "");
+
+union C { int a; A b; };
+
+static_assert (std::is_trivially_relocatable_v <C>, "");
+static_assert (std::is_replaceable_v <C>, "");
+
+union D { int a; A b; B c; };
+
+static_assert (!std::is_trivially_relocatable_v <D>, "");
+static_assert (!std::is_replaceable_v <D>, "");
+
+union E { E (); int a; A b; };
+
+static_assert (std::is_trivially_relocatable_v <E>, "");
+static_assert (std::is_replaceable_v <E>, "");
+
+union F { F () = default; int a; A b; };
+
+static_assert (std::is_trivially_relocatable_v <F>, "");
+static_assert (std::is_replaceable_v <F>, "");
+
+union G { G (const G &); int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <G>, "");
+static_assert (!std::is_replaceable_v <G>, "");
+
+union H { H (const H &) = default; int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <H>, "");
+static_assert (!std::is_replaceable_v <H>, "");
+
+union I { I (I &&); int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <I>, "");
+static_assert (!std::is_replaceable_v <I>, "");
+
+union J { J (J &&) = default; int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <J>, "");
+static_assert (!std::is_replaceable_v <J>, "");
+
+union K { K &operator= (const K &); int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (!std::is_replaceable_v <K>, "");
+
+union L { L &operator= (const L &) = default; int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <L>, "");
+static_assert (!std::is_replaceable_v <L>, "");
+
+union M { M &operator= (M &&); int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <M>, "");
+static_assert (!std::is_replaceable_v <M>, "");
+
+union N { N &operator= (N &&) = default; int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <N>, "");
+static_assert (!std::is_replaceable_v <N>, "");
+
+union O { ~O (); int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <O>, "");
+static_assert (!std::is_replaceable_v <O>, "");
+
+union P { ~P () = default; int a; A b; };
+
+static_assert (!std::is_trivially_relocatable_v <P>, "");
+static_assert (!std::is_replaceable_v <P>, "");
+
+union Q { int a; const A b; };
+
+static_assert (std::is_trivially_relocatable_v <Q>, "");
+static_assert (!std::is_replaceable_v <Q>, "");
+
+union S { volatile int a; A b; };
+
+static_assert (std::is_trivially_relocatable_v <S>, "");
+static_assert (!std::is_replaceable_v <S>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable2.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable2.C
new file mode 100644
index 0000000..b740061
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable2.C
@@ -0,0 +1,204 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+class A {};
+
+static_assert (std::is_trivially_relocatable_v <A>, "");
+static_assert (std::is_replaceable_v <A>, "");
+static_assert (std::is_trivially_relocatable <A>::value, "");
+static_assert (std::is_replaceable <A>::value, "");
+
+struct B { ~B (); };
+static B z;
+
+static_assert (!std::is_trivially_relocatable_v <B>, "");
+static_assert (!std::is_replaceable_v <B>, "");
+static_assert (!std::is_trivially_relocatable <B>::value, "");
+static_assert (!std::is_replaceable <B>::value, "");
+
+class C trivially_relocatable_if_eligible {};
+
+static_assert (std::is_trivially_relocatable_v <C>, "");
+static_assert (std::is_replaceable_v <C>, "");
+
+class D trivially_relocatable_if_eligible : A {};
+
+static_assert (std::is_trivially_relocatable_v <D>, "");
+static_assert (std::is_replaceable_v <D>, "");
+
+class E trivially_relocatable_if_eligible {
+  int a;
+  void *b;
+  int c[3];
+  A d[3];
+  B &e = z;
+};
+
+static_assert (std::is_trivially_relocatable_v <E>, "");
+static_assert (!std::is_replaceable_v <E>, "");
+
+class F trivially_relocatable_if_eligible : A {};
+
+static_assert (std::is_trivially_relocatable_v <F>, "");
+static_assert (std::is_replaceable_v <F>, "");
+
+class G trivially_relocatable_if_eligible : virtual A {};
+
+static_assert (!std::is_trivially_relocatable_v <G>, "");
+static_assert (std::is_replaceable_v <G>, "");
+
+class H trivially_relocatable_if_eligible : B {};
+
+static_assert (!std::is_trivially_relocatable_v <H>, "");
+static_assert (!std::is_replaceable_v <H>, "");
+
+class I trivially_relocatable_if_eligible { I (I &&); };
+
+static_assert (std::is_trivially_relocatable_v <I>, "");
+static_assert (!std::is_replaceable_v <I>, "");
+
+class J trivially_relocatable_if_eligible { ~J (); };
+
+static_assert (std::is_trivially_relocatable_v <J>, "");
+static_assert (!std::is_replaceable_v <J>, "");
+
+class K trivially_relocatable_if_eligible {
+  B a;
+  B b[1];
+  const B c;
+  const B d[1];
+};
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (!std::is_replaceable_v <K>, "");
+
+class L trivially_relocatable_if_eligible: virtual A, B { B a; };
+
+static_assert (!std::is_trivially_relocatable_v <L>, "");
+static_assert (!std::is_replaceable_v <L>, "");
+
+static_assert (!std::is_trivially_relocatable_v <void>, "");
+static_assert (!std::is_trivially_relocatable_v <const void>, "");
+static_assert (std::is_trivially_relocatable_v <int>, "");
+static_assert (std::is_trivially_relocatable_v <const int>, "");
+static_assert (std::is_trivially_relocatable_v <char>, "");
+static_assert (std::is_trivially_relocatable_v <char const volatile>, "");
+static_assert (std::is_trivially_relocatable_v <unsigned long long>, "");
+static_assert (std::is_trivially_relocatable_v <void *>, "");
+static_assert (std::is_trivially_relocatable_v <const int *>, "");
+static_assert (!std::is_trivially_relocatable_v <int &>, "");
+static_assert (!std::is_trivially_relocatable_v <A &>, "");
+static_assert (std::is_trivially_relocatable_v <const A>, "");
+static_assert (std::is_trivially_relocatable_v <A [1]>, "");
+static_assert (std::is_trivially_relocatable_v <A []>, "");
+static_assert (!std::is_replaceable_v <void>, "");
+static_assert (!std::is_replaceable_v <const void>, "");
+static_assert (std::is_replaceable_v <int>, "");
+static_assert (!std::is_replaceable_v <const int>, "");
+static_assert (std::is_replaceable_v <char>, "");
+static_assert (!std::is_replaceable_v <char const volatile>, "");
+static_assert (std::is_replaceable_v <unsigned long long>, "");
+static_assert (std::is_replaceable_v <void *>, "");
+static_assert (std::is_replaceable_v <const int *>, "");
+static_assert (!std::is_replaceable_v <int &>, "");
+static_assert (!std::is_replaceable_v <A &>, "");
+static_assert (!std::is_replaceable_v <const A>, "");
+static_assert (std::is_replaceable_v <A [1]>, "");
+static_assert (std::is_replaceable_v <A []>, "");
+
+struct M { const int i; };
+
+static_assert (!std::is_trivially_relocatable_v <M>, "");
+static_assert (!std::is_replaceable_v <M>, "");
+
+struct N trivially_relocatable_if_eligible { const int i; };
+
+static_assert (std::is_trivially_relocatable_v <N>, "");
+static_assert (!std::is_replaceable_v <N>, "");
+
+struct O { ~O (); };
+
+static_assert (!std::is_trivially_relocatable_v <O>, "");
+static_assert (!std::is_replaceable_v <O>, "");
+
+struct P { ~P () = default; };
+
+static_assert (std::is_trivially_relocatable_v <P>, "");
+static_assert (std::is_replaceable_v <P>, "");
+
+struct Q { Q (Q &&); Q (const Q &) = default; };
+
+static_assert (!std::is_trivially_relocatable_v <Q>, "");
+static_assert (!std::is_replaceable_v <Q>, "");
+
+struct R { R (R &&); };
+
+static_assert (!std::is_trivially_relocatable_v <R>, "");
+static_assert (!std::is_replaceable_v <R>, "");
+
+struct S { S (S &&) = default; };
+
+static_assert (!std::is_trivially_relocatable_v <S>, "");
+static_assert (!std::is_replaceable_v <S>, "");
+
+struct T { T (T &&) = default; T &operator= (T &&) = default; };
+
+static_assert (std::is_trivially_relocatable_v <T>, "");
+static_assert (std::is_replaceable_v <T>, "");
+
+struct U { U (const U &); };
+
+static_assert (!std::is_trivially_relocatable_v <U>, "");
+static_assert (!std::is_replaceable_v <U>, "");
+
+struct V { V (const V&) = default; };
+
+static_assert (std::is_trivially_relocatable_v <V>, "");
+static_assert (std::is_replaceable_v <V>, "");
+
+struct W { W (W &&) = delete; W (const W &) = default; };
+
+static_assert (!std::is_trivially_relocatable_v <W>, "");
+static_assert (!std::is_replaceable_v <W>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable3.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable3.C
new file mode 100644
index 0000000..312c11b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable3.C
@@ -0,0 +1,213 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+class A {};
+
+struct B { ~B (); };
+
+class C trivially_relocatable_if_eligible { C (C &&); };
+
+template <typename T>
+class D trivially_relocatable_if_eligible : T {};
+D<A> a;
+D<B> b;
+
+static_assert (std::is_trivially_relocatable_v <D<A>>, "");
+static_assert (!std::is_trivially_relocatable_v <D<B>>, "");
+static_assert (std::is_replaceable_v <D<A>>, "");
+static_assert (!std::is_replaceable_v <D<B>>, "");
+
+struct E { E (E &&) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <E>, "");
+static_assert (!std::is_replaceable_v <E>, "");
+
+struct F { F (const F &) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <F>, "");
+static_assert (!std::is_replaceable_v <F>, "");
+
+struct G { G &operator= (G &&) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <G>, "");
+static_assert (!std::is_replaceable_v <G>, "");
+
+struct H { ~H () = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <H>, "");
+static_assert (!std::is_replaceable_v <H>, "");
+
+union U { C u; };
+
+static_assert (std::is_trivially_relocatable_v <U>, "");
+static_assert (!std::is_replaceable_v <U>, "");
+
+template <typename T>
+struct I { int s; T t; };
+
+static_assert (std::is_trivially_relocatable_v <I<int>>, "");
+static_assert (std::is_trivially_relocatable_v <I<volatile int>>, "");
+static_assert (!std::is_trivially_relocatable_v <I<const int>>, "");
+static_assert (!std::is_trivially_relocatable_v <I<const int &>>, "");
+static_assert (!std::is_trivially_relocatable_v <I<int &>>, "");
+static_assert (std::is_trivially_relocatable_v <I<int [2]>>, "");
+static_assert (!std::is_trivially_relocatable_v <I<const int [2]>>, "");
+static_assert (std::is_trivially_relocatable_v <I<int []>>, "");
+static_assert (std::is_replaceable_v <I<int>>, "");
+static_assert (!std::is_replaceable_v <I<volatile int>>, "");
+static_assert (!std::is_replaceable_v <I<const int>>, "");
+static_assert (!std::is_replaceable_v <I<const int &>>, "");
+static_assert (!std::is_replaceable_v <I<int &>>, "");
+static_assert (std::is_replaceable_v <I<int [2]>>, "");
+static_assert (!std::is_replaceable_v <I<const int [2]>>, "");
+
+template <typename T>
+struct J trivially_relocatable_if_eligible { int s; T t; };
+
+static_assert (std::is_trivially_relocatable_v <J<int>>, "");
+static_assert (std::is_trivially_relocatable_v <J<volatile int>>, "");
+static_assert (std::is_trivially_relocatable_v <J<const int>>, "");
+static_assert (std::is_trivially_relocatable_v <J<const int &>>, "");
+static_assert (std::is_trivially_relocatable_v <J<int &>>, "");
+static_assert (std::is_trivially_relocatable_v <J<int [2]>>, "");
+static_assert (std::is_trivially_relocatable_v <J<const int [2]>>, "");
+static_assert (std::is_trivially_relocatable_v <J<int []>>, "");
+static_assert (std::is_replaceable_v <J<int>>, "");
+static_assert (!std::is_replaceable_v <J<volatile int>>, "");
+static_assert (!std::is_replaceable_v <J<const int>>, "");
+static_assert (!std::is_replaceable_v <J<const int &>>, "");
+static_assert (!std::is_replaceable_v <J<int &>>, "");
+static_assert (std::is_replaceable_v <J<int [2]>>, "");
+static_assert (!std::is_replaceable_v <J<const int [2]>>, "");
+static_assert (std::is_replaceable_v <J<int []>>, "");
+
+struct K { K (K &&) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (!std::is_replaceable_v <K>, "");
+
+struct L { L (const L &) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <L>, "");
+static_assert (!std::is_replaceable_v <L>, "");
+
+struct M { M &operator= (M &&) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <M>, "");
+static_assert (!std::is_replaceable_v <M>, "");
+
+struct N { N (N &&) = default; N &operator= (N &&) = default; };
+
+static_assert (std::is_trivially_relocatable_v <N>, "");
+static_assert (std::is_replaceable_v <N>, "");
+
+struct O {
+  O (const O &) = default;
+  O (O &&) = default;
+  O &operator= (O &&) = default;
+};
+
+static_assert (std::is_trivially_relocatable_v <O>, "");
+static_assert (std::is_replaceable_v <O>, "");
+
+struct P { P (P &&) = default; P &operator= (P &&) = default; };
+
+static_assert (std::is_trivially_relocatable_v <P>, "");
+static_assert (std::is_replaceable_v <P>, "");
+
+struct Q { Q (Q &&) {} };
+
+static_assert (!std::is_trivially_relocatable_v <Q>, "");
+static_assert (!std::is_replaceable_v <Q>, "");
+
+struct R { R (const R &) {} };
+
+static_assert (!std::is_trivially_relocatable_v <R>, "");
+static_assert (!std::is_replaceable_v <R>, "");
+
+struct S { S &operator= (const S &) { return *this; }; };
+
+static_assert (!std::is_trivially_relocatable_v <S>, "");
+static_assert (!std::is_replaceable_v <S>, "");
+
+struct T {};
+
+static_assert (std::is_trivially_relocatable_v <T>, "");
+static_assert (std::is_replaceable_v <T>, "");
+
+struct V replaceable_if_eligible {};
+
+static_assert (std::is_trivially_relocatable_v <V>, "");
+static_assert (std::is_replaceable_v <V>, "");
+
+struct W { template <typename U> W (const U &) = delete; };
+
+static_assert (std::is_trivially_relocatable_v <W>, "");
+static_assert (std::is_replaceable_v <W>, "");
+
+template <typename T>
+struct X : T {};
+
+static_assert (!std::is_trivially_relocatable_v <X<Q>>, "");
+static_assert (!std::is_replaceable_v <X<Q>>, "");
+
+template <typename T>
+struct Y : virtual T {};
+
+static_assert (!std::is_trivially_relocatable_v <Y<I<int>>>, "");
+static_assert (!std::is_trivially_relocatable_v <Y<I<const int>>>, "");
+static_assert (!std::is_trivially_relocatable_v <Y<Q>>, "");
+static_assert (std::is_replaceable_v <Y<I<int>>>, "");
+static_assert (!std::is_replaceable_v <Y<I<const int>>>, "");
+static_assert (!std::is_replaceable_v <Y<Q>>, "");
+
+struct Z {
+  virtual ~Z () = default;
+  Z (Z &&) = default;
+  Z &operator= (Z &&) = default;
+};
+
+static_assert (std::is_trivially_relocatable_v <Z>, "");
+static_assert (std::is_replaceable_v <Z>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable4.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable4.C
new file mode 100644
index 0000000..10aafc1
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable4.C
@@ -0,0 +1,128 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+struct A replaceable_if_eligible {
+  ~A () = delete;
+  A (A &&) = default;
+  A &operator= (A &&) = default;
+};
+
+static_assert (!std::is_trivially_relocatable_v <A>, "");
+static_assert (!std::is_replaceable_v <A>, "");
+
+struct B replaceable_if_eligible { B (const B &) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <B>, "");
+static_assert (!std::is_replaceable_v <B>, "");
+
+template <typename T>
+struct C replaceable_if_eligible : virtual T {};
+
+static_assert (!std::is_trivially_relocatable_v <C<A>>, "");
+static_assert (!std::is_trivially_relocatable_v <C<B>>, "");
+static_assert (!std::is_replaceable_v <C<A>>, "");
+static_assert (!std::is_replaceable_v <C<B>>, "");
+
+template <typename T>
+struct D { int s; T t; };
+
+static_assert (!std::is_trivially_relocatable_v <C<D<int>>>, "");
+static_assert (std::is_replaceable_v <C<D<int>>>, "");
+
+struct E trivially_relocatable_if_eligible replaceable_if_eligible {
+  E (E &&);
+  E &operator= (E &&) = default;
+};
+
+static_assert (std::is_trivially_relocatable_v <E>, "");
+static_assert (std::is_replaceable_v <E>, "");
+
+struct F trivially_relocatable_if_eligible replaceable_if_eligible {
+  F (F &&) = default;
+  F &operator= (F &&);
+};
+
+static_assert (std::is_trivially_relocatable_v <F>, "");
+static_assert (std::is_replaceable_v <F>, "");
+
+struct G replaceable_if_eligible { G (G const &) = default; };
+
+static_assert (std::is_trivially_relocatable_v <G>, "");
+static_assert (std::is_replaceable_v <G>, "");
+
+struct H { H (H const &) = default; };
+
+static_assert (std::is_trivially_relocatable_v <H>, "");
+static_assert (std::is_replaceable_v <H>, "");
+
+struct I replaceable_if_eligible { I &operator= (const I &) = default; };
+
+static_assert (std::is_trivially_relocatable_v <I>, "");
+static_assert (std::is_replaceable_v <I>, "");
+
+struct J { J &operator= (J const &) = default; };
+
+static_assert (std::is_trivially_relocatable_v <J>, "");
+static_assert (std::is_replaceable_v <J>, "");
+
+struct K { K (const K &) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (!std::is_replaceable_v <K>, "");
+
+struct L { L (L&&) = delete; };
+
+static_assert (!std::is_trivially_relocatable_v <L>, "");
+static_assert (!std::is_replaceable_v <L>, "");
+
+struct M { M operator= (M); };
+
+static_assert (!std::is_trivially_relocatable_v <M>, "");
+static_assert (!std::is_replaceable_v <M>, "");
+
+struct N { N operator= (N &&); };
+
+static_assert (!std::is_trivially_relocatable_v <N>, "");
+static_assert (!std::is_replaceable_v <N>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable5.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable5.C
new file mode 100644
index 0000000..0416137
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable5.C
@@ -0,0 +1,77 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-additional-options "-Wc++26-compat" }
+
+struct A __trivially_relocatable_if_eligible { A (const A &&); };
+struct B __replaceable_if_eligible { B (const B &&); B &operator= (B &&); };
+struct C __replaceable_if_eligible __final __trivially_relocatable_if_eligible { C (const C &&); C &operator= (C &&); };
+#if __cpp_trivial_relocatability >= 202502L
+struct D trivially_relocatable_if_eligible { D (const D &&); };
+struct E replaceable_if_eligible { E (const E &&); E &operator= (E &&); };
+struct F trivially_relocatable_if_eligible replaceable_if_eligible final { F (const F &&); F &operator= (F &&); };
+#else
+struct D trivially_relocatable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+// { dg-error "variable 'D trivially_relocatable_if_eligible' has initializer but incomplete type" "" { target c++23_down } .-1 }
+struct E replaceable_if_eligible {};			// { dg-warning "identifier 'replaceable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+// { dg-error "variable 'E replaceable_if_eligible' has initializer but incomplete type" "" { target c++23_down } .-1 }
+struct F trivially_relocatable_if_eligible replaceable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+// { dg-error "expected initializer before 'replaceable_if_eligible'" "" { target c++23_down } .-1 }
+#endif
+#if __cplusplus <= 202302L
+struct G {};
+struct G trivially_relocatable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+struct H {};
+struct H replaceable_if_eligible {};			// { dg-warning "identifier 'replaceable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+struct I {};
+struct I trivially_relocatable_if_eligible replaceable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+#endif							// { dg-error "expected initializer before 'replaceable_if_eligible'" "" { target c++23_down } .-1 }
+struct J {};
+struct J __trivially_relocatable_if_eligible {};	// { dg-error "redefinition of 'struct J'" }
+struct K {};
+struct K __replaceable_if_eligible {};			// { dg-error "redefinition of 'struct K'" }
+struct L {};
+struct L __trivially_relocatable_if_eligible __replaceable_if_eligible {};	// { dg-error "redefinition of 'struct L'" }
+struct M __trivially_relocatable_if_eligible __trivially_relocatable_if_eligible {}; // { dg-error "duplicate '__trivially_relocatable_if_eligible' specifier" }
+struct N __replaceable_if_eligible __replaceable_if_eligible {};		// { dg-error "duplicate '__replaceable_if_eligible' specifier" }
+struct O __trivially_relocatable_if_eligible __replaceable_if_eligible __replaceable_if_eligible __trivially_relocatable_if_eligible final final {};
+// { dg-error "duplicate '__replaceable_if_eligible' specifier" "" { target *-*-* } .-1 }
+// { dg-error "duplicate '__trivially_relocatable_if_eligible' specifier" "" { target *-*-* } .-2 }
+// { dg-error "duplicate 'final' specifier" "" { target *-*-* } .-3 }
+#if __cpp_trivial_relocatability >= 202502L
+struct P trivially_relocatable_if_eligible trivially_relocatable_if_eligible {}; // { dg-error "duplicate 'trivially_relocatable_if_eligible' specifier" "" { target c++26 } }
+struct Q replaceable_if_eligible replaceable_if_eligible {};			// { dg-error "duplicate 'replaceable_if_eligible' specifier" "" { target c++26 } }
+struct R trivially_relocatable_if_eligible replaceable_if_eligible replaceable_if_eligible trivially_relocatable_if_eligible final final {};
+// { dg-error "duplicate 'replaceable_if_eligible' specifier" "" { target c++26 } .-1 }
+// { dg-error "duplicate 'trivially_relocatable_if_eligible' specifier" "" { target c++26 } .-2 }
+// { dg-error "duplicate 'final' specifier" "" { target c++26 } .-3 }
+struct S trivially_relocatable_if_eligible __trivially_relocatable_if_eligible {}; // { dg-error "duplicate '__trivially_relocatable_if_eligible' specifier" "" { target c++26 } }
+struct T replaceable_if_eligible __replaceable_if_eligible {};			// { dg-error "duplicate '__replaceable_if_eligible' specifier" "" { target c++26 } }
+struct U trivially_relocatable_if_eligible replaceable_if_eligible __replaceable_if_eligible __trivially_relocatable_if_eligible final __final {};
+// { dg-error "duplicate '__replaceable_if_eligible' specifier" "" { target c++26 } .-1 }
+// { dg-error "duplicate '__trivially_relocatable_if_eligible' specifier" "" { target c++26 } .-2 }
+// { dg-error "duplicate '__final' specifier" "" { target c++26 } .-3 }
+struct V __trivially_relocatable_if_eligible trivially_relocatable_if_eligible {}; // { dg-error "duplicate 'trivially_relocatable_if_eligible' specifier" "" { target c++26 } }
+struct W __replaceable_if_eligible replaceable_if_eligible {};			// { dg-error "duplicate 'replaceable_if_eligible' specifier" "" { target c++26 } }
+struct X __trivially_relocatable_if_eligible __replaceable_if_eligible replaceable_if_eligible trivially_relocatable_if_eligible __final final {};
+// { dg-error "duplicate 'replaceable_if_eligible' specifier" "" { target c++26 } .-1 }
+// { dg-error "duplicate 'trivially_relocatable_if_eligible' specifier" "" { target c++26 } .-2 }
+// { dg-error "duplicate 'final' specifier" "" { target c++26 } .-3 }
+#else
+struct Y {};
+Y foo ();
+struct Y trivially_relocatable_if_eligible = foo ();    // { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+struct Z {};
+Z bar ();
+struct Z replaceable_if_eligible = bar ();		// { dg-warning "identifier 'replaceable_if_eligible' is a conditional keyword in" "" { target c++23_down } }
+#endif
+
+static_assert (__builtin_is_trivially_relocatable (A), "");
+static_assert (__builtin_is_replaceable (B), "");
+static_assert (__builtin_is_trivially_relocatable (C), "");
+static_assert (__builtin_is_replaceable (C), "");
+#if __cpp_trivial_relocatability >= 202502L
+static_assert (__builtin_is_trivially_relocatable (D), "");
+static_assert (__builtin_is_replaceable (E), "");
+static_assert (__builtin_is_trivially_relocatable (F), "");
+static_assert (__builtin_is_replaceable (F), "");
+#endif
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable6.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable6.C
new file mode 100644
index 0000000..ffcf12b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable6.C
@@ -0,0 +1,30 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++98_only } }
+// { dg-additional-options "-Wc++26-compat" }
+
+struct A __trivially_relocatable_if_eligible {};
+struct B __replaceable_if_eligible {};
+struct C __replaceable_if_eligible __final __trivially_relocatable_if_eligible {};
+struct D trivially_relocatable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" }
+// { dg-error "variable 'D trivially_relocatable_if_eligible' has initializer but incomplete type" "" { target *-*-* } .-1 }
+// { dg-error "extended initializer lists only available with" "" { target *-*-* } .-2 }
+struct E replaceable_if_eligible {};			// { dg-warning "identifier 'replaceable_if_eligible' is a conditional keyword in" }
+// { dg-error "variable 'E replaceable_if_eligible' has initializer but incomplete type" "" { target *-*-* } .-1 }
+// { dg-error "extended initializer lists only available with" "" { target *-*-* } .-2 }
+struct F trivially_relocatable_if_eligible replaceable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" }
+// { dg-error "expected initializer before 'replaceable_if_eligible'" "" { target *-*-* } .-1 }
+struct G {};
+struct G trivially_relocatable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" }
+// { dg-error "extended initializer lists only available with" "" { target *-*-* } .-1 }
+struct H {};
+struct H replaceable_if_eligible {};			// { dg-warning "identifier 'replaceable_if_eligible' is a conditional keyword in" }
+// { dg-error "extended initializer lists only available with" "" { target *-*-* } .-1 }
+struct I {};
+struct I trivially_relocatable_if_eligible replaceable_if_eligible {};		// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" }
+// { dg-error "expected initializer before 'replaceable_if_eligible'" "" { target *-*-* } .-1 }
+struct J {};
+J foo ();
+struct J trivially_relocatable_if_eligible = foo ();	// { dg-warning "identifier 'trivially_relocatable_if_eligible' is a conditional keyword in" }
+struct K {};
+K bar ();
+struct K replaceable_if_eligible = bar ();		// { dg-warning "identifier 'replaceable_if_eligible' is a conditional keyword in" }
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable7.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable7.C
new file mode 100644
index 0000000..608c245
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable7.C
@@ -0,0 +1,33 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+							// { dg-error "invalid use of incomplete type 'struct A'" "" { target *-*-* } .-1 }
+
+template <typename T>
+inline constexpr bool is_nothrow_relocatable_v		// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_nothrow_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+							// { dg-error "invalid use of incomplete type 'struct A'" "" { target *-*-* } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}							// { dg-error "invalid use of incomplete type 'struct A'" "" { target *-*-* } .-1 }
+
+struct A;						// { dg-message "forward declaration of 'struct A'" }
+
+auto a = std::is_trivially_relocatable_v <A>;		// { dg-message "required from here" }
+auto b = std::is_nothrow_relocatable_v <A>;		// { dg-message "required from here" }
+auto c = std::is_replaceable_v <A>;			// { dg-message "required from here" }
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable8.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable8.C
new file mode 100644
index 0000000..5f8390d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable8.C
@@ -0,0 +1,190 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_nothrow_relocatable
+  : public integral_constant <bool, __builtin_is_nothrow_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_nothrow_relocatable_v		// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_nothrow_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+struct A { A (A &&) = default; A &operator= (A &&) = default; ~A () = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <A>, "");
+static_assert (std::is_nothrow_relocatable_v <A>, "");
+static_assert (std::is_replaceable_v <A>, "");
+
+struct B { B (B &&); B &operator= (B &&) = default; ~B () = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <B>, "");
+static_assert (!std::is_nothrow_relocatable_v <B>, "");
+static_assert (!std::is_replaceable_v <B>, "");
+
+struct C { C (C &&) = default; C &operator= (C &&); ~C () = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <C>, "");
+static_assert (std::is_nothrow_relocatable_v <C>, "");
+static_assert (!std::is_replaceable_v <C>, "");
+
+struct D { D (D &&) = delete; D &operator= (D &&) = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <D>, "");
+static_assert (!std::is_nothrow_relocatable_v <D>, "");
+static_assert (!std::is_replaceable_v <D>, "");
+
+struct E { E (E &&) = default; E &operator= (E &&) = delete; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <E>, "");
+static_assert (std::is_nothrow_relocatable_v <E>, "");
+static_assert (!std::is_replaceable_v <E>, "");
+
+struct F { F (F &&) = default; F &operator= (F &&) = default; ~F () = delete; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <F>, "");
+static_assert (!std::is_nothrow_relocatable_v <F>, "");
+static_assert (!std::is_replaceable_v <F>, "");
+
+struct G { G (const G &) = default; G &operator= (const G &) = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <G>, "");
+static_assert (std::is_nothrow_relocatable_v <G>, "");
+static_assert (std::is_replaceable_v <G>, "");
+
+struct H { H (const H &); H &operator= (const H &) = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <H>, "");
+static_assert (!std::is_nothrow_relocatable_v <H>, "");
+static_assert (!std::is_replaceable_v <H>, "");
+
+struct I { I (const I &) = default; I &operator= (const I &); ~I () = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <I>, "");
+static_assert (std::is_nothrow_relocatable_v <I>, "");
+static_assert (!std::is_replaceable_v <I>, "");
+
+struct J { J (const J &) = delete; J &operator= (const J &) = default; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <J>, "");
+static_assert (!std::is_nothrow_relocatable_v <J>, "");
+static_assert (!std::is_replaceable_v <J>, "");
+
+struct K { K (const K &) = default; K &operator= (const K &) = delete; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (std::is_nothrow_relocatable_v <K>, "");
+static_assert (!std::is_replaceable_v <K>, "");
+
+struct M;
+struct L { L (L &&) = default; L (M &&); L &operator= (L &&) = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <L>, "");
+static_assert (std::is_nothrow_relocatable_v <L>, "");
+static_assert (std::is_replaceable_v <L>, "");
+
+struct M : public L { using L::L; M (const M &); M &operator= (M &&) = default; int b; };
+
+static_assert (!std::is_trivially_relocatable_v <M>, "");
+static_assert (!std::is_nothrow_relocatable_v <M>, "");
+static_assert (!std::is_replaceable_v <M>, "");
+
+struct O;
+struct N { N (N &&) = default; N &operator= (N &&) = default; N &operator= (O &&); int a; };
+
+static_assert (std::is_trivially_relocatable_v <N>, "");
+static_assert (std::is_nothrow_relocatable_v <N>, "");
+static_assert (std::is_replaceable_v <N>, "");
+
+struct O : public N { using N::operator=; O (O &&) = default; int b; };
+
+static_assert (!std::is_trivially_relocatable_v <O>, "");
+static_assert (std::is_nothrow_relocatable_v <O>, "");
+static_assert (!std::is_replaceable_v <O>, "");
+
+struct Q;
+struct P { template <typename T> P (T &&) {} };
+
+static_assert (std::is_trivially_relocatable_v <P>, "");
+static_assert (std::is_nothrow_relocatable_v <P>, "");
+static_assert (std::is_replaceable_v <P>, "");
+
+struct Q : public P { using P::P; Q (const Q &); };
+
+static_assert (!std::is_trivially_relocatable_v <Q>, "");
+static_assert (!std::is_nothrow_relocatable_v <Q>, "");
+static_assert (!std::is_replaceable_v <Q>, "");
+
+struct S;
+struct R { R (const R &) = default; R (const M &); R &operator= (R &&) = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <R>, "");
+static_assert (std::is_nothrow_relocatable_v <R>, "");
+static_assert (std::is_replaceable_v <R>, "");
+
+struct S : public R { using R::R; S &operator= (S &&) = default; int b; };
+
+static_assert (!std::is_trivially_relocatable_v <S>, "");
+static_assert (!std::is_nothrow_relocatable_v <S>, "");
+static_assert (!std::is_replaceable_v <S>, "");
+
+struct T { T (T &&) = default; T &operator= (T &&) = default; ~T (); int a; };
+
+static_assert (!std::is_trivially_relocatable_v <T>, "");
+static_assert (std::is_nothrow_relocatable_v <T>, "");
+static_assert (!std::is_replaceable_v <T>, "");
+
+struct U { U (const U &) = default; U &operator= (const U &) = default; ~U (); int a; };
+
+static_assert (!std::is_trivially_relocatable_v <U>, "");
+static_assert (std::is_nothrow_relocatable_v <U>, "");
+static_assert (!std::is_replaceable_v <U>, "");
+
+struct V { public: V (); private: V (V &&) = default; V &operator= (V &&) = default; ~V () = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <V>, "");
+static_assert (std::is_nothrow_relocatable_v <V>, "");
+static_assert (std::is_replaceable_v <V>, "");
diff --git a/gcc/testsuite/g++.dg/cpp26/trivially-relocatable9.C b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable9.C
new file mode 100644
index 0000000..1c45b53
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp26/trivially-relocatable9.C
@@ -0,0 +1,134 @@
+// P2786R13 - C++26 Trivial Relocatability
+// { dg-do compile { target c++11 } }
+// { dg-options "" }
+// { dg-additional-options "-pedantic" { target c++17 } }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+namespace std
+{
+template <typename T, T v>
+struct integral_constant
+{
+  static constexpr T value = v;
+};
+
+template <typename>
+struct is_trivially_relocatable;
+
+template <typename>
+struct is_replaceable;
+
+template<typename T>
+struct is_trivially_relocatable
+  : public integral_constant <bool, __builtin_is_trivially_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_nothrow_relocatable
+  : public integral_constant <bool, __builtin_is_nothrow_relocatable (T)>
+{
+};
+
+template<typename T>
+struct is_replaceable
+  : public integral_constant <bool, __builtin_is_replaceable (T)>
+{
+};
+
+template <typename T>
+inline constexpr bool is_trivially_relocatable_v	// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_trivially_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_nothrow_relocatable_v		// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_nothrow_relocatable (T);		// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+
+template <typename T>
+inline constexpr bool is_replaceable_v			// { dg-warning "inline variables are only available with" "" { target c++14_down } }
+  = __builtin_is_replaceable (T);			// { dg-warning "variable templates only available with" "" { target c++11_down } .-1 }
+}
+
+struct A trivially_relocatable_if_eligible { A (A &&); A &operator= (A &&); ~A (); int a; };
+
+static_assert (std::is_trivially_relocatable_v <A>, "");
+static_assert (std::is_nothrow_relocatable_v <A>, "");
+
+struct B { B (B &&); B &operator= (B &&); ~B (); int a; };
+
+static_assert (!std::is_trivially_relocatable_v <B>, "");
+static_assert (!std::is_nothrow_relocatable_v <B>, "");
+
+struct C trivially_relocatable_if_eligible : public A { C (C &&); C &operator= (C &&); ~C (); int a; };
+
+static_assert (std::is_trivially_relocatable_v <C>, "");
+static_assert (std::is_nothrow_relocatable_v <C>, "");
+
+struct D trivially_relocatable_if_eligible : public B { D (D &&); D &operator= (D &&); ~D (); int a; };
+
+static_assert (!std::is_trivially_relocatable_v <D>, "");
+static_assert (!std::is_nothrow_relocatable_v <D>, "");
+
+struct E trivially_relocatable_if_eligible { E (E &&); E &operator= (E &&); ~E (); A a; };
+
+static_assert (std::is_trivially_relocatable_v <E>, "");
+static_assert (std::is_nothrow_relocatable_v <E>, "");
+
+struct F trivially_relocatable_if_eligible { F (F &&) noexcept; F &operator= (F &&); ~F (); B a; };
+
+static_assert (!std::is_trivially_relocatable_v <F>, "");
+static_assert (std::is_nothrow_relocatable_v <F>, "");
+
+struct G trivially_relocatable_if_eligible { G (G &&); G &operator= (G &&); ~G () = delete; int a; };
+
+static_assert (!std::is_trivially_relocatable_v <G>, "");
+static_assert (!std::is_nothrow_relocatable_v <G>, "");
+
+struct H trivially_relocatable_if_eligible : virtual A { H (H &&); H &operator= (H &&); ~H (); int a; };
+
+static_assert (!std::is_trivially_relocatable_v <H>, "");
+static_assert (!std::is_nothrow_relocatable_v <H>, "");
+
+struct I trivially_relocatable_if_eligible { I (I &&); I &operator= (I &&); ~I (); A &a; int &b; };
+
+static_assert (std::is_trivially_relocatable_v <I>, "");
+static_assert (std::is_nothrow_relocatable_v <I>, "");
+
+struct J trivially_relocatable_if_eligible { J (J &&); J &operator= (J &&); ~J (); B &a; int &b; };
+
+static_assert (std::is_trivially_relocatable_v <J>, "");
+static_assert (std::is_nothrow_relocatable_v <J>, "");
+
+struct K trivially_relocatable_if_eligible { K (K &&) noexcept; K &operator= (K &&); ~K (); union { A a; int b; char c; }; };
+
+static_assert (!std::is_trivially_relocatable_v <K>, "");
+static_assert (std::is_nothrow_relocatable_v <K>, "");
+
+struct L trivially_relocatable_if_eligible { L (L &&); L &operator= (L &&); ~L (); union { int a; B b; short c; }; };
+
+static_assert (!std::is_trivially_relocatable_v <L>, "");
+static_assert (!std::is_nothrow_relocatable_v <L>, "");
+
+struct M trivially_relocatable_if_eligible { M (M &&); M &operator= (M &&); ~M () = default; int a; };
+
+static_assert (std::is_trivially_relocatable_v <M>, "");
+static_assert (std::is_nothrow_relocatable_v <M>, "");
+
+struct N trivially_relocatable_if_eligible { N (N &&); N &operator= (N &&); ~N (); union { M a; int b; char c; }; };
+
+static_assert (std::is_trivially_relocatable_v <N>, "");
+static_assert (std::is_nothrow_relocatable_v <N>, "");
+
+struct O trivially_relocatable_if_eligible { O (O &&); O &operator= (O &&); ~O (); union { unsigned long long a; int b; char c; }; };
+
+static_assert (std::is_trivially_relocatable_v <O>, "");
+static_assert (std::is_nothrow_relocatable_v <O>, "");
+
+struct P { P (P &&) noexcept; P &operator= (P &&); ~P (); int a; };
+
+static_assert (!std::is_trivially_relocatable_v <P>, "");
+static_assert (std::is_nothrow_relocatable_v <P>, "");
diff --git a/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210.C b/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210.C
index a31750e..baddc1c 100644
--- a/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210.C
+++ b/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210.C
@@ -7,4 +7,4 @@ concept C = true;
 template <class T>
 struct A {};
 
-void f(A<C<int> auto >) {} 
+void f(A<C<int> auto >) {} 	// { dg-error "auto" }
diff --git a/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210a.C b/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210a.C
new file mode 100644
index 0000000..1d63a84
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/concepts-pr67210a.C
@@ -0,0 +1,11 @@
+// PR c++/67210
+// { dg-do compile { target c++20 } }
+// { dg-additional-options -Wno-abbreviated-auto-in-template-arg }
+
+template <class T, class U>
+concept C = true;
+
+template <class T>
+struct A {};
+
+void f(A<C<int> auto >) {} 
diff --git a/gcc/testsuite/g++.dg/cpp2a/concepts-traits3.C b/gcc/testsuite/g++.dg/cpp2a/concepts-traits3.C
index 3e87da4..90d859a 100644
--- a/gcc/testsuite/g++.dg/cpp2a/concepts-traits3.C
+++ b/gcc/testsuite/g++.dg/cpp2a/concepts-traits3.C
@@ -1,49 +1,58 @@
 // PR c++/100474
 // { dg-do compile { target c++20 } }
 
-struct S { S() = delete; S(const S&); };
+struct S { S() = delete; S(const S&); };  // { dg-line S }
 
 template<class T>
 concept Aggregate = __is_aggregate(T);
-// { dg-message "'S' is not an aggregate" "" { target *-*-* } .-1  }
+// { dg-message "'S' is not an aggregate" "" { target *-*-* } S }
 
 template<class T>
 concept TriviallyCopyable = __is_trivially_copyable(T);
-// { dg-message "'S' is not trivially copyable" "" { target *-*-* } .-1  }
+// { dg-message "'S' is not trivially copyable" "" { target *-*-* } S }
 
 template<class T, class U>
 concept Assignable = __is_assignable(T, U);
-// { dg-message "'S' is not assignable from 'int'" "" { target *-*-* } .-1  }
+// { dg-message "'S' is not assignable from 'int', because" "" { target *-*-* } .-1  }
+// { dg-error "no match for 'operator='" "" { target *-*-* } .-2 }
 
 template<class T, class U>
 concept TriviallyAssignable = __is_trivially_assignable(T, U);
 // { dg-message "'S' is not trivially assignable from 'int'" "" { target *-*-* } .-1  }
+// { dg-error "no match for 'operator='" "" { target *-*-* } .-2 }
 
 template<class T, class U>
 concept NothrowAssignable = __is_nothrow_assignable(T, U);
 // { dg-message "'S' is not nothrow assignable from 'int'" "" { target *-*-* } .-1  }
+// { dg-error "no match for 'operator='" "" { target *-*-* } .-2 }
 
 template<class T, class... Args>
 concept Constructible = __is_constructible(T, Args...);
 // { dg-message "'S' is not default constructible" "" { target *-*-* } .-1  }
-// { dg-message "'S' is not constructible from 'int'" "" { target *-*-* } .-2  }
-// { dg-message "'S' is not constructible from 'int, char'" "" { target *-*-* } .-3  }
+// { dg-error "use of deleted function 'S::S\\(\\)'" "" { target *-*-* } .-2 }
+// { dg-message "'S' is not constructible from 'int'" "" { target *-*-* } .-3  }
+// { dg-message "'S' is not constructible from 'int, char'" "" { target *-*-* } .-4  }
+// { dg-error "no matching function for call to 'S::S" "" { target *-*-* } .-5 }
 
 template<class T, class... Args>
 concept TriviallyConstructible = __is_trivially_constructible(T, Args...);
 // { dg-message "'S' is not trivially default constructible" "" { target *-*-* } .-1  }
-// { dg-message "'S' is not trivially constructible from 'int'" "" { target *-*-* } .-2  }
-// { dg-message "'S' is not trivially constructible from 'int, char'" "" { target *-*-* } .-3  }
+// { dg-error "use of deleted function 'S::S\\(\\)'" "" { target *-*-* } .-2 }
+// { dg-message "'S' is not trivially constructible from 'int'" "" { target *-*-* } .-3  }
+// { dg-message "'S' is not trivially constructible from 'int, char'" "" { target *-*-* } .-4  }
+// { dg-error "no matching function for call to 'S::S" "" { target *-*-* } .-5 }
 
 template<class T, class... Args>
 concept NothrowConstructible = __is_nothrow_constructible(T, Args...);
 // { dg-message "'S' is not nothrow default constructible" "" { target *-*-* } .-1  }
-// { dg-message "'S' is not nothrow constructible from 'int'" "" { target *-*-* } .-2  }
-// { dg-message "'S' is not nothrow constructible from 'int, char'" "" { target *-*-* } .-3  }
+// { dg-error "use of deleted function 'S::S\\(\\)'" "" { target *-*-* } .-2 }
+// { dg-message "'S' is not nothrow constructible from 'int'" "" { target *-*-* } .-3  }
+// { dg-message "'S' is not nothrow constructible from 'int, char'" "" { target *-*-* } .-4  }
+// { dg-error "no matching function for call to 'S::S" "" { target *-*-* } .-5 }
 
 template<class T>
 concept UniqueObjReps = __has_unique_object_representations(T);
-// { dg-message "'S' does not have unique object representations" "" { target *-*-* } .-1  }
+// { dg-message "'S' does not have unique object representations" "" { target *-*-* } S }
 
 static_assert(Aggregate<S>); // { dg-error "assert" }
 static_assert(TriviallyCopyable<S>); // { dg-error "assert" }
diff --git a/gcc/testsuite/g++.dg/cpp2a/concepts-traits4.C b/gcc/testsuite/g++.dg/cpp2a/concepts-traits4.C
new file mode 100644
index 0000000..caad816
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/concepts-traits4.C
@@ -0,0 +1,77 @@
+// PR c++/117294
+// { dg-do compile { target c++20 } }
+// { dg-additional-options "-fconcepts-diagnostics-depth=2" }
+
+template <typename T> struct norm
+  { static constexpr bool value = __is_constructible(T); };
+template <typename T> constexpr bool norm_v = __is_constructible(T);
+
+template <typename T> struct part
+  { static constexpr bool value = __is_constructible(T); };
+template <typename T> struct part<T*>
+  { static constexpr bool value = false; };
+template <typename T> struct part<const T>
+  { static constexpr bool value = __is_same(T, void); };
+template <typename T> constexpr bool part_v = __is_constructible(T);
+template <typename T> constexpr bool part_v<T*> = false;
+template <typename T> constexpr bool part_v<const T> = __is_same(T, void);
+
+template <typename T> struct expl
+  { static constexpr bool value = __is_constructible(T); };
+template <> struct expl<int*>
+  { static constexpr bool value = false; };
+template <> struct expl<const int>
+  { static constexpr bool value = __is_same(int, void); };
+template <typename T> constexpr bool expl_v = __is_constructible(T);
+template <> constexpr bool expl_v<int*> = false;
+template <> constexpr bool expl_v<const int> = __is_same(int, void);
+
+template <typename T> concept test_norm = norm<T>::value;  // { dg-line norm }
+template <typename T> concept test_part = part<T>::value;  // { dg-line part }
+template <typename T> concept test_expl = expl<T>::value;  // { dg-line expl }
+template <typename T> concept test_norm_v = norm_v<T>;  // { dg-line norm_v }
+template <typename T> concept test_part_v = part_v<T>;  // { dg-line part_v }
+template <typename T> concept test_expl_v = expl_v<T>;  // { dg-line expl_v }
+
+static_assert(test_norm<void>);  // { dg-error "assert" }
+static_assert(test_part<void>);  // { dg-error "assert" }
+static_assert(test_expl<void>);  // { dg-error "assert" }
+static_assert(test_norm_v<void>);  // { dg-error "assert" }
+static_assert(test_part_v<void>);  // { dg-error "assert" }
+static_assert(test_expl_v<void>);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } norm }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } part }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } expl }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } norm_v }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } part_v }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } expl_v }
+// { dg-prune-output "'void' is incomplete" }
+
+static_assert(test_part<int*>);  // { dg-error "assert" }
+static_assert(test_expl<int*>);  // { dg-error "assert" }
+static_assert(test_part_v<int*>);  // { dg-error "assert" }
+static_assert(test_expl_v<int*>);  // { dg-error "assert" }
+// { dg-message ".with T = int\\*.. evaluated to .false." "" { target *-*-* } part }
+// { dg-message ".with T = int\\*.. evaluated to .false." "" { target *-*-* } expl }
+// { dg-message ".with T = int\\*.. evaluated to .false." "" { target *-*-* } part_v }
+// { dg-message ".with T = int\\*.. evaluated to .false." "" { target *-*-* } expl_v }
+
+static_assert(test_part<const int>);  // { dg-error "assert" }
+static_assert(test_part_v<const int>);  // { dg-error "assert" }
+// { dg-message "'int' is not the same as 'void'" "" { target *-*-* } part }
+// { dg-message "'int' is not the same as 'void'" "" { target *-*-* } part_v }
+
+struct S { S(int); };
+static_assert(requires { requires test_norm<S>; });  // { dg-error "assert" }
+static_assert(requires { requires test_part<S>; });  // { dg-error "assert" }
+static_assert(requires { requires test_expl<S>; });  // { dg-error "assert" }
+static_assert(requires { requires test_norm_v<S>; });  // { dg-error "assert" }
+static_assert(requires { requires test_part_v<S>; });  // { dg-error "assert" }
+static_assert(requires { requires test_expl_v<S>; });  // { dg-error "assert" }
+// { dg-message "'S' is not default constructible" "" { target *-*-* } norm }
+// { dg-message "'S' is not default constructible" "" { target *-*-* } part }
+// { dg-message "'S' is not default constructible" "" { target *-*-* } expl }
+// { dg-message "'S' is not default constructible" "" { target *-*-* } norm_v }
+// { dg-message "'S' is not default constructible" "" { target *-*-* } part_v }
+// { dg-message "'S' is not default constructible" "" { target *-*-* } expl_v }
+// { dg-prune-output "no matching function for call" }
diff --git a/gcc/testsuite/g++.dg/cpp2a/consteval-memfn1.C b/gcc/testsuite/g++.dg/cpp2a/consteval-memfn1.C
index ca92351..17a391b 100644
--- a/gcc/testsuite/g++.dg/cpp2a/consteval-memfn1.C
+++ b/gcc/testsuite/g++.dg/cpp2a/consteval-memfn1.C
@@ -4,13 +4,13 @@
 
 struct fixed_string {
   consteval int size(int n) const {
-    if (n < 0) throw; // { dg-error "not a constant" }
-    return n;
+    if (n < 0) throw; // { dg-error "not a constant" "" { target c++23_down } }
+    return n; // { dg-error "'void __cxa_rethrow\\\(\\\)' called with no caught exceptions active" "" { target c++26 } .-1 }
   }
 
   static consteval int size_static(int n) {
-    if (n < 0) throw; // { dg-error "not a constant" }
-    return n;
+    if (n < 0) throw; // { dg-error "not a constant" "" { target c++23_down } }
+    return n; // { dg-error "'void __cxa_rethrow\\\(\\\)' called with no caught exceptions active" "" { target c++26 } .-1 }
   }
 
   consteval void operator()() const { }
diff --git a/gcc/testsuite/g++.dg/cpp2a/consteval34.C b/gcc/testsuite/g++.dg/cpp2a/consteval34.C
index 7562f40..73bcf77 100644
--- a/gcc/testsuite/g++.dg/cpp2a/consteval34.C
+++ b/gcc/testsuite/g++.dg/cpp2a/consteval34.C
@@ -1,42 +1,51 @@
 // { dg-do compile { target c++20 } }
 // Explicit { dg-require-effective-target exceptions_enabled } to avoid verify compiler messages FAILs for '-fno-exceptions'.
 
-consteval int bar (int i) { if (i != 1) throw 1; return 0; }	// { dg-error "is not a constant expression" }
+consteval int bar (int i) { if (i != 1) throw 1; return 0; }	// { dg-error "is not a constant expression" "" { target c++23_down } }
 
 constexpr int
 foo (bool b)
 {
-  return b ? bar (3) : 2; // { dg-message "in .constexpr. expansion" }
+  return b ? bar (3) : 2; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
-}
+} // { dg-error "uncaught exception '1'" "" { target c++26 } }
 
 static_assert (foo (false) == 2);
 
-__extension__ constexpr int g1 = false ?: bar (2); // { dg-message "in .constexpr. expansion" }
-__extension__ constexpr int g2 = false ?: (1 + bar (2)); // { dg-message "in .constexpr. expansion" }
+__extension__ constexpr int g1 = false ?: bar (2); // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
+__extension__ constexpr int g2 = false ?: (1 + bar (2)); // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
 __extension__ constexpr int g3 = true ?: bar (2);
 __extension__ constexpr int g4 = true ?: (1 + bar (2));
-constexpr int g5 = bar (2) ? 1 : 2; // { dg-message "in .constexpr. expansion" }
-constexpr int g6 = bar (2) - 1 ? 1 : 2; // { dg-message "in .constexpr. expansion" }
+constexpr int g5 = bar (2) ? 1 : 2; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
+constexpr int g6 = bar (2) - 1 ? 1 : 2; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-1 }
 
 void
 g ()
 {
-  __extension__ int a1[bar(3)]; // { dg-message "in .constexpr. expansion" }
+  __extension__ int a1[bar(3)]; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
   int a2[sizeof (bar(3))];
 
-  int a3 = false ? (1 + bar (8)) : 1; // { dg-message "in .constexpr. expansion" }
+  int a3 = false ? (1 + bar (8)) : 1; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
-  a3 += false ? (1 + bar (8)) : 1; // { dg-message "in .constexpr. expansion" }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
+  a3 += false ? (1 + bar (8)) : 1; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
 
-  __extension__ int a4 = false ?: (1 + bar (8)); // { dg-message "in .constexpr. expansion" }
+  __extension__ int a4 = false ?: (1 + bar (8)); // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
-  __extension__ int a5 = true ?: (1 + bar (8)); // { dg-message "in .constexpr. expansion" }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
+  __extension__ int a5 = true ?: (1 + bar (8)); // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
-  int a6 = bar (2) ? 1 : 2; // { dg-message "in .constexpr. expansion" }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
+  int a6 = bar (2) ? 1 : 2; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
-  int a7 = bar (2) - 1 ? 1 : 2; // { dg-message "in .constexpr. expansion" }
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
+  int a7 = bar (2) - 1 ? 1 : 2; // { dg-message "in .constexpr. expansion" "" { target c++23_down } }
 // { dg-error "call to consteval function" "" { target *-*-* } .-1 }
-}
+// { dg-error "uncaught exception '1'" "" { target c++26 } .-2 }
+} // { dg-error "uncaught exception '1'" "" { target c++26 } }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic11.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic11.C
index 9ee93c3..12a071a 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic11.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic11.C
@@ -8,6 +8,22 @@
 // a pointer to or object of the constructor or destructor's own class or one
 // of its bases, the dynamic_cast results in undefined behavior.
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 struct V {
   virtual void f();
 };
@@ -19,7 +35,7 @@ struct B : V {
 };
 
 struct D : A, B {
-  constexpr D() : B((A*)this, this) { } // { dg-message "in 'constexpr' expansion of" }
+  constexpr D() : B((A*)this, this) { } // { dg-message "in 'constexpr' expansion of" "" { target c++23_down } }
 };
 
 constexpr B::B(V* v, A* a)
@@ -29,8 +45,9 @@ constexpr B::B(V* v, A* a)
   if (b != nullptr)
     __builtin_abort ();
 
-  B& br = dynamic_cast<B&>(*v); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "dynamic type .A. of its operand does not have an unambiguous public base class .B." "" { target *-*-* } .-1 }
+  B& br = dynamic_cast<B&>(*v); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "dynamic type .A. of its operand does not have an unambiguous public base class .B." "" { target c++23_down } .-1 }
 }
 
-constexpr D d; // { dg-message "in 'constexpr' expansion of" }
+constexpr D d; // { dg-message "in 'constexpr' expansion of" "" { target c++23_down } }
+// { dg-error "uncaught exception" "" { target c++26 } .-1 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic14.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic14.C
index cc1cada..f966db4 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic14.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic14.C
@@ -4,6 +4,22 @@
 // Adopted from g++.old-deja/g++.other/dyncast1.C.
 // But use reference dynamic_cast.
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 // 1. downcast
 // 1.1. single inheritance case
 
@@ -21,27 +37,35 @@ class CCC : protected B {};
 class DDD : protected CCC {};
 
 constexpr D d;
-constexpr bool b01 = (dynamic_cast<D&> ((A&)d), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const A. of its operand is a non-public base class of dynamic type .D." "" { target *-*-* } .-1 }
-constexpr bool b02 = (dynamic_cast<D&> ((B&)d), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .D." "" { target *-*-* } .-1 }
+constexpr bool b01 = (dynamic_cast<D&> ((A&)d), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const A. of its operand is a non-public base class of dynamic type .D." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b02 = (dynamic_cast<D&> ((B&)d), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .D." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 static_assert (&d == &dynamic_cast<const D&> ((C&)d));
-constexpr bool b03 = (dynamic_cast<C&> ((B&)d), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .D." "" { target *-*-* } .-1 }
+constexpr bool b03 = (dynamic_cast<C&> ((B&)d), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .D." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 constexpr DD dd;
-constexpr bool b04 = (dynamic_cast<DD&> ((A&)dd), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const A. of its operand is a non-public base class of dynamic type .DD." "" { target *-*-* } .-1 }
-constexpr bool b05 = (dynamic_cast<DD&> ((B&)dd), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .DD." "" { target *-*-* } .-1 }
+constexpr bool b04 = (dynamic_cast<DD&> ((A&)dd), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const A. of its operand is a non-public base class of dynamic type .DD." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b05 = (dynamic_cast<DD&> ((B&)dd), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .DD." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 constexpr DDD ddd;
-constexpr bool b06 = (dynamic_cast<DDD&> ((A&)ddd), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const A. of its operand is a non-public base class of dynamic type .DDD." "" { target *-*-* } .-1 }
-constexpr bool b07 = (dynamic_cast<DDD&> ((B&)ddd), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .DDD." "" { target *-*-* } .-1 }
-constexpr bool b08 = (dynamic_cast<CCC&> ((B&)ddd), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .DDD." "" { target *-*-* } .-1 }
+constexpr bool b06 = (dynamic_cast<DDD&> ((A&)ddd), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const A. of its operand is a non-public base class of dynamic type .DDD." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b07 = (dynamic_cast<DDD&> ((B&)ddd), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .DDD." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b08 = (dynamic_cast<CCC&> ((B&)ddd), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .DDD." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 // 1.2. multiple inheritance case
 // 1.2.1. all bases are public
@@ -50,19 +74,23 @@ struct E : D, CC {};
 struct EE : CC, D {}; //Will search in reverse order.
 
 constexpr E e;
-constexpr bool b09 = (dynamic_cast<E&> ((A&)(D&)e), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .A. of its operand is a non-public base class of dynamic type .E." "" { target *-*-* } .-1 }
-constexpr bool b10 = (dynamic_cast<E&> ((B&)(D&)e), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .E." "" { target *-*-* } .-1 }
+constexpr bool b09 = (dynamic_cast<E&> ((A&)(D&)e), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .A. of its operand is a non-public base class of dynamic type .E." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b10 = (dynamic_cast<E&> ((B&)(D&)e), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .E." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 static_assert (&e == &dynamic_cast<E&> ((C&)(D&)e));
 static_assert (&e == &dynamic_cast<E&> ((B&)(CC&)e));
 static_assert (&(CC&)e == &dynamic_cast<CC&> ((B&)(CC&)e));
 
 constexpr EE ee;
-constexpr bool b11 = (dynamic_cast<EE&> ((A&)(D&)ee), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .A. of its operand is a non-public base class of dynamic type .EE." "" { target *-*-* } .-1 }
-constexpr bool b12 = (dynamic_cast<EE&> ((B&)(D&)ee), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .EE." "" { target *-*-* } .-1 }
+constexpr bool b11 = (dynamic_cast<EE&> ((A&)(D&)ee), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .A. of its operand is a non-public base class of dynamic type .EE." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b12 = (dynamic_cast<EE&> ((B&)(D&)ee), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .EE." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 static_assert (&ee == &dynamic_cast<EE&> ((C&)(D&)ee));
 static_assert (&ee == &dynamic_cast<EE&> ((B&)(CC&)ee));
 static_assert (&(CC&)ee == &dynamic_cast<CC&> ((B&)(CC&)ee));
@@ -78,14 +106,17 @@ constexpr X x;
 static_assert (&x == &dynamic_cast<X&>((B&)(CC&)(E&)x));
 
 constexpr XX xx;
-constexpr bool b13 = (dynamic_cast<XX&>((B&)(CC&)(E&)xx), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .XX." "" { target *-*-* } .-1 }
+constexpr bool b13 = (dynamic_cast<XX&>((B&)(CC&)(E&)xx), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .XX." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 constexpr Y y;
-constexpr bool b14 = (dynamic_cast<Y&>((B&)y), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .Y." "" { target *-*-* } .-1 }
-constexpr bool b15 = (dynamic_cast<Y&>((A&)(B&)y), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .A. of its operand is a non-public base class of dynamic type .Y." "" { target *-*-* } .-1 }
+constexpr bool b14 = (dynamic_cast<Y&>((B&)y), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const B. of its operand is a non-public base class of dynamic type .Y." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b15 = (dynamic_cast<Y&>((A&)(B&)y), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .A. of its operand is a non-public base class of dynamic type .Y." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 // 2. crosscast
 
@@ -93,13 +124,16 @@ struct J { virtual void j(); };
 struct K : CC, private J {}; 
 class KK : J, CC{};
 		
-constexpr bool b16 = (dynamic_cast<CC&> ((B&)(D&)e), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .CC." "" { target *-*-* } .-1 }
+constexpr bool b16 = (dynamic_cast<CC&> ((B&)(D&)e), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .B. of its operand is a non-public base class of dynamic type .CC." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 static_assert (&(CC&)e == &dynamic_cast<CC&> ((C&)(D&)e));
 
 constexpr K k;
-constexpr bool b17 = (dynamic_cast<J&> ((B&)k), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "dynamic type .K. of its operand does not have an unambiguous public base class .J." "" { target *-*-* } .-1 }
+constexpr bool b17 = (dynamic_cast<J&> ((B&)k), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "dynamic type .K. of its operand does not have an unambiguous public base class .J." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 constexpr KK kk;
-constexpr bool b18 = (dynamic_cast<J&> ((CC&)kk), true); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const CC. of its operand is a non-public base class of dynamic type .KK." "" { target *-*-* } .-1 }
+constexpr bool b18 = (dynamic_cast<J&> ((CC&)kk), true); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const CC. of its operand is a non-public base class of dynamic type .KK." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic18.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic18.C
index 25d98c2..112ff8a 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic18.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic18.C
@@ -2,6 +2,22 @@
 // { dg-do compile { target c++20 } }
 // Here 'b' doesn't point/refer to a public base of Derived.
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 struct Base {
     constexpr virtual ~Base(){}
 };
@@ -11,12 +27,12 @@ struct Derived: Base {
 };
 
 constexpr const Derived& cast(const Base& b) {
-    return dynamic_cast<const Derived&>(b); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "dynamic type .const Base. of its operand does not have a base class of type .Derived." "" { target *-*-* } .-1 }
+    return dynamic_cast<const Derived&>(b); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "dynamic type .const Base. of its operand does not have a base class of type .Derived." "" { target c++23_down } .-1 }
 }
 
 auto test() {
     static constexpr Base b;
-    constexpr auto res = cast(b);
+    constexpr auto res = cast(b);	// { dg-error "uncaught exception" "" { target c++26 } }
     return res;
 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic4.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic4.C
index da647bf..a237134 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic4.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic4.C
@@ -4,6 +4,22 @@
 
 // From clang's constant-expression-cxx2a.cpp.
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 struct A2 { virtual void a2(); };
 struct A : A2 { virtual void a(); };
 struct B : A {}; 
@@ -26,31 +42,36 @@ static_assert(dynamic_cast<const A*>(static_cast<const C2*>(&g)) == nullptr);
 static_assert(g.f == (void*)(F*)&g);
 static_assert(dynamic_cast<const void*>(static_cast<const D*>(&g)) == &g);
 
-constexpr int d_a = (dynamic_cast<const A&>(static_cast<const D&>(g)), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message ".A. is an ambiguous base class of dynamic type .G." "" { target *-*-* } .-1 }
-
+constexpr int d_a = (dynamic_cast<const A&>(static_cast<const D&>(g)), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message ".A. is an ambiguous base class of dynamic type .G." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 // Can navigate from A2 to its A...
 static_assert(&dynamic_cast<A&>((A2&)(B&)g) == &(A&)(B&)g);
 // ... and from B to its A ...
 static_assert(&dynamic_cast<A&>((B&)g) == &(A&)(B&)g);
 // ... but not from D.
-static_assert(&dynamic_cast<A&>((D&)g) == &(A&)(B&)g); // { dg-error "non-constant condition for static assertion|reference .dynamic_cast. failed" }
-// { dg-message ".A. is an ambiguous base class of dynamic type .G." "" { target *-*-* } .-1 }
-
+static_assert(&dynamic_cast<A&>((D&)g) == &(A&)(B&)g); // { dg-error "non-constant condition for static assertion" }
+// { dg-message ".A. is an ambiguous base class of dynamic type .G." "" { target c++23_down } .-1 }
+// { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } .-2 }
+// { dg-error "uncaught exception" "" { target c++26 } .-3 }
 // Can cast from A2 to sibling class D.
 static_assert(&dynamic_cast<D&>((A2&)(B&)g) == &(D&)g);
 
 // Cannot cast from private base E to derived class F.
-constexpr int e_f = (dynamic_cast<F&>((E&)g), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const E. of its operand is a non-public base class of dynamic type .G." "" { target *-*-* } .-1 }
+constexpr int e_f = (dynamic_cast<F&>((E&)g), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const E. of its operand is a non-public base class of dynamic type .G." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 // Cannot cast from B to private sibling E.
-constexpr int b_e = (dynamic_cast<E&>((B&)g), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "dynamic type .G. of its operand does not have an unambiguous public base class .E." "" { target *-*-* } .-1 }
+constexpr int b_e = (dynamic_cast<E&>((B&)g), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "dynamic type .G. of its operand does not have an unambiguous public base class .E." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 struct Unrelated { virtual void unrelated(); };
 
-constexpr int b_unrelated = (dynamic_cast<Unrelated&>((B&)g), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "dynamic type .G. of its operand does not have an unambiguous public base class .Unrelated." "" { target *-*-* } .-1 }
-constexpr int e_unrelated = (dynamic_cast<Unrelated&>((E&)g), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const E. of its operand is a non-public base class of dynamic type .G." "" { target *-*-* } .-1 }
+constexpr int b_unrelated = (dynamic_cast<Unrelated&>((B&)g), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "dynamic type .G. of its operand does not have an unambiguous public base class .Unrelated." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr int e_unrelated = (dynamic_cast<Unrelated&>((E&)g), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const E. of its operand is a non-public base class of dynamic type .G." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic6.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic6.C
index d67c307..4434a1a 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic6.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic6.C
@@ -1,6 +1,22 @@
 // PR c++/88337 - Implement P1327R1: Allow dynamic_cast/typeid in constexpr.
 // { dg-do compile { target c++20 } }
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 // Private base.
 
 struct P1 { virtual void p1(); };
@@ -12,14 +28,16 @@ struct A : B, C, private P2 { virtual void a(); };
 constexpr A a;
 
 // P1 is a non-public base of A.
-constexpr bool b1 = (dynamic_cast<B&>((P1&)a), false); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const P1. of its operand is a non-public base class of dynamic type .A." "" { target *-*-* } .-1 }
+constexpr bool b1 = (dynamic_cast<B&>((P1&)a), false); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const P1. of its operand is a non-public base class of dynamic type .A." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 // Don't error here.
 static_assert (dynamic_cast<B*>((P1*)&a) == nullptr);
 
-constexpr bool b2 = (dynamic_cast<C&>((P2&)a), false); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const P2. of its operand is a non-public base class of dynamic type .A." "" { target *-*-* } .-1 }
+constexpr bool b2 = (dynamic_cast<C&>((P2&)a), false); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const P2. of its operand is a non-public base class of dynamic type .A." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 static_assert (dynamic_cast<C*>((P1*)&a) == nullptr);
 static_assert (dynamic_cast<C*>((P2*)&a) == nullptr);
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic7.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic7.C
index bc3efd0..d504efd 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic7.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic7.C
@@ -1,6 +1,22 @@
 // PR c++/88337 - Implement P1327R1: Allow dynamic_cast/typeid in constexpr.
 // { dg-do compile { target c++20 } }
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 // Protected base.
 
 struct P1 { virtual void p1(); };
@@ -12,14 +28,16 @@ struct A : B, C, protected P2 { virtual void a(); };
 constexpr A a;
 
 // P1 is a non-public base of A.
-constexpr bool b1 = (dynamic_cast<B&>((P1&)a), false); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const P1. of its operand is a non-public base class of dynamic type .A." "" { target *-*-* } .-1 }
+constexpr bool b1 = (dynamic_cast<B&>((P1&)a), false); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const P1. of its operand is a non-public base class of dynamic type .A." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 // Don't error here.
 static_assert (dynamic_cast<B*>((P1*)&a) == nullptr);
 
-constexpr bool b2 = (dynamic_cast<C&>((P2&)a), false); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const P2. of its operand is a non-public base class of dynamic type .A." "" { target *-*-* } .-1 }
+constexpr bool b2 = (dynamic_cast<C&>((P2&)a), false); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const P2. of its operand is a non-public base class of dynamic type .A." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 static_assert (dynamic_cast<C*>((P1*)&a) == nullptr);
 static_assert (dynamic_cast<C*>((P2*)&a) == nullptr);
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic8.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic8.C
index 1958cae4..2fc8242 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic8.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic8.C
@@ -1,6 +1,22 @@
 // PR c++/88337 - Implement P1327R1: Allow dynamic_cast/typeid in constexpr.
 // { dg-do compile { target c++20 } }
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 // Unrelated type.
 
 struct B { virtual void b(); };
@@ -12,12 +28,15 @@ constexpr A a;
 
 struct U { virtual void u(); };
 
-constexpr bool b1 = (dynamic_cast<U&>((B&)a), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "dynamic type .A. of its operand does not have an unambiguous public base class .U." "" { target *-*-* } .-1 }
-constexpr bool b2 = (dynamic_cast<U&>((P1&)a), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const P1. of its operand is a non-public base class of dynamic type .A." "" { target *-*-* } .-1 }
-constexpr bool b3 = (dynamic_cast<U&>((P2&)a), 0); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message "static type .const P2. of its operand is a non-public base class of dynamic type .A." "" { target *-*-* } .-1 }
+constexpr bool b1 = (dynamic_cast<U&>((B&)a), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "dynamic type .A. of its operand does not have an unambiguous public base class .U." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b2 = (dynamic_cast<U&>((P1&)a), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const P1. of its operand is a non-public base class of dynamic type .A." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
+constexpr bool b3 = (dynamic_cast<U&>((P2&)a), 0); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message "static type .const P2. of its operand is a non-public base class of dynamic type .A." "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 static_assert (dynamic_cast<U*>((B*)&a) == nullptr);
 static_assert (dynamic_cast<U*>((P1*)&a) == nullptr);
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic9.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic9.C
index 7d42ffa..3a50dde 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic9.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-dynamic9.C
@@ -1,6 +1,22 @@
 // PR c++/88337 - Implement P1327R1: Allow dynamic_cast/typeid in constexpr.
 // { dg-do compile { target c++20 } }
 
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_cast : public exception {
+    constexpr virtual ~bad_cast () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_cast"; }
+  };
+}
+#endif
+
 // Ambiguous base.
 
 struct A { virtual void a(); };
@@ -11,7 +27,8 @@ struct E : B, C, D { virtual void d(); };
 
 constexpr E e;
 
-constexpr bool b1 = (dynamic_cast<A&>((D&)e), false); // { dg-error "reference .dynamic_cast. failed" }
-// { dg-message ".A. is an ambiguous base class of dynamic type .E. of its operand" "" { target *-*-* } .-1 }
+constexpr bool b1 = (dynamic_cast<A&>((D&)e), false); // { dg-error "reference .dynamic_cast. failed" "" { target c++23_down } }
+// { dg-message ".A. is an ambiguous base class of dynamic type .E. of its operand" "" { target c++23_down } .-1 }
+// { dg-error "uncaught exception" "" { target c++26 } .-2 }
 
 static_assert (dynamic_cast<A*>((D*)&e) == nullptr);
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-new27.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-new27.C
new file mode 100644
index 0000000..a26fc7e
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-new27.C
@@ -0,0 +1,41 @@
+// { dg-do compile { target c++20 } }
+
+#if __cpp_constexpr_exceptions >= 202411L
+namespace std {
+  struct exception {
+    constexpr exception () noexcept {}
+    constexpr virtual ~exception () noexcept {}
+    constexpr exception (const exception &) = default;
+    constexpr exception &operator= (const exception &) = default;
+    constexpr virtual const char *what () const noexcept { return "std::exception"; }
+  };
+  struct bad_alloc : public exception {
+    constexpr virtual ~bad_alloc () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_alloc"; }
+  };
+  struct bad_array_new_length : public bad_alloc {
+    constexpr virtual ~bad_array_new_length () noexcept {}
+    constexpr virtual const char *what () const noexcept { return "std::bad_array_new_length"; }
+  };
+}
+#endif
+
+constexpr int
+foo (__SIZE_TYPE__ x, int y, int z)
+{
+  char (*a)[2] = new char[x][2];	// { dg-error "call to non-'constexpr' function 'void __cxa_throw_bad_array_new_length\\\(\\\)'" "" { target c++23_down } }
+  delete[] a;				// { dg-message "declared here" "" { target c++23_down } .-1 }
+  int *b = new int[y];			// { dg-error "call to non-'constexpr' function 'void __cxa_throw_bad_array_new_length\\\(\\\)'" "" { target c++23_down } }
+  delete[] b;
+  int *c = new int[z]{1, 2, 3};		// { dg-error "call to non-'constexpr' function 'void __cxa_throw_bad_array_new_length\\\(\\\)'" "" { target c++23_down } }
+  delete[] c;
+  return 0;
+}
+
+constexpr int a = foo (16, 2, 3);
+constexpr int b = foo (-64, 2, 3);	// { dg-message "in 'constexpr' expansion of" "" { target c++23_down } }
+					// { dg-error "uncaught exception" "" { target c++26 } .-1 }
+constexpr int c = foo (16, -1, 3);	// { dg-message "in 'constexpr' expansion of" "" { target c++23_down } }
+					// { dg-error "uncaught exception" "" { target c++26 } .-1 }
+constexpr int d = foo (16, 2, 1);	// { dg-message "in 'constexpr' expansion of" "" { target c++23_down } }
+					// { dg-error "uncaught exception" "" { target c++26 } .-1 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-typeid5.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-typeid5.C
new file mode 100644
index 0000000..567383d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-typeid5.C
@@ -0,0 +1,20 @@
+// PR c++/88337 - Implement P1327R1: Allow dynamic_cast/typeid in constexpr.
+// { dg-do compile { target c++17 } }
+
+#include <typeinfo>
+
+template <class T>
+constexpr bool foo ()
+{
+   bool r = false;
+   const std::type_info &s = typeid( (r = true), *(T *) 0);	// { dg-error "call to non-'constexpr' function 'void __cxa_bad_typeid\\\(\\\)'" "" { target c++23_down } }
+   return r;							// { dg-message "declared here" "" { target c++23_down } .-1 }
+}
+
+struct A {};
+struct B { virtual ~B () {} };
+
+static_assert (!foo <int> ());
+static_assert (!foo <A> ());
+constexpr bool a = foo <B> ();	// { dg-message "in 'constexpr' expansion of" "" { target c++23_down } }
+// { dg-error "uncaught exception" "" { target c++26 } .-1 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-union6.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-union6.C
index 00bda53..ab8c979 100644
--- a/gcc/testsuite/g++.dg/cpp2a/constexpr-union6.C
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-union6.C
@@ -45,9 +45,9 @@ constexpr int test5() {
   union {
     int data[1];
   } u;
-  std::construct_at(u.data, 0);  // { dg-message "in .constexpr. expansion" }
+  std::construct_at(u.data, 0);  // { dg-bogus "in .constexpr. expansion" }
   return 0;
 }
-constexpr int x5 = test5();  // { dg-message "in .constexpr. expansion" }
+constexpr int x5 = test5();  // { dg-bogus "in .constexpr. expansion" }
 
 // { dg-error "accessing (uninitialized member|.* member instead of)" "" { target *-*-* } 0 }
diff --git a/gcc/testsuite/g++.dg/cpp2a/constexpr-union9.C b/gcc/testsuite/g++.dg/cpp2a/constexpr-union9.C
new file mode 100644
index 0000000..7db1030
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/constexpr-union9.C
@@ -0,0 +1,33 @@
+// PR c++/120577
+// { dg-do compile { target c++20 } }
+
+template <class _Tp> struct optional {
+  union {
+    _Tp __val_;
+  };
+  template <class... _Args>
+  constexpr optional(_Args... __args)
+      : __val_(__args...) {}
+};
+template <class _Tp, class... _Args>
+constexpr optional<_Tp> make_optional(_Args... __args) {
+  return optional<_Tp>(__args...);
+}
+
+struct __non_trivial_if {
+  constexpr __non_trivial_if() {}
+};
+struct allocator : __non_trivial_if {};
+struct __padding {};
+struct __short {
+  [[__no_unique_address__]] __padding __padding_;
+  int __data_;
+};
+struct basic_string {
+  union {
+    __short __s;
+  };
+  [[__no_unique_address__]] allocator __alloc_;
+  constexpr basic_string(int, int) {}
+};
+auto opt = make_optional<basic_string>(4, 'X');
diff --git a/gcc/testsuite/g++.dg/cpp2a/lambda-targ16.C b/gcc/testsuite/g++.dg/cpp2a/lambda-targ16.C
new file mode 100644
index 0000000..11f2337
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/lambda-targ16.C
@@ -0,0 +1,29 @@
+// PR c++/120748
+// From Clang cxx20-lambda-decltype-this.cpp.
+// { dg-do compile { target c++20 } }
+
+namespace PR45881 {
+struct A {
+    void f();
+};
+int id(A*);
+void A::f() {
+    auto z = [*this](auto z2, decltype(z2(this)) z3){};
+    z(id,3);
+}
+
+struct B {
+    void f();
+};
+void B::f() {
+    auto z = []<typename TT, typename TTT=decltype(TT()(this))>(){return 0;};
+    z.template operator()<int(*)(B*)>();
+}
+struct C {
+    void f();
+};
+void C::f() {
+    auto z = []<typename TT, decltype(TT()(this)) n>(){return 0;};
+    z.template operator()<int(*)(C*), 8>();
+}
+} // namespace PR45881
diff --git a/gcc/testsuite/g++.dg/cpp2a/lambda-targ17.C b/gcc/testsuite/g++.dg/cpp2a/lambda-targ17.C
new file mode 100644
index 0000000..84955ae
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/lambda-targ17.C
@@ -0,0 +1,8 @@
+// PR c++/12012
+// { dg-do compile { target c++20 } }
+
+template<auto> int x;
+
+int main() {
+	x<[](auto) {}>;
+}
diff --git a/gcc/testsuite/g++.dg/cpp2a/lambda-uneval28.C b/gcc/testsuite/g++.dg/cpp2a/lambda-uneval28.C
new file mode 100644
index 0000000..4b1bfc8
--- /dev/null
+++ b/gcc/testsuite/g++.dg/cpp2a/lambda-uneval28.C
@@ -0,0 +1,10 @@
+// PR c++/121008
+// { dg-do compile { target c++20 } }
+
+struct A {
+  void f()
+    noexcept(noexcept([this]() noexcept(noexcept(this)) {}))
+  {}
+};
+
+int main() {}
diff --git a/gcc/testsuite/g++.dg/diagnostic/static_assert5.C b/gcc/testsuite/g++.dg/diagnostic/static_assert5.C
new file mode 100644
index 0000000..16681b2
--- /dev/null
+++ b/gcc/testsuite/g++.dg/diagnostic/static_assert5.C
@@ -0,0 +1,70 @@
+// PR c++/117294
+// { dg-do compile { target c++14 } }
+
+template <typename T> struct norm
+  { static constexpr bool value = __is_constructible(T); };
+template <typename T> constexpr bool norm_v = __is_constructible(T);
+
+template <typename T> struct part
+  { static constexpr bool value = __is_constructible(T); };
+template <typename T> struct part<T*>
+  { static constexpr bool value = false; };
+template <typename T> struct part<const T>
+  { static constexpr bool value = __is_same(T, void); };
+template <typename T> constexpr bool part_v = __is_constructible(T);
+template <typename T> constexpr bool part_v<T*> = false;
+template <typename T> constexpr bool part_v<const T> = __is_same(T, void);
+
+template <typename T> struct expl
+  { static constexpr bool value = __is_constructible(T); };
+template <> struct expl<int*>
+  { static constexpr bool value = false; };
+template <> struct expl<const int>
+  { static constexpr bool value = __is_same(int, void); };
+template <typename T> constexpr bool expl_v = __is_constructible(T);
+template <> constexpr bool expl_v<int*> = false;
+template <> constexpr bool expl_v<const int> = __is_same(int, void);
+
+// === Primary template can give customised diagnostics when using traits
+static_assert(norm<void>::value);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } .-1 }
+static_assert(part<void>::value);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } .-1 }
+static_assert(expl<void>::value);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } .-1 }
+static_assert(norm_v<void>);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } .-1 }
+static_assert(part_v<void>);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } .-1 }
+static_assert(expl_v<void>);  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible" "" { target *-*-* } .-1 }
+
+// { dg-prune-output "'void' is incomplete" }
+
+
+// === Specialisations don't customise just because primary template had trait
+static_assert(part<int*>::value);  // { dg-error "assert" }
+// { dg-bogus "default constructible" "" { target *-*-* } .-1 }
+static_assert(expl<int*>::value);  // { dg-error "assert" }
+// { dg-bogus "default constructible" "" { target *-*-* } .-1 }
+static_assert(part_v<int*>);  // { dg-error "assert" }
+// { dg-bogus "default constructible" "" { target *-*-* } .-1 }
+static_assert(expl_v<int*>);  // { dg-error "assert" }
+// { dg-bogus "default constructible" "" { target *-*-* } .-1 }
+
+
+// === But partial specialisations actually using a trait can customise
+static_assert(part<const int>::value);  // { dg-error "assert" }
+// { dg-message "'int' is not the same as 'void'" "" { target *-*-* } .-1 }
+static_assert(part_v<const int>);  // { dg-error "assert" }
+// { dg-message "'int' is not the same as 'void'" "" { target *-*-* } .-1 }
+
+
+// === For these cases, we no longer know that the error was caused by the trait
+// === because it's been folded away before we process the failure.
+static_assert(expl<const int>::value);  // { dg-error "assert" }
+// { dg-bogus "because" "" { target *-*-* } .-1 }
+static_assert(expl_v<const int>);  // { dg-error "assert" }
+// { dg-bogus "because" "" { target *-*-* } .-1 }
+static_assert(__is_constructible(void));  // { dg-error "assert" }
+// { dg-bogus "because" "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/g++.dg/ext/has_virtual_destructor2.C b/gcc/testsuite/g++.dg/ext/has_virtual_destructor2.C
new file mode 100644
index 0000000..14eea80
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/has_virtual_destructor2.C
@@ -0,0 +1,27 @@
+// { dg-do compile { target c++11 } }
+
+template <typename T> struct has_virtual_destructor {
+  static constexpr bool value = __has_virtual_destructor(T);
+};
+
+static_assert(has_virtual_destructor<int>::value, "");  // { dg-error "assert" }
+// { dg-message "'int' does not have a virtual destructor" "" { target *-*-* } .-1 }
+
+struct A {};  // { dg-message "'A' does not have a virtual destructor" }
+static_assert(has_virtual_destructor<A>::value, "");  // { dg-error "assert" }
+
+struct B {
+  ~B();  // { dg-message "'B' does not have a virtual destructor" }
+};
+static_assert(has_virtual_destructor<B>::value, "");  // { dg-error "assert" }
+
+struct C {  // { dg-bogus "" }
+  virtual ~C();  // { dg-bogus "" }
+};
+static_assert(has_virtual_destructor<C[5]>::value, "");  // { dg-error "assert" }
+// { dg-message "'C \\\[5\\\]' does not have a virtual destructor" "" { target *-*-* } .-1 }
+
+union U {  // { dg-message "'U' does not have a virtual destructor" }
+  ~U();
+};
+static_assert(has_virtual_destructor<U>::value, "");  // { dg-error "assert" }
diff --git a/gcc/testsuite/g++.dg/ext/is_assignable2.C b/gcc/testsuite/g++.dg/ext/is_assignable2.C
new file mode 100644
index 0000000..b346d7b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_assignable2.C
@@ -0,0 +1,36 @@
+// { dg-do compile { target c++11 } }
+
+template <typename T>
+struct is_copy_assignable {
+  static constexpr bool value = __is_assignable(T&, const T&);
+};
+
+static_assert(is_copy_assignable<const int>::value, "");  // { dg-error "assert" }
+// { dg-error "assignment to read-only type 'const int'" "" { target *-*-* } .-1 }
+
+struct A {
+  void operator=(A) = delete;  // { dg-message "declared here" }
+};
+static_assert(is_copy_assignable<A>::value, "");  // { dg-error "assert" }
+// { dg-message "is not assignable" "" { target *-*-* } .-1 }
+// { dg-error "use of deleted function" "" { target *-*-* } .-2 }
+
+template <typename T>
+struct is_nothrow_copy_assignable {
+  static constexpr bool value = __is_nothrow_assignable(T&, const T&);
+};
+struct B {
+  void operator=(const B&);  // { dg-message "noexcept" }
+};
+static_assert(is_nothrow_copy_assignable<B>::value, "");  // { dg-error "assert" }
+// { dg-message "is not nothrow assignable" "" { target *-*-* } .-1 }
+
+template <typename T>
+struct is_trivially_copy_assignable {
+  static constexpr bool value = __is_trivially_assignable(T&, const T&);
+};
+struct C {
+  void operator=(const C&);  // { dg-message "non-trivial" }
+};
+static_assert(is_trivially_copy_assignable<C>::value, "");  // { dg-error "assert" }
+// { dg-message "is not trivially assignable" "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/g++.dg/ext/is_constructible9.C b/gcc/testsuite/g++.dg/ext/is_constructible9.C
new file mode 100644
index 0000000..5448878
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_constructible9.C
@@ -0,0 +1,60 @@
+// { dg-do compile { target c++11 } }
+
+template <typename T, typename... Args>
+struct is_constructible {
+  static constexpr bool value = __is_constructible(T, Args...);
+};
+
+static_assert(is_constructible<void>::value, "");  // { dg-error "assert" }
+// { dg-message "'void' is not default constructible, because" "" { target *-*-* } .-1 }
+// { dg-error "'void' is incomplete" "" { target *-*-* } .-2 }
+
+static_assert(is_constructible<int&, const int&>::value, "");  // { dg-error "assert" }
+// { dg-message "'int&' is not constructible from 'const int&', because" "" { target *-*-* } .-1 }
+// { dg-error "discards qualifiers" "" { target *-*-* } .-2 }
+
+static_assert(is_constructible<int, int, int>::value, "");  // { dg-error "assert" }
+// { dg-message "'int' is not constructible from 'int, int', because" "" { target *-*-* } .-1 }
+// { dg-error "too many initializers for non-class type 'int'" "" { target *-*-* } .-2 }
+
+struct A {
+  A(int);  // { dg-message "candidate" }
+};
+static_assert(is_constructible<A, int, int>::value, "");  // { dg-error "assert" }
+// { dg-message "'A' is not constructible from 'int, int', because" "" { target *-*-* } .-1 }
+// { dg-error "no matching function for call to" "" { target *-*-* } .-2 }
+
+template <typename T, typename... Args>
+struct is_nothrow_constructible {
+  static constexpr bool value = __is_nothrow_constructible(T, Args...);
+};
+
+struct B {
+  B(int);  // { dg-message "candidate" }
+};
+static_assert(is_nothrow_constructible<B>::value, "");  // { dg-error "assert" }
+// { dg-message "'B' is not nothrow default constructible, because" "" { target *-*-* } .-1 }
+// { dg-error "no matching function for call to" "" { target *-*-* } .-2 }
+
+struct C {
+  C(int);  // { dg-message "noexcept" }
+};
+static_assert(is_nothrow_constructible<C, int>::value, "");  // { dg-error "assert" }
+// { dg-message "'C' is not nothrow constructible from 'int', because" "" { target *-*-* } .-1 }
+
+template <typename T, typename... Args>
+struct is_trivially_constructible {
+  static constexpr bool value = __is_trivially_constructible(T, Args...);
+};
+
+struct D {
+  D();  // { dg-message "non-trivial" }
+};
+static_assert(is_trivially_constructible<D>::value, "");  // { dg-error "assert" }
+// { dg-message "'D' is not trivially default constructible, because" "" { target *-*-* } .-1 }
+
+struct E {
+  operator int();  // { dg-message "non-trivial" }
+};
+static_assert(is_trivially_constructible<int, E>::value, "");  // { dg-error "assert" }
+// { dg-message "'int' is not trivially constructible from 'E', because" "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/g++.dg/ext/is_convertible7.C b/gcc/testsuite/g++.dg/ext/is_convertible7.C
new file mode 100644
index 0000000..b38fc04
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_convertible7.C
@@ -0,0 +1,29 @@
+// { dg-do compile { target c++11 } }
+
+template <typename T, typename U>
+struct is_convertible {
+  static constexpr bool value = __is_convertible(T, U);
+};
+
+static_assert(is_convertible<int*, int>::value, "");  // { dg-error "assert" }
+// { dg-error "invalid conversion" "" { target *-*-* } .-1 }
+
+static_assert(is_convertible<int(), double (*)()>::value, "");  // { dg-error "assert" }
+// { dg-error "invalid conversion" "" { target *-*-* } .-1 }
+
+struct A {
+  explicit A(int);
+};
+static_assert(is_convertible<int, A>::value, "");  // { dg-error "assert" }
+// { dg-error "could not convert 'int' to 'A'" "" { target *-*-* } .-1 }
+
+template <typename T, typename U>
+struct is_nothrow_convertible {
+  static constexpr bool value = __is_nothrow_convertible(T, U);
+};
+
+struct B {
+  B(int);  // { dg-message "noexcept" }
+};
+static_assert(is_nothrow_convertible<int, B>::value, "");  // { dg-error "assert" }
+// { dg-message "'int' is not nothrow convertible from 'B', because" "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/g++.dg/ext/is_destructible3.C b/gcc/testsuite/g++.dg/ext/is_destructible3.C
new file mode 100644
index 0000000..a8501d6
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_destructible3.C
@@ -0,0 +1,65 @@
+// { dg-do compile { target c++11 } }
+
+template <typename T>
+struct is_destructible {
+  static constexpr bool value = __is_destructible(T);
+};
+
+static_assert(is_destructible<void>::value, "");  // { dg-error "assert" }
+// { dg-message "'void' is not destructible, because" "" { target *-*-* } .-1 }
+// { dg-error "'void' is incomplete" "" { target *-*-* } .-2 }
+
+struct A {
+  ~A() = delete;  // { dg-message "declared here" }
+};
+static_assert(is_destructible<A>::value, "");  // { dg-error "assert" }
+// { dg-message "'A' is not destructible, because" "" { target *-*-* } .-1 }
+// { dg-error "use of deleted function" "" { target *-*-* } .-2 }
+
+struct B {
+private:
+  ~B();  // { dg-message "declared private here" }
+};
+static_assert(is_destructible<B>::value, "");  // { dg-error "assert" }
+// { dg-message "'B' is not destructible, because" "" { target *-*-* } .-1 }
+// { dg-error "private within this context" "" { target *-*-* } .-2 }
+
+template <typename T>
+struct is_nothrow_destructible {
+  static constexpr bool value = __is_nothrow_destructible(T);
+};
+
+struct C {
+  ~C() noexcept(false);  // { dg-message "noexcept" }
+};
+static_assert(is_nothrow_destructible<C>::value, "");  // { dg-error "assert" }
+// { dg-message "'C' is not nothrow destructible, because" "" { target *-*-* } .-1 }
+
+struct D {
+private:
+  ~D() {}  // { dg-message "declared private here" }
+};
+static_assert(is_nothrow_destructible<D>::value, "");  // { dg-error "assert" }
+// { dg-message "'D' is not nothrow destructible, because" "" { target *-*-* } .-1 }
+// { dg-error "private within this context" "" { target *-*-* } .-2 }
+
+template <typename T>
+struct is_trivially_destructible {
+  static constexpr bool value = __is_trivially_destructible(T);
+};
+
+struct E {
+  ~E();
+};
+struct F { E d; };  // { dg-message "non-trivial" }
+static_assert(is_trivially_destructible<F>::value, "");  // { dg-error "assert" }
+// { dg-message "'F' is not trivially destructible, because" "" { target *-*-* } .-1 }
+
+struct G {
+private:
+  ~G();  // { dg-message "declared private here" }
+};
+struct H { G g; };  // { dg-error "private within this context" }
+static_assert(is_trivially_destructible<H>::value, "");  // { dg-error "assert" }
+// { dg-message "'H' is not trivially destructible, because" "" { target *-*-* } .-1 }
+// { dg-error "use of deleted function" "" { target *-*-* } .-2 }
diff --git a/gcc/testsuite/g++.dg/ext/is_invocable5.C b/gcc/testsuite/g++.dg/ext/is_invocable5.C
new file mode 100644
index 0000000..460eed5
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_invocable5.C
@@ -0,0 +1,15 @@
+// PR c++/121055
+// { dg-do compile { target c++11 } }
+// { dg-skip-if "requires hosted libstdc++ for functional function" { ! hostedlib } }
+
+#include <functional>
+
+#define SA(X) static_assert((X),#X)
+
+struct F;
+
+SA( __is_invocable(void (F::*)() &, std::reference_wrapper<F>) );
+SA( ! __is_invocable(void (F::*)() &&, std::reference_wrapper<F>) );
+
+SA( __is_invocable(void (F::*)(int) &, std::reference_wrapper<F>, int) );
+SA( ! __is_invocable(void (F::*)(int) &&, std::reference_wrapper<F>, int) );
diff --git a/gcc/testsuite/g++.dg/ext/is_invocable6.C b/gcc/testsuite/g++.dg/ext/is_invocable6.C
new file mode 100644
index 0000000..64c5c76
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_invocable6.C
@@ -0,0 +1,45 @@
+// { dg-do compile { target c++11 } }
+
+template <typename F, typename... Args>
+struct is_invocable {
+  static constexpr bool value = __is_invocable(F, Args...);
+};
+
+static_assert(is_invocable<int>::value, "");  // { dg-error "assert" }
+// { dg-message "'int' is not invocable, because" "" { target *-*-* } .-1 }
+// { dg-error "'int' cannot be used as a function" "" { target *-*-* } .-2 }
+
+static_assert(is_invocable<void(*)(), int>::value, "");  // { dg-error "assert" }
+// { dg-message "'void \[^'\]*' is not invocable by 'int', because" "" { target *-*-* } .-1 }
+// { dg-error "too many arguments" "" { target *-*-* } .-2 }
+
+static_assert(is_invocable<void(void*), void() const>::value, "");  // { dg-error "assert" }
+// { dg-message "'void.void..' is not invocable by 'void.. const', because" "" { target *-*-* } .-1 }
+// { dg-error "qualified function type" "" { target *-*-* } .-2 }
+
+struct A {};
+static_assert(is_invocable<const A&&, int, double>::value, "");  // { dg-error "assert" }
+// { dg-message "'const A&&' is not invocable by 'int, double', because" "" { target *-*-* } .-1 }
+// { dg-error "no match for call to " "" { target *-*-* } .-2 }
+
+struct B {
+  void operator()() = delete;  // { dg-message "declared here" }
+};
+static_assert(is_invocable<B>::value, "");  // { dg-error "assert" }
+// { dg-message "'B' is not invocable, because" "" { target *-*-* } .-1 }
+// { dg-error "use of deleted function" "" { target *-*-* } .-2 }
+
+template <typename F, typename... Args>
+struct is_nothrow_invocable {
+  static constexpr bool value = __is_nothrow_invocable(F, Args...);
+};
+
+static_assert(is_nothrow_invocable<void(*)()>::value, "");  // { dg-error "assert" }
+// { dg-message "'void \[^'\]*' is not nothrow invocable, because" "" { target *-*-* } .-1 }
+// { dg-message "'void \[^'\]*' is not 'noexcept'" "" { target *-*-* } .-2 }
+
+struct C {
+  int operator()(int, double) const;  // { dg-message "noexcept" }
+};
+static_assert(is_nothrow_invocable<const C&, int, int>::value, "");  // { dg-error "assert" }
+// { dg-message "'const C&' is not nothrow invocable by 'int, int', because" "" { target *-*-* } .-1 }
diff --git a/gcc/testsuite/g++.dg/ext/is_invocable7.C b/gcc/testsuite/g++.dg/ext/is_invocable7.C
new file mode 100644
index 0000000..5c852fc
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_invocable7.C
@@ -0,0 +1,21 @@
+// PR c++/121291
+// { dg-do compile { target c++17 } }
+
+template <typename T>
+constexpr bool is_invocable = __is_invocable(T);
+
+template <typename T>
+constexpr bool is_nothrow_invocable = __is_nothrow_invocable(T);
+
+struct S {
+private:
+  int operator()() noexcept;  // { dg-message "here" }
+};
+
+static_assert(is_invocable<S>);  // { dg-error "assert" }
+// { dg-message "not invocable" "" { target *-*-* } .-1 }
+// { dg-error "private within this context" "" { target *-*-* } .-2 }
+
+static_assert(is_nothrow_invocable<S>);  // { dg-error "assert" }
+// { dg-message "not nothrow invocable" "" { target *-*-* } .-1 }
+// { dg-error "private within this context" "" { target *-*-* } .-2 }
diff --git a/gcc/testsuite/g++.dg/ext/is_nothrow_convertible5.C b/gcc/testsuite/g++.dg/ext/is_nothrow_convertible5.C
new file mode 100644
index 0000000..0ce8fb8
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_nothrow_convertible5.C
@@ -0,0 +1,15 @@
+// PR c++/121291
+// { dg-do compile { target c++17 } }
+
+template <typename T, typename U>
+constexpr bool is_nothrow_convertible = __is_nothrow_convertible(T, U);
+
+struct A {};
+struct B {
+private:
+  operator A() noexcept;  // { dg-message "here" }
+};
+
+static_assert(is_nothrow_convertible<B, A>);  // { dg-error "assert" }
+// { dg-message "not nothrow convertible" "" { target *-*-* } .-1 }
+// { dg-error "private within this context" "" { target *-*-* } .-2 }
diff --git a/gcc/testsuite/g++.dg/ext/is_virtual_base_of_diagnostic2.C b/gcc/testsuite/g++.dg/ext/is_virtual_base_of_diagnostic2.C
new file mode 100644
index 0000000..ac28121
--- /dev/null
+++ b/gcc/testsuite/g++.dg/ext/is_virtual_base_of_diagnostic2.C
@@ -0,0 +1,13 @@
+// { dg-do compile { target c++11 } }
+
+template <typename T, typename U>
+struct is_virtual_base_of {
+  static constexpr bool value = __builtin_is_virtual_base_of(T, U);
+};
+
+static_assert(is_virtual_base_of<int, int>::value, "");  // { dg-error "assert" }
+// { dg-message "'int' is not a virtual base of 'int'" "" { target *-*-* } .-1 }
+
+struct A {};  // { dg-message "'A' is not a virtual base of 'B'" }
+struct B : A {};  // { dg-message "declared here" }
+static_assert(is_virtual_base_of<A, B>::value, "");  // { dg-error "assert" }
diff --git a/gcc/testsuite/g++.dg/lookup/operator-8.C b/gcc/testsuite/g++.dg/lookup/operator-8.C
index 64d8a97..32d432d 100644
--- a/gcc/testsuite/g++.dg/lookup/operator-8.C
+++ b/gcc/testsuite/g++.dg/lookup/operator-8.C
@@ -16,11 +16,16 @@ struct A {
 template<class T>
 void f() {
   A a;
-  (void)(a != 0, 0 != a); // { dg-bogus "deleted" "" { xfail *-*-* } }
-  (void)(a < 0, 0 < a);   // { dg-bogus "deleted" "" { xfail *-*-* } }
-  (void)(a <= 0, 0 <= a); // { dg-bogus "deleted" "" { xfail *-*-* } }
-  (void)(a > 0, 0 > a);   // { dg-bogus "deleted" "" { xfail *-*-* } }
-  (void)(a >= 0, 0 >= a); // { dg-bogus "deleted" "" { xfail *-*-* } }
+  (void)(a != 0);
+  (void)(0 != a);
+  (void)(a < 0);
+  (void)(0 < a);
+  (void)(a <= 0);
+  (void)(0 <= a);
+  (void)(a > 0);
+  (void)(0 > a);
+  (void)(a >= 0);
+  (void)(0 >= a);
 }
 
 // These later-declared namespace-scope overloads shouldn't be considered
@@ -31,4 +36,10 @@ bool operator<=(A, int) = delete;
 bool operator>(A, int) = delete;
 bool operator>=(A, int) = delete;
 
+bool operator!=(int, A) = delete;
+bool operator<(int, A) = delete;
+bool operator<=(int, A) = delete;
+bool operator>(int, A) = delete;
+bool operator>=(int, A) = delete;
+
 template void f<int>();
diff --git a/gcc/testsuite/g++.dg/lto/pr114790_0.C b/gcc/testsuite/g++.dg/lto/pr114790_0.C
new file mode 100644
index 0000000..eed112d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/lto/pr114790_0.C
@@ -0,0 +1,16 @@
+// { dg-lto-do link }
+// { dg-lto-options { { -w -flto -g -flto-partition=1to1 -O2 -shared -fPIC -fvisibility=hidden} } }
+// { dg-require-effective-target fpic }
+// { dg-require-effective-target shared }
+struct APITracerContext {
+  virtual ~APITracerContext() = default;
+  virtual void releaseActivetracersList() = 0;
+};
+struct APITracerContextImp : APITracerContext {
+  ~APITracerContextImp() override;
+  void releaseActivetracersList() override;
+};
+struct APITracerContextImp globalAPITracerContextImp;
+struct APITracerContextImp *pGlobalAPITracerContextImp = &globalAPITracerContextImp;
+APITracerContextImp::~APITracerContextImp() {}
+
diff --git a/gcc/testsuite/g++.dg/lto/pr114790_1.C b/gcc/testsuite/g++.dg/lto/pr114790_1.C
new file mode 100644
index 0000000..511fae4
--- /dev/null
+++ b/gcc/testsuite/g++.dg/lto/pr114790_1.C
@@ -0,0 +1,15 @@
+struct APITracerContext {
+  virtual void releaseActivetracersList() = 0;
+};
+extern struct APITracerContextImp *pGlobalAPITracerContextImp;
+struct APITracerContextImp : APITracerContext { void releaseActivetracersList();};
+int g();
+inline int
+apiTracerWrapperImp(  ) {
+  for (int i = 0; i < g(); i++) 
+  pGlobalAPITracerContextImp->releaseActivetracersList();
+}
+__attribute__((visibility("default"))) int
+zeCommandListAppendMemoryCopyTracing() {
+  return apiTracerWrapperImp(      );
+}
diff --git a/gcc/testsuite/g++.dg/missing-return.C b/gcc/testsuite/g++.dg/missing-return.C
index 5f8e2cc..f6934b0 100644
--- a/gcc/testsuite/g++.dg/missing-return.C
+++ b/gcc/testsuite/g++.dg/missing-return.C
@@ -5,4 +5,6 @@ int foo(int a)
 {
 } /* { dg-warning "no return statement" } */
 
-/* { dg-final { scan-tree-dump "__builtin_unreachable" "optimized" } } */
+/* For targets without traps, it will be an infinite loop */
+/* { dg-final { scan-tree-dump "__builtin_unreachable" "optimized" { target trap } } } */
+/* { dg-final { scan-tree-dump "goto <" "optimized" { target { ! trap } } } } */
diff --git a/gcc/testsuite/g++.dg/modules/class-11_a.H b/gcc/testsuite/g++.dg/modules/class-11_a.H
new file mode 100644
index 0000000..799dbdd
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/class-11_a.H
@@ -0,0 +1,36 @@
+// Check for some additional lang_type flags that we'd missed.
+// { dg-additional-options "-fmodule-header -fabi-version=21 -Wabi=15" }
+// { dg-module-cmi {} }
+
+#if __cpp_trivial_relocatability < 202502L
+#define trivially_relocatable_if_eligible __trivially_relocatable_if_eligible
+#define replaceable_if_eligible __replaceable_if_eligible
+#endif
+
+struct A trivially_relocatable_if_eligible { A(A&&); };
+struct B replaceable_if_eligible { B(B&&); B& operator=(B&&); };
+struct C {};
+static_assert(__builtin_is_trivially_relocatable(C) && __builtin_is_replaceable(C), "");
+
+
+struct pr106381 {
+  long l;
+  char c = -1;
+};
+struct L1 : pr106381 {
+  char x;  // { dg-warning "offset" "" { target c++14 } }
+};
+static_assert(sizeof(L1) == sizeof(pr106381), "");
+
+
+struct pr120012 {
+  pr120012(const pr120012&) = default;
+  pr120012(pr120012&&) = default;
+  pr120012& operator=(pr120012&&) = default;
+  unsigned int a;
+  unsigned char b;
+};
+struct L2 : pr120012 {
+  unsigned char y;  // { dg-warning "offset" "" { target c++20 } }
+};
+static_assert(sizeof(L2) > sizeof(pr120012), "");
diff --git a/gcc/testsuite/g++.dg/modules/class-11_b.C b/gcc/testsuite/g++.dg/modules/class-11_b.C
new file mode 100644
index 0000000..2450a45
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/class-11_b.C
@@ -0,0 +1,15 @@
+// { dg-additional-options "-fmodules -fabi-version=21 -Wabi=15" }
+
+import "class-11_a.H";
+
+static_assert(__builtin_is_trivially_relocatable(A), "");
+static_assert(__builtin_is_replaceable(B), "");
+static_assert(__builtin_is_trivially_relocatable(C) && __builtin_is_replaceable(C), "");
+
+struct M1 : pr106381 {
+  char x;  // { dg-warning "offset" "" { target c++14 } }
+};
+
+struct M2 : pr120012 {
+  unsigned char y;  // { dg-warning "offset" "" { target c++20 } }
+};
diff --git a/gcc/testsuite/g++.dg/modules/internal-14_a.C b/gcc/testsuite/g++.dg/modules/internal-14_a.C
new file mode 100644
index 0000000..07eb965
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/internal-14_a.C
@@ -0,0 +1,17 @@
+// PR c++/120412
+// { dg-additional-options "-fmodules -std=c++20 -Wtemplate-names-tu-local" }
+// { dg-module-cmi m:part }
+
+export module m:part;
+
+export template <typename F>
+auto fun1(F) {
+  return true;
+}
+
+using Dodgy = decltype([]{});
+
+export template <typename T>
+auto fun2(T&&) {  // { dg-warning "TU-local" }
+  return fun1(Dodgy{});
+}
diff --git a/gcc/testsuite/g++.dg/modules/internal-14_b.C b/gcc/testsuite/g++.dg/modules/internal-14_b.C
new file mode 100644
index 0000000..ad3b09d
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/internal-14_b.C
@@ -0,0 +1,6 @@
+// PR c++/120412
+// { dg-additional-options "-fmodules -std=c++20 -Wtemplate-names-tu-local" }
+// { dg-module-cmi m }
+
+export module m;
+export import :part;
diff --git a/gcc/testsuite/g++.dg/modules/internal-14_c.C b/gcc/testsuite/g++.dg/modules/internal-14_c.C
new file mode 100644
index 0000000..4f8e785c
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/internal-14_c.C
@@ -0,0 +1,9 @@
+// PR c++/120412
+// { dg-additional-options "-fmodules -std=c++20" }
+
+import m;
+
+int main() {
+  // { dg-error "instantiation exposes TU-local entity '(fun1|Dodgy)'" "" { target *-*-* } 0 }
+  fun2(123);  // { dg-message "required from here" }
+}
diff --git a/gcc/testsuite/g++.dg/modules/merge-19.h b/gcc/testsuite/g++.dg/modules/merge-19.h
new file mode 100644
index 0000000..c3faadc
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/merge-19.h
@@ -0,0 +1,21 @@
+// PR c++/121238
+
+inline void inc(const char*& __first) {
+  ++__first;
+}
+
+template <typename = void>
+bool parse_integer(const char *first) {
+  const char *start = first;
+  inc(first);
+  return first != start;
+}
+template bool parse_integer<void>(const char*);
+
+
+struct S { ~S() {} int x; };
+template <typename = void>
+bool take_by_invisiref(S s) {
+  return s.x == 5;
+}
+template bool take_by_invisiref<void>(S);
diff --git a/gcc/testsuite/g++.dg/modules/merge-19_a.H b/gcc/testsuite/g++.dg/modules/merge-19_a.H
new file mode 100644
index 0000000..149a447
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/merge-19_a.H
@@ -0,0 +1,5 @@
+// PR c++/121238
+// { dg-additional-options "-fmodule-header" }
+// { dg-module-cmi {} }
+
+#include "merge-19.h"
diff --git a/gcc/testsuite/g++.dg/modules/merge-19_b.C b/gcc/testsuite/g++.dg/modules/merge-19_b.C
new file mode 100644
index 0000000..345e7fe
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/merge-19_b.C
@@ -0,0 +1,16 @@
+// PR c++/121238
+// { dg-module-do run }
+// { dg-additional-options "-fmodules -fno-module-lazy" }
+
+#include "merge-19.h"
+import "merge-19_a.H";
+
+int main() {
+  const char fmt[] = "5";
+  if (!parse_integer<void>(fmt))
+    __builtin_abort();
+
+  S s{ 5 };
+  if (!take_by_invisiref(s))
+    __builtin_abort();
+}
diff --git a/gcc/testsuite/g++.dg/modules/pr108080.H b/gcc/testsuite/g++.dg/modules/pr108080.H
new file mode 100644
index 0000000..b05d957
--- /dev/null
+++ b/gcc/testsuite/g++.dg/modules/pr108080.H
@@ -0,0 +1,5 @@
+// PR c++/108080
+// { dg-additional-options "-fmodules" }
+// Give a diagnostic message rather than a crash for unsupported features.
+
+[[gnu::optimize("-O3")]] void foo();  // { dg-warning "optimize" }
diff --git a/gcc/testsuite/g++.dg/parse/pr120940.C b/gcc/testsuite/g++.dg/parse/pr120940.C
new file mode 100644
index 0000000..5da36b2
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/pr120940.C
@@ -0,0 +1,18 @@
+// PR c++/120940
+// { dg-do run }
+
+int a[8] = { 1, 2, 3, 4, 5, 6, 7, 8 };
+int b[8] = { 9, 10, 11, 12, 13, 14, 15, 16 };
+
+__attribute__((noipa)) int
+foo (int x, int y)
+{
+  return (x ? a : b)[y];
+}
+
+int
+main ()
+{
+  if (foo (1, 4) != 5 || foo (0, 6) != 15)
+    __builtin_abort ();
+}
diff --git a/gcc/testsuite/g++.dg/parse/template32.C b/gcc/testsuite/g++.dg/parse/template32.C
new file mode 100644
index 0000000..b090f40
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/template32.C
@@ -0,0 +1,13 @@
+// PR c++/119838
+// { dg-do compile { target c++11 } }
+
+template<typename T>
+struct S { using U = T; static const int x = 0; };
+void
+g ()
+{
+  ::S<int>::U a;
+  ::template S<int>::U b;
+  auto c = ::S<int>::x;
+  auto d = ::template S<int>::x;
+}
diff --git a/gcc/testsuite/g++.dg/parse/union1.C b/gcc/testsuite/g++.dg/parse/union1.C
new file mode 100644
index 0000000..d567ea3
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/union1.C
@@ -0,0 +1,19 @@
+// PR c++/83469
+// { dg-do compile }
+
+struct S {
+  union U { int m; };
+};
+
+template <typename T>
+void
+f ()
+{
+  union T::U u;
+}
+
+int
+main()
+{
+  f<S>();
+}
diff --git a/gcc/testsuite/g++.dg/parse/union2.C b/gcc/testsuite/g++.dg/parse/union2.C
new file mode 100644
index 0000000..cdb1392
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/union2.C
@@ -0,0 +1,19 @@
+// PR c++/83469
+// { dg-do compile }
+
+struct S {
+  union U { int m; };
+};
+
+template <typename T>
+void
+f ()
+{
+  struct T::U u;  // { dg-error "not a non-union class type" }
+}
+
+int
+main()
+{
+  f<S>();
+}
diff --git a/gcc/testsuite/g++.dg/parse/union3.C b/gcc/testsuite/g++.dg/parse/union3.C
new file mode 100644
index 0000000..61552a4
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/union3.C
@@ -0,0 +1,19 @@
+// PR c++/83469
+// { dg-do compile }
+
+struct S {
+  struct C { int m; };
+};
+
+template <typename T>
+void
+f ()
+{
+  union T::C u; // { dg-error "not a union type" }
+}
+
+int
+main()
+{
+  f<S>();
+}
diff --git a/gcc/testsuite/g++.dg/parse/union4.C b/gcc/testsuite/g++.dg/parse/union4.C
new file mode 100644
index 0000000..709f6a4
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/union4.C
@@ -0,0 +1,12 @@
+// PR c++/93809
+// { dg-do compile }
+
+class C { };
+enum E { };
+struct S { };
+union U { };
+
+typedef typename ::C C2;
+typedef typename ::E E2;
+typedef typename ::S S2;
+typedef typename ::U U2;
diff --git a/gcc/testsuite/g++.dg/parse/union5.C b/gcc/testsuite/g++.dg/parse/union5.C
new file mode 100644
index 0000000..18238dd
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/union5.C
@@ -0,0 +1,5 @@
+// PR c++/93809
+// { dg-do compile { target c++11 } }
+
+union U {};
+auto var = new (typename ::U);
diff --git a/gcc/testsuite/g++.dg/parse/union6.C b/gcc/testsuite/g++.dg/parse/union6.C
new file mode 100644
index 0000000..61b9568
--- /dev/null
+++ b/gcc/testsuite/g++.dg/parse/union6.C
@@ -0,0 +1,5 @@
+// PR c++/93809
+// { dg-do compile }
+
+typedef union{} U;
+typename ::U foo () { return U();  }
diff --git a/gcc/testsuite/g++.dg/plugin/show-template-tree-color-labels.C b/gcc/testsuite/g++.dg/plugin/show-template-tree-color-labels.C
index 3e93126..71ae8f2 100644
--- a/gcc/testsuite/g++.dg/plugin/show-template-tree-color-labels.C
+++ b/gcc/testsuite/g++.dg/plugin/show-template-tree-color-labels.C
@@ -1,4 +1,4 @@
-/* Verify colorization of the labels in diagnostic-show-locus.c
+/* Verify colorization of the labels in diagnostics/source-printing.cc
    for template comparisons.
    Doing so requires a plugin; see the comments in the plugin for the
    rationale.  */
diff --git a/gcc/testsuite/g++.dg/plugin/show_template_tree_color_plugin.cc b/gcc/testsuite/g++.dg/plugin/show_template_tree_color_plugin.cc
index 6151b6e..5d185ff 100644
--- a/gcc/testsuite/g++.dg/plugin/show_template_tree_color_plugin.cc
+++ b/gcc/testsuite/g++.dg/plugin/show_template_tree_color_plugin.cc
@@ -21,7 +21,8 @@
 int plugin_is_GPL_compatible;
 
 void
-noop_text_starter_fn (diagnostic_text_output_format &, const diagnostic_info *)
+noop_text_starter_fn (diagnostics::text_sink &,
+		      const diagnostics::diagnostic_info *)
 {
 }
 
@@ -32,7 +33,7 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  diagnostic_text_starter (global_dc) = noop_text_starter_fn;
+  diagnostics::text_starter (global_dc) = noop_text_starter_fn;
 
   return 0;
 }
diff --git a/gcc/testsuite/g++.dg/tc1/dr49.C b/gcc/testsuite/g++.dg/tc1/dr49.C
index 753d96b..6ddea6b 100644
--- a/gcc/testsuite/g++.dg/tc1/dr49.C
+++ b/gcc/testsuite/g++.dg/tc1/dr49.C
@@ -10,8 +10,8 @@ template struct R<&p>; // OK
 template struct S<&p>; // OK due to parameter adjustment
 
 int *ptr;
-template struct R<ptr>; // { dg-error "argument" }
-template struct S<ptr>; // { dg-error "argument" }
+template struct R<ptr>; // { dg-error "template argument|constant expression" }
+template struct S<ptr>; // { dg-error "template argument|constant expression" }
 
 int v[5];
 template struct R<v>; // OK due to implicit argument conversion
diff --git a/gcc/testsuite/g++.dg/template/error45.C b/gcc/testsuite/g++.dg/template/error45.C
index 064554d..f4c6560 100644
--- a/gcc/testsuite/g++.dg/template/error45.C
+++ b/gcc/testsuite/g++.dg/template/error45.C
@@ -11,7 +11,7 @@ struct enable_if< true, T >
 
 template < typename T >
 struct enable_if< true, T >::type
-f( T x );			// { dg-error "not a class type" }
+f( T x );			// { dg-error "not a non-union class type" }
 
 void
 g( void )
diff --git a/gcc/testsuite/g++.dg/template/func2.C b/gcc/testsuite/g++.dg/template/func2.C
index 0116f23..360f430 100644
--- a/gcc/testsuite/g++.dg/template/func2.C
+++ b/gcc/testsuite/g++.dg/template/func2.C
@@ -4,8 +4,7 @@ typedef void (*fptr)();
 fptr zeroptr = 0;
 template<typename T, fptr F> struct foo { };
 template<typename T> struct foo<T,zeroptr> { };
-// { dg-error "not a valid template argument" "not valid" { target *-*-* } .-1 } 
-// { dg-message "must be the address" "must be the address " { target *-*-* } .-2 }
+// { dg-error "template argument|constant expression" "not valid" { target *-*-* } .-1 } 
 
 // The rest is needed to trigger the ICE in 4.0 to 4.3:
 void f() { }
diff --git a/gcc/testsuite/g++.dg/template/permissive-error3.C b/gcc/testsuite/g++.dg/template/permissive-error3.C
new file mode 100644
index 0000000..988b7fa
--- /dev/null
+++ b/gcc/testsuite/g++.dg/template/permissive-error3.C
@@ -0,0 +1,12 @@
+// PR c++/120575
+// { dg-additional-options -Wno-template-body }
+
+template< int >
+struct T {};
+
+template< int >
+struct S {
+
+  operator typename T< oops >::anything () {};
+
+};
diff --git a/gcc/testsuite/g++.dg/torture/pr120119-1.C b/gcc/testsuite/g++.dg/torture/pr120119-1.C
new file mode 100644
index 0000000..1206feb
--- /dev/null
+++ b/gcc/testsuite/g++.dg/torture/pr120119-1.C
@@ -0,0 +1,15 @@
+// { dg-do compile }
+// { dg-additional-options "-mcpu=cortex-a57" { target aarch64*-*-* } }
+
+// PR target/120119
+
+struct a {
+  float operator()(int b, int c) { return d[c * 4 + b]; }
+  float *d;
+};
+float e(float *);
+auto f(a b) {
+  float g[]{b(1, 1), b(2, 1), b(3, 1), b(1, 2), b(2, 2), b(3, 2), b(1, 3),
+            b(2, 3), b(3, 3), b(3, 2), b(1, 3), b(2, 3), b(3, 3)};
+  return b.d[0] * e(g);
+}
diff --git a/gcc/testsuite/g++.dg/tree-prof/eh1.C b/gcc/testsuite/g++.dg/tree-prof/eh1.C
new file mode 100644
index 0000000..10a3596
--- /dev/null
+++ b/gcc/testsuite/g++.dg/tree-prof/eh1.C
@@ -0,0 +1,34 @@
+/* { dg-options "-O3 -fdump-ipa-profile-details -fno-inline -fdump-tree-fixup_cfg3-details -fdump-tree-optimized-details" } */
+char a[10000];
+char b[10000];
+int sz = 1000;
+
+__attribute__((noipa))
+     void test2 ()
+{
+  throw (sz);
+}
+void
+test ()
+{
+  try
+  {
+    test2 ();
+  }
+  catch (int v)
+  {
+    __builtin_memcpy (b, a, v);
+  }
+}
+int
+main ()
+{
+  for (int i = 0; i < 100000; i++)
+    test ();
+}
+/* { dg-final-use-not-autofdo { scan-ipa-dump-times "Average value sum:100000000" 2 "profile" } } */
+/* 1 zero count for resx block.  */
+/* { dg-final-use-not-autofdo { scan-tree-dump-times "count: 0" 1 "fixup_cfg3" } } */
+/* 2 zero count for resx block and return block since return gets duplicated by tracer.  */
+/* { dg-final-use-not-autofdo { scan-tree-dump-times "count: 0" 2 "optimized" } } */
+/* { dg-final-use-not-autofdo { scan-tree-dump-times "Average value sum:100000000" 1 "optimized" } } */
diff --git a/gcc/testsuite/g++.dg/warn/Wduplicated-branches9.C b/gcc/testsuite/g++.dg/warn/Wduplicated-branches9.C
new file mode 100644
index 0000000..f9fafcd
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/Wduplicated-branches9.C
@@ -0,0 +1,11 @@
+// PR c++/120940
+// { dg-do compile }
+// { dg-options "-Wduplicated-branches" }
+
+static char a[16][8], b[16][8];
+
+char *
+foo (int x, int y)
+{
+  return (x ? a : b)[y];
+}
diff --git a/gcc/testsuite/g++.dg/warn/Wformat-gcc_diag-1.C b/gcc/testsuite/g++.dg/warn/Wformat-gcc_diag-1.C
index dd41b08..db85150 100644
--- a/gcc/testsuite/g++.dg/warn/Wformat-gcc_diag-1.C
+++ b/gcc/testsuite/g++.dg/warn/Wformat-gcc_diag-1.C
@@ -29,6 +29,8 @@ typedef struct diagnostic_event_id_t diagnostic_event_id_t;
 namespace pp_markup { class element; }
 typedef pp_markup::element pp_element;
 
+typedef class string_slice string_slice;
+
 #define FORMAT(kind) __attribute__ ((format (__gcc_## kind ##__, 1, 2)))
 
 void diag (const char*, ...) FORMAT (diag);
@@ -63,7 +65,7 @@ void test_diag (tree t, gimple *gc, diagnostic_event_id_t *event_id_ptr,
   diag ("%e", 42); /* { dg-warning "format" } */
 }
 
-void test_cdiag (tree t, gimple *gc)
+void test_cdiag (tree t, gimple *gc, string_slice *s)
 {
   cdiag ("%<");   /* { dg-warning "unterminated quoting directive" } */
   cdiag ("%>");   /* { dg-warning "unmatched quoting directive " } */
@@ -74,6 +76,7 @@ void test_cdiag (tree t, gimple *gc)
   cdiag ("%F", t);       /* { dg-warning ".F. conversion used unquoted" } */
   cdiag ("%G", gc);      /* { dg-warning "format" } */
   cdiag ("%K", t);       /* { dg-warning "format" } */
+  cdiag ("%B", s);
 
   cdiag ("%R");       /* { dg-warning "unmatched color reset directive" } */
   cdiag ("%r", "");   /* { dg-warning "unterminated color directive" } */
@@ -90,6 +93,7 @@ void test_cdiag (tree t, gimple *gc)
   cdiag ("%<%F%>", t);
   cdiag ("%<%G%>", gc);  /* { dg-warning "format" } */
   cdiag ("%<%K%>", t);   /* { dg-warning "format" } */
+  cdiag ("%<%B%>", s);
 
   cdiag ("%<%R%>");      /* { dg-warning "unmatched color reset directive" } */
   cdiag ("%<%r%>", "");  /* { dg-warning "unterminated color directive" } */
@@ -101,9 +105,10 @@ void test_cdiag (tree t, gimple *gc)
   cdiag ("%<%qD%>", t);  /* { dg-warning ".q. flag used within a quoted sequence" } */
   cdiag ("%<%qE%>", t);  /* { dg-warning ".q. flag used within a quoted sequence" } */
   cdiag ("%<%qT%>", t);  /* { dg-warning ".q. flag used within a quoted sequence" } */
+  cdiag ("%<%qB%>", s);  /* { dg-warning ".q. flag used within a quoted sequence" } */
 }
 
-void test_tdiag (tree t, gimple *gc)
+void test_tdiag (tree t, gimple *gc, string_slice *s)
 {
   tdiag ("%<");       /* { dg-warning "unterminated quoting directive" } */
   tdiag ("%>");       /* { dg-warning "unmatched quoting directive " } */
@@ -113,6 +118,7 @@ void test_tdiag (tree t, gimple *gc)
   tdiag ("%E", t);
   tdiag ("%G", gc);     /* { dg-warning "format" } */
   tdiag ("%K", t);      /* { dg-warning "format" } */
+  tdiag ("%B", s);
 
   tdiag ("%R");          /* { dg-warning "unmatched color reset directive" } */
   tdiag ("%r", "");   /* { dg-warning "unterminated color directive" } */
@@ -138,9 +144,10 @@ void test_tdiag (tree t, gimple *gc)
   tdiag ("%<%qD%>", t);  /* { dg-warning ".q. flag used within a quoted sequence" } */
   tdiag ("%<%qE%>", t);  /* { dg-warning ".q. flag used within a quoted sequence" } */
   tdiag ("%<%qT%>", t);  /* { dg-warning ".q. flag used within a quoted sequence" } */
+  tdiag ("%<%qB%>", s);  /* { dg-warning ".q. flag used within a quoted sequence" } */
 }
 
-void test_cxxdiag (tree t, gimple *gc)
+void test_cxxdiag (tree t, gimple *gc, string_slice *s)
 {
   cxxdiag ("%A", t);     /* { dg-warning ".A. conversion used unquoted" } */
   cxxdiag ("%D", t);     /* { dg-warning ".D. conversion used unquoted" } */
@@ -148,6 +155,7 @@ void test_cxxdiag (tree t, gimple *gc)
   cxxdiag ("%F", t);     /* { dg-warning ".F. conversion used unquoted" } */
   cxxdiag ("%G", gc);    /* { dg-warning "format" } */
   cxxdiag ("%K", t);     /* { dg-warning "format" } */
+  cxxdiag ("%B", s);
 
   cxxdiag ("%R");        /* { dg-warning "unmatched color reset directive" } */
   cxxdiag ("%r", "");    /* { dg-warning "unterminated color directive" } */
@@ -172,9 +180,10 @@ void test_cxxdiag (tree t, gimple *gc)
   cxxdiag ("%<%T%>", t);
   cxxdiag ("%<%V%>", t);
   cxxdiag ("%<%X%>", t);
+  cxxdiag ("%<%B%>", s);
 }
 
-void test_dump (tree t, gimple *stmt, cgraph_node *node)
+void test_dump (tree t, gimple *stmt, cgraph_node *node, string_slice *s)
 {
   dump ("%<");   /* { dg-warning "unterminated quoting directive" } */
   dump ("%>");   /* { dg-warning "unmatched quoting directive " } */
@@ -197,4 +206,5 @@ void test_dump (tree t, gimple *stmt, cgraph_node *node)
   dump ("%C", node);
   dump ("%f", 1.0);
   dump ("%4.2f", 1.0); /* { dg-warning "format" } */
+  dump ("%B", s);
 }
diff --git a/gcc/testsuite/g++.dg/warn/Wmismatched-new-delete-5.C b/gcc/testsuite/g++.dg/warn/Wmismatched-new-delete-5.C
index bac2b68..a21e864 100644
--- a/gcc/testsuite/g++.dg/warn/Wmismatched-new-delete-5.C
+++ b/gcc/testsuite/g++.dg/warn/Wmismatched-new-delete-5.C
@@ -31,7 +31,7 @@ void warn_placement_new ()
 
 void warn_placement_array_new ()
 {
-  void *p = malloc (sizeof (int));
+  void *p = malloc (sizeof (int) * 2);
   int *q = new (p) int[2];
   delete q;                   // { dg-warning "-Wmismatched-new-delete" }
 }
diff --git a/gcc/testsuite/g++.dg/warn/Wredundant-tags-3.C b/gcc/testsuite/g++.dg/warn/Wredundant-tags-3.C
index 0eeee34..dcccdca 100644
--- a/gcc/testsuite/g++.dg/warn/Wredundant-tags-3.C
+++ b/gcc/testsuite/g++.dg/warn/Wredundant-tags-3.C
@@ -28,7 +28,7 @@ struct N::S s3;       // { dg-warning "-Wredundant-tags" }
 
 N::U u1;
 typename N::U u2;     // { dg-bogus "-Wredundant-tags" }
-                      // { dg-bogus "'class' tag used in naming 'union N::U" "pr93809" { xfail *-*-*} .-1 }
+                      // { dg-bogus "'class' tag used in naming 'union N::U" "pr93809" { target *-*-*} .-1 }
 union N::U u3;        // { dg-warning "-Wredundant-tags" }
 
 
diff --git a/gcc/testsuite/g++.dg/warn/Wunused-parm-12.C b/gcc/testsuite/g++.dg/warn/Wunused-parm-12.C
new file mode 100644
index 0000000..03029f9
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/Wunused-parm-12.C
@@ -0,0 +1,59 @@
+// PR c/44677
+// { dg-do compile }
+// { dg-options "-O2 -Wunused-but-set-parameter" }
+
+void baz (int);
+
+template <int N>
+void
+foo (int a,		// { dg-warning "parameter 'a' set but not used" }
+     int b,		// { dg-warning "parameter 'b' set but not used" }
+     int c,		// { dg-warning "parameter 'c' set but not used" }
+     int d,		// { dg-warning "parameter 'd' set but not used" }
+     int e,		// { dg-warning "parameter 'e' set but not used" }
+     int f,		// { dg-warning "parameter 'f' set but not used" }
+     int g,		// { dg-warning "parameter 'g' set but not used" }
+     int h,		// { dg-warning "parameter 'h' set but not used" }
+     int i,		// { dg-warning "parameter 'i' set but not used" }
+     int j,		// { dg-warning "parameter 'j' set but not used" }
+     int k,		// { dg-warning "parameter 'k' set but not used" }
+     int l,		// { dg-warning "parameter 'l' set but not used" }
+     int m)		// { dg-warning "parameter 'm' set but not used" }
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+template <int N>
+int
+bar (int a, int b, int c, int d, int e, int f, int g, int h, int i, int j,
+     int k, int l, int m, int n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
+
+void
+test ()
+{
+  foo <0> (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+  bar <0> (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+}
diff --git a/gcc/testsuite/g++.dg/warn/Wunused-parm-13.C b/gcc/testsuite/g++.dg/warn/Wunused-parm-13.C
new file mode 100644
index 0000000..f2d357f
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/Wunused-parm-13.C
@@ -0,0 +1,59 @@
+// PR c/44677
+// { dg-do compile }
+// { dg-options "-O2 -Wunused-but-set-parameter" }
+
+void baz (int);
+
+template <typename T>
+void
+foo (T a,		// { dg-warning "parameter 'a' set but not used" }
+     T b,		// { dg-warning "parameter 'b' set but not used" }
+     T c,		// { dg-warning "parameter 'c' set but not used" }
+     T d,		// { dg-warning "parameter 'd' set but not used" }
+     T e,		// { dg-warning "parameter 'e' set but not used" }
+     T f,		// { dg-warning "parameter 'f' set but not used" }
+     T g,		// { dg-warning "parameter 'g' set but not used" }
+     T h,		// { dg-warning "parameter 'h' set but not used" }
+     T i,		// { dg-warning "parameter 'i' set but not used" }
+     T j,		// { dg-warning "parameter 'j' set but not used" }
+     T k,		// { dg-warning "parameter 'k' set but not used" }
+     T l,		// { dg-warning "parameter 'l' set but not used" }
+     T m)		// { dg-warning "parameter 'm' set but not used" }
+{
+  a = 1;
+  ++b;
+  c++;
+  --d;
+  e--;
+  f += 2;
+  g |= 2;
+  h -= 2;
+  i &= 2;
+  j ^= 2;
+  k *= 2;
+  l %= 2;
+  for (T n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+template <typename T>
+T
+bar (T a, T b, T c, T d, T e, T f, T g, T h, T i, T j,
+     T k, T l, T m, T n)
+{
+  b = ++a;
+  d = --c;
+  f = e--;
+  h = g++;
+  j = i += 42;
+  l = k *= 4;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
+
+void
+test ()
+{
+  foo <int> (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+  bar <int> (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+}
diff --git a/gcc/testsuite/g++.dg/warn/Wunused-var-2.C b/gcc/testsuite/g++.dg/warn/Wunused-var-2.C
index 0b21ef1..869065f 100644
--- a/gcc/testsuite/g++.dg/warn/Wunused-var-2.C
+++ b/gcc/testsuite/g++.dg/warn/Wunused-var-2.C
@@ -18,9 +18,9 @@ f1 ()
 }
 
 void
-f2 (int x)
+f2 (int x)	// { dg-warning "parameter 'x' set but not used" }
 {
-  int a = 0;
+  int a = 0;	// { dg-warning "variable 'a' set but not used" }
   x++;
   ++a;
 }
diff --git a/gcc/testsuite/g++.dg/warn/Wunused-var-40.C b/gcc/testsuite/g++.dg/warn/Wunused-var-40.C
new file mode 100644
index 0000000..9351367
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/Wunused-var-40.C
@@ -0,0 +1,69 @@
+// PR c/44677
+// { dg-do compile }
+// { dg-options "-O2 -Wunused-but-set-variable" }
+
+void baz (int);
+
+template <int N>
+void
+foo (void)
+{
+  int a = 0;		// { dg-warning "variable 'a' set but not used" }
+  a = 1;
+  int b = 0;		// { dg-warning "variable 'b' set but not used" }
+  ++b;
+  int c = 0;		// { dg-warning "variable 'c' set but not used" }
+  c++;
+  int d = 0;		// { dg-warning "variable 'd' set but not used" }
+  --d;
+  int e = 0;		// { dg-warning "variable 'e' set but not used" }
+  e--;
+  int f = 0;		// { dg-warning "variable 'f' set but not used" }
+  f += 2;
+  int g = 0;		// { dg-warning "variable 'g' set but not used" }
+  g |= 2;
+  int h = 0;		// { dg-warning "variable 'h' set but not used" }
+  h -= 2;
+  int i = 0;		// { dg-warning "variable 'i' set but not used" }
+  i &= 2;
+  int j = 0;		// { dg-warning "variable 'j' set but not used" }
+  j ^= 2;
+  int k = 0;		// { dg-warning "variable 'k' set but not used" }
+  k *= 2;
+  int l = 0;		// { dg-warning "variable 'l' set but not used" }
+  l %= 2;
+  int m = 0;		// { dg-warning "variable 'm' set but not used" }
+  for (int n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+template <int N>
+int
+bar (void)
+{
+  int a = 0;
+  int b = ++a;
+  int c = 0;
+  int d = --c;
+  int e = 0;
+  int f = e--;
+  int g = 0;
+  int h = g++;
+  int i = 0;
+  int j;
+  j = i += 42;
+  int k = 0;
+  int l;
+  l = k *= 4;
+  int m = 0;
+  int n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
+
+void
+test ()
+{
+  foo <0> ();
+  bar <0> ();
+}
diff --git a/gcc/testsuite/g++.dg/warn/Wunused-var-41.C b/gcc/testsuite/g++.dg/warn/Wunused-var-41.C
new file mode 100644
index 0000000..ff981ee
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/Wunused-var-41.C
@@ -0,0 +1,69 @@
+// PR c/44677
+// { dg-do compile }
+// { dg-options "-O2 -Wunused-but-set-variable" }
+
+void baz (int);
+
+template <typename T>
+void
+foo (void)
+{
+  T a = 0;		// { dg-warning "variable 'a' set but not used" }
+  a = 1;
+  T b = 0;		// { dg-warning "variable 'b' set but not used" }
+  ++b;
+  T c = 0;		// { dg-warning "variable 'c' set but not used" }
+  c++;
+  T d = 0;		// { dg-warning "variable 'd' set but not used" }
+  --d;
+  T e = 0;		// { dg-warning "variable 'e' set but not used" }
+  e--;
+  T f = 0;		// { dg-warning "variable 'f' set but not used" }
+  f += 2;
+  T g = 0;		// { dg-warning "variable 'g' set but not used" }
+  g |= 2;
+  T h = 0;		// { dg-warning "variable 'h' set but not used" }
+  h -= 2;
+  T i = 0;		// { dg-warning "variable 'i' set but not used" }
+  i &= 2;
+  T j = 0;		// { dg-warning "variable 'j' set but not used" }
+  j ^= 2;
+  T k = 0;		// { dg-warning "variable 'k' set but not used" }
+  k *= 2;
+  T l = 0;		// { dg-warning "variable 'l' set but not used" }
+  l %= 2;
+  T m = 0;		// { dg-warning "variable 'm' set but not used" }
+  for (T n = 4; n < 10; n++, m++)
+    baz (n);
+}
+
+template <typename T>
+T
+bar (void)
+{
+  T a = 0;
+  T b = ++a;
+  T c = 0;
+  T d = --c;
+  T e = 0;
+  T f = e--;
+  T g = 0;
+  T h = g++;
+  T i = 0;
+  T j;
+  j = i += 42;
+  T k = 0;
+  T l;
+  l = k *= 4;
+  T m = 0;
+  T n;
+  n = m |= 2;
+  return b + d + f + h + j + l + n;
+}
+
+void
+test ()
+{
+  foo <int> ();
+  bar <int> ();
+}
diff --git a/gcc/testsuite/g++.dg/warn/pr121133-1.C b/gcc/testsuite/g++.dg/warn/pr121133-1.C
new file mode 100644
index 0000000..6d6e13b
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/pr121133-1.C
@@ -0,0 +1,16 @@
+// PR c++/121133
+// { dg-do compile }
+// { dg-options "-std=c++98 -Wno-long-long -pedantic-errors" }
+
+__extension__ typedef long long L;
+__extension__ long long a;
+struct S {
+  __extension__ long long b;
+};
+
+void
+foo ()
+{
+  __extension__ long long c;
+  c = c + (__extension__ (long long) 1);
+}
diff --git a/gcc/testsuite/g++.dg/warn/pr121133-2.C b/gcc/testsuite/g++.dg/warn/pr121133-2.C
new file mode 100644
index 0000000..cd97a76
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/pr121133-2.C
@@ -0,0 +1,5 @@
+// PR c++/121133
+// { dg-do compile }
+// { dg-options "-std=c++98 -pedantic-errors" }
+
+#include "pr121133-1.C"
diff --git a/gcc/testsuite/g++.dg/warn/pr121133-3.C b/gcc/testsuite/g++.dg/warn/pr121133-3.C
new file mode 100644
index 0000000..9ffd407
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/pr121133-3.C
@@ -0,0 +1,5 @@
+// PR c++/121133
+// { dg-do compile { target c++11 } }
+// { dg-options "-pedantic-errors" }
+
+#include "pr121133-1.C"
diff --git a/gcc/testsuite/g++.dg/warn/pr121133-4.C b/gcc/testsuite/g++.dg/warn/pr121133-4.C
new file mode 100644
index 0000000..76885ba
--- /dev/null
+++ b/gcc/testsuite/g++.dg/warn/pr121133-4.C
@@ -0,0 +1,5 @@
+// PR c++/121133
+// { dg-do compile { target c++11 } }
+// { dg-options "-pedantic-errors -Wlong-long" }
+
+#include "pr121133-1.C"
diff --git a/gcc/testsuite/g++.target/aarch64/mv-cpu-features.C b/gcc/testsuite/g++.target/aarch64/mv-cpu-features.C
new file mode 100644
index 0000000..ad6accd
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/mv-cpu-features.C
@@ -0,0 +1,82 @@
+/* { dg-do run }  */
+/* { dg-require-ifunc "" } */
+/* { dg-require-effective-target mmap } */
+/* { dg-options "-Wno-experimental-fmv-target" } */
+
+#include <cstdint>
+#include <sys/auxv.h>
+
+__attribute__((target_version ("default")))
+int foo ()
+{
+  return 0;
+}
+
+__attribute__((target_version ("rng")))
+int foo ()
+{
+  return 1;
+}
+
+__attribute__((target_version ("lse")))
+int foo ()
+{
+  return 2;
+}
+
+typedef struct {
+  uint64_t size;
+  uint64_t hwcap;
+  uint64_t hwcap2;
+  uint64_t hwcap3;
+  uint64_t hwcap4;
+} ifunc_arg_t;
+
+int impl ()
+{
+  return 0;
+}
+
+#ifndef _IFUNC_ARG_HWCAP
+#define _IFUNC_ARG_HWCAP (1ULL << 62)
+#endif
+
+extern "C" void
+__init_cpu_features_resolver (unsigned long hwcap, const ifunc_arg_t *arg);
+
+extern "C" void *
+fun_resolver (uint64_t a0, const ifunc_arg_t *a1)
+{
+  ifunc_arg_t arg = {};
+  arg.size = sizeof (ifunc_arg_t);
+  /* These flags determine that the implementation of foo ()
+     that returns 2 will be selected.  */
+  arg.hwcap = HWCAP_ATOMICS;
+  arg.hwcap2 = HWCAP2_RNG;
+  __init_cpu_features_resolver (arg.hwcap | _IFUNC_ARG_HWCAP, &arg);
+  return (void *)(uintptr_t)impl;
+}
+
+extern "C" int fun (void) __attribute__((ifunc ("fun_resolver")));
+
+/* In this test we expect that the manual resolver for the fun ()
+   function will be executed before the automatic resolver for the
+   FMV function foo ().  This is because resolvers from the same TU
+   are executed according to the offset of corresponding relocations.
+
+   Automatic resolver is generated in a dedicated section while the
+   manually written resolver will be put in the .text section which
+   will come first.
+
+   The manual resolver above calls __init_cpu_features_resolver()
+   supplying synthetic ifunc_arg_t fields that will determine the
+   choice for the FMV implementation.
+   */
+
+int main ()
+{
+  int res = fun ();
+  if (res == 0 && foo () == 2)
+    return 0;
+  return 1;
+}
diff --git a/gcc/testsuite/g++.target/aarch64/pr119498.C b/gcc/testsuite/g++.target/aarch64/pr119498.C
new file mode 100644
index 0000000..03f1659
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/pr119498.C
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-require-ifunc "" } */
+/* { dg-options "-O0" } */
+/* { dg-additional-options "-Wno-experimental-fmv-target" } */
+
+__attribute__ ((target_version ("default"))) int
+foo ();
+
+__attribute__ ((target_version ("default"))) int
+foo () { return 1; } /* { dg-message "old declaration" } */
+
+__attribute__ ((target_version ("dotprod"))) float
+foo () { return 3; } /* { dg-error "ambiguating new declaration" } */
+
+__attribute__ ((target_version ("sve"))) int
+foo2 () { return 1; } /* { dg-message "old declaration" } */
+
+__attribute__ ((target_version ("dotprod"))) float
+foo2 () { return 3; } /* { dg-error "ambiguating new declaration of" } */
diff --git a/gcc/testsuite/g++.target/aarch64/sme/sme_throw_1.C b/gcc/testsuite/g++.target/aarch64/sme/sme_throw_1.C
new file mode 100644
index 0000000..76f1e8b
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sme/sme_throw_1.C
@@ -0,0 +1,55 @@
+/* { dg-do run { target { aarch64*-linux-gnu* && aarch64_sme_hw } } } */
+
+#include <signal.h>
+#include <arm_sme.h>
+
+static bool caught;
+
+[[gnu::noipa]] void thrower(int)
+{
+  throw 1;
+}
+
+[[gnu::noipa]] void bar()
+{
+  *(volatile int *)0 = 0;
+}
+
+[[gnu::noipa]] void foo()
+{
+  try
+    {
+      bar();
+    }
+  catch (int)
+    {
+      caught = true;
+    }
+}
+
+__arm_new("za") __arm_locally_streaming void sme_user()
+{
+  svbool_t all = svptrue_b8();
+  for (unsigned int i = 0; i < svcntb(); ++i)
+    {
+      svint8_t expected = svindex_s8(i + 1, i);
+      svwrite_hor_za8_m(0, i, all, expected);
+    }
+  foo();
+  for (unsigned int i = 0; i < svcntb(); ++i)
+    {
+      svint8_t expected = svindex_s8(i + 1, i);
+      svint8_t actual = svread_hor_za8_m(svdup_s8(0), all, 0, i);
+      if (svptest_any(all, svcmpne(all, expected, actual)))
+	__builtin_abort();
+    }
+  if (!caught)
+    __builtin_abort();
+}
+
+int main()
+{
+  signal(SIGSEGV, thrower);
+  sme_user();
+  return 0;
+}
diff --git a/gcc/testsuite/g++.target/aarch64/sme/sme_throw_2.C b/gcc/testsuite/g++.target/aarch64/sme/sme_throw_2.C
new file mode 100644
index 0000000..db3197c
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sme/sme_throw_2.C
@@ -0,0 +1,4 @@
+/* { dg-do run { target { aarch64*-linux-gnu* && aarch64_sme_hw } } } */
+/* { dg-options "-O2" } */
+
+#include "sme_throw_1.C"
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_binary_bf16_1.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_binary_bf16_1.C
new file mode 100644
index 0000000..9d6342b
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_binary_bf16_1.C
@@ -0,0 +1,35 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -msve-vector-bits=2048" } */
+
+#pragma GCC target "arch=armv9-a+sve-b16b16"
+
+#define ADD(a, b) a + b
+#define SUB(a, b) a - b
+#define MUL(a, b) a * b
+#define MAX(a, b) (a > b) ? a : b
+#define MIN(a, b) (a > b) ? b : a
+
+#define TEST_OP(TYPE, OP)                                        \
+  TYPE test_##TYPE##_##OP (TYPE a, TYPE b) { return OP (a, b); } \
+
+#define TEST_ALL(TYPE, SIZE)                                        \
+  typedef TYPE TYPE##SIZE __attribute__((vector_size(SIZE)));       \
+  TEST_OP (TYPE##SIZE, ADD)                                         \
+  TEST_OP (TYPE##SIZE, SUB)                                         \
+  TEST_OP (TYPE##SIZE, MUL)                                         \
+  TEST_OP (TYPE##SIZE, MIN)                                         \
+  TEST_OP (TYPE##SIZE, MAX)
+
+TEST_ALL (__bf16, 64)
+
+TEST_ALL (__bf16, 128)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 5 } } */
+
+/* { dg-final { scan-assembler-times {\tbfadd\tz[0-9]+\.h, p[0-7]/m. z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tbfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_binary_bf16_2.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_binary_bf16_2.C
new file mode 100644
index 0000000..63de293
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_binary_bf16_2.C
@@ -0,0 +1,15 @@
+/* { dg-do compile }*/
+/* { dg-options "-O2 -ffinite-math-only -fno-signed-zeros -fno-trapping-math -msve-vector-bits=2048 " } */
+
+#include "unpacked_binary_bf16_1.C"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 10 } } */
+
+/* { dg-final { scan-assembler-times {\tbfadd\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfsub\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfmul\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tbfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_binary_bf16_1.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_binary_bf16_1.C
new file mode 100644
index 0000000..560d874
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_binary_bf16_1.C
@@ -0,0 +1,46 @@
+/* { dg-do compile }*/
+/* { dg-options "-O -ffinite-math-only -fno-signed-zeros -fno-trapping-math -msve-vector-bits=2048 " } */
+
+#include <stdint.h>
+#pragma GCC target "arch=armv9-a+sve-b16b16"
+
+#define ADD(a, b) a + b
+#define SUB(a, b) a - b
+#define MUL(a, b) a * b
+#define MAX(a, b) (a > b) ? a : b
+#define MIN(a, b) (a > b) ? b : a
+
+#define COND_OP(OP, TYPE, PRED_TYPE, ARG2, MERGE)                                  \
+  TYPE test_##OP##_##TYPE##_##ARG2##_##MERGE (TYPE a, TYPE b, TYPE c, PRED_TYPE p) \
+  {return p ? OP (a, ARG2) : MERGE; }
+
+#define TEST_OP(OP, TYPE, PRED_TYPE, T)     \
+  T (OP, TYPE, PRED_TYPE, b, a)             \
+  T (OP, TYPE, PRED_TYPE, b, b)             \
+  T (OP, TYPE, PRED_TYPE, b, c)
+
+#define TEST_ALL(TYPE, PRED_TYPE, T) \
+  TEST_OP (ADD, TYPE, PRED_TYPE, T)  \
+  TEST_OP (SUB, TYPE, PRED_TYPE, T)  \
+  TEST_OP (MUL, TYPE, PRED_TYPE, T)  \
+  TEST_OP (MAX, TYPE, PRED_TYPE, T)  \
+  TEST_OP (MIN, TYPE, PRED_TYPE, T)
+
+#define TEST(TYPE, PTYPE, SIZE)                                   \
+  typedef TYPE TYPE##SIZE __attribute__ ((vector_size (SIZE)));   \
+  typedef PTYPE PTYPE##SIZE __attribute__ ((vector_size (SIZE))); \
+  TEST_ALL (TYPE##SIZE, PTYPE##SIZE, COND_OP)
+
+TEST (__bf16, uint16_t, 128)
+
+TEST (__bf16, uint16_t, 64)
+
+/* { dg-final { scan-assembler-times {\tbfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tbfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tbfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tbfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tbfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+
+// There's no BFSUBR.
+/* { dg-final { scan-assembler-times {\tsel\t} 2 } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_binary_bf16_2.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_binary_bf16_2.C
new file mode 100644
index 0000000..02880ef
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_binary_bf16_2.C
@@ -0,0 +1,18 @@
+/* { dg-do compile }*/
+/* { dg-options "-O -ffinite-math-only -fno-signed-zeros -msve-vector-bits=2048 " } */
+
+#include "unpacked_cond_binary_bf16_1.C"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 15 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 15 } } */
+/* { dg-final { scan-assembler-times {\tand} 30 } } */
+
+/* { dg-final { scan-assembler-times {\tbfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tbfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tbfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tbfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tbfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+
+// There's no BFSUBR.
+/* { dg-final { scan-assembler-times {\tsel\t} 2 } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_ternary_bf16_1.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_ternary_bf16_1.C
new file mode 100644
index 0000000..95cd698
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_ternary_bf16_1.C
@@ -0,0 +1,35 @@
+/* { dg-do compile }*/
+/* { dg-options "-O2  -fno-trapping-math -msve-vector-bits=2048 " } */
+
+#include <stdint.h>
+#pragma GCC target "arch=armv9-a+sve-b16b16"
+
+#define COND_BFMLA(TYPE, PRED_TYPE, MERGE)                               \
+  TYPE test_bfmla_##TYPE##_##MERGE (TYPE a, TYPE b, TYPE c, PRED_TYPE p) \
+  {return p ? a * b + c : MERGE; }
+
+#define COND_BFMLS(TYPE, PRED_TYPE, MERGE)                               \
+  TYPE test_bfmls_##TYPE##_##MERGE (TYPE a, TYPE b, TYPE c, PRED_TYPE p) \
+  {return p ? a * -b + c : MERGE; }
+
+#define TEST_OP(TYPE, PRED_TYPE, T) \
+  T (TYPE, PRED_TYPE, c)            \
+  T (TYPE, PRED_TYPE, 0)
+
+#define TEST(TYPE, PTYPE, SIZE)                                   \
+  typedef TYPE TYPE##SIZE __attribute__ ((vector_size (SIZE)));   \
+  typedef PTYPE PTYPE##SIZE __attribute__ ((vector_size (SIZE))); \
+  TEST_OP (TYPE##SIZE, PTYPE##SIZE, COND_BFMLA)                   \
+  TEST_OP (TYPE##SIZE, PTYPE##SIZE, COND_BFMLS)
+
+TEST (__bf16, uint16_t, 128)
+
+TEST (__bf16, uint16_t, 64)
+
+/* { dg-final { scan-assembler-times {\tptrue} 8 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tbfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tbfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_ternary_bf16_2.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_ternary_bf16_2.C
new file mode 100644
index 0000000..c0d7c50
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_cond_ternary_bf16_2.C
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -msve-vector-bits=2048" } */
+
+#include "unpacked_cond_ternary_bf16_1.C"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tand} 8 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tbfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tbfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_ternary_bf16_1.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_ternary_bf16_1.C
new file mode 100644
index 0000000..19bfe95
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_ternary_bf16_1.C
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -msve-vector-bits=2048" } */
+
+#define BFMLA(TYPE)                               \
+  TYPE test_bfmla_##TYPE (TYPE a, TYPE b, TYPE c) \
+  { return a * b + c; }
+
+#define BFMLS(TYPE)                               \
+  TYPE test_bfmls_##TYPE (TYPE a, TYPE b, TYPE c) \
+  { return a * -b + c; }
+
+#define TEST_TYPE(TYPE, SIZE)                                 \
+  typedef TYPE TYPE##SIZE __attribute__((vector_size(SIZE))); \
+  BFMLA (TYPE##SIZE)                                          \
+  BFMLS (TYPE##SIZE)
+
+#pragma GCC target "arch=armv9-a+sve-b16b16"
+
+TEST_TYPE (__bf16,  128)
+
+TEST_TYPE (__bf16,  64)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tbfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/g++.target/aarch64/sve/unpacked_ternary_bf16_2.C b/gcc/testsuite/g++.target/aarch64/sve/unpacked_ternary_bf16_2.C
new file mode 100644
index 0000000..ef37400
--- /dev/null
+++ b/gcc/testsuite/g++.target/aarch64/sve/unpacked_ternary_bf16_2.C
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -msve-vector-bits=2048 -fno-trapping-math" } */
+
+#include "unpacked_ternary_bf16_1.C"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tbfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tbfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.c-torture/compile/pr120807.c b/gcc/testsuite/gcc.c-torture/compile/pr120807.c
new file mode 100644
index 0000000..9b37e60
--- /dev/null
+++ b/gcc/testsuite/gcc.c-torture/compile/pr120807.c
@@ -0,0 +1,20 @@
+typedef __UINT8_TYPE__ uint8_t;
+typedef __UINT32_TYPE__ uint32_t;
+
+typedef struct
+{
+  uint32_t dword[2];
+  uint8_t byte[8];
+} reg64_t;
+reg64_t TestF20F_opgd, TestF20F_oped;
+
+void
+TestF20F ()
+{
+  TestF20F_opgd.dword[0] ^= TestF20F_oped.byte[0];
+  for (int i = 0; i < 8; i++)
+    if (TestF20F_opgd.dword[0] & 1)
+      TestF20F_opgd.dword[0] = TestF20F_opgd.dword[0] >> 1 ^ (uint32_t)2197175160UL;
+    else
+      TestF20F_opgd.dword[0] = TestF20F_opgd.dword[0] >> 1;
+}
diff --git a/gcc/testsuite/gcc.dg/20021014-1.c b/gcc/testsuite/gcc.dg/20021014-1.c
index e43f7b2..ee5d459 100644
--- a/gcc/testsuite/gcc.dg/20021014-1.c
+++ b/gcc/testsuite/gcc.dg/20021014-1.c
@@ -2,6 +2,7 @@
 /* { dg-require-profiling "-p" } */
 /* { dg-options "-O2 -p" } */
 /* { dg-options "-O2 -p -static" { target hppa*-*-hpux* } } */
+/* { dg-additional-options "-mfentry -fno-pic" { target i?86-*-gnu* x86_64-*-gnu* } } */
 /* { dg-error "profiler" "No profiler support" { target xstormy16-*-* } 0 } */
 /* { dg-message "" "consider using `-pg' instead of `-p' with gprof(1)" { target *-*-freebsd* } 0 } */
 
diff --git a/gcc/testsuite/gcc.dg/Warray-parameter-11.c b/gcc/testsuite/gcc.dg/Warray-parameter-11.c
index 8ca1b55..e05835c 100644
--- a/gcc/testsuite/gcc.dg/Warray-parameter-11.c
+++ b/gcc/testsuite/gcc.dg/Warray-parameter-11.c
@@ -9,7 +9,7 @@ typedef __INTPTR_TYPE__ intptr_t;
 void f0 (double[!copysign (~2, 3)]);
 
 void f1 (double[!copysign (~2, 3)]);
-void f1 (double[1]);                    // { dg-warning "-Warray-parameter" }
+void f1 (double[1]);                    // { dg-warning "-Wvla-parameter" }
 
 void f2 (int[(int)+1.0]);
 void f2 (int[(int)+1.1]);
@@ -21,4 +21,4 @@ extern struct S *sp;
 
 void f3 (int[(intptr_t)((char*)sp->a - (char*)sp)]);
 void f3 (int[(intptr_t)((char*)&sp->a[0] - (char*)sp)]);
-void f3 (int[(intptr_t)((char*)&sp->a[1] - (char*)sp)]);   // { dg-warning "-Warray-parameter" }
+void f3 (int[(intptr_t)((char*)&sp->a[1] - (char*)sp)]);   // { dg-warning "-Wvla-parameter" }
diff --git a/gcc/testsuite/gcc.dg/Warray-parameter.c b/gcc/testsuite/gcc.dg/Warray-parameter.c
index 6c5195a..31879a8 100644
--- a/gcc/testsuite/gcc.dg/Warray-parameter.c
+++ b/gcc/testsuite/gcc.dg/Warray-parameter.c
@@ -118,8 +118,7 @@ typedef int IA2[2];
 typedef int IA3[3];
 
 // The message should differentiate between the [] form and *.
-void f1IAx_A1 (IAx);            // { dg-message "previously declared as 'int\\\[]'" "pr?????" { xfail *-*-* } }
-                                // { dg-message "previously declared as 'int *\\\*'" "note" { target *-*-* } .-1 }
+void f1IAx_A1 (IAx);            // { dg-message "previously declared as 'int\\\[]'" }
 void f1IAx_A1 (IA1);            // { dg-message "argument 1 of type 'int\\\[1]' with mismatched bound" }
 
 void f1IA1_A2 (IA1);            // { dg-message "previously declared as 'int\\\[1]'" }
diff --git a/gcc/testsuite/gcc.dg/analyzer/state-diagram-1-sarif.py b/gcc/testsuite/gcc.dg/analyzer/state-diagram-1-sarif.py
index d2967d4..d92af83 100644
--- a/gcc/testsuite/gcc.dg/analyzer/state-diagram-1-sarif.py
+++ b/gcc/testsuite/gcc.dg/analyzer/state-diagram-1-sarif.py
@@ -6,7 +6,7 @@ import pytest
 def sarif():
     return sarif_from_env()
 
-def test_xml_state(sarif):
+def test_state_graph(sarif):
     result = get_result_by_index(sarif, 0)
 
     assert result['level'] == 'warning'
@@ -17,16 +17,49 @@ def test_xml_state(sarif):
 
     # Event "(1)": "entry to 'test'" (index == 0)
     assert events[0]['location']['message']['text'] == "entry to 'test'"
-    state0 = get_xml_state(events, 0)
-    memory_regions = state0.find('memory-regions')
-    assert memory_regions is not None
-    stack = memory_regions.find('stack')
-    assert stack is not None
-    frame = stack.find('stack-frame')
-    assert frame.get('function') == 'test'
+    state0 = get_state_graph(events, 0)
+
+    stack = state0['nodes'][0]
+    assert stack['id'] == 'stack'
+    assert get_state_node_kind(stack) == 'stack'
+
+    frame = stack['children'][0]
+    assert frame['id'].startswith('frame-region-')
+    assert get_state_node_kind(frame) == 'stack-frame'
+    assert get_state_node_attr(frame, 'function') == 'test'
+    assert frame['location']['logicalLocations'][0]['fullyQualifiedName'] == 'test'
 
     # Final event:
     assert events[-1]['location']['message']['text'].startswith("use after 'free' of ")
-    state = get_xml_state(events, -1)
-    # TODO
+    state = get_state_graph(events, -1)
+
+    stack = state['nodes'][0]
+    assert stack['id'] == 'stack'
+    assert get_state_node_kind(stack) == 'stack'
+
+    frame = stack['children'][0]
+    assert frame['id'].startswith('frame-region-')
+    assert get_state_node_kind(frame) == 'stack-frame'
+    assert get_state_node_attr(frame, 'function') == 'test'
+    assert frame['location']['logicalLocations'][0]['fullyQualifiedName'] == 'test'
+
+    heap = state['nodes'][1]
+    assert heap['id'] == 'heap'
+    assert get_state_node_kind(heap) == 'heap'
+
+    assert len(heap['children']) == 3
+    heap_buffer0 = heap['children'][0]
+    assert heap_buffer0['id'].startswith('heap-allocated-region-')
+    assert get_state_node_kind(heap_buffer0) == 'dynalloc-buffer'
+
+    globals_ = state['nodes'][2]
+    assert globals_['id'] == 'globals'
+    assert get_state_node_kind(globals_) == 'globals'
 
+    first = globals_['children'][0]
+    assert first['id'].startswith('decl-region-')
+    assert get_state_node_kind(first) == 'variable'
+    assert get_state_node_name(first) == 'first'
+    assert get_state_node_type(first) == 'struct node *'
+    
+    assert len(state['edges']) == 3
diff --git a/gcc/testsuite/gcc.dg/analyzer/state-diagram-1.c b/gcc/testsuite/gcc.dg/analyzer/state-diagram-1.c
index 3d853d2..3b35cfa 100644
--- a/gcc/testsuite/gcc.dg/analyzer/state-diagram-1.c
+++ b/gcc/testsuite/gcc.dg/analyzer/state-diagram-1.c
@@ -1,4 +1,4 @@
-/* { dg-additional-options "-fdiagnostics-add-output=sarif:xml-state=yes" } */
+/* { dg-additional-options "-fdiagnostics-add-output=sarif:state-graphs=yes" } */
 
 #include "analyzer-decls.h"
 
diff --git a/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.c b/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.c
index 28cf580..b981cf9 100644
--- a/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.c
+++ b/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.c
@@ -1,4 +1,4 @@
-/* { dg-additional-options "-fdiagnostics-add-output=sarif:xml-state=yes" } */
+/* { dg-additional-options "-fdiagnostics-add-output=sarif:state-graphs=yes" } */
 
 #include "analyzer-decls.h"
 
diff --git a/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.py b/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.py
index 484da96..3a2c6f8 100644
--- a/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.py
+++ b/gcc/testsuite/gcc.dg/analyzer/state-diagram-5-sarif.py
@@ -1,5 +1,3 @@
-import xml.etree.ElementTree as ET
-
 from sarif import *
 
 import pytest
@@ -8,7 +6,7 @@ import pytest
 def sarif():
     return sarif_from_env()
 
-def test_nested_types_in_xml_state(sarif):
+def test_nested_types_in_state_graph(sarif):
     result = get_result_by_index(sarif, 0)
 
     assert result['level'] == 'note'
@@ -16,16 +14,17 @@ def test_nested_types_in_xml_state(sarif):
     events = result["codeFlows"][0]["threadFlows"][0]['locations']
 
     assert events[0]['location']['message']['text'] == 'here'
-    state = get_xml_state(events, 0)
-
-    memory_regions = state.find('memory-regions')
-    assert memory_regions is not None
+    state = get_state_graph(events, 0)
 
-    stack = memory_regions.find('stack')
-    assert stack is not None
+    stack = state['nodes'][0]
+    assert stack['id'] == 'stack'
+    assert get_state_node_kind(stack) == 'stack'
 
-    frame = stack.find('stack-frame')
-    assert frame.get('function') == 'test'
+    frame = stack['children'][0]
+    assert frame['id'].startswith('frame-region-')
+    assert get_state_node_kind(frame) == 'stack-frame'
+    assert get_state_node_attr(frame, 'function') == 'test'
+    assert frame['location']['logicalLocations'][0]['fullyQualifiedName'] == 'test'
 
     # We have:
     #   baz_arr[1].m_bars[1].m_foos[2].m_ints[1] = 42;
@@ -34,40 +33,57 @@ def test_nested_types_in_xml_state(sarif):
     # representation to nested elements and fields.
 
     # "baz_arr":
-    baz_arr = frame.find("variable[@name='baz_arr']")
-    assert baz_arr.get('type') == 'struct baz[2]'
+    baz_arr = frame['children'][0]
+    assert get_state_node_kind(baz_arr) == 'variable'
+    assert get_state_node_type(baz_arr) == 'struct baz[2]'
+
+    assert len(baz_arr['children']) == 2
+
+    bindings = baz_arr['children'][0]
+    assert bindings['id'].startswith('concrete-bindings-')
+    assert get_state_node_kind(bindings) == 'other'
+    assert get_state_node_value(bindings['children'][0]) == '(int)42'
 
     # "baz_arr[1]":
-    baz_arr_1 = baz_arr.find("element[@index='1']")
-    assert baz_arr_1.get('type') == 'struct baz'
+    baz_arr_1 = baz_arr['children'][1]
+    assert get_state_node_type(baz_arr_1) == 'struct baz'
+    assert get_state_node_kind(baz_arr_1) == 'element'
+    assert get_state_node_attr(baz_arr_1, 'index') == '1'
 
-    assert baz_arr.find("element[@index='0']") is None
+    assert len(baz_arr_1['children']) == 1
 
     # "baz_arr[1].m_bars":
-    baz_arr_1_m_bars = baz_arr_1.find("field[@name='m_bars']")
-    assert baz_arr_1_m_bars.get('type') == 'struct bar[2]'
+    baz_arr_1_m_bars = baz_arr_1['children'][0]
+    assert get_state_node_name(baz_arr_1_m_bars) == 'm_bars'
+    assert get_state_node_type(baz_arr_1_m_bars) == 'struct bar[2]'
 
     # "baz_arr[1].m_bars[1]"
-    baz_arr_1_m_bars_1 = baz_arr_1_m_bars.find("element[@index='1']")
-    assert baz_arr_1_m_bars_1.get('type') == 'struct bar'
+    baz_arr_1_m_bars_1 = baz_arr_1_m_bars['children'][0]
+    assert get_state_node_type(baz_arr_1_m_bars_1) == 'struct bar'
+    assert get_state_node_kind(baz_arr_1_m_bars_1) == 'element'
+    assert get_state_node_attr(baz_arr_1_m_bars_1, 'index') == '1'
 
     # "baz_arr[1].m_bars[1].m_foos"
-    baz_arr_1_m_bars_1_m_foos = baz_arr_1_m_bars_1.find("field[@name='m_foos']")
-    assert baz_arr_1_m_bars_1_m_foos.get('type') == 'struct foo[3]'
+    baz_arr_1_m_bars_1_m_foos = baz_arr_1_m_bars_1['children'][0]
+    assert get_state_node_kind(baz_arr_1_m_bars_1_m_foos) == 'field'
+    assert get_state_node_name(baz_arr_1_m_bars_1_m_foos) == 'm_foos'
+    assert get_state_node_type(baz_arr_1_m_bars_1_m_foos) == 'struct foo[3]'
 
     # "baz_arr[1].m_bars[1].m_foos[2]"
-    baz_arr_1_m_bars_1_m_foos_2 = baz_arr_1_m_bars_1_m_foos.find("element[@index='2']")
-    assert baz_arr_1_m_bars_1_m_foos_2.get('type') == 'struct foo'
-    
+    baz_arr_1_m_bars_1_m_foos_2 = baz_arr_1_m_bars_1_m_foos['children'][0]
+    assert get_state_node_type(baz_arr_1_m_bars_1_m_foos_2) == 'struct foo'
+    assert get_state_node_kind(baz_arr_1_m_bars_1_m_foos_2) == 'element'
+    assert get_state_node_attr(baz_arr_1_m_bars_1_m_foos_2, 'index') == '2'
+
     # "baz_arr[1].m_bars[1].m_foos[2].m_ints"
-    baz_arr_1_m_bars_1_m_foos_2_m_ints = baz_arr_1_m_bars_1_m_foos_2.find('field[@name="m_ints"]')
-    assert baz_arr_1_m_bars_1_m_foos_2_m_ints.get('type') == 'int[4]'
-    
-    # "baz_arr[1].m_bars[1].m_foos[2].m_ints[1]"
-    baz_arr_1_m_bars_1_m_foos_2_m_ints_1 = baz_arr_1_m_bars_1_m_foos_2_m_ints.find('element[@index="1"]')
-    assert baz_arr_1_m_bars_1_m_foos_2_m_ints_1.get('type') == 'int'
+    baz_arr_1_m_bars_1_m_foos_2_m_ints = baz_arr_1_m_bars_1_m_foos_2['children'][0]
+    assert get_state_node_kind(baz_arr_1_m_bars_1_m_foos_2_m_ints) == 'field'
+    assert get_state_node_name(baz_arr_1_m_bars_1_m_foos_2_m_ints) == 'm_ints'
+    assert get_state_node_type(baz_arr_1_m_bars_1_m_foos_2_m_ints) == 'int[4]'
 
-    value = baz_arr_1_m_bars_1_m_foos_2_m_ints_1.find('value-of-region')
-    constant = value.find('constant')
-    assert constant.get('value') == '42'
-    assert constant.get('type') == 'int'
+    # "baz_arr[1].m_bars[1].m_foos[2].m_ints[1]"
+    baz_arr_1_m_bars_1_m_foos_2_m_ints_1 = baz_arr_1_m_bars_1_m_foos_2_m_ints['children'][0]
+    assert get_state_node_type(baz_arr_1_m_bars_1_m_foos_2_m_ints_1) == 'int'
+    assert get_state_node_kind(baz_arr_1_m_bars_1_m_foos_2_m_ints_1) == 'element'
+    assert get_state_node_attr(baz_arr_1_m_bars_1_m_foos_2_m_ints_1, 'index') == '1'
+    assert get_state_node_value(baz_arr_1_m_bars_1_m_foos_2_m_ints_1) == '(int)42'
diff --git a/gcc/testsuite/gcc.dg/aru-2.c b/gcc/testsuite/gcc.dg/aru-2.c
index 054223c..61898de 100644
--- a/gcc/testsuite/gcc.dg/aru-2.c
+++ b/gcc/testsuite/gcc.dg/aru-2.c
@@ -1,6 +1,7 @@
 /* { dg-do run } */
 /* { dg-require-profiling "-pg" } */
 /* { dg-options "-O2 -pg" } */
+/* { dg-additional-options "-mfentry -fno-pic" { target i?86-*-gnu* x86_64-*-gnu* } } */
 
 static int __attribute__((noinline))
 bar (int x)
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-1.c b/gcc/testsuite/gcc.dg/asm-hard-reg-1.c
new file mode 100644
index 0000000..6a5a9ad
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-1.c
@@ -0,0 +1,85 @@
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* riscv*-*-* s390*-*-* x86_64-*-* } } */
+
+#if defined (__aarch64__)
+# define GPR "{x4}"
+/* { dg-final { scan-assembler-times "foo\tx4" 8 { target { aarch64*-*-* } } } } */
+#elif defined (__arm__)
+# define GPR "{r4}"
+/* { dg-final { scan-assembler-times "foo\tr4" 8 { target { arm*-*-* } } } } */
+#elif defined (__i386__)
+# define GPR "{ecx}"
+/* { dg-final { scan-assembler-times "foo\t%cl" 2 { target { i?86-*-* } } } } */
+/* { dg-final { scan-assembler-times "foo\t%cx" 2 { target { i?86-*-* } } } } */
+/* { dg-final { scan-assembler-times "foo\t%ecx" 4 { target { i?86-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define GPR "{r5}"
+/* { dg-final { scan-assembler-times "foo\t5" 8 { target { powerpc*-*-* } } } } */
+#elif defined (__riscv)
+# define GPR "{t5}"
+/* { dg-final { scan-assembler-times "foo\tt5" 8 { target { riscv*-*-* } } } } */
+#elif defined (__s390__)
+# define GPR "{r4}"
+/* { dg-final { scan-assembler-times "foo\t%r4" 8 { target { s390*-*-* } } } } */
+#elif defined (__x86_64__)
+# define GPR "{rcx}"
+/* { dg-final { scan-assembler-times "foo\t%cl" 2 { target { x86_64-*-* } } } } */
+/* { dg-final { scan-assembler-times "foo\t%cx" 2 { target { x86_64-*-* } } } } */
+/* { dg-final { scan-assembler-times "foo\t%ecx" 2 { target { x86_64-*-* } } } } */
+/* { dg-final { scan-assembler-times "foo\t%rcx" 2 { target { x86_64-*-* } } } } */
+#endif
+
+char
+test_char (char x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (x));
+  return x;
+}
+
+char
+test_char_from_mem (char *x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (*x));
+  return *x;
+}
+
+short
+test_short (short x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (x));
+  return x;
+}
+
+short
+test_short_from_mem (short *x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (*x));
+  return *x;
+}
+
+int
+test_int (int x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (x));
+  return x;
+}
+
+int
+test_int_from_mem (int *x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (*x));
+  return *x;
+}
+
+long
+test_long (long x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (x));
+  return x;
+}
+
+long
+test_long_from_mem (long *x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (*x));
+  return *x;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-2.c b/gcc/testsuite/gcc.dg/asm-hard-reg-2.c
new file mode 100644
index 0000000..7dabf96
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-2.c
@@ -0,0 +1,33 @@
+/* { dg-do compile { target aarch64*-*-* powerpc64*-*-* riscv64-*-* s390*-*-* x86_64-*-* } } */
+/* { dg-options "-std=c99" } we need long long */
+
+#if defined (__aarch64__)
+# define GPR "{x4}"
+/* { dg-final { scan-assembler-times "foo\tx4" 2 { target { aarch64*-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define GPR "{r5}"
+/* { dg-final { scan-assembler-times "foo\t5" 2 { target { powerpc64*-*-* } } } } */
+#elif defined (__riscv)
+# define GPR "{t5}"
+/* { dg-final { scan-assembler-times "foo\tt5" 2 { target { riscv64-*-* } } } } */
+#elif defined (__s390__)
+# define GPR "{r4}"
+/* { dg-final { scan-assembler-times "foo\t%r4" 2 { target { s390*-*-* } } } } */
+#elif defined (__x86_64__)
+# define GPR "{rcx}"
+/* { dg-final { scan-assembler-times "foo\t%rcx" 2 { target { x86_64-*-* } } } } */
+#endif
+
+long long
+test_longlong (long long x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (x));
+  return x;
+}
+
+long long
+test_longlong_from_mem (long long *x)
+{
+  __asm__ ("foo\t%0" : "+"GPR (*x));
+  return *x;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-3.c b/gcc/testsuite/gcc.dg/asm-hard-reg-3.c
new file mode 100644
index 0000000..fa4472a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-3.c
@@ -0,0 +1,25 @@
+/* { dg-do compile { target { { aarch64*-*-* powerpc64*-*-* riscv64-*-* s390*-*-* x86_64-*-* } && int128 } } } */
+/* { dg-options "-O2" } get rid of -ansi since we use __int128 */
+
+#if defined (__aarch64__)
+# define REG "{x4}"
+/* { dg-final { scan-assembler-times "foo\tx4" 1 { target { aarch64*-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define REG "{r5}"
+/* { dg-final { scan-assembler-times "foo\t5" 1 { target { powerpc*-*-* } } } } */
+#elif defined (__riscv)
+# define REG "{t5}"
+/* { dg-final { scan-assembler-times "foo\tt5" 1 { target { riscv*-*-* } } } } */
+#elif defined (__s390__)
+# define REG "{r4}"
+/* { dg-final { scan-assembler-times "foo\t%r4" 1 { target { s390*-*-* } } } } */
+#elif defined (__x86_64__)
+# define REG "{xmm0}"
+/* { dg-final { scan-assembler-times "foo\t%xmm0" 1 { target { x86_64-*-* } } } } */
+#endif
+
+void
+test (void)
+{
+  __asm__ ("foo\t%0" :: REG ((__int128) 42));
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-4.c b/gcc/testsuite/gcc.dg/asm-hard-reg-4.c
new file mode 100644
index 0000000..0134bf0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-4.c
@@ -0,0 +1,50 @@
+/* { dg-do compile { target aarch64*-*-* arm*-*-* powerpc*-*-* riscv*-*-* s390*-*-* x86_64-*-* } } */
+
+#if defined (__aarch64__)
+# define FPR "{d5}"
+/* { dg-final { scan-assembler-times "foo\tv5" 4 { target { aarch64*-*-* } } } } */
+#elif defined (__arm__)
+# define FPR "{d5}"
+/* { dg-additional-options "-march=armv7-a+fp -mfloat-abi=hard" { target arm*-*-* } } */
+/* { dg-final { scan-assembler-times "foo\ts10" 4 { target { arm*-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define FPR "{5}"
+/* { dg-final { scan-assembler-times "foo\t5" 4 { target { powerpc*-*-* } } } } */
+#elif defined (__riscv)
+# define FPR "{fa5}"
+/* { dg-final { scan-assembler-times "foo\tfa5" 4 { target { rsicv*-*-* } } } } */
+#elif defined (__s390__)
+# define FPR "{f5}"
+/* { dg-final { scan-assembler-times "foo\t%f5" 4 { target { s390*-*-* } } } } */
+#elif defined (__x86_64__)
+# define FPR "{xmm5}"
+/* { dg-final { scan-assembler-times "foo\t%xmm5" 4 { target { x86_64-*-* } } } } */
+#endif
+
+float
+test_float (float x)
+{
+  __asm__ ("foo\t%0" : "+"FPR (x));
+  return x;
+}
+
+float
+test_float_from_mem (float *x)
+{
+  __asm__ ("foo\t%0" : "+"FPR (*x));
+  return *x;
+}
+
+double
+test_double (double x)
+{
+  __asm__ ("foo\t%0" : "+"FPR (x));
+  return x;
+}
+
+double
+test_double_from_mem (double *x)
+{
+  __asm__ ("foo\t%0" : "+"FPR (*x));
+  return *x;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-5.c b/gcc/testsuite/gcc.dg/asm-hard-reg-5.c
new file mode 100644
index 0000000..a9e25ce
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-5.c
@@ -0,0 +1,36 @@
+/* { dg-do compile { target aarch64*-*-* powerpc64*-*-* riscv64-*-* s390*-*-* x86_64-*-* } } */
+
+typedef int V __attribute__ ((vector_size (4 * sizeof (int))));
+
+#if defined (__aarch64__)
+# define VR "{v20}"
+/* { dg-final { scan-assembler-times "foo\tv20" 2 { target { aarch64*-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define VR "{v5}"
+/* { dg-final { scan-assembler-times "foo\t5" 2 { target { powerpc64*-*-* } } } } */
+#elif defined (__riscv)
+# define VR "{v5}"
+/* { dg-additional-options "-march=rv64imv" { target riscv64-*-* } } */
+/* { dg-final { scan-assembler-times "foo\tv5" 2 { target { riscv*-*-* } } } } */
+#elif defined (__s390__)
+# define VR "{v5}"
+/* { dg-require-effective-target s390_mvx { target s390*-*-* } } */
+/* { dg-final { scan-assembler-times "foo\t%v5" 2 { target s390*-*-* } } } */
+#elif defined (__x86_64__)
+# define VR "{xmm9}"
+/* { dg-final { scan-assembler-times "foo\t%xmm9" 2 { target { x86_64-*-* } } } } */
+#endif
+
+V
+test (V x)
+{
+  __asm__ ("foo\t%0" : "+"VR (x));
+  return x;
+}
+
+V
+test_from_mem (V *x)
+{
+  __asm__ ("foo\t%0" : "+"VR (*x));
+  return *x;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-6.c b/gcc/testsuite/gcc.dg/asm-hard-reg-6.c
new file mode 100644
index 0000000..d9b7fae
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-6.c
@@ -0,0 +1,60 @@
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* riscv*-*-* s390*-*-* x86_64-*-* } } */
+/* { dg-options "-O2" } */
+
+/* Test multiple alternatives.  */
+
+#if defined (__aarch64__)
+# define GPR1 "{x1}"
+# define GPR2 "{x2}"
+# define GPR3 "{x3}"
+/* { dg-final { scan-assembler-times "foo\tx1,x3" 1 { target { aarch64*-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\tx2,\\\[x1\\\]" 1 { target { aarch64*-*-* } } } } */
+#elif defined (__arm__)
+# define GPR1 "{r1}"
+# define GPR2 "{r2}"
+# define GPR3 "{r3}"
+/* { dg-final { scan-assembler-times "foo\tr1,r3" 1 { target { arm*-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\tr2,\\\[r1\\\]" 1 { target { arm*-*-* } } } } */
+#elif defined (__i386__)
+# define GPR1 "{eax}"
+# define GPR2 "{ebx}"
+# define GPR3 "{ecx}"
+/* { dg-final { scan-assembler-times "foo\t4\\(%esp\\),%ecx" 1 { target { i?86-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\t%ebx,\\(%eax\\)" 1 { target { i?86-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define GPR1 "{r4}"
+# define GPR2 "{r5}"
+# define GPR3 "{r6}"
+/* { dg-final { scan-assembler-times "foo\t4,6" 1 { target { powerpc*-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\t5,0\\(4\\)" 1 { target { powerpc*-*-* } } } } */
+#elif defined (__riscv)
+# define GPR1 "{t1}"
+# define GPR2 "{t2}"
+# define GPR3 "{t3}"
+/* { dg-final { scan-assembler-times "foo\tt1,t3" 1 { target { riscv*-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\tt2,0\\(a1\\)" 1 { target { riscv*-*-* } } } } */
+#elif defined (__s390__)
+# define GPR1 "{r0}"
+# define GPR2 "{r1}"
+# define GPR3 "{r2}"
+/* { dg-final { scan-assembler-times "foo\t%r0,%r2" 1 { target { s390*-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\t%r1,0\\(%r3\\)" 1 { target { s390*-*-* } } } } */
+#elif defined (__x86_64__)
+# define GPR1 "{eax}"
+# define GPR2 "{ebx}"
+# define GPR3 "{rcx}"
+/* { dg-final { scan-assembler-times "foo\t%eax,%rcx" 1 { target { x86_64-*-* } } } } */
+/* { dg-final { scan-assembler-times "bar\t%ebx,\\(%rsi\\)" 1 { target { x86_64-*-* } } } } */
+#endif
+
+void
+test_reg_reg (int x, long long *y)
+{
+  __asm__ ("foo\t%0,%1" :: GPR1"m,"GPR2 (x), GPR3",m" (y));
+}
+
+void
+test_reg_mem (int x, long long *y)
+{
+  __asm__ ("bar\t%0,%1" :: GPR1"m,"GPR2 (x), GPR3",m" (*y));
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-7.c b/gcc/testsuite/gcc.dg/asm-hard-reg-7.c
new file mode 100644
index 0000000..761a6b7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-7.c
@@ -0,0 +1,41 @@
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* riscv*-*-* s390*-*-* x86_64-*-* } } */
+/* { dg-options "-O2" } */
+
+/* Test multiple alternatives.  */
+
+#if defined (__aarch64__)
+# define GPR "{x1}"
+/* { dg-final { scan-assembler-times "foo\tx1,x1" 2 { target { aarch64*-*-* } } } } */
+#elif defined (__arm__)
+# define GPR "{r1}"
+/* { dg-final { scan-assembler-times "foo\tr1,r1" 2 { target { arm*-*-* } } } } */
+#elif defined (__i386__)
+# define GPR "{eax}"
+/* { dg-final { scan-assembler-times "foo\t%eax,%eax" 2 { target { i?86-*-* } } } } */
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define GPR "{r4}"
+/* { dg-final { scan-assembler-times "foo\t4,4" 2 { target { powerpc*-*-* } } } } */
+#elif defined (__riscv)
+# define GPR "{t1}"
+/* { dg-final { scan-assembler-times "foo\tt1,t1" 2 { target { riscv*-*-* } } } } */
+#elif defined (__s390__)
+# define GPR "{r0}"
+/* { dg-final { scan-assembler-times "foo\t%r0,%r0" 2 { target { s390*-*-* } } } } */
+#elif defined (__x86_64__)
+# define GPR "{eax}"
+/* { dg-final { scan-assembler-times "foo\t%eax,%eax" 2 { target { x86_64-*-* } } } } */
+#endif
+
+int
+test_1 (int x)
+{
+  __asm__ ("foo\t%0,%0" : "+"GPR (x));
+  return x;
+}
+
+int
+test_2 (int x, int y)
+{
+  __asm__ ("foo\t%0,%1" : "="GPR (x) : GPR (y));
+  return x;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-8.c b/gcc/testsuite/gcc.dg/asm-hard-reg-8.c
new file mode 100644
index 0000000..cda5e3e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-8.c
@@ -0,0 +1,49 @@
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* riscv*-*-* s390*-*-* x86_64-*-* } } */
+
+/* Due to hard register constraints, X must be copied.  */
+
+#if defined (__aarch64__)
+# define GPR1 "{x1}"
+# define GPR2 "{x2}"
+#elif defined (__arm__)
+# define GPR1 "{r1}"
+# define GPR2 "{r2}"
+#elif defined (__i386__)
+# define GPR1 "{eax}"
+# define GPR2 "{ebx}"
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define GPR1 "{r4}"
+# define GPR2 "{r5}"
+#elif defined (__riscv)
+# define GPR1 "{t1}"
+# define GPR2 "{t2}"
+#elif defined (__s390__)
+# define GPR1 "{r0}"
+# define GPR2 "{r1}"
+#elif defined (__x86_64__)
+# define GPR1 "{eax}"
+# define GPR2 "{ebx}"
+#endif
+
+#define TEST(T) \
+int \
+test_##T (T x) \
+{ \
+  int out; \
+  __asm__ ("foo" : "=r" (out) : GPR1 (x), GPR2 (x)); \
+  return out; \
+}
+
+TEST(char)
+TEST(short)
+TEST(int)
+TEST(long)
+
+int
+test_subreg (long x)
+{
+  int out;
+  short subreg_x = x;
+  __asm__ ("foo" : "=r" (out) : GPR1 (x), GPR2 (subreg_x));
+  return out;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-error-1.c b/gcc/testsuite/gcc.dg/asm-hard-reg-error-1.c
new file mode 100644
index 0000000..0d7c2f2
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-error-1.c
@@ -0,0 +1,83 @@
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* riscv*-*-* s390*-*-* x86_64-*-* } } */
+
+#if defined (__aarch64__)
+# define GPR1_RAW "x0"
+# define GPR2 "{x1}"
+# define GPR3 "{x2}"
+# define INVALID_GPR_A "{x31}"
+#elif defined (__arm__)
+# define GPR1_RAW "r0"
+# define GPR2 "{r1}"
+# define GPR3 "{r2}"
+# define INVALID_GPR_A "{r16}"
+#elif defined (__i386__)
+# define GPR1_RAW "%eax"
+# define GPR2 "{%ebx}"
+# define GPR3 "{%edx}"
+# define INVALID_GPR_A "{%eex}"
+#elif defined (__powerpc__) || defined (__POWERPC__)
+# define GPR1_RAW "r4"
+# define GPR2 "{r5}"
+# define GPR3 "{r6}"
+# define INVALID_GPR_A "{r33}"
+#elif defined (__riscv)
+# define GPR1_RAW "t4"
+# define GPR2 "{t5}"
+# define GPR3 "{t6}"
+# define INVALID_GPR_A "{t7}"
+#elif defined (__s390__)
+# define GPR1_RAW "r4"
+# define GPR2 "{r5}"
+# define GPR3 "{r6}"
+# define INVALID_GPR_A "{r17}"
+#elif defined (__x86_64__)
+# define GPR1_RAW "rax"
+# define GPR2 "{rbx}"
+# define GPR3 "{rcx}"
+# define INVALID_GPR_A "{rex}"
+#endif
+
+#define GPR1 "{"GPR1_RAW"}"
+#define INVALID_GPR_B "{"GPR1_RAW
+
+struct { int a[128]; } s = {0};
+
+void
+test (void)
+{
+  int x, y;
+  register int gpr1 __asm__ (GPR1_RAW) = 0;
+
+  __asm__ ("" :: "{}" (42)); /* { dg-error "invalid input constraint: \{\}" } */
+  __asm__ ("" :: INVALID_GPR_A (42)); /* { dg-error "invalid input constraint" } */
+  __asm__ ("" :: INVALID_GPR_B (42)); /* { dg-error "invalid input constraint" } */
+
+  __asm__ ("" :: GPR1 (s)); /* { dg-error "data type isn't suitable for register .* of operand 0" } */
+
+  __asm__ ("" :: "r" (gpr1), GPR1 (42)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" :: GPR1 (42), "r" (gpr1)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" :: GPR1 (42), GPR1 (42)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" :: GPR1","GPR2 (42), GPR2","GPR3 (42));
+  __asm__ ("" :: GPR1","GPR2 (42), GPR3","GPR2 (42)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" :: GPR1","GPR2 (42), GPR1","GPR3 (42)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" :: GPR1 GPR2 (42), GPR2 (42)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" : "+"GPR1 (x), "="GPR1 (y)); /* { dg-error "multiple outputs to hard register" } */
+  __asm__ ("" : "="GPR1 (y) : GPR1 (42), "0" (42)); /* { dg-error "multiple inputs to hard register" } */
+  __asm__ ("" : "+"GPR1 (x) : GPR1 (42)); /* { dg-error "multiple inputs to hard register" } */
+
+  __asm__ ("" : "="GPR1 (gpr1));
+  __asm__ ("" : "="GPR2 (gpr1)); /* { dg-error "constraint and register 'asm' for output operand 0 are unsatisfiable" } */
+  __asm__ ("" :: GPR2 (gpr1)); /* { dg-error "constraint and register 'asm' for input operand 0 are unsatisfiable" } */
+  __asm__ ("" : "="GPR1 (x) : "0" (gpr1));
+  __asm__ ("" : "="GPR1 GPR2 (x) : "0" (gpr1)); /* { dg-error "constraint and register 'asm' for input operand 0 are unsatisfiable" } */
+
+  __asm__ ("" : "=&"GPR1 (x) : "0" (gpr1));
+  __asm__ ("" : "=&"GPR1 (x) : "0" (42));
+  __asm__ ("" : "=&"GPR2","GPR1 (x) : "r,"GPR1 (42));
+  __asm__ ("" : "="GPR2",&"GPR1 (x) : "r,"GPR1 (42)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+  __asm__ ("" : "=&"GPR1 (x) : GPR1 (42)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+  __asm__ ("" : "=&"GPR2","GPR1 (x) : "r,r" (gpr1));
+  __asm__ ("" : "="GPR2",&"GPR1 (x) : "r,r" (gpr1)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+  __asm__ ("" : "=&r" (gpr1) : GPR1 (42)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+  __asm__ ("" : "=&"GPR1 (x),  "=r" (y) : "1" (gpr1)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-error-2.c b/gcc/testsuite/gcc.dg/asm-hard-reg-error-2.c
new file mode 100644
index 0000000..d0d5cfe
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-error-2.c
@@ -0,0 +1,26 @@
+/* { dg-do compile { target { { aarch64*-*-* s390x-*-* } && int128 } } } */
+/* { dg-options "-O2" } get rid of -ansi since we use __int128 */
+
+/* Test register pairs.  */
+
+#if defined (__aarch64__)
+# define GPR1 "{x4}"
+# define GPR2_RAW "x5"
+#elif defined (__s390__)
+# define GPR1 "{r4}"
+# define GPR2_RAW "r5"
+#endif
+
+#define GPR2 "{"GPR2_RAW"}"
+
+void
+test (void)
+{
+  __asm__ ("" :: GPR1 ((__int128) 42));
+  __asm__ ("" :: GPR2 ((__int128) 42)); /* { dg-error "register .* for operand 0 isn't suitable for data type" } */
+  __asm__ ("" :: GPR1 ((__int128) 42), GPR2 (42)); /* { dg-error "multiple inputs to hard register" } */
+
+  __int128 x;
+  __asm__ ("" : "="GPR1 (x) :: GPR2_RAW); /* { dg-error "hard register constraint for output 0 conflicts with 'asm' clobber list" } */
+  __asm__ ("" : "=r" (x) : GPR1 (x) : GPR2_RAW); /* { dg-error "hard register constraint for input 0 conflicts with 'asm' clobber list" } */
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-error-3.c b/gcc/testsuite/gcc.dg/asm-hard-reg-error-3.c
new file mode 100644
index 0000000..17b2317
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-error-3.c
@@ -0,0 +1,27 @@
+/* { dg-do compile { target arm-*-* s390-*-* } } */
+/* { dg-options "-std=c99" } we need long long */
+/* { dg-additional-options "-march=armv7-a" { target arm-*-* } } */
+
+/* Test register pairs.  */
+
+#if defined (__arm__)
+# define GPR1 "{r4}"
+# define GPR2_RAW "r5"
+#elif defined (__s390__)
+# define GPR1 "{r4}"
+# define GPR2_RAW "r5"
+#endif
+
+#define GPR2 "{"GPR2_RAW"}"
+
+void
+test (void)
+{
+  __asm__ ("" :: GPR1 (42ll));
+  __asm__ ("" :: GPR2 (42ll)); /* { dg-error "register .* for operand 0 isn't suitable for data type" } */
+  __asm__ ("" :: GPR1 (42ll), GPR2 (42)); /* { dg-error "multiple inputs to hard register" } */
+
+  long long x;
+  __asm__ ("" : "="GPR1 (x) :: GPR2_RAW); /* { dg-error "hard register constraint for output 0 conflicts with 'asm' clobber list" } */
+  __asm__ ("" : "=r" (x) : GPR1 (x) : GPR2_RAW); /* { dg-error "hard register constraint for input 0 conflicts with 'asm' clobber list" } */
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-error-4.c b/gcc/testsuite/gcc.dg/asm-hard-reg-error-4.c
new file mode 100644
index 0000000..465f24b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-error-4.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+
+/* Verify output operands.  */
+
+int
+test (void)
+{
+  int x;
+  register int y __asm__ ("0");
+
+  /* Preserve status quo and don't error out.  */
+  __asm__ ("" : "=r" (x), "=r" (x));
+
+  /* Be more strict for hard register constraints and error out.  */
+  __asm__ ("" : "={0}" (x), "={1}" (x)); /* { dg-error "multiple outputs to lvalue 'x'" } */
+
+  /* Still error out in case of a mixture.  */
+  __asm__ ("" : "=r" (x), "={1}" (x)); /* { dg-error "multiple outputs to lvalue 'x'" } */
+
+  return x + y;
+}
diff --git a/gcc/testsuite/gcc.dg/asm-hard-reg-error-5.c b/gcc/testsuite/gcc.dg/asm-hard-reg-error-5.c
new file mode 100644
index 0000000..85398f0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/asm-hard-reg-error-5.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+
+/* Test clobbers.
+   See asm-hard-reg-error-{2,3}.c for tests involving register pairs.  */
+
+int
+test (void)
+{
+  int x, y;
+  __asm__ ("" : "={0}" (x), "={1}" (y) : : "1"); /* { dg-error "hard register constraint for output 1 conflicts with 'asm' clobber list" } */
+  __asm__ ("" : "={0}" (x) : "{0}" (y), "{1}" (y) : "1"); /* { dg-error "hard register constraint for input 1 conflicts with 'asm' clobber list" } */
+  return x + y;
+}
diff --git a/gcc/testsuite/gcc.dg/bitint-124.c b/gcc/testsuite/gcc.dg/bitint-124.c
new file mode 100644
index 0000000..160a1e3
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/bitint-124.c
@@ -0,0 +1,30 @@
+/* PR tree-optimization/121131 */
+/* { dg-do run { target bitint } } */
+/* { dg-options "-O2" } */
+
+#if __BITINT_MAXWIDTH__ >= 156
+struct A { _BitInt(156) b : 135; };
+
+static inline _BitInt(156)
+foo (struct A *x)
+{
+  return x[1].b;
+}
+
+__attribute__((noipa)) _BitInt(156)
+bar (void)
+{
+  struct A a[] = { 1, 1, -13055525270329736316393717310914023773847wb,
+		   1, 1, 1, 1, 1, 1, 1, 1, 1 };
+  return foo (&a[1]);
+}
+#endif
+
+int
+main ()
+{
+#if __BITINT_MAXWIDTH__ >= 156
+  if (bar () != -13055525270329736316393717310914023773847wb)
+    __builtin_abort ();
+#endif
+}
diff --git a/gcc/testsuite/gcc.dg/builtin-dynamic-object-size-pr120780.c b/gcc/testsuite/gcc.dg/builtin-dynamic-object-size-pr120780.c
index 0d6593e..12e6c29 100644
--- a/gcc/testsuite/gcc.dg/builtin-dynamic-object-size-pr120780.c
+++ b/gcc/testsuite/gcc.dg/builtin-dynamic-object-size-pr120780.c
@@ -207,7 +207,7 @@ test5 (size_t sz)
 }
 
 int
-main (size_t sz)
+main (void)
 {
   test1 (sizeof (struct container));
   test1 (sizeof (struct container) - sizeof (int));
diff --git a/gcc/testsuite/gcc.dg/crc-non-cst-poly-1.c b/gcc/testsuite/gcc.dg/crc-non-cst-poly-1.c
new file mode 100644
index 0000000..0c3d905
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/crc-non-cst-poly-1.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "" } */
+
+/* PR middle-end/120709 */
+/* Make sure we don't ICE on a non-constant poly argument. */
+
+
+typedef unsigned char uint8_t;
+uint8_t crc8_data8(uint8_t crc, uint8_t data, uint8_t polynomial) {
+  return __builtin_rev_crc32_data8 (crc, data, polynomial); /* { dg-error "must be a constant" } */
+}
diff --git a/gcc/testsuite/gcc.dg/darwin-minversion-link.c b/gcc/testsuite/gcc.dg/darwin-minversion-link.c
index af712a1b..55f7c7e 100644
--- a/gcc/testsuite/gcc.dg/darwin-minversion-link.c
+++ b/gcc/testsuite/gcc.dg/darwin-minversion-link.c
@@ -20,6 +20,7 @@
 /* { dg-additional-options "-mmacosx-version-min=013.000.00 -DCHECK=130000" { target *-*-darwin22* } } */
 /* { dg-additional-options "-mmacosx-version-min=014.000.00 -DCHECK=140000" { target *-*-darwin23* } } */
 /* { dg-additional-options "-mmacosx-version-min=015.000.00 -DCHECK=150000" { target *-*-darwin24* } } */
+/* { dg-additional-options "-mmacosx-version-min=026.000.00 -DCHECK=260000" { target *-*-darwin25* } } */
 
 int
 main ()
diff --git a/gcc/testsuite/gcc.dg/flex-array-counted-by-pr121000.c b/gcc/testsuite/gcc.dg/flex-array-counted-by-pr121000.c
new file mode 100644
index 0000000..5b9a2c6
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/flex-array-counted-by-pr121000.c
@@ -0,0 +1,43 @@
+/* PR middle-end/121000 */
+/* { dg-do run } */
+/* { dg-options "-O" } */
+
+#include "builtin-object-size-common.h"
+
+/* The parameter m must be const qualified to avoid the m is 
+   marked as TREE_SIDE_EFFECTS in IR.
+   The __builtin_dynamic_object_size will be folded as -1 by
+   fold_builtin_object_size when m is NOT const qualified.  */
+
+void
+foo (int n, const int m)
+{
+  typedef int A[m];
+  struct S { int n, m; A a[2]; A b[] __attribute__((counted_by (n))); } *p;
+  p = __builtin_malloc (sizeof (struct S) + sizeof (A) * n);
+  p->n = n;
+  p->m = m;
+  EXPECT (__builtin_dynamic_object_size (p->b, 1), sizeof (A) * n);
+}
+
+/* The parameter m1, m2 must be const qualified to avoid the m is 
+   marked as TREE_SIDE_EFFECTS in IR.
+   The __builtin_dynamic_object_size will be folded as -1 by
+   fold_builtin_object_size when m1 or m2 is NOT const qualified.  */
+void
+foo_1 (int n, const int m1, const int m2)
+{
+  typedef int A[m1][m2];
+  struct S { int n; A a[2]; A b[] __attribute__((counted_by (n))); } *p;
+  p = __builtin_malloc (sizeof (struct S) + sizeof (A) * n);
+  p->n = n;
+  EXPECT (__builtin_dynamic_object_size (p->b, 1), sizeof (A) * n);
+}
+
+int
+main ()
+{
+  foo (2, 10);
+  foo_1 (2, 10, 20);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/flex-array-counted-by.c b/gcc/testsuite/gcc.dg/flex-array-counted-by.c
index 4fa91ff..16eb2c6 100644
--- a/gcc/testsuite/gcc.dg/flex-array-counted-by.c
+++ b/gcc/testsuite/gcc.dg/flex-array-counted-by.c
@@ -10,7 +10,7 @@ int x __attribute ((counted_by (size))); /* { dg-error "attribute is not allowed
 
 struct trailing {
   int count;
-  int field __attribute ((counted_by (count))); /* { dg-error "attribute is not allowed for a non-array or non-pointer field" } */
+  int field __attribute ((counted_by (count))); /* { dg-error "attribute is not allowed for a non-array field" } */
 };
 
 struct trailing_1 {
diff --git a/gcc/testsuite/gcc.dg/guality/guality.h b/gcc/testsuite/gcc.dg/guality/guality.h
index d41327c..48b59d2e 100644
--- a/gcc/testsuite/gcc.dg/guality/guality.h
+++ b/gcc/testsuite/gcc.dg/guality/guality.h
@@ -204,9 +204,10 @@ int volatile guality_attached;
    of this wrapping, guality_main may not have an empty argument
    list.  */
 
-extern int guality_main (int argc, char *argv[]);
+extern int __attribute__((noipa))
+guality_main (int argc, char *argv[]);
 
-static void __attribute__((noinline))
+static void __attribute__((noipa))
 guality_check (const char *name, gualchk_t value, int unknown_ok);
 
 /* Set things up, run guality_main, then print a summary and quit.  */
diff --git a/gcc/testsuite/gcc.dg/memchr-3.c b/gcc/testsuite/gcc.dg/memchr-3.c
index 9a35735..9caa2ac 100644
--- a/gcc/testsuite/gcc.dg/memchr-3.c
+++ b/gcc/testsuite/gcc.dg/memchr-3.c
@@ -17,9 +17,10 @@ struct SX
 const struct SX sx = { 0x1221 };
 const char sx_rep[] = { };
 
-void test_find (void)
+int test_find (void)
 {
   int n = 0, nb = (const char*)&sx.a - (const char*)&sx;
   const char *p = (const char*)&sx, *q = sx_rep;
   n += p + 1 == memchr (p, q[1], nb);
+  return n;
 }
diff --git a/gcc/testsuite/gcc.dg/nest.c b/gcc/testsuite/gcc.dg/nest.c
index 5734c11..2dce65e 100644
--- a/gcc/testsuite/gcc.dg/nest.c
+++ b/gcc/testsuite/gcc.dg/nest.c
@@ -3,6 +3,7 @@
 /* { dg-require-profiling "-pg" } */
 /* { dg-options "-O2 -pg" } */
 /* { dg-options "-O2 -pg -static" { target hppa*-*-hpux* } } */
+/* { dg-additional-options "-mfentry -fno-pic" { target i?86-*-gnu* x86_64-*-gnu* } } */
 /* { dg-error "profiler" "No profiler support" { target xstormy16-*-* } 0 } */
 
 extern void abort (void);
diff --git a/gcc/testsuite/gcc.dg/plugin/analyzer_cpython_plugin.cc b/gcc/testsuite/gcc.dg/plugin/analyzer_cpython_plugin.cc
index 1fe5b5c..01ab766 100644
--- a/gcc/testsuite/gcc.dg/plugin/analyzer_cpython_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/analyzer_cpython_plugin.cc
@@ -23,8 +23,8 @@
 #include "target.h"
 #include "fold-const.h"
 #include "tree-pretty-print.h"
-#include "diagnostic-color.h"
-#include "diagnostic-metadata.h"
+#include "diagnostics/color.h"
+#include "diagnostics/metadata.h"
 #include "tristate.h"
 #include "bitmap.h"
 #include "selftest.h"
diff --git a/gcc/testsuite/gcc.dg/plugin/analyzer_gil_plugin.cc b/gcc/testsuite/gcc.dg/plugin/analyzer_gil_plugin.cc
index fa2f2fa..c101c45 100644
--- a/gcc/testsuite/gcc.dg/plugin/analyzer_gil_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/analyzer_gil_plugin.cc
@@ -16,7 +16,7 @@
 #include "gimple.h"
 #include "gimple-iterator.h"
 #include "gimple-walk.h"
-#include "diagnostic-event-id.h"
+#include "diagnostics/event-id.h"
 #include "analyzer/common.h"
 #include "analyzer/analyzer-logging.h"
 #include "json.h"
@@ -120,20 +120,22 @@ public:
     return false;
   }
 
-  diagnostic_event::meaning
+  diagnostics::paths::event::meaning
   get_meaning_for_state_change (const evdesc::state_change &change)
     const final override
   {
+    using event = diagnostics::paths::event;
+
     if (change.is_global_p ())
       {
 	if (change.m_new_state == m_sm.m_released_gil)
-	  return diagnostic_event::meaning (diagnostic_event::verb::release,
-					    diagnostic_event::noun::lock);
+	  return event::meaning (event::verb::release,
+				 event::noun::lock);
 	else if (change.m_new_state == m_sm.get_start_state ())
-	  return diagnostic_event::meaning (diagnostic_event::verb::acquire,
-					    diagnostic_event::noun::lock);
+	  return event::meaning (event::verb::acquire,
+				 event::noun::lock);
       }
-    return diagnostic_event::meaning ();
+    return event::meaning ();
   }
  protected:
   gil_diagnostic (const gil_state_machine &sm) : m_sm (sm)
diff --git a/gcc/testsuite/gcc.dg/plugin/analyzer_kernel_plugin.cc b/gcc/testsuite/gcc.dg/plugin/analyzer_kernel_plugin.cc
index 18e054b..fc282a7 100644
--- a/gcc/testsuite/gcc.dg/plugin/analyzer_kernel_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/analyzer_kernel_plugin.cc
@@ -23,8 +23,8 @@
 #include "target.h"
 #include "fold-const.h"
 #include "tree-pretty-print.h"
-#include "diagnostic-color.h"
-#include "diagnostic-metadata.h"
+#include "diagnostics/color.h"
+#include "diagnostics/metadata.h"
 #include "tristate.h"
 #include "bitmap.h"
 #include "selftest.h"
diff --git a/gcc/testsuite/gcc.dg/plugin/analyzer_known_fns_plugin.cc b/gcc/testsuite/gcc.dg/plugin/analyzer_known_fns_plugin.cc
index 5a6e075..44fcf37 100644
--- a/gcc/testsuite/gcc.dg/plugin/analyzer_known_fns_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/analyzer_known_fns_plugin.cc
@@ -23,8 +23,8 @@
 #include "target.h"
 #include "fold-const.h"
 #include "tree-pretty-print.h"
-#include "diagnostic-color.h"
-#include "diagnostic-metadata.h"
+#include "diagnostics/color.h"
+#include "diagnostics/metadata.h"
 #include "tristate.h"
 #include "bitmap.h"
 #include "selftest.h"
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-html.c b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-html.c
new file mode 100644
index 0000000..2256a63
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-html.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-fdiagnostics-add-output=experimental-html:javascript=no" } */
+
+extern void here (void);
+
+void test_graphs (void)
+{
+  here (); /* { dg-error "this is a placeholder error, with graphs" } */
+}
+
+/* Use a Python script to verify various properties about the generated
+   HTML file:
+   { dg-final { run-html-pytest diagnostic-test-graphs-html.c "diagnostic-test-graphs-html.py" } } */
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-html.py b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-html.py
new file mode 100644
index 0000000..11e5fd1
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-html.py
@@ -0,0 +1,48 @@
+# Verify that metadata works in HTML output.
+
+from htmltest import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def html_tree():
+    return html_tree_from_env()
+
+def test_result_graph(html_tree):
+    root = html_tree.getroot ()
+    assert root.tag == make_tag('html')
+
+    body = root.find('xhtml:body', ns)
+    assert body is not None
+
+    diag_list = body.find('xhtml:div', ns)
+    assert diag_list is not None
+    assert diag_list.attrib['class'] == 'gcc-diagnostic-list'
+
+    diag = diag_list.find('xhtml:div', ns)
+    assert diag is not None
+
+    message = diag.find("./xhtml:div[@class='gcc-message']", ns)
+    assert message.attrib['id'] == 'gcc-diag-0-message'
+
+    assert message[0].tag == make_tag('strong')
+    assert message[0].tail == ' this is a placeholder error, with graphs '
+
+    graph = diag.find("./xhtml:div[@class='gcc-directed-graph']", ns)
+    assert graph is not None
+
+    header = graph.find("./xhtml:h2", ns)
+    assert header.text == 'foo'
+
+def test_run_graph(html_tree):
+    root = html_tree.getroot ()
+    assert root.tag == make_tag('html')
+
+    body = root.find('xhtml:body', ns)
+    assert body is not None
+
+    graph = body.find("./xhtml:div[@class='gcc-directed-graph']", ns)
+    assert graph is not None
+
+    header = graph.find("./xhtml:h2", ns)
+    assert header.text == 'Optimization Passes'
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c
new file mode 100644
index 0000000..4170f51
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-fdiagnostics-add-output=sarif" } */
+
+extern void here (void);
+
+void test_graphs (void)
+{
+  here (); /* { dg-error "this is a placeholder error, with graphs" } */
+}
+
+/* Verify that some JSON was written to a file with the expected name.  */
+/* { dg-final { verify-sarif-file } } */
+
+/* Use a Python script to verify various properties about the generated
+   .sarif file:
+   { dg-final { run-sarif-pytest diagnostic-test-graphs-sarif.c "diagnostic-test-graphs-sarif.py" } } */
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.py b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.py
new file mode 100644
index 0000000..4bb7535
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.py
@@ -0,0 +1,55 @@
+from sarif import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def sarif():
+    return sarif_from_env()
+
+def test_basics(sarif):
+    schema = sarif['$schema']
+    assert schema == "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json"
+
+    version = sarif['version']
+    assert version == "2.1.0"
+
+def test_result_graph(sarif):
+    runs = sarif['runs']
+    run = runs[0]
+    results = run['results']
+
+    assert len(results) == 1
+    
+    result = results[0]
+    assert result['level'] == 'error'
+    assert result['message']['text'] == "this is a placeholder error, with graphs"
+
+    assert len(result['graphs']) == 2
+
+    assert result['graphs'][0]['description']['text'] == 'foo'
+
+    assert len(result['graphs'][0]['nodes']) == 2
+    assert result['graphs'][0]['nodes'][0]['id'] == 'a'
+    assert result['graphs'][0]['nodes'][1]['id'] == 'b'
+    assert result['graphs'][0]['nodes'][1]['properties']['/placeholder-prefix/color'] == 'red'
+    assert len(result['graphs'][0]['nodes'][1]['children']) == 1
+    assert result['graphs'][0]['nodes'][1]['children'][0]['id'] == 'c'
+    assert result['graphs'][0]['nodes'][1]['children'][0]['label']['text'] == 'I am a node label'
+
+    assert len(result['graphs'][0]['edges']) == 1
+    result['graphs'][0]['edges'][0]['id'] == 'my-edge'
+    assert result['graphs'][0]['edges'][0]['label']['text'] == 'I am an edge label'
+    assert result['graphs'][0]['edges'][0]['sourceNodeId'] == 'a'
+    assert result['graphs'][0]['edges'][0]['targetNodeId'] == 'c'
+    
+    assert result['graphs'][1]['description']['text'] == 'bar'
+
+def test_run_graph(sarif):
+    runs = sarif['runs']
+    run = runs[0]
+
+    assert len(run['graphs']) == 1
+
+    assert run['graphs'][0]['description']['text'] == 'Optimization Passes'
+    assert run['graphs'][0]['nodes'][0]['id'] == 'all_lowering_passes'
+    assert run['graphs'][0]['edges'][0]['id'] == 'edge0'
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs.c b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs.c
new file mode 100644
index 0000000..7973954
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs.c
@@ -0,0 +1,8 @@
+/* { dg-do compile } */
+
+extern void here (void);
+
+void test_graphs (void)
+{
+  here (); /* { dg-error "this is a placeholder error, with graphs" } */
+}
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-show-locus.py b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-show-locus.py
index aca1b6c..eaca35a 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic-test-show-locus.py
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic-test-show-locus.py
@@ -1,4 +1,4 @@
-# Verify that diagnostic-show-locus.cc works with HTML output.
+# Verify that diagnostics/source-printing.cc works with HTML output.
 
 from htmltest import *
 
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_group_plugin.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_group_plugin.cc
index 4ade232..48f8325 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_group_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_group_plugin.cc
@@ -27,7 +27,7 @@
 #include "plugin-version.h"
 #include "c-family/c-common.h"
 #include "diagnostic.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 #include "context.h"
 
 int plugin_is_GPL_compatible;
@@ -165,8 +165,8 @@ pass_test_groups::execute (function *fun)
    expected output.  */
 
 void
-test_diagnostic_text_starter (diagnostic_text_output_format &text_output,
-			      const diagnostic_info *diagnostic)
+test_diagnostic_text_starter (diagnostics::text_sink &text_output,
+			      const diagnostics::diagnostic_info *diagnostic)
 {
   pp_set_prefix (text_output.get_printer (), xstrdup ("PREFIX: "));
 }
@@ -175,22 +175,22 @@ test_diagnostic_text_starter (diagnostic_text_output_format &text_output,
    expected output.  */
 
 void
-test_diagnostic_start_span_fn (const diagnostic_location_print_policy &,
-			       to_text &sink,
+test_diagnostic_start_span_fn (const diagnostics::location_print_policy &,
+			       diagnostics::to_text &sink,
 			       expanded_location)
 {
-  pretty_printer *pp = get_printer (sink);
+  pretty_printer *pp = diagnostics::get_printer (sink);
   pp_string (pp, "START_SPAN_FN: ");
   pp_newline (pp);
 }
 
-/* Custom output format subclass.  */
+/* Custom text_sink subclass.  */
 
-class test_output_format : public diagnostic_text_output_format
+class custom_test_sink : public diagnostics::text_sink
 {
  public:
-  test_output_format (diagnostic_context &context)
-  : diagnostic_text_output_format (context)
+  custom_test_sink (diagnostics::context &context)
+  : diagnostics::text_sink (context)
   {}
 
   void on_begin_group () final override
@@ -228,10 +228,9 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  diagnostic_text_starter (global_dc) = test_diagnostic_text_starter;
-  diagnostic_start_span (global_dc) = test_diagnostic_start_span_fn;
-  global_dc->set_output_format
-    (::std::make_unique<test_output_format> (*global_dc));
+  diagnostics::text_starter (global_dc) = test_diagnostic_text_starter;
+  diagnostics::start_span (global_dc) = test_diagnostic_start_span_fn;
+  global_dc->set_sink (::std::make_unique<custom_test_sink> (*global_dc));
 
   pass_info.pass = new pass_test_groups (g);
   pass_info.reference_pass_name = "*warn_function_noreturn";
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_show_trees.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_show_trees.cc
index 44b94da..7e34a42 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_show_trees.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_show_trees.cc
@@ -115,7 +115,7 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  global_dc->m_source_printing.max_width = 80;
+  global_dc->get_source_printing_options ().max_width = 80;
 
   register_callback (plugin_name,
 		     PLUGIN_PRE_GENERICIZE,
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_graphs.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_graphs.cc
new file mode 100644
index 0000000..7398a29
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_graphs.cc
@@ -0,0 +1,283 @@
+/* This plugin exercises diagnostic graphs.
+   We emit an error with a pair of digraphs associated with it,
+   and a global digraph showing the optimization passes.  */
+
+#define INCLUDE_MAP
+#define INCLUDE_STRING
+#define INCLUDE_VECTOR
+#include "gcc-plugin.h"
+#include "config.h"
+#include "system.h"
+#include "coretypes.h"
+#include "tm.h"
+#include "tree.h"
+#include "stringpool.h"
+#include "toplev.h"
+#include "basic-block.h"
+#include "hash-table.h"
+#include "vec.h"
+#include "ggc.h"
+#include "basic-block.h"
+#include "tree-ssa-alias.h"
+#include "internal-fn.h"
+#include "gimple.h"
+#include "gimple-iterator.h"
+#include "gimple-fold.h"
+#include "tree-eh.h"
+#include "gimple-expr.h"
+#include "is-a.h"
+#include "tree.h"
+#include "tree-pass.h"
+#include "intl.h"
+#include "plugin-version.h"
+#include "diagnostic.h"
+#include "context.h"
+#include "gcc-rich-location.h"
+#include "diagnostics/metadata.h"
+#include "diagnostics/digraphs.h"
+#include "pass_manager.h"
+
+
+int plugin_is_GPL_compatible;
+
+const pass_data pass_data_test_graph_emission =
+{
+  GIMPLE_PASS, /* type */
+  "test_graph_emission", /* name */
+  OPTGROUP_NONE, /* optinfo_flags */
+  TV_NONE, /* tv_id */
+  PROP_ssa, /* properties_required */
+  0, /* properties_provided */
+  0, /* properties_destroyed */
+  0, /* todo_flags_start */
+  0, /* todo_flags_finish */
+};
+
+class pass_test_graph_emission : public gimple_opt_pass
+{
+public:
+  pass_test_graph_emission(gcc::context *ctxt)
+    : gimple_opt_pass(pass_data_test_graph_emission, ctxt)
+  {}
+
+  /* opt_pass methods: */
+  bool gate (function *) { return true; }
+  virtual unsigned int execute (function *);
+
+}; // class pass_test_graph_emission
+
+/* Determine if STMT is a call with NUM_ARGS arguments to a function
+   named FUNCNAME.
+   If so, return STMT as a gcall *.  Otherwise return NULL.  */
+
+static gcall *
+check_for_named_call (gimple *stmt,
+		      const char *funcname, unsigned int num_args)
+{
+  gcc_assert (funcname);
+
+  gcall *call = dyn_cast <gcall *> (stmt);
+  if (!call)
+    return NULL;
+
+  tree fndecl = gimple_call_fndecl (call);
+  if (!fndecl)
+    return NULL;
+
+  if (strcmp (IDENTIFIER_POINTER (DECL_NAME (fndecl)), funcname))
+    return NULL;
+
+  if (gimple_call_num_args (call) != num_args)
+    {
+      error_at (stmt->location, "expected number of args: %i (got %i)",
+		num_args, gimple_call_num_args (call));
+      return NULL;
+    }
+
+  return call;
+}
+
+class lazy_passes_graph : public lazily_created<diagnostics::digraphs::digraph>
+{
+public:
+  lazy_passes_graph (const ::gcc::pass_manager &pass_manager_)
+  : m_pass_manager (pass_manager_)
+  {
+  }
+
+private:
+  std::unique_ptr<diagnostics::digraphs::digraph>
+  create_object () const final override
+  {
+    auto g = std::make_unique<diagnostics::digraphs::digraph> ();
+    g->set_description ("Optimization Passes");
+
+#define DEF_PASS_LIST(NAME)					\
+    add_top_level_pass_list (*g, #NAME, m_pass_manager.NAME);
+
+    GCC_PASS_LISTS
+
+#undef DEF_PASS_LIST
+
+    return g;
+  }
+
+  void
+  add_top_level_pass_list (diagnostics::digraphs::digraph &g,
+			   const char *pass_list_name,
+			   const opt_pass *p) const
+  {
+    gcc_assert (p);
+    auto n = std::make_unique<diagnostics::digraphs::node> (g, pass_list_name);
+    n->set_label (pass_list_name);
+    add_child_pass (g, *n, *p);
+    g.add_node (std::move (n));
+  }
+
+  diagnostics::digraphs::node &
+  add_child_pass (diagnostics::digraphs::digraph &g,
+		  diagnostics::digraphs::node &parent_node,
+		  const opt_pass &p) const
+  {
+    std::string node_label;
+    std::string node_id;
+    if (p.static_pass_number > 0 )
+      {
+	node_label = std::to_string (p.static_pass_number) + "_" + p.name;
+	node_id = node_label;
+      }
+    else
+      {
+	node_label = std::string (p.name);
+	pretty_printer pp;
+	pp_printf (&pp, "%s_%p", p.name, &p);
+	node_id = pp_formatted_text (&pp);
+      }
+    auto n
+      = std::make_unique<diagnostics::digraphs::node> (g,
+						       std::move (node_id));
+    n->set_label (node_label.c_str ());
+    diagnostics::digraphs::node &result = *n;
+    parent_node.add_child (std::move (n));
+
+    // TODO: add attrs for things like type, properties_*, etc
+
+    if (p.sub)
+      {
+	auto &other_node = add_child_pass (g, parent_node, *p.sub);
+	g.add_edge (nullptr, result, other_node, "sub");
+      }
+
+    if (p.next)
+      {
+	auto &other_node = add_child_pass (g, parent_node, *p.next);
+	g.add_edge (nullptr, result, other_node, "next");
+      }
+
+    return result;
+  }
+
+  const ::gcc::pass_manager &m_pass_manager;
+};
+
+static void
+report_diag_with_graphs (location_t loc)
+{
+  class my_lazy_digraphs : public diagnostics::metadata::lazy_digraphs
+  {
+  public:
+    using diagnostic_graph = diagnostics::digraphs::digraph;
+    using diagnostic_node = diagnostics::digraphs::node;
+    using diagnostic_edge = diagnostics::digraphs::edge;
+
+    std::unique_ptr<std::vector<std::unique_ptr<diagnostic_graph>>>
+    create_object () const final override
+    {
+      auto graphs
+	= std::make_unique<std::vector<std::unique_ptr<diagnostic_graph>>> ();
+
+      graphs->push_back (make_test_graph ("foo"));
+      graphs->push_back (make_test_graph ("bar"));
+
+      return graphs;
+    }
+
+  private:
+    std::unique_ptr<diagnostic_graph>
+    make_test_graph (const char *desc) const
+    {
+      auto g = std::make_unique<diagnostic_graph> ();
+      g->set_description (desc);
+      auto a = std::make_unique<diagnostic_node> (*g, "a");
+      auto b = std::make_unique<diagnostic_node> (*g, "b");
+#define KEY_PREFIX "/placeholder-prefix/"
+      b->set_attr (KEY_PREFIX, "color", "red");
+#undef KEY_PREFIX
+      auto c = std::make_unique<diagnostic_node> (*g, "c");
+      c->set_label ("I am a node label");
+
+      auto e = std::make_unique<diagnostic_edge> (*g, "my-edge", *a, *c);
+      e->set_label ("I am an edge label");
+      g->add_edge (std::move (e));
+
+      g->add_node (std::move (a));
+
+      b->add_child (std::move (c));
+      g->add_node (std::move (b));
+
+      return g;
+    }
+  };
+
+  gcc_rich_location rich_loc (loc);
+  diagnostics::metadata meta;
+  my_lazy_digraphs ldg;
+  meta.set_lazy_digraphs (&ldg);
+  error_meta (&rich_loc, meta,
+	      "this is a placeholder error, with graphs");
+}
+
+/* Exercise diagnostic graph emission.  */
+
+unsigned int
+pass_test_graph_emission::execute (function *fun)
+{
+  gimple_stmt_iterator gsi;
+  basic_block bb;
+
+  FOR_EACH_BB_FN (bb, fun)
+    for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+      {
+	gimple *stmt = gsi_stmt (gsi);
+
+	if (gcall *call = check_for_named_call (stmt, "here", 0))
+	  report_diag_with_graphs (gimple_location (call));
+      }
+
+  return 0;
+}
+
+int
+plugin_init (struct plugin_name_args *plugin_info,
+	     struct plugin_gcc_version *version)
+{
+  struct register_pass_info pass_info;
+  const char *plugin_name = plugin_info->base_name;
+  int argc = plugin_info->argc;
+  struct plugin_argument *argv = plugin_info->argv;
+
+  if (!plugin_default_version_check (version, &gcc_version))
+    return 1;
+
+  pass_info.pass = new pass_test_graph_emission (g);
+  pass_info.reference_pass_name = "ssa";
+  pass_info.ref_pass_instance_number = 1;
+  pass_info.pos_op = PASS_POS_INSERT_AFTER;
+  register_callback (plugin_name, PLUGIN_PASS_MANAGER_SETUP, NULL,
+		     &pass_info);
+
+  gcc_assert (::g->get_passes ());
+  global_dc->report_global_digraph (lazy_passes_graph (*::g->get_passes ()));
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_inlining.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_inlining.cc
index 7edce1f..d38538d 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_inlining.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_inlining.cc
@@ -169,7 +169,7 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  global_dc->m_source_printing.max_width = 80;
+  global_dc->get_source_printing_options ().max_width = 80;
 
   pass_info.pass = new pass_test_inlining (g);
   pass_info.reference_pass_name = "*warn_function_noreturn";
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_metadata.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_metadata.cc
index b86a8b3..f172258 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_metadata.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_metadata.cc
@@ -1,5 +1,6 @@
-/* This plugin exercises diagnostic_metadata.  */
+/* This plugin exercises diagnostics::metadata.  */
 
+#define INCLUDE_VECTOR
 #include "gcc-plugin.h"
 #include "config.h"
 #include "system.h"
@@ -28,7 +29,7 @@
 #include "diagnostic.h"
 #include "context.h"
 #include "gcc-rich-location.h"
-#include "diagnostic-metadata.h"
+#include "diagnostics/metadata.h"
 
 int plugin_is_GPL_compatible;
 
@@ -89,7 +90,7 @@ check_for_named_call (gimple *stmt,
   return call;
 }
 
-/* Exercise diagnostic_metadata.  */
+/* Exercise diagnostics::metadata.  */
 
 unsigned int
 pass_test_metadata::execute (function *fun)
@@ -106,13 +107,13 @@ pass_test_metadata::execute (function *fun)
 	if (gcall *call = check_for_named_call (stmt, "gets", 1))
 	  {
 	    gcc_rich_location richloc (gimple_location (call));
-	    diagnostic_metadata m;
+	    diagnostics::metadata m;
 
 	    /* CWE-242: Use of Inherently Dangerous Function.  */
 	    m.add_cwe (242);
 
-	    /* Example of a diagnostic_metadata::rule.  */
-	    diagnostic_metadata::precanned_rule
+	    /* Example of a diagnostics::metadata::rule.  */
+	    diagnostics::metadata::precanned_rule
 	      test_rule ("STR34-C", "https://example.com/");
 	    m.add_rule (test_rule);
 
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_paths.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_paths.cc
index a7963fa..875f4a8 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_paths.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_paths.cc
@@ -6,6 +6,7 @@
    specific tests within the compiler's IR.  We can't use any real
    diagnostics for this, so we have to fake it, hence this plugin.  */
 
+#define INCLUDE_VECTOR
 #include "gcc-plugin.h"
 #include "config.h"
 #include "system.h"
@@ -32,8 +33,7 @@
 #include "intl.h"
 #include "plugin-version.h"
 #include "diagnostic.h"
-#include "diagnostic-path.h"
-#include "diagnostic-metadata.h"
+#include "diagnostics/metadata.h"
 #include "context.h"
 #include "print-tree.h"
 #include "gcc-rich-location.h"
@@ -223,7 +223,7 @@ class test_diagnostic_path : public simple_diagnostic_path
   diagnostic_event_id_t
   add_event_2 (event_location_t evloc, int stack_depth,
 	       const char *desc,
-	       diagnostic_thread_id_t thread_id = 0)
+	       diagnostics::paths::thread_id_t thread_id = 0)
   {
     gcc_assert (evloc.m_fun);
     return add_thread_event (thread_id, evloc.m_loc, evloc.m_fun->decl,
@@ -232,7 +232,7 @@ class test_diagnostic_path : public simple_diagnostic_path
   diagnostic_event_id_t
   add_event_2_with_event_id (event_location_t evloc, int stack_depth,
 			     const char *fmt,
-			     diagnostic_thread_id_t thread_id,
+			     diagnostics::paths::thread_id_t thread_id,
 			     diagnostic_event_id_t event_id)
   {
     gcc_assert (evloc.m_fun);
@@ -242,7 +242,7 @@ class test_diagnostic_path : public simple_diagnostic_path
   }
   void add_entry (event_location_t evloc, int stack_depth,
 		  const char *funcname,
-		  diagnostic_thread_id_t thread_id = 0)
+		  diagnostics::paths::thread_id_t thread_id = 0)
   {
     gcc_assert (evloc.m_fun);
     add_thread_event (thread_id, evloc.m_loc, evloc.m_fun->decl, stack_depth,
@@ -363,7 +363,7 @@ example_2 ()
 
       richloc.set_path (&path);
 
-      diagnostic_metadata m;
+      diagnostics::metadata m;
       m.add_cwe (415); /* CWE-415: Double Free.  */
 
       warning_meta (&richloc, m, 0,
@@ -441,7 +441,7 @@ example_3 ()
 
       richloc.set_path (&path);
 
-      diagnostic_metadata m;
+      diagnostics::metadata m;
       /* CWE-479: Signal Handler Use of a Non-reentrant Function.  */
       m.add_cwe (479);
 
@@ -502,8 +502,8 @@ example_4 ()
 
       gcc_rich_location richloc (call_to_acquire_lock_a_in_bar.m_loc);
       test_diagnostic_path path (global_dc->get_reference_printer ());
-      diagnostic_thread_id_t thread_1 = path.add_thread ("Thread 1");
-      diagnostic_thread_id_t thread_2 = path.add_thread ("Thread 2");
+      diagnostics::paths::thread_id_t thread_1 = path.add_thread ("Thread 1");
+      diagnostics::paths::thread_id_t thread_2 = path.add_thread ("Thread 2");
       path.add_entry (entry_to_foo, 0, "foo", thread_1);
       diagnostic_event_id_t event_a_acquired
 	= path.add_event_2 (call_to_acquire_lock_a_in_foo, 1,
@@ -524,7 +524,7 @@ example_4 ()
 	 thread_2, event_a_acquired);
       richloc.set_path (&path);
 
-      diagnostic_metadata m;
+      diagnostics::metadata m;
       warning_meta (&richloc, m, 0,
 		    "deadlock due to inconsistent lock acquisition order");
     }
@@ -559,7 +559,7 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  global_dc->m_source_printing.max_width = 80;
+  global_dc->get_source_printing_options ().max_width = 80;
 
   pass_info.pass = make_pass_test_show_path (g);
   pass_info.reference_pass_name = "whole-program";
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc
index cd3834b..92839cd 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_show_locus.cc
@@ -62,7 +62,8 @@
 #include "print-tree.h"
 #include "gcc-rich-location.h"
 #include "text-range-label.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
+#include "diagnostics/file-cache.h"
 
 int plugin_is_GPL_compatible;
 
@@ -125,14 +126,14 @@ static bool force_show_locus_color = false;
 /* We want to verify the colorized output of diagnostic_show_locus,
    but turning on colorization for everything confuses "dg-warning" etc.
    Hence we special-case it within this plugin by using this modified
-   version of default_diagnostic_text_finalizer, which, if "color" is
+   version of diagnostics::default_text_finalizer, which, if "color" is
    passed in as a plugin argument turns on colorization, but just
    for diagnostic_show_locus.  */
 
 static void
-custom_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
-				  const diagnostic_info *diagnostic,
-				  diagnostic_t)
+custom_diagnostic_text_finalizer (diagnostics::text_sink &text_output,
+				  const diagnostics::diagnostic_info *diag,
+				  enum diagnostics::kind)
 {
   pretty_printer *const pp = text_output.get_printer ();
   bool old_show_color = pp_show_color (pp);
@@ -143,7 +144,7 @@ custom_diagnostic_text_finalizer (diagnostic_text_output_format &text_output,
   pp_newline (pp);
   diagnostic_show_locus (&text_output.get_context (),
 			 text_output.get_source_printing_options (),
-			 diagnostic->richloc, diagnostic->kind, pp);
+			 diag->m_richloc, diag->m_kind, pp);
   pp_show_color (pp) = old_show_color;
   pp_set_prefix (pp, saved_prefix);
   pp_flush (pp);
@@ -176,11 +177,11 @@ test_show_locus (function *fun)
   location_t fnstart = fun->function_start_locus;
   int fnstart_line = LOCATION_LINE (fnstart);
 
-  diagnostic_text_finalizer (global_dc) = custom_diagnostic_text_finalizer;
+  diagnostics::text_finalizer (global_dc) = custom_diagnostic_text_finalizer;
 
   /* Hardcode the "terminal width", to verify the behavior of
      very wide lines.  */
-  global_dc->m_source_printing.max_width = 71;
+  global_dc->get_source_printing_options ().max_width = 71;
 
   if (0 == strcmp (fnname, "test_simple"))
     {
@@ -251,7 +252,7 @@ test_show_locus (function *fun)
   if (0 == strcmp (fnname, "test_very_wide_line"))
     {
       const int line = fnstart_line + 2;
-      global_dc->m_source_printing.show_ruler_p = true;
+      global_dc->get_source_printing_options ().show_ruler_p = true;
       text_range_label label0 ("label 0");
       text_range_label label1 ("label 1");
       rich_location richloc (line_table,
@@ -263,7 +264,7 @@ test_show_locus (function *fun)
 			 &label1);
       richloc.add_fixit_replace ("bar * foo");
       warning_at (&richloc, 0, "test");
-      global_dc->m_source_printing.show_ruler_p = false;
+      global_dc->get_source_printing_options ().show_ruler_p = false;
     }
 
   /* Likewise, but with a secondary location that's immediately before
@@ -271,7 +272,7 @@ test_show_locus (function *fun)
   if (0 == strcmp (fnname, "test_very_wide_line_2"))
     {
       const int line = fnstart_line + 2;
-      global_dc->m_source_printing.show_ruler_p = true;
+      global_dc->get_source_printing_options ().show_ruler_p = true;
       text_range_label label0 ("label 0");
       text_range_label label1 ("label 1");
       rich_location richloc (line_table,
@@ -283,7 +284,7 @@ test_show_locus (function *fun)
       richloc.add_range (get_loc (line, 34), SHOW_RANGE_WITHOUT_CARET,
 			 &label1);
       warning_at (&richloc, 0, "test");
-      global_dc->m_source_printing.show_ruler_p = false;
+      global_dc->get_source_printing_options ().show_ruler_p = false;
     }
 
   /* Example of multiple carets.  */
@@ -294,11 +295,11 @@ test_show_locus (function *fun)
       location_t caret_b = get_loc (line, 11);
       rich_location richloc (line_table, caret_a);
       add_range (&richloc, caret_b, caret_b, SHOW_RANGE_WITH_CARET);
-      global_dc->m_source_printing.caret_chars[0] = 'A';
-      global_dc->m_source_printing.caret_chars[1] = 'B';
+      global_dc->get_source_printing_options ().caret_chars[0] = 'A';
+      global_dc->get_source_printing_options ().caret_chars[1] = 'B';
       warning_at (&richloc, 0, "test");
-      global_dc->m_source_printing.caret_chars[0] = '^';
-      global_dc->m_source_printing.caret_chars[1] = '^';
+      global_dc->get_source_printing_options ().caret_chars[0] = '^';
+      global_dc->get_source_printing_options ().caret_chars[1] = '^';
     }
 
   /* Tests of rendering fixit hints.  */
@@ -412,11 +413,11 @@ test_show_locus (function *fun)
       location_t caret_b = get_loc (line - 1, 19);
       rich_location richloc (line_table, caret_a);
       richloc.add_range (caret_b, SHOW_RANGE_WITH_CARET);
-      global_dc->m_source_printing.caret_chars[0] = '1';
-      global_dc->m_source_printing.caret_chars[1] = '2';
+      global_dc->get_source_printing_options ().caret_chars[0] = '1';
+      global_dc->get_source_printing_options ().caret_chars[1] = '2';
       warning_at (&richloc, 0, "test");
-      global_dc->m_source_printing.caret_chars[0] = '^';
-      global_dc->m_source_printing.caret_chars[1] = '^';
+      global_dc->get_source_printing_options ().caret_chars[0] = '^';
+      global_dc->get_source_printing_options ().caret_chars[1] = '^';
     }
 
   /* Example of using the "%q+D" format code, which as well as printing
@@ -443,8 +444,8 @@ test_show_locus (function *fun)
       rich_location richloc (line_table, loc);
       for (int line = start_line; line <= finish_line; line++)
 	{
-	  file_cache &fc = global_dc->get_file_cache ();
-	  char_span content = fc.get_source_line (file, line);
+	  diagnostics::file_cache &fc = global_dc->get_file_cache ();
+	  diagnostics::char_span content = fc.get_source_line (file, line);
 	  gcc_assert (content);
 	  /* Split line up into words.  */
 	  for (int idx = 0; idx < content.length (); idx++)
@@ -464,7 +465,8 @@ test_show_locus (function *fun)
 		      richloc.add_range (word, SHOW_RANGE_WITH_CARET, &label);
 
 		      /* Add a fixit, converting to upper case.  */
-		      char_span word_span = content.subspan (start_idx, idx - start_idx);
+		      diagnostics::char_span word_span
+			= content.subspan (start_idx, idx - start_idx);
 		      char *copy = word_span.xstrdup ();
 		      for (char *ch = copy; *ch; ch++)
 			*ch = TOUPPER (*ch);
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_string_literals.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_string_literals.cc
index 1b5fad2..b64d60f 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_string_literals.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_string_literals.cc
@@ -208,7 +208,7 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  global_dc->m_source_printing.max_width = 80;
+  global_dc->get_source_printing_options ().max_width = 80;
 
   pass_info.pass = new pass_test_string_literals (g);
   pass_info.reference_pass_name = "ssa";
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_text_art.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_text_art.cc
index e28d697..ce2f1d3 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_text_art.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_text_art.cc
@@ -9,7 +9,7 @@
 #include "coretypes.h"
 #include "plugin-version.h"
 #include "diagnostic.h"
-#include "diagnostic-diagram.h"
+#include "diagnostics/diagram.h"
 #include "text-art/canvas.h"
 #include "text-art/table.h"
 
@@ -22,7 +22,7 @@ using namespace text_art;
 static void
 emit_canvas (const canvas &c, const char *alt_text)
 {
-  diagnostic_diagram diagram (c, alt_text);
+  diagnostics::diagram diagram (c, alt_text);
   global_dc->emit_diagram (diagram);
 }
 
diff --git a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_tree_expression_range.cc b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_tree_expression_range.cc
index fbdb2f8..bbd0faa 100644
--- a/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_tree_expression_range.cc
+++ b/gcc/testsuite/gcc.dg/plugin/diagnostic_plugin_test_tree_expression_range.cc
@@ -89,7 +89,7 @@ plugin_init (struct plugin_name_args *plugin_info,
   if (!plugin_default_version_check (version, &gcc_version))
     return 1;
 
-  global_dc->m_source_printing.max_width = 130;
+  global_dc->get_source_printing_options ().max_width = 130;
 
   register_callback (plugin_name,
 		     PLUGIN_PRE_GENERICIZE,
diff --git a/gcc/testsuite/gcc.dg/plugin/expensive_selftests_plugin.cc b/gcc/testsuite/gcc.dg/plugin/expensive_selftests_plugin.cc
index 7b9b8d4..67722d4 100644
--- a/gcc/testsuite/gcc.dg/plugin/expensive_selftests_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/expensive_selftests_plugin.cc
@@ -6,9 +6,9 @@
 #include "system.h"
 #include "coretypes.h"
 #include "diagnostic.h"
-#include "edit-context.h"
+#include "diagnostics/changes.h"
 #include "selftest.h"
-#include "selftest-diagnostic.h"
+#include "diagnostics/selftest-context.h"
 
 int plugin_is_GPL_compatible;
 
@@ -47,14 +47,15 @@ static void
 test_richloc (rich_location *richloc)
 {
   /* Run the diagnostic and fix-it printing code.  */
-  test_diagnostic_context dc;
-  diagnostic_show_locus (&dc, dc.m_source_printing,
-			 richloc, DK_ERROR, dc.get_reference_printer ());
+  diagnostics::selftest::test_context dc;
+  diagnostic_show_locus (&dc, dc.get_source_printing_options (),
+			 richloc, diagnostics::kind::error,
+			 dc.get_reference_printer ());
 
   /* Generate a diff.  */
-  edit_context ec (global_dc->get_file_cache ());
-  ec.add_fixits (richloc);
-  char *diff = ec.generate_diff (true);
+  diagnostics::changes::change_set edit (global_dc->get_file_cache ());
+  edit.add_fixits (richloc);
+  char *diff = edit.generate_diff (true);
   free (diff);
 }
 
diff --git a/gcc/testsuite/gcc.dg/plugin/location_overflow_plugin.cc b/gcc/testsuite/gcc.dg/plugin/location_overflow_plugin.cc
index f770d35..00ad870 100644
--- a/gcc/testsuite/gcc.dg/plugin/location_overflow_plugin.cc
+++ b/gcc/testsuite/gcc.dg/plugin/location_overflow_plugin.cc
@@ -7,7 +7,7 @@
 #include "coretypes.h"
 #include "spellcheck.h"
 #include "diagnostic.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 
 int plugin_is_GPL_compatible;
 
@@ -39,12 +39,12 @@ on_pragma_registration (void */*gcc_data*/, void */*user_data*/)
 /* We add some extra testing during diagnostics by chaining up
    to the text finalizer.  */
 
-static diagnostic_text_finalizer_fn original_text_finalizer = NULL;
+static diagnostics::text_finalizer_fn original_text_finalizer = NULL;
 
 static void
-verify_unpacked_ranges  (diagnostic_text_output_format &text_output,
-			 const diagnostic_info *diagnostic,
-			 diagnostic_t orig_diag_kind)
+verify_unpacked_ranges  (diagnostics::text_sink &text_output,
+			 const diagnostics::diagnostic_info *diagnostic,
+			 enum diagnostics::kind orig_diag_kind)
 {
   /* Verify that the locations are ad-hoc, not packed. */
   location_t loc = diagnostic_location (diagnostic);
@@ -56,9 +56,9 @@ verify_unpacked_ranges  (diagnostic_text_output_format &text_output,
 }
 
 static void
-verify_no_columns  (diagnostic_text_output_format &text_output,
-		    const diagnostic_info *diagnostic,
-		    diagnostic_t orig_diag_kind)
+verify_no_columns  (diagnostics::text_sink &text_output,
+		    const diagnostics::diagnostic_info *diagnostic,
+		    enum diagnostics::kind orig_diag_kind)
 {
   /* Verify that the locations have no columns. */
   location_t loc = diagnostic_location (diagnostic);
@@ -104,15 +104,15 @@ plugin_init (struct plugin_name_args *plugin_info,
 		     NULL); /* void *user_data */
 
   /* Hack in additional testing, based on the exact value supplied.  */
-  original_text_finalizer = diagnostic_text_finalizer (global_dc);
+  original_text_finalizer = diagnostics::text_finalizer (global_dc);
   switch (base_location)
     {
     case LINE_MAP_MAX_LOCATION_WITH_PACKED_RANGES + 1:
-      diagnostic_text_finalizer (global_dc) = verify_unpacked_ranges;
+      diagnostics::text_finalizer (global_dc) = verify_unpacked_ranges;
       break;
 
     case LINE_MAP_MAX_LOCATION_WITH_COLS + 1:
-      diagnostic_text_finalizer (global_dc) = verify_no_columns;
+      diagnostics::text_finalizer (global_dc) = verify_no_columns;
       break;
 
     default:
diff --git a/gcc/testsuite/gcc.dg/plugin/must-tail-call-2.c b/gcc/testsuite/gcc.dg/plugin/must-tail-call-2.c
index d51d15c..6f65f4a 100644
--- a/gcc/testsuite/gcc.dg/plugin/must-tail-call-2.c
+++ b/gcc/testsuite/gcc.dg/plugin/must-tail-call-2.c
@@ -55,5 +55,5 @@ volatile fn_ptr_t fn_ptr;
 void
 test_5 (void)
 {
-  fn_ptr (); /* { dg-error "cannot tail-call: " } */
+  fn_ptr ();
 }
diff --git a/gcc/testsuite/gcc.dg/plugin/plugin.exp b/gcc/testsuite/gcc.dg/plugin/plugin.exp
index d1d7f5d..ce25c0a 100644
--- a/gcc/testsuite/gcc.dg/plugin/plugin.exp
+++ b/gcc/testsuite/gcc.dg/plugin/plugin.exp
@@ -107,6 +107,10 @@ set plugin_test_list [list \
 	  diagnostic-test-metadata.c \
 	  diagnostic-test-metadata-html.c \
 	  diagnostic-test-metadata-sarif.c } \
+    { diagnostic_plugin_test_graphs.cc
+	  diagnostic-test-graphs.c \
+	  diagnostic-test-graphs-html.c \
+	  diagnostic-test-graphs-sarif.c } \
     { diagnostic_plugin_test_nesting.cc \
 	  diagnostic-test-nesting-text-plain.c \
 	  diagnostic-test-nesting-text-indented.c \
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-1.c b/gcc/testsuite/gcc.dg/pointer-counted-by-1.c
deleted file mode 100644
index 395af34..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-1.c
+++ /dev/null
@@ -1,34 +0,0 @@
-/* More testing the correct usage of attribute counted_by for pointer field.  */
-/* { dg-do compile } */
-/* { dg-options "-O0" } */
-
-typedef struct item1 Item1;
-typedef union item2 Item2;
-
-struct pointer_array {
-  int count1;
-  Item1 *array_1 __attribute__ ((counted_by (count1))); 
-  Item2 *array_2 __attribute__ ((counted_by (count2))); 
-  int count2;
-} *pointer_data; 
-
-struct item1 {
-  int a;
-  float b[];
-};
-
-union item2 {
-  int c;
-  float d[];
-};
-
-void foo ()
-{
-  pointer_data 
-    = (struct pointer_array *) __builtin_malloc (sizeof (struct pointer_array));
-  pointer_data->array_1 /* { dg-error "attribute is not allowed for a pointer to structure or union with flexible array member" } */ 
-    = (Item1 *) __builtin_malloc (sizeof (Item1) + 3 * sizeof (float));
-  pointer_data->array_2 /* { dg-error "attribute is not allowed for a pointer to structure or union with flexible array member" } */
-    = (Item2 *) __builtin_malloc (sizeof (Item2) + 3 * sizeof (float));
-  return;
-}
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-2.c b/gcc/testsuite/gcc.dg/pointer-counted-by-2.c
deleted file mode 100644
index 1f4a278..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-2.c
+++ /dev/null
@@ -1,10 +0,0 @@
-/* Testing the correct usage of attribute counted_by for pointer: _BitInt  */
-/* { dg-do compile { target bitint } } */
-/* { dg-options "-O2 -std=c23" } */
-
-struct pointer_array {
-  _BitInt(24) count; 
-  int *array __attribute__ ((counted_by (count)));
-  int *array1 __attribute__ ((counted_by (count1)));
-  _BitInt(24) count1; 
-};
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-3.c b/gcc/testsuite/gcc.dg/pointer-counted-by-3.c
deleted file mode 100644
index 7005609..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-3.c
+++ /dev/null
@@ -1,127 +0,0 @@
- /* Testing the correct usage of attribute counted_by for pointer in c23,
-    multiple definitions of the same tag in same or different scopes.
-    { dg-do compile }
-    { dg-options "-std=c23" }
- */
-
-/* Allowed redefinitions of the same struct in the same scope, with the
-   same counted_by attribute.  */
-struct f {
-  int b;
-  int c;
-  int *a __attribute__ ((counted_by (b))); };
-struct f {
-  int b;
-  int c;
-  int *a __attribute__ ((counted_by (b))); };
-struct f {
-  int b;
-  int c;
-  int *a; }; /* { dg-error "redefinition of struct or union" } */
-
-/* Error when the counted_by attribute is defined differently.  */
-struct f {
-  int b;
-  int c;
-  int *a __attribute__ ((counted_by (c))); }; /* { dg-error "redefinition of struct or union" } */
-
-struct h {
-  int b;
-  int c;
-  int *a __attribute__ ((counted_by (b))); } p;  
-
-void test (void)
-{
-  struct h {
-  int b;
-  int c;
-  int *a __attribute__ ((counted_by (b))); } x;
-
-  p = x;
-}
-
-void test1 (void)
-{
-  struct h {
-  int b;
-  int c;
-  int *a __attribute__ ((counted_by (c))); } y;
-
-  p = y;   /* { dg-error "incompatible types when assigning to type" } */
-}
-
-struct nested_f {
-  struct {
-    union {
-      int b;
-      float f;
-    };
-    int n;
-  };
-  char *c __attribute__ ((counted_by (b)));
-}; 
-
-struct nested_f {
-  struct {
-    union {
-      int b;
-      float f;
-    };
-    int n;
-  };
-  char *c __attribute__ ((counted_by (b)));
-}; 
-
-struct nested_f {
-  struct {
-    union {
-      int b;
-      float f;
-    };
-    int n;
-  };
-  char *c __attribute__ ((counted_by (n)));
-};  /* { dg-error "redefinition of struct or union" } */
-
-struct nested_h {
-  struct {
-    union {
-      int b;
-      float f;
-    };
-    int n;
-  };
-  char *c __attribute__ ((counted_by (b)));
-} nested_p; 
-
-void test_2 (void)
-{
-struct nested_h {
-  struct {
-    union {
-      int b;
-      float f;
-    };
-    int n;
-  };
-  char *c __attribute__ ((counted_by (b)));
-} nested_x; 
-
- nested_p = nested_x;
-}
-
-void test_3 (void)
-{
-struct nested_h {
-  struct {
-    union {
-      int b;
-      float f;
-    };
-    int n;
-  };
-  char *c __attribute__ ((counted_by (n)));
-} nested_y; 
-
- nested_p = nested_y; /* { dg-error "incompatible types when assigning to type" } */
-}
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-4-char.c b/gcc/testsuite/gcc.dg/pointer-counted-by-4-char.c
deleted file mode 100644
index c404e5b..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-4-char.c
+++ /dev/null
@@ -1,6 +0,0 @@
-/* Test the attribute counted_by for pointer field and its usage in
- * __builtin_dynamic_object_size.  */ 
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-#define PTR_TYPE char
-#include "pointer-counted-by-4.c"
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-4-float.c b/gcc/testsuite/gcc.dg/pointer-counted-by-4-float.c
deleted file mode 100644
index 383d8fb..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-4-float.c
+++ /dev/null
@@ -1,6 +0,0 @@
-/* Test the attribute counted_by for pointer field and its usage in
- * __builtin_dynamic_object_size.  */ 
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-#define PTR_TYPE float 
-#include "pointer-counted-by-4.c"
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-4-struct.c b/gcc/testsuite/gcc.dg/pointer-counted-by-4-struct.c
deleted file mode 100644
index 50246d2..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-4-struct.c
+++ /dev/null
@@ -1,10 +0,0 @@
-/* Test the attribute counted_by for pointer field and its usage in
- * __builtin_dynamic_object_size.  */ 
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-struct A {
-  int a;
-  char *b;
-};
-#define PTR_TYPE struct A 
-#include "pointer-counted-by-4.c"
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-4-union.c b/gcc/testsuite/gcc.dg/pointer-counted-by-4-union.c
deleted file mode 100644
index e786d99..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-4-union.c
+++ /dev/null
@@ -1,10 +0,0 @@
-/* Test the attribute counted_by for pointer field and its usage in
- * __builtin_dynamic_object_size.  */ 
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-union A {
-  int a;
-  float b;
-};
-#define PTR_TYPE union A 
-#include "pointer-counted-by-4.c"
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-4.c b/gcc/testsuite/gcc.dg/pointer-counted-by-4.c
deleted file mode 100644
index c4b3631..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-4.c
+++ /dev/null
@@ -1,77 +0,0 @@
-/* Test the attribute counted_by for pointer field and its usage in
- * __builtin_dynamic_object_size.  */ 
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-
-#include "builtin-object-size-common.h"
-#ifndef PTR_TYPE
-#define PTR_TYPE int
-#endif
-struct pointer_array {
-  int b;
-  PTR_TYPE *c;
-} *p_array;
-
-struct annotated {
-  PTR_TYPE *c __attribute__ ((counted_by (b)));
-  int b;
-} *p_array_annotated;
-
-struct nested_annotated {
-  PTR_TYPE *c __attribute__ ((counted_by (b)));
-  struct {
-    union {
-      int b;
-      float f;	
-    };
-    int n;
-  };
-} *p_array_nested_annotated;
-
-void __attribute__((__noinline__)) setup (int normal_count, int attr_count)
-{
-  p_array
-    = (struct pointer_array *) malloc (sizeof (struct pointer_array));
-  p_array->c = (PTR_TYPE *) malloc (sizeof (PTR_TYPE) * normal_count);
-  p_array->b = normal_count;
-
-  p_array_annotated
-    = (struct annotated *) malloc (sizeof (struct annotated));
-  p_array_annotated->c = (PTR_TYPE *) malloc (sizeof (PTR_TYPE) * attr_count);
-  p_array_annotated->b = attr_count;
-
-  p_array_nested_annotated
-    = (struct nested_annotated *) malloc (sizeof (struct nested_annotated));
-  p_array_nested_annotated->c = (PTR_TYPE *) malloc (sizeof (PTR_TYPE) * attr_count);
-  p_array_nested_annotated->b = attr_count;
-
-  return;
-}
-
-void __attribute__((__noinline__)) test ()
-{
-  EXPECT(__builtin_dynamic_object_size(p_array->c, 1), -1);
-  EXPECT(__builtin_dynamic_object_size(p_array_annotated->c, 1),
-	 p_array_annotated->b * sizeof (PTR_TYPE));
-  EXPECT(__builtin_dynamic_object_size(p_array_nested_annotated->c, 1),
-	 p_array_nested_annotated->b * sizeof (PTR_TYPE));
-}
-
-void cleanup ()
-{
-  free (p_array->c); 
-  free (p_array);
-  free (p_array_annotated->c);
-  free (p_array_annotated);
-  free (p_array_nested_annotated->c);
-  free (p_array_nested_annotated);
-}
-
-int main(int argc, char *argv[])
-{
-  setup (10,10);   
-  test ();
-  DONE ();
-  cleanup ();
-  return 0;
-}
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-5.c b/gcc/testsuite/gcc.dg/pointer-counted-by-5.c
deleted file mode 100644
index b43ffdf..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-5.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in
- * __builtin_dynamic_object_size: when the counted_by field is negative.  */
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-
-#include "builtin-object-size-common.h"
-
-struct annotated {
-  int b;
-  int *c __attribute__ ((counted_by (b)));
-} *array_annotated;
-
-struct nested_annotated {
-  int *c __attribute__ ((counted_by (b)));
-  struct {
-    union {
-      int b;
-      float f;	
-    };
-    int n;
-  };
-} *array_nested_annotated;
-
-void __attribute__((__noinline__)) setup (int attr_count)
-{
-  array_annotated
-    = (struct annotated *)malloc (sizeof (struct annotated));
-  array_annotated->b = attr_count;
-
-  array_nested_annotated
-    = (struct nested_annotated *)malloc (sizeof (struct nested_annotated));
-  array_nested_annotated->b = attr_count - 1;
-
-  return;
-}
-
-void __attribute__((__noinline__)) test ()
-{
-  EXPECT(__builtin_dynamic_object_size(array_annotated->c, 1), 0);
-  EXPECT(__builtin_dynamic_object_size(array_nested_annotated->c, 1), 0);
-}
-
-void cleanup ()
-{
-  free (array_annotated);
-  free (array_nested_annotated);
-}
-
-int main(int argc, char *argv[])
-{
-  setup (-10);   
-  test ();
-  DONE ();
-  cleanup ();
-  return 0;
-}
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-6.c b/gcc/testsuite/gcc.dg/pointer-counted-by-6.c
deleted file mode 100644
index 355558c..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-6.c
+++ /dev/null
@@ -1,56 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in
- * __builtin_dynamic_object_size: when the type of the pointer  
- * is casting to another type.  */
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-
-#include "builtin-object-size-common.h"
-
-typedef unsigned short u16;
-
-struct info {
-  u16 data_len;
-  char *data __attribute__((counted_by(data_len)));
-};
-
-struct foo {
-  int a;
-  int b;
-};
-
-static __attribute__((__noinline__))
-struct info *setup ()
-{
-  struct info *p;
-  size_t bytes = 3 * sizeof(struct foo);
-
-  p = (struct info *) malloc (sizeof (struct info));
-  p->data = (char *) malloc (bytes);
-  p->data_len = bytes;
-
-  return p;
-}
-
-static void
-__attribute__((__noinline__)) report (struct info *p)
-{
-  struct foo *bar = (struct foo *)p->data;
-  EXPECT(__builtin_dynamic_object_size((char *)(bar + 1), 1),
-	 sizeof (struct foo) * 2);
-  EXPECT(__builtin_dynamic_object_size((char *)(bar + 2), 1),
-	 sizeof (struct foo));
-}
-
-void cleanup (struct info *p)
-{
-  free (p->data);
-  free (p);
-}
-
-int main(int argc, char *argv[])
-{
-  struct info *p = setup();
-  report(p);
-  cleanup (p);
-  return 0;
-}
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by-7.c b/gcc/testsuite/gcc.dg/pointer-counted-by-7.c
deleted file mode 100644
index af1ab27..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by-7.c
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Additional test of the attribute counted_by for pointer field and its usage
-   in __builtin_dynamic_object_size.  */ 
-/* { dg-do run } */
-/* { dg-options "-O2" } */
-
-#include "builtin-object-size-common.h"
-
-struct annotated {
-  int b;
-  int *c __attribute__ ((counted_by (b)));
-};
-
-struct annotated __attribute__((__noinline__)) setup (int attr_count)
-{
-  struct annotated p_array_annotated;
-  p_array_annotated.c = (int *) malloc (sizeof (int) * attr_count);
-  p_array_annotated.b = attr_count;
-
-  return p_array_annotated;
-}
-
-int main(int argc, char *argv[])
-{
-  struct annotated x = setup (10);   
-  int *p = x.c;
-  x = setup (20);
-  EXPECT(__builtin_dynamic_object_size (p, 1), 10 * sizeof (int));
-  EXPECT(__builtin_dynamic_object_size (x.c, 1), 20 * sizeof (int));
-  free (p);
-  free (x.c);
-  DONE ();
-}
diff --git a/gcc/testsuite/gcc.dg/pointer-counted-by.c b/gcc/testsuite/gcc.dg/pointer-counted-by.c
deleted file mode 100644
index 0f18828..0000000
--- a/gcc/testsuite/gcc.dg/pointer-counted-by.c
+++ /dev/null
@@ -1,111 +0,0 @@
-/* Testing the correct usage of attribute counted_by for pointer field.
-   and also mixed pointer field and FMA field in the same structure.  */
-/* { dg-do compile } */
-/* { dg-options "-O0" } */
-
-int size;
-int *x __attribute__ ((counted_by (size))); /* { dg-error "attribute is not allowed for a non-field declaration" } */
-
-struct pointer_array_0 {
-  int count;
-  int array __attribute__ ((counted_by (count))); /* { dg-error "attribute is not allowed for a non-array or non-pointer field" } */
-  int other;
-};
-
-int count;
-struct pointer_array_1 {
-  int other;
-  int *array_1 __attribute__ ((counted_by (count))); /* { dg-error "attribute is not a field declaration in the same structure as" } */
-  int array_fam[] __attribute__ ((counted_by (count))); /* { dg-error "attribute is not a field declaration in the same structure as" } */
-};
-
-struct pointer_array_2 {
-  float count1;
-  float count2;
-  int *array_2 __attribute__ ((counted_by (count1))); /* { dg-error "attribute is not a field declaration with an integer type" } */
-  int array_fam[] __attribute__ ((counted_by (count2))); /* { dg-error "attribute is not a field declaration with an integer type" } */
-}; 
-
-struct pointer_array_3 {
-  int count;
-  int *array_3 __attribute__ ((counted_by (count))) __attribute__ ((counted_by (count)));
-}; 
-
-struct pointer_array_4 {
-  int count1;
-  int count2;
-  int *array_4 __attribute__ ((counted_by (count1))) __attribute__ ((counted_by (count2))); /* { dg-error "conflicts with previous declaration" } */
-  float array_fam[] __attribute__ ((counted_by (count2))) __attribute__ ((counted_by (count1))); /* { dg-error "conflicts with previous declaration" } */
-}; 
-
-struct pointer_array_5 {
-  _Bool count;
-  int *array_5 __attribute__ ((counted_by (count)));
-}; 
-
-enum week {Mon, Tue, Wed};
-struct pointer_array_6 {
-  enum week days;
-  int *array_6 __attribute__ ((counted_by (days)));
-}; 
-
-struct pointer_array_7 {
-  int count;
-  void *array_7 __attribute__ ((counted_by (count))); /* { dg-error "attribute is not allowed for a pointer to void" } */
-}; 
-
-struct pointer_array_8 {
-  int count;
-  int (*fpr)(int,int) __attribute__ ((counted_by (count))); /* { dg-error "attribute is not allowed for a pointer to function" } */
-}; 
-
-struct item1 {
-  int a;
-  float b;
-};
-
-union item2 {
-  char *a;
-  int *b; 
-};
-
-typedef struct item3 Item3;
-typedef union item4 Item4;
-
-struct item5 {
-  int a;
-  float b[];
-};
-
-/* Incomplete structure and union are allowed.  */
-struct pointer_array_9 {
-  int count1;
-  int count2;
-  int count3;
-  struct item1 *array_1 __attribute__ ((counted_by (count1)));
-  union item2 *array_2 __attribute__ ((counted_by (count2)));
-  Item3 *array_3 __attribute__ ((counted_by (count3))); 
-  Item4 *array_4 __attribute__ ((counted_by (count4))); 
-  int count4;
-  int count5;
-  /* structure with flexible array member is not allowed.  */
-  struct item5 *array_5 __attribute__ ((counted_by (count5))); /* { dg-error "attribute is not allowed for a pointer to structure or union with flexible array member" } */
-}; 
-
-struct mixed_array {
-  int count1;
-  float *array_1 __attribute__ ((counted_by (count1)));
-  float *array_2 __attribute__ ((counted_by (count1)));
-  int count2;
-  long *array_3 __attribute__ ((counted_by (count2)));
-  long array_4[] __attribute__ ((counted_by (count2)));
-};
-
-struct mixed_array_2 {
-  float *array_1 __attribute__ ((counted_by (count1)));
-  int count1;
-  float *array_2 __attribute__ ((counted_by (count1)));
-  long *array_3 __attribute__ ((counted_by (count2)));
-  int count2;
-  long array_4[] __attribute__ ((counted_by (count2)));
-};
diff --git a/gcc/testsuite/gcc.dg/pr109267-1.c b/gcc/testsuite/gcc.dg/pr109267-1.c
new file mode 100644
index 0000000..e762e59
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr109267-1.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+
+/* PR middle-end/109267 */
+
+int f(void)
+{
+  __builtin_unreachable();
+}
+
+/* This unreachable should be changed to be a trap. */
+
+/* { dg-final { scan-tree-dump-times "__builtin_unreachable trap \\\(" 1 "optimized" { target trap }  } } */
+/* { dg-final { scan-tree-dump-times "goto <" 1 "optimized" { target { ! trap } } } } */
+/* { dg-final { scan-tree-dump-not "__builtin_unreachable \\\(" "optimized"} } */
diff --git a/gcc/testsuite/gcc.dg/pr109267-2.c b/gcc/testsuite/gcc.dg/pr109267-2.c
new file mode 100644
index 0000000..6cd1419
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr109267-2.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+
+/* PR middle-end/109267 */
+void g(void);
+int f(int *t)
+{
+  g();
+  __builtin_unreachable();
+}
+
+/* The unreachable should stay a unreachable. */
+/* { dg-final { scan-tree-dump-not "__builtin_unreachable trap \\\(" "optimized"} } */
+/* { dg-final { scan-tree-dump-times "__builtin_unreachable \\\(" 1 "optimized"} } */
diff --git a/gcc/testsuite/gcc.dg/pr116906-1.c b/gcc/testsuite/gcc.dg/pr116906-1.c
index 7187507..ee60ad6 100644
--- a/gcc/testsuite/gcc.dg/pr116906-1.c
+++ b/gcc/testsuite/gcc.dg/pr116906-1.c
@@ -1,3 +1,4 @@
+/* { dg-do run } */
 /* { dg-require-effective-target alarm } */
 /* { dg-require-effective-target signal } */
 /* { dg-options "-O2" } */
diff --git a/gcc/testsuite/gcc.dg/pr116906-2.c b/gcc/testsuite/gcc.dg/pr116906-2.c
index 41a352b..4172ec3 100644
--- a/gcc/testsuite/gcc.dg/pr116906-2.c
+++ b/gcc/testsuite/gcc.dg/pr116906-2.c
@@ -1,3 +1,4 @@
+/* { dg-do run } */
 /* { dg-require-effective-target alarm } */
 /* { dg-require-effective-target signal } */
 /* { dg-options "-O2 -fno-tree-ch" } */
diff --git a/gcc/testsuite/gcc.dg/pr118948-1.c b/gcc/testsuite/gcc.dg/pr118948-1.c
new file mode 100644
index 0000000..2a46cf1
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr118948-1.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "" } */
+
+/* PR c/118948 */
+
+/* Used to ICE in tree_expr_nonnegative_p after an error. */
+
+void f(void) {
+    int i; /* { dg-note "previous" } */
+    for (i = 0; i < 2; i++) ;
+    float i; /* { dg-error "conflicting types for" } */
+}
diff --git a/gcc/testsuite/gcc.dg/pr120660.c b/gcc/testsuite/gcc.dg/pr120660.c
new file mode 100644
index 0000000..6e8c5e8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr120660.c
@@ -0,0 +1,19 @@
+/* { dg-do run } */
+/* { dg-options "-O -favoid-store-forwarding" } */
+
+int c;
+
+short
+foo (short s)
+{
+  __builtin_memset (&s, c, 1);
+  return s;
+}
+
+int
+main ()
+{
+  short x = foo (0x1111);
+  if (x != 0x1100 && x != 0x0011)
+    __builtin_abort();
+}
diff --git a/gcc/testsuite/gcc.dg/pr121035.c b/gcc/testsuite/gcc.dg/pr121035.c
new file mode 100644
index 0000000..fc0edce
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr121035.c
@@ -0,0 +1,94 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fgimple" } */
+
+int printf(const char *, ...);
+int a, b, d;
+unsigned c;
+int __GIMPLE (ssa,startwith("pre"))
+main ()
+{
+  unsigned int g;
+  int f;
+  unsigned int _1;
+  unsigned int _2;
+  int _3;
+  int _4;
+  int _5;
+  unsigned int _6;
+  unsigned int _7;
+  int _10;
+  unsigned int _11;
+  _Bool _19;
+  _Bool _20;
+  _Bool _22;
+  int _25;
+
+  __BB(2):
+  _25 = a;
+  if (_25 != 0)
+    goto __BB11;
+  else
+    goto __BB10;
+
+  __BB(11):
+  goto __BB3;
+
+  __BB(3):
+  f_26 = __PHI (__BB12: f_18, __BB11: 0);
+  g_15 = c;
+  if (f_26 != 0)
+    goto __BB4;
+  else
+    goto __BB5;
+
+  __BB(4):
+  __builtin_putchar (48);
+  goto __BB5;
+
+  __BB(5):
+  _1 = c;
+  _2 = _1 << 1;
+  _3 = a;
+  _4 = d;
+  _5 = _3 * _4;
+  if (_5 != 0)
+    goto __BB7;
+  else
+    goto __BB6;
+
+  __BB(6):
+  goto __BB7;
+
+  __BB(7):
+  _11 = __PHI (__BB5: 0u, __BB6: 4294967295u);
+  _6 = g_15 * 4294967294u;
+  _7 = _6 | _11;
+  _20 = _3 != 0;
+  _19 = _7 != 0u;
+  _22 = _19 & _20;
+  if (_22 != _Literal (_Bool) 0)
+    goto __BB9;
+  else
+    goto __BB8;
+
+  __BB(8):
+  goto __BB9;
+
+  __BB(9):
+  _10 = __PHI (__BB7: 1, __BB8: 0);
+  b = _10;
+  f_18 = (int) _1;
+  if (_3 != 0)
+    goto __BB12;
+  else
+    goto __BB10;
+
+  __BB(12):
+  goto __BB3;
+
+  __BB(10):
+  return 0;
+
+}
+
+
diff --git a/gcc/testsuite/gcc.dg/pr121202.c b/gcc/testsuite/gcc.dg/pr121202.c
new file mode 100644
index 0000000..30ecf4a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr121202.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fno-tree-copy-prop" } */
+
+int a, b, c;
+int e(int f, int g) { return f >> g; }
+int h(int f) { return a > 1 ? 0 : f << a; }
+int main() {
+  while (c--)
+    b = e(h(1), a);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/pr121216.c b/gcc/testsuite/gcc.dg/pr121216.c
new file mode 100644
index 0000000..a695b40
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr121216.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "" } */
+
+int foo (void)
+{
+    const char *key = "obscurelevelofabstraction";
+    const char reverse_key[__builtin_strlen(key)] = {'\0'}; /* { dg-error "variable-sized object may not be initialized except with an empty initializer" } */
+    return __builtin_strlen(reverse_key);
+}
diff --git a/gcc/testsuite/gcc.dg/pr121322.c b/gcc/testsuite/gcc.dg/pr121322.c
new file mode 100644
index 0000000..2fad5b5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr121322.c
@@ -0,0 +1,14 @@
+/* PR middle-end/121322 */
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+unsigned long long
+foo (unsigned long long *p)
+{
+  unsigned long long a = *p;
+  unsigned long long b = __builtin_bswap64 (a);
+  return ((b << 32)
+	  | ((b >> 8) & 0xff000000ULL)
+	  | ((b >> 24) & 0xff0000ULL)
+	  | ((b >> 40) & 0xff00ULL));
+}
diff --git a/gcc/testsuite/gcc.dg/pr32450.c b/gcc/testsuite/gcc.dg/pr32450.c
index 9606e30..0af262f 100644
--- a/gcc/testsuite/gcc.dg/pr32450.c
+++ b/gcc/testsuite/gcc.dg/pr32450.c
@@ -4,6 +4,7 @@
 /* { dg-require-profiling "-pg" } */
 /* { dg-options "-O2 -pg" } */
 /* { dg-options "-O2 -pg -mtune=core2" { target { i?86-*-* x86_64-*-* } } } */
+/* { dg-additional-options "-mfentry -fno-pic" { target i?86-*-gnu* x86_64-*-gnu* } } */
 /* { dg-options "-O2 -pg -static" { target hppa*-*-hpux* } } */
 
 extern void abort (void);
diff --git a/gcc/testsuite/gcc.dg/pr43643.c b/gcc/testsuite/gcc.dg/pr43643.c
index 43896ab..41c00c8 100644
--- a/gcc/testsuite/gcc.dg/pr43643.c
+++ b/gcc/testsuite/gcc.dg/pr43643.c
@@ -4,6 +4,7 @@
 /* { dg-require-profiling "-pg" } */
 /* { dg-options "-O2 -pg" } */
 /* { dg-options "-O2 -pg -static" { target hppa*-*-hpux* } } */
+/* { dg-additional-options "-mfentry -fno-pic" { target i?86-*-gnu* x86_64-*-gnu* } } */
 
 extern char *strdup (const char *);
 
diff --git a/gcc/testsuite/gcc.dg/pr78185.c b/gcc/testsuite/gcc.dg/pr78185.c
index ada8b1b..4c3af4f 100644
--- a/gcc/testsuite/gcc.dg/pr78185.c
+++ b/gcc/testsuite/gcc.dg/pr78185.c
@@ -1,3 +1,4 @@
+/* { dg-do run } */
 /* { dg-require-effective-target alarm } */
 /* { dg-require-effective-target signal } */
 /* { dg-options "-O" } */
diff --git a/gcc/testsuite/gcc.dg/pr87600-1.c b/gcc/testsuite/gcc.dg/pr87600-1.c
index 3517957..9d74cad 100644
--- a/gcc/testsuite/gcc.dg/pr87600-1.c
+++ b/gcc/testsuite/gcc.dg/pr87600-1.c
@@ -1,5 +1,5 @@
 /* PR rtl-optimization/87600  */
-/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* s390*-*-* x86_64-*-* } } */
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* loongarch*-*-* powerpc*-*-* s390*-*-* x86_64-*-* } } */
 /* { dg-options "-O2" } */
 
 #include "pr87600.h"
diff --git a/gcc/testsuite/gcc.dg/pr87600-2.c b/gcc/testsuite/gcc.dg/pr87600-2.c
index e8a9f19..822afe0 100644
--- a/gcc/testsuite/gcc.dg/pr87600-2.c
+++ b/gcc/testsuite/gcc.dg/pr87600-2.c
@@ -1,5 +1,5 @@
 /* PR rtl-optimization/87600  */
-/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* s390*-*-* x86_64-*-* } } */
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* loongarch*-*-* powerpc*-*-* s390*-*-* x86_64-*-* } } */
 /* { dg-options "-O2" } */
 
 #include "pr87600.h"
@@ -23,22 +23,3 @@ test1 (void)
   asm ("blah %0 %1" : "=r" (var1) : "0" (var2)); /* { dg-error "invalid hard register usage between output operand and matching constraint operand" } */
   return var1;
 }
-
-long
-test2 (void)
-{
-  register long var1 asm (REG1);
-  register long var2 asm (REG1);
-  asm ("blah %0 %1" : "=&r" (var1) : "r" (var2)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
-  return var1;
-}
-
-long
-test3 (void)
-{
-  register long var1 asm (REG1);
-  register long var2 asm (REG1);
-  long var3;
-  asm ("blah %0 %1" : "=&r" (var1), "=r" (var3) : "1" (var2)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
-  return var1 + var3;
-}
diff --git a/gcc/testsuite/gcc.dg/pr87600-3.c b/gcc/testsuite/gcc.dg/pr87600-3.c
new file mode 100644
index 0000000..4f43a5f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr87600-3.c
@@ -0,0 +1,26 @@
+/* PR rtl-optimization/87600  */
+/* { dg-do compile { target aarch64*-*-* arm*-*-* i?86-*-* powerpc*-*-* s390*-*-* x86_64-*-* } } */
+/* { dg-options "-O2" } */
+
+#include "pr87600.h"
+
+/* The following are all invalid uses of local register variables.  */
+
+long
+test2 (void)
+{
+  register long var1 asm (REG1);
+  register long var2 asm (REG1);
+  asm ("blah %0 %1" : "=&r" (var1) : "r" (var2)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+  return var1;
+}
+
+long
+test3 (void)
+{
+  register long var1 asm (REG1);
+  register long var2 asm (REG1);
+  long var3;
+  asm ("blah %0 %1" : "=&r" (var1), "=r" (var3) : "1" (var2)); /* { dg-error "invalid hard register usage between earlyclobber operand and input operand" } */
+  return var1 + var3;
+}
diff --git a/gcc/testsuite/gcc.dg/pr87600.h b/gcc/testsuite/gcc.dg/pr87600.h
index c89071eb..29f065e 100644
--- a/gcc/testsuite/gcc.dg/pr87600.h
+++ b/gcc/testsuite/gcc.dg/pr87600.h
@@ -7,6 +7,9 @@
 #elif defined (__i386__)
 # define REG1 "%eax"
 # define REG2 "%edx"
+#elif defined (__loongarch__)
+# define REG1 "$t0"
+# define REG2 "$t1"
 #elif defined (__powerpc__) || defined (__POWERPC__) || defined (__PPC__)
 # define REG1 "r3"
 # define REG2 "r4"
diff --git a/gcc/testsuite/gcc.dg/rtl/aarch64/vec-series-1.c b/gcc/testsuite/gcc.dg/rtl/aarch64/vec-series-1.c
new file mode 100644
index 0000000..6f795c6
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/rtl/aarch64/vec-series-1.c
@@ -0,0 +1,35 @@
+/* { dg-do compile { target aarch64*-*-* } } */
+/* { dg-options "-O2 -msve-vector-bits=256 -mlittle-endian" } */
+
+#include <arm_sve.h>
+
+#pragma GCC target "+sve"
+
+svint64x2_t __RTL (startwith ("vregs")) foo ()
+{
+  (function "foo"
+    (insn-chain
+      (block 2
+	(edge-from entry (flags "FALLTHRU"))
+	(cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+        (cnote 2 NOTE_INSN_FUNCTION_BEG)
+	(insn 3 (set (reg:VNx4DI <0>)
+		     (const_vector:VNx4DI [(const_int 11)
+					   (const_int 12)
+					   (const_int 13)
+					   (const_int 14)
+					   (const_int 15)
+					   (const_int 16)
+					   (const_int 17)
+					   (const_int 18)])))
+	(insn 4 (set (reg:VNx4DI v0) (reg:VNx4DI <0>)))
+        (insn 5 (use (reg:VNx4DI v0)))
+	(edge-to exit (flags "FALLTHRU"))
+      ) ;; block 2
+    ) ;; insn-chain
+    (crtl (return_rtx (reg:VNx4DI v0)))
+  ) ;; function
+}
+
+/* { dg-final { scan-assembler {\tindex\tz0\.d, #11, #1\n} } } */
+/* { dg-final { scan-assembler {\tindex\tz1\.d, #15, #1\n} } } */
diff --git a/gcc/testsuite/gcc.dg/rtl/aarch64/vec-series-2.c b/gcc/testsuite/gcc.dg/rtl/aarch64/vec-series-2.c
new file mode 100644
index 0000000..17e46cb
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/rtl/aarch64/vec-series-2.c
@@ -0,0 +1,35 @@
+/* { dg-do compile { target aarch64*-*-* } } */
+/* { dg-options "-O2 -msve-vector-bits=256 -mlittle-endian" } */
+
+#include <arm_sve.h>
+
+#pragma GCC target "+sve"
+
+svint64x2_t __RTL (startwith ("vregs")) foo ()
+{
+  (function "foo"
+    (insn-chain
+      (block 2
+	(edge-from entry (flags "FALLTHRU"))
+	(cnote 1 [bb 2] NOTE_INSN_BASIC_BLOCK)
+        (cnote 2 NOTE_INSN_FUNCTION_BEG)
+	(insn 3 (set (reg:VNx4DI <0>)
+		     (const_vector:VNx4DI [(const_int -16)
+					   (const_int -15)
+					   (const_int -14)
+					   (const_int -13)
+					   (const_int -12)
+					   (const_int -11)
+					   (const_int -10)
+					   (const_int -9)])))
+	(insn 4 (set (reg:VNx4DI v0) (reg:VNx4DI <0>)))
+        (insn 5 (use (reg:VNx4DI v0)))
+	(edge-to exit (flags "FALLTHRU"))
+      ) ;; block 2
+    ) ;; insn-chain
+    (crtl (return_rtx (reg:VNx4DI v0)))
+  ) ;; function
+}
+
+/* { dg-final { scan-assembler {\tindex\tz0\.d, #-16, #1\n} } } */
+/* { dg-final { scan-assembler {\tindex\tz1\.d, #-12, #1\n} } } */
diff --git a/gcc/testsuite/gcc.dg/sarif-output/include-chain-2.h b/gcc/testsuite/gcc.dg/sarif-output/include-chain-2.h
index 382ac02..f60fb33 100644
--- a/gcc/testsuite/gcc.dg/sarif-output/include-chain-2.h
+++ b/gcc/testsuite/gcc.dg/sarif-output/include-chain-2.h
@@ -1,4 +1,4 @@
-/* Generate a warning with a diagnostic_path within a header.  */
+/* Generate a warning with a diagnostic path within a header.  */
 
 void test (void *ptr)
 {
diff --git a/gcc/testsuite/gcc.dg/torture/pr120654.c b/gcc/testsuite/gcc.dg/torture/pr120654.c
index 3819b78..aacfeea 100644
--- a/gcc/testsuite/gcc.dg/torture/pr120654.c
+++ b/gcc/testsuite/gcc.dg/torture/pr120654.c
@@ -2,8 +2,6 @@
 
 int a, c, e, f, h, j;
 long g, k;
-void *malloc(long);
-void free(void *);
 int b(int m) {
   if (m || a)
     return 1;
@@ -16,9 +14,9 @@ int i() {
 }
 void n() {
   long o;
-  int *p = malloc(sizeof(int));
+  int *p = __builtin_malloc(sizeof(int));
   k = 1 % j;
   for (; i() + f + h; o++)
     if (p[d(j + 6, (int)k + 1992695866) + h + f + j + (int)k - 1 + o])
-      free(p);
+      __builtin_free(p);
 }
diff --git a/gcc/testsuite/gcc.dg/torture/pr120944.c b/gcc/testsuite/gcc.dg/torture/pr120944.c
new file mode 100644
index 0000000..92f3c77
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr120944.c
@@ -0,0 +1,34 @@
+/* { dg-skip-if "" { *-*-* } { "-flto" } { "" } } */
+/* { dg-additional-options "-fdump-tree-optimized" } */
+
+#include <stdlib.h>
+
+typedef union {
+  int u32;
+  struct
+  {
+     int A:1;
+     int B:2;
+     int C:3;
+  };
+} u_t;
+
+typedef union {
+   volatile int u[3];
+   volatile struct {
+        u_t a;
+        int b;
+        int c;
+   };
+} DATA;
+
+void foo (volatile DATA *d)
+{
+     d->a.u32 = ~0;
+     u_t u = d->a;
+     int v = u.A;
+     if (v)
+        abort();
+}
+
+/* { dg-final { scan-tree-dump-times "if \\\(" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/torture/pr120951-1.c b/gcc/testsuite/gcc.dg/torture/pr120951-1.c
new file mode 100644
index 0000000..4e2b41d
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr120951-1.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-fnon-call-exceptions -fsignaling-nans" } */
+
+/* PR tree-optimization/120951 */
+
+/* cdce would create a trapping comparison inside a condition.
+   tests to make sure that does not happen.  */
+
+double f(double r, double i) {
+   return __builtin_fmod(r, i);
+}
+
diff --git a/gcc/testsuite/gcc.dg/torture/pr121116.c b/gcc/testsuite/gcc.dg/torture/pr121116.c
new file mode 100644
index 0000000..637324f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr121116.c
@@ -0,0 +1,21 @@
+/* { dg-do run { target bitint } } */
+
+#include <stdlib.h>
+#include <stdckdint.h>
+#include <string.h>
+typedef _BitInt(16) bit16;
+[[nodiscard]] static bit16 process_data(bit16 input) {
+    _Static_assert(sizeof(bit16) == 2, "Unexpected size of bit16");
+    return (input << 5) | (input >> 9);
+}
+int main(void) {
+    const bit16 data = 0b101'0101'0000'0000;
+    bit16 result = 0;
+    for (bit16 i = 0; i < 0b1000; ++i) {
+        result ^= process_data(data ^ i);
+    }
+    if (ckd_add(&result, result, 0x1234)) {
+        return EXIT_FAILURE;
+    }
+    return (result & 0xFF00) ? 0 : 1;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr121194.c b/gcc/testsuite/gcc.dg/torture/pr121194.c
new file mode 100644
index 0000000..20f5ff7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr121194.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+
+int a, b, c, d;
+void e() {
+  int *f = &b;
+  for (a = 0; a < 8; a++) {
+    *f = 0;
+    for (c = 0; c < 2; c++)
+      *f = *f == 0;
+  }
+}
+int main() {
+  e();
+  int *g = &b;
+  *g = *g == (d == 0);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/torture/pr121236-1.c b/gcc/testsuite/gcc.dg/torture/pr121236-1.c
new file mode 100644
index 0000000..2b397e3
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr121236-1.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* PR tree-optimization/121236 */
+
+
+unsigned func_26(short *p_27, int gg, int p) {
+  unsigned l_184 = 0;
+  unsigned m = 0;
+  for (int g_59 = 0; g_59 < 10; g_59++)
+    {
+      if (gg)
+	l_184--;
+      else
+	{
+	  m |= l_184 |= p;
+	  (l_184)--;
+	}
+    }
+ return m;
+}
+
diff --git a/gcc/testsuite/gcc.dg/torture/pr121295-1.c b/gcc/testsuite/gcc.dg/torture/pr121295-1.c
new file mode 100644
index 0000000..7825c6e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/torture/pr121295-1.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-additional-options " -fno-tree-copy-prop -fno-tree-pre -fno-code-hoisting" */
+
+/* PR tree-optimization/121295 */
+
+
+int a, b, c;
+int main() {
+  int *d = &a;
+  while (b)
+    b = (*d &= 10) <= 0 || (*d = c);
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/tree-prof/afdo-crossmodule-1b.c b/gcc/testsuite/gcc.dg/tree-prof/afdo-crossmodule-1b.c
index dd53295..79ba529 100644
--- a/gcc/testsuite/gcc.dg/tree-prof/afdo-crossmodule-1b.c
+++ b/gcc/testsuite/gcc.dg/tree-prof/afdo-crossmodule-1b.c
@@ -1,3 +1,8 @@
+/* { dg-require-effective-target lto } */
+/* { dg-additional-sources "afdo-crossmodule-1.c" } */
+/* { dg-options "-O3 -flto -fdump-ipa-afdo_offline -fdump-tree-einline-details" } */
+/* { dg-require-profiling "-fauto-profile" } */
+
 extern int foo2 ();
 
 int bar (int (*fooptr) (int (*)()))
diff --git a/gcc/testsuite/gcc.dg/tree-prof/afdo-inline.c b/gcc/testsuite/gcc.dg/tree-prof/afdo-inline.c
index b67b3cb..ded4068 100644
--- a/gcc/testsuite/gcc.dg/tree-prof/afdo-inline.c
+++ b/gcc/testsuite/gcc.dg/tree-prof/afdo-inline.c
@@ -1,6 +1,15 @@
 /* { dg-options "-O2 -fdump-tree-einline-details --param early-inlining-insns=1" } */
 /* { dg-require-profiling "-fauto-profile" } */ 
 volatile int a[1000];
+
+#define STR1(X) #X
+#define STR2(X) STR1(X)
+
+int reta (int i)
+asm(STR2(__USER_LABEL_PREFIX__) "renamed_reta");
+int test ()
+asm(STR2(__USER_LABEL_PREFIX__) "renamed_test");
+
 int reta (int i)
 {
 	if (a[i])
diff --git a/gcc/testsuite/gcc.dg/tree-prof/clone-merge-1.c b/gcc/testsuite/gcc.dg/tree-prof/clone-merge-1.c
index 43a9090..904dd0c 100644
--- a/gcc/testsuite/gcc.dg/tree-prof/clone-merge-1.c
+++ b/gcc/testsuite/gcc.dg/tree-prof/clone-merge-1.c
@@ -31,4 +31,4 @@ int main()
 }
 /* We will have profiles for test2 and test2.constprop.0 that will have to be
    merged,  */
-/* { dg-final-use-autofdo { scan-ipa-dump "Merging duplicate symbol test2" "afdo_offline"} } */
+/* { dg-final-use-autofdo { scan-ipa-dump "Merging duplicate instance: test2" "afdo_offline"} } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/cmp-2.c b/gcc/testsuite/gcc.dg/tree-ssa/cmp-2.c
new file mode 100644
index 0000000..9b02901
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/cmp-2.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O1 -fdump-tree-forwprop" } */
+
+/* PR tree-optimization/110949 */
+/* Transform `(cmp) - 1` into `-icmp`. */
+
+int f1(int a)
+{
+  int t = a == 115;
+  return t - 1;
+}
+
+/* { dg-final { scan-tree-dump " != 115" "forwprop1" } } */
+/* { dg-final { scan-tree-dump-not " == 115" "forwprop1" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/cswtch-7.c b/gcc/testsuite/gcc.dg/tree-ssa/cswtch-7.c
new file mode 100644
index 0000000..7b797807
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/cswtch-7.c
@@ -0,0 +1,48 @@
+/* PR tree-optimization/120523 */
+/* PR tree-optimization/120451 */
+/* { dg-do compile { target elf } } */
+/* { dg-options "-O2" } */
+
+void foo (int, int);
+
+__attribute__((noinline, noclone)) void
+f1 (int v, int w)
+{
+  int i, j;
+  if (w)
+    {
+      i = 129;
+      j = i - 1;
+      goto lab;
+    }
+  switch (v)
+    {
+    case 170:
+      j = 7;
+      i = 27;
+      break;
+    case 171:
+      i = 8;
+      j = 122;
+      break;
+    case 172:
+      i = 21;
+      j = -19;
+      break;
+    case 173:
+      i = 18;
+      j = 17;
+      break;
+    case 174:
+      i = 33;
+      j = 55;
+      break;
+    default:
+      __builtin_abort ();
+    }
+
+ lab:
+  foo (i, j);
+}
+
+/* { dg-final { scan-assembler ".rodata.cst32" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/max-bitcmp-1.c b/gcc/testsuite/gcc.dg/tree-ssa/max-bitcmp-1.c
new file mode 100644
index 0000000..81b5a27
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/max-bitcmp-1.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O1 -fdump-tree-forwprop -fdump-tree-optimized" } */
+
+/* PR tree-optimization/95906 */
+/* this should become MAX_EXPR<a,b> */
+
+int f2(int a, int b)
+{
+    int cmp = -(a > b);
+    return (cmp & a) | (~cmp & b);
+}
+
+/*  we should not end up with -_2 */
+/*  we should not end up and & nor a `+ -1` */
+/* In optimized we should have a max.  */
+/* { dg-final { scan-tree-dump-not " -\[a-zA-Z_\]" "forwprop1" } } */
+/* { dg-final { scan-tree-dump-not " & " "forwprop1" } } */
+/* { dg-final { scan-tree-dump-not " . -1" "forwprop1" } } */
+/* { dg-final { scan-tree-dump-times "MAX_EXPR " 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr117423.c b/gcc/testsuite/gcc.dg/tree-ssa/pr117423.c
new file mode 100644
index 0000000..a5d3b29
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr117423.c
@@ -0,0 +1,49 @@
+/* { dg-do run } */
+/* { dg-options "-O1" } */
+
+struct s4 {
+    int _0;
+};
+struct s1 {
+    unsigned char _4;
+    long _1;
+};
+struct s2 {
+    union {
+        struct s3 {
+            unsigned char _1;
+            struct s4 _0;
+        } var_0;
+        struct s1 var_1;
+    } DATA;
+};
+int f1(int arg0) { return arg0 > 12345; }
+__attribute__((noinline))
+struct s4 f2(int arg0) {
+    struct s4 rv = {arg0};
+    return rv;
+}
+struct s2 f3(int arg0) {
+    struct s2 rv;
+    struct s1 var6 = {0};
+    struct s4 var7;
+    if (f1(arg0)) {
+        rv.DATA.var_1 = var6;
+        return rv;
+    } else {
+        rv.DATA.var_0._1 = 2;
+        var7 = f2(arg0);
+        rv.DATA.var_0._0 = var7;
+        return rv;
+    }
+}
+int main() {
+  if (f3(12345).DATA.var_0._0._0 == 12345)
+    ;
+  else
+    __builtin_abort();
+  if (f3(12344).DATA.var_0._0._0 == 12344)
+    ;
+  else
+    __builtin_abort();
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr119085.c b/gcc/testsuite/gcc.dg/tree-ssa/pr119085.c
new file mode 100644
index 0000000..e9811ce
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr119085.c
@@ -0,0 +1,37 @@
+/* { dg-do run } */
+/* { dg-options "-O1" } */
+
+struct with_hole {
+  int x;
+  long y;
+};
+struct without_hole {
+  int x;
+  int y;
+};
+union u {
+  struct with_hole with_hole;
+  struct without_hole without_hole;
+};
+
+void __attribute__((noinline))
+test (union u *up, union u u)
+{
+  union u u2;
+  volatile int f = 0;
+  u2 = u;
+  if (f)
+    u2.with_hole = u.with_hole;
+  *up = u2;
+}
+
+int main(void)
+{
+  union u u;
+  union u u2;
+  u2.without_hole.y = -1;
+  test (&u, u2);
+  if (u.without_hole.y != -1)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr121264.c b/gcc/testsuite/gcc.dg/tree-ssa/pr121264.c
new file mode 100644
index 0000000..bd5acc0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr121264.c
@@ -0,0 +1,12 @@
+/* PR tree-optimization/121264 */
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump " \\\| " "optimized" } } */
+
+struct A { char b; char c[0x20000010]; } a;
+
+int
+foo ()
+{
+  return a.c[0x20000000] || a.c[1];
+}
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr81627.c b/gcc/testsuite/gcc.dg/tree-ssa/pr81627.c
index 9ba43be..ef35b29 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/pr81627.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/pr81627.c
@@ -4,6 +4,7 @@
 int a, b, c, d[6], e = 3, f;
 
 void abort (void);
+void fn1 () __attribute__((noinline));
 void fn1 ()
 {
   for (b = 1; b < 5; b++)
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-sink-23.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-sink-23.c
new file mode 100644
index 0000000..f632dc8
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-sink-23.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-sink1-details" } */
+
+struct S {
+    int* x;
+    int* y;
+};
+
+void __attribute__((noreturn)) bar(const struct S* s);
+
+void foo(int a, int b) {
+    struct S s;
+    s.x = &a;
+    s.y = &b;
+    if (a < b) {
+        bar(&s);
+    }
+}
+
+/* { dg-final { scan-tree-dump "Sinking.*s.y" "sink1" } } */
+/* { dg-final { scan-tree-dump "Sinking.*s.x" "sink1" } } */
diff --git a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-2.c b/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-2.c
deleted file mode 100644
index 0653ecc..0000000
--- a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-2.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in
-   bounds sanitizer combined with VLA.  */
-/* { dg-do run } */
-/* { dg-options "-fsanitize=bounds" } */
-/* { dg-output "index 11 out of bounds for type 'int \\\[\\\*\\\]\\\[\\\*\\\]'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*index 20 out of bounds for type 'int \\\[\\\*\\\]\\\[\\\*\\\]\\\[\\\*\\\]'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*index 11 out of bounds for type 'int \\\[\\\*\\\]\\\[\\\*\\\]'\[^\n\r]*(\n|\r\n|\r)" } */
-/* { dg-output "\[^\n\r]*index 10 out of bounds for type 'int \\\[\\\*\\\]'\[^\n\r]*(\n|\r\n|\r)" } */
-
-
-#include <stdlib.h>
-
-void __attribute__((__noinline__)) setup_and_test_vla (int n, int m)
-{
-  struct foo {
-    int n;
-    int (*p)[n] __attribute__((counted_by(n)));
-  } *f;
-
-  f = (struct foo *) malloc (sizeof (struct foo));
-  f->p = (int (*)[n]) malloc (m * sizeof (int[n]));
-  f->n = m;
-  f->p[m][n-1] = 1;
-  free (f->p);
-  free (f);
-  return;
-}
-
-void __attribute__((__noinline__)) setup_and_test_vla_1 (int n1, int n2, int m)
-{
-  struct foo {
-    int n;
-    int (*p)[n2][n1] __attribute__((counted_by(n)));
-  } *f;
-
-  f = (struct foo *) malloc (sizeof(struct foo));
-  f->p = (int (*)[n2][n1]) malloc (m * sizeof (int[n2][n1]));
-  f->n = m;
-  f->p[m][n2][n1] = 1;
-  free (f->p);
-  free (f);
-  return;
-}
-
-int main(int argc, char *argv[])
-{
-  setup_and_test_vla (10, 11);
-  setup_and_test_vla_1 (10, 11, 20);
-  return 0;
-}
-
diff --git a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-3.c b/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-3.c
deleted file mode 100644
index 731422d..0000000
--- a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-3.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in bounds
-   sanitizer. when counted_by field is negative value.  */
-/* { dg-do run } */
-/* { dg-options "-fsanitize=bounds" } */
-
-#include <stdlib.h>
-
-struct annotated {
-  int b;
-  int *c __attribute__ ((counted_by (b)));
-} *array_annotated;
-
-void __attribute__((__noinline__)) setup (int annotated_count)
-{
-  array_annotated
-    = (struct annotated *)malloc (sizeof (struct annotated));
-  array_annotated->c = (int *) malloc (sizeof (int) * 10);
-  array_annotated->b = annotated_count;
-
-  return;
-}
-
-void __attribute__((__noinline__)) test (int annotated_index)
-{
-  array_annotated->c[annotated_index] = 2;
-}
-
-void cleanup ()
-{
-  free (array_annotated->c);
-  free (array_annotated);
-}
-
-int main(int argc, char *argv[])
-{
-  setup (-3);   
-  test (2);
-  cleanup ();
-  return 0;
-}
-
-/* { dg-output "25:21: runtime error: index 2 out of bounds for type" } */
diff --git a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-4.c b/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-4.c
deleted file mode 100644
index 52f202f..0000000
--- a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-4.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in bounds
-   sanitizer. when counted_by field is zero value.  */
-/* { dg-do run } */
-/* { dg-options "-fsanitize=bounds" } */
-
-#include <stdlib.h>
-
-struct annotated {
-  int b;
-  int *c __attribute__ ((counted_by (b)));
-} *array_annotated;
-
-void __attribute__((__noinline__)) setup (int annotated_count)
-{
-  array_annotated
-    = (struct annotated *)malloc (sizeof (struct annotated));
-  array_annotated->c = (int *)malloc (sizeof (int) * 10);
-  array_annotated->b = annotated_count;
-
-  return;
-}
-
-void __attribute__((__noinline__)) test (int annotated_index)
-{
-  array_annotated->c[annotated_index] = 2;
-}
-
-void cleanup ()
-{
-  free (array_annotated->c);
-  free (array_annotated);
-}
-
-int main(int argc, char *argv[])
-{
-  setup (0);   
-  test (1);
-  cleanup ();
-  return 0;
-}
-
-/* { dg-output "25:21: runtime error: index 1 out of bounds for type" } */
diff --git a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-5.c b/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-5.c
deleted file mode 100644
index 8ad7572..0000000
--- a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds-5.c
+++ /dev/null
@@ -1,40 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in
-   bounds sanitizer.  */
-/* { dg-do run } */
-/* { dg-options "-fsanitize=bounds" } */
-
-#include <stdlib.h>
-
-struct annotated {
-  int b;
-  int *c __attribute__ ((counted_by (b)));
-} *p_array_annotated;
-
-void __attribute__((__noinline__)) setup (int annotated_count)
-{
-  p_array_annotated
-    = (struct annotated *)malloc (sizeof (struct annotated));
-  p_array_annotated->c = (int *) malloc (annotated_count *  sizeof (int));
-  p_array_annotated->b = annotated_count;
-
-  return;
-}
-
-void cleanup ()
-{
-  free (p_array_annotated->c);
-  free (p_array_annotated);
-}
-
-int main(int argc, char *argv[])
-{
-  int i;
-  setup (10);
-  for (i = 0; i < 11; i++)
-    p_array_annotated->c[i] = 2; // goes boom at i == 10
-  cleanup ();
-  return 0;
-}
-
-
-/* { dg-output "34:25: runtime error: index 10 out of bounds for type" } */
diff --git a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds.c b/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds.c
deleted file mode 100644
index c5a1ac5..0000000
--- a/gcc/testsuite/gcc.dg/ubsan/pointer-counted-by-bounds.c
+++ /dev/null
@@ -1,46 +0,0 @@
-/* Test the attribute counted_by for pointer fields and its usage in
-   bounds sanitizer.  */
-/* { dg-do run } */
-/* { dg-options "-fsanitize=bounds" } */
-
-#include <stdlib.h>
-
-struct pointer_array {
-  int b;
-  int *c;
-} *p_array;
-
-struct annotated {
-  int b;
-  int *c __attribute__ ((counted_by (b)));
-} *p_array_annotated;
-
-void __attribute__((__noinline__)) setup (int normal_count, int annotated_count)
-{
-  p_array
-    = (struct pointer_array *) malloc (sizeof (struct pointer_array));
-  p_array->c = (int *) malloc (normal_count * sizeof (int));
-  p_array->b = normal_count;
-
-  p_array_annotated
-    = (struct annotated *) malloc (sizeof (struct annotated));
-  p_array_annotated->c = (int *) malloc (annotated_count * sizeof (int));
-  p_array_annotated->b = annotated_count;
-
-  return;
-}
-
-void __attribute__((__noinline__)) test (int normal_index, int annotated_index)
-{
-  p_array->c[normal_index] = 1;
-  p_array_annotated->c[annotated_index] = 2;
-}
-
-int main(int argc, char *argv[])
-{
-  setup (10, 10);   
-  test (10, 10);
-  return 0;
-}
-
-/* { dg-output "36:23: runtime error: index 10 out of bounds for type" } */
diff --git a/gcc/testsuite/gcc.dg/ubsan/pr120837.c b/gcc/testsuite/gcc.dg/ubsan/pr120837.c
new file mode 100644
index 0000000..97c85c7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/ubsan/pr120837.c
@@ -0,0 +1,32 @@
+/* PR c/120837 */
+/* { dg-do run } */
+/* { dg-options "-O1 -fsanitize=undefined -fno-sanitize-recover=undefined" } */
+
+[[gnu::noipa]] void
+bar (void **x, void **y)
+{
+  x[0] = 0;
+  x[1] = 0;
+  x[2] = 0;
+  y[0] = 0;
+  y[1] = 0;
+  y[2] = 0;
+  y[3] = 0;
+  y[4] = 0;
+}
+
+[[gnu::noipa]] void *
+foo (int x, int y)
+{
+  void *a[3];
+  void *b[5];
+  bar (a, b);
+  return (x > y ? b : a)[y - 1];
+}
+
+int
+main ()
+{
+  if (foo (2, 1) != 0)
+    __builtin_abort ();
+}
diff --git a/gcc/testsuite/gcc.dg/uninit-pr120924.c b/gcc/testsuite/gcc.dg/uninit-pr120924.c
new file mode 100644
index 0000000..bfc8ae9
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/uninit-pr120924.c
@@ -0,0 +1,34 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -Wmaybe-uninitialized" } */
+
+int foo(int);
+enum {
+  BPF_TRACE_RAW_TP,
+  BPF_MODIFY_RETURN,
+  BPF_LSM_MAC,
+  BPF_TRACE_ITER,
+  BPF_LSM_CGROUP
+};
+int btf_get_kernel_prefix_kind_prefix, obj_1, attach_name___trans_tmp_1;
+char attach_name_fn_name;
+void attach_name(int attach_type)
+{
+  int mod_len;
+  char mod_name = attach_name_fn_name;
+  if (attach_name_fn_name)
+    mod_len = mod_name;
+  for (; obj_1;) {
+    if (mod_name && foo(mod_len))
+      continue;
+    switch (attach_type) {
+    case BPF_TRACE_RAW_TP:
+    case BPF_LSM_MAC:
+    case BPF_LSM_CGROUP:
+      btf_get_kernel_prefix_kind_prefix = 1;
+    case BPF_TRACE_ITER:
+      attach_name_fn_name = 2;
+    }
+    if (attach_name___trans_tmp_1)
+      return;
+  }
+}
diff --git a/gcc/testsuite/gcc.dg/unused-9.c b/gcc/testsuite/gcc.dg/unused-9.c
index bdf36e1..ad1ad0e 100644
--- a/gcc/testsuite/gcc.dg/unused-9.c
+++ b/gcc/testsuite/gcc.dg/unused-9.c
@@ -2,12 +2,9 @@
 /* { dg-do compile } */
 /* { dg-options "-Wunused" } */
 
-
 void g(void)
 {
-  int i = 0;
-  volatile int x;
-  (x, i++);	/* { dg-bogus "set but not used" } */
+  int i = 0;		/* { dg-warning "variable 'i' set but not used" } */
+  volatile int x;	/* { dg-bogus "variable 'x' set but not used" } */
+  (x, i++);
 }
-
-
diff --git a/gcc/testsuite/gcc.dg/vect/bb-slp-39.c b/gcc/testsuite/gcc.dg/vect/bb-slp-39.c
index f05ce8f..255bb10 100644
--- a/gcc/testsuite/gcc.dg/vect/bb-slp-39.c
+++ b/gcc/testsuite/gcc.dg/vect/bb-slp-39.c
@@ -16,5 +16,4 @@ void foo (double *p)
 }
 
 /* See that we vectorize three SLP instances.  */
-/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "slp2" { target { ! { s390*-*-* riscv*-*-* } } } } } */
-/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 5 "slp2" { target {   s390*-*-* riscv*-*-* } } } } */
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "slp2" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr101145.c b/gcc/testsuite/gcc.dg/vect/pr101145.c
index cd11c03..c055ae6 100644
--- a/gcc/testsuite/gcc.dg/vect/pr101145.c
+++ b/gcc/testsuite/gcc.dg/vect/pr101145.c
@@ -2,7 +2,7 @@
 /* { dg-additional-options "-O3" } */
 #include <limits.h>
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 foo (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {
   while (n < ++l)
@@ -10,7 +10,7 @@ foo (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 foo_1 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned)
 {
   while (UINT_MAX - 64 < ++l)
@@ -18,7 +18,7 @@ foo_1 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 foo_2 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {
   l = UINT_MAX - 32;
@@ -27,7 +27,7 @@ foo_2 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 foo_3 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {
   while (n <= ++l)
@@ -35,7 +35,7 @@ foo_3 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 foo_4 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {  // infininate 
   while (0 <= ++l)
@@ -43,7 +43,7 @@ foo_4 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 foo_5 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {
   //no loop
@@ -53,7 +53,7 @@ foo_5 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 bar (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {
   while (--l < n)
@@ -61,7 +61,7 @@ bar (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 bar_1 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned)
 {
   while (--l < 64)
@@ -69,7 +69,7 @@ bar_1 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned)
   return l;
 }
 
-unsigned __attribute__ ((noinline))
+unsigned __attribute__ ((noipa))
 bar_2 (int *__restrict__ a, int *__restrict__ b, unsigned l, unsigned n)
 {
   l = 32;
diff --git a/gcc/testsuite/gcc.dg/vect/pr112325.c b/gcc/testsuite/gcc.dg/vect/pr112325.c
index 8689fbf..d380595 100644
--- a/gcc/testsuite/gcc.dg/vect/pr112325.c
+++ b/gcc/testsuite/gcc.dg/vect/pr112325.c
@@ -5,6 +5,7 @@
 /* { dg-additional-options "-mavx2" { target x86_64-*-* i?86-*-* } } */
 /* { dg-additional-options "--param max-completely-peeled-insns=200" { target powerpc64*-*-* } } */
 /* { dg-additional-options "-mlsx" { target loongarch64-*-* } } */
+/* { dg-additional-options "--param max-completely-peeled-insns=200 --param min-vect-loop-bound=0" { target s390*-*-* } } */
 
 typedef unsigned short ggml_fp16_t;
 static float table_f32_f16[1 << 16];
diff --git a/gcc/testsuite/gcc.dg/vect/pr116125.c b/gcc/testsuite/gcc.dg/vect/pr116125.c
index eab9efd..1b882ec 100644
--- a/gcc/testsuite/gcc.dg/vect/pr116125.c
+++ b/gcc/testsuite/gcc.dg/vect/pr116125.c
@@ -17,12 +17,12 @@ main (void)
 {
   check_vect ();
 
-  struct st a[9] = {};
+  struct st a[10] = {};
 
-  // input a = 0, 0, 0, 0, 0, 0, 0, 0, 0
+  // input a = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
   mem_overlap (&a[1], a);
 
-  // output a = 0, 1, 2, 3, 4, 5, 6, 7, 8
+  // output a = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9
   if (a[2].num == 2)
     return 0;
   else
diff --git a/gcc/testsuite/gcc.dg/vect/pr117888-1.c b/gcc/testsuite/gcc.dg/vect/pr117888-1.c
index 0b31fcd..884aed2 100644
--- a/gcc/testsuite/gcc.dg/vect/pr117888-1.c
+++ b/gcc/testsuite/gcc.dg/vect/pr117888-1.c
@@ -5,6 +5,7 @@
 /* { dg-additional-options "-mavx2" { target x86_64-*-* i?86-*-* } } */
 /* { dg-additional-options "--param max-completely-peeled-insns=200" { target powerpc64*-*-* } } */
 /* { dg-additional-options "-mlsx" { target loongarch64-*-* } } */
+/* { dg-additional-options "--param max-completely-peeled-insns=200 --param min-vect-loop-bound=0" { target s390*-*-* } } */
 
 typedef unsigned short ggml_fp16_t;
 static float table_f32_f16[1 << 16];
diff --git a/gcc/testsuite/gcc.dg/vect/pr120687-1.c b/gcc/testsuite/gcc.dg/vect/pr120687-1.c
new file mode 100644
index 0000000..ce9cf63
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr120687-1.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_int } */
+
+unsigned
+frd (unsigned *p, unsigned *lastone)
+{
+  unsigned sum = 0;
+  for (; p <= lastone; p += 16)
+    sum += p[0] + p[1] + p[2] + p[3] + p[4] + p[5] + p[6] + p[7]
+           + p[8] + p[9] + p[10] + p[11] + p[12] + p[13] + p[14] + p[15];
+  return sum;
+}
+
+/* { dg-final { scan-tree-dump "reduction: detected reduction chain" "vect" } } */
+/* { dg-final { scan-tree-dump-not "SLP discovery of reduction chain failed" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr120687-2.c b/gcc/testsuite/gcc.dg/vect/pr120687-2.c
new file mode 100644
index 0000000..dfc6dc7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr120687-2.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_float } */
+/* { dg-additional-options "-ffast-math" } */
+
+float
+frd (float *p, float *lastone)
+{
+  float sum = 0;
+  for (; p <= lastone; p += 16)
+    sum += p[0] + p[1] + p[2] + p[3] + p[4] + p[5] + p[6] + p[7]
+           + p[8] + p[9] + p[10] + p[11] + p[12] + p[13] + p[14] + p[15];
+  return sum;
+}
+
+/* { dg-final { scan-tree-dump "reduction: detected reduction chain" "vect" } } */
+/* { dg-final { scan-tree-dump-not "SLP discovery of reduction chain failed" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr120687-3.c b/gcc/testsuite/gcc.dg/vect/pr120687-3.c
new file mode 100644
index 0000000..f20a66a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr120687-3.c
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_double } */
+/* { dg-additional-options "-ffast-math" } */
+
+float
+frd (float *p, float *lastone)
+{
+  float sum = 0;
+  for (; p <= lastone; p += 2)
+    sum += p[0] + p[1];
+  return sum;
+}
+
+/* { dg-final { scan-tree-dump "reduction: detected reduction chain" "vect" } } */
+/* { dg-final { scan-tree-dump-not "SLP discovery of reduction chain failed" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized" "vect" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr120817.c b/gcc/testsuite/gcc.dg/vect/pr120817.c
new file mode 100644
index 0000000..199189a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr120817.c
@@ -0,0 +1,41 @@
+/* { dg-additional-options "-O1" } */
+/* { dg-require-effective-target aarch64_sve_hw { target aarch64*-*-* } } */
+/* { dg-additional-options "-march=armv8-a+sve -mtune=neoverse-n2" { target aarch64*-*-* } } */
+
+#include "tree-vect.h"
+
+typedef struct {
+    int _M_current;
+} __normal_iterator;
+
+typedef struct {
+    char _M_elems[5];
+} array_5;
+
+__normal_iterator __trans_tmp_1 = {-5};
+
+__attribute__((noipa))
+array_5 copySourceIntoTarget() {
+    array_5 target;
+    char* target_it = target._M_elems;
+
+    while (__trans_tmp_1._M_current != 0) {
+        *target_it = 1;
+        __trans_tmp_1._M_current++;
+        target_it++;
+    }
+
+    return target;
+}
+
+int main ()
+{
+  check_vect ();
+
+  array_5 res = copySourceIntoTarget();
+
+#pragma GCC novector
+  for (int i = 0; i < 5; i++)
+    if (res._M_elems[i] != 1)
+      __builtin_abort ();
+}
diff --git a/gcc/testsuite/gcc.dg/vect/pr120922.c b/gcc/testsuite/gcc.dg/vect/pr120922.c
new file mode 100644
index 0000000..1a7247a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr120922.c
@@ -0,0 +1,18 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-additional-options "-fsigned-char -fno-strict-aliasing -fwrapv" } */
+/* { dg-additional-options "-march=rv64gcv_zvl1024b -mrvv-vector-bits=zvl -mrvv-max-lmul=m8 -O3" { target { riscv_v } } } */
+
+char g;
+unsigned char h;
+int i[9][6];
+int main() {
+  int k[5];
+  if (g)
+    goto l;
+  for (; h <= 5; h++)
+    i[0][h] = *k;
+l:
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump "loop vectorized" "vect" { target riscv_v } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/pr121034.c b/gcc/testsuite/gcc.dg/vect/pr121034.c
new file mode 100644
index 0000000..de20781
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr121034.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3" } */
+
+int b, e;
+char c, d;
+unsigned g;
+int abs(int);
+void f() {
+  char *a = &d;
+  int h;
+  for (; e; e++) {
+    h = 0;
+    for (; h < 16; h++)
+      g += __builtin_abs(a[h] - c);
+    a += b;
+  }
+}
diff --git a/gcc/testsuite/gcc.dg/vect/pr121049.c b/gcc/testsuite/gcc.dg/vect/pr121049.c
new file mode 100644
index 0000000..558c92a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr121049.c
@@ -0,0 +1,25 @@
+/* { dg-additional-options "--param vect-partial-vector-usage=1" } */
+/* { dg-additional-options "-march=x86-64-v4" { target avx512f_runtime } } */
+
+#include "tree-vect.h"
+
+int mon_lengths[12] = { 1, 10, 100 };
+
+__attribute__ ((noipa)) long
+transtime (int mon)
+{
+  long value = 0;
+  for (int i = 0; i < mon; ++i)
+    value += mon_lengths[i] * 2l;
+  return value;
+}
+
+int
+main ()
+{
+  check_vect ();
+  if (transtime (3) != 222)
+    __builtin_abort ();
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.dg/vect/pr121059.c b/gcc/testsuite/gcc.dg/vect/pr121059.c
new file mode 100644
index 0000000..2bbfcea
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr121059.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3 --param vect-partial-vector-usage=1" } */
+/* { dg-additional-options "-march=x86-64-v4" { target avx512f } } */
+
+typedef struct {
+  long left, right, top, bottom;
+} MngBox;
+typedef struct {
+  MngBox object_clip[6];
+  char exists[256], frozen[];
+} MngReadInfo;
+MngReadInfo mng_info;
+
+long ReadMNGImage_i;
+
+void ReadMNGImage(int ReadMNGImage_i)
+{
+  for (; ReadMNGImage_i < 256; ReadMNGImage_i++)
+    if (mng_info.exists[ReadMNGImage_i] && mng_info.frozen[ReadMNGImage_i])
+      mng_info.object_clip[ReadMNGImage_i].left =
+          mng_info.object_clip[ReadMNGImage_i].right =
+              mng_info.object_clip[ReadMNGImage_i].top =
+                  mng_info.object_clip[ReadMNGImage_i].bottom = 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/pr121126.c b/gcc/testsuite/gcc.dg/vect/pr121126.c
new file mode 100644
index 0000000..ae6603b
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/pr121126.c
@@ -0,0 +1,30 @@
+/* { dg-additional-options "--param vect-partial-vector-usage=2" } */
+
+#include "tree-vect.h"
+
+unsigned char a;
+unsigned b;
+int r[11];
+static void __attribute__((noipa))
+c(int e, unsigned s[][11][11])
+{
+  for (int u = -(e ? 2000424973 : 0) - 2294542319; u < 7; u += 4)
+    for (int x = 0; x < 300000011; x += 4)
+      for (int y = 0; y < (0 < s[u][4][1]) + 11; y += 3) {
+        a = a ?: 1;
+        b = r[2];
+      }
+}
+long long ab;
+int e = 1;
+unsigned s[11][11][11];
+int main()
+{
+  check_vect ();
+  for (int t = 0; t < 11; ++t)
+    r[t] = 308100;
+  c(e,s);
+  ab = b;
+  if (ab != 308100)
+    __builtin_abort ();
+}
diff --git a/gcc/testsuite/gcc.dg/vect/slp-28.c b/gcc/testsuite/gcc.dg/vect/slp-28.c
index 67b7be2..1f98787 100644
--- a/gcc/testsuite/gcc.dg/vect/slp-28.c
+++ b/gcc/testsuite/gcc.dg/vect/slp-28.c
@@ -59,8 +59,8 @@ main1 ()
         abort ();
     }
   
-  /* Not vectorizable because of data dependencies: distance 3 is greater than 
-     the actual VF with SLP (2), but the analysis fail to detect that for now.  */
+  /* Dependence distance 3 is greater than the actual VF with SLP (2),
+     thus vectorizable.  */
   for (i = 3; i < N/4; i++)
     {
       in3[i*4] = in3[(i-3)*4] + 5;
@@ -91,7 +91,6 @@ int main (void)
   return 0;
 }
 
-/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { target { ! vect32 } } } } */
-/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" { target vect32 } } } */
-/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 1 "vect" { target { ! vect32 } } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 2 "vect" } } */
   
diff --git a/gcc/testsuite/gcc.dg/vect/vect-127.c b/gcc/testsuite/gcc.dg/vect/vect-127.c
new file mode 100644
index 0000000..8b913dc
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-127.c
@@ -0,0 +1,15 @@
+// { dg-do compile }
+// { dg-require-effective-target vect_int }
+
+void foo (int *p)
+{
+  for (int i = 0; i < 1024; ++i)
+    {
+      int a0 = p[2*i + 0];
+      int a1 = p[2*i + 1];
+      p[2*i + 4] = a0;
+      p[2*i + 5] = a1;
+    }
+}
+
+/* { dg-final { scan-tree-dump "loop vectorized using 16 byte vectors" "vect" { target { vect128 && vect_hw_misalign } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_137-pr121190.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_137-pr121190.c
new file mode 100644
index 0000000..e6b071c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_137-pr121190.c
@@ -0,0 +1,62 @@
+/* PR tree-optimization/121190 */
+/* { dg-options "-O3" } */
+/* { dg-additional-options "-march=znver2" { target x86_64-*-* i?86-*-* } } */
+/* { dg-require-effective-target mmap } */
+/* { dg-require-effective-target vect_early_break } */
+
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include "tree-vect.h"
+
+#define MAX_COMPARE 5000
+
+__attribute__((noipa))
+int diff (uint64_t *restrict p, uint64_t *restrict q)
+{
+  int i = 0;
+  while (i < MAX_COMPARE) {
+    if (*(p + i) != *(q + i))
+      return i;
+    i++;
+  }
+  return -1;
+}
+
+int main ()
+{
+  check_vect ();
+
+  long pgsz = sysconf (_SC_PAGESIZE);
+  if (pgsz == -1) {
+    fprintf (stderr, "sysconf failed\n");
+    return 0;
+  }
+
+  /* Allocate 2 consecutive pages of memory and let p1 and p2 point to the
+     beginning of each.  */
+  void *mem = mmap (NULL, pgsz * 2, PROT_READ | PROT_WRITE,
+		    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  if (mem == MAP_FAILED) {
+    fprintf (stderr, "mmap failed\n");
+    return 0;
+  }
+  uint64_t *p1 = (uint64_t *) mem;
+  uint64_t *p2 = (uint64_t *) mem + pgsz / sizeof (uint64_t);
+
+  /* Fill the first page with zeros, except for its last 64 bits.  */
+  memset (p1, 0, pgsz);
+  *(p2 - 1) = -1;
+
+  /* Make the 2nd page not accessable.  */
+  mprotect (p2, pgsz, PROT_NONE);
+
+  /* Calls to diff should not read the 2nd page.  */
+  for (int i = 1; i <= 20; i++) {
+    if (diff (p2 - i, p1) != i - 1)
+      __builtin_abort ();
+  }
+}
+
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_138-pr121020.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_138-pr121020.c
new file mode 100644
index 0000000..8cb62bf
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_138-pr121020.c
@@ -0,0 +1,54 @@
+/* PR tree-optimization/121020 */
+/* { dg-options "-O3 --vect-cost-model=unlimited" } */
+/* { dg-additional-options "-march=znver2" { target x86_64-*-* i?86-*-* } } */
+/* { dg-require-effective-target mmap } */
+/* { dg-require-effective-target vect_early_break } */
+
+#include <stdint.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include "tree-vect.h"
+
+__attribute__((noipa))
+bool equal (uint64_t *restrict p, uint64_t *restrict q, int length)
+{
+  for (int i = 0; i < length; i++) {
+    if (*(p + i) != *(q + i))
+      return false;
+  }
+  return true;
+}
+
+int main ()
+{
+  check_vect ();
+
+  long pgsz = sysconf (_SC_PAGESIZE);
+  if (pgsz == -1) {
+    fprintf (stderr, "sysconf failed\n");
+    return 0;
+  }
+
+  /* Allocate a whole page of memory.  */
+  void *mem = mmap (NULL, pgsz, PROT_READ | PROT_WRITE,
+		    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  if (mem == MAP_FAILED) {
+    fprintf (stderr, "mmap failed\n");
+    return 0;
+  }
+  uint64_t *p1 = (uint64_t *) mem;
+  uint64_t *p2 = (uint64_t *) mem + 32;
+
+  /* The first 16 elements pointed to by p1 and p2 are the same.  */
+  for (int i = 0; i < 32; i++) {
+    *(p1 + i) = 0;
+    *(p2 + i) = (i < 16 ? 0 : -1);
+  }
+
+  /* All calls to equal should return true.  */
+  for (int len = 0; len < 16; len++) {
+    if (!equal (p1 + 1, p2 + 1, len))
+      __builtin_abort();
+  }
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-early-break_52.c b/gcc/testsuite/gcc.dg/vect/vect-early-break_52.c
index 86a632f..6abfcd6 100644
--- a/gcc/testsuite/gcc.dg/vect/vect-early-break_52.c
+++ b/gcc/testsuite/gcc.dg/vect/vect-early-break_52.c
@@ -18,4 +18,4 @@ int main1 (short X)
     }
 }
 
-/* { dg-final { scan-tree-dump "vectorized 1 loops in function" "vect" { target { ! "x86_64-*-* i?86-*-*" } } } } */
+/* { dg-final { scan-tree-dump "vectorized 1 loops in function" "vect" { target { ! "x86_64-*-* i?86-*-* arm*-*-*" } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-pr120927-2.c b/gcc/testsuite/gcc.dg/vect/vect-pr120927-2.c
new file mode 100644
index 0000000..e38cebe
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-pr120927-2.c
@@ -0,0 +1,24 @@
+/* { dg-additional-options "--param vect-partial-vector-usage=1" } */
+/* { dg-additional-options "-mavx512bw -mavx512vl" { target avx512f_runtime } } */
+
+#include "tree-vect.h"
+
+static const double __attribute__((aligned(__BIGGEST_ALIGNMENT__))) a[] = { 1., 2., 3., 4., 5. };
+
+void __attribute__((noipa))
+foo (double *b, double *bp, double c, int n)
+{
+  for (int i = 0; i < n; ++i)
+    b[i] = bp[i] = a[i] * c;
+}
+
+int main()
+{
+  double b[6], bp[6];
+  b[5] = bp[5] = 13.;
+  check_vect ();
+  foo (b, bp, 3., 5);
+  if (b[5] != 13. || bp[5] != 13.)
+    abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-pr120927.c b/gcc/testsuite/gcc.dg/vect/vect-pr120927.c
new file mode 100644
index 0000000..793593f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-pr120927.c
@@ -0,0 +1,24 @@
+/* { dg-additional-options "--param vect-partial-vector-usage=1" } */
+/* { dg-additional-options "-mavx512bw -mavx512vl" { target avx512f_runtime } } */
+
+#include "tree-vect.h"
+
+static const double a[] = { 1., 2., 3., 4., 5. };
+
+void __attribute__((noipa))
+foo (double *b, double *bp, double c, int n)
+{
+  for (int i = 0; i < n; ++i)
+    b[i] = bp[i] = a[i] * c;
+}
+
+int main()
+{
+  double b[6], bp[6];
+  b[5] = bp[5] = 13.;
+  check_vect ();
+  foo (b, bp, 3., 5);
+  if (b[5] != 13. || bp[5] != 13.)
+    abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256-2.c b/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256-2.c
new file mode 100644
index 0000000..7350fd9
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256-2.c
@@ -0,0 +1,49 @@
+/* { dg-additional-options "-mavx2" { target avx2_runtime } } */
+
+#include "tree-vect.h"
+
+#define B 0
+#define G 1
+#define R 2
+
+int red = 153;
+int green = 66;
+int blue = 187;
+
+static void __attribute__((noipa))
+sub_left_prediction_bgr32(int *restrict dst, int *restrict src)
+{
+  for (int i = 0; i < 8; i++) {
+    int rt = src[i * 3 + R];
+    int gt = src[i * 3 + G];
+    int bt = src[i * 3 + B];
+
+    dst[i * 3 + R] = rt - red;
+    dst[i * 3 + G] = gt - green;
+    dst[i * 3 + B] = bt - blue;
+
+    red = rt;
+    green = gt;
+    blue = bt;
+  }
+}
+
+int main()
+{
+  int dst[8*3];
+  int src[8*3] = { 160, 73, 194, 17, 33, 99, 0, 12, 283, 87, 73, 11,
+		   9, 7, 1, 23, 19, 13, 77, 233, 97, 78, 2, 5 };
+  int dst2[8*3] = {-27, 7, 41, -143, -40, -95, -17, -21, 184, 87, 61,
+      -272, -78, -66, -10, 14, 12, 12, 54, 214, 84, 1, -231, -92};
+
+  check_vect ();
+
+  sub_left_prediction_bgr32(dst, src);
+
+#pragma GCC novector
+  for (int i = 0; i < 8*3; ++i)
+    if (dst[i] != dst2[i])
+      __builtin_abort();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256.c b/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256.c
new file mode 100644
index 0000000..c895e94
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256.c
@@ -0,0 +1,54 @@
+/* { dg-additional-options "-mavx2" { target avx2_runtime } } */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include "tree-vect.h"
+
+#define B 0
+#define G 1
+#define R 2
+#define A 3
+
+int red = 153;
+int green = 66;
+int blue = 187;
+int alpha = 255;
+
+static void __attribute__((noipa))
+sub_left_prediction_bgr32(uint8_t *restrict dst, uint8_t *restrict src, int w)
+{
+  for (int i = 0; i < 8; i++) {
+    int rt = src[i * 4 + R];
+    int gt = src[i * 4 + G];
+    int bt = src[i * 4 + B];
+    int at = src[i * 4 + A];
+
+    dst[i * 4 + R] = rt - red;
+    dst[i * 4 + G] = gt - green;
+    dst[i * 4 + B] = bt - blue;
+    dst[i * 4 + A] = at - alpha;
+
+    red = rt;
+    green = gt;
+    blue = bt;
+    alpha = at;
+  }
+}
+
+int main()
+{
+  check_vect ();
+
+  uint8_t *dst = calloc(36, sizeof(uint8_t));
+  uint8_t *src = calloc(36, sizeof(uint8_t));
+
+  src[R] = 160;
+  src[G] = 73;
+  src[B] = 194;
+  src[A] = 255;
+
+  sub_left_prediction_bgr32(dst, src, 33);
+  if (dst[R] != 7 || dst[B] != 7 || dst[A] != 0)
+    __builtin_abort();
+}
diff --git a/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-1.c b/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-1.c
new file mode 100644
index 0000000..258f17e
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-1.c
@@ -0,0 +1,60 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_condition } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+/* PR tree-optimization/119920 */
+
+#define N 32
+
+unsigned int ub[N];
+
+/* Test vectorization of reduction of unsigned-int.  */
+
+__attribute__ ((noinline, noipa))
+void init(void)
+{
+  #pragma GCC novector
+  for(int i = 0;i < N; i++)
+    ub[i] = i;
+}
+
+
+__attribute__ ((noinline, noipa))
+void main1 (unsigned int b, unsigned int c)
+{
+  int i;
+  unsigned int usum = 0;
+
+  init();
+
+  /* Summation.  */
+  for (i = 0; i < N; i++) {
+    if ( ub[i] < N/2 )
+    {
+      usum += b;
+    }
+    else
+    {
+      usum += c;
+    }
+  }
+
+  /* check results:  */
+ /* __builtin_printf("%d : %d\n", usum, (N/2*b + N/2*c)); */
+  if (usum != N/2*b + N/2*c)
+    abort ();
+}
+
+int main (void)
+{ 
+  check_vect ();
+  
+  main1 (0, 0);
+  main1 (1, 1);
+  main1 (10, 1);
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail { vect_no_int_add } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-2.c b/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-2.c
new file mode 100644
index 0000000..126a50f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-2.c
@@ -0,0 +1,62 @@
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target vect_condition } */
+/* { dg-additional-options "-fdump-tree-ifcvt-details" } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+/* PR tree-optimization/119920 */
+
+#define N 32
+
+unsigned int ub[N];
+unsigned int ua[N];
+
+/* Test vectorization of reduction of unsigned-int.  */
+
+__attribute__ ((noinline, noipa))
+void init(void)
+{
+  #pragma GCC novector
+  for(int i = 0;i < N; i++) {
+    ub[i] = i;
+    ua[i] = 1;
+  }
+}
+
+
+__attribute__ ((noinline, noipa))
+void main1 (unsigned int b, unsigned int c)
+{
+  int i;
+  unsigned int usum = 0;
+
+  init();
+
+  /* Summation.  */
+  for (i = 0; i < N; i++) {
+    unsigned t = ua[i];
+    if ( ub[i] < N/2 )
+      usum += b * t;
+    else
+      usum += c * t;
+  }
+
+  /* check results:  */
+  /* __builtin_printf("%d : %d\n", usum, (N/2*b*1 + N/2*c*1)); */
+  if (usum != N/2*b + N/2*c)
+    abort ();
+}
+
+int main (void)
+{ 
+  check_vect ();
+  
+  main1 (0, 0);
+  main1 (1, 1);
+  main1 (10, 1);
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail { vect_no_int_add } } } } */
+/* { dg-final { scan-tree-dump-times "changed to factor operation out from COND_EXPR" 2 "ifcvt" } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-3.c b/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-3.c
new file mode 100644
index 0000000..e425869
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-reduc-cond-3.c
@@ -0,0 +1,56 @@
+/* { dg-require-effective-target vect_int } */
+
+#include <stdarg.h>
+#include "tree-vect.h"
+
+/* PR tree-optimization/112324 */
+/* PR tree-optimization/110015 */
+
+#define N 32
+
+int ub[N];
+
+/* Test vectorization of reduction of int max with some extra code involed.  */
+
+__attribute__ ((noinline, noipa))
+void init(void)
+{
+  #pragma GCC novector
+  for(int i = 0;i < N; i++)
+    ub[i] = (i&4) && (i&1) ? -i : i;
+}
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b)) 
+
+__attribute__ ((noinline, noipa))
+void main1 (void)
+{
+  int i;
+  int max = 0;
+
+  init();
+
+  /* Summation.  */
+  for (i = 0; i < N; i++) {
+    int tmp = ub[i];
+    if (tmp < 0)
+      max = MAX (-tmp, max);
+    else
+      max = MAX (tmp, max);
+  }
+
+  /* check results:  */
+  /* __builtin_printf("%d : %d\n", max, N); */
+  if (max != N - 1)
+    abort ();
+}
+
+int main (void)
+{ 
+  check_vect ();
+  
+  main1 ();
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail { vect_no_int_min_max } } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-simd-pr121130.c b/gcc/testsuite/gcc.dg/vect/vect-simd-pr121130.c
new file mode 100644
index 0000000..c882ded
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-simd-pr121130.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+
+int n2;
+
+__attribute__((simd)) char
+w7(void)
+{
+  short int xb = n2;
+    xb = w7() < 1;
+  return xb;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/asm-hard-reg-1.c b/gcc/testsuite/gcc.target/aarch64/asm-hard-reg-1.c
new file mode 100644
index 0000000..7441dd7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/asm-hard-reg-1.c
@@ -0,0 +1,55 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+/* Ensure that we error out in case no hard regs are available for an operand
+   with constraint y.  The position/order of the y-constrained operand does not
+   matter.  */
+
+void
+test (void)
+{
+  int x, a, b, c, d, e, f, g, h;
+
+  __asm__ __volatile__ ("" :
+      "={v0}" (a),
+      "={v1}" (b),
+      "={v2}" (c),
+      "={v3}" (d),
+      "={v4}" (e),
+      "={v5}" (f),
+      "={v6}" (g),
+      "={v7}" (h));
+
+  __asm__ __volatile__ ("" : /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+      "=y" (x),
+      "={v0}" (a),
+      "={v1}" (b),
+      "={v2}" (c),
+      "={v3}" (d),
+      "={v4}" (e),
+      "={v5}" (f),
+      "={v6}" (g),
+      "={v7}" (h));
+
+  __asm__ __volatile__ ("" : /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+      "={v0}" (a),
+      "={v1}" (b),
+      "={v2}" (c),
+      "={v3}" (d),
+      "=y" (x),
+      "={v4}" (e),
+      "={v5}" (f),
+      "={v6}" (g),
+      "={v7}" (h));
+
+  __asm__ __volatile__ ("" : /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+      "={v0}" (a),
+      "={v1}" (b),
+      "={v2}" (c),
+      "={v3}" (d),
+      "={v4}" (e),
+      "={v5}" (f),
+      "={v6}" (g),
+      "={v7}" (h),
+      "=y" (x));
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/asm-hard-reg-2.c b/gcc/testsuite/gcc.target/aarch64/asm-hard-reg-2.c
new file mode 100644
index 0000000..7434063
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/asm-hard-reg-2.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=armv8-a+sve" } */
+
+/* Test register pairs.  */
+
+#include <arm_sve.h>
+
+void
+test (void)
+{
+  svuint32x2_t x, y;
+  svuint32x4_t z;
+
+  __asm__ __volatile__ ("" : "={z4}" (x), "={z6}" (y));
+  __asm__ __volatile__ ("" : "={z5}" (x), "={z6}" (y));  /* { dg-error "multiple outputs to hard register: v6" } */
+  __asm__ __volatile__ ("" : "={z4}" (z), "={z6}" (y));  /* { dg-error "multiple outputs to hard register: v6" } */
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/avoid-store-forwarding-be.c b/gcc/testsuite/gcc.target/aarch64/avoid-store-forwarding-be.c
new file mode 100644
index 0000000..2e8946b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/avoid-store-forwarding-be.c
@@ -0,0 +1,23 @@
+/* { dg-do run } */
+/* { dg-require-effective-target aarch64_big_endian } */
+/* { dg-options "-O2 -favoid-store-forwarding" } */
+
+typedef union {
+    char arr[2];
+    short value;
+} DataUnion;
+
+short __attribute__ ((noinline))
+ssll (DataUnion *data, char x, char y)
+{
+  data->arr[0] = x;
+  data->arr[1] = y;
+  return data->value;
+}
+
+int main () {
+  DataUnion data = {};
+  short value = ssll (&data, 0, 1);
+  if (value != 1)
+    __builtin_abort ();
+}
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/aarch64/cmpbr.c b/gcc/testsuite/gcc.target/aarch64/cmpbr.c
index a86af9d..34630f9 100644
--- a/gcc/testsuite/gcc.target/aarch64/cmpbr.c
+++ b/gcc/testsuite/gcc.target/aarch64/cmpbr.c
@@ -121,7 +121,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u8_x0_eq_x1:
-**	cbbeq	w1, w0, .L([0-9]+)
+**	cbbeq	(?:w1, w0|w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -129,7 +129,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u8_x0_ne_x1:
-**	cbbne	w1, w0, .L([0-9]+)
+**	cbbne	(?:w1, w0|w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -137,7 +137,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u8_x0_ult_x1:
-**	cbbhi	w1, w0, .L([0-9]+)
+**	(?:cbbhi	w1, w0|cbblo	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -145,7 +145,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u8_x0_ule_x1:
-**	cbbhs	w1, w0, .L([0-9]+)
+**	(?:cbbhs	w1, w0|cbbls	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -153,7 +153,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u8_x0_ugt_x1:
-**	cbblo	w1, w0, .L([0-9]+)
+**	(?:cbblo	w1, w0|cbbhi	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -161,7 +161,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u8_x0_uge_x1:
-**	cbbls	w1, w0, .L([0-9]+)
+**	(?:cbbls	w1, w0|cbbhs	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -169,7 +169,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i8_x0_slt_x1:
-**	cbbgt	w1, w0, .L([0-9]+)
+**	(?:cbbgt	w1, w0|cbblt	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -177,7 +177,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i8_x0_sle_x1:
-**	cbbge	w1, w0, .L([0-9]+)
+**	(?:cbbge	w1, w0|cbble	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -185,7 +185,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i8_x0_sgt_x1:
-**	cbblt	w1, w0, .L([0-9]+)
+**	(?:cbblt	w1, w0|cbbgt	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -193,7 +193,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i8_x0_sge_x1:
-**	cbble	w1, w0, .L([0-9]+)
+**	(?:cbble	w1, w0|cbbge	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -201,7 +201,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u16_x0_eq_x1:
-**	cbheq	w1, w0, .L([0-9]+)
+**	cbheq	(?:w1, w0|w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -209,7 +209,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u16_x0_ne_x1:
-**	cbhne	w0|w1, w1|w0, .L([0-9]+)
+**	cbhne	(?:w1, w0|w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -217,7 +217,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u16_x0_ult_x1:
-**	cbhhi	w1, w0, .L([0-9]+)
+**	(?:cbhhi	w1, w0|cbhlo	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -225,7 +225,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u16_x0_ule_x1:
-**	cbhhs	w1, w0, .L([0-9]+)
+**	(?:cbhhs	w1, w0|cbhls	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -233,7 +233,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u16_x0_ugt_x1:
-**	cbhlo	w1, w0, .L([0-9]+)
+**	(?:cbhlo	w1, w0|cbhhi	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -241,7 +241,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** u16_x0_uge_x1:
-**	cbhls	w1, w0, .L([0-9]+)
+**	(?:cbhls	w1, w0|cbhhs	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -249,7 +249,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i16_x0_slt_x1:
-**	cbhgt	w1, w0, .L([0-9]+)
+**	(?:cbhgt	w1, w0|cbhlt	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -257,7 +257,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i16_x0_sle_x1:
-**	cbhge	w1, w0, .L([0-9]+)
+**	(?:cbhge	w1, w0|cbhle	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -265,7 +265,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i16_x0_sgt_x1:
-**	cbhlt	w1, w0, .L([0-9]+)
+**	(?:cbhlt	w1, w0|cbhgt	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
@@ -273,7 +273,7 @@ FAR_BRANCH(u64, 42);
 
 /*
 ** i16_x0_sge_x1:
-**	cbhle	w1, w0, .L([0-9]+)
+**	(?:cbhle	w1, w0|cbhge	w0, w1), .L([0-9]+)
 **	b	not_taken
 ** .L\1:
 **	b	taken
diff --git a/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-0.c b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-0.c
new file mode 100644
index 0000000..e544b04f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-0.c
@@ -0,0 +1,12 @@
+/* { dg-do run }  */
+/* { dg-require-ifunc "" } */
+/* { dg-require-effective-target mmap } */
+/* { dg-options "-Wno-experimental-fmv-target" } */
+
+#include <stdint.h>
+
+typedef struct {
+  uint64_t size;
+} ifunc_arg_t;
+
+#include "ifunc-resolver.in"
diff --git a/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-1.c b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-1.c
new file mode 100644
index 0000000..be70687
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-1.c
@@ -0,0 +1,13 @@
+/* { dg-do run }  */
+/* { dg-require-ifunc "" } */
+/* { dg-require-effective-target mmap } */
+/* { dg-options "-Wno-experimental-fmv-target" } */
+
+#include <stdint.h>
+
+typedef struct {
+  uint64_t size;
+  uint64_t hwcap;
+} ifunc_arg_t;
+
+#include "ifunc-resolver.in"
diff --git a/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-2.c b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-2.c
new file mode 100644
index 0000000..bf594d0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-2.c
@@ -0,0 +1,14 @@
+/* { dg-do run }  */
+/* { dg-require-ifunc "" } */
+/* { dg-require-effective-target mmap } */
+/* { dg-options "-Wno-experimental-fmv-target" } */
+
+#include <stdint.h>
+
+typedef struct {
+  uint64_t size;
+  uint64_t hwcap;
+  uint64_t hwcap2;
+} ifunc_arg_t;
+
+#include "ifunc-resolver.in"
diff --git a/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-3.c b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-3.c
new file mode 100644
index 0000000..f16d01b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-3.c
@@ -0,0 +1,15 @@
+/* { dg-do run }  */
+/* { dg-require-ifunc "" } */
+/* { dg-require-effective-target mmap } */
+/* { dg-options "-Wno-experimental-fmv-target" } */
+
+#include <stdint.h>
+
+typedef struct {
+  uint64_t size;
+  uint64_t hwcap;
+  uint64_t hwcap2;
+  uint64_t hwcap3;
+} ifunc_arg_t;
+
+#include "ifunc-resolver.in"
diff --git a/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-4.c b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-4.c
new file mode 100644
index 0000000..1b4ccbd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver-4.c
@@ -0,0 +1,16 @@
+/* { dg-do run }  */
+/* { dg-require-ifunc "" } */
+/* { dg-require-effective-target mmap } */
+/* { dg-options "-Wno-experimental-fmv-target" } */
+
+#include <stdint.h>
+
+typedef struct {
+  uint64_t size;
+  uint64_t hwcap;
+  uint64_t hwcap2;
+  uint64_t hwcap3;
+  uint64_t hwcap4;
+} ifunc_arg_t;
+
+#include "ifunc-resolver.in"
diff --git a/gcc/testsuite/gcc.target/aarch64/ifunc-resolver.in b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver.in
new file mode 100644
index 0000000..ada0b33
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ifunc-resolver.in
@@ -0,0 +1,48 @@
+#include <unistd.h>
+#include <string.h>
+#include <sys/mman.h>
+
+/* Allocate memory buffer of size LEN with a protected page
+   following right after the buffer end so that any memory
+   accesses past the end of the buffer would trigger SEGFAUL.  */
+void *allocate_mem (size_t len)
+{
+  size_t pagesize = sysconf (_SC_PAGESIZE);
+  char *m = mmap (NULL, pagesize * 2,
+		  PROT_READ | PROT_WRITE,
+		  MAP_PRIVATE | MAP_ANONYMOUS,
+		  -1, 0);
+  mprotect (m + pagesize, pagesize, PROT_NONE);
+  m = m + pagesize - len;
+  memset(m, 0, len);
+  return m;
+}
+
+int impl ()
+{
+  return 0;
+}
+
+#ifndef _IFUNC_ARG_HWCAP
+#define _IFUNC_ARG_HWCAP (1ULL << 62)
+#endif
+
+void
+__init_cpu_features_resolver (unsigned long hwcap, const void *arg);
+
+static void *
+fun_resolver (uint64_t a0, const uint64_t *a1)
+{
+  ifunc_arg_t *arg = allocate_mem (sizeof (ifunc_arg_t));
+  arg->size = sizeof (ifunc_arg_t);
+  /* Call this function with synthetic ifunc_arg_t arg.  */
+  __init_cpu_features_resolver (_IFUNC_ARG_HWCAP, arg);
+  return (void *)(uintptr_t)impl;
+}
+
+int fun (void) __attribute__ ((ifunc ("fun_resolver")));
+
+int main (int argc, char *argv[])
+{
+  return fun ();
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/inszero_split_1.c b/gcc/testsuite/gcc.target/aarch64/inszero_split_1.c
new file mode 100644
index 0000000..5c739bd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/inszero_split_1.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/* Avoid INS from WZR register when optimizing for speed.  */
+
+#include <arm_neon.h>
+
+/*
+** foo:
+**	movi?	[vdz]([0-9]+)\.?(?:[0-9]*[bhsd])?, #?0
+**	ins	v0.h\[2\], v(\1).h\[0\]
+**	ret
+*/
+uint16x8_t foo(uint16x8_t a) {
+  a[2] = 0;
+  return a;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/ldapr-sext.c b/gcc/testsuite/gcc.target/aarch64/ldapr-sext.c
index f57c09d..e8a545a 100644
--- a/gcc/testsuite/gcc.target/aarch64/ldapr-sext.c
+++ b/gcc/testsuite/gcc.target/aarch64/ldapr-sext.c
@@ -33,7 +33,7 @@ TEST(s8_s64, s8, long long)
 /*
 **test_s16_s64:
 **...
-**	ldapursh	x0, \[x[0-9]+\]
+**	ldapursh	x0, \[x[0-9]+, [0-9]+\]
 **	ret
 */
 
@@ -42,7 +42,7 @@ TEST(s16_s64, s16, long long)
 /*
 **test_s32_s64:
 **...
-**	ldapursw	x0, \[x[0-9]+\]
+**	ldapursw	x0, \[x[0-9]+, [0-9]+\]
 **	ret
 */
 
@@ -60,7 +60,7 @@ TEST(s8_s32, s8, int)
 /*
 **test_s16_s32:
 **...
-**	ldapursh	w0, \[x[0-9]+\]
+**	ldapursh	w0, \[x[0-9]+, [0-9]+\]
 **	ret
 */
 
diff --git a/gcc/testsuite/gcc.target/aarch64/ldapur.c b/gcc/testsuite/gcc.target/aarch64/ldapur.c
new file mode 100644
index 0000000..5c68bdd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ldapur.c
@@ -0,0 +1,77 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -std=c99" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <stdatomic.h>
+#include <stdint.h>
+
+#pragma GCC target "arch=armv8.8-a"
+
+atomic_ullong u64;
+atomic_uint u32;
+atomic_ushort u16;
+atomic_uchar u8[2]; /* Force an offset for u8 */
+
+#define TEST(name, ldsize, rettype)				\
+rettype								\
+test_##name (void)						\
+{								\
+  return atomic_load_explicit (&ldsize, memory_order_acquire);	\
+}								\
+
+
+/*
+** test_u8_u64:
+**	...
+**	ldapurb	w[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u8_u64, u8[1], uint64_t)
+
+/*
+** test_u16_u64:
+**	...
+**	ldapurh	w[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u16_u64, u16, uint64_t)
+
+/*
+**test_u32_u64:
+**	...
+**	ldapur	w[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u32_u64, u32, uint64_t)
+
+/*
+**test_u64_u64:
+**	...
+**	ldapur	x[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u64_u64, u64, uint64_t)
+
+/*
+**test_u8_u32:
+**	...
+**	ldapurb	w[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u8_u32, u8[1], uint32_t)
+
+/*
+**test_u16_u32:
+**	...
+**	ldapurh	w[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u16_u32, u16, uint32_t)
+
+/*
+**test_u32_u32:
+**	...
+**	ldapur	w[0-9]+, \[x[0-9]+, [0-9]+\]
+**	ret
+*/
+TEST(u32_u32, u32, uint32_t)
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/aarch64/ldapur_avoid.c b/gcc/testsuite/gcc.target/aarch64/ldapur_avoid.c
new file mode 100644
index 0000000..ad87a30
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/ldapur_avoid.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -std=c99 -moverride=tune=avoid_ldapur" } */
+
+#include <stdatomic.h>
+#include <stdint.h>
+
+#pragma GCC target "arch=armv8.8-a"
+/* LDAPUR is only avoided for armv8.4 to armv8.7. This checks for the working
+of avoid_ldapur flag. */
+
+/* { dg-final { scan-assembler-not "ldapur\t" } } */
+
+atomic_ullong u64;
+atomic_uint u32;
+atomic_ushort u16;
+atomic_uchar u8[2]; /* Force an offset for u8 */
+
+#define TEST(name, ldsize, rettype)				\
+rettype								\
+test_##name (void)						\
+{								\
+  return atomic_load_explicit (&ldsize, memory_order_acquire);	\
+}								\
+
+TEST(u8_u64, u8[1], uint64_t)
+TEST(u16_u64, u16, uint64_t)
+TEST(u32_u64, u32, uint64_t)
+TEST(u64_u64, u64, uint64_t)
+TEST(u8_u32, u8[1], uint32_t)
+TEST(u16_u32, u16, uint32_t)
+TEST(u32_u32, u32, uint32_t)
+
+/* { dg-final { scan-assembler-times "ldapr\t" 3 } } */
+/* { dg-final { scan-assembler-times "ldaprh\t" 2 } } */
+/* { dg-final { scan-assembler-times "ldaprb\t" 2 } } */
+
+
diff --git a/gcc/testsuite/gcc.target/aarch64/popcnt13.c b/gcc/testsuite/gcc.target/aarch64/popcnt13.c
new file mode 100644
index 0000000..2a30e98
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/popcnt13.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#pragma GCC target "+nocssc+sve"
+
+/*
+** h128:
+**	ldr	q([0-9]+), \[x0\]
+**	ptrue	p([0-9]+).b, vl16
+**	cnt	z([0-9]+).d, p\2/m, z\1.d
+**	addp	d([0-9]+), v\3.2d
+**	fmov	x0, d\4
+**	ret
+*/
+
+unsigned h128 (const unsigned __int128 *a) {
+	  return __builtin_popcountg (a[0]);
+}
+
+/* There should be only one POPCOUNT. */
+/* { dg-final { scan-tree-dump-times "POPCOUNT " 1 "optimized" } } */
+/* { dg-final { scan-tree-dump-not " __builtin_popcount"  "optimized" } } */
+
diff --git a/gcc/testsuite/gcc.target/aarch64/popcnt9.c b/gcc/testsuite/gcc.target/aarch64/popcnt9.c
index c778fc7..cfed8c5 100644
--- a/gcc/testsuite/gcc.target/aarch64/popcnt9.c
+++ b/gcc/testsuite/gcc.target/aarch64/popcnt9.c
@@ -3,7 +3,7 @@
 /* { dg-final { check-function-bodies "**" "" } } */
 /* PR target/113042 */
 
-#pragma GCC target "+nocssc"
+#pragma GCC target "+nocssc+nosve"
 
 /*
 ** h128:
diff --git a/gcc/testsuite/gcc.target/aarch64/pr118348_1.c b/gcc/testsuite/gcc.target/aarch64/pr118348_1.c
index 75f6dad..2715dcb 100644
--- a/gcc/testsuite/gcc.target/aarch64/pr118348_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/pr118348_1.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target aarch64_sve128_hw } } */
+/* { dg-do run { target { aarch64_sve128_hw && fstack_protector } } } */
 /* { dg-options "-O2 -fopenmp-simd -fno-trapping-math -msve-vector-bits=128 --param aarch64-autovec-preference=sve-only -fstack-protector-strong" } */
 
 #pragma GCC target "+sve"
diff --git a/gcc/testsuite/gcc.target/aarch64/pr118348_2.c b/gcc/testsuite/gcc.target/aarch64/pr118348_2.c
index 2e20004..4ce8d20 100644
--- a/gcc/testsuite/gcc.target/aarch64/pr118348_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/pr118348_2.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target aarch64_sve256_hw } } */
+/* { dg-do run { target { aarch64_sve256_hw && fstack_protector } } } */
 /* { dg-options "-O2 -fopenmp-simd -fno-trapping-math -msve-vector-bits=256 --param aarch64-autovec-preference=sve-only -fstack-protector-strong" } */
 
 #include "pr118348_1.c"
diff --git a/gcc/testsuite/gcc.target/aarch64/pr121300.c b/gcc/testsuite/gcc.target/aarch64/pr121300.c
new file mode 100644
index 0000000..5f2cd9a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/pr121300.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-S -O3 -march=armv9-a+sme2" } */
+
+#include <arm_sme.h>
+
+svfloat16x2_t test (svfloat16x2_t zd, svfloat16x2_t zm) __arm_streaming
+{
+    return svamin_f16_x2 (zd, zm); // { dg-error "ACLE function .svamin_f16_x2. requires ISA extension .faminmax." }
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c
index acd2e11..8fc1569 100644
--- a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_1.c
@@ -4,24 +4,24 @@
 
 /*
 ** uadd:
-**	dup	v([0-9]+).8b, w1
-**	dup	v([0-9]+).8b, w0
+**	dup	v([0-9]+).8b, w[01]
+**	dup	v([0-9]+).8b, w[01]
 **	uqadd	b([0-9]+), (?:b\2, b\1|b\1, b\2)
 **	umov	w0, v\3.b\[0\]
 **	ret
 */
 /*
 ** uadd2:
-**	dup	v([0-9]+).8b, w1
-**	dup	v([0-9]+).8b, w0
+**	dup	v([0-9]+).8b, w[01]
+**	dup	v([0-9]+).8b, w[01]
 **	uqadd	b([0-9]+), (?:b\2, b\1|b\1, b\2)
 **	umov	w0, v\3.b\[0\]
 **	ret
 */
 /*
 ** usub: { xfail *-*-* }
-**	dup	v([0-9]+).8b, w1
-**	dup	v([0-9]+).8b, w0
+**	dup	v([0-9]+).8b, w[01]
+**	dup	v([0-9]+).8b, w[01]
 **	uqsub	b([0-9]+), b\1, b\2
 **	umov	w0, v\3.b\[0\]
 **	ret
diff --git a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c
index 86c88f8..dd0fefa 100644
--- a/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/saturating_arithmetic_2.c
@@ -4,16 +4,16 @@
 
 /*
 ** uadd:
-**	dup	v([0-9]+).4h, w1
-**	dup	v([0-9]+).4h, w0
+**	dup	v([0-9]+).4h, w[01]
+**	dup	v([0-9]+).4h, w[01]
 **	uqadd	h([0-9]+), (?:h\2, h\1|h\1, h\2)
 **	umov	w0, v\3.h\[0\]
 **	ret
 */
 /*
 ** uadd2:
-**	dup	v([0-9]+).4h, w1
-**	dup	v([0-9]+).4h, w0
+**	dup	v([0-9]+).4h, w[01]
+**	dup	v([0-9]+).4h, w[01]
 **	uqadd	h([0-9]+), (?:h\2, h\1|h\1, h\2)
 **	umov	w0, v\3.h\[0\]
 **	ret
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/bcax_d.c b/gcc/testsuite/gcc.target/aarch64/simd/bcax_d.c
new file mode 100644
index 0000000..a7640c3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/bcax_d.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_neon.h>
+
+#pragma GCC target "+sha3"
+
+#define BCAX(x,y,z)  ((x) ^ ((y) & ~(z)))
+
+/* When the inputs come from GP regs don't form a BCAX.  */
+uint64_t bcax_d_gp (uint64_t a, uint64_t b, uint64_t c) { return BCAX (a, b, c); }
+
+uint64x1_t bcax_d (uint64x1_t a, uint64x1_t b, uint64x1_t c) { return BCAX (a, b, c); }
+uint32x2_t bcax_s (uint32x2_t a, uint32x2_t b, uint32x2_t c) { return BCAX (a, b, c); }
+uint16x4_t bcax_h (uint16x4_t a, uint16x4_t b, uint16x4_t c) { return BCAX (a, b, c); }
+uint8x8_t bcax_b (uint8x8_t a, uint8x8_t b, uint8x8_t c) { return BCAX (a, b, c); }
+
+/* { dg-final { scan-assembler-times {bcax\tv0.16b, v0.16b, v1.16b, v2.16b} 4 } } */
+
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/eor3_d.c b/gcc/testsuite/gcc.target/aarch64/simd/eor3_d.c
new file mode 100644
index 0000000..7f2b2b4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/eor3_d.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+#include <arm_neon.h>
+
+#pragma GCC target "+sha3"
+
+#define EOR3(x,y,z)  ((x) ^ (y) ^ (z))
+
+uint32x2_t bcax_s (uint32x2_t a, uint32x2_t b, uint32x2_t c) { return EOR3 (a, b, c); }
+uint16x4_t bcax_h (uint16x4_t a, uint16x4_t b, uint16x4_t c) { return EOR3 (a, b, c); }
+uint8x8_t bcax_b (uint8x8_t a, uint8x8_t b, uint8x8_t c) { return EOR3 (a, b, c); }
+
+/* { dg-final { scan-assembler-times {eor3\tv0.16b, v[0-9]+.16b, v[0-9]+.16b, v[0-9]+.16b} 3 } } */
+
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_1.c b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_1.c
new file mode 100644
index 0000000..f082198
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_1.c
@@ -0,0 +1,716 @@
+/* { dg-do compile } */
+/* { dg-options "-O -march=armv9-a+bf16" } */
+
+#include <arm_neon.h>
+
+/* We should use the highpart instruction where doing so would avoid data
+   movement instructions.  This case, where all the arguments are non-constant
+   vector highparts, can be handled by either gimple_fold_builtin or combine.  */
+
+#ifndef TEST_UN_HIGHPARTS
+#define TEST_UN_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF) \
+  RETTYPE test_##FN##_##SUFF (INTYPE a)		     \
+  {                                                  \
+    return FN##_##SUFF (vget_high_##SUFF (a));	     \
+  }
+#endif
+
+#ifndef TEST_BIN_W_HIGHPARTS
+#define TEST_BIN_W_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)  \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, INTYPE b)	 \
+  {                                                      \
+    return FN##_##SUFF (a, vget_high_##SUFF (b));	 \
+  }
+#endif
+
+#ifndef TEST_BIN_N_HIGHPARTS
+#define TEST_BIN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)    \
+  RETTYPE test_##FN##_##SUFF (INTYPE a)			   \
+  {                                                        \
+    return FN##_##SUFF (vget_high_##SUFF (a), a[1]);	   \
+  }
+#endif
+
+#ifndef TEST_TERN_N_HIGHPARTS
+#define TEST_TERN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)      \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, INTYPE b)	      \
+  {                                                           \
+    return FN##_##SUFF (a, vget_high_##SUFF (b), b[1]);	      \
+  }
+#endif
+
+#ifndef TEST_BIN_HIGHPARTS
+#define TEST_BIN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF) \
+  RETTYPE test_##FN##_##SUFF (INTYPE a, INTYPE b)		\
+  {                                                             \
+    return FN##_##SUFF (vget_high_##SUFF (a),			\
+			vget_high_##SUFF (b));			\
+  }
+#endif
+
+#ifndef TEST_TERN_HIGHPARTS
+#define TEST_TERN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF)   \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, INTYPE b, INTYPE c)	   \
+  {                                                                \
+    return FN##_##SUFF(a, vget_high_##SUFF (b),			   \
+		       vget_high_##SUFF (c));			   \
+  }
+#endif
+
+#define TEST_UNOP(FN) \
+  TEST_UN_HIGHPARTS (FN, int16x8_t,  int8x16_t,  s8)  \
+  TEST_UN_HIGHPARTS (FN, uint16x8_t, uint8x16_t, u8)  \
+  TEST_UN_HIGHPARTS (FN, int32x4_t,  int16x8_t,  s16) \
+  TEST_UN_HIGHPARTS (FN, uint32x4_t, uint16x8_t, u16) \
+  TEST_UN_HIGHPARTS (FN, int64x2_t,  int32x4_t,  s32) \
+  TEST_UN_HIGHPARTS (FN, uint64x2_t, uint32x4_t, u32)
+
+#define TEST_BINOP(FN)						   \
+  TEST_BIN_HIGHPARTS (FN, int16x8_t,  int8x16_t,  int8x8_t,   s8)  \
+  TEST_BIN_HIGHPARTS (FN, uint16x8_t, uint8x16_t, uint8x8_t,  u8)  \
+  TEST_BIN_HIGHPARTS (FN, int32x4_t,  int16x8_t,  int16x4_t,  s16) \
+  TEST_BIN_HIGHPARTS (FN, uint32x4_t, uint16x8_t, uint16x4_t, u16) \
+  TEST_BIN_HIGHPARTS (FN, int64x2_t,  int32x4_t,  int32x2_t,  s32) \
+  TEST_BIN_HIGHPARTS (FN, uint64x2_t, uint32x4_t, uint32x2_t, u32)
+
+#define TEST_BINOP_N(FN)				 \
+  TEST_BIN_N_HIGHPARTS (FN, int32x4_t,  int16x8_t,  s16) \
+  TEST_BIN_N_HIGHPARTS (FN, uint32x4_t, uint16x8_t, u16) \
+  TEST_BIN_N_HIGHPARTS (FN, int64x2_t,  int32x4_t,  s32) \
+  TEST_BIN_N_HIGHPARTS (FN, uint64x2_t, uint32x4_t, u32)
+
+#define TEST_BINOP_W(FN)				 \
+  TEST_BIN_W_HIGHPARTS (FN, int16x8_t,  int8x16_t,   s8) \
+  TEST_BIN_W_HIGHPARTS (FN, uint16x8_t, uint8x16_t,  u8) \
+  TEST_BIN_W_HIGHPARTS (FN, int32x4_t,  int16x8_t,  s16) \
+  TEST_BIN_W_HIGHPARTS (FN, uint32x4_t, uint16x8_t, u16) \
+  TEST_BIN_W_HIGHPARTS (FN, int64x2_t,  int32x4_t,  s32) \
+  TEST_BIN_W_HIGHPARTS (FN, uint64x2_t, uint32x4_t, u32)
+
+#define TEST_TERNOP_N(FN)				  \
+  TEST_TERN_N_HIGHPARTS (FN, int32x4_t,  int16x8_t,  s16) \
+  TEST_TERN_N_HIGHPARTS (FN, uint32x4_t, uint16x8_t, u16) \
+  TEST_TERN_N_HIGHPARTS (FN, int64x2_t,  int32x4_t,  s32) \
+  TEST_TERN_N_HIGHPARTS (FN, uint64x2_t, uint32x4_t, u32)
+
+#define TEST_TERNOP(FN)                                           \
+  TEST_TERN_HIGHPARTS (FN, int16x8_t,  int8x16_t,  int8x8_t,   s8)  \
+  TEST_TERN_HIGHPARTS (FN, uint16x8_t, uint8x16_t, uint8x8_t,  u8)  \
+  TEST_TERN_HIGHPARTS (FN, int32x4_t,  int16x8_t,  int16x4_t,  s16) \
+  TEST_TERN_HIGHPARTS (FN, uint32x4_t, uint16x8_t, uint16x4_t, u16) \
+  TEST_TERN_HIGHPARTS (FN, int64x2_t,  int32x4_t,  int32x2_t,  s32) \
+  TEST_TERN_HIGHPARTS (FN, uint64x2_t, uint32x4_t, uint32x2_t, u32)
+
+#define TEST_VQDMULL                                                 \
+  TEST_BIN_HIGHPARTS (vqdmull, int32x4_t, int16x8_t, int16x4_t, s16) \
+  TEST_BIN_HIGHPARTS (vqdmull, int64x2_t, int32x4_t, int32x2_t, s32)
+
+#define TEST_VQDMULL_N                                        \
+  TEST_BIN_N_HIGHPARTS (vqdmull_n, int32x4_t, int16x8_t, s16) \
+  TEST_BIN_N_HIGHPARTS (vqdmull_n, int64x2_t, int32x4_t, s32)
+
+#define TEST_VQMLAL                                                   \
+  TEST_TERN_HIGHPARTS (vqdmlal, int32x4_t, int16x8_t, int16x4_t, s16) \
+  TEST_TERN_HIGHPARTS (vqdmlal, int64x2_t, int32x4_t, int32x2_t, s32)
+
+#define TEST_VQMLAL_N                                          \
+  TEST_TERN_N_HIGHPARTS (vqdmlal_n, int32x4_t, int16x8_t, s16) \
+  TEST_TERN_N_HIGHPARTS (vqdmlal_n, int64x2_t, int32x4_t, s32)
+
+#define TEST_VQMLSL                                                   \
+  TEST_TERN_HIGHPARTS (vqdmlsl, int32x4_t, int16x8_t, int16x4_t, s16) \
+  TEST_TERN_HIGHPARTS (vqdmlsl, int64x2_t, int32x4_t, int32x2_t, s32)
+
+#define TEST_VQMLSL_N                                          \
+  TEST_TERN_N_HIGHPARTS (vqdmlsl_n, int32x4_t, int16x8_t, s16) \
+  TEST_TERN_N_HIGHPARTS (vqdmlsl_n, int64x2_t, int32x4_t, s32)
+
+#define TEST_VMOVL \
+  TEST_UNOP (vmovl)
+
+#define TEST_VMULL \
+  TEST_BINOP (vmull) \
+  TEST_BIN_HIGHPARTS (vmull, poly16x8_t, poly8x16_t, poly8x8_t, p8)
+
+#define TEST_VMULL_N \
+  TEST_BINOP_N (vmull_n)
+
+#define TEST_VADDL \
+  TEST_BINOP (vaddl)
+
+#define TEST_VSUBL \
+  TEST_BINOP (vsubl)
+
+#define TEST_VMLAL \
+  TEST_TERNOP (vmlal)
+
+#define TEST_VMLAL_N \
+  TEST_TERNOP_N (vmlal_n)
+
+#define TEST_VMLSL \
+  TEST_TERNOP (vmlsl)
+
+#define TEST_VMLSL_N \
+  TEST_TERNOP_N (vmlsl_n)
+
+#define TEST_VABDL \
+  TEST_BINOP (vabdl)
+
+#define TEST_VABAL \
+  TEST_TERNOP (vabal)
+
+#define TEST_VSUBW \
+  TEST_BINOP_W (vsubw)
+
+#define TEST_VADDW \
+  TEST_BINOP_W (vaddw)
+
+/*
+** test_vmovl_s8:
+**	sxtl2	v0\.8h, v0\.16b
+**	ret
+*/
+
+/*
+** test_vmovl_u8:
+**	uxtl2	v0\.8h, v0\.16b
+**	ret
+*/
+
+/*
+** test_vmovl_s16:
+**	sxtl2	v0\.4s, v0\.8h
+**	ret
+*/
+
+/*
+** test_vmovl_u16:
+**	uxtl2	v0\.4s, v0\.8h
+**	ret
+*/
+
+/*
+** test_vmovl_s32:
+**	sxtl2	v0\.2d, v0\.4s
+**	ret
+*/
+
+/*
+** test_vmovl_u32:
+**	uxtl2	v0\.2d, v0\.4s
+**	ret
+*/
+
+TEST_VMOVL
+
+/*
+** test_vmull_s8:
+**	smull2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+/*
+** test_vmull_u8:
+**	umull2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+/*
+** test_vmull_s16:
+**	smull2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vmull_u16:
+**	umull2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vmull_s32:
+**	smull2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+/*
+** test_vmull_u32:
+**	umull2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+/*
+** test_vmull_p8:
+**	pmull2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+TEST_VMULL
+
+/*
+** test_vmull_n_s16:
+**	smull2	v0\.4s, v0\.8h, v0\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vmull_n_u16:
+**	umull2	v0\.4s, v0\.8h, v0\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vmull_n_s32:
+**	smull2	v0\.2d, v0\.4s, v0\.s\[[0-3]\]
+**	ret
+*/
+
+/*
+** test_vmull_n_u32:
+**	umull2	v0\.2d, v0\.4s, v0\.s\[[0-3]\]
+**	ret
+*/
+
+TEST_VMULL_N
+
+/*
+** test_vaddl_s8:
+**	saddl2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+/*
+** test_vaddl_u8:
+**	uaddl2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+/*
+** test_vaddl_s16:
+**	saddl2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vaddl_u16:
+**	uaddl2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vaddl_s32:
+**	saddl2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+/*
+** test_vaddl_u32:
+**	uaddl2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+TEST_VADDL
+
+/*
+** test_vsubl_s8:
+**	ssubl2	v0\.8h, v0\.16b, v1\.16b
+**	ret
+*/
+
+/*
+** test_vsubl_u8:
+**	usubl2	v0\.8h, v0\.16b, v1\.16b
+**	ret
+*/
+
+/*
+** test_vsubl_s16:
+**	ssubl2	v0\.4s, v0\.8h, v1\.8h
+**	ret
+*/
+
+/*
+** test_vsubl_u16:
+**	usubl2	v0\.4s, v0\.8h, v1\.8h
+**	ret
+*/
+
+/*
+** test_vsubl_s32:
+**	ssubl2	v0\.2d, v0\.4s, v1\.4s
+**	ret
+*/
+
+/*
+** test_vsubl_u32:
+**	usubl2	v0\.2d, v0\.4s, v1\.4s
+**	ret
+*/
+
+TEST_VSUBL
+
+/*
+** test_vabal_s8:
+**	sabal2	v0\.8h, (v1\.16b, v2\.16b|v2\.16b, v1\.16b)
+**	ret
+*/
+
+/*
+** test_vabal_u8:
+**	uabal2	v0\.8h, (v1\.16b, v2\.16b|v2\.16b, v1\.16b)
+**	ret
+*/
+
+/*
+** test_vabal_s16:
+**	sabal2	v0\.4s, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+**	ret
+*/
+
+/*
+** test_vabal_u16:
+**	uabal2	v0\.4s, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+**	ret
+*/
+
+/*
+** test_vabal_s32:
+**	sabal2	v0\.2d, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+**	ret
+*/
+
+/*
+** test_vabal_u32:
+**	uabal2	v0\.2d, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+**	ret
+*/
+
+TEST_VABAL
+
+/*
+** test_vsubw_s8:
+**	ssubw2	v0\.8h, v0\.8h, v1\.16b
+**	ret
+*/
+
+/*
+** test_vsubw_u8:
+**	usubw2	v0\.8h, v0\.8h, v1\.16b
+**	ret
+*/
+
+/*
+** test_vsubw_s16:
+**	ssubw2	v0\.4s, v0\.4s, v1\.8h
+**	ret
+*/
+
+/*
+** test_vsubw_u16:
+**	usubw2	v0\.4s, v0\.4s, v1\.8h
+**	ret
+*/
+
+/*
+** test_vsubw_s32:
+**	ssubw2	v0\.2d, v0\.2d, v1\.4s
+**	ret
+*/
+
+/*
+** test_vsubw_u32:
+**	usubw2	v0\.2d, v0\.2d, v1\.4s
+**	ret
+*/
+
+TEST_VSUBW
+
+/*
+** test_vaddw_s8:
+**	saddw2	v0\.8h, v0\.8h, v1\.16b
+**	ret
+*/
+
+/*
+** test_vaddw_u8:
+**	uaddw2	v0\.8h, v0\.8h, v1\.16b
+**	ret
+*/
+
+/*
+** test_vaddw_s16:
+**	saddw2	v0\.4s, v0\.4s, v1\.8h
+**	ret
+*/
+
+/*
+** test_vaddw_u16:
+**	uaddw2	v0\.4s, v0\.4s, v1\.8h
+**	ret
+*/
+
+/*
+** test_vaddw_s32:
+**	saddw2	v0\.2d, v0\.2d, v1\.4s
+**	ret
+*/
+
+/*
+** test_vaddw_u32:
+**	uaddw2	v0\.2d, v0\.2d, v1\.4s
+**	ret
+*/
+
+TEST_VADDW
+
+/*
+** test_vabdl_s8:
+**	sabdl2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+/*
+** test_vabdl_u8:
+**	uabdl2	v0\.8h, (v0\.16b, v1\.16b|v1\.16b, v0\.16b)
+**	ret
+*/
+
+/*
+** test_vabdl_s16:
+**	sabdl2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vabdl_u16:
+**	uabdl2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vabdl_s32:
+**	sabdl2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+/*
+** test_vabdl_u32:
+**	uabdl2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+TEST_VABDL
+
+/*
+** test_vmlal_s8:
+**	smlal2	v0\.8h, (v1\.16b, v2\.16b|v2\.16b, v1\.16b)
+**	ret
+*/
+
+/*
+** test_vmlal_u8:
+**	umlal2	v0\.8h, (v1\.16b, v2\.16b|v2\.16b, v1\.16b)
+**	ret
+*/
+
+/*
+** test_vmlal_s16:
+**	smlal2	v0\.4s, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+**	ret
+*/
+
+/*
+** test_vmlal_u16:
+**	umlal2	v0\.4s, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+**	ret
+*/
+
+/*
+** test_vmlal_s32:
+**	smlal2	v0\.2d, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+**	ret
+*/
+
+/*
+** test_vmlal_u32:
+**	umlal2	v0\.2d, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+**	ret
+*/
+
+TEST_VMLAL
+
+/*
+** test_vmlal_n_s16:
+**	smlal2	v0\.4s, v1\.8h, v1\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vmlal_n_u16:
+**	umlal2	v0\.4s, v1\.8h, v1\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vmlal_n_s32:
+**	smlal2	v0\.2d, v1\.4s, v1\.s\[[0-3]\]
+**	ret
+*/
+
+/*
+** test_vmlal_n_u32:
+**	umlal2	v0\.2d, v1\.4s, v1\.s\[[0-3]\]
+**	ret
+*/
+
+TEST_VMLAL_N
+
+/*
+** test_vmlsl_s8:
+**	smlsl2	v0\.8h, v1\.16b, v2\.16b
+**	ret
+*/
+
+/*
+** test_vmlsl_u8:
+**	umlsl2	v0\.8h, v1\.16b, v2\.16b
+**	ret
+*/
+
+/*
+** test_vmlsl_s16:
+**	smlsl2	v0\.4s, v1\.8h, v2\.8h
+**	ret
+*/
+
+/*
+** test_vmlsl_u16:
+**	umlsl2	v0\.4s, v1\.8h, v2\.8h
+**	ret
+*/
+
+/*
+** test_vmlsl_s32:
+**	smlsl2	v0\.2d, v1\.4s, v2\.4s
+**	ret
+*/
+
+/*
+** test_vmlsl_u32:
+**	umlsl2	v0\.2d, v1\.4s, v2\.4s
+**	ret
+*/
+
+TEST_VMLSL
+
+/*
+** test_vmlsl_n_s16:
+**	smlsl2	v0\.4s, v1\.8h, v1\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vmlsl_n_u16:
+**	umlsl2	v0\.4s, v1\.8h, v1\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vmlsl_n_s32:
+**	smlsl2	v0\.2d, v1\.4s, v1\.s\[[0-3]\]
+**	ret
+*/
+
+/*
+** test_vmlsl_n_u32:
+**	umlsl2	v0\.2d, v1\.4s, v1\.s\[[0-3]\]
+**	ret
+*/
+
+TEST_VMLSL_N
+
+/*
+** test_vqdmull_s16:
+**	sqdmull2	v0\.4s, (v0\.8h, v1\.8h|v1\.8h, v0\.8h)
+**	ret
+*/
+
+/*
+** test_vqdmull_s32:
+**	sqdmull2	v0\.2d, (v0\.4s, v1\.4s|v1\.4s, v0\.4s)
+**	ret
+*/
+
+TEST_VQDMULL
+
+/*
+** test_vqdmull_n_s16:
+**	sqdmull2	v0\.4s, v0\.8h, v0\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vqdmull_n_s32:
+**	sqdmull2	v0\.2d, v0\.4s, v0\.s\[[0-3]\]
+**	ret
+*/
+
+TEST_VQDMULL_N
+
+/*
+** test_vqdmlal_s16:
+**	sqdmlal2	v0\.4s, (v1\.8h, v2\.8h|v2\.8h, v1\.8h)
+**	ret
+*/
+
+/*
+** test_vqdmlal_s32:
+**	sqdmlal2	v0\.2d, (v1\.4s, v2\.4s|v2\.4s, v1\.4s)
+**	ret
+*/
+
+TEST_VQMLAL
+
+/*
+** test_vqdmlal_n_s16:
+**	sqdmlal2	v0\.4s, v1\.8h, v1\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vqdmlal_n_s32:
+**	sqdmlal2	v0\.2d, v1\.4s, v1\.s\[[0-3]\]
+**	ret
+*/
+
+TEST_VQMLAL_N
+
+/*
+** test_vqdmlsl_s16:
+**	sqdmlsl2	v0\.4s, v1\.8h, v2\.8h
+**	ret
+*/
+
+/*
+** test_vqdmlsl_s32:
+**	sqdmlsl2	v0\.2d, v1\.4s, v2\.4s
+**	ret
+*/
+
+TEST_VQMLSL
+
+/*
+** test_vqdmlsl_n_s16:
+**	sqdmlsl2	v0\.4s, v1\.8h, v1\.h\[[0-7]\]
+**	ret
+*/
+
+/*
+** test_vqdmlsl_n_s32:
+**	sqdmlsl2	v0\.2d, v1\.4s, v1\.s\[[0-3]\]
+**	ret
+*/
+
+TEST_VQMLSL_N
+
+/* { dg-final { check-function-bodies "**" ""} } */
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_2.c b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_2.c
new file mode 100644
index 0000000..5885b28
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_2.c
@@ -0,0 +1,88 @@
+/* { dg-do compile } */
+/* { dg-options "-O -march=armv9-a+bf16" } */
+
+/* We should not use the highpart instruction unless doing so would avoid
+   data movement instructions.  That is, unless at least one argument is a
+   reference to the highpart of a non-constant vector.  */
+
+#define TEST_UN_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF) \
+  RETTYPE test_##FN##_## SUFF ()		     \
+  {                                                  \
+    INTYPE a = vdupq_n_##SUFF (0x1A);		     \
+    return FN##_##SUFF (vget_high_##SUFF (a));	     \
+  }
+
+#define TEST_BIN_W_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF) \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a)		\
+  {                                                     \
+    INTYPE b = vdupq_n_##SUFF (0x1A);			\
+    return FN##_##SUFF (a, vget_high_##SUFF (b));	\
+  }
+
+#define TEST_BIN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)     \
+  RETTYPE test_##FN##_##SUFF (INTYPE c)			    \
+  {							    \
+    INTYPE a = vdupq_n_##SUFF (0x1A);			    \
+    return FN##_##SUFF (vget_high_##SUFF (a), c[1]);	    \
+  }
+
+#define TEST_TERN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)      \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a)		      \
+  {                                                           \
+    INTYPE b = vdupq_n_##SUFF (0x1A);			      \
+    return FN##_##SUFF (a, vget_high_##SUFF (b), b[1]);      \
+  }
+
+#define TEST_BIN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF) \
+  RETTYPE test_##FN##_## SUFF (H_INTYPE b)			\
+  {                                                             \
+    INTYPE a = vdupq_n_##SUFF (0x1A);				\
+    return FN##_##SUFF (vget_high_##SUFF (a), b);		\
+  }
+
+#define TEST_TERN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF) \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, H_INTYPE b)		 \
+  {                                                              \
+    INTYPE c = vdupq_n_##SUFF (0x1A);				 \
+    return FN##_##SUFF (a, vget_high_##SUFF (c), b);		 \
+  }
+
+#include "fold_to_highpart_1.c"
+
+
+/* { dg-final { scan-assembler-not {uxtl2\t} } } */
+/* { dg-final { scan-assembler-not {sxtl2\t} } } */
+
+/* { dg-final { scan-assembler-not {umull2\t} } } */
+/* { dg-final { scan-assembler-not {smull2\t} } } */
+/* { dg-final { scan-assembler-not {pmull2\t} } } */
+
+/* { dg-final { scan-assembler-not {uaddl2\t} } } */
+/* { dg-final { scan-assembler-not {saddl2\t} } } */
+
+/* { dg-final { scan-assembler-not {usubl2\t} } } */
+/* { dg-final { scan-assembler-not {ssubl2\t} } } */
+
+/* { dg-final { scan-assembler-not {uabal2\t} } } */
+/* { dg-final { scan-assembler-not {sabal2\t} } } */
+
+/* { dg-final { scan-assembler-not {uabdl2\t} } } */
+/* { dg-final { scan-assembler-not {sabdl2\t} } } */
+
+/* { dg-final { scan-assembler-not {usubw2\t} } } */
+/* { dg-final { scan-assembler-not {ssubw2\t} } } */
+
+/* { dg-final { scan-assembler-not {uaddw2\t} } } */
+/* { dg-final { scan-assembler-not {saddw2\t} } } */
+
+/* { dg-final { scan-assembler-not {umlal2\t} } } */
+/* { dg-final { scan-assembler-not {smlal2\t} } } */
+
+/* { dg-final { scan-assembler-not {umlsl2\t} } } */
+/* { dg-final { scan-assembler-not {smlsl2\t} } } */
+
+/* { dg-final { scan-assembler-not {sqdmull2\t} } } */
+
+/* { dg-final { scan-assembler-not {sqdmlal2\t} } } */
+
+/* { dg-final { scan-assembler-not {sqdmlsl2\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_3.c b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_3.c
new file mode 100644
index 0000000..3baf826
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_3.c
@@ -0,0 +1,83 @@
+/* { dg-do compile } */
+/* { dg-options "-O" } */
+
+/* PR117850 */
+
+/* We should use the highpart instruction where doing so would avoid data
+   movement instructions.  We avoid a DUP here after extending the
+   VECTOR_CSTs to 128-bits.  */
+
+#define TEST_UN_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)
+#define TEST_BIN_W_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)
+#define TEST_BIN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)
+#define TEST_TERN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)
+
+#define TEST_BIN_HIGHPART_A1(FN, RETTYPE, INTYPE, SUFF) \
+  RETTYPE test_a1_##FN##_##SUFF (INTYPE a)		\
+  {                                                     \
+    INTYPE b = vdupq_n_##SUFF (0x1A);			\
+    return FN##_##SUFF (vget_high_##SUFF (a),		\
+			vget_high_##SUFF (b));		\
+  }
+
+#define TEST_BIN_HIGHPART_A2(FN, RETTYPE, INTYPE, SUFF) \
+  RETTYPE test_a2_##FN##_##SUFF (INTYPE a)		\
+  {                                                     \
+    INTYPE b = vdupq_n_##SUFF (0x1A);			\
+    return FN##_##SUFF (vget_high_##SUFF (b),		\
+			vget_high_##SUFF (a));		\
+  }
+
+#define TEST_TERN_HIGHPART_A1(FN, RETTYPE, INTYPE, SUFF)    \
+  RETTYPE test_a1_##FN##_##SUFF (RETTYPE a, INTYPE b)	    \
+  {                                                         \
+    INTYPE c = vdupq_n_##SUFF (0x1A);			    \
+    return FN##_##SUFF (a, vget_high_##SUFF (b),	    \
+			vget_high_##SUFF (c));		    \
+  }
+
+#define TEST_TERN_HIGHPART_A2(FN, RETTYPE, INTYPE, SUFF)    \
+  RETTYPE test_a2_##FN##_##SUFF (RETTYPE a, INTYPE b)	    \
+  {                                                         \
+    INTYPE c = vdupq_n_##SUFF (0x1A);			    \
+    return FN##_##SUFF (a, vget_high_##SUFF (c),	    \
+			vget_high_##SUFF (b));		    \
+  }
+
+#define TEST_BIN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF) \
+  TEST_BIN_HIGHPART_A1 (FN, RETTYPE, INTYPE, SUFF)              \
+  TEST_BIN_HIGHPART_A2 (FN, RETTYPE, INTYPE, SUFF)
+
+#define TEST_TERN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF) \
+  TEST_TERN_HIGHPART_A1 (FN, RETTYPE, INTYPE, SUFF)              \
+  TEST_TERN_HIGHPART_A2 (FN, RETTYPE, INTYPE, SUFF)
+
+
+#include "fold_to_highpart_1.c"
+
+/* { dg-final { scan-assembler-not {dup\t} } } */
+
+/* { dg-final { scan-assembler-times {smull2\t} 6} } */
+/* { dg-final { scan-assembler-times {umull2\t} 6} } */
+/* { dg-final { scan-assembler-times {pmull2\t} 2} } */
+
+/* { dg-final { scan-assembler-times {saddl2\t} 6} } */
+/* { dg-final { scan-assembler-times {uaddl2\t} 6} } */
+
+/* { dg-final { scan-assembler-times {ssubl2\t} 6} } */
+/* { dg-final { scan-assembler-times {usubl2\t} 6} } */
+
+/* { dg-final { scan-assembler-times {sabdl2\t} 6} } */
+/* { dg-final { scan-assembler-times {uabdl2\t} 6} } */
+
+/* { dg-final { scan-assembler-times {smlal2\t} 6} } */
+/* { dg-final { scan-assembler-times {umlal2\t} 6} } */
+
+/* { dg-final { scan-assembler-times {smlsl2\t} 6} } */
+/* { dg-final { scan-assembler-times {umlsl2\t} 6} } */
+
+/* { dg-final { scan-assembler-times {sqdmull2\t} 4} } */
+
+/* { dg-final { scan-assembler-times {sqdmlal2\t} 4} } */
+
+/* { dg-final { scan-assembler-times {sqdmlsl2\t} 4} } */
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_4.c b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_4.c
new file mode 100644
index 0000000..046c7a0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_4.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_little_endian } */
+/* { dg-options "-O -fdump-tree-optimized" } */
+
+#include "arm_neon.h"
+
+#define VEC_CST_u8  0x0102030405060708
+#define VEC_CST_u16 0x0001000200030004
+#define VEC_CST_u32 0x0000000100000002
+
+/* Extend the 64b VECTOR_CST to the type required by the hi builtin.  */
+
+uint16x8_t
+test_u8 (uint8x16_t a)
+{
+  const uint8x8_t b = vcreate_u8 (VEC_CST_u8);
+  return vmull_u8 (vget_high_u8 (a), b);
+}
+
+/* { dg-final { scan-tree-dump-times "\{ 8, 7, 6, 5, 4, 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, 1 \}" 1 "optimized" } } */
+
+uint32x4_t
+test_u16 (uint16x8_t a)
+{
+  const uint16x4_t b = vcreate_u16 (VEC_CST_u16);
+  return vmull_u16 (vget_high_u16 (a), b);
+}
+
+/* { dg-final { scan-tree-dump-times "\{ 4, 3, 2, 1, 4, 3, 2, 1 \}" 1 "optimized" } } */
+
+uint64x2_t
+test_u32 (uint32x4_t a)
+{
+  const uint32x2_t b = vcreate_u32 (VEC_CST_u32);
+  return vmull_u32 (vget_high_u32 (a), b);
+}
+
+/* { dg-final { scan-tree-dump-times "\{ 2, 1, 2, 1 \}" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_5.c b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_5.c
new file mode 100644
index 0000000..4f39b67
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_5.c
@@ -0,0 +1,93 @@
+/* { dg-do compile } */
+/* { dg-options "-O -march=armv9-a+bf16" } */
+
+/* Test that we can still fold when the base type of the vector who's
+   highpart we are referring to is incompatible with that of the hi
+   builtin.
+
+   Use float64x2_t as it is never INTYPE.  */
+
+#define TEST_UN_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF) \
+  RETTYPE test_##FN##_##SUFF (float64x2_t a)	     \
+  {                                                  \
+    INTYPE x = vreinterpretq_##SUFF##_f64 (a);	     \
+    return FN##_##SUFF(vget_high_##SUFF (x));	     \
+  }
+
+#define TEST_BIN_W_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)       \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, float64x2_t b)	      \
+  {                                                           \
+    INTYPE x = vreinterpretq_##SUFF##_f64 (b);		      \
+    return FN##_##SUFF (a, vget_high_##SUFF (x));	      \
+  }
+
+#define TEST_BIN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)    \
+  RETTYPE test_##FN##_##SUFF (float64x2_t a)		   \
+  {                                                        \
+    INTYPE x = vreinterpretq_##SUFF##_f64 (a);		   \
+    return FN##_##SUFF (vget_high_##SUFF (x), x[1]);	   \
+  }
+
+#define TEST_TERN_N_HIGHPARTS(FN, RETTYPE, INTYPE, SUFF)         \
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, float64x2_t b)		 \
+  {                                                              \
+    INTYPE x = vreinterpretq_##SUFF##_f64 (b);			 \
+    return FN##_##SUFF (a, vget_high_##SUFF (x), x[1]);	 \
+  }
+
+#define TEST_BIN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF)   \
+  RETTYPE test_##FN##_##SUFF (float64x2_t a, float64x2_t b)	  \
+  {                                                               \
+    INTYPE x = vreinterpretq_##SUFF##_f64 (a);			  \
+    INTYPE y = vreinterpretq_##SUFF##_f64 (b);			  \
+    return FN##_##SUFF (vget_high_##SUFF (x),			  \
+			vget_high_##SUFF (y));			  \
+  }
+
+#define TEST_TERN_HIGHPARTS(FN, RETTYPE, INTYPE, H_INTYPE, SUFF)	\
+  RETTYPE test_##FN##_##SUFF (RETTYPE a, float64x2_t b, float64x2_t c)	\
+  {									\
+    INTYPE x = vreinterpretq_##SUFF##_f64 (b);				\
+    INTYPE y = vreinterpretq_##SUFF##_f64 (c);				\
+    return FN##_##SUFF (a, vget_high_## SUFF (x),                       \
+			vget_high_## SUFF (y));				\
+  }
+
+#include "fold_to_highpart_1.c"
+
+/* { dg-final { scan-assembler-times {sxtl2\t} 3} } */
+/* { dg-final { scan-assembler-times {uxtl2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {smull2\t} 5} } */
+/* { dg-final { scan-assembler-times {umull2\t} 5} } */
+/* { dg-final { scan-assembler-times {pmull2\t} 1} } */
+
+/* { dg-final { scan-assembler-times {saddl2\t} 3} } */
+/* { dg-final { scan-assembler-times {uaddl2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {ssubl2\t} 3} } */
+/* { dg-final { scan-assembler-times {usubl2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {sabdl2\t} 3} } */
+/* { dg-final { scan-assembler-times {uabdl2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {saddw2\t} 3} } */
+/* { dg-final { scan-assembler-times {uaddw2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {ssubw2\t} 3} } */
+/* { dg-final { scan-assembler-times {usubw2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {sabdl2\t} 3} } */
+/* { dg-final { scan-assembler-times {uabdl2\t} 3} } */
+
+/* { dg-final { scan-assembler-times {smlal2\t} 5} } */
+/* { dg-final { scan-assembler-times {umlal2\t} 5} } */
+
+/* { dg-final { scan-assembler-times {smlsl2\t} 5} } */
+/* { dg-final { scan-assembler-times {umlsl2\t} 5} } */
+
+/* { dg-final { scan-assembler-times {sqdmull2\t} 4} } */
+
+/* { dg-final { scan-assembler-times {sqdmlal2\t} 4} } */
+
+/* { dg-final { scan-assembler-times {sqdmlsl2\t} 4} } */
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_6.c b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_6.c
new file mode 100644
index 0000000..3570d4d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/simd/fold_to_highpart_6.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_little_endian } */
+/* { dg-options "-O2 -march=armv8-a+sve" } */
+
+#include <arm_neon_sve_bridge.h>
+
+typedef int16_t int16x16_t __attribute__ ((vector_size (32)));
+
+/* Edge cases where we don't/can't fold, reject these gracefully.  */
+
+int8x16_t z;
+
+int16x8_t
+test_addressable ()
+{
+  return vmovl_s8 (vget_high_s8 (z));
+}
+
+int16x8_t
+test_scalable_type (svint8_t scalable)
+{
+  return vmovl_s8 (vget_high_s8 (svget_neonq_s8 (scalable)));
+}
+
+int16x8_t
+test_scalar_type (__int128_t foo)
+{
+  return vmovl_s8 (vget_high_s8 (vreinterpretq_s8_p128 (foo)));
+}
+
+int32x4_t
+test_256b_type (int16x16_t foo)
+{
+  return vmovl_s16 ((int16x4_t) { foo[4], foo[5], foo[6], foo[7] });
+}
+
+/* { dg-final { scan-assembler-not {sxtl2\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/mf8_data_1.c b/gcc/testsuite/gcc.target/aarch64/simd/mf8_data_1.c
index a3fd9b8..79d1ccf 100644
--- a/gcc/testsuite/gcc.target/aarch64/simd/mf8_data_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/simd/mf8_data_1.c
@@ -1016,7 +1016,12 @@ mfloat8x8_t test_set_lane3(mfloat8x8_t a, const mfloat8_t *ptr)
 
 /*
 ** test_set_lane4:
+**	(
 **	ins	v0.b\[6\], wzr
+**	|
+**	movi?	[vdz]([0-9]+)\.?(?:[0-9]*[bhsd])?, #?0
+**	ins	v0.b\[6\], v(\1).b\[0\]
+**	)
 **	ret
 */
 mfloat8x8_t test_set_lane4(mfloat8x8_t a)
@@ -1056,7 +1061,12 @@ mfloat8x16_t test_setq_lane3(mfloat8x16_t a, const mfloat8_t *ptr)
 
 /*
 ** test_setq_lane4:
+**	(
 **	ins	v0.b\[14\], wzr
+**	|
+**	movi?	[vdz]([0-9]+)\.?(?:[0-9]*[bhsd])?, #?0
+**	ins	v0.b\[14\], v(\1).b\[0\]
+**	)
 **	ret
 */
 mfloat8x16_t test_setq_lane4(mfloat8x16_t a)
diff --git a/gcc/testsuite/gcc.target/aarch64/simd/vabal_combine.c b/gcc/testsuite/gcc.target/aarch64/simd/vabal_combine.c
deleted file mode 100644
index c51878a..0000000
--- a/gcc/testsuite/gcc.target/aarch64/simd/vabal_combine.c
+++ /dev/null
@@ -1,72 +0,0 @@
-/* { dg-do compile } */
-/* { dg-options "-O" } */
-/* { dg-final { check-function-bodies "**" "" "" } } */
-
-#include <arm_neon.h>
-
-/*
-** test_vabal_s8:
-**      sabal2	v0.8h, v2.16b, v1.16b
-**      ret
-*/
-int16x8_t
-test_vabal_s8 (int16x8_t sadv, int8x16_t pv, int8x16_t sv)
-{
-  return vabal_s8 (sadv, vget_high_s8 (pv), vget_high_s8 (sv));
-}
-
-/*
-** test_vabal_u8:
-**      uabal2	v0.8h, v2.16b, v1.16b
-**      ret
-*/
-uint16x8_t
-test_vabal_u8 (uint16x8_t sadv, uint8x16_t pv, uint8x16_t sv)
-{
-  return vabal_u8 (sadv, vget_high_u8 (pv), vget_high_u8 (sv));
-}
-
-/*
-** test_vabal_s16:
-**      sabal2	v0.4s, v2.8h, v1.8h
-**      ret
-*/
-int32x4_t
-test_vabal_s16 (int32x4_t sadv, int16x8_t pv, int16x8_t sv)
-{
-  return vabal_s16 (sadv, vget_high_s16 (pv), vget_high_s16 (sv));
-}
-
-/*
-** test_vabal_u16:
-**      uabal2	v0.4s, v2.8h, v1.8h
-**      ret
-*/
-uint32x4_t
-test_vabal_u16 (uint32x4_t sadv, uint16x8_t pv, uint16x8_t sv)
-{
-  return vabal_u16 (sadv, vget_high_u16 (pv), vget_high_u16 (sv));
-}
-
-/*
-** test_vabal_s32:
-**      sabal2	v0.2d, v2.4s, v1.4s
-**      ret
-*/
-int64x2_t
-test_vabal_s32 (int64x2_t sadv, int32x4_t pv, int32x4_t sv)
-{
-  return vabal_s32 (sadv, vget_high_s32 (pv), vget_high_s32 (sv));
-}
-
-/*
-** test_vabal_u32:
-**      uabal2	v0.2d, v2.4s, v1.4s
-**      ret
-*/
-uint64x2_t
-test_vabal_u32 (uint64x2_t sadv, uint32x4_t pv, uint32x4_t sv)
-{
-  return vabal_u32 (sadv, vget_high_u32 (pv), vget_high_u32 (sv));
-}
-
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_1.c b/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_1.c
index 98922aa..3a63da7 100644
--- a/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_1.c
@@ -1,5 +1,5 @@
 // { dg-options "-O -fomit-frame-pointer -fno-optimize-sibling-calls -funwind-tables" }
-// { dg-final { check-function-bodies "**" "" } }
+// { dg-final { check-function-bodies "**" "" "" { target "*-*-*" } {\t\.inst} } }
 
 void ns_callee ();
  void s_callee () [[arm::streaming]];
@@ -218,7 +218,7 @@ sc_caller_x1 (int *ptr, int a) [[arm::streaming_compatible]]
 **	bl	ns_callee_stack
 **	ldr	x16, \[x29, #?16\]
 **	tbz	x16, 0, .*
-**	smstart	sm
+**	.inst 0xd503437f // smstart sm
 **	...
 */
 void
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_11.c b/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_11.c
index ee6f987..c72d03f 100644
--- a/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_11.c
+++ b/gcc/testsuite/gcc.target/aarch64/sme/call_sm_switch_11.c
@@ -1,5 +1,6 @@
 // { dg-options "-O -fomit-frame-pointer -fno-optimize-sibling-calls -funwind-tables -mtrack-speculation" }
-// { dg-final { check-function-bodies "**" "" } }
+// { dg-final { check-function-bodies "**" "" "" { target "*-*-*" } {\t\.inst} } }
+
 
 void ns_callee ();
  void s_callee () [[arm::streaming]];
@@ -196,7 +197,7 @@ sc_caller_x1 (int *ptr, int a) [[arm::streaming_compatible]]
 **	tst	x16, #?1
 **	beq	[^\n]*
 **	csel	x15, x15, xzr, ne
-**	smstart	sm
+**	.inst 0xd503437f // smstart sm
 **	...
 */
 void
diff --git a/gcc/testsuite/gcc.target/aarch64/sme/pr121028.c b/gcc/testsuite/gcc.target/aarch64/sme/pr121028.c
new file mode 100644
index 0000000..a6aa119
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme/pr121028.c
@@ -0,0 +1,46 @@
+// PR121028
+// { dg-do assemble { target aarch64_asm_sme_ok } }
+// { dg-options "-O --save-temps" }
+// { dg-final { check-function-bodies "**" "" "" { target "*-*-*" } {\t\.inst} } }
+
+void ns_callee ();
+
+/*
+** sc_caller_sme:
+**	...
+**	mrs	x16, svcr
+**	str	x16, \[x29, #?16\]
+**	ldr	x16, \[x29, #?16\]
+**	tbz	x16, 0, .*
+**	smstop	sm
+**	bl	ns_callee
+**	ldr	x16, \[x29, #?16\]
+**	tbz	x16, 0, .*
+**	smstart	sm
+**	...
+*/
+void sc_caller_sme() __arm_streaming_compatible
+{
+    ns_callee ();
+}
+
+#pragma GCC target "+nosme"
+
+/*
+** sc_caller_nosme:
+**	...
+**	bl	__arm_sme_state
+**	str	x0, \[x29, #?16\]
+**	ldr	x16, \[x29, #?16\]
+**	tbz	x16, 0, .*
+**	.inst 0xd503427f // smstop sm
+**	bl	ns_callee
+**	ldr	x16, \[x29, #?16\]
+**	tbz	x16, 0, .*
+**	.inst 0xd503437f // smstart sm
+**	...
+*/
+void sc_caller_nosme() __arm_streaming_compatible
+{
+    ns_callee ();
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f16_x2.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f16_x2.c
new file mode 100644
index 0000000..b9fd96a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f16_x2.c
@@ -0,0 +1,99 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amax_z0_z0_z4:
+**	famax	{z0\.h - z1\.h}, {z0\.h - z1\.h}, {z4\.h - z5\.h}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z4, svfloat16x2_t, z0,
+	 svamax_f16_x2 (z0, z4),
+	 svamax (z0, z4))
+
+/*
+** amax_z0_z4_z0:
+**	famax	{z0\.h - z1\.h}, {z0\.h - z1\.h}, {z4\.h - z5\.h}
+**	ret
+*/
+TEST_XN (amax_z0_z4_z0, svfloat16x2_t, z0,
+	 svamax_f16_x2 (z4, z0),
+	 svamax (z4, z0))
+
+/*
+** amax_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.h - z29\.h}
+** |
+**	famax	[^\n]+, {z28\.h - z29\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z4_z28, svfloat16x2_t, z0,
+	 svamax_f16_x2 (z4, z28),
+	 svamax (z4, z28))
+
+/*
+** amax_z18_z18_z4:
+**	famax	{z18\.h - z19\.h}, {z18\.h - z19\.h}, {z4\.h - z5\.h}
+**	ret
+*/
+TEST_XN (amax_z18_z18_z4, svfloat16x2_t, z18,
+	 svamax_f16_x2 (z18, z4),
+	 svamax (z18, z4))
+
+/*
+** amax_z23_z23_z18:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z18\.h - z19\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z23_z23_z18, svfloat16x2_t, z23,
+	 svamax_f16_x2 (z23, z18),
+	 svamax (z23, z18))
+
+/*
+** amax_z28_z28_z0:
+**	famax	{z28\.h - z29\.h}, {z28\.h - z29\.h}, {z0\.h - z1\.h}
+**	ret
+*/
+TEST_XN (amax_z28_z28_z0, svfloat16x2_t, z28,
+	 svamax_f16_x2 (z28, z0),
+	 svamax (z28, z0))
+
+/*
+** amax_z0_z0_z18:
+**	famax	{z0\.h - z1\.h}, {z0\.h - z1\.h}, {z18\.h - z19\.h}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z18, svfloat16x2_t, z0,
+	 svamax_f16_x2 (z0, z18),
+	 svamax (z0, z18))
+
+/*
+** amax_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z4\.h - z5\.h}, {z4\.h - z5\.h}, [^\n]+
+** |
+**	famax	{z4\.h - z5\.h}, {z4\.h - z5\.h}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z4_z4_z23, svfloat16x2_t, z4,
+	 svamax_f16_x2 (z4, z23),
+	 svamax (z4, z23))
+
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f16_x4.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f16_x4.c
new file mode 100644
index 0000000..70e2697
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f16_x4.c
@@ -0,0 +1,130 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amax_z0_z0_z4:
+**	famax	{z0\.h - z3\.h}, {z0\.h - z3\.h}, {z4\.h - z7\.h}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z4, svfloat16x4_t, z0,
+	 svamax_f16_x4 (z0, z4),
+	 svamax (z0, z4))
+
+/*
+** amax_z0_z4_z0:
+**	famax	{z0\.h - z3\.h}, {z0\.h - z3\.h}, {z4\.h - z7\.h}
+**	ret
+*/
+TEST_XN (amax_z0_z4_z0, svfloat16x4_t, z0,
+	 svamax_f16_x4 (z4, z0),
+	 svamax (z4, z0))
+
+/*
+** amax_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.h - z31\.h}
+** |
+**	famax	[^\n]+, {z28\.h - z31\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z4_z28, svfloat16x4_t, z0,
+	 svamax_f16_x4 (z4, z28),
+	 svamax (z4, z28))
+
+/*
+** amax_z18_z18_z4:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z4\.h - z7\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z18_z18_z4, svfloat16x4_t, z18,
+	 svamax_f16_x4 (z18, z4),
+	 svamax (z18, z4))
+
+/*
+** amax_z23_z23_z28:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.h - z31\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z23_z23_z28, svfloat16x4_t, z23,
+	 svamax_f16_x4 (z23, z28),
+	 svamax (z23, z28))
+
+/*
+** amax_z28_z28_z0:
+**	famax	{z28\.h - z31\.h}, {z28\.h - z31\.h}, {z0\.h - z3\.h}
+**	ret
+*/
+TEST_XN (amax_z28_z28_z0, svfloat16x4_t, z28,
+	 svamax_f16_x4 (z28, z0),
+	 svamax (z28, z0))
+
+/*
+** amax_z0_z0_z18:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z0\.h - z3\.h}, {z0\.h - z3\.h}, [^\n]+
+** |
+**	famax	{z0\.h - z3\.h}, {z0\.h - z3\.h}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z0_z18, svfloat16x4_t, z0,
+	 svamax_f16_x4 (z0, z18),
+	 svamax (z0, z18))
+
+/*
+** amax_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z4\.h - z7\.h}, {z4\.h - z7\.h}, [^\n]+
+** |
+**	famax	{z4\.h - z7\.h}, {z4\.h - z7\.h}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z4_z4_z23, svfloat16x4_t, z4,
+	 svamax_f16_x4 (z4, z23),
+	 svamax (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f32_x2.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f32_x2.c
new file mode 100644
index 0000000..cf57d1b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f32_x2.c
@@ -0,0 +1,98 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amax_z0_z0_z4:
+**	famax	{z0\.s - z1\.s}, {z0\.s - z1\.s}, {z4\.s - z5\.s}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z4, svfloat32x2_t, z0,
+	 svamax_f32_x2 (z0, z4),
+	 svamax (z0, z4))
+
+/*
+** amax_z0_z4_z0:
+**	famax	{z0\.s - z1\.s}, {z0\.s - z1\.s}, {z4\.s - z5\.s}
+**	ret
+*/
+TEST_XN (amax_z0_z4_z0, svfloat32x2_t, z0,
+	 svamax_f32_x2 (z4, z0),
+	 svamax (z4, z0))
+
+/*
+** amax_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.s - z29\.s}
+** |
+**	famax	[^\n]+, {z28\.s - z29\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z4_z28, svfloat32x2_t, z0,
+	 svamax_f32_x2 (z4, z28),
+	 svamax (z4, z28))
+
+/*
+** amax_z18_z18_z4:
+**	famax	{z18\.s - z19\.s}, {z18\.s - z19\.s}, {z4\.s - z5\.s}
+**	ret
+*/
+TEST_XN (amax_z18_z18_z4, svfloat32x2_t, z18,
+	 svamax_f32_x2 (z18, z4),
+	 svamax (z18, z4))
+
+/*
+** amax_z23_z23_z18:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z18\.s - z19\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z23_z23_z18, svfloat32x2_t, z23,
+	 svamax_f32_x2 (z23, z18),
+	 svamax (z23, z18))
+
+/*
+** amax_z28_z28_z0:
+**	famax	{z28\.s - z29\.s}, {z28\.s - z29\.s}, {z0\.s - z1\.s}
+**	ret
+*/
+TEST_XN (amax_z28_z28_z0, svfloat32x2_t, z28,
+	 svamax_f32_x2 (z28, z0),
+	 svamax (z28, z0))
+
+/*
+** amax_z0_z0_z18:
+**	famax	{z0\.s - z1\.s}, {z0\.s - z1\.s}, {z18\.s - z19\.s}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z18, svfloat32x2_t, z0,
+	 svamax_f32_x2 (z0, z18),
+	 svamax (z0, z18))
+
+/*
+** amax_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z4\.s - z5\.s}, {z4\.s - z5\.s}, [^\n]+
+** |
+**	famax	{z4\.s - z5\.s}, {z4\.s - z5\.s}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z4_z4_z23, svfloat32x2_t, z4,
+	 svamax_f32_x2 (z4, z23),
+	 svamax (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f32_x4.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f32_x4.c
new file mode 100644
index 0000000..10d9175
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f32_x4.c
@@ -0,0 +1,131 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amax_z0_z0_z4:
+**	famax	{z0\.s - z3\.s}, {z0\.s - z3\.s}, {z4\.s - z7\.s}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z4, svfloat32x4_t, z0,
+	 svamax_f32_x4 (z0, z4),
+	 svamax (z0, z4))
+
+/*
+** amax_z0_z4_z0:
+**	famax	{z0\.s - z3\.s}, {z0\.s - z3\.s}, {z4\.s - z7\.s}
+**	ret
+*/
+TEST_XN (amax_z0_z4_z0, svfloat32x4_t, z0,
+	 svamax_f32_x4 (z4, z0),
+	 svamax (z4, z0))
+
+/*
+** amax_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.s - z31\.s}
+** |
+**	famax	[^\n]+, {z28\.s - z31\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z4_z28, svfloat32x4_t, z0,
+	 svamax_f32_x4 (z4, z28),
+	 svamax (z4, z28))
+
+/*
+** amax_z18_z18_z4:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z4\.s - z7\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z18_z18_z4, svfloat32x4_t, z18,
+	 svamax_f32_x4 (z18, z4),
+	 svamax (z18, z4))
+
+/*
+** amax_z23_z23_z28:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.s - z31\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z23_z23_z28, svfloat32x4_t, z23,
+	 svamax_f32_x4 (z23, z28),
+	 svamax (z23, z28))
+
+/*
+** amax_z28_z28_z0:
+**	famax	{z28\.s - z31\.s}, {z28\.s - z31\.s}, {z0\.s - z3\.s}
+**	ret
+*/
+TEST_XN (amax_z28_z28_z0, svfloat32x4_t, z28,
+	 svamax_f32_x4 (z28, z0),
+	 svamax (z28, z0))
+
+/*
+** amax_z0_z0_z18:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z0\.s - z3\.s}, {z0\.s - z3\.s}, [^\n]+
+** |
+**	famax	{z0\.s - z3\.s}, {z0\.s - z3\.s}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z0_z18, svfloat32x4_t, z0,
+	 svamax_f32_x4 (z0, z18),
+	 svamax (z0, z18))
+
+/*
+** amax_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z4\.s - z7\.s}, {z4\.s - z7\.s}, [^\n]+
+** |
+**	famax	{z4\.s - z7\.s}, {z4\.s - z7\.s}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z4_z4_z23, svfloat32x4_t, z4,
+	 svamax_f32_x4 (z4, z23),
+	 svamax (z4, z23))
+
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f64_x2.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f64_x2.c
new file mode 100644
index 0000000..b7918ab
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f64_x2.c
@@ -0,0 +1,98 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amax_z0_z0_z4:
+**	famax	{z0\.d - z1\.d}, {z0\.d - z1\.d}, {z4\.d - z5\.d}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z4, svfloat64x2_t, z0,
+	 svamax_f64_x2 (z0, z4),
+	 svamax (z0, z4))
+
+/*
+** amax_z0_z4_z0:
+**	famax	{z0\.d - z1\.d}, {z0\.d - z1\.d}, {z4\.d - z5\.d}
+**	ret
+*/
+TEST_XN (amax_z0_z4_z0, svfloat64x2_t, z0,
+	 svamax_f64_x2 (z4, z0),
+	 svamax (z4, z0))
+
+/*
+** amax_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.d - z29\.d}
+** |
+**	famax	[^\n]+, {z28\.d - z29\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z4_z28, svfloat64x2_t, z0,
+	 svamax_f64_x2 (z4, z28),
+	 svamax (z4, z28))
+
+/*
+** amax_z18_z18_z4:
+**	famax	{z18\.d - z19\.d}, {z18\.d - z19\.d}, {z4\.d - z5\.d}
+**	ret
+*/
+TEST_XN (amax_z18_z18_z4, svfloat64x2_t, z18,
+	 svamax_f64_x2 (z18, z4),
+	 svamax (z18, z4))
+
+/*
+** amax_z23_z23_z18:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z18\.d - z19\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z23_z23_z18, svfloat64x2_t, z23,
+	 svamax_f64_x2 (z23, z18),
+	 svamax (z23, z18))
+
+/*
+** amax_z28_z28_z0:
+**	famax	{z28\.d - z29\.d}, {z28\.d - z29\.d}, {z0\.d - z1\.d}
+**	ret
+*/
+TEST_XN (amax_z28_z28_z0, svfloat64x2_t, z28,
+	 svamax_f64_x2 (z28, z0),
+	 svamax (z28, z0))
+
+/*
+** amax_z0_z0_z18:
+**	famax	{z0\.d - z1\.d}, {z0\.d - z1\.d}, {z18\.d - z19\.d}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z18, svfloat64x2_t, z0,
+	 svamax_f64_x2 (z0, z18),
+	 svamax (z0, z18))
+
+/*
+** amax_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z4\.d - z5\.d}, {z4\.d - z5\.d}, [^\n]+
+** |
+**	famax	{z4\.d - z5\.d}, {z4\.d - z5\.d}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z4_z4_z23, svfloat64x2_t, z4,
+	 svamax_f64_x2 (z4, z23),
+	 svamax (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f64_x4.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f64_x4.c
new file mode 100644
index 0000000..153a37a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amax_f64_x4.c
@@ -0,0 +1,130 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amax_z0_z0_z4:
+**	famax	{z0\.d - z3\.d}, {z0\.d - z3\.d}, {z4\.d - z7\.d}
+**	ret
+*/
+TEST_XN (amax_z0_z0_z4, svfloat64x4_t, z0,
+	 svamax_f64_x4 (z0, z4),
+	 svamax (z0, z4))
+
+/*
+** amax_z0_z4_z0:
+**	famax	{z0\.d - z3\.d}, {z0\.d - z3\.d}, {z4\.d - z7\.d}
+**	ret
+*/
+TEST_XN (amax_z0_z4_z0, svfloat64x4_t, z0,
+	 svamax_f64_x4 (z4, z0),
+	 svamax (z4, z0))
+
+/*
+** amax_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.d - z31\.d}
+** |
+**	famax	[^\n]+, {z28\.d - z31\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z4_z28, svfloat64x4_t, z0,
+	 svamax_f64_x4 (z4, z28),
+	 svamax (z4, z28))
+
+/*
+** amax_z18_z18_z4:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z4\.d - z7\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z18_z18_z4, svfloat64x4_t, z18,
+	 svamax_f64_x4 (z18, z4),
+	 svamax (z18, z4))
+
+/*
+** amax_z23_z23_z28:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	[^\n]+, {z28\.d - z31\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amax_z23_z23_z28, svfloat64x4_t, z23,
+	 svamax_f64_x4 (z23, z28),
+	 svamax (z23, z28))
+
+/*
+** amax_z28_z28_z0:
+**	famax	{z28\.d - z31\.d}, {z28\.d - z31\.d}, {z0\.d - z3\.d}
+**	ret
+*/
+TEST_XN (amax_z28_z28_z0, svfloat64x4_t, z28,
+	 svamax_f64_x4 (z28, z0),
+	 svamax (z28, z0))
+
+/*
+** amax_z0_z0_z18:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z0\.d - z3\.d}, {z0\.d - z3\.d}, [^\n]+
+** |
+**	famax	{z0\.d - z3\.d}, {z0\.d - z3\.d}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z0_z0_z18, svfloat64x4_t, z0,
+	 svamax_f64_x4 (z0, z18),
+	 svamax (z0, z18))
+
+/*
+** amax_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famax	{z4\.d - z7\.d}, {z4\.d - z7\.d}, [^\n]+
+** |
+**	famax	{z4\.d - z7\.d}, {z4\.d - z7\.d}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amax_z4_z4_z23, svfloat64x4_t, z4,
+	 svamax_f64_x4 (z4, z23),
+	 svamax (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f16_x2.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f16_x2.c
new file mode 100644
index 0000000..bd6e13b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f16_x2.c
@@ -0,0 +1,98 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amin_z0_z0_z4:
+**	famin	{z0\.h - z1\.h}, {z0\.h - z1\.h}, {z4\.h - z5\.h}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z4, svfloat16x2_t, z0,
+	 svamin_f16_x2 (z0, z4),
+	 svamin (z0, z4))
+
+/*
+** amin_z0_z4_z0:
+**	famin	{z0\.h - z1\.h}, {z0\.h - z1\.h}, {z4\.h - z5\.h}
+**	ret
+*/
+TEST_XN (amin_z0_z4_z0, svfloat16x2_t, z0,
+	 svamin_f16_x2 (z4, z0),
+	 svamin (z4, z0))
+
+/*
+** amin_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.h - z29\.h}
+** |
+**	famin	[^\n]+, {z28\.h - z29\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z4_z28, svfloat16x2_t, z0,
+	 svamin_f16_x2 (z4, z28),
+	 svamin (z4, z28))
+
+/*
+** amin_z18_z18_z4:
+**	famin	{z18\.h - z19\.h}, {z18\.h - z19\.h}, {z4\.h - z5\.h}
+**	ret
+*/
+TEST_XN (amin_z18_z18_z4, svfloat16x2_t, z18,
+	 svamin_f16_x2 (z18, z4),
+	 svamin (z18, z4))
+
+/*
+** amin_z23_z23_z18:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z18\.h - z19\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z23_z23_z18, svfloat16x2_t, z23,
+	 svamin_f16_x2 (z23, z18),
+	 svamin (z23, z18))
+
+/*
+** amin_z28_z28_z0:
+**	famin	{z28\.h - z29\.h}, {z28\.h - z29\.h}, {z0\.h - z1\.h}
+**	ret
+*/
+TEST_XN (amin_z28_z28_z0, svfloat16x2_t, z28,
+	 svamin_f16_x2 (z28, z0),
+	 svamin (z28, z0))
+
+/*
+** amin_z0_z0_z18:
+**	famin	{z0\.h - z1\.h}, {z0\.h - z1\.h}, {z18\.h - z19\.h}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z18, svfloat16x2_t, z0,
+	 svamin_f16_x2 (z0, z18),
+	 svamin (z0, z18))
+
+/*
+** amin_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z4\.h - z5\.h}, {z4\.h - z5\.h}, [^\n]+
+** |
+**	famin	{z4\.h - z5\.h}, {z4\.h - z5\.h}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z4_z4_z23, svfloat16x2_t, z4,
+	 svamin_f16_x2 (z4, z23),
+	 svamin (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f16_x4.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f16_x4.c
new file mode 100644
index 0000000..9f71b1f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f16_x4.c
@@ -0,0 +1,130 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amin_z0_z0_z4:
+**	famin	{z0\.h - z3\.h}, {z0\.h - z3\.h}, {z4\.h - z7\.h}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z4, svfloat16x4_t, z0,
+	 svamin_f16_x4 (z0, z4),
+	 svamin (z0, z4))
+
+/*
+** amin_z0_z4_z0:
+**	famin	{z0\.h - z3\.h}, {z0\.h - z3\.h}, {z4\.h - z7\.h}
+**	ret
+*/
+TEST_XN (amin_z0_z4_z0, svfloat16x4_t, z0,
+	 svamin_f16_x4 (z4, z0),
+	 svamin (z4, z0))
+
+/*
+** amin_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.h - z31\.h}
+** |
+**	famin	[^\n]+, {z28\.h - z31\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z4_z28, svfloat16x4_t, z0,
+	 svamin_f16_x4 (z4, z28),
+	 svamin (z4, z28))
+
+/*
+** amin_z18_z18_z4:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z4\.h - z7\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z18_z18_z4, svfloat16x4_t, z18,
+	 svamin_f16_x4 (z18, z4),
+	 svamin (z18, z4))
+
+/*
+** amin_z23_z23_z28:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.h - z31\.h}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z23_z23_z28, svfloat16x4_t, z23,
+	 svamin_f16_x4 (z23, z28),
+	 svamin (z23, z28))
+
+/*
+** amin_z28_z28_z0:
+**	famin	{z28\.h - z31\.h}, {z28\.h - z31\.h}, {z0\.h - z3\.h}
+**	ret
+*/
+TEST_XN (amin_z28_z28_z0, svfloat16x4_t, z28,
+	 svamin_f16_x4 (z28, z0),
+	 svamin (z28, z0))
+
+/*
+** amin_z0_z0_z18:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z0\.h - z3\.h}, {z0\.h - z3\.h}, [^\n]+
+** |
+**	famin	{z0\.h - z3\.h}, {z0\.h - z3\.h}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z0_z18, svfloat16x4_t, z0,
+	 svamin_f16_x4 (z0, z18),
+	 svamin (z0, z18))
+
+/*
+** amin_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z4\.h - z7\.h}, {z4\.h - z7\.h}, [^\n]+
+** |
+**	famin	{z4\.h - z7\.h}, {z4\.h - z7\.h}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z4_z4_z23, svfloat16x4_t, z4,
+	 svamin_f16_x4 (z4, z23),
+	 svamin (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f32_x2.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f32_x2.c
new file mode 100644
index 0000000..aaa6a2e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f32_x2.c
@@ -0,0 +1,98 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amin_z0_z0_z4:
+**	famin	{z0\.s - z1\.s}, {z0\.s - z1\.s}, {z4\.s - z5\.s}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z4, svfloat32x2_t, z0,
+ svamin_f32_x2 (z0, z4),
+ svamin (z0, z4))
+
+/*
+** amin_z0_z4_z0:
+**	famin	{z0\.s - z1\.s}, {z0\.s - z1\.s}, {z4\.s - z5\.s}
+**	ret
+*/
+TEST_XN (amin_z0_z4_z0, svfloat32x2_t, z0,
+ svamin_f32_x2 (z4, z0),
+ svamin (z4, z0))
+
+/*
+** amin_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.s - z29\.s}
+** |
+**	famin	[^\n]+, {z28\.s - z29\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z4_z28, svfloat32x2_t, z0,
+ svamin_f32_x2 (z4, z28),
+ svamin (z4, z28))
+
+/*
+** amin_z18_z18_z4:
+**	famin	{z18\.s - z19\.s}, {z18\.s - z19\.s}, {z4\.s - z5\.s}
+**	ret
+*/
+TEST_XN (amin_z18_z18_z4, svfloat32x2_t, z18,
+ svamin_f32_x2 (z18, z4),
+ svamin (z18, z4))
+
+/*
+** amin_z23_z23_z18:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z18\.s - z19\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z23_z23_z18, svfloat32x2_t, z23,
+ svamin_f32_x2 (z23, z18),
+ svamin (z23, z18))
+
+/*
+** amin_z28_z28_z0:
+**	famin	{z28\.s - z29\.s}, {z28\.s - z29\.s}, {z0\.s - z1\.s}
+**	ret
+*/
+TEST_XN (amin_z28_z28_z0, svfloat32x2_t, z28,
+ svamin_f32_x2 (z28, z0),
+ svamin (z28, z0))
+
+/*
+** amin_z0_z0_z18:
+**	famin	{z0\.s - z1\.s}, {z0\.s - z1\.s}, {z18\.s - z19\.s}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z18, svfloat32x2_t, z0,
+ svamin_f32_x2 (z0, z18),
+ svamin (z0, z18))
+
+/*
+** amin_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z4\.s - z5\.s}, {z4\.s - z5\.s}, [^\n]+
+** |
+**	famin	{z4\.s - z5\.s}, {z4\.s - z5\.s}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z4_z4_z23, svfloat32x2_t, z4,
+ svamin_f32_x2 (z4, z23),
+ svamin (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f32_x4.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f32_x4.c
new file mode 100644
index 0000000..34c1098
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f32_x4.c
@@ -0,0 +1,130 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amin_z0_z0_z4:
+**	famin	{z0\.s - z3\.s}, {z0\.s - z3\.s}, {z4\.s - z7\.s}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z4, svfloat32x4_t, z0,
+	 svamin_f32_x4 (z0, z4),
+	 svamin (z0, z4))
+
+/*
+** amin_z0_z4_z0:
+**	famin	{z0\.s - z3\.s}, {z0\.s - z3\.s}, {z4\.s - z7\.s}
+**	ret
+*/
+TEST_XN (amin_z0_z4_z0, svfloat32x4_t, z0,
+	 svamin_f32_x4 (z4, z0),
+	 svamin (z4, z0))
+
+/*
+** amin_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.s - z31\.s}
+** |
+**	famin	[^\n]+, {z28\.s - z31\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z4_z28, svfloat32x4_t, z0,
+	 svamin_f32_x4 (z4, z28),
+	 svamin (z4, z28))
+
+/*
+** amin_z18_z18_z4:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z4\.s - z7\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z18_z18_z4, svfloat32x4_t, z18,
+	 svamin_f32_x4 (z18, z4),
+	 svamin (z18, z4))
+
+/*
+** amin_z23_z23_z28:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.s - z31\.s}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z23_z23_z28, svfloat32x4_t, z23,
+	 svamin_f32_x4 (z23, z28),
+	 svamin (z23, z28))
+
+/*
+** amin_z28_z28_z0:
+**	famin	{z28\.s - z31\.s}, {z28\.s - z31\.s}, {z0\.s - z3\.s}
+**	ret
+*/
+TEST_XN (amin_z28_z28_z0, svfloat32x4_t, z28,
+	 svamin_f32_x4 (z28, z0),
+	 svamin (z28, z0))
+
+/*
+** amin_z0_z0_z18:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z0\.s - z3\.s}, {z0\.s - z3\.s}, [^\n]+
+** |
+**	famin	{z0\.s - z3\.s}, {z0\.s - z3\.s}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z0_z18, svfloat32x4_t, z0,
+	 svamin_f32_x4 (z0, z18),
+	 svamin (z0, z18))
+
+/*
+** amin_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z4\.s - z7\.s}, {z4\.s - z7\.s}, [^\n]+
+** |
+**	famin	{z4\.s - z7\.s}, {z4\.s - z7\.s}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z4_z4_z23, svfloat32x4_t, z4,
+	 svamin_f32_x4 (z4, z23),
+	 svamin (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f64_x2.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f64_x2.c
new file mode 100644
index 0000000..e4138e0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f64_x2.c
@@ -0,0 +1,98 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amin_z0_z0_z4:
+**	famin	{z0\.d - z1\.d}, {z0\.d - z1\.d}, {z4\.d - z5\.d}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z4, svfloat64x2_t, z0,
+	 svamin_f64_x2 (z0, z4),
+	 svamin (z0, z4))
+
+/*
+** amin_z0_z4_z0:
+**	famin	{z0\.d - z1\.d}, {z0\.d - z1\.d}, {z4\.d - z5\.d}
+**	ret
+*/
+TEST_XN (amin_z0_z4_z0, svfloat64x2_t, z0,
+	 svamin_f64_x2 (z4, z0),
+	 svamin (z4, z0))
+
+/*
+** amin_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.d - z29\.d}
+** |
+**	famin	[^\n]+, {z28\.d - z29\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z4_z28, svfloat64x2_t, z0,
+	 svamin_f64_x2 (z4, z28),
+	 svamin (z4, z28))
+
+/*
+** amin_z18_z18_z4:
+**	famin	{z18\.d - z19\.d}, {z18\.d - z19\.d}, {z4\.d - z5\.d}
+**	ret
+*/
+TEST_XN (amin_z18_z18_z4, svfloat64x2_t, z18,
+	 svamin_f64_x2 (z18, z4),
+	 svamin (z18, z4))
+
+/*
+** amin_z23_z23_z18:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z18\.d - z19\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z23_z23_z18, svfloat64x2_t, z23,
+	 svamin_f64_x2 (z23, z18),
+	 svamin (z23, z18))
+
+/*
+** amin_z28_z28_z0:
+**	famin	{z28\.d - z29\.d}, {z28\.d - z29\.d}, {z0\.d - z1\.d}
+**	ret
+*/
+TEST_XN (amin_z28_z28_z0, svfloat64x2_t, z28,
+	 svamin_f64_x2 (z28, z0),
+	 svamin (z28, z0))
+
+/*
+** amin_z0_z0_z18:
+**	famin	{z0\.d - z1\.d}, {z0\.d - z1\.d}, {z18\.d - z19\.d}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z18, svfloat64x2_t, z0,
+	 svamin_f64_x2 (z0, z18),
+	 svamin (z0, z18))
+
+/*
+** amin_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z4\.d - z5\.d}, {z4\.d - z5\.d}, [^\n]+
+** |
+**	famin	{z4\.d - z5\.d}, {z4\.d - z5\.d}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z4_z4_z23, svfloat64x2_t, z4,
+	 svamin_f64_x2 (z4, z23),
+	 svamin (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f64_x4.c b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f64_x4.c
new file mode 100644
index 0000000..8fbabe7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sme2/acle-asm/amin_f64_x4.c
@@ -0,0 +1,130 @@
+/* { dg-do assemble { target { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } */
+/* { dg-do compile { target { ! { aarch64_asm_sme2_ok && aarch64_asm_faminmax_ok } } } } */
+/* { dg-final { check-function-bodies "**" "" "-DCHECK_ASM" } } */
+
+#include "test_sme2_acle.h"
+#pragma GCC target "+faminmax"
+
+/*
+** amin_z0_z0_z4:
+**	famin	{z0\.d - z3\.d}, {z0\.d - z3\.d}, {z4\.d - z7\.d}
+**	ret
+*/
+TEST_XN (amin_z0_z0_z4, svfloat64x4_t, z0,
+	 svamin_f64_x4 (z0, z4),
+	 svamin (z0, z4))
+
+/*
+** amin_z0_z4_z0:
+**	famin	{z0\.d - z3\.d}, {z0\.d - z3\.d}, {z4\.d - z7\.d}
+**	ret
+*/
+TEST_XN (amin_z0_z4_z0, svfloat64x4_t, z0,
+	 svamin_f64_x4 (z4, z0),
+	 svamin (z4, z0))
+
+/*
+** amin_z0_z4_z28:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.d - z31\.d}
+** |
+**	famin	[^\n]+, {z28\.d - z31\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z4_z28, svfloat64x4_t, z0,
+	 svamin_f64_x4 (z4, z28),
+	 svamin (z4, z28))
+
+/*
+** amin_z18_z18_z4:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z4\.d - z7\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z18_z18_z4, svfloat64x4_t, z18,
+	 svamin_f64_x4 (z18, z4),
+	 svamin (z18, z4))
+
+/*
+** amin_z23_z23_z28:
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	[^\n]+, {z28\.d - z31\.d}
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	ret
+*/
+TEST_XN (amin_z23_z23_z28, svfloat64x4_t, z23,
+	 svamin_f64_x4 (z23, z28),
+	 svamin (z23, z28))
+
+/*
+** amin_z28_z28_z0:
+**	famin	{z28\.d - z31\.d}, {z28\.d - z31\.d}, {z0\.d - z3\.d}
+**	ret
+*/
+TEST_XN (amin_z28_z28_z0, svfloat64x4_t, z28,
+	 svamin_f64_x4 (z28, z0),
+	 svamin (z28, z0))
+
+/*
+** amin_z0_z0_z18:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z0\.d - z3\.d}, {z0\.d - z3\.d}, [^\n]+
+** |
+**	famin	{z0\.d - z3\.d}, {z0\.d - z3\.d}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z0_z0_z18, svfloat64x4_t, z0,
+	 svamin_f64_x4 (z0, z18),
+	 svamin (z0, z18))
+
+/*
+** amin_z4_z4_z23:
+** (
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	famin	{z4\.d - z7\.d}, {z4\.d - z7\.d}, [^\n]+
+** |
+**	famin	{z4\.d - z7\.d}, {z4\.d - z7\.d}, [^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+**	mov	[^\n]+
+** )
+**	ret
+*/
+TEST_XN (amin_z4_z4_z23, svfloat64x4_t, z4,
+	 svamin_f64_x4 (z4, z23),
+	 svamin (z4, z23))
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_2.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_2.c
index 218a660..13ebb9f 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_2.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_2.c
@@ -10,6 +10,6 @@ dupq (int x)
   return svdupq_s32 (x, 1, 2, 3);
 }
 
-/* { dg-final { scan-assembler {\tindex\tz[0-9]+\.s, #3, #-1} } } */
+/* { dg-final { scan-assembler {\tindex\tz[0-9]+\.s, #0, #1\n} } } */
 /* { dg-final { scan-assembler {\tins\tv[0-9]+\.s\[0\], w0\n} } } */
 /* { dg-final { scan-assembler {\tdup\tz[0-9]+\.q, z[0-9]+\.q\[0\]\n} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_4.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_4.c
index cbee6f2..13d27e2 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_4.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/dupq_4.c
@@ -10,6 +10,6 @@ dupq (int x)
   return svdupq_s32 (0, 1, x, 3);
 }
 
-/* { dg-final { scan-assembler {\tindex\tz[0-9]+\.s, #3, #-1} } } */
+/* { dg-final { scan-assembler {\tindex\tz[0-9]+\.s, #0, #1\n} } } */
 /* { dg-final { scan-assembler {\tins\tv[0-9]+\.s\[2\], w0\n} } } */
 /* { dg-final { scan-assembler {\tdup\tz[0-9]+\.q, z[0-9]+\.q\[0\]\n} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_1.c
new file mode 100644
index 0000000..6b920b8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/acle/general/perm_1.c
@@ -0,0 +1,14 @@
+/* { dg-options "-O2 -msve-vector-bits=256" } */
+
+#include <arm_sve.h>
+typedef svbfloat16_t vls_bfloat16_t __attribute__((arm_sve_vector_bits(32 * 8)));
+svbfloat16_t foo(vls_bfloat16_t a, vls_bfloat16_t b)
+{
+  svbfloat16_t zero = svreinterpret_bf16_f32 (svdup_n_f32 (0.0f));
+  return svzip2_bf16(zero, svuzp1_bf16(a,b));
+}
+
+
+/* { dg-final { scan-assembler-times {\tuzp1\t} 1 } } */
+/* { dg-final { scan-assembler-times {\tzip2\t} 1 } } */
+/* { dg-final { scan-assembler-not {\ttbl\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/mask_load_2.c b/gcc/testsuite/gcc.target/aarch64/sve/mask_load_2.c
new file mode 100644
index 0000000..66d9510
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/mask_load_2.c
@@ -0,0 +1,23 @@
+// { dg-do compile }
+// { dg-options "-march=armv8-a+sve -msve-vector-bits=128 -O3" }
+
+typedef struct Array {
+    int elems[3];
+} Array;
+
+int loop(Array **pp, int len, int idx) {
+    int nRet = 0;
+
+    #pragma GCC unroll 0
+    for (int i = 0; i < len; i++) {
+        Array *p = pp[i];
+        if (p) {
+            nRet += p->elems[idx];
+        }
+    }
+
+    return nRet;
+}
+
+// { dg-final { scan-assembler-times {ld1w\tz[0-9]+\.d, p[0-7]/z} 1 } }
+// { dg-final { scan-assembler-times {add\tz[0-9]+\.s, p[0-7]/m}  1 } }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary.c
index a8fd4c8..4708d57 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_int_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_int_opt_n.c
index 08cd6a0..4530b18 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_int_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_int_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_n.c
index f5c9cbf..3097459 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_single_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_single_n.c
index 91ae3c8..5e9d21c 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_single_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_opt_single_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_rotate.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_rotate.c
index 12368ce..768a740 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_rotate.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_rotate.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint64_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint64_opt_n.c
index dd52a58..ce14abb 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint64_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint64_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint_opt_n.c
index e55ddfb..ceeb5ae 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binary_uint_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binaryxn.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binaryxn.c
index 6796229..f8b6b82 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binaryxn.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-binaryxn.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-clast.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-clast.c
index 7f2ec4a..45f74ed 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-clast.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-clast.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_opt_n.c
index d18427b..fc601a1 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_wide_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_wide_opt_n.c
index 983ab5c..4959f1d 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_wide_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-compare_wide_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-count_pred.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-count_pred.c
index de36b66..d8a8a81 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-count_pred.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-count_pred.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-fold_left.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-fold_left.c
index 333140d..6cf2683 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-fold_left.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-fold_left.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
index 93d6693..a32b636 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext.c
index c88686a..72e743b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_index.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_index.c
index 5f4b562fc..1178104 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_index.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_index.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_offset.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_offset.c
index 0fe8ab3..ebd313a 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_offset.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_ext_gather_offset.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_sv.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_sv.c
index 758f00f..d531987 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_sv.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_sv.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_vs.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_vs.c
index f82471f..55c9cef 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_vs.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_gather_vs.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_replicate.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_replicate.c
index ba500b6..5532232 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_replicate.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load_replicate.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2 -march=armv8.2-a+sve+f64mm" } */
+/* { dg-options "-O2 -march=armv8.2-a+sve+f64mm -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch.c
index 71894c4..78bdb0b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_index.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_index.c
index 1b7cc42..e219007 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_index.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_index.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_offset.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_offset.c
index 7f4ff2d..98897e9 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_offset.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-prefetch_gather_offset.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ptest.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ptest.c
index 0a587fc..c6fe6b9 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ptest.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ptest.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-rdffr.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-rdffr.c
index d795f8e..7e2c1b9 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-rdffr.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-rdffr.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction.c
index 42b37ae..f7f75f6 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2 -fdump-tree-optimized" } */
+/* { dg-options "-O2 -fdump-tree-optimized -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction_wide.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction_wide.c
index bd9a980..54b6197 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction_wide.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-reduction_wide.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-shift_right_imm.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-shift_right_imm.c
index 62a0755..e8b8a55 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-shift_right_imm.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-shift_right_imm.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store.c
index 751e60e..1539f58 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_index.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_index.c
index 44792d3..21c8f6b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_index.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_index.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_offset.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_offset.c
index f3820e0..a908289 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_offset.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-store_scatter_offset.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-storexn.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-storexn.c
index e49266d..12b5e14 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-storexn.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-storexn.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_opt_n.c
index acdd141..89873fc 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_rotate.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_rotate.c
index 7698045..c6d2cfb 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_rotate.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-ternary_rotate.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary.c
index 037376b..8a3b3e0 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convert_narrowt.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convert_narrowt.c
index 1287a70..04bc049 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convert_narrowt.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convert_narrowt.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2 -march=armv8.2-a+sve+bf16" } */
+/* { dg-options "-O2 -march=armv8.2-a+sve+bf16 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convertxn.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convertxn.c
index f519266..f39d2c5 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convertxn.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_convertxn.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2 -march=armv8.2-a+sve+bf16" } */
+/* { dg-options "-O2 -march=armv8.2-a+sve+bf16 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_n.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_n.c
index fabde3e..4403e50 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_pred.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_pred.c
index 46c9592..f06b067 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_pred.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_pred.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_to_uint.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_to_uint.c
index b820bde..a851c4a 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_to_uint.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unary_to_uint.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unaryxn.c b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unaryxn.c
index 1e99b7f..dde812b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unaryxn.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-unaryxn.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_1.c
new file mode 100644
index 0000000..e6aa047
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_1.c
@@ -0,0 +1,44 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (FN (a[i], RHS) > c[i])				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (__builtin_fmaxf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fmaxf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fmaxf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_2.c
new file mode 100644
index 0000000..87125a6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmax_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_builtin_fmax_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_1.c
new file mode 100644
index 0000000..b9fded0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_1.c
@@ -0,0 +1,44 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (FN (a[i], RHS) > c[i])				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (__builtin_fminf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fminf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fminf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_2.c
new file mode 100644
index 0000000..5923b67
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_builtin_fmin_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_builtin_fmin_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_1.c
new file mode 100644
index 0000000..d328b37
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, b_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)
+
+TEST_ALL (__builtin_fmaxf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fmaxf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fmaxf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_2.c
new file mode 100644
index 0000000..f84ded5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmax_2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_builtin_fmax_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_1.c
new file mode 100644
index 0000000..1821f03
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, b_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)
+
+TEST_ALL (__builtin_fminf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fminf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fminf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_2.c
new file mode 100644
index 0000000..bceddf9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_builtin_fmin_2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_builtin_fmin_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_cvtf_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_cvtf_1.c
new file mode 100644
index 0000000..fa4dd15
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_cvtf_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define COND_CVT(TYPE0, TYPE1, TYPE2, COUNT)		    \
+  void                                                      \
+  test_##TYPE0##_##TYPE1##_##TYPE2 (TYPE0 *__restrict out,  \
+				    TYPE1 *__restrict a,    \
+				    TYPE0 *__restrict b,    \
+				    TYPE2 *__restrict p)    \
+  {							    \
+    for (unsigned int i = 0; i < COUNT; i++)                \
+      out[i] = p[i] ?  (TYPE0)a[i] : b[i];		    \
+  }
+
+#define TEST_CVTF(PFX, T)		   \
+  T (_Float16, PFX##int16_t, uint64_t, 32) \
+  T (_Float16, PFX##int16_t, uint32_t, 64) \
+  T (_Float16, PFX##int32_t, uint64_t, 32) \
+  T (_Float16, PFX##int32_t, uint32_t, 64) \
+  T (_Float16, PFX##int64_t, uint64_t, 32) \
+  T (float, PFX##int32_t, uint64_t, 32)    \
+  T (float, PFX##int64_t, uint64_t, 32)
+
+#define TEST_ALL(T) \
+  TEST_CVTF (, T)   \
+  TEST_CVTF (u, T)
+
+TEST_ALL (COND_CVT)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 6 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 8 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tscvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tucvtf\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fabs_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fabs_1.c
new file mode 100644
index 0000000..d959aa9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fabs_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_fabsf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_fabsf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_fabsf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_1.c
new file mode 100644
index 0000000..666cf89
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_1.c
@@ -0,0 +1,62 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+#define imm_p5 0.5
+
+#define ADD(A, B) A + B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, NAME, RHS, MERGE)	 \
+  void								 \
+  f_##TYPE0##_##TYPE1##_##NAME##_##MERGE (TYPE0 *__restrict out, \
+					 TYPE0 *__restrict a,	 \
+					 TYPE0 *__restrict b,	 \
+					 TYPE0 *__restrict c,	 \
+					 TYPE1 *__restrict p)	 \
+  {								 \
+    for (unsigned int i = 0; i < COUNT; i++)			 \
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		 \
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)	     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b[i], a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b[i], b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b[i], c_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, one, 1, a_i)     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, one, 1, b_i)     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, none, -1, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, none, -1, b_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, p5, 0.5, a_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, p5, 0.5, b_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, np5, -0.5, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, np5, -0.5, b_i)
+
+TEST_ALL (ADD, _Float16, uint64_t, 32)
+
+TEST_ALL (ADD, _Float16, uint32_t, 64)
+
+TEST_ALL (ADD, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 19 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 19 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 19 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 5 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 10 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_2.c
new file mode 100644
index 0000000..e59864b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fadd_2.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fadd_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 11 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 22 } } */
+/* { dg-final { scan-assembler-times {\tand} 33 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 19 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 19 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 19 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 5 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 10 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvt_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvt_1.c
new file mode 100644
index 0000000..3caae19
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvt_1.c
@@ -0,0 +1,41 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define COND_CVT(TYPE0, TYPE1, TYPE2, COUNT)		    \
+  void                                                      \
+  test_##TYPE0##_##TYPE1##_##TYPE2 (TYPE0 *__restrict out,  \
+				    TYPE1 *__restrict a,    \
+				    TYPE0 *__restrict b,    \
+				    TYPE2 *__restrict p)    \
+  {							    \
+    for (unsigned int i = 0; i < COUNT; i++)                \
+      out[i] = p[i] ?  (TYPE0)a[i] : b[i];		    \
+  }
+
+#define TEST_FCVT(T)		     \
+  T (_Float16, float, uint64_t, 32)  \
+  T (_Float16, float, uint32_t, 64)  \
+  T (_Float16, double, uint64_t, 32) \
+  T (float, double, uint64_t, 32)    \
+  T (float, _Float16, uint64_t, 32)  \
+  T (float, _Float16, uint32_t, 64)  \
+  T (double, _Float16, uint64_t,32)  \
+  T (double, float, uint64_t, 32)
+
+TEST_FCVT (COND_CVT)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvt\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvtz_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvtz_1.c
new file mode 100644
index 0000000..426d3af
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fcvtz_1.c
@@ -0,0 +1,55 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define COND_CVT(TYPE0, TYPE1, TYPE2, COUNT)		    \
+  void                                                      \
+  test_##TYPE0##_##TYPE1##_##TYPE2 (TYPE0 *__restrict out,  \
+				    TYPE1 *__restrict a,    \
+				    TYPE0 *__restrict b,    \
+				    TYPE2 *__restrict p)    \
+  {							    \
+    for (unsigned int i = 0; i < COUNT; i++)                \
+      out[i] = p[i] ?  (TYPE0)a[i] : b[i];		    \
+  }
+
+#define TEST_FCVTZ(PFX, T)		   \
+  T (PFX##int16_t, _Float16, uint64_t, 32) \
+  T (PFX##int16_t, _Float16, uint32_t, 64) \
+  T (PFX##int32_t, _Float16, uint64_t, 32) \
+  T (PFX##int32_t, _Float16, uint32_t, 64) \
+  T (PFX##int64_t, _Float16, uint64_t, 32) \
+  T (PFX##int32_t, float, uint64_t, 32)    \
+  T (PFX##int64_t, float, uint64_t, 32)    \
+  T (PFX##int32_t, double, uint64_t, 32)
+
+#define TEST_ALL(T) \
+  TEST_FCVTZ (, T)  \
+  TEST_FCVTZ (u, T)
+
+TEST_ALL (COND_CVT)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 6 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 8 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.h\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.d, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfcvtzs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcvtzu\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.d\n} 1 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_1.c
new file mode 100644
index 0000000..ec5653e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_1.c
@@ -0,0 +1,47 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define DIV(A, B) A / B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)	 \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i)
+
+TEST_ALL (DIV, _Float16, uint64_t, 32)
+
+TEST_ALL (DIV, _Float16, uint32_t, 64)
+
+TEST_ALL (DIV, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_2.c
new file mode 100644
index 0000000..1ca3dbf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fdiv_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fdiv_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 6 } } */
+/* { dg-final { scan-assembler-times {\tand} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfdivr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfdiv\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_1.c
new file mode 100644
index 0000000..d34872f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_1.c
@@ -0,0 +1,53 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-signed-zeros -ffinite-math-only -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define MAX(A, B) (A > B) ? A : B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, b_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)
+
+TEST_ALL (MAX, _Float16, uint64_t, 32)
+
+TEST_ALL (MAX, _Float16, uint32_t, 64)
+
+TEST_ALL (MAX, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_2.c
new file mode 100644
index 0000000..282f3ed
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmaxnm_2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-signed-zeros -ffinite-math-only" } */
+
+#include "unpacked_cond_fmaxnm_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_1.c
new file mode 100644
index 0000000..d6c3c38
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_1.c
@@ -0,0 +1,53 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-signed-zeros -ffinite-math-only -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define MIN(A, B) (A < B) ? A : B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)     \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0, b_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)
+
+TEST_ALL (MIN, _Float16, uint64_t, 32)
+
+TEST_ALL (MIN, _Float16, uint32_t, 64)
+
+TEST_ALL (MIN, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_2.c
new file mode 100644
index 0000000..8226a6f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fminnm_2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-signed-zeros -ffinite-math-only" } */
+
+#include "unpacked_cond_fminnm_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_1.c
new file mode 100644
index 0000000..cae9242
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##TYPE0##_##TYPE1##_##MERGE (TYPE0 *__restrict out,		\
+				 TYPE0 *__restrict a,		\
+				 TYPE0 *__restrict b,		\
+				 TYPE0 *__restrict c,		\
+				 TYPE1 *__restrict p)		\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN : MERGE;				\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, c_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)
+
+TEST_ALL (FMLA (f16), _Float16, uint64_t, 32)
+
+TEST_ALL (FMLA (f16), _Float16, uint32_t, 64)
+
+TEST_ALL (FMLA (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_2.c
new file mode 100644
index 0000000..72e04a4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmla_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fmla_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tand} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmad\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_1.c
new file mode 100644
index 0000000..db0f818
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##TYPE0##_##TYPE1##_##MERGE (TYPE0 *__restrict out,		\
+				 TYPE0 *__restrict a,		\
+				 TYPE0 *__restrict b,		\
+				 TYPE0 *__restrict c,		\
+				 TYPE1 *__restrict p)		\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN : MERGE;				\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, c_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)
+
+TEST_ALL (FMLS (f16), _Float16, uint64_t, 32)
+
+TEST_ALL (FMLS (f16), _Float16, uint32_t, 64)
+
+TEST_ALL (FMLS (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_2.c
new file mode 100644
index 0000000..3012052
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmls_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fmls_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tand} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmsb\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_1.c
new file mode 100644
index 0000000..1ae7678
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_1.c
@@ -0,0 +1,50 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+#define imm_p5 0.5
+
+#define MUL(A, B) A * B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)	 \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, imm_p5, a_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, imm_p5, b_i)
+
+TEST_ALL (MUL, _Float16, uint64_t, 32)
+
+TEST_ALL (MUL, _Float16, uint32_t, 64)
+
+TEST_ALL (MUL, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 10 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_2.c
new file mode 100644
index 0000000..21713f5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fmul_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fmul_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 10 } } */
+/* { dg-final { scan-assembler-times {\tand} 15 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 10 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 10 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fneg_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fneg_1.c
new file mode 100644
index 0000000..7280f4e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fneg_1.c
@@ -0,0 +1,39 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define NEG(X) -X
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (NEG, _Float16, uint64_t, 32)
+
+TEST_ALL (NEG, _Float16, uint32_t, 64)
+
+TEST_ALL (NEG, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c
new file mode 100644
index 0000000..07bab63
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##TYPE0##_##TYPE1##_##MERGE (TYPE0 *__restrict out,		\
+				 TYPE0 *__restrict a,		\
+				 TYPE0 *__restrict b,		\
+				 TYPE0 *__restrict c,		\
+				 TYPE1 *__restrict p)		\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN : MERGE;				\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, c_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)
+
+TEST_ALL (FNMLA (f16), _Float16, uint64_t, 32)
+
+TEST_ALL (FNMLA (f16), _Float16, uint32_t, 64)
+
+TEST_ALL (FNMLA (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_2.c
new file mode 100644
index 0000000..daef4e49
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmla_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fnmla_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tand} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmad\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfnmla\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c
new file mode 100644
index 0000000..5526378
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_1.c
@@ -0,0 +1,51 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##TYPE0##_##TYPE1##_##MERGE (TYPE0 *__restrict out,		\
+				 TYPE0 *__restrict a,		\
+				 TYPE0 *__restrict b,		\
+				 TYPE0 *__restrict c,		\
+				 TYPE1 *__restrict p)		\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN : MERGE;				\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, c_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)
+
+TEST_ALL (FNMLS (f16), _Float16, uint64_t, 32)
+
+TEST_ALL (FNMLS (f16), _Float16, uint32_t, 64)
+
+TEST_ALL (FNMLS (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_2.c
new file mode 100644
index 0000000..8a8f348
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fnmls_2.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fnmls_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 8 } } */
+/* { dg-final { scan-assembler-times {\tand} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 12 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/z, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/z, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfnmsb\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfnmls\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_1.c
new file mode 100644
index 0000000..ed4efb6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_roundf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_roundf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_roundf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_2.c
new file mode 100644
index 0000000..f20e2e6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinta_2.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -mtune=generic -ftree-vectorize" } */
+
+#include "unpacked_cond_frinta_1.c"
+
+/* Test that we don't drop SELs without -fno-trapping-math.  */
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tsel\t} 6 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinti_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinti_1.c
new file mode 100644
index 0000000..d682d15
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frinti_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_nearbyintf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_nearbyintf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_nearbyintf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintm_1.c
new file mode 100644
index 0000000..7d429b3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintm_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_floorf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_floorf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_floorf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintp_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintp_1.c
new file mode 100644
index 0000000..c6d0c8c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintp_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_ceilf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_ceilf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_ceilf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintx_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintx_1.c
new file mode 100644
index 0000000..b8afef1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintx_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_rintf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_rintf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_rintf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintz_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintz_1.c
new file mode 100644
index 0000000..d55279b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_frintz_1.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, MERGE)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##MERGE (TYPE1 *__restrict p,	\
+    					TYPE0 *__restrict out,	\
+					TYPE0 *__restrict a,	\
+					TYPE0 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i]) : MERGE;			\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, a_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (__builtin_truncf16, _Float16, uint64_t, 32)
+
+TEST_ALL (__builtin_truncf16, _Float16, uint32_t, 64)
+
+TEST_ALL (__builtin_truncf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_1.c
new file mode 100644
index 0000000..eafd169
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_1.c
@@ -0,0 +1,56 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include <stdint.h>
+
+#define a_i a[i]
+#define b_i b[i]
+#define c_i c[i]
+#define imm_p5 0.5
+
+#define SUBR(A, B) B - A
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS, MERGE)		\
+  void								\
+  f_##TYPE0##_##TYPE1##_##RHS##_##MERGE (TYPE0 *__restrict out,	\
+					 TYPE0 *__restrict a,	\
+					 TYPE0 *__restrict b,	\
+					 TYPE0 *__restrict c,	\
+					 TYPE1 *__restrict p)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = p[i] ? FN (a[i], (TYPE0)RHS) : MERGE;		\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)	 \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, a_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, c_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, a_i)	 \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1, b_i)	 \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, imm_p5, a_i) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, imm_p5, b_i)
+
+TEST_ALL (SUBR, _Float16, uint64_t, 32)
+
+TEST_ALL (SUBR, _Float16, uint32_t, 64)
+
+TEST_ALL (SUBR, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_2.c
new file mode 100644
index 0000000..cd7a0e1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_cond_fsubr_2.c
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include "unpacked_cond_fsubr_1.c"
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-9]+\.d} 14 } } */
+/* { dg-final { scan-assembler-times {\tand} 21 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 13 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 13 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tmovprfx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 6 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 4 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 4 } } */
+
+/* { dg-final { scan-assembler-not {\tsel\t} } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fabs_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fabs_1.c
new file mode 100644
index 0000000..f09cfe8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fabs_1.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_fabsf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_fabsf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_fabsf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfabs\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_1.c
new file mode 100644
index 0000000..9675f56
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_1.c
@@ -0,0 +1,52 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define ADD(A, B)  A + B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, NAME, RHS)		\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##NAME (TYPE1 *__restrict out,	\
+				       TYPE0 *__restrict a,     \
+				       TYPE0 *__restrict b,     \
+                                       TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (FN (a[i], (TYPE0)RHS) > c[i])				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)      \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i, b[i]) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, p5, 0.5)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, np5, -0.5) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, one, 1)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, none, -1)
+
+TEST_ALL (ADD, _Float16, uint64_t, 32)
+
+TEST_ALL (ADD, _Float16, uint32_t, 64)
+
+TEST_ALL (ADD, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 10 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 11 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 11 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 11 } } */
+
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
new file mode 100644
index 0000000..7a74efd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
@@ -0,0 +1,26 @@
+/* { dg-do compile }*/
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fadd_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 12 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 11 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 11 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 11 } } */
+
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfadd\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_1.c
new file mode 100644
index 0000000..bf9c127
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_1.c
@@ -0,0 +1,602 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 --param=aarch64-autovec-preference=sve-only -fno-schedule-insns -fno-schedule-insns2" } */
+
+#include <stdint.h>
+
+#define UNLT(A, B) (!__builtin_isgreaterequal (A, B))
+#define UNLE(A, B) (!__builtin_isgreater (A, B))
+#define UNGT(A, B) (!__builtin_islessequal (A, B))
+#define UNGE(A, B) (!__builtin_isless (A, B))
+#define UNEQ(A, B) (!__builtin_islessgreater (A, B))
+
+#define EQ(A, B) ((A) == (B))
+#define NE(A, B) ((A) != (B))
+#define LE(A, B) ((A) <= (B))
+#define LT(A, B) ((A) < (B))
+#define GE(A, B) ((A) >= (B))
+#define GT(A, B) ((A) > (B))
+#define ORDERED(A, B) (!__builtin_isunordered (A, B))
+#define UNORDERED(A, B) (__builtin_isunordered (A, B))
+
+#define b_i b[i]
+
+#define TEST_FCM(TYPE0, TYPE1, CMP, RHS, COUNT)			\
+  void								\
+  f_##TYPE0##_##TYPE1##_##CMP##_##RHS (TYPE0 *__restrict out,	\
+				       TYPE1 *__restrict a,	\
+				       TYPE1 *__restrict b)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      out[i] = CMP (a[i], RHS) ? 3 : out[i];			\
+  }
+
+#define TEST_CC_REG(CMP)		      \
+  TEST_FCM (uint64_t, float, CMP, b_i, 32)    \
+  TEST_FCM (uint32_t, _Float16, CMP, b_i, 64) \
+  TEST_FCM (uint64_t, _Float16, CMP, b_i, 32)
+
+#define TEST_CC_ALL(CMP)		    \
+  TEST_CC_REG (CMP)			    \
+  TEST_FCM (uint64_t, float, CMP, 0, 32)    \
+  TEST_FCM (uint32_t, _Float16, CMP, 0, 64) \
+  TEST_FCM (uint64_t, _Float16, CMP, 0, 32)
+
+
+/*
+** f_uint64_t_float_UNLT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmge	p[0-9]+\.s, \3/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_UNLT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmge	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_UNLT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmge	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (UNLT)
+
+/*
+** f_uint64_t_float_UNLE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmgt	p[0-9]+\.s, \3/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_UNLE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmgt	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_UNLE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmgt	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (UNLE)
+
+/*
+** f_uint64_t_float_UNGT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmle	p[0-9]+\.s, \3/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_UNGT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmle	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_UNGT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmle	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (UNGT)
+
+/*
+** f_uint64_t_float_UNGE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmlt	p[0-9]+\.s, \3/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_UNGE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmlt	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_UNGE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmlt	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (UNGE)
+
+/*
+** f_uint64_t_float_UNEQ_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmne	p[0-9]+\.s, \3/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_UNEQ_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmne	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_UNEQ_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	(p[0-9]+)\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**	not	(p[0-9]+)\.b, \1/z, \2\.b
+**	fcmne	p[0-9]+\.h, \3/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (UNEQ)
+
+/*
+** f_uint64_t_float_EQ_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmeq	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_EQ_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmeq	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_EQ_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmeq	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t_float_EQ_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmeq	p[0-9]+\.s, \1/z, z[0-9]+\.s, #0.0
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_EQ_0:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmeq	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_EQ_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmeq	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+TEST_CC_ALL (EQ)
+
+/*
+** f_uint64_t_float_NE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmne	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_NE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmne	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_NE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmne	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t_float_NE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmne	p[0-9]+\.s, \1/z, z[0-9]+\.s, #0.0
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_NE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmne	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_NE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmne	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+TEST_CC_ALL (NE)
+
+/*
+** f_uint64_t_float_LE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmle	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_LE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmle	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_LE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmle	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t_float_LE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmle	p[0-9]+\.s, \1/z, z[0-9]+\.s, #0.0
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_LE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmle	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_LE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmle	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+TEST_CC_ALL (LE)
+
+/*
+** f_uint64_t_float_LT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmlt	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_LT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmlt	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_LT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmlt	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t_float_LT_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmlt	p[0-9]+\.s, \1/z, z[0-9]+\.s, #0.0
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_LT_0:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmlt	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_LT_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmlt	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+TEST_CC_ALL (LT)
+
+/*
+** f_uint64_t_float_GE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmge	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_GE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmge	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_GE_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmge	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t_float_GE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmge	p[0-9]+\.s, \1/z, z[0-9]+\.s, #0.0
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_GE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmge	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_GE_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmge	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+TEST_CC_ALL (GE)
+
+/*
+** f_uint64_t_float_GT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmgt	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_GT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmgt	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_GT_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmgt	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t_float_GT_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmgt	p[0-9]+\.s, \1/z, z[0-9]+\.s, #0.0
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_GT_0:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmgt	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_GT_0:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmgt	p[0-9]+\.h, \1/z, z[0-9]+\.h, #0.0
+**  ...
+*/
+TEST_CC_ALL (GT)
+
+/*
+** f_uint64_t_float_ORDERED_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_ORDERED_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_ORDERED_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (ORDERED)
+
+/*
+** f_uint64_t_float_UNORDERED_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	p[0-9]+\.s, \1/z, z[0-9]+\.s, z[0-9]+\.s
+**  ...
+*/
+
+/*
+** f_uint32_t__Float16_UNORDERED_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.s, all
+**  ...
+**	fcmuo	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+
+/*
+** f_uint64_t__Float16_UNORDERED_b_i:
+**  ...
+**	ptrue	(p[0-9]+)\.d, all
+**  ...
+**	fcmuo	p[0-9]+\.h, \1/z, z[0-9]+\.h, z[0-9]+\.h
+**  ...
+*/
+TEST_CC_REG (UNORDERED)
+
+
+/* { dg-final { check-function-bodies "**" "" ""} } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_2.c
new file mode 100644
index 0000000..ab210da
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_2.c
@@ -0,0 +1,50 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 --param=aarch64-autovec-preference=sve-only -fno-trapping-math" } */
+
+#include "unpacked_fcm_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 32 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 32 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 32 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmeq\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcmeq\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmeq\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmeq\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, #0.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmne\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmne\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmne\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmne\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, #0.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmle\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcmle\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmle\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmle\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, #0.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmlt\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcmlt\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmlt\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmlt\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, #0.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmge\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcmge\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmge\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmge\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, #0.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmgt\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfcmgt\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmgt\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfcmgt\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, #0.0\n} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfcmuo\tp[0-9]+\.s, p[0-7]/z, z[0-9]+\.s, z[0-9]+\.s\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tfcmuo\tp[0-9]+\.h, p[0-7]/z, z[0-9]+\.h, z[0-9]+\.h\n} 6 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_combines_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_combines_1.c
new file mode 100644
index 0000000..d793a6c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_combines_1.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 --param=aarch64-autovec-preference=sve-only -fno-trapping-math" } */
+
+#include "unpacked_fcm_1.c"
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 32 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 32 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 32 } } */
+
+/* Drop a PTRUE predicated AND with the loop mask and comparison result in
+   favour of predicating the comparison with the loop mask.  */
+/* { dg-final { scan-assembler-not {\tand\t} } } */
+
+/* Similarly, for codes that are implemented via an inversion, prefer
+   NOT (predicated with the loop mask) over BIC+PTRUE.  */
+/* { dg-final { scan-assembler-not {\tbic\t} } } */
+/* { dg-final { scan-assembler-times {\tnot\t} 15 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_combines_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_combines_2.c
new file mode 100644
index 0000000..b85391b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fcm_combines_2.c
@@ -0,0 +1,35 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 --param=aarch64-autovec-preference=sve-only -fno-trapping-math" } */
+
+#include <stdint.h>
+
+/* Ensure that we still emit NOR here, rather than two NOTs.  */
+
+#define TEST_FCM_NOR(TYPE0, TYPE1, CMP, COUNT)		\
+  void							\
+  f_##TYPE0##_##TYPE1##_##CMP (TYPE0 *__restrict out,	\
+			       TYPE1 *__restrict a,	\
+			       TYPE1 *__restrict b,	\
+			       TYPE1 *__restrict c)	\
+  {								    \
+    for (unsigned int i = 0; i < COUNT; i++)			    \
+      out[i] = !(CMP (a[i], c[i]) | CMP (b[i], c[i])) ? 3 : out[i]; \
+  }
+
+#define GT(A, B) ((A) > (B))
+
+TEST_FCM_NOR (uint64_t, float, GT, 32)
+TEST_FCM_NOR (uint64_t, _Float16, GT, 32)
+TEST_FCM_NOR (uint32_t, _Float16, GT, 64)
+
+TEST_FCM_NOR (uint64_t, float, __builtin_isunordered, 32)
+TEST_FCM_NOR (uint64_t, _Float16, __builtin_isunordered, 32)
+TEST_FCM_NOR (uint32_t, _Float16, __builtin_isunordered, 64)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 6 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 6 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 6 } } */
+
+/* { dg-final { scan-assembler-not {\tbic\t} } } */
+/* { dg-final { scan-assembler-not {\tnot\t} } } */
+/* { dg-final { scan-assembler-times {\tnor\tp[0-9]+\.b, p[0-9]+/z, p[0-9]+\.b, p[0-9]+\.b\n} 6 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_1.c
new file mode 100644
index 0000000..78d0d9c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define DIV(A, B) A / B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (FN (a[i], (TYPE0)RHS) > c[i])				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)
+
+TEST_ALL (DIV, _Float16, uint64_t, 32)
+
+TEST_ALL (DIV, _Float16, uint32_t, 64)
+
+TEST_ALL (DIV, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfdivr?\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdivr?\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_2.c
new file mode 100644
index 0000000..a8f70e1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fdiv_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfdivr?\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfdivr?\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_3.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_3.c
new file mode 100644
index 0000000..ecd088f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fdiv_3.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-Ofast -moverride=sve_width=2048 -mlow-precision-div" } */
+
+#include "unpacked_fdiv_1.c"
+
+/* { dg-final { scan-assembler-not {\tfrecpe\tz[0-9]+\.h} } } */
+/* { dg-final { scan-assembler-not {\tfrecps\tz[0-9]+\.h} } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tfrecpe\tz[0-9]+\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrecps\tz[0-9]+\.s} 1 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_1.c
new file mode 100644
index 0000000..5239e4b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_1.c
@@ -0,0 +1,45 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define MAX(A, B) (A > B) ? A : B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (c[i] = FN (a[i], RHS))				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (MAX, _Float16, uint64_t, 32)
+
+TEST_ALL (MAX, _Float16, uint32_t, 64)
+
+TEST_ALL (MAX, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_2.c
new file mode 100644
index 0000000..11aa7c0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmaxnm_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -fno-trapping-math -moverride=sve_width=2048" } */
+
+#include "unpacked_fmaxnm_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmaxnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_1.c
new file mode 100644
index 0000000..02a5f46
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_1.c
@@ -0,0 +1,46 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define MIN(A, B) (A < B) ? A : B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (c[i] = FN (a[i], RHS) )				\
+	out[i] = 3;						\
+  }
+
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT) \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 0)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (MIN, _Float16, uint64_t, 32)
+
+TEST_ALL (MIN, _Float16, uint32_t, 64)
+
+TEST_ALL (MIN, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_2.c
new file mode 100644
index 0000000..81f583b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fminnm_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fno-signed-zeros -ffinite-math-only -fno-trapping-math -moverride=sve_width=2048" } */
+
+#include "unpacked_fminnm_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.0\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.0\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfminnm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_1.c
new file mode 100644
index 0000000..312bccc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)	\
+  void						\
+  f_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+		       TYPE0 *__restrict a,	\
+		       TYPE0 *__restrict b,	\
+		       TYPE0 *__restrict c,	\
+		       TYPE0 *__restrict d)	\
+  {						\
+    for (unsigned int i = 0; i < COUNT; i++)	\
+      if (FN > d[i])				\
+	out[i] = 3;				\
+  }
+
+TEST_FN (FMLA (f16), _Float16, uint64_t, 32)
+
+TEST_FN (FMLA (f16), _Float16, uint32_t, 64)
+
+TEST_FN (FMLA (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fmla|fmad)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fmla|fmad)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_2.c
new file mode 100644
index 0000000..ca3f94d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmla_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fmla_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fmla|fmad)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fmla|fmad)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_1.c
new file mode 100644
index 0000000..f7cbfb3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)	\
+  void						\
+  f_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+		       TYPE0 *__restrict a,	\
+		       TYPE0 *__restrict b,	\
+		       TYPE0 *__restrict c,	\
+		       TYPE0 *__restrict d)	\
+  {						\
+    for (unsigned int i = 0; i < COUNT; i++)	\
+      if (FN > d[i])				\
+	out[i] = 3;				\
+  }
+
+TEST_FN (FMLS (f16), _Float16, uint64_t, 32)
+
+TEST_FN (FMLS (f16), _Float16, uint32_t, 64)
+
+TEST_FN (FMLS (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fmls|fmsb)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fmls|fmsb)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_2.c
new file mode 100644
index 0000000..387dbec
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmls_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fmls_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fmls|fmsb)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fmls|fmsb)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_1.c
new file mode 100644
index 0000000..a180a07
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_1.c
@@ -0,0 +1,43 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define immp5 0.5
+#define MUL(A, B) A * B
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (FN (a[i], (TYPE0)RHS) > c[i])				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, immp5)
+
+TEST_ALL (MUL, _Float16, uint64_t, 32)
+
+TEST_ALL (MUL, _Float16, uint32_t, 64)
+
+TEST_ALL (MUL, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 5 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 5 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
new file mode 100644
index 0000000..eb05600
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fmul_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 5 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 5 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 5 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfmul\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fneg_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fneg_1.c
new file mode 100644
index 0000000..d489ecb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fneg_1.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define NEG(X) -X
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (NEG, _Float16, uint64_t, 32)
+
+TEST_FN (NEG, _Float16, uint32_t, 64)
+
+TEST_FN (NEG, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfneg\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_1.c
new file mode 100644
index 0000000..bf13ff5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)	\
+  void						\
+  f_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+		       TYPE0 *__restrict a,	\
+		       TYPE0 *__restrict b,	\
+		       TYPE0 *__restrict c,	\
+		       TYPE0 *__restrict d)	\
+  {						\
+    for (unsigned int i = 0; i < COUNT; i++)	\
+      if (FN > d[i])				\
+	out[i] = 3;				\
+  }
+
+TEST_FN (FNMLA (f16), _Float16, uint64_t, 32)
+
+TEST_FN (FNMLA (f16), _Float16, uint32_t, 64)
+
+TEST_FN (FNMLA (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fnmla|fnmad)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fnmla|fnmad)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_2.c
new file mode 100644
index 0000000..64130ba
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmla_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fnmla_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fnmla|fnmad)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fnmla|fnmad)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_1.c
new file mode 100644
index 0000000..399920a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_1.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define FMLA(SUFF)  __builtin_fma##SUFF (a[i], b[i], c[i])
+#define FMLS(SUFF)  __builtin_fma##SUFF (a[i], -b[i], c[i])
+#define FNMLA(SUFF) -FMLA (SUFF)
+#define FNMLS(SUFF) -FMLS (SUFF)
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)	\
+  void						\
+  f_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+		       TYPE0 *__restrict a,	\
+		       TYPE0 *__restrict b,	\
+		       TYPE0 *__restrict c,	\
+		       TYPE0 *__restrict d)	\
+  {						\
+    for (unsigned int i = 0; i < COUNT; i++)	\
+      if (FN > d[i])				\
+	out[i] = 3;				\
+  }
+
+TEST_FN (FNMLS (f16), _Float16, uint64_t, 32)
+
+TEST_FN (FNMLS (f16), _Float16, uint32_t, 64)
+
+TEST_FN (FNMLS (f32), float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fnmls|fnmsb)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fnmls|fnmsb)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_2.c
new file mode 100644
index 0000000..59fb7f9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fnmls_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fnmls_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 4 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 4 } } */
+
+/* { dg-final { scan-assembler-times {\t(fnmls|fnmsb)\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\t(fnmls|fnmsb)\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_1.c
new file mode 100644
index 0000000..3cbdef3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_roundf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_roundf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_roundf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_2.c
new file mode 100644
index 0000000..4564686
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinta_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frinta_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrinta\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_1.c
new file mode 100644
index 0000000..7645fed
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_nearbyintf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_nearbyintf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_nearbyintf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_2.c
new file mode 100644
index 0000000..eadce07
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frinti_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frinti_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrinti\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_1.c
new file mode 100644
index 0000000..98f85fb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_floorf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_floorf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_floorf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_2.c
new file mode 100644
index 0000000..56988be
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintm_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frintm_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintm\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_1.c
new file mode 100644
index 0000000..f233697
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_ceilf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_ceilf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_ceilf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_2.c
new file mode 100644
index 0000000..c24c632
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintp_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frintp_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintp\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_1.c
new file mode 100644
index 0000000..73403a5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_rintf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_rintf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_rintf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_2.c
new file mode 100644
index 0000000..e8b8924
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintx_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frintx_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintx\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_1.c
new file mode 100644
index 0000000..7377843
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize" } */
+
+#include <stdint.h>
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT)		\
+  void							\
+  f_##FN##_##TYPE0##_##TYPE1 (TYPE1 *__restrict out,	\
+			      TYPE0 *__restrict a,      \
+			      TYPE0 *__restrict b)      \
+  {							\
+    for (unsigned int i = 0; i < COUNT; i++)		\
+      if (FN (a[i]) > b[i])				\
+	out[i] = 3;					\
+  }
+
+TEST_FN (__builtin_truncf16, _Float16, uint64_t, 32)
+
+TEST_FN (__builtin_truncf16, _Float16, uint32_t, 64)
+
+TEST_FN (__builtin_truncf32, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_2.c
new file mode 100644
index 0000000..1779122
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_frintz_2.c
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048 -ftree-vectorize -fno-trapping-math" } */
+
+#include "unpacked_frintz_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 2 } } */
+
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfrintz\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_1.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_1.c
new file mode 100644
index 0000000..2cc8ec2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_1.c
@@ -0,0 +1,46 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -moverride=sve_width=2048" } */
+
+#include <stdint.h>
+
+#define b_i b[i]
+#define immp5 0.5
+#define SUBR(A, B) B - A
+
+#define TEST_FN(FN, TYPE0, TYPE1, COUNT, RHS)			\
+  void								\
+  f_##FN##_##TYPE0##_##TYPE1##_##RHS (TYPE1 *__restrict out,	\
+				      TYPE0 *__restrict a,      \
+				      TYPE0 *__restrict b,      \
+                                      TYPE0 *__restrict c)	\
+  {								\
+    for (unsigned int i = 0; i < COUNT; i++)			\
+      if (FN (a[i], (TYPE0)RHS) > c[i])				\
+	out[i] = 3;						\
+  }
+
+#define TEST_ALL(FN, TYPE0, TYPE1, COUNT)   \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, b_i)    \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, immp5)  \
+  TEST_FN (FN, TYPE0, TYPE1, COUNT, 1)
+
+TEST_ALL (SUBR, _Float16, uint64_t, 32)
+
+TEST_ALL (SUBR, _Float16, uint32_t, 64)
+
+TEST_ALL (SUBR, float, uint64_t, 32)
+
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.d} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfsubr?\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfsubr?\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
new file mode 100644
index 0000000..de9325c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
@@ -0,0 +1,20 @@
+/* { dg-do compile }*/
+/* { dg-options "-O2 -moverride=sve_width=2048 -fno-trapping-math" } */
+
+#include "unpacked_fsubr_1.c"
+
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
+/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.d} 7 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.s, z[0-9]+\.s, z[0-9]+\.s\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #0.5\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.s, p[0-7]/m, z[0-9]+\.s, #1.0\n} 1 } } */
+
+/* { dg-final { scan-assembler-times {\tfsub\tz[0-9]+\.h, z[0-9]+\.h, z[0-9]+\.h\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #0.5\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tfsubr\tz[0-9]+\.h, p[0-7]/m, z[0-9]+\.h, #1.0\n} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vec_init_3.c b/gcc/testsuite/gcc.target/aarch64/sve/vec_init_3.c
index 25910db..5100a87 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/vec_init_3.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/vec_init_3.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -mlittle-endian" } */
 /* { dg-final { check-function-bodies "**" "" "" } } */
 
 typedef char v16qi __attribute__ ((vector_size (16)));
@@ -8,7 +8,7 @@ typedef short v8hi __attribute__ ((vector_size (16)));
 typedef short v4hi __attribute__ ((vector_size (8)));
 typedef int v4si __attribute__ ((vector_size (16)));
 typedef int v2si __attribute__ ((vector_size (8)));
-typedef long v2di __attribute__ ((vector_size (16)));
+typedef long long v2di __attribute__ ((vector_size (16)));
 
 /*
 ** f_v16qi:
@@ -97,3 +97,113 @@ g_v4si (void)
 {
   return (v4si){ 3, -1, -5, -9 };
 }
+
+/*
+** g_min_1:
+**	index	z0\.s, #-16, #1
+**	ret
+*/
+v4si
+g_min_1 (void)
+{
+  return (v4si){ -16, -15, -14, -13 };
+}
+
+/*
+** g_min_min:
+**	index	z0\.s, #-16, #-16
+**	ret
+*/
+v4si
+g_min_min (void)
+{
+  return (v4si){ -16, -32, -48, -64 };
+}
+
+/*
+** g_min_max:
+**	index	z0\.s, #-16, #15
+**	ret
+*/
+v4si
+g_min_max (void)
+{
+  return (v4si){ -16, -1, 14, 29 };
+}
+
+/*
+** g_max_1:
+**	index	z0\.s, #15, #1
+**	ret
+*/
+v4si
+g_max_1 (void)
+{
+  return (v4si){ 15, 16, 17, 18 };
+}
+
+/*
+** g_max_min:
+**	index	z0\.s, #15, #-16
+**	ret
+*/
+v4si
+g_max_min (void)
+{
+  return (v4si){ 15, -1, -17, -33 };
+}
+
+/*
+** g_max_max:
+**	index	z0\.s, #15, #15
+**	ret
+*/
+v4si
+g_max_max (void)
+{
+  return (v4si){ 15, 30, 45, 60 };
+}
+
+/*
+** g_ob_1:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_1 (void)
+{
+  return (v4si){ -17, -16, -15, -14 };
+}
+
+/*
+** g_ob_2:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_2 (void)
+{
+  return (v4si){ 16, 17, 18, 19 };
+}
+
+/*
+** g_ob_3:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_3 (void)
+{
+  return (v4si){ 0, -17, -34, -51 };
+}
+
+/*
+** g_ob_4:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_4 (void)
+{
+  return (v4si){ 0, 16, 32, 48 };
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vec_init_4.c b/gcc/testsuite/gcc.target/aarch64/sve/vec_init_4.c
new file mode 100644
index 0000000..0681d95
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/vec_init_4.c
@@ -0,0 +1,209 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mbig-endian" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+typedef char v16qi __attribute__ ((vector_size (16)));
+typedef char v8qi __attribute__ ((vector_size (8)));
+typedef short v8hi __attribute__ ((vector_size (16)));
+typedef short v4hi __attribute__ ((vector_size (8)));
+typedef int v4si __attribute__ ((vector_size (16)));
+typedef int v2si __attribute__ ((vector_size (8)));
+typedef long long v2di __attribute__ ((vector_size (16)));
+
+/*
+** f_v16qi:
+**	index	z0\.b, #15, #-1
+**	ret
+*/
+v16qi
+f_v16qi (void)
+{
+  return (v16qi){ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
+}
+
+/*
+** f_v8qi:
+**	index	z0\.b, #7, #-1
+**	ret
+*/
+v8qi
+f_v8qi (void)
+{
+  return (v8qi){ 0, 1, 2, 3, 4, 5, 6, 7 };
+}
+
+/*
+** f_v8hi:
+**	index	z0\.h, #7, #-1
+**	ret
+*/
+v8hi
+f_v8hi (void)
+{
+  return (v8hi){ 0, 1, 2, 3, 4, 5, 6, 7 };
+}
+
+/*
+** f_v4hi:
+**	index	z0\.h, #3, #-1
+**	ret
+*/
+v4hi
+f_v4hi (void)
+{
+  return (v4hi){ 0, 1, 2, 3 };
+}
+
+/*
+** f_v4si:
+**	index	z0\.s, #3, #-1
+**	ret
+*/
+v4si
+f_v4si (void)
+{
+  return (v4si){ 0, 1, 2, 3 };
+}
+
+/*
+** f_v2si:
+**	index	z0\.s, #1, #-1
+**	ret
+*/
+v2si
+f_v2si (void)
+{
+  return (v2si){ 0, 1 };
+}
+
+/*
+** f_v2di:
+**	index	z0\.d, #1, #-1
+**	ret
+*/
+v2di
+f_v2di (void)
+{
+  return (v2di){ 0, 1 };
+}
+
+/*
+** g_v4si:
+**	index	z0\.s, #-9, #4
+**	ret
+*/
+v4si
+g_v4si (void)
+{
+  return (v4si){ 3, -1, -5, -9 };
+}
+
+/*
+** g_min_1:
+**	index	z0\.s, #-16, #1
+**	ret
+*/
+v4si
+g_min_1 (void)
+{
+  return (v4si){ -13, -14, -15, -16 };
+}
+
+/*
+** g_min_min:
+**	index	z0\.s, #-16, #-16
+**	ret
+*/
+v4si
+g_min_min (void)
+{
+  return (v4si){ -64, -48, -32, -16 };
+}
+
+/*
+** g_min_max:
+**	index	z0\.s, #-16, #15
+**	ret
+*/
+v4si
+g_min_max (void)
+{
+  return (v4si){ 29, 14, -1, -16 };
+}
+
+/*
+** g_max_1:
+**	index	z0\.s, #15, #1
+**	ret
+*/
+v4si
+g_max_1 (void)
+{
+  return (v4si){ 18, 17, 16, 15 };
+}
+
+/*
+** g_max_min:
+**	index	z0\.s, #15, #-16
+**	ret
+*/
+v4si
+g_max_min (void)
+{
+  return (v4si){ -33, -17, -1, 15 };
+}
+
+/*
+** g_max_max:
+**	index	z0\.s, #15, #15
+**	ret
+*/
+v4si
+g_max_max (void)
+{
+  return (v4si){ 60, 45, 30, 15 };
+}
+
+/*
+** g_ob_1:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_1 (void)
+{
+  return (v4si){ -14, -15, -16, -17 };
+}
+
+/*
+** g_ob_2:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_2 (void)
+{
+  return (v4si){ 19, 18, 17, 16 };
+}
+
+/*
+** g_ob_3:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_3 (void)
+{
+  return (v4si){ -51, -34, -17, 0 };
+}
+
+/*
+** g_ob_4:
+**	((?!index).)*
+**	ret
+*/
+v4si
+g_ob_4 (void)
+{
+  return (v4si){ 48, 32, 16, 0 };
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1.c
index 5472e30..9db60b1 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1.c
@@ -1,5 +1,5 @@
 /* { dg-options "-O2 -msve-vector-bits=256" } */
-/* { dg-final { check-function-bodies "**" "" "" { target { le } } } } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
 
 #include <arm_sve.h>
 
@@ -15,7 +15,7 @@ typedef svuint64_t fixed_uint64_t __attribute__((arm_sve_vector_bits(256)));
 **	trn1	z0\.d, z0\.d, z0\.d
 **	ret
 */
-fixed_uint64_t
+[[gnu::noipa]] fixed_uint64_t
 f1 (fixed_uint64_t z0)
 {
   return __builtin_shufflevector (z0, z0, 0, 0, 2, 2);
@@ -26,7 +26,7 @@ f1 (fixed_uint64_t z0)
 **	trn2	z0\.d, z0\.d, z0\.d
 **	ret
 */
-fixed_uint64_t
+[[gnu::noipa]] fixed_uint64_t
 f2 (fixed_uint64_t z0)
 {
   return __builtin_shufflevector (z0, z0, 1, 1, 3, 3);
@@ -37,7 +37,7 @@ f2 (fixed_uint64_t z0)
 **	dupq	z0\.s, z0\.s\[0\]
 **	ret
 */
-fixed_int32_t
+[[gnu::noipa]] fixed_int32_t
 f3 (fixed_int32_t z0)
 {
   return __builtin_shufflevector (z0, z0, 0, 0, 0, 0, 4, 4, 4, 4);
@@ -48,7 +48,7 @@ f3 (fixed_int32_t z0)
 **	dupq	z0\.s, z0\.s\[1\]
 **	ret
 */
-fixed_int32_t
+[[gnu::noipa]] fixed_int32_t
 f4 (fixed_int32_t z0)
 {
   return __builtin_shufflevector (z0, z0, 1, 1, 1, 1, 5, 5, 5, 5);
@@ -59,7 +59,7 @@ f4 (fixed_int32_t z0)
 **	dupq	z0\.s, z0\.s\[2\]
 **	ret
 */
-fixed_int32_t
+[[gnu::noipa]] fixed_int32_t
 f5 (fixed_int32_t z0)
 {
   return __builtin_shufflevector (z0, z0, 2, 2, 2, 2, 6, 6, 6, 6);
@@ -70,7 +70,7 @@ f5 (fixed_int32_t z0)
 **	dupq	z0\.s, z0\.s\[3\]
 **	ret
 */
-fixed_int32_t
+[[gnu::noipa]] fixed_int32_t
 f6 (fixed_int32_t z0)
 {
   return __builtin_shufflevector (z0, z0, 3, 3, 3, 3, 7, 7, 7, 7);
@@ -81,7 +81,7 @@ f6 (fixed_int32_t z0)
 **	dupq	z0\.h, z0\.h\[0\]
 **	ret
 */
-fixed_uint16_t
+[[gnu::noipa]] fixed_uint16_t
 f7 (fixed_uint16_t z0)
 {
   return __builtin_shufflevector (z0, z0,
@@ -95,7 +95,7 @@ f7 (fixed_uint16_t z0)
 **	dupq	z0\.h, z0\.h\[5\]
 **	ret
 */
-fixed_uint16_t
+[[gnu::noipa]] fixed_uint16_t
 f8 (fixed_uint16_t z0)
 {
   return __builtin_shufflevector (z0, z0,
@@ -108,7 +108,7 @@ f8 (fixed_uint16_t z0)
 **	dupq	z0\.h, z0\.h\[7\]
 **	ret
 */
-fixed_uint16_t
+[[gnu::noipa]] fixed_uint16_t
 f9 (fixed_uint16_t z0)
 {
   return __builtin_shufflevector (z0, z0,
@@ -121,7 +121,7 @@ f9 (fixed_uint16_t z0)
 **	dupq	z0\.b, z0\.b\[0\]
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f10 (fixed_uint8_t z0)
 {
   return __builtin_shufflevector (z0, z0,
@@ -136,7 +136,7 @@ f10 (fixed_uint8_t z0)
 **	dupq	z0\.b, z0\.b\[13\]
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f11 (fixed_uint8_t z0)
 {
   return __builtin_shufflevector (z0, z0,
@@ -151,7 +151,7 @@ f11 (fixed_uint8_t z0)
 **	dupq	z0\.b, z0\.b\[15\]
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f12 (fixed_uint8_t z0)
 {
   return __builtin_shufflevector (z0, z0,
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1_run.c
new file mode 100644
index 0000000..fd25034
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/dupq_1_run.c
@@ -0,0 +1,87 @@
+/* { dg-do run { target { aarch64_sve256_hw && aarch64_sve2p1_hw } } } */
+/* { dg-options "-O2 -msve-vector-bits=256" } */
+
+#include "dupq_1.c"
+
+#define TEST(A, B)							\
+  do {									\
+    typeof(B) actual_ = (A);						\
+    if (__builtin_memcmp (&actual_, &(B), sizeof (actual_)) != 0)	\
+      __builtin_abort ();						\
+  } while (0)
+
+int
+main ()
+{
+  fixed_uint64_t a64 = { 0x1122, -1, 0x5566, -2 };
+  fixed_int32_t a32 = { 0x1122, -0x3344, 0x5566, -0x7788,
+			0x99aa, -0xbbcc, 0xddee, -0xff00 };
+  fixed_uint16_t a16 = { 0x9a12, 0xbc34, 0xde56, 0xf078,
+			 0x00ff, 0x11ee, 0x22dd, 0x33cc,
+			 0x44bb, 0x55aa, 0x6699, 0x7788,
+			 0xfe01, 0xdc23, 0xba45, 0x9867 };
+  fixed_uint8_t a8 = { 0x01, 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x70,
+		       0x89, 0x9a, 0xab, 0xbc, 0xcd, 0xde, 0xef, 0xf8,
+		       0xfe, 0xed, 0xdc, 0xcb, 0xba, 0xa9, 0x98, 0x8f,
+		       0x76, 0x65, 0x54, 0x43, 0x32, 0x21, 0x10, 0x07 };
+
+  fixed_uint64_t expected1 = { 0x1122, 0x1122, 0x5566, 0x5566 };
+  TEST (f1 (a64), expected1);
+
+  fixed_uint64_t expected2 = { -1, -1, -2, -2 };
+  TEST (f2 (a64), expected2);
+
+  fixed_int32_t expected3 = { 0x1122, 0x1122, 0x1122, 0x1122,
+                              0x99aa, 0x99aa, 0x99aa, 0x99aa };
+  TEST (f3 (a32), expected3);
+
+  fixed_int32_t expected4 = { -0x3344, -0x3344, -0x3344, -0x3344,
+			      -0xbbcc, -0xbbcc, -0xbbcc, -0xbbcc };
+  TEST (f4 (a32), expected4);
+
+  fixed_int32_t expected5 = { 0x5566, 0x5566, 0x5566, 0x5566,
+                              0xddee, 0xddee, 0xddee, 0xddee };
+  TEST (f5 (a32), expected5);
+
+  fixed_int32_t expected6 = { -0x7788, -0x7788, -0x7788, -0x7788,
+			      -0xff00, -0xff00, -0xff00, -0xff00 };
+  TEST (f6 (a32), expected6);
+
+  fixed_uint16_t expected7 = { 0x9a12, 0x9a12, 0x9a12, 0x9a12,
+			       0x9a12, 0x9a12, 0x9a12, 0x9a12,
+			       0x44bb, 0x44bb, 0x44bb, 0x44bb,
+			       0x44bb, 0x44bb, 0x44bb, 0x44bb };
+  TEST (f7 (a16), expected7);
+
+  fixed_uint16_t expected8 = { 0x11ee, 0x11ee, 0x11ee, 0x11ee,
+			       0x11ee, 0x11ee, 0x11ee, 0x11ee,
+			       0xdc23, 0xdc23, 0xdc23, 0xdc23,
+			       0xdc23, 0xdc23, 0xdc23, 0xdc23 };
+  TEST (f8 (a16), expected8);
+
+  fixed_uint16_t expected9 = { 0x33cc, 0x33cc, 0x33cc, 0x33cc,
+			       0x33cc, 0x33cc, 0x33cc, 0x33cc,
+			       0x9867, 0x9867, 0x9867, 0x9867,
+			       0x9867, 0x9867, 0x9867, 0x9867 };
+  TEST (f9 (a16), expected9);
+
+  fixed_uint8_t expected10 = { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+			       0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+			       0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe,
+			       0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe };
+  TEST (f10 (a8), expected10);
+
+  fixed_uint8_t expected11 = { 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde,
+			       0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde, 0xde,
+			       0x21, 0x21, 0x21, 0x21, 0x21, 0x21, 0x21, 0x21,
+			       0x21, 0x21, 0x21, 0x21, 0x21, 0x21, 0x21, 0x21 };
+  TEST (f11 (a8), expected11);
+
+  fixed_uint8_t expected12 = { 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8,
+			       0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8,
+			       0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
+			       0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07 };
+  TEST (f12 (a8), expected12);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/eon_bsl2n.c b/gcc/testsuite/gcc.target/aarch64/sve2/eon_bsl2n.c
new file mode 100644
index 0000000..74b4637
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/eon_bsl2n.c
@@ -0,0 +1,52 @@
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <arm_neon.h>
+#include <arm_sve.h>
+
+#define EON(x, y)   (~((x) ^ (y)))
+
+/*
+** eon_d:
+** 	bsl2n	z0.d, z0.d, z0.d, z1.d
+** 	ret
+*/
+uint32x2_t eon_d(uint32x2_t a, uint32x2_t b) { return EON(a, b); }
+
+/*
+** eon_d_mp:
+**	movprfx	z0, z1
+** 	bsl2n	z0.d, z0.d, z1.d, z2.d
+** 	ret
+*/
+uint32x2_t eon_d_mp(uint32x2_t c, uint32x2_t a, uint32x2_t b) { return EON(a, b); }
+
+/*
+** eon_q:
+** 	bsl2n	z0.d, z0.d, z0.d, z1.d
+** 	ret
+*/
+uint64x2_t eon_q(uint64x2_t a, uint64x2_t b) { return EON(a, b); }
+
+/*
+** eon_q_mp:
+**	movprfx	z0, z1
+** 	bsl2n	z0.d, z0.d, z1.d, z2.d
+** 	ret
+*/
+uint64x2_t eon_q_mp(uint64x2_t c, uint64x2_t a, uint64x2_t b) { return EON(a, b); }
+
+/*
+** eon_z:
+** 	bsl2n	z0.d, z0.d, z0.d, z1.d
+** 	ret
+*/
+svuint64_t eon_z(svuint64_t a, svuint64_t b) { return EON(a, b); }
+
+/*
+** eon_z_mp:
+**	movprfx	z0, z1
+** 	bsl2n	z0.d, z0.d, z1.d, z2.d
+** 	ret
+*/
+svuint64_t eon_z_mp(svuint64_t c, svuint64_t a, svuint64_t b) { return EON(a, b); }
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/extq_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/extq_1.c
index 03c5fb1..be5ae71 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/extq_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/extq_1.c
@@ -1,5 +1,5 @@
 /* { dg-options "-O2 -msve-vector-bits=256" } */
-/* { dg-final { check-function-bodies "**" "" "" { target { le } } } } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
 
 #include <arm_sve.h>
 
@@ -15,7 +15,7 @@ typedef svfloat64_t fixed_float64_t __attribute__((arm_sve_vector_bits(256)));
 **	extq	z0\.b, z0\.b, z1\.b, #8
 **	ret
 */
-fixed_float64_t
+[[gnu::noipa]] fixed_float64_t
 f1 (fixed_float64_t z0, fixed_float64_t z1)
 {
   return __builtin_shufflevector (z0, z1, 1, 4, 3, 6);
@@ -26,7 +26,7 @@ f1 (fixed_float64_t z0, fixed_float64_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #4
 **	ret
 */
-fixed_uint32_t
+[[gnu::noipa]] fixed_uint32_t
 f2 (fixed_uint32_t z0, fixed_uint32_t z1)
 {
   return __builtin_shufflevector (z0, z1, 1, 2, 3, 8, 5, 6, 7, 12);
@@ -37,7 +37,7 @@ f2 (fixed_uint32_t z0, fixed_uint32_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #12
 **	ret
 */
-fixed_uint32_t
+[[gnu::noipa]] fixed_uint32_t
 f3 (fixed_uint32_t z0, fixed_uint32_t z1)
 {
   return __builtin_shufflevector (z0, z1, 3, 8, 9, 10, 7, 12, 13, 14);
@@ -48,7 +48,7 @@ f3 (fixed_uint32_t z0, fixed_uint32_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #2
 **	ret
 */
-fixed_float16_t
+[[gnu::noipa]] fixed_float16_t
 f4 (fixed_float16_t z0, fixed_float16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -61,7 +61,7 @@ f4 (fixed_float16_t z0, fixed_float16_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #10
 **	ret
 */
-fixed_float16_t
+[[gnu::noipa]] fixed_float16_t
 f5 (fixed_float16_t z0, fixed_float16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -74,7 +74,7 @@ f5 (fixed_float16_t z0, fixed_float16_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #14
 **	ret
 */
-fixed_float16_t
+[[gnu::noipa]] fixed_float16_t
 f6 (fixed_float16_t z0, fixed_float16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -87,7 +87,7 @@ f6 (fixed_float16_t z0, fixed_float16_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #1
 **	ret
 */
-fixed_int8_t
+[[gnu::noipa]] fixed_int8_t
 f7 (fixed_int8_t z0, fixed_int8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -102,7 +102,7 @@ f7 (fixed_int8_t z0, fixed_int8_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #11
 **	ret
 */
-fixed_int8_t
+[[gnu::noipa]] fixed_int8_t
 f8 (fixed_int8_t z0, fixed_int8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -117,7 +117,7 @@ f8 (fixed_int8_t z0, fixed_int8_t z1)
 **	extq	z0\.b, z0\.b, z1\.b, #15
 **	ret
 */
-fixed_int8_t
+[[gnu::noipa]] fixed_int8_t
 f9 (fixed_int8_t z0, fixed_int8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/extq_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve2/extq_1_run.c
new file mode 100644
index 0000000..6b72c98
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/extq_1_run.c
@@ -0,0 +1,73 @@
+/* { dg-do run { target { aarch64_sve256_hw && aarch64_sve2p1_hw } } } */
+/* { dg-options "-O2 -msve-vector-bits=256" } */
+
+#include "extq_1.c"
+
+#define TEST(A, B)							\
+  do {									\
+    typeof(B) actual_ = (A);						\
+    if (__builtin_memcmp (&actual_, &(B), sizeof (actual_)) != 0)	\
+      __builtin_abort ();						\
+  } while (0)
+
+int
+main ()
+{
+  fixed_float64_t a64 = { 1.5, 3.75, -5.25, 9 };
+  fixed_float64_t b64 = { -2, 4.125, -6.375, 11.5 };
+  fixed_float64_t expected1 = { 3.75, -2, 9, -6.375 };
+  TEST (f1 (a64, b64), expected1);
+
+  fixed_uint32_t a32 = { 0x1122, -0x3344, 0x5566, -0x7788,
+			 0x99aa, -0xbbcc, 0xddee, -0xff00 };
+  fixed_uint32_t b32 = { 1 << 20, 1 << 21, 1 << 22, 1 << 23,
+			 5 << 6, 5 << 7, 5 << 8, 5 << 9 };
+  fixed_uint32_t expected2 = { -0x3344, 0x5566, -0x7788, 1 << 20,
+			       -0xbbcc, 0xddee, -0xff00, 5 << 6 };
+  fixed_uint32_t expected3 = { -0x7788, 1 << 20, 1 << 21, 1 << 22,
+			       -0xff00, 5 << 6, 5 << 7, 5 << 8 };
+  TEST (f2 (a32, b32), expected2);
+  TEST (f3 (a32, b32), expected3);
+
+  fixed_float16_t a16 = { 0.5, 0.75, 1, 1.25, 1.5, 1.75, 2, 2.25,
+			  2.5, 2.75, 3, 3.25, 3.5, 3.75, 4, 4.25 };
+  fixed_float16_t b16 = { -0.5, -0.75, -1, -1.25, -1.5, -1.75, -2, -2.25,
+			  -2.5, -2.75, -3, -3.25, -3.5, -3.75, -4, -4.25 };
+  fixed_float16_t expected4 = { 0.75, 1, 1.25, 1.5, 1.75, 2, 2.25, -0.5,
+				2.75, 3, 3.25, 3.5, 3.75, 4, 4.25, -2.5 };
+  fixed_float16_t expected5 = { 1.75, 2, 2.25, -0.5, -0.75, -1, -1.25, -1.5,
+				3.75, 4, 4.25, -2.5, -2.75, -3, -3.25, -3.5 };
+  fixed_float16_t expected6 = { 2.25, -0.5, -0.75, -1,
+				-1.25, -1.5, -1.75, -2,
+				4.25, -2.5, -2.75, -3,
+				-3.25, -3.5, -3.75, -4 };
+  TEST (f4 (a16, b16), expected4);
+  TEST (f5 (a16, b16), expected5);
+  TEST (f6 (a16, b16), expected6);
+
+  fixed_int8_t a8 = { 0x01, 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x70,
+		      0x89, 0x9a, 0xab, 0xbc, 0xcd, 0xde, 0xef, 0xf8,
+		      0xfe, 0xed, 0xdc, 0xcb, 0xba, 0xa9, 0x98, 0x8f,
+		      0x76, 0x65, 0x54, 0x43, 0x32, 0x21, 0x10, 0x07 };
+  fixed_int8_t b8 = { 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+		      0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00,
+		      0x13, 0x24, 0x35, 0x46, 0x57, 0x68, 0x79, 0x8a,
+		      0x9b, 0xac, 0xbd, 0xce, 0xdf, 0xe0, 0xf1, 0x02 };
+  fixed_int8_t expected7 = { 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x70, 0x89,
+			     0x9a, 0xab, 0xbc, 0xcd, 0xde, 0xef, 0xf8, 0x11,
+			     0xed, 0xdc, 0xcb, 0xba, 0xa9, 0x98, 0x8f, 0x76,
+			     0x65, 0x54, 0x43, 0x32, 0x21, 0x10, 0x07, 0x13 };
+  fixed_int8_t expected8 = { 0xbc, 0xcd, 0xde, 0xef, 0xf8, 0x11, 0x22, 0x33,
+			     0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb,
+			     0x43, 0x32, 0x21, 0x10, 0x07, 0x13, 0x24, 0x35,
+			     0x46, 0x57, 0x68, 0x79, 0x8a, 0x9b, 0xac, 0xbd };
+  fixed_int8_t expected9 = { 0xf8, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
+			     0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+			     0x07, 0x13, 0x24, 0x35, 0x46, 0x57, 0x68, 0x79,
+			     0x8a, 0x9b, 0xac, 0xbd, 0xce, 0xdf, 0xe0, 0xf1 };
+  TEST (f7 (a8, b8), expected7);
+  TEST (f8 (a8, b8), expected8);
+  TEST (f9 (a8, b8), expected9);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/nbsl_nor_nand_neon.c b/gcc/testsuite/gcc.target/aarch64/sve2/nbsl_nor_nand_neon.c
new file mode 100644
index 0000000..09bfc19
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/nbsl_nor_nand_neon.c
@@ -0,0 +1,68 @@
+/* { dg-options "-O2" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <arm_neon.h>
+
+#define NAND(x, y)  (~((x) & (y)))
+#define NOR(x, y)   (~((x) | (y)))
+
+/*
+** nand_d:
+** 	nbsl	z0.d, z0.d, z1.d, z1.d
+** 	ret
+*/
+uint32x2_t nand_d(uint32x2_t a, uint32x2_t b) { return NAND(a, b); }
+
+/*
+** nand_d_mp:
+** 	movprfx	z0, z1
+** 	nbsl	z0.d, z0.d, z2.d, z2.d
+** 	ret
+*/
+uint32x2_t nand_d_mp(uint32x2_t c, uint32x2_t a, uint32x2_t b) { return NAND(a, b); }
+
+/*
+** nor_d:
+** 	nbsl	z0.d, z0.d, z1.d, z0.d
+** 	ret
+*/
+uint32x2_t nor_d(uint32x2_t a, uint32x2_t b) { return NOR(a, b); }
+
+/*
+** nor_d_mp:
+** 	movprfx	z0, z1
+** 	nbsl	z0.d, z0.d, z2.d, z1.d
+** 	ret
+*/
+uint32x2_t nor_d_mp(uint32x2_t c, uint32x2_t a, uint32x2_t b) { return NOR(a, b); }
+
+/*
+** nand_q:
+** 	nbsl	z0.d, z0.d, z1.d, z1.d
+** 	ret
+*/
+uint64x2_t nand_q(uint64x2_t a, uint64x2_t b) { return NAND(a, b); }
+
+/*
+** nand_q_mp:
+** 	movprfx	z0, z1
+** 	nbsl	z0.d, z0.d, z2.d, z2.d
+** 	ret
+*/
+uint32x4_t nand_q_mp(uint32x4_t c, uint32x4_t a, uint32x4_t b) { return NAND(a, b); }
+
+/*
+** nor_q:
+** 	nbsl	z0.d, z0.d, z1.d, z0.d
+** 	ret
+*/
+uint64x2_t nor_q(uint64x2_t a, uint64x2_t b) { return NOR(a, b); }
+
+/*
+** nor_q_mp:
+** 	movprfx	z0, z1
+** 	nbsl	z0.d, z0.d, z2.d, z1.d
+** 	ret
+*/
+uint32x4_t nor_q_mp(uint32x4_t c, uint32x4_t a, uint32x4_t b) { return NOR(a, b); }
+
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary.c
index 94470a5..977fa39 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_n.c
index b8747b8..b816fa1 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_single_n.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_single_n.c
index 7cb7ee5..0e4427a 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_single_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_int_opt_single_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_n.c
index 787126f..81d0c82 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_single_n.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_single_n.c
index 6b2b0a42..3920bdb 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_single_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_opt_single_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2 -march=armv8.2-a+sve2+faminmax" } */
+/* { dg-options "-O2 -march=armv8.2-a+sve2+faminmax -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_to_uint.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_to_uint.c
index a0a7f80..c7d10b3 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_to_uint.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_to_uint.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_uint_opt_n.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_uint_opt_n.c
index c13db48..122fba7 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_uint_opt_n.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_uint_opt_n.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_wide.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_wide.c
index 145b077..7f35859 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_wide.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-binary_wide.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-compare.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-compare.c
index da175db..b079a56 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-compare.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-compare.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-binary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_index_restricted.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_index_restricted.c
index c0476ce..14e77c00 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_index_restricted.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_index_restricted.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_offset_restricted.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_offset_restricted.c
index f644024..b680548 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_offset_restricted.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_ext_gather_offset_restricted.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_sv_restricted.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_sv_restricted.c
index a48a8a9..6d1a356 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_sv_restricted.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_sv_restricted.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_vs.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_vs.c
index 1fc08a3..9cb4471 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_vs.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-load_gather_vs.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_left_imm_to_uint.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_left_imm_to_uint.c
index bd2c937..e57a650 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_left_imm_to_uint.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_left_imm_to_uint.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_right_imm.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_right_imm.c
index f4994de..710ca73 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_right_imm.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-shift_right_imm.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_index_restricted.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_index_restricted.c
index 6bec3b3..dc9cf46 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_index_restricted.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_index_restricted.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_offset_restricted.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_offset_restricted.c
index bcb4a14..2728c9b 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_offset_restricted.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-store_scatter_offset_restricted.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary.c
index ba7e931..9f33295 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2 -march=armv9.2-a+sve+sme" } */
+/* { dg-options "-O2 -march=armv9.2-a+sve+sme -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert.c
index 7aa59ff..68769fe 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert_narrowt.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert_narrowt.c
index 1a4525c..692891f 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert_narrowt.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_convert_narrowt.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include <arm_sve.h>
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_to_int.c b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_to_int.c
index b64bfc3..7dffa1c 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_to_int.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pfalse-unary_to_int.c
@@ -1,6 +1,6 @@
 /* { dg-do compile } */
 /* { dg-require-effective-target elf } */
-/* { dg-options "-O2" } */
+/* { dg-options "-O2 -funwind-tables" } */
 
 #include "../pfalse-unary_0.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/pr120999.c b/gcc/testsuite/gcc.target/aarch64/sve2/pr120999.c
new file mode 100644
index 0000000..2dca36a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/pr120999.c
@@ -0,0 +1,17 @@
+/* PR target/120999.  */
+/* { dg-do assemble } */
+/* { dg-options "-O2 --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <arm_sve.h>
+
+#define NOR(x, y)   (~((x) | (y)))
+
+/*
+** nor_z:
+** 	movprfx	z0, z1
+**	nbsl	z0.d, z0.d, z2.d, z1.d
+** 	ret
+*/
+svuint64_t nor_z(svuint64_t c, svuint64_t a, svuint64_t b) { return NOR(a, b); }
+
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1.c
index f923e94..587f670 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1.c
@@ -1,5 +1,5 @@
 /* { dg-options "-O2 -msve-vector-bits=256" } */
-/* { dg-final { check-function-bodies "**" "" "" { target { le } } } } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
 
 #include <arm_sve.h>
 
@@ -15,7 +15,7 @@ typedef svint64_t fixed_int64_t __attribute__((arm_sve_vector_bits(256)));
 **	trn1	z0\.d, z0\.d, z1\.d
 **	ret
 */
-fixed_int64_t
+[[gnu::noipa]] fixed_int64_t
 f1 (fixed_int64_t z0, fixed_int64_t z1)
 {
   return __builtin_shufflevector (z0, z1, 0, 4, 2, 6);
@@ -26,7 +26,7 @@ f1 (fixed_int64_t z0, fixed_int64_t z1)
 **	trn2	z0\.d, z0\.d, z1\.d
 **	ret
 */
-fixed_int64_t
+[[gnu::noipa]] fixed_int64_t
 f2 (fixed_int64_t z0, fixed_int64_t z1)
 {
   return __builtin_shufflevector (z0, z1, 1, 5, 3, 7);
@@ -37,7 +37,7 @@ f2 (fixed_int64_t z0, fixed_int64_t z1)
 **	uzpq1	z0\.s, z0\.s, z1\.s
 **	ret
 */
-fixed_float32_t
+[[gnu::noipa]] fixed_float32_t
 f3 (fixed_float32_t z0, fixed_float32_t z1)
 {
   return __builtin_shufflevector (z0, z1, 0, 2, 8, 10, 4, 6, 12, 14);
@@ -48,7 +48,7 @@ f3 (fixed_float32_t z0, fixed_float32_t z1)
 **	uzpq2	z0\.s, z0\.s, z1\.s
 **	ret
 */
-fixed_float32_t
+[[gnu::noipa]] fixed_float32_t
 f4 (fixed_float32_t z0, fixed_float32_t z1)
 {
   return __builtin_shufflevector (z0, z1, 1, 3, 9, 11, 5, 7, 13, 15);
@@ -59,7 +59,7 @@ f4 (fixed_float32_t z0, fixed_float32_t z1)
 **	uzpq1	z0\.h, z0\.h, z1\.h
 **	ret
 */
-fixed_bfloat16_t
+[[gnu::noipa]] fixed_bfloat16_t
 f5 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -72,7 +72,7 @@ f5 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 **	uzpq2	z0\.h, z0\.h, z1\.h
 **	ret
 */
-fixed_bfloat16_t
+[[gnu::noipa]] fixed_bfloat16_t
 f6 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -85,7 +85,7 @@ f6 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 **	uzpq1	z0\.b, z0\.b, z1\.b
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f7 (fixed_uint8_t z0, fixed_uint8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -100,7 +100,7 @@ f7 (fixed_uint8_t z0, fixed_uint8_t z1)
 **	uzpq2	z0\.b, z0\.b, z1\.b
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f8 (fixed_uint8_t z0, fixed_uint8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1_run.c
new file mode 100644
index 0000000..9044cae
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/uzpq_1_run.c
@@ -0,0 +1,78 @@
+/* { dg-do run { target { aarch64_sve256_hw && aarch64_sve2p1_hw } } } */
+/* { dg-options "-O2 -msve-vector-bits=256" } */
+
+#include "uzpq_1.c"
+
+typedef svuint16_t fixed_uint16_t __attribute__((arm_sve_vector_bits(256)));
+
+#define TEST(A, B)							\
+  do {									\
+    typeof(A) actual_ = (A);						\
+    if (__builtin_memcmp (&actual_, &(B), sizeof (actual_)) != 0)	\
+      __builtin_abort ();						\
+  } while (0)
+
+int
+main ()
+{
+  fixed_int64_t a64 = { 0x1122LL << 31, -1LL << 47, 0x5566 << 15, -2 };
+  fixed_int64_t b64 = { 42, -0x3344LL << 19, 303, -0x7788LL << 27 };
+  fixed_int64_t expected1 = { 0x1122LL << 31, 42,
+			      0x5566 << 15, 303 };
+  fixed_int64_t expected2 = { -1LL << 47, -0x3344LL << 19,
+			      -2, -0x7788LL << 27 };
+  TEST (f1 (a64, b64), expected1);
+  TEST (f2 (a64, b64), expected2);
+
+  fixed_float32_t a32 = { 0.5, 0.75, 1, 1.25, 2.5, 2.75, 3, 3.25 };
+  fixed_float32_t b32 = { -0.5, -0.75, -1, -1.25, -2.5, -2.75, -3, -3.25 };
+  fixed_float32_t expected3 = { 0.5, 1, -0.5, -1,
+				2.5, 3, -2.5, -3 };
+  fixed_float32_t expected4 = { 0.75, 1.25, -0.75, -1.25,
+				2.75, 3.25, -2.75, -3.25 };
+  TEST (f3 (a32, b32), expected3);
+  TEST (f4 (a32, b32), expected4);
+
+  fixed_uint16_t a16_i = { 0x9a12, 0xbc34, 0xde56, 0xf078,
+			   0x00ff, 0x11ee, 0x22dd, 0x33cc,
+			   0x44bb, 0x55aa, 0x6699, 0x7788,
+			   0xfe01, 0xdc23, 0xba45, 0x9867 };
+  fixed_uint16_t b16_i = { 0x1010, 0x2020, 0x3030, 0x4040,
+			   0x5050, 0x6060, 0x7070, 0x8080,
+			   0x9090, 0xa0a0, 0xb0b0, 0xc0c0,
+			   0xd0d0, 0xe0e0, 0xf0f0, 0x0f0f };
+  fixed_uint16_t expected5 = { 0x9a12, 0xde56, 0x00ff, 0x22dd,
+			       0x1010, 0x3030, 0x5050, 0x7070,
+			       0x44bb, 0x6699, 0xfe01, 0xba45,
+			       0x9090, 0xb0b0, 0xd0d0, 0xf0f0 };
+  fixed_uint16_t expected6 = { 0xbc34, 0xf078, 0x11ee, 0x33cc,
+			       0x2020, 0x4040, 0x6060, 0x8080,
+			       0x55aa, 0x7788, 0xdc23, 0x9867,
+			       0xa0a0, 0xc0c0, 0xe0e0, 0x0f0f };
+  fixed_bfloat16_t a16, b16;
+  __builtin_memcpy (&a16, &a16_i, sizeof (a16));
+  __builtin_memcpy (&b16, &b16_i, sizeof (b16));
+  TEST (f5 (a16, b16), expected5);
+  TEST (f6 (a16, b16), expected6);
+
+  fixed_uint8_t a8 = { 0x01, 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x70,
+		       0x89, 0x9a, 0xab, 0xbc, 0xcd, 0xde, 0xef, 0xf8,
+		       0xfe, 0xed, 0xdc, 0xcb, 0xba, 0xa9, 0x98, 0x8f,
+		       0x76, 0x65, 0x54, 0x43, 0x32, 0x21, 0x10, 0x07 };
+  fixed_uint8_t b8 = { 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+		       0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00,
+		       0x13, 0x24, 0x35, 0x46, 0x57, 0x68, 0x79, 0x8a,
+		       0x9b, 0xac, 0xbd, 0xce, 0xdf, 0xe0, 0xf1, 0x02 };
+  fixed_uint8_t expected7 = { 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
+			      0x11, 0x33, 0x55, 0x77, 0x99, 0xbb, 0xdd, 0xff,
+			      0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10,
+			      0x13, 0x35, 0x57, 0x79, 0x9b, 0xbd, 0xdf, 0xf1 };
+  fixed_uint8_t expected8 = { 0x12, 0x34, 0x56, 0x70, 0x9a, 0xbc, 0xde, 0xf8,
+			      0x22, 0x44, 0x66, 0x88, 0xaa, 0xcc, 0xee, 0x00,
+			      0xed, 0xcb, 0xa9, 0x8f, 0x65, 0x43, 0x21, 0x07,
+			      0x24, 0x46, 0x68, 0x8a, 0xac, 0xce, 0xe0, 0x02 };
+  TEST (f7 (a8, b8), expected7);
+  TEST (f8 (a8, b8), expected8);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1.c b/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1.c
index fa420a9..76fb4b4 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1.c
@@ -1,5 +1,5 @@
 /* { dg-options "-O2 -msve-vector-bits=256" } */
-/* { dg-final { check-function-bodies "**" "" "" { target { le } } } } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
 
 #include <arm_sve.h>
 
@@ -15,7 +15,7 @@ typedef svint64_t fixed_int64_t __attribute__((arm_sve_vector_bits(256)));
 **	trn1	z0\.d, z0\.d, z1\.d
 **	ret
 */
-fixed_int64_t
+[[gnu::noipa]] fixed_int64_t
 f1 (fixed_int64_t z0, fixed_int64_t z1)
 {
   return __builtin_shufflevector (z0, z1, 0, 4, 2, 6);
@@ -26,7 +26,7 @@ f1 (fixed_int64_t z0, fixed_int64_t z1)
 **	trn2	z0\.d, z0\.d, z1\.d
 **	ret
 */
-fixed_int64_t
+[[gnu::noipa]] fixed_int64_t
 f2 (fixed_int64_t z0, fixed_int64_t z1)
 {
   return __builtin_shufflevector (z0, z1, 1, 5, 3, 7);
@@ -37,7 +37,7 @@ f2 (fixed_int64_t z0, fixed_int64_t z1)
 **	zipq1	z0\.s, z0\.s, z1\.s
 **	ret
 */
-fixed_float32_t
+[[gnu::noipa]] fixed_float32_t
 f3 (fixed_float32_t z0, fixed_float32_t z1)
 {
   return __builtin_shufflevector (z0, z1, 0, 8, 1, 9, 4, 12, 5, 13);
@@ -48,7 +48,7 @@ f3 (fixed_float32_t z0, fixed_float32_t z1)
 **	zipq2	z0\.s, z0\.s, z1\.s
 **	ret
 */
-fixed_float32_t
+[[gnu::noipa]] fixed_float32_t
 f4 (fixed_float32_t z0, fixed_float32_t z1)
 {
   return __builtin_shufflevector (z0, z1, 2, 10, 3, 11, 6, 14, 7, 15);
@@ -59,7 +59,7 @@ f4 (fixed_float32_t z0, fixed_float32_t z1)
 **	zipq1	z0\.h, z0\.h, z1\.h
 **	ret
 */
-fixed_bfloat16_t
+[[gnu::noipa]] fixed_bfloat16_t
 f5 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -72,7 +72,7 @@ f5 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 **	zipq2	z0\.h, z0\.h, z1\.h
 **	ret
 */
-fixed_bfloat16_t
+[[gnu::noipa]] fixed_bfloat16_t
 f6 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -85,7 +85,7 @@ f6 (fixed_bfloat16_t z0, fixed_bfloat16_t z1)
 **	zipq1	z0\.b, z0\.b, z1\.b
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f7 (fixed_uint8_t z0, fixed_uint8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
@@ -100,7 +100,7 @@ f7 (fixed_uint8_t z0, fixed_uint8_t z1)
 **	zipq2	z0\.b, z0\.b, z1\.b
 **	ret
 */
-fixed_uint8_t
+[[gnu::noipa]] fixed_uint8_t
 f8 (fixed_uint8_t z0, fixed_uint8_t z1)
 {
   return __builtin_shufflevector (z0, z1,
diff --git a/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1_run.c
new file mode 100644
index 0000000..211f9d9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve2/zipq_1_run.c
@@ -0,0 +1,78 @@
+/* { dg-do run { target { aarch64_sve256_hw && aarch64_sve2p1_hw } } } */
+/* { dg-options "-O2 -msve-vector-bits=256" } */
+
+#include "zipq_1.c"
+
+typedef svuint16_t fixed_uint16_t __attribute__((arm_sve_vector_bits(256)));
+
+#define TEST(A, B)							\
+  do {									\
+    typeof(A) actual_ = (A);						\
+    if (__builtin_memcmp (&actual_, &(B), sizeof (actual_)) != 0)	\
+      __builtin_abort ();						\
+  } while (0)
+
+int
+main ()
+{
+  fixed_int64_t a64 = { 0x1122LL << 31, -1LL << 47, 0x5566 << 15, -2 };
+  fixed_int64_t b64 = { 42, -0x3344LL << 19, 303, -0x7788LL << 27 };
+  fixed_int64_t expected1 = { 0x1122LL << 31, 42,
+			      0x5566 << 15, 303 };
+  fixed_int64_t expected2 = { -1LL << 47, -0x3344LL << 19,
+			      -2, -0x7788LL << 27 };
+  TEST (f1 (a64, b64), expected1);
+  TEST (f2 (a64, b64), expected2);
+
+  fixed_float32_t a32 = { 0.5, 0.75, 1, 1.25, 2.5, 2.75, 3, 3.25 };
+  fixed_float32_t b32 = { -0.5, -0.75, -1, -1.25, -2.5, -2.75, -3, -3.25 };
+  fixed_float32_t expected3 = { 0.5, -0.5, 0.75, -0.75,
+				2.5, -2.5, 2.75, -2.75 };
+  fixed_float32_t expected4 = { 1, -1, 1.25, -1.25,
+				3, -3, 3.25, -3.25 };
+  TEST (f3 (a32, b32), expected3);
+  TEST (f4 (a32, b32), expected4);
+
+  fixed_uint16_t a16_i = { 0x9a12, 0xbc34, 0xde56, 0xf078,
+			   0x00ff, 0x11ee, 0x22dd, 0x33cc,
+			   0x44bb, 0x55aa, 0x6699, 0x7788,
+			   0xfe01, 0xdc23, 0xba45, 0x9867 };
+  fixed_uint16_t b16_i = { 0x1010, 0x2020, 0x3030, 0x4040,
+			   0x5050, 0x6060, 0x7070, 0x8080,
+			   0x9090, 0xa0a0, 0xb0b0, 0xc0c0,
+			   0xd0d0, 0xe0e0, 0xf0f0, 0x0f0f };
+  fixed_uint16_t expected5 = { 0x9a12, 0x1010, 0xbc34, 0x2020,
+			       0xde56, 0x3030, 0xf078, 0x4040,
+			       0x44bb, 0x9090, 0x55aa, 0xa0a0,
+			       0x6699, 0xb0b0, 0x7788, 0xc0c0 };
+  fixed_uint16_t expected6 = { 0x00ff, 0x5050, 0x11ee, 0x6060,
+			       0x22dd, 0x7070, 0x33cc, 0x8080,
+			       0xfe01, 0xd0d0, 0xdc23, 0xe0e0,
+			       0xba45, 0xf0f0, 0x9867, 0x0f0f };
+  fixed_bfloat16_t a16, b16;
+  __builtin_memcpy (&a16, &a16_i, sizeof (a16));
+  __builtin_memcpy (&b16, &b16_i, sizeof (b16));
+  TEST (f5 (a16, b16), expected5);
+  TEST (f6 (a16, b16), expected6);
+
+  fixed_uint8_t a8 = { 0x01, 0x12, 0x23, 0x34, 0x45, 0x56, 0x67, 0x70,
+		       0x89, 0x9a, 0xab, 0xbc, 0xcd, 0xde, 0xef, 0xf8,
+		       0xfe, 0xed, 0xdc, 0xcb, 0xba, 0xa9, 0x98, 0x8f,
+		       0x76, 0x65, 0x54, 0x43, 0x32, 0x21, 0x10, 0x07 };
+  fixed_uint8_t b8 = { 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
+		       0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00,
+		       0x13, 0x24, 0x35, 0x46, 0x57, 0x68, 0x79, 0x8a,
+		       0x9b, 0xac, 0xbd, 0xce, 0xdf, 0xe0, 0xf1, 0x02 };
+  fixed_uint8_t expected7 = { 0x01, 0x11, 0x12, 0x22, 0x23, 0x33, 0x34, 0x44,
+			      0x45, 0x55, 0x56, 0x66, 0x67, 0x77, 0x70, 0x88,
+			      0xfe, 0x13, 0xed, 0x24, 0xdc, 0x35, 0xcb, 0x46,
+			      0xba, 0x57, 0xa9, 0x68, 0x98, 0x79, 0x8f, 0x8a };
+  fixed_uint8_t expected8 = { 0x89, 0x99, 0x9a, 0xaa, 0xab, 0xbb, 0xbc, 0xcc,
+			      0xcd, 0xdd, 0xde, 0xee, 0xef, 0xff, 0xf8, 0x00,
+			      0x76, 0x9b, 0x65, 0xac, 0x54, 0xbd, 0x43, 0xce,
+			      0x32, 0xdf, 0x21, 0xe0, 0x10, 0xf1, 0x07, 0x02 };
+  TEST (f7 (a8, b8), expected7);
+  TEST (f8 (a8, b8), expected8);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/aarch64/vec-set-zero.c b/gcc/testsuite/gcc.target/aarch64/vec-set-zero.c
index b34b902c..ba4696e 100644
--- a/gcc/testsuite/gcc.target/aarch64/vec-set-zero.c
+++ b/gcc/testsuite/gcc.target/aarch64/vec-set-zero.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O2" } */
+/* { dg-options "-Os" } */
 
 #include "arm_neon.h"
 
diff --git a/gcc/testsuite/gcc.target/aarch64/vector-compare-5.c b/gcc/testsuite/gcc.target/aarch64/vector-compare-5.c
new file mode 100644
index 0000000..a1a601d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vector-compare-5.c
@@ -0,0 +1,67 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+/* { dg-additional-options "-fdump-tree-original-all" } */
+
+typedef int v4i __attribute__((vector_size(4*sizeof(int))));
+
+/* Ensure we can simplify `VEC_COND_EXPR(a OP1 b) OP2 VEC_COND_EXPR(a OP3 b)`
+ * into `VEC_COND_EXPR(a OP4 b)`
+ */
+
+void use (v4i const *z);
+
+void
+g (v4i *x, v4i const *y, v4i *z, v4i *t)
+{
+  *z = *x > *y | *x == *y; // expect >=
+  *t = *x > *y | *x <= *y; // expect true
+}
+
+void
+h (v4i *x, v4i const *y, v4i *z, v4i *t)
+{
+  *z = *x <= *y & *x >= *y; // expect x == y
+  *t = *x <= *y & *x != *y; // expect x<y
+}
+
+void
+i (v4i *x, v4i const *y, v4i *z, v4i *t)
+{
+  *z = *x == *y | *x != *y; // expect true
+  *t = *x == *y & *x != *y; // expect false
+}
+
+void
+k (v4i *x, v4i const *y, v4i *z, v4i *t)
+{
+  *z = *x < *y | *x == *y;  // x <= y
+  *t = *x < *y & *x > *y;   // expect false
+}
+
+void
+m (v4i *x, v4i const *y, v4i *z, v4i *t)
+{
+  *z = *x <= *y ^ *x >= *y; /* expect x != y */
+  *t = *x <= *y ^ *x != *y; /* expect x <= y */
+}
+
+void
+n (v4i *x, v4i const *y, v4i *z, v4i *t)
+{
+  *z = *x == *y ^ *x != *y; /* expect true */
+  *t = *x == *y ^ *x == *y; /* expect false */
+}
+
+
+/* { dg-final { scan-tree-dump ".*\\*zD\\.\\d+\\s*=\\s*VEC_COND_EXPR\\s*<\\s*\\*xD\\.\\d+\\s*>=\\s*VIEW_CONVERT_EXPR<v4iD\\.\\d+>\\(\\*yD\\.\\d+\\)\\s*,\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*,\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*>\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*tD\\.\\d+\\s*=\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*zD\\.\\d+\\s*=\\s*VEC_COND_EXPR\\s*<\\s*\\*xD\\.\\d+\\s*==\\s*VIEW_CONVERT_EXPR<v4iD\\.\\d+>\\(\\*yD\\.\\d+\\)\\s*,\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*,\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*>\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*tD\\.\\d+\\s*=\\s*VEC_COND_EXPR\\s*<\\s*\\*xD\\.\\d+\\s*<\\s*VIEW_CONVERT_EXPR<v4iD\\.\\d+>\\(\\*yD\\.\\d+\\)\\s*,\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*,\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*>\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*zD\\.\\d+\\s*=\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*tD\\.\\d+\\s*=\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*zD\\.\\d+\\s*=\\s*VEC_COND_EXPR\\s*<\\s*\\*xD\\.\\d+\\s*<=\\s*VIEW_CONVERT_EXPR<v4iD\\.\\d+>\\(\\*yD\\.\\d+\\)\\s*,\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*,\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*>\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*tD\\.\\d+\\s*=\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*zD\\.\\d+\\s*=\\s*VEC_COND_EXPR\\s*<\\s*\\*xD\\.\\d+\\s*!=\\s*VIEW_CONVERT_EXPR<v4iD\\.\\d+>\\(\\*yD\\.\\d+\\)\\s*,\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*,\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*>\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*tD\\.\\d+\\s*=\\s*VEC_COND_EXPR\\s*<\\s*\\*xD\\.\\d+\\s*>=\\s*VIEW_CONVERT_EXPR<v4iD\\.\\d+>\\(\\*yD\\.\\d+\\)\\s*,\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*,\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*>\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*zD\\.\\d+\\s*=\\s*\\{\\s*-1(,\\s*-1){3}\\s*\\}\\s*;" "original" } } */
+/* { dg-final { scan-tree-dump ".*\\*tD\\.\\d+\\s*=\\s*\\{\\s*0(,\\s*0){3}\\s*\\}\\s*;" "original" } } */
diff --git a/gcc/testsuite/gcc.target/arm/pr121065.c b/gcc/testsuite/gcc.target/arm/pr121065.c
new file mode 100644
index 0000000..dfc6059
--- /dev/null
+++ b/gcc/testsuite/gcc.target/arm/pr121065.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-mcpu=cortex-m55" } */
+
+_Accum sa;
+char c;
+
+void
+div_csa ()
+{
+  c /= sa;
+}
diff --git a/gcc/testsuite/gcc.target/i386/20020224-1.c b/gcc/testsuite/gcc.target/i386/20020224-1.c
index 2905719..769332b 100644
--- a/gcc/testsuite/gcc.target/i386/20020224-1.c
+++ b/gcc/testsuite/gcc.target/i386/20020224-1.c
@@ -4,6 +4,7 @@
    while callee was actually not poping it up (as the hidden argument
    was passed in register).  */
 /* { dg-do run } */
+/* { dg-require-effective-target ia32 } */
 /* { dg-options "-O2 -fomit-frame-pointer" } */
 
 extern void abort (void);
diff --git a/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowd2ps-2.c b/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowd2ps-2.c
index cfd5644..c9a2d19 100644
--- a/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowd2ps-2.c
+++ b/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowd2ps-2.c
@@ -1,6 +1,6 @@
 /* { dg-do run { target { ! ia32 } } } */
 /* { dg-require-effective-target amx_avx512 } */
-/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512" } */
+/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512 -mavx512fp16" } */
 #define AMX_AVX512
 #define DO_TEST test_amx_avx512_cvtrowd2ps
 void test_amx_avx512_cvtrowd2ps();
diff --git a/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2bf16-2.c b/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2bf16-2.c
index acd5f76..2014ec6 100644
--- a/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2bf16-2.c
+++ b/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2bf16-2.c
@@ -1,6 +1,6 @@
 /* { dg-do run { target { ! ia32 } } } */
 /* { dg-require-effective-target amx_avx512 } */
-/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512" } */
+/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512 -mavx512fp16" } */
 #define AMX_AVX512
 #define DO_TEST test_amx_avx512_cvtrowps2bf16
 void test_amx_avx512_cvtrowps2bf16();
diff --git a/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2ph-2.c b/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2ph-2.c
index 1fd28de..ca53ed00 100644
--- a/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2ph-2.c
+++ b/gcc/testsuite/gcc.target/i386/amxavx512-cvtrowps2ph-2.c
@@ -1,6 +1,6 @@
 /* { dg-do run { target { ! ia32 } } } */
 /* { dg-require-effective-target amx_avx512 } */
-/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512" } */
+/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512 -mavx512fp16" } */
 #define AMX_AVX512
 #define DO_TEST test_amx_avx512_cvtrowps2ph
 void test_amx_avx512_cvtrowps2ph();
diff --git a/gcc/testsuite/gcc.target/i386/amxavx512-movrow-2.c b/gcc/testsuite/gcc.target/i386/amxavx512-movrow-2.c
index ea28d82..b2dee14 100644
--- a/gcc/testsuite/gcc.target/i386/amxavx512-movrow-2.c
+++ b/gcc/testsuite/gcc.target/i386/amxavx512-movrow-2.c
@@ -1,6 +1,6 @@
 /* { dg-do run { target { ! ia32 } } } */
 /* { dg-require-effective-target amx_avx512 } */
-/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512" } */
+/* { dg-options "-O2 -march=x86-64-v3 -mamx-avx512 -mavx512fp16" } */
 #define AMX_AVX512
 #define DO_TEST test_amx_avx512_movrow
 void test_amx_avx512_movrow();
diff --git a/gcc/testsuite/gcc.target/i386/apx-1.c b/gcc/testsuite/gcc.target/i386/apx-1.c
index 4e580ec..b118928 100644
--- a/gcc/testsuite/gcc.target/i386/apx-1.c
+++ b/gcc/testsuite/gcc.target/i386/apx-1.c
@@ -3,6 +3,6 @@
 /* { dg-error "'-mapxf' is not supported for 32-bit code" "" { target ia32 } 0 } */
 
 void
-apx_hanlder ()
+apx_handler ()
 {
 }
diff --git a/gcc/testsuite/gcc.target/i386/asm-hard-reg-1.c b/gcc/testsuite/gcc.target/i386/asm-hard-reg-1.c
new file mode 100644
index 0000000..8080f56
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/asm-hard-reg-1.c
@@ -0,0 +1,80 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+void
+test (void)
+{
+  int x, y;
+
+  __asm__ __volatile__ ("" : "=a" (x), "={rbx}" (y));
+  __asm__ __volatile__ ("" : "=a" (x), "={rax}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=a" (x) : "{rax}" (y));
+  __asm__ __volatile__ ("" : "=&a" (x) : "{rax}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "a" (x), "{rax}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rbx}" (x), "=a" (y));
+  __asm__ __volatile__ ("" : "={rax}" (x), "=a" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rax}" (x) : "a" (y));
+  __asm__ __volatile__ ("" : "=&{rax}" (x) : "a" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rax}" (x), "a" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "=b" (x), "={rax}" (y));
+  __asm__ __volatile__ ("" : "=b" (x), "={rbx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=b" (x) : "{rbx}" (y));
+  __asm__ __volatile__ ("" : "=&b" (x) : "{rbx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "b" (x), "{rbx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rax}" (x), "=b" (y));
+  __asm__ __volatile__ ("" : "={rbx}" (x), "=b" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rbx}" (x) : "b" (y));
+  __asm__ __volatile__ ("" : "=&{rbx}" (x) : "b" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rbx}" (x), "b" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "=c" (x), "={rax}" (y));
+  __asm__ __volatile__ ("" : "=c" (x), "={rcx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=c" (x) : "{rcx}" (y));
+  __asm__ __volatile__ ("" : "=&c" (x) : "{rcx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "c" (x), "{rcx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rax}" (x), "=c" (y));
+  __asm__ __volatile__ ("" : "={rcx}" (x), "=c" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rcx}" (x) : "c" (y));
+  __asm__ __volatile__ ("" : "=&{rcx}" (x) : "c" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rcx}" (x), "c" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "=d" (x), "={rax}" (y));
+  __asm__ __volatile__ ("" : "=d" (x), "={rdx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=d" (x) : "{rdx}" (y));
+  __asm__ __volatile__ ("" : "=&d" (x) : "{rdx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "d" (x), "{rdx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rax}" (x), "=d" (y));
+  __asm__ __volatile__ ("" : "={rdx}" (x), "=d" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rdx}" (x) : "d" (y));
+  __asm__ __volatile__ ("" : "=&{rdx}" (x) : "d" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rdx}" (x), "d" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "=S" (x), "={rax}" (y));
+  __asm__ __volatile__ ("" : "=S" (x), "={rsi}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=S" (x) : "{rsi}" (y));
+  __asm__ __volatile__ ("" : "=&S" (x) : "{rsi}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "S" (x), "{rsi}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rax}" (x), "=S" (y));
+  __asm__ __volatile__ ("" : "={rsi}" (x), "=S" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rsi}" (x) : "S" (y));
+  __asm__ __volatile__ ("" : "=&{rsi}" (x) : "S" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rsi}" (x), "S" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "=D" (x), "={rax}" (y));
+  __asm__ __volatile__ ("" : "=D" (x), "={rdi}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=D" (x) : "{rdi}" (y));
+  __asm__ __volatile__ ("" : "=&D" (x) : "{rdi}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "D" (x), "{rdi}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rax}" (x), "=D" (y));
+  __asm__ __volatile__ ("" : "={rdi}" (x), "=D" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rdi}" (x) : "D" (y));
+  __asm__ __volatile__ ("" : "=&{rdi}" (x) : "D" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rdi}" (x), "D" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/asm-hard-reg-2.c b/gcc/testsuite/gcc.target/i386/asm-hard-reg-2.c
new file mode 100644
index 0000000..b35cf53
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/asm-hard-reg-2.c
@@ -0,0 +1,43 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+void
+test (void)
+{
+  int x, y, yy;
+#ifdef __x86_64__
+  int z __attribute__ ((mode (TI)));
+#else
+  long z;
+#endif
+
+  __asm__ __volatile__ ("" : "=A" (z), "={rbx}" (y));
+  __asm__ __volatile__ ("" : "=A" (z), "={rax}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=A" (z), "={rdx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=A" (z) : "{rax}" (y));
+  __asm__ __volatile__ ("" : "=A" (z) : "{rdx}" (y));
+  __asm__ __volatile__ ("" : "=&A" (z) : "{rax}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=&A" (z) : "{rdx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "A" (z), "{rax}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "A" (z), "{rdx}" (y)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  __asm__ __volatile__ ("" : "={rbx}" (y), "=A" (z));
+  __asm__ __volatile__ ("" : "={rax}" (y), "=A" (z)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rdx}" (y), "=A" (z)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "={rax}" (y) : "A" (z));
+  __asm__ __volatile__ ("" : "={rdx}" (y) : "A" (z));
+  __asm__ __volatile__ ("" : "=&{rax}" (y) : "A" (z)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=&{rdx}" (y) : "A" (z)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rax}" (y), "A" (z)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" :: "{rdx}" (y), "A" (z)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+
+  /* Note, we do not error for */
+  __asm__ __volatile__ ("" : "=A" (x), "={rax}" (y));
+  __asm__ __volatile__ ("" : "=A" (x), "={rdx}" (y));
+  /* This is due to how constraint A is implemented.  RA has the freedom to
+     choose between rax or rdx for operand 0 since x fits into a single
+     register and does not require a register pair.  Of course, we error out if
+     rax and rdx are taken by other operands as in the following:  */
+  __asm__ __volatile__ ("" : "=A" (x), "={rax}" (y), "={rdx}" (yy)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+  __asm__ __volatile__ ("" : "=A" (x), "={rdx}" (y), "={rax}" (yy)); /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/attributes-error.c b/gcc/testsuite/gcc.target/i386/attributes-error.c
index 405eda5..5d1c77d 100644
--- a/gcc/testsuite/gcc.target/i386/attributes-error.c
+++ b/gcc/testsuite/gcc.target/i386/attributes-error.c
@@ -1,12 +1,40 @@
+/* { dg-options "-msse2" } */
 /* { dg-do compile } */
 /* { dg-require-effective-target ia32 } */
 
-void foo1(int i, int j) __attribute__((fastcall, cdecl)); /* { dg-error "not compatible" } */
-void foo2(int i, int j) __attribute__((fastcall, stdcall)); /* { dg-error "not compatible" } */
+void foo1(int i, int j) __attribute__((cdecl, regparm(2)));
+void foo2(int i, int j) __attribute__((stdcall, regparm(2)));
 void foo3(int i, int j) __attribute__((fastcall, regparm(2))); /* { dg-error "not compatible" } */
-void foo4(int i, int j) __attribute__((stdcall, cdecl)); /* { dg-error "not compatible" } */
-void foo5(int i, int j) __attribute__((stdcall, fastcall)); /* { dg-error "not compatible" } */
-void foo6(int i, int j) __attribute__((cdecl, fastcall)); /* { dg-error "not compatible" } */
-void foo7(int i, int j) __attribute__((cdecl, stdcall)); /* { dg-error "not compatible" } */
-void foo8(int i, int j) __attribute__((regparm(2), fastcall)); /* { dg-error "not compatible" } */
+void foo4(int i, int j) __attribute__((thiscall, regparm(2))); /* { dg-error "not compatible" } */
+void foo5(int i, int j) __attribute__((sseregparm, regparm(2)));
+
+void foo6(int i, int j) __attribute__((stdcall, fastcall)); /* { dg-error "not compatible" } */
+void foo7(int i, int j) __attribute__((regparm(2), fastcall)); /* { dg-error "not compatible" } */
+void foo8(int i, int j) __attribute__((cdecl, fastcall)); /* { dg-error "not compatible" } */
+void foo9(int i, int j) __attribute__((thiscall, fastcall)); /* { dg-error "not compatible" } */
+void foo10(int i, int j) __attribute__((sseregparm, fastcall));
+
+void foo11(int i, int j) __attribute__((cdecl, stdcall)); /* { dg-error "not compatible" } */
+void foo12(int i, int j) __attribute__((fastcall, stdcall)); /* { dg-error "not compatible" } */
+void foo13(int i, int j) __attribute__((thiscall, stdcall)); /* { dg-error "not compatible" } */
+void foo14(int i, int j) __attribute__((regparm(2), stdcall));
+void foo15(int i, int j) __attribute__((sseregparm, stdcall));
+
+void foo16(int i, int j) __attribute__((stdcall, cdecl)); /* { dg-error "not compatible" } */
+void foo17(int i, int j) __attribute__((fastcall, cdecl)); /* { dg-error "not compatible" } */
+void foo18(int i, int j) __attribute__((thiscall, cdecl)); /* { dg-error "not compatible" } */
+void foo19(int i, int j) __attribute__((regparm(2), cdecl));
+void foo20(int i, int j) __attribute__((sseregparm, cdecl));
+
+void foo21(int i, int j) __attribute__((stdcall, thiscall)); /* { dg-error "not compatible" } */
+void foo22(int i, int j) __attribute__((fastcall, thiscall)); /* { dg-error "not compatible" } */
+void foo23(int i, int j) __attribute__((cdecl, thiscall)); /* { dg-error "not compatible" } */
+void foo24(int i, int j) __attribute__((regparm(2), thiscall)); /* { dg-error "not compatible" } */
+void foo25(int i, int j) __attribute__((sseregparm, thiscall));
+
+void foo26(int i, int j) __attribute__((cdecl, sseregparm));
+void foo27(int i, int j) __attribute__((fastcall, sseregparm));
+void foo28(int i, int j) __attribute__((stdcall, sseregparm));
+void foo29(int i, int j) __attribute__((thiscall, sseregparm));
+void foo30(int i, int j) __attribute__((regparm(2), sseregparm));
 
diff --git a/gcc/testsuite/gcc.target/i386/attributes-ignore.c b/gcc/testsuite/gcc.target/i386/attributes-ignore.c
new file mode 100644
index 0000000..93a3770
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/attributes-ignore.c
@@ -0,0 +1,8 @@
+/* { dg-do compile { target { ! ia32 } } } */
+
+void foo1(int i, int j) __attribute__((regparm(0))); /* { dg-warning "ignored" } */
+void foo2(int i, int j) __attribute__((stdcall)); /* { dg-warning "ignored" } */
+void foo3(int i, int j) __attribute__((fastcall)); /* { dg-warning "ignored" } */
+void foo4(int i, int j) __attribute__((cdecl)); /* { dg-warning "ignored" } */
+void foo5(int i, int j) __attribute__((thiscall)); /* { dg-warning "ignored" } */
+void foo6(int i, int j) __attribute__((sseregparm)); /* { dg-warning "ignored" } */
diff --git a/gcc/testsuite/gcc.target/i386/auto-init-padding-9.c b/gcc/testsuite/gcc.target/i386/auto-init-padding-9.c
index 102217c..4f26aa4 100644
--- a/gcc/testsuite/gcc.target/i386/auto-init-padding-9.c
+++ b/gcc/testsuite/gcc.target/i386/auto-init-padding-9.c
@@ -8,17 +8,28 @@
 /*
 **foo:
 **...
+**	leaq	-160\(%rbp\), %rax
+**	movq	%rax, %rcx
 **	pxor	%xmm0, %xmm0
-**...
+**	movl	\$160, %edx
+**	movl	%edx, %edi
+**	andl	\$-64, %edi
+**	movl	\$0, %esi
 **.L[0-9]+:
-**	movl	%esi, %ecx
-**	movaps	%xmm0, \(%rdx,%rcx\)
-**	movaps	%xmm0, 16\(%rdx,%rcx\)
-**	movaps	%xmm0, 32\(%rdx,%rcx\)
-**	movaps	%xmm0, 48\(%rdx,%rcx\)
+**	movl	%esi, %edx
+**	movaps	%xmm0, \(%rax,%rdx\)
+**	movaps	%xmm0, 16\(%rax,%rdx\)
+**	movaps	%xmm0, 32\(%rax,%rdx\)
+**	movaps	%xmm0, 48\(%rax,%rdx\)
 **	addl	\$64, %esi
 **	cmpl	%edi, %esi
 **	jb	.L[0-9]+
+**	movl	%esi, %eax
+**	addq	%rax, %rcx
+**	movaps	%xmm0, \(%rcx\)
+**	movaps	%xmm0, 16\(%rcx\)
+**	movzbl	-116\(%rbp\), %eax
+**	movsbl	%al, %eax
 **...
 */
 
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-1.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-1.c
new file mode 100644
index 0000000..b1f6678
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-1.c
@@ -0,0 +1,42 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mno-sse -mmemcpy-strategy=unrolled_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movq	221\(%rsi\), %rax
+**	xorl	%edx, %edx
+**	movq	%rax, 221\(%rdi\)
+**	movq	229\(%rsi\), %rax
+**	movq	%rax, 229\(%rdi\)
+**	movq	237\(%rsi\), %rax
+**	movq	%rax, 237\(%rdi\)
+**	movq	245\(%rsi\), %rax
+**	movq	%rax, 245\(%rdi\)
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	addl	\$32, %edx
+**	movq	\(%rsi,%rax\), %r10
+**	movq	8\(%rsi,%rax\), %r9
+**	movq	16\(%rsi,%rax\), %r8
+**	movq	24\(%rsi,%rax\), %rcx
+**	movq	%r10, \(%rdi,%rax\)
+**	movq	%r9, 8\(%rdi,%rax\)
+**	movq	%r8, 16\(%rdi,%rax\)
+**	movq	%rcx, 24\(%rdi,%rax\)
+**	cmpl	\$224, %edx
+**	jb	.L[0-9]+
+**	ret
+**...
+*/
+
+void
+foo (char *dest, char *src)
+{
+  __builtin_memcpy (dest, src, 253);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-2.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-2.c
new file mode 100644
index 0000000..0d0e348
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-2.c
@@ -0,0 +1,41 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemcpy-strategy=vector_loop:2048:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	xorl	%edx, %edx
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	addl	\$64, %edx
+**	movdqa	src\(%rax\), %xmm3
+**	movdqa	src\+16\(%rax\), %xmm2
+**	movdqa	src\+32\(%rax\), %xmm1
+**	movdqa	src\+48\(%rax\), %xmm0
+**	movaps	%xmm3, dest\(%rax\)
+**	movaps	%xmm2, dest\+16\(%rax\)
+**	movaps	%xmm1, dest\+32\(%rax\)
+**	movaps	%xmm0, dest\+48\(%rax\)
+**	cmpl	\$256, %edx
+**	jb	.L[0-9]+
+**	movdqa	src\(%rdx\), %xmm0
+**	movaps	%xmm0, dest\(%rdx\)
+**	ret
+**...
+*/
+
+#define SIZE (16 + 1) * 16
+
+char dest[SIZE];
+char src[SIZE];
+
+void
+foo (void)
+{
+  __builtin_memcpy (dest, src, SIZE);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-3.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-3.c
new file mode 100644
index 0000000..e5aca32
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-3.c
@@ -0,0 +1,43 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemcpy-strategy=vector_loop:2048:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	xorl	%edx, %edx
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	addl	\$64, %edx
+**	movdqa	src\(%rax\), %xmm3
+**	movdqa	src\+16\(%rax\), %xmm2
+**	movdqa	src\+32\(%rax\), %xmm1
+**	movdqa	src\+48\(%rax\), %xmm0
+**	movaps	%xmm3, dest\(%rax\)
+**	movaps	%xmm2, dest\+16\(%rax\)
+**	movaps	%xmm1, dest\+32\(%rax\)
+**	movaps	%xmm0, dest\+48\(%rax\)
+**	cmpl	\$256, %edx
+**	jb	.L[0-9]+
+**	movdqa	src\(%rdx\), %xmm0
+**	movaps	%xmm0, dest\(%rdx\)
+**	movdqu	src\+15\(%rdx\), %xmm0
+**	movups	%xmm0, dest\+15\(%rdx\)
+**	ret
+**...
+*/
+
+#define SIZE 16 * 16 + 31
+
+char dest[SIZE];
+char src[SIZE];
+
+void
+foo (void)
+{
+  __builtin_memcpy (dest, src, SIZE);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-4.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-4.c
new file mode 100644
index 0000000..27f7bed
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-4.c
@@ -0,0 +1,42 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemcpy-strategy=vector_loop:2048:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	xorl	%edx, %edx
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	subl	\$-128, %edx
+**	vmovdqa	src\(%rax\), %ymm3
+**	vmovdqa	src\+32\(%rax\), %ymm2
+**	vmovdqa	src\+64\(%rax\), %ymm1
+**	vmovdqa	src\+96\(%rax\), %ymm0
+**	vmovdqa	%ymm3, dest\(%rax\)
+**	vmovdqa	%ymm2, dest\+32\(%rax\)
+**	vmovdqa	%ymm1, dest\+64\(%rax\)
+**	vmovdqa	%ymm0, dest\+96\(%rax\)
+**	cmpl	\$512, %edx
+**	jb	.L[0-9]+
+**	vmovdqa	src\(%rdx\), %ymm0
+**	vmovdqa	%ymm0, dest\(%rdx\)
+**	vzeroupper
+**	ret
+**...
+*/
+
+#define SIZE (16 + 1) * 32
+
+char dest[SIZE];
+char src[SIZE];
+
+void
+foo (void)
+{
+  __builtin_memcpy (dest, src, SIZE);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-5.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-5.c
new file mode 100644
index 0000000..34a7408
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-5.c
@@ -0,0 +1,44 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemcpy-strategy=vector_loop:2048:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	xorl	%edx, %edx
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	subl	\$-128, %edx
+**	vmovdqa	src\(%rax\), %ymm3
+**	vmovdqa	src\+32\(%rax\), %ymm2
+**	vmovdqa	src\+64\(%rax\), %ymm1
+**	vmovdqa	src\+96\(%rax\), %ymm0
+**	vmovdqa	%ymm3, dest\(%rax\)
+**	vmovdqa	%ymm2, dest\+32\(%rax\)
+**	vmovdqa	%ymm1, dest\+64\(%rax\)
+**	vmovdqa	%ymm0, dest\+96\(%rax\)
+**	cmpl	\$512, %edx
+**	jb	.L[0-9]+
+**	vmovdqa	src\(%rdx\), %ymm0
+**	vmovdqa	%ymm0, dest\(%rdx\)
+**	vmovdqu	src\+31\(%rdx\), %ymm0
+**	vmovdqu	%ymm0, dest\+31\(%rdx\)
+**	vzeroupper
+**	ret
+**...
+*/
+
+#define SIZE 16 * 32 + 32 + 31
+
+char dest[SIZE];
+char src[SIZE];
+
+void
+foo (void)
+{
+  __builtin_memcpy (dest, src, SIZE);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-6.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-6.c
new file mode 100644
index 0000000..aa5d90d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-6.c
@@ -0,0 +1,42 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v4 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemcpy-strategy=vector_loop:2048:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	xorl	%edx, %edx
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	addl	\$256, %edx
+**	vmovdqa64	src\(%rax\), %zmm3
+**	vmovdqa64	src\+64\(%rax\), %zmm2
+**	vmovdqa64	src\+128\(%rax\), %zmm1
+**	vmovdqa64	src\+192\(%rax\), %zmm0
+**	vmovdqa64	%zmm3, dest\(%rax\)
+**	vmovdqa64	%zmm2, dest\+64\(%rax\)
+**	vmovdqa64	%zmm1, dest\+128\(%rax\)
+**	vmovdqa64	%zmm0, dest\+192\(%rax\)
+**	cmpl	\$1024, %edx
+**	jb	.L[0-9]+
+**	vmovdqa64	src\(%rdx\), %zmm0
+**	vmovdqa64	%zmm0, dest\(%rdx\)
+**	vzeroupper
+**	ret
+**...
+*/
+
+#define SIZE (16 + 1) * 64
+
+char dest[SIZE] __attribute__((aligned(64)));
+char src[SIZE] __attribute__((aligned(64)));
+
+void
+foo (void)
+{
+  __builtin_memcpy (dest, src, SIZE);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-pr120683-7.c b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-7.c
new file mode 100644
index 0000000..63d8a15
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memcpy-pr120683-7.c
@@ -0,0 +1,44 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v4 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemcpy-strategy=vector_loop:2048:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	xorl	%edx, %edx
+**.L[0-9]+:
+**	movl	%edx, %eax
+**	addl	\$256, %edx
+**	vmovdqa64	src\(%rax\), %zmm3
+**	vmovdqa64	src\+64\(%rax\), %zmm2
+**	vmovdqa64	src\+128\(%rax\), %zmm1
+**	vmovdqa64	src\+192\(%rax\), %zmm0
+**	vmovdqa64	%zmm3, dest\(%rax\)
+**	vmovdqa64	%zmm2, dest\+64\(%rax\)
+**	vmovdqa64	%zmm1, dest\+128\(%rax\)
+**	vmovdqa64	%zmm0, dest\+192\(%rax\)
+**	cmpl	\$1024, %edx
+**	jb	.L[0-9]+
+**	vmovdqa	src\(%rdx\), %ymm0
+**	vmovdqa	%ymm0, dest\(%rdx\)
+**	vmovdqu	src\+31\(%rdx\), %ymm0
+**	vmovdqu	%ymm0, dest\+31\(%rdx\)
+**	vzeroupper
+**	ret
+**...
+*/
+
+#define SIZE 16 * 64 + 63
+
+char dest[SIZE] __attribute__((aligned(64)));
+char src[SIZE] __attribute__((aligned(64)));
+
+void
+foo (void)
+{
+  __builtin_memcpy (dest, src, SIZE);
+}
+
+/* { dg-final { scan-assembler-not "rep mov" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memcpy-strategy-12.c b/gcc/testsuite/gcc.target/i386/memcpy-strategy-12.c
index d0316ef..c60cef0 100644
--- a/gcc/testsuite/gcc.target/i386/memcpy-strategy-12.c
+++ b/gcc/testsuite/gcc.target/i386/memcpy-strategy-12.c
@@ -1,14 +1,21 @@
 /* { dg-do compile } */
-/* { dg-options "-O2 -mtune=generic -mno-sse" } */
+/* { dg-options "-O2 -mtune=generic -mno-sse -fasynchronous-unwind-tables -fdwarf2-cfi-asm" } */
 /* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
 /* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
 
 /*
 **foo:
 **.LFB[0-9]+:
-**...
+**	.cfi_startproc
+**	movq	221\(%rsi\), %rax
 **	xorl	%edx, %edx
-**...
+**	movq	%rax, 221\(%rdi\)
+**	movq	229\(%rsi\), %rax
+**	movq	%rax, 229\(%rdi\)
+**	movq	237\(%rsi\), %rax
+**	movq	%rax, 237\(%rdi\)
+**	movq	245\(%rsi\), %rax
+**	movq	%rax, 245\(%rdi\)
 **.L[0-9]+:
 **	movl	%edx, %eax
 **	addl	\$32, %edx
@@ -22,6 +29,7 @@
 **	movq	%rcx, 24\(%rdi,%rax\)
 **	cmpl	\$224, %edx
 **	jb	.L[0-9]+
+**	ret
 **...
 */
 
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-1.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-1.c
new file mode 100644
index 0000000..06e3892
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-1.c
@@ -0,0 +1,35 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	pxor	%xmm0, %xmm0
+**	xorl	%eax, %eax
+**	movups	%xmm0, 190\(%rdi\)
+**	movups	%xmm0, 206\(%rdi\)
+**	movups	%xmm0, 222\(%rdi\)
+**	movups	%xmm0, 238\(%rdi\)
+**.L[0-9]+:
+**	movl	%eax, %edx
+**	addl	\$64, %eax
+**	movups	%xmm0, \(%rdi,%rdx\)
+**	movups	%xmm0, 16\(%rdi,%rdx\)
+**	movups	%xmm0, 32\(%rdi,%rdx\)
+**	movups	%xmm0, 48\(%rdi,%rdx\)
+**	cmpl	\$192, %eax
+**	jb	.L[0-9]+
+**	ret
+**...
+*/
+
+void
+foo (char *dest)
+{
+  __builtin_memset (dest, 0, 254);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-10.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-10.c
new file mode 100644
index 0000000..36a924d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-10.c
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mno-sse -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=unrolled_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movq	\$0, 48\(%rdi\)
+**	movq	\$0, \(%rdi\)
+**	movq	\$0, 8\(%rdi\)
+**	movq	\$0, 16\(%rdi\)
+**	movq	\$0, 24\(%rdi\)
+**	movq	\$0, 32\(%rdi\)
+**	movq	\$0, 40\(%rdi\)
+**	movq	\$0, 53\(%rdi\)
+**	ret
+**...
+*/
+
+void
+foo (char *dest)
+{
+  __builtin_memset (dest, 0, 61);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-11.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-11.c
new file mode 100644
index 0000000..4868e56
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-11.c
@@ -0,0 +1,29 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mno-sse -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=unrolled_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movabsq	\$289360691352306692, %rax
+**	movq	%rax, 48\(%rdi\)
+**	movq	%rax, \(%rdi\)
+**	movq	%rax, 8\(%rdi\)
+**	movq	%rax, 16\(%rdi\)
+**	movq	%rax, 24\(%rdi\)
+**	movq	%rax, 32\(%rdi\)
+**	movq	%rax, 40\(%rdi\)
+**	movq	%rax, 53\(%rdi\)
+**	ret
+**...
+*/
+
+void
+foo (char *dest)
+{
+  __builtin_memset (dest, 4, 61);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-12.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-12.c
new file mode 100644
index 0000000..9112897
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-12.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mno-sse -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=unrolled_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movabsq	\$72340172838076673, %rax
+**	movzbl	%sil, %esi
+**	imulq	%rax, %rsi
+**	movq	%rsi, 48\(%rdi\)
+**	movq	%rsi, \(%rdi\)
+**	movq	%rsi, 8\(%rdi\)
+**	movq	%rsi, 16\(%rdi\)
+**	movq	%rsi, 24\(%rdi\)
+**	movq	%rsi, 32\(%rdi\)
+**	movq	%rsi, 40\(%rdi\)
+**	movq	%rsi, 53\(%rdi\)
+**	ret
+**...
+*/
+
+void
+foo (char *dest, int c)
+{
+  __builtin_memset (dest, c, 61);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-13.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-13.c
new file mode 100644
index 0000000..69ec6c6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-13.c
@@ -0,0 +1,36 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	pxor	%xmm0, %xmm0
+**	xorl	%eax, %eax
+**.L[0-9]+:
+**	movl	%eax, %edx
+**	addl	\$64, %eax
+**	movaps	%xmm0, dest\(%rdx\)
+**	movaps	%xmm0, dest\+16\(%rdx\)
+**	movaps	%xmm0, dest\+32\(%rdx\)
+**	movaps	%xmm0, dest\+48\(%rdx\)
+**	cmpl	\$192, %eax
+**	jb	.L[0-9]+
+**	movaps	%xmm0, dest\(%rax\)
+**	movaps	%xmm0, dest\+16\(%rax\)
+**	movaps	%xmm0, dest\+32\(%rax\)
+**	ret
+**...
+*/
+
+char dest[240];
+
+void
+foo (void)
+{
+  __builtin_memset (dest, 0, sizeof (dest));
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-14.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-14.c
new file mode 100644
index 0000000..209cd67
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-14.c
@@ -0,0 +1,91 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	pxor	%xmm0, %xmm0
+**	cmpq	\$64, %rsi
+**	jnb	.L2
+**	testb	\$32, %sil
+**	jne	.L19
+**	testb	\$16, %sil
+**	jne	.L20
+**	testb	\$8, %sil
+**	jne	.L21
+**	testb	\$4, %sil
+**	jne	.L22
+**	testq	%rsi, %rsi
+**	jne	.L23
+**.L1:
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	movups	%xmm0, -64\(%rdi,%rsi\)
+**	movups	%xmm0, -48\(%rdi,%rsi\)
+**	movups	%xmm0, -32\(%rdi,%rsi\)
+**	movups	%xmm0, -16\(%rdi,%rsi\)
+**	subq	\$1, %rsi
+**	cmpq	\$64, %rsi
+**	jb	.L1
+**	andq	\$-64, %rsi
+**	xorl	%eax, %eax
+**.L9:
+**	movups	%xmm0, \(%rdi,%rax\)
+**	movups	%xmm0, 16\(%rdi,%rax\)
+**	movups	%xmm0, 32\(%rdi,%rax\)
+**	movups	%xmm0, 48\(%rdi,%rax\)
+**	addq	\$64, %rax
+**	cmpq	%rsi, %rax
+**	jb	.L9
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	movb	\$0, \(%rdi\)
+**	testb	\$2, %sil
+**	je	.L1
+**	xorl	%eax, %eax
+**	movw	%ax, -2\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L19:
+**	movups	%xmm0, \(%rdi\)
+**	movups	%xmm0, 16\(%rdi\)
+**	movups	%xmm0, -32\(%rdi,%rsi\)
+**	movups	%xmm0, -16\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L20:
+**	movups	%xmm0, \(%rdi\)
+**	movups	%xmm0, -16\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L21:
+**	movq	\$0, \(%rdi\)
+**	movq	\$0, -8\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	movl	\$0, \(%rdi\)
+**	movl	\$0, -4\(%rdi,%rsi\)
+**	ret
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, 0, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-15.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-15.c
new file mode 100644
index 0000000..d19188f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-15.c
@@ -0,0 +1,103 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	vpxor	%xmm0, %xmm0, %xmm0
+**	cmpq	\$128, %rsi
+**	jnb	.L2
+**	testb	\$64, %sil
+**	jne	.L22
+**	testb	\$32, %sil
+**	jne	.L23
+**	testb	\$16, %sil
+**	jne	.L24
+**	testb	\$8, %sil
+**	jne	.L25
+**	testb	\$4, %sil
+**	jne	.L26
+**	testq	%rsi, %rsi
+**	jne	.L27
+**.L20:
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	vmovdqu	%ymm0, -128\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -96\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -64\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	subq	\$1, %rsi
+**	cmpq	\$128, %rsi
+**	jb	.L19
+**	andq	\$-128, %rsi
+**	xorl	%eax, %eax
+**.L10:
+**	vmovdqu	%ymm0, \(%rdi,%rax\)
+**	vmovdqu	%ymm0, 32\(%rdi,%rax\)
+**	vmovdqu	%ymm0, 64\(%rdi,%rax\)
+**	vmovdqu	%ymm0, 96\(%rdi,%rax\)
+**	subq	\$-128, %rax
+**	cmpq	%rsi, %rax
+**	jb	.L10
+**.L19:
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L27:
+**	movb	\$0, \(%rdi\)
+**	testb	\$2, %sil
+**	je	.L20
+**	xorl	%eax, %eax
+**	movw	%ax, -2\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, 32\(%rdi\)
+**	vmovdqu	%ymm0, -64\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L24:
+**	vmovdqu	%xmm0, \(%rdi\)
+**	vmovdqu	%xmm0, -16\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L25:
+**	movq	\$0, \(%rdi\)
+**	movq	\$0, -8\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L26:
+**	movl	\$0, \(%rdi\)
+**	movl	\$0, -4\(%rdi,%rsi\)
+**	ret
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, 0, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-16.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-16.c
new file mode 100644
index 0000000..539714c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-16.c
@@ -0,0 +1,112 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v4 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	vpxor	%xmm0, %xmm0, %xmm0
+**	cmpq	\$256, %rsi
+**	jnb	.L2
+**	testb	\$-128, %sil
+**	jne	.L23
+**	testb	\$64, %sil
+**	jne	.L24
+**	testb	\$32, %sil
+**	jne	.L25
+**	testb	\$16, %sil
+**	jne	.L26
+**	testb	\$8, %sil
+**	jne	.L27
+**	testb	\$4, %sil
+**	jne	.L28
+**	testq	%rsi, %rsi
+**	jne	.L29
+**.L21:
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	vmovdqu64	%zmm0, -256\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -192\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -128\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rsi\)
+**	subq	\$1, %rsi
+**	cmpq	\$256, %rsi
+**	jb	.L20
+**	xorb	%sil, %sil
+**	xorl	%eax, %eax
+**.L11:
+**	vmovdqu64	%zmm0, \(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 64\(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 128\(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 192\(%rdi,%rax\)
+**	addq	\$256, %rax
+**	cmpq	%rsi, %rax
+**	jb	.L11
+**.L20:
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L29:
+**	movb	\$0, \(%rdi\)
+**	testb	\$2, %sil
+**	je	.L21
+**	xorl	%eax, %eax
+**	movw	%ax, -2\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	vmovdqu64	%zmm0, \(%rdi\)
+**	vmovdqu64	%zmm0, 64\(%rdi\)
+**	vmovdqu64	%zmm0, -128\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rsi\)
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L24:
+**	vmovdqu64	%zmm0, \(%rdi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rsi\)
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L25:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L26:
+**	vmovdqu	%xmm0, \(%rdi\)
+**	vmovdqu	%xmm0, -16\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L27:
+**	movq	\$0, \(%rdi\)
+**	movq	\$0, -8\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L28:
+**	movl	\$0, \(%rdi\)
+**	movl	\$0, -4\(%rdi,%rsi\)
+**	ret
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, 0, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-17.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-17.c
new file mode 100644
index 0000000..f58cb28
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-17.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	pxor	%xmm0, %xmm0
+**	xorl	%eax, %eax
+**.L[0-9]+:
+**	movl	%eax, %edx
+**	addl	\$64, %eax
+**	movaps	%xmm0, dest\(%rdx\)
+**	movaps	%xmm0, dest\+16\(%rdx\)
+**	movaps	%xmm0, dest\+32\(%rdx\)
+**	movaps	%xmm0, dest\+48\(%rdx\)
+**	cmpl	\$128, %eax
+**	jb	.L[0-9]+
+**	movq	\$0, dest\+48\(%rax\)
+**	movaps	%xmm0, dest\(%rax\)
+**	movaps	%xmm0, dest\+16\(%rax\)
+**	movaps	%xmm0, dest\+32\(%rax\)
+**	ret
+**...
+*/
+
+char dest[184];
+
+void
+foo (void)
+{
+  __builtin_memset (dest, 0, sizeof (dest));
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-18.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-18.c
new file mode 100644
index 0000000..a127028
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-18.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	pxor	%xmm0, %xmm0
+**	xorl	%eax, %eax
+**.L[0-9]+:
+**	movl	%eax, %edx
+**	addl	\$64, %eax
+**	movaps	%xmm0, dest\(%rdx\)
+**	movaps	%xmm0, dest\+16\(%rdx\)
+**	movaps	%xmm0, dest\+32\(%rdx\)
+**	movaps	%xmm0, dest\+48\(%rdx\)
+**	cmpl	\$128, %eax
+**	jb	.L[0-9]+
+**	movaps	%xmm0, dest\+32\(%rax\)
+**	movaps	%xmm0, dest\(%rax\)
+**	movl	\$0, dest\+47\(%rax\)
+**	movaps	%xmm0, dest\+16\(%rax\)
+**	ret
+**...
+*/
+
+char dest[179];
+
+void
+foo (void)
+{
+  __builtin_memset (dest, 0, sizeof (dest));
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-19.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-19.c
new file mode 100644
index 0000000..8dd5ae6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-19.c
@@ -0,0 +1,37 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	pxor	%xmm0, %xmm0
+**	xorl	%eax, %eax
+**.L[0-9]+:
+**	movl	%eax, %edx
+**	addl	\$64, %eax
+**	movaps	%xmm0, dest\(%rdx\)
+**	movaps	%xmm0, dest\+16\(%rdx\)
+**	movaps	%xmm0, dest\+32\(%rdx\)
+**	movaps	%xmm0, dest\+48\(%rdx\)
+**	cmpl	\$128, %eax
+**	jb	.L[0-9]+
+**	movb	\$0, dest\+48\(%rax\)
+**	movaps	%xmm0, dest\(%rax\)
+**	movaps	%xmm0, dest\+16\(%rax\)
+**	movaps	%xmm0, dest\+32\(%rax\)
+**	ret
+**...
+*/
+
+char dest[177];
+
+void
+foo (void)
+{
+  __builtin_memset (dest, 0, sizeof (dest));
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-2.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-2.c
new file mode 100644
index 0000000..3b84b29
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-2.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	vpxor	%xmm0, %xmm0, %xmm0
+**	vmovdqu	%ymm0, 192\(%rdi\)
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, 32\(%rdi\)
+**	vmovdqu	%ymm0, 64\(%rdi\)
+**	vmovdqu	%ymm0, 96\(%rdi\)
+**	vmovdqu	%ymm0, 128\(%rdi\)
+**	vmovdqu	%ymm0, 160\(%rdi\)
+**	vmovdqu	%ymm0, 222\(%rdi\)
+**	vzeroupper
+**	ret
+**...
+*/
+
+void
+foo (char *dest)
+{
+  __builtin_memset (dest, 0, 254);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-20.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-20.c
new file mode 100644
index 0000000..b8b9cb7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-20.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movd	%edi, %xmm0
+**	punpcklbw	%xmm0, %xmm0
+**	punpcklwd	%xmm0, %xmm0
+**	pshufd	\$0, %xmm0, %xmm0
+**	movaps	%xmm0, dest\+160\(%rip\)
+**	movaps	%xmm0, dest\(%rip\)
+**	movaps	%xmm0, dest\+16\(%rip\)
+**	movaps	%xmm0, dest\+32\(%rip\)
+**	movaps	%xmm0, dest\+48\(%rip\)
+**	movaps	%xmm0, dest\+64\(%rip\)
+**	movaps	%xmm0, dest\+80\(%rip\)
+**	movaps	%xmm0, dest\+96\(%rip\)
+**	movaps	%xmm0, dest\+112\(%rip\)
+**	movaps	%xmm0, dest\+128\(%rip\)
+**	movaps	%xmm0, dest\+144\(%rip\)
+**	movd	%xmm0, dest\+175\(%rip\)
+**	ret
+**...
+*/
+
+char dest[179];
+
+void
+foo (int c)
+{
+  __builtin_memset (dest, c, sizeof (dest));
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-21.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-21.c
new file mode 100644
index 0000000..3c7bb7c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-21.c
@@ -0,0 +1,38 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movd	%edi, %xmm0
+**	movb	%dil, dest\+176\(%rip\)
+**	punpcklbw	%xmm0, %xmm0
+**	punpcklwd	%xmm0, %xmm0
+**	pshufd	\$0, %xmm0, %xmm0
+**	movaps	%xmm0, dest\(%rip\)
+**	movaps	%xmm0, dest\+16\(%rip\)
+**	movaps	%xmm0, dest\+32\(%rip\)
+**	movaps	%xmm0, dest\+48\(%rip\)
+**	movaps	%xmm0, dest\+64\(%rip\)
+**	movaps	%xmm0, dest\+80\(%rip\)
+**	movaps	%xmm0, dest\+96\(%rip\)
+**	movaps	%xmm0, dest\+112\(%rip\)
+**	movaps	%xmm0, dest\+128\(%rip\)
+**	movaps	%xmm0, dest\+144\(%rip\)
+**	movaps	%xmm0, dest\+160\(%rip\)
+**	ret
+**...
+*/
+
+char dest[177];
+
+void
+foo (int c)
+{
+  __builtin_memset (dest, c, sizeof (dest));
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-22.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-22.c
new file mode 100644
index 0000000..96a21c8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-22.c
@@ -0,0 +1,27 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=rep_8byte:8192:align,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	movl	\$25, %ecx
+**	xorl	%eax, %eax
+**	movl	\$dest, %edi
+**	rep stosq
+**	movl	\$0, \(%rdi\)
+**	ret
+**...
+*/
+
+#define SIZE 204
+
+char dest[SIZE];
+
+void
+foo (void)
+{
+  __builtin_memset (dest, 0, sizeof (dest));
+}
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-23.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-23.c
new file mode 100644
index 0000000..f3f5d80
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-23.c
@@ -0,0 +1,67 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -minline-all-stringops -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movzbl	%dil, %edi
+**	movl	\$p, %eax
+**	movabsq	\$72340172838076673, %rdx
+**	imulq	%rdx, %rdi
+**	movq	%rdi, %xmm0
+**	punpcklqdq	%xmm0, %xmm0
+**	cmpq	\$64, %rsi
+**	jnb	.L18
+**.L2:
+**	movq	%rsi, %rcx
+**	andl	\$63, %ecx
+**	je	.L1
+**	xorl	%edx, %edx
+**	andl	\$1, %esi
+**	je	.L5
+**	movl	\$1, %edx
+**	movb	%dil, \(%rax\)
+**	cmpq	%rcx, %rdx
+**	jnb	.L19
+**.L5:
+**	movb	%dil, \(%rax,%rdx\)
+**	movb	%dil, 1\(%rax,%rdx\)
+**	addq	\$2, %rdx
+**	cmpq	%rcx, %rdx
+**	jb	.L5
+**.L1:
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L18:
+**	movq	%rsi, %rdx
+**	xorl	%eax, %eax
+**	andq	\$-64, %rdx
+**.L3:
+**	movaps	%xmm0, p\(%rax\)
+**	addq	\$64, %rax
+**	movaps	%xmm0, p-48\(%rax\)
+**	movaps	%xmm0, p-32\(%rax\)
+**	movaps	%xmm0, p-16\(%rax\)
+**	cmpq	%rdx, %rax
+**	jb	.L3
+**	addq	\$p, %rax
+**	jmp	.L2
+**.L19:
+**	ret
+**	.cfi_endproc
+**...
+*/
+
+
+#define WRITE_CHUNK 256
+char p[WRITE_CHUNK];
+
+void
+foo (int c, __SIZE_TYPE__ nbyte)
+{
+ __builtin_memset (p, c, nbyte);
+}
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-3.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-3.c
new file mode 100644
index 0000000..faa47ca
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-3.c
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v4 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	vpxor	%xmm0, %xmm0, %xmm0
+**	vmovdqu8	%zmm0, 128\(%rdi\)
+**	vmovdqu8	%zmm0, \(%rdi\)
+**	vmovdqu8	%zmm0, 64\(%rdi\)
+**	vmovdqu8	%zmm0, 190\(%rdi\)
+**	vzeroupper
+**	ret
+**...
+*/
+
+void
+foo (char *dest)
+{
+  __builtin_memset (dest, 0, 254);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-4.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-4.c
new file mode 100644
index 0000000..dc3aa57b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-4.c
@@ -0,0 +1,93 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movabsq	\$289360691352306692, %rax
+**	movq	%rax, %xmm0
+**	punpcklqdq	%xmm0, %xmm0
+**	cmpq	\$64, %rsi
+**	jnb	.L2
+**	testb	\$32, %sil
+**	jne	.L19
+**	testb	\$16, %sil
+**	jne	.L20
+**	testb	\$8, %sil
+**	jne	.L21
+**	testb	\$4, %sil
+**	jne	.L22
+**	testq	%rsi, %rsi
+**	jne	.L23
+**.L1:
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	movups	%xmm0, -64\(%rdi,%rsi\)
+**	movups	%xmm0, -48\(%rdi,%rsi\)
+**	movups	%xmm0, -32\(%rdi,%rsi\)
+**	movups	%xmm0, -16\(%rdi,%rsi\)
+**	subq	\$1, %rsi
+**	cmpq	\$64, %rsi
+**	jb	.L1
+**	andq	\$-64, %rsi
+**	xorl	%eax, %eax
+**.L9:
+**	movups	%xmm0, \(%rdi,%rax\)
+**	movups	%xmm0, 16\(%rdi,%rax\)
+**	movups	%xmm0, 32\(%rdi,%rax\)
+**	movups	%xmm0, 48\(%rdi,%rax\)
+**	addq	\$64, %rax
+**	cmpq	%rsi, %rax
+**	jb	.L9
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	movb	\$4, \(%rdi\)
+**	testb	\$2, %sil
+**	je	.L1
+**	movl	\$1028, %eax
+**	movw	%ax, -2\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L19:
+**	movups	%xmm0, \(%rdi\)
+**	movups	%xmm0, 16\(%rdi\)
+**	movups	%xmm0, -32\(%rdi,%rsi\)
+**	movups	%xmm0, -16\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L20:
+**	movups	%xmm0, \(%rdi\)
+**	movups	%xmm0, -16\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L21:
+**	movq	%rax, \(%rdi\)
+**	movq	%rax, -8\(%rdi,%rsi\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	movl	\$67372036, \(%rdi\)
+**	movl	\$67372036, -4\(%rdi,%rsi\)
+**	ret
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, 4, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-5.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-5.c
new file mode 100644
index 0000000..a324f8e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-5.c
@@ -0,0 +1,102 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movabsq	\$289360691352306692, %rax
+**	vmovq	%rax, %xmm1
+**	vpbroadcastq	%xmm1, %ymm0
+**	cmpq	\$128, %rsi
+**	jnb	.L2
+**	testb	\$64, %sil
+**	jne	.L21
+**	testb	\$32, %sil
+**	jne	.L22
+**	testb	\$16, %sil
+**	jne	.L23
+**	testb	\$8, %sil
+**	jne	.L24
+**	testb	\$4, %sil
+**	jne	.L25
+**	testq	%rsi, %rsi
+**	jne	.L26
+**.L19:
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	vmovdqu	%ymm0, -128\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -96\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -64\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	subq	\$1, %rsi
+**	cmpq	\$128, %rsi
+**	jb	.L19
+**	andq	\$-128, %rsi
+**	xorl	%eax, %eax
+**.L10:
+**	vmovdqu	%ymm0, \(%rdi,%rax\)
+**	vmovdqu	%ymm0, 32\(%rdi,%rax\)
+**	vmovdqu	%ymm0, 64\(%rdi,%rax\)
+**	vmovdqu	%ymm0, 96\(%rdi,%rax\)
+**	subq	\$-128, %rax
+**	cmpq	%rsi, %rax
+**	jb	.L10
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L26:
+**	movb	\$4, \(%rdi\)
+**	testb	\$2, %sil
+**	je	.L19
+**	movl	\$1028, %eax
+**	movw	%ax, -2\(%rdi,%rsi\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L21:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, 32\(%rdi\)
+**	vmovdqu	%ymm0, -64\(%rdi,%rsi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	vmovdqu	%xmm0, \(%rdi\)
+**	vmovdqu	%xmm0, -16\(%rdi,%rsi\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L24:
+**	movq	%rax, \(%rdi\)
+**	movq	%rax, -8\(%rdi,%rsi\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L25:
+**	movl	\$67372036, \(%rdi\)
+**	movl	\$67372036, -4\(%rdi,%rsi\)
+**	jmp	.L19
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, 4, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-6.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-6.c
new file mode 100644
index 0000000..64e7589
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-6.c
@@ -0,0 +1,109 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v4 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movabsq	\$289360691352306692, %rax
+**	vpbroadcastq	%rax, %zmm0
+**	cmpq	\$256, %rsi
+**	jnb	.L2
+**	testb	\$-128, %sil
+**	jne	.L22
+**	testb	\$64, %sil
+**	jne	.L23
+**	testb	\$32, %sil
+**	jne	.L24
+**	testb	\$16, %sil
+**	jne	.L25
+**	testb	\$8, %sil
+**	jne	.L26
+**	testb	\$4, %sil
+**	jne	.L27
+**	testq	%rsi, %rsi
+**	jne	.L28
+**.L20:
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	vmovdqu64	%zmm0, -256\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -192\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -128\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rsi\)
+**	subq	\$1, %rsi
+**	cmpq	\$256, %rsi
+**	jb	.L20
+**	xorb	%sil, %sil
+**	xorl	%eax, %eax
+**.L11:
+**	vmovdqu64	%zmm0, \(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 64\(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 128\(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 192\(%rdi,%rax\)
+**	addq	\$256, %rax
+**	cmpq	%rsi, %rax
+**	jb	.L11
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L28:
+**	movb	\$4, \(%rdi\)
+**	testb	\$2, %sil
+**	je	.L20
+**	movl	\$1028, %eax
+**	movw	%ax, -2\(%rdi,%rsi\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	vmovdqu64	%zmm0, \(%rdi\)
+**	vmovdqu64	%zmm0, 64\(%rdi\)
+**	vmovdqu64	%zmm0, -128\(%rdi,%rsi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rsi\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	vmovdqu64	%zmm0, \(%rdi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rsi\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L24:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rsi\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L25:
+**	vmovdqu	%xmm0, \(%rdi\)
+**	vmovdqu	%xmm0, -16\(%rdi,%rsi\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L26:
+**	movq	%rax, \(%rdi\)
+**	movq	%rax, -8\(%rdi,%rsi\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L27:
+**	movl	\$67372036, \(%rdi\)
+**	movl	\$67372036, -4\(%rdi,%rsi\)
+**	jmp	.L20
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, 4, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-7.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-7.c
new file mode 100644
index 0000000..022f6f9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-7.c
@@ -0,0 +1,94 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movabsq	\$72340172838076673, %rax
+**	movzbl	%sil, %esi
+**	imulq	%rax, %rsi
+**	movq	%rsi, %xmm0
+**	punpcklqdq	%xmm0, %xmm0
+**	cmpq	\$64, %rdx
+**	jnb	.L2
+**	testb	\$32, %dl
+**	jne	.L19
+**	testb	\$16, %dl
+**	jne	.L20
+**	testb	\$8, %dl
+**	jne	.L21
+**	testb	\$4, %dl
+**	jne	.L22
+**	testq	%rdx, %rdx
+**	jne	.L23
+**.L1:
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	movups	%xmm0, -64\(%rdi,%rdx\)
+**	movups	%xmm0, -48\(%rdi,%rdx\)
+**	movups	%xmm0, -32\(%rdi,%rdx\)
+**	movups	%xmm0, -16\(%rdi,%rdx\)
+**	subq	\$1, %rdx
+**	cmpq	\$64, %rdx
+**	jb	.L1
+**	andq	\$-64, %rdx
+**	xorl	%eax, %eax
+**.L9:
+**	movups	%xmm0, \(%rdi,%rax\)
+**	movups	%xmm0, 16\(%rdi,%rax\)
+**	movups	%xmm0, 32\(%rdi,%rax\)
+**	movups	%xmm0, 48\(%rdi,%rax\)
+**	addq	\$64, %rax
+**	cmpq	%rdx, %rax
+**	jb	.L9
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	movb	%sil, \(%rdi\)
+**	testb	\$2, %dl
+**	je	.L1
+**	movw	%si, -2\(%rdi,%rdx\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L19:
+**	movups	%xmm0, \(%rdi\)
+**	movups	%xmm0, 16\(%rdi\)
+**	movups	%xmm0, -32\(%rdi,%rdx\)
+**	movups	%xmm0, -16\(%rdi,%rdx\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L20:
+**	movups	%xmm0, \(%rdi\)
+**	movups	%xmm0, -16\(%rdi,%rdx\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L21:
+**	movq	%rsi, \(%rdi\)
+**	movq	%rsi, -8\(%rdi,%rdx\)
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	movl	%esi, \(%rdi\)
+**	movl	%esi, -4\(%rdi,%rdx\)
+**	ret
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, int c, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, c, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-8.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-8.c
new file mode 100644
index 0000000..5254e21
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-8.c
@@ -0,0 +1,103 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movabsq	\$72340172838076673, %rax
+**	movzbl	%sil, %esi
+**	imulq	%rax, %rsi
+**	vmovq	%rsi, %xmm1
+**	vpbroadcastq	%xmm1, %ymm0
+**	cmpq	\$128, %rdx
+**	jnb	.L2
+**	testb	\$64, %dl
+**	jne	.L21
+**	testb	\$32, %dl
+**	jne	.L22
+**	testb	\$16, %dl
+**	jne	.L23
+**	testb	\$8, %dl
+**	jne	.L24
+**	testb	\$4, %dl
+**	jne	.L25
+**	testq	%rdx, %rdx
+**	jne	.L26
+**.L19:
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	vmovdqu	%ymm0, -128\(%rdi,%rdx\)
+**	vmovdqu	%ymm0, -96\(%rdi,%rdx\)
+**	vmovdqu	%ymm0, -64\(%rdi,%rdx\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rdx\)
+**	subq	\$1, %rdx
+**	cmpq	\$128, %rdx
+**	jb	.L19
+**	andq	\$-128, %rdx
+**	xorl	%eax, %eax
+**.L10:
+**	vmovdqu	%ymm0, \(%rdi,%rax\)
+**	vmovdqu	%ymm0, 32\(%rdi,%rax\)
+**	vmovdqu	%ymm0, 64\(%rdi,%rax\)
+**	vmovdqu	%ymm0, 96\(%rdi,%rax\)
+**	subq	\$-128, %rax
+**	cmpq	%rdx, %rax
+**	jb	.L10
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L26:
+**	movb	%sil, \(%rdi\)
+**	testb	\$2, %dl
+**	je	.L19
+**	movw	%si, -2\(%rdi,%rdx\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L21:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, 32\(%rdi\)
+**	vmovdqu	%ymm0, -64\(%rdi,%rdx\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rdx\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rdx\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	vmovdqu	%xmm0, \(%rdi\)
+**	vmovdqu	%xmm0, -16\(%rdi,%rdx\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L24:
+**	movq	%rsi, \(%rdi\)
+**	movq	%rsi, -8\(%rdi,%rdx\)
+**	jmp	.L19
+**	.p2align 4,,10
+**	.p2align 3
+**.L25:
+**	movl	%esi, \(%rdi\)
+**	movl	%esi, -4\(%rdi,%rdx\)
+**	jmp	.L19
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, int c, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, c, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-pr120683-9.c b/gcc/testsuite/gcc.target/i386/memset-pr120683-9.c
new file mode 100644
index 0000000..1719de6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/memset-pr120683-9.c
@@ -0,0 +1,110 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v4 -fasynchronous-unwind-tables -fdwarf2-cfi-asm -mmemset-strategy=vector_loop:256:noalign,libcall:-1:noalign -minline-all-stringops" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target lp64 } {^\t?\.} } } */
+
+/*
+**foo:
+**.LFB0:
+**	.cfi_startproc
+**	movabsq	\$72340172838076673, %rax
+**	movzbl	%sil, %esi
+**	imulq	%rax, %rsi
+**	vpbroadcastq	%rsi, %zmm0
+**	cmpq	\$256, %rdx
+**	jnb	.L2
+**	testb	\$-128, %dl
+**	jne	.L22
+**	testb	\$64, %dl
+**	jne	.L23
+**	testb	\$32, %dl
+**	jne	.L24
+**	testb	\$16, %dl
+**	jne	.L25
+**	testb	\$8, %dl
+**	jne	.L26
+**	testb	\$4, %dl
+**	jne	.L27
+**	testq	%rdx, %rdx
+**	jne	.L28
+**.L20:
+**	vzeroupper
+**	ret
+**	.p2align 4,,10
+**	.p2align 3
+**.L2:
+**	vmovdqu64	%zmm0, -256\(%rdi,%rdx\)
+**	vmovdqu64	%zmm0, -192\(%rdi,%rdx\)
+**	vmovdqu64	%zmm0, -128\(%rdi,%rdx\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rdx\)
+**	subq	\$1, %rdx
+**	cmpq	\$256, %rdx
+**	jb	.L20
+**	xorb	%dl, %dl
+**	xorl	%eax, %eax
+**.L11:
+**	vmovdqu64	%zmm0, \(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 64\(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 128\(%rdi,%rax\)
+**	vmovdqu64	%zmm0, 192\(%rdi,%rax\)
+**	addq	\$256, %rax
+**	cmpq	%rdx, %rax
+**	jb	.L11
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L28:
+**	movb	%sil, \(%rdi\)
+**	testb	\$2, %dl
+**	je	.L20
+**	movw	%si, -2\(%rdi,%rdx\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L22:
+**	vmovdqu64	%zmm0, \(%rdi\)
+**	vmovdqu64	%zmm0, 64\(%rdi\)
+**	vmovdqu64	%zmm0, -128\(%rdi,%rdx\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rdx\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L23:
+**	vmovdqu64	%zmm0, \(%rdi\)
+**	vmovdqu64	%zmm0, -64\(%rdi,%rdx\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L24:
+**	vmovdqu	%ymm0, \(%rdi\)
+**	vmovdqu	%ymm0, -32\(%rdi,%rdx\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L25:
+**	vmovdqu	%xmm0, \(%rdi\)
+**	vmovdqu	%xmm0, -16\(%rdi,%rdx\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L26:
+**	movq	%rsi, \(%rdi\)
+**	movq	%rsi, -8\(%rdi,%rdx\)
+**	jmp	.L20
+**	.p2align 4,,10
+**	.p2align 3
+**.L27:
+**	movl	%esi, \(%rdi\)
+**	movl	%esi, -4\(%rdi,%rdx\)
+**	jmp	.L20
+**	.cfi_endproc
+**...
+*/
+
+void
+foo (char *dest, int c, __SIZE_TYPE__ n)
+{
+  __builtin_memset (dest, c, n);
+}
+
+/* { dg-final { scan-assembler-not "rep stos" } } */
diff --git a/gcc/testsuite/gcc.target/i386/memset-strategy-25.c b/gcc/testsuite/gcc.target/i386/memset-strategy-25.c
index 1cc3de7..7bd5d43 100644
--- a/gcc/testsuite/gcc.target/i386/memset-strategy-25.c
+++ b/gcc/testsuite/gcc.target/i386/memset-strategy-25.c
@@ -7,7 +7,11 @@
 **foo:
 **.LFB[0-9]+:
 **	.cfi_startproc
+**	movq	\$0, 221\(%rdi\)
 **	xorl	%eax, %eax
+**	movq	\$0, 229\(%rdi\)
+**	movq	\$0, 237\(%rdi\)
+**	movq	\$0, 245\(%rdi\)
 **.L[0-9]+:
 **	movl	%eax, %edx
 **	addl	\$32, %eax
@@ -17,6 +21,7 @@
 **	movq	\$0, 24\(%rdi,%rdx\)
 **	cmpl	\$224, %eax
 **	jb	.L[0-9]+
+**	ret
 **...
 */
 
diff --git a/gcc/testsuite/gcc.target/i386/memset-strategy-29.c b/gcc/testsuite/gcc.target/i386/memset-strategy-29.c
index 61aef92..a33bf92 100644
--- a/gcc/testsuite/gcc.target/i386/memset-strategy-29.c
+++ b/gcc/testsuite/gcc.target/i386/memset-strategy-29.c
@@ -8,7 +8,11 @@
 **...
 **.LFB[0-9]+:
 **	.cfi_startproc
+**	movq	\$0, 49\(%rdi\)
 **	xorl	%eax, %eax
+**	movq	\$0, 57\(%rdi\)
+**	movq	\$0, 65\(%rdi\)
+**	movq	\$0, 73\(%rdi\)
 **.L[0-9]+:
 **	movl	%eax, %edx
 **	addl	\$32, %eax
@@ -18,6 +22,7 @@
 **	movq	\$0, 24\(%rdi,%rdx\)
 **	cmpl	\$64, %eax
 **	jb	.L[0-9]+
+**	ret
 **...
 */
 
diff --git a/gcc/testsuite/gcc.target/i386/memset-strategy-30.c b/gcc/testsuite/gcc.target/i386/memset-strategy-30.c
index 917f151..f3912f8 100644
--- a/gcc/testsuite/gcc.target/i386/memset-strategy-30.c
+++ b/gcc/testsuite/gcc.target/i386/memset-strategy-30.c
@@ -8,7 +8,11 @@
 **...
 **.LFB[0-9]+:
 **	.cfi_startproc
+**	movq	\$0, 63\(%rdi\)
 **	xorl	%eax, %eax
+**	movq	\$0, 71\(%rdi\)
+**	movq	\$0, 79\(%rdi\)
+**	movq	\$0, 87\(%rdi\)
 **.L[0-9]+:
 **	movl	%eax, %edx
 **	addl	\$32, %eax
@@ -18,6 +22,7 @@
 **	movq	\$0, 24\(%rdi,%rdx\)
 **	cmpl	\$64, %eax
 **	jb	.L[0-9]+
+**	ret
 **...
 */
 
diff --git a/gcc/testsuite/gcc.target/i386/memset-strategy-31.c b/gcc/testsuite/gcc.target/i386/memset-strategy-31.c
index 17a4df2..4791c4d 100644
--- a/gcc/testsuite/gcc.target/i386/memset-strategy-31.c
+++ b/gcc/testsuite/gcc.target/i386/memset-strategy-31.c
@@ -9,6 +9,10 @@
 **...
 **	pxor	%xmm0, %xmm0
 **	xorl	%eax, %eax
+**	movups	%xmm0, 190\(%rdi\)
+**	movups	%xmm0, 206\(%rdi\)
+**	movups	%xmm0, 222\(%rdi\)
+**	movups	%xmm0, 238\(%rdi\)
 **.L[0-9]+:
 **	movl	%eax, %edx
 **	addl	\$64, %eax
diff --git a/gcc/testsuite/gcc.target/i386/pr103785.c b/gcc/testsuite/gcc.target/i386/pr103785.c
index 5503b96..49d6c56 100644
--- a/gcc/testsuite/gcc.target/i386/pr103785.c
+++ b/gcc/testsuite/gcc.target/i386/pr103785.c
@@ -11,7 +11,10 @@ struct wrapper_t
 
 struct wrapper_t **table;
 
-__attribute__ ((weak, regparm (2)))
+#ifndef __x86_64__
+__attribute__ ((regparm (2)))
+#endif
+__attribute__ ((weak))
 void
 update (long k, long e)
 {
diff --git a/gcc/testsuite/gcc.target/i386/pr104447.c b/gcc/testsuite/gcc.target/i386/pr104447.c
index cb618c7..145ba90 100644
--- a/gcc/testsuite/gcc.target/i386/pr104447.c
+++ b/gcc/testsuite/gcc.target/i386/pr104447.c
@@ -1,6 +1,7 @@
 /* { dg-do compile } */
 /* { dg-require-profiling "-pg" } */
 /* { dg-options "-O2 -pg" } */
+/* { dg-additional-options "-mfentry -fno-pic" { target *-*-gnu* } } */
 
 int
 bar (int x)
diff --git a/gcc/testsuite/gcc.target/i386/pr113122-3.c b/gcc/testsuite/gcc.target/i386/pr113122-3.c
index 71aa240..87b76de 100644
--- a/gcc/testsuite/gcc.target/i386/pr113122-3.c
+++ b/gcc/testsuite/gcc.target/i386/pr113122-3.c
@@ -2,6 +2,7 @@
 /* { dg-do assemble { target *-*-linux* } } */
 /* { dg-require-effective-target masm_intel } */
 /* { dg-options "-fprofile -O2 -masm=intel" } */
+/* { dg-additional-options "-mfentry -fno-pic" { target *-*-gnu* } } */
 
 void
 func (void)
diff --git a/gcc/testsuite/gcc.target/i386/pr119386-1.c b/gcc/testsuite/gcc.target/i386/pr119386-1.c
index 9a0dc64..7a56eac 100644
--- a/gcc/testsuite/gcc.target/i386/pr119386-1.c
+++ b/gcc/testsuite/gcc.target/i386/pr119386-1.c
@@ -1,7 +1,9 @@
 /* PR target/119386 */
 /* { dg-do compile { target *-*-linux* } } */
 /* { dg-options "-O2 -fpic -pg" } */
-/* { dg-final { scan-assembler "call\[ \t\]+mcount@PLT" } } */
+/* { dg-additional-options "-mfentry" { target { *-*-gnu* && { ! ia32 } } } } */
+/* { dg-final { scan-assembler "call\[ \t\]+mcount@PLT" { target ia32 } } } */
+/* { dg-final { scan-assembler "call\[ \t\]+__fentry__@PLT" { target { *-*-gnu* && { ! ia32 } } } } } */
 
 int
 main ()
diff --git a/gcc/testsuite/gcc.target/i386/pr119386-2.c b/gcc/testsuite/gcc.target/i386/pr119386-2.c
index 3ea978e..cddaaf0 100644
--- a/gcc/testsuite/gcc.target/i386/pr119386-2.c
+++ b/gcc/testsuite/gcc.target/i386/pr119386-2.c
@@ -1,7 +1,8 @@
 /* PR target/119386 */
 /* { dg-do compile { target *-*-linux* } } */
 /* { dg-options "-O2 -fpic -fno-plt -pg" } */
-/* { dg-final { scan-assembler "call\[ \t\]+\\*mcount@GOTPCREL\\(" { target { ! ia32 } } } } */
+/* { dg-additional-options "-mfentry" { target { *-*-gnu* && { ! ia32 } } } } */
+/* { dg-final { scan-assembler "call\[ \t\]+\\*__fentry__@GOTPCREL" { target { *-*-gnu* && { ! ia32 } } } } } */
 /* { dg-final { scan-assembler "call\[ \t\]+\\*mcount@GOT\\(" { target ia32 } } } */
 
 
diff --git a/gcc/testsuite/gcc.target/i386/pr119795.c b/gcc/testsuite/gcc.target/i386/pr119795.c
new file mode 100644
index 0000000..03c91cc
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr119795.c
@@ -0,0 +1,26 @@
+/* { dg-do run } */
+/* { dg-options "-O -fschedule-insns -favoid-store-forwarding" } */
+
+unsigned a, b, c;
+
+void
+foo (_BitInt(2) b2, unsigned _BitInt(255) by, unsigned _BitInt(5) b5,
+     unsigned _BitInt(256) *ret)
+{
+  unsigned _BitInt(255) bx = b2;
+  by += 0x80000000000000000000000000000000wb;
+  __builtin_memmove (&b, &c, 3);
+  unsigned d = b;
+  unsigned e = __builtin_stdc_rotate_right (0x1uwb % b5, a);
+  unsigned _BitInt(256) r = by + bx + d + e;
+  *ret = r;
+}
+
+int
+main ()
+{
+  unsigned  _BitInt(256) x;
+  foo (0, -1, 2, &x);
+  if (x != 0x80000000000000000000000000000000wb)
+    __builtin_abort();
+}
\ No newline at end of file
diff --git a/gcc/testsuite/gcc.target/i386/pr120427-5.c b/gcc/testsuite/gcc.target/i386/pr120427-5.c
new file mode 100644
index 0000000..7199aef
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120427-5.c
@@ -0,0 +1,10 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-Oz" } */
+
+long long
+func1 (void)
+{
+  return -1;
+}
+/* { dg-final { scan-assembler-times "pushq\[ \\t\]+\\\$-1" 1 } } */
+/* { dg-final { scan-assembler-times "popq\[ \\t\]+%rax" 1 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr120881-1a.c b/gcc/testsuite/gcc.target/i386/pr120881-1a.c
new file mode 100644
index 0000000..3d9ac0e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120881-1a.c
@@ -0,0 +1,4 @@
+/* { dg-do compile { target fpic } } */
+/* { dg-require-profiling "-pg" } */
+/* { dg-options "-O2 -pg -mno-fentry -fno-pic" } */
+/* { dg-message "'-pg' without '-mfentry' may be unreliable with shrink wrapping" "" { target *-*-* } 0 } */
diff --git a/gcc/testsuite/gcc.target/i386/pr120881-1b.c b/gcc/testsuite/gcc.target/i386/pr120881-1b.c
new file mode 100644
index 0000000..0826407
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120881-1b.c
@@ -0,0 +1,4 @@
+/* { dg-do compile { target { fpic && { ! ia32 } } } } */
+/* { dg-require-profiling "-pg" } */
+/* { dg-options "-O2 -pg -mno-fentry -fpic" } */
+/* { dg-message "'-pg' without '-mfentry' may be unreliable with shrink wrapping" "" { target *-*-* } 0 } */
diff --git a/gcc/testsuite/gcc.target/i386/pr120881-1c.c b/gcc/testsuite/gcc.target/i386/pr120881-1c.c
new file mode 100644
index 0000000..c21979f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120881-1c.c
@@ -0,0 +1,3 @@
+/* { dg-do compile { target { fpic && ia32 } } } */
+/* { dg-require-profiling "-pg" } */
+/* { dg-options "-O2 -pg -mno-fentry -fpic" } */
diff --git a/gcc/testsuite/gcc.target/i386/pr120881-1d.c b/gcc/testsuite/gcc.target/i386/pr120881-1d.c
new file mode 100644
index 0000000..f74af23
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120881-1d.c
@@ -0,0 +1,3 @@
+/* { dg-do compile { target { fpic && ia32 } } } */
+/* { dg-require-profiling "-pg" } */
+/* { dg-options "-O2 -pg -mno-fentry -fno-shrink-wrap -fno-pic" } */
diff --git a/gcc/testsuite/gcc.target/i386/pr120881-2a.c b/gcc/testsuite/gcc.target/i386/pr120881-2a.c
new file mode 100644
index 0000000..52e3e52
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120881-2a.c
@@ -0,0 +1,21 @@
+/* { dg-do compile { target fentry } } */
+/* { dg-options "-O2 -pg" } */
+/* Keep labels and directives ('.cfi_startproc', '.cfi_endproc').  */
+/* { dg-final { check-function-bodies "**" "" "" { target "*-*-*" } {^\t?\.} } } */
+
+/*
+**f2:
+**.LFB[0-9]+:
+**	.cfi_startproc
+**	call	__fentry__
+**...
+*/
+
+extern void f1 (void);
+
+void
+f2 (int count)
+{
+  for (int i = 0; i < count; ++i)
+    f1 ();
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr120881-2b.c b/gcc/testsuite/gcc.target/i386/pr120881-2b.c
new file mode 100644
index 0000000..43a12f0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr120881-2b.c
@@ -0,0 +1,6 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-rtl-pro_and_epilogue -march=x86-64" } */
+/* { dg-final { scan-rtl-dump "Now spread 1 times" "pro_and_epilogue" } } */
+
+#include "pr120881-2a.c"
+
diff --git a/gcc/testsuite/gcc.target/i386/pr121015.c b/gcc/testsuite/gcc.target/i386/pr121015.c
new file mode 100644
index 0000000..57c8bff
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121015.c
@@ -0,0 +1,32 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3" } */
+
+extern union {
+  int i;
+  float f;
+} int_as_float_u;
+
+extern int render_result_from_bake_w;
+extern int render_result_from_bake_h_seed_pass;
+extern float *render_result_from_bake_h_primitive;
+extern float *render_result_from_bake_h_seed;
+
+float
+int_as_float(int i)
+{
+  int_as_float_u.i = i;
+  return int_as_float_u.f;
+}
+
+void
+render_result_from_bake_h(int tx)
+{
+  while (render_result_from_bake_w) {
+    for (; tx < render_result_from_bake_w; tx++)
+      render_result_from_bake_h_primitive[1] =
+          render_result_from_bake_h_primitive[2] = int_as_float(-1);
+    if (render_result_from_bake_h_seed_pass) {
+      *render_result_from_bake_h_seed = 0;
+    }
+  }
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-1.c b/gcc/testsuite/gcc.target/i386/pr121062-1.c
new file mode 100644
index 0000000..799f856
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-1.c
@@ -0,0 +1,34 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3" } */
+
+extern union {
+  int i;
+  float f;
+} int_as_float_u;
+
+extern int render_result_from_bake_w;
+extern int render_result_from_bake_h_seed_pass;
+extern float *render_result_from_bake_h_primitive;
+extern float *render_result_from_bake_h_seed;
+
+float
+int_as_float(int i)
+{
+  int_as_float_u.i = i;
+  return int_as_float_u.f;
+}
+
+void
+render_result_from_bake_h(int tx)
+{
+  while (render_result_from_bake_w) {
+    for (; tx < render_result_from_bake_w; tx++)
+      render_result_from_bake_h_primitive[1] =
+          render_result_from_bake_h_primitive[2] = int_as_float(-1);
+    if (render_result_from_bake_h_seed_pass) {
+      *render_result_from_bake_h_seed = 0;
+    }
+  }
+}
+
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+\\\$-1, %r\[a-z0-9\]+" 2 { target { ! ia32 } } } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-2.c b/gcc/testsuite/gcc.target/i386/pr121062-2.c
new file mode 100644
index 0000000..723d68a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-2.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-Og -fno-dce -mtune=generic" } */
+
+typedef int __attribute__((__vector_size__ (4))) S;
+extern void bar (S);
+
+void
+foo ()
+{
+  bar ((S){-1});
+}
+
+/* { dg-final { scan-assembler-times "movl\[ \\t\]+\\\$-1, \\(%esp\\)" 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "movl\[ \\t\]+\\\$-1, %edi" 1 { target { ! ia32 } } } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-3a.c b/gcc/testsuite/gcc.target/i386/pr121062-3a.c
new file mode 100644
index 0000000..effd4ff
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-3a.c
@@ -0,0 +1,23 @@
+/* { dg-do compile { target fpic } } */
+/* { dg-options "-O2 -march=x86-64 -fpic" } */
+
+typedef struct {
+  struct {
+    unsigned short lo4;
+    unsigned short lo3;
+    unsigned short lo2;
+    unsigned short lo1;
+  } i;
+} BID_BINARY80LDOUBLE;
+extern BID_BINARY80LDOUBLE __bid64_to_binary80_x_out;
+void
+__bid64_to_binary80 (void)
+{
+  __bid64_to_binary80_x_out.i.lo4
+    = __bid64_to_binary80_x_out.i.lo3
+    = __bid64_to_binary80_x_out.i.lo2
+    = __bid64_to_binary80_x_out.i.lo1 = 65535;
+}
+
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+%xmm\[0-9\]+, " 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+\\\$-1, \\(%(e|r)\[a-z0-9\]+\\)" 1 { target { ! ia32 } } } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-3b.c b/gcc/testsuite/gcc.target/i386/pr121062-3b.c
new file mode 100644
index 0000000..eb89b5d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-3b.c
@@ -0,0 +1,6 @@
+/* { dg-do compile { target { fpic && lp64 } } } */
+/* { dg-options "-O2 -march=x86-64 -fno-pic -mcmodel=large" } */
+
+#include "pr121062-3a.c"
+
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+\\\$-1, \\(%r\[a-z0-9\]+\\)" 1 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-3c.c b/gcc/testsuite/gcc.target/i386/pr121062-3c.c
new file mode 100644
index 0000000..4c07029
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-3c.c
@@ -0,0 +1,6 @@
+/* { dg-do compile { target { fpic && lp64 } } } */
+/* { dg-options "-O2 -march=x86-64 -fpic -mcmodel=large" } */
+
+#include "pr121062-3a.c"
+
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+\\\$-1, \\(%r\[a-z0-9\]+\\)" 1 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-4.c b/gcc/testsuite/gcc.target/i386/pr121062-4.c
new file mode 100644
index 0000000..77a0c2e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-4.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64" } */
+
+typedef long long int __attribute__((__vector_size__ (8))) S;
+
+void
+foo (S *c)
+{
+  *c = (S){0x12345678badbeefULL};
+}
+
+
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+%xmm\[0-9\]+, " 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "movabsq\[ \\t\]+\\\$81985529250168559, %r\[a-z0-9\]+" 1 { target { ! ia32 } } } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-5.c b/gcc/testsuite/gcc.target/i386/pr121062-5.c
new file mode 100644
index 0000000..22c09a6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-5.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64" } */
+
+typedef int __attribute__((__vector_size__ (4))) S;
+
+void
+foo (S *c)
+{
+  *c = (S){0x12345678};
+}
+
+
+/* { dg-final { scan-assembler-times "movl\[ \\t\]+\\\$305419896, \\(%(e|r)\[a-z0-9\]+\\)" 1 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-6.c b/gcc/testsuite/gcc.target/i386/pr121062-6.c
new file mode 100644
index 0000000..780b496
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-6.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-Og -fno-dce -mtune=generic" } */
+
+typedef int __attribute__((__vector_size__ (8))) S;
+
+void
+foo (S *c)
+{
+  *c = (S){0x12345678,0xbadbeefULL};
+}
+
+/* { dg-final { scan-assembler-times "movq\[ \\t\]+%xmm\[0-9\]+, " 1 { target ia32 } } } */
+/* { dg-final { scan-assembler-times "movabsq\[ \\t\]+\\\$841538639400031864, %r\[a-z0-9\]+" 1 { target { ! ia32 } } } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121062-7.c b/gcc/testsuite/gcc.target/i386/pr121062-7.c
new file mode 100644
index 0000000..f1834f8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121062-7.c
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64" } */
+
+typedef __bf16 __attribute__((__vector_size__ (4))) S;
+
+void
+foo (S *c)
+{
+  *c = (S){-0.1, 2.1};
+}
+
+
+/* { dg-final { scan-assembler-times "movl\[ \\t\]+\\\$1074183629, \\(%(e|r)\[a-z0-9\]+\\)" 1 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-1a.c b/gcc/testsuite/gcc.target/i386/pr121208-1a.c
new file mode 100644
index 0000000..cb8bd0b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-1a.c
@@ -0,0 +1,15 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mno-80387 -mtls-dialect=gnu" } */
+
+extern __thread int bar;
+extern void func (void);
+
+__attribute__((no_caller_saved_registers))
+void
+foo (int error)
+{
+  bar = 1; /* { dg-error -mtls-dialect=gnu2 } */
+  if (error == 0)
+    func ();
+  bar = 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-1b.c b/gcc/testsuite/gcc.target/i386/pr121208-1b.c
new file mode 100644
index 0000000..037e9a0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-1b.c
@@ -0,0 +1,4 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mno-80387 -mtls-dialect=gnu2" } */
+
+#include "pr121208-1a.c"
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-2a.c b/gcc/testsuite/gcc.target/i386/pr121208-2a.c
new file mode 100644
index 0000000..c1891ae
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-2a.c
@@ -0,0 +1,17 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mtls-dialect=gnu" } */
+
+typedef unsigned int uword_t __attribute__ ((mode (__word__)));
+extern __thread int bar;
+extern void func (void);
+
+__attribute__((target("general-regs-only")))
+__attribute__((interrupt))
+void
+foo (void *frame, uword_t error)
+{
+  bar = 1; /* { dg-error -mtls-dialect=gnu2 } */
+  if (error == 0)
+    func ();
+  bar = 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-2b.c b/gcc/testsuite/gcc.target/i386/pr121208-2b.c
new file mode 100644
index 0000000..269b120
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-2b.c
@@ -0,0 +1,4 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mtls-dialect=gnu2" } */
+
+#include "pr121208-2a.c"
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-3a.c b/gcc/testsuite/gcc.target/i386/pr121208-3a.c
new file mode 100644
index 0000000..26fe687
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-3a.c
@@ -0,0 +1,17 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mtls-dialect=gnu" } */
+
+typedef unsigned int uword_t __attribute__ ((mode (__word__)));
+extern __thread int bar;
+extern void func (void);
+
+__attribute__((target("general-regs-only")))
+__attribute__((interrupt))
+void
+foo (void *frame)
+{
+  bar = 1; /* { dg-error -mtls-dialect=gnu2 } */
+  if (frame == 0)
+    func ();
+  bar = 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr121208-3b.c b/gcc/testsuite/gcc.target/i386/pr121208-3b.c
new file mode 100644
index 0000000..b672d75
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121208-3b.c
@@ -0,0 +1,4 @@
+/* { dg-do compile { target *-*-linux* } } */
+/* { dg-options "-O2 -fPIC -mtls-dialect=gnu2" } */
+
+#include "pr121208-3a.c"
diff --git a/gcc/testsuite/gcc.target/i386/pr121274.c b/gcc/testsuite/gcc.target/i386/pr121274.c
new file mode 100644
index 0000000..16760cf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr121274.c
@@ -0,0 +1,24 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-march=x86-64-v4 -O2" } */
+/* { dg-final { scan-assembler-not "vpextrq" } } */
+/* { dg-final { scan-assembler-not "vpinsrq" } } */
+
+typedef int v16si __attribute__((vector_size(64)));
+typedef int v4si __attribute__((vector_size(16)));
+
+v4si f(v16si x)
+{
+  return __builtin_shufflevector(x, x, 0, 1, 2, 3);
+}
+
+v4si g(v16si x)
+{
+return __builtin_shufflevector(x, x, 4, 5, 6, 7);
+}
+
+v4si f1(__int128 *x)
+{
+  __int128 t = *x;
+  asm("":"+x"(t));
+  return (v4si)t;
+}
diff --git a/gcc/testsuite/gcc.target/i386/pr15184-2.c b/gcc/testsuite/gcc.target/i386/pr15184-2.c
index cb8201f..dd50c42 100644
--- a/gcc/testsuite/gcc.target/i386/pr15184-2.c
+++ b/gcc/testsuite/gcc.target/i386/pr15184-2.c
@@ -1,4 +1,4 @@
-/* PR 15184 second two tests
+/* PR 15184 second two tests */
 /* { dg-do compile { target ia32 } } */
 /* { dg-options "-O2 -march=pentiumpro" } */
 /* { dg-additional-options "-fno-PIE" { target ia32 } } */
diff --git a/gcc/testsuite/gcc.target/i386/pr36533.c b/gcc/testsuite/gcc.target/i386/pr36533.c
index 8d71ece..8699d26 100644
--- a/gcc/testsuite/gcc.target/i386/pr36533.c
+++ b/gcc/testsuite/gcc.target/i386/pr36533.c
@@ -55,14 +55,22 @@ typedef struct
   S1 *s18;
 } S7;
 
-__attribute__((regparm (3), noinline)) int
+#ifndef __x86_64__
+__attribute__((regparm (3)))
+#endif
+__attribute__((noinline))
+int
 fn1 (const char *x, void *y, S1 *z)
 {
   asm volatile ("" : : : "memory");
   return *x + (y != 0);
 }
 
-__attribute__((regparm (3), noinline)) int
+#ifndef __x86_64__
+__attribute__((regparm (3)))
+#endif
+__attribute__((noinline))
+int
 fn2 (const char *x, int y, S2 *z)
 {
   asm volatile ("" : : : "memory");
@@ -84,7 +92,11 @@ fn3 (S3 *p)
   return (S3 *) ((char *) p + fn4 (p->s9));
 }
 
-__attribute__((regparm (3), noinline)) int
+#ifndef __x86_64__
+__attribute__((regparm (3)))
+#endif
+__attribute__((noinline))
+int
 fn5 (void)
 {
   asm volatile ("" : : : "memory");
@@ -116,7 +128,11 @@ fn6 (S3 *w, int x, S2 *y, S4 *z)
   return a;
 }
 
-__attribute__((regparm (3), noinline)) unsigned int
+#ifndef __x86_64__
+__attribute__((regparm (3)))
+#endif
+__attribute__((noinline))
+unsigned int
 test (void *u, S6 *v, S1 **w, S7 *x, S2 *y, S1 *z)
 {
   unsigned b = v->s17->s16;
diff --git a/gcc/testsuite/gcc.target/i386/pr59099.c b/gcc/testsuite/gcc.target/i386/pr59099.c
index cf4a8da..21dfbc2 100644
--- a/gcc/testsuite/gcc.target/i386/pr59099.c
+++ b/gcc/testsuite/gcc.target/i386/pr59099.c
@@ -13,10 +13,17 @@ struct s
 };
 
 
-void* f (struct s *, struct s *) __attribute__ ((noinline, regparm(1)));
+void* f (struct s *, struct s *)
+#ifndef __x86_64__
+__attribute__ ((regparm(1)))
+#endif
+__attribute__ ((noinline))
+;
 
 void*
+#ifndef __x86_64__
 __attribute__ ((regparm(1)))
+#endif
 f (struct s *p, struct s *p2)
 {
   void *gp, *gp1;
diff --git a/gcc/testsuite/gcc.target/i386/pr82699-1.c b/gcc/testsuite/gcc.target/i386/pr82699-1.c
index 272d079..96e3ccb 100644
--- a/gcc/testsuite/gcc.target/i386/pr82699-1.c
+++ b/gcc/testsuite/gcc.target/i386/pr82699-1.c
@@ -1,5 +1,5 @@
 /* { dg-do compile { target *-*-linux* } } */
-/* { dg-options "-O2 -fno-pic -fcf-protection -pg -fasynchronous-unwind-tables" } */
+/* { dg-options "-O2 -mfentry -fno-pic -fcf-protection -pg -fasynchronous-unwind-tables" } */
 /* { dg-final { scan-assembler-times {\t\.cfi_startproc\n\tendbr} 1 } } */
 
 extern int bar (int);
diff --git a/gcc/testsuite/gcc.target/i386/pr91384-1.c b/gcc/testsuite/gcc.target/i386/pr91384-1.c
new file mode 100644
index 0000000..4f8823d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr91384-1.c
@@ -0,0 +1,20 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O2 -mapxf" } */
+
+void foo (void);
+void bar (void);
+
+int
+test (int a)
+{
+  int r;
+
+  if (r = -a)
+    foo ();
+  else
+    bar ();
+
+  return r;
+}
+
+/* { dg-final { scan-assembler-not "testl" } } */
diff --git a/gcc/testsuite/gcc.target/i386/sibcall-8.c b/gcc/testsuite/gcc.target/i386/sibcall-8.c
index 3ab3809..29ebfe5 100644
--- a/gcc/testsuite/gcc.target/i386/sibcall-8.c
+++ b/gcc/testsuite/gcc.target/i386/sibcall-8.c
@@ -1,23 +1,29 @@
 /* { dg-do run } */
 /* { dg-options "-O2" } */
 
+#ifndef __x86_64__
+#define REGPARM __attribute__((regparm(1)))
+#else
+#define REGPARM
+#endif
+
 extern void abort (void);
 
-static int __attribute__((regparm(1)))
+static int REGPARM
 bar(void *arg)
 {
   return arg != bar;
 }
 
-static int __attribute__((noinline,noclone,regparm(1)))
-foo(int (__attribute__((regparm(1))) **bar)(void*))
+static int __attribute__((noinline,noclone)) REGPARM
+foo(int (REGPARM **bar)(void*))
 {
   return (*bar)(*bar);
 }
 
 int main()
 {
-  int (__attribute__((regparm(1))) *p)(void*) = bar;
+  int (REGPARM *p)(void*) = bar;
   if (foo(&p))
     abort();
   return 0;
diff --git a/gcc/testsuite/gcc.target/i386/sw-1.c b/gcc/testsuite/gcc.target/i386/sw-1.c
index 14db3ce..025f0e1 100644
--- a/gcc/testsuite/gcc.target/i386/sw-1.c
+++ b/gcc/testsuite/gcc.target/i386/sw-1.c
@@ -7,7 +7,10 @@
 
 int c;
 int x[2000];
-__attribute__((regparm(1))) void foo (int a, int b)
+#ifndef __x86_64__
+__attribute__((regparm(1)))
+#endif
+void foo (int a, int b)
  {
    int t[200];
    if (a == 0 || c == 0)
diff --git a/gcc/testsuite/gcc.target/i386/uintr-2.c b/gcc/testsuite/gcc.target/i386/uintr-2.c
index 0a83c66..a0d2514 100644
--- a/gcc/testsuite/gcc.target/i386/uintr-2.c
+++ b/gcc/testsuite/gcc.target/i386/uintr-2.c
@@ -15,6 +15,6 @@ foo (void *frame, uword_t uirrv)
 
 void
 __attribute__((interrupt))
-UINTR_hanlder (struct __uintr_frame *frame, uword_t uirrv)
+UINTR_handler (struct __uintr_frame *frame, uword_t uirrv)
 {
 }
diff --git a/gcc/testsuite/gcc.target/i386/uintr-5.c b/gcc/testsuite/gcc.target/i386/uintr-5.c
index 49cb2ec..7c7c12f 100644
--- a/gcc/testsuite/gcc.target/i386/uintr-5.c
+++ b/gcc/testsuite/gcc.target/i386/uintr-5.c
@@ -7,6 +7,6 @@
 typedef unsigned int uword_t __attribute__ ((mode (__word__)));
 
 void
-UINTR_hanlder (struct __uintr_frame *frame, uword_t uirrv)
+UINTR_handler (struct __uintr_frame *frame, uword_t uirrv)
 {
 }
diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-3.c b/gcc/testsuite/gcc.target/i386/vect-epilogues-3.c
index 0ee610f..e88ab30 100644
--- a/gcc/testsuite/gcc.target/i386/vect-epilogues-3.c
+++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-3.c
@@ -1,5 +1,5 @@
 /* { dg-do compile } */
-/* { dg-options "-O3 -mavx512bw -mtune=znver4 -fdump-tree-vect-optimized" } */
+/* { dg-options "-O3 -mavx512bw -mtune=znver4 --param vect-partial-vector-usage=0 -fdump-tree-vect-optimized" } */
 
 int test (signed char *data, int n)
 {
diff --git a/gcc/testsuite/gcc.target/i386/vect-mask-epilogue-1.c b/gcc/testsuite/gcc.target/i386/vect-mask-epilogue-1.c
new file mode 100644
index 0000000..55519aa
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vect-mask-epilogue-1.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=znver5 -fdump-tree-vect-optimized" } */
+
+void bar (double *a, double *b, double c, int n, int m)
+{
+  for (int j = 0; j < m; ++j)
+    for (int i = 0; i < n; ++i)
+      a[j*n + i] = b[j*n + i] + c;
+}
+
+/* { dg-final { scan-tree-dump "epilogue loop vectorized using masked 64 byte vectors" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/i386/vect-mask-epilogue-2.c b/gcc/testsuite/gcc.target/i386/vect-mask-epilogue-2.c
new file mode 100644
index 0000000..3dc28b3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/vect-mask-epilogue-2.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -march=znver5 -fdump-tree-vect-optimized" } */
+
+void foo (double *a, double b, double c, int n, int m)
+{
+  for (int j = 0; j < m; ++j)
+    for (int i = 0; i < n; ++i)
+      a[j*n + i] = a[j*n + i] * b + c;
+}
+
+/* We do not want to use a masked epilogue for the inner loop as the next
+   outer iteration will possibly immediately read from elements masked of
+   the previous inner loop epilogue and that never forwards.  */
+/* { dg-final { scan-tree-dump "epilogue loop vectorized using 32 byte vectors" "vect" } } */
diff --git a/gcc/testsuite/gcc.target/loongarch/pr121064.c b/gcc/testsuite/gcc.target/loongarch/pr121064.c
new file mode 100644
index 0000000..a466c7a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/loongarch/pr121064.c
@@ -0,0 +1,38 @@
+/* { dg-require-effective-target loongarch_sx_hw } */
+/* { dg-do run } */
+/* { dg-options "-march=loongarch64 -mfpu=64 -mlsx -O3" } */
+
+typedef __INT32_TYPE__ int32_t;
+typedef unsigned __INT32_TYPE__ uint32_t;
+
+__attribute__ ((noipa)) static int32_t
+long_filter_ehigh_3830_1 (int32_t *buffer, int length)
+{
+  int i, j;
+  int32_t dotprod = 0;
+  int32_t delay[4] = { 0 };
+  uint32_t coeffs[4] = { 0 };
+
+  for (i = 0; i < length; i++)
+    {
+      dotprod = 0;
+      for (j = 3; j >= 0; j--)
+        {
+          dotprod += delay[j] * coeffs[j];
+          coeffs[j] += ((delay[j] >> 31) | 1);
+        }
+      for (j = 3; j > 0; j--)
+        delay[j] = delay[j - 1];
+      delay[0] = buffer[i];
+    }
+
+  return dotprod;
+}
+
+int
+main ()
+{
+  int32_t buffer[] = { -1, 1 };
+  if (long_filter_ehigh_3830_1 (buffer, 2) != -1)
+    __builtin_trap ();
+}
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_100.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100.c
new file mode 100644
index 0000000..e759a11
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_100 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_100a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100a.c
new file mode 100644
index 0000000..153ed1e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_100a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_100f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100f.c
new file mode 100644
index 0000000..9bb9127
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_100f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_100f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_101.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101.c
new file mode 100644
index 0000000..06b3ceb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_101 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_101a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101a.c
new file mode 100644
index 0000000..0cca3f3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_101a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_101f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101f.c
new file mode 100644
index 0000000..9548be5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_101f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_101f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_103.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103.c
new file mode 100644
index 0000000..5731249
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_103 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_103a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103a.c
new file mode 100644
index 0000000..aea501e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_103a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_103f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103f.c
new file mode 100644
index 0000000..59d8987
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_103f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_103f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_120.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120.c
new file mode 100644
index 0000000..d28a671
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_120 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_120a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120a.c
new file mode 100644
index 0000000..613dd65
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_120a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_120f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120f.c
new file mode 100644
index 0000000..1b23350
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_120f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_120f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_121.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121.c
new file mode 100644
index 0000000..240332b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_121 -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_121a.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121a.c
new file mode 100644
index 0000000..1e7fb70
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121a.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_121a -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/nvptx/march-map=sm_121f.c b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121f.c
new file mode 100644
index 0000000..2cbec51
--- /dev/null
+++ b/gcc/testsuite/gcc.target/nvptx/march-map=sm_121f.c
@@ -0,0 +1,19 @@
+/* { dg-do assemble } */
+/* { dg-options {-march-map=sm_121f -mptx=_} } */
+/* { dg-additional-options -save-temps } */
+/* { dg-final { scan-assembler-times {(?n)^	\.version	7\.8$} 1 } } */
+/* { dg-final { scan-assembler-times {(?n)^	\.target	sm_89$} 1 } } */
+
+#if __PTX_ISA_VERSION_MAJOR__ != 7
+#error wrong value for __PTX_ISA_VERSION_MAJOR__
+#endif
+
+#if __PTX_ISA_VERSION_MINOR__ != 8
+#error wrong value for __PTX_ISA_VERSION_MINOR__
+#endif
+
+#if __PTX_SM__ != 890
+#error wrong value for __PTX_SM__
+#endif
+
+int dummy;
diff --git a/gcc/testsuite/gcc.target/powerpc/pr121007.c b/gcc/testsuite/gcc.target/powerpc/pr121007.c
new file mode 100644
index 0000000..9e6b1be
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/pr121007.c
@@ -0,0 +1,40 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mdejagnu-cpu=power9" } */
+
+typedef struct { int a; } A;
+unsigned char *a;
+char b;
+int c;
+void foo (vector char, vector char, vector char);
+
+void
+bar (long stride)
+{
+  vector char v0, v1, v2, v3, v5;
+  vector char r0 = __builtin_vec_vsx_ld (0, a);
+  vector char r2 = __builtin_vec_vsx_ld (2 * stride, a - 3);
+  vector char r3 = __builtin_vec_vsx_ld (3 * stride, a - 3);
+  vector char r4;
+  vector char r6 = __builtin_vec_vsx_ld (6 * stride, a - 3);
+  vector char r7 = __builtin_vec_vsx_ld (7 * stride, a - 3);
+  vector char r14, h, i, j;
+  if (b)
+    return;
+  v1 = __builtin_vec_vsx_ld (9 * stride, a);
+  v2 = __builtin_vec_vsx_ld (10 * stride, a - 3);
+  v3 = __builtin_vec_vsx_ld (11 * stride, a - 3);
+  r3 = __builtin_vec_mergeh (r3, v3);
+  v5 = __builtin_vec_mergel (r2, r6);
+  r14 = __builtin_vec_mergeh (r3, r7);
+  r4 = __builtin_vec_mergeh (v2, r14);
+  v0 = __builtin_vec_mergeh (r0, r4);
+  union { unsigned char a[16]; A b; } temp;
+  vector signed char k;
+  h = __builtin_vec_ld (0, temp.a);
+  i = __builtin_vec_splat (h, 1);
+  temp.b.a = c;
+  k = __builtin_vec_ld (0, (signed char *) temp.a);
+  j = __builtin_vec_and (i, (vector char) k);
+  foo (v1, v0, j);
+  foo (v1, v5, j);
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/vsx-builtin-7.c b/gcc/testsuite/gcc.target/powerpc/vsx-builtin-7.c
index 5095d50..312043b 100644
--- a/gcc/testsuite/gcc.target/powerpc/vsx-builtin-7.c
+++ b/gcc/testsuite/gcc.target/powerpc/vsx-builtin-7.c
@@ -1,8 +1,16 @@
 /* { dg-do compile { target { powerpc*-*-* } } } */
 /* { dg-skip-if "" { powerpc*-*-darwin* } } */
-/* { dg-options "-O2 -mdejagnu-cpu=power7 -fno-inline-functions" } */
+/* { dg-options "-O2 -mdejagnu-cpu=power7 -fno-inline-functions -fno-ipa-icf" } */
 /* { dg-require-effective-target powerpc_vsx } */
 
+/* PR testsuite/119382
+   Note: Added -fno-ipa-icf to disable Interprocedural Identical Code
+   Folding (ICF). Without this, insert_di_0_v2 is merged with insert_di_0
+   due to improved alias analysis introduced in commit r15-7961-gdc47161c1f32c3.
+   This results in the compiler replacing insert_di_0_v2 with a tail call to
+   insert_di_0, altering expected test behavior. Disabling ICF ensures correct
+   execution of the test.  */
+
 /* Test simple extract/insert/slat operations.  Make sure all types are
    supported with various options.  */
 
diff --git a/gcc/testsuite/gcc.target/pru/pragma-ctable_entry-2.c b/gcc/testsuite/gcc.target/pru/pragma-ctable_entry-2.c
new file mode 100644
index 0000000..a1c707d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/pru/pragma-ctable_entry-2.c
@@ -0,0 +1,22 @@
+/* Test for base addresses with bit 31 set (PR121124).  */
+
+/* { dg-do compile } */
+/* { dg-options "-O1" } */
+
+/* -O1 in the options is significant.  Without it LBCO/SBCO operations may
+   not be optimized to the respective instructions.  */
+
+
+#pragma ctable_entry 12 0x80beef00
+
+unsigned int
+test_ctable (unsigned int val1, unsigned int val2)
+{
+  ((volatile unsigned short int *)0x80beef00)[0] = val2;
+  ((volatile unsigned int *)0x80beef00)[val1] = val2;
+  return ((volatile unsigned int *)0x80beef00)[5];
+}
+
+/* { dg-final { scan-assembler "sbco\\tr15.b\[012\]?, 12, 0, 2" } } */
+/* { dg-final { scan-assembler "sbco\\tr15.b0, 12, r14, 4" } } */
+/* { dg-final { scan-assembler "lbco\\tr14.b0, 12, 20, 4" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/amo/zabha-zacas-atomic-cas.c b/gcc/testsuite/gcc.target/riscv/amo/zabha-zacas-atomic-cas.c
new file mode 100644
index 0000000..d3d84fd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/amo/zabha-zacas-atomic-cas.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* PR target/120995 ICE unrecognized subword atomic cas */
+/* { dg-options "-O" } */
+/* { dg-add-options riscv_zacas } */
+/* { dg-add-options riscv_zabha } */
+
+_Bool b;
+void atomic_bool_cmpxchg()
+{
+  __sync_bool_compare_and_swap(&b, 1, 0);
+}
diff --git a/gcc/testsuite/gcc.target/riscv/amo/zalrsc-rvwmo-amo-add-int.c b/gcc/testsuite/gcc.target/riscv/amo/zalrsc-rvwmo-amo-add-int.c
index 4cf617d..0dfe816 100644
--- a/gcc/testsuite/gcc.target/riscv/amo/zalrsc-rvwmo-amo-add-int.c
+++ b/gcc/testsuite/gcc.target/riscv/amo/zalrsc-rvwmo-amo-add-int.c
@@ -9,7 +9,7 @@
 
 /*
 ** atomic_add_fetch_int_relaxed:
-**	1:
+**...
 **	lr.w\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -23,7 +23,7 @@ void atomic_add_fetch_int_relaxed (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_acquire:
-**	1:
+**...
 **	lr.w.aq\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -37,7 +37,7 @@ void atomic_add_fetch_int_acquire (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_release:
-**	1:
+**...
 **	lr.w\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w.rl\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -51,7 +51,7 @@ void atomic_add_fetch_int_release (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_acq_rel:
-**	1:
+**...
 **	lr.w.aq\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w.rl\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -65,7 +65,7 @@ void atomic_add_fetch_int_acq_rel (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_seq_cst:
-**	1:
+**...
 **	lr.w.aqrl\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w.rl\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
diff --git a/gcc/testsuite/gcc.target/riscv/amo/zalrsc-ztso-amo-add-int.c b/gcc/testsuite/gcc.target/riscv/amo/zalrsc-ztso-amo-add-int.c
index 3fb16c0..658b040 100644
--- a/gcc/testsuite/gcc.target/riscv/amo/zalrsc-ztso-amo-add-int.c
+++ b/gcc/testsuite/gcc.target/riscv/amo/zalrsc-ztso-amo-add-int.c
@@ -9,7 +9,7 @@
 
 /*
 ** atomic_add_fetch_int_relaxed:
-**	1:
+**...
 **	lr.w\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -23,7 +23,7 @@ void atomic_add_fetch_int_relaxed (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_acquire:
-**	1:
+**...
 **	lr.w\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -37,7 +37,7 @@ void atomic_add_fetch_int_acquire (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_release:
-**	1:
+**...
 **	lr.w\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -51,7 +51,7 @@ void atomic_add_fetch_int_release (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_acq_rel:
-**	1:
+**...
 **	lr.w\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
@@ -65,7 +65,7 @@ void atomic_add_fetch_int_acq_rel (int* bar, int baz)
 
 /*
 ** atomic_add_fetch_int_seq_cst:
-**	1:
+**...
 **	lr.w.aqrl\t[atx][0-9]+, 0\(a0\)
 **	add\t[atx][0-9]+, [atx][0-9]+, a1
 **	sc.w.rl\t[atx][0-9]+, [atx][0-9]+, 0\(a0\)
diff --git a/gcc/testsuite/gcc.target/riscv/interrupt-conflict-mode.c b/gcc/testsuite/gcc.target/riscv/interrupt-conflict-mode.c
index 81ebf5f..15cc3ee 100644
--- a/gcc/testsuite/gcc.target/riscv/interrupt-conflict-mode.c
+++ b/gcc/testsuite/gcc.target/riscv/interrupt-conflict-mode.c
@@ -1,7 +1,7 @@
 /* Verify proper errors are generated for conflicted interrupt type.  */
 /* { dg-do compile } */
 /* { dg-options "" } */
-void __attribute__ ((interrupt ("user")))
+void __attribute__ ((interrupt ("supervisor")))
 foo(void);
 
 void __attribute__ ((interrupt ("machine")))
diff --git a/gcc/testsuite/gcc.target/riscv/interrupt-rnmi.c b/gcc/testsuite/gcc.target/riscv/interrupt-rnmi.c
new file mode 100644
index 0000000..f340108
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/interrupt-rnmi.c
@@ -0,0 +1,11 @@
+/* Verify the return instruction is mnret.  */
+/* { dg-do compile } */
+/* { dg-options "-march=rv32gc_smrnmi" { target { rv32 } } } */
+/* { dg-options "-march=rv64gc_smrnmi" { target { rv64 } } } */
+
+void __attribute__ ((interrupt ("rnmi")))
+foo (void)
+{
+}
+
+/* { dg-final { scan-assembler {\mmnret} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/interrupt-umode.c b/gcc/testsuite/gcc.target/riscv/interrupt-umode.c
deleted file mode 100644
index 042abf0..0000000
--- a/gcc/testsuite/gcc.target/riscv/interrupt-umode.c
+++ /dev/null
@@ -1,8 +0,0 @@
-/* Verify the return instruction is mret.  */
-/* { dg-do compile } */
-/* { dg-options "" } */
-void __attribute__ ((interrupt ("user")))
-foo (void)
-{
-}
-/* { dg-final { scan-assembler {\muret} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/mipscondmov.c b/gcc/testsuite/gcc.target/riscv/mipscondmov.c
new file mode 100644
index 0000000..5485133
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/mipscondmov.c
@@ -0,0 +1,29 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv32imafd_xmipscmov" { target { rv32 } } } */
+/* { dg-options "-march=rv64imafd_xmipscmov -mabi=lp64d" { target { rv64 } } } */
+/* { dg-skip-if "" { *-*-* } { "-O0" "-Og" } } */
+
+#define MYTEST(name, mytype) \
+mytype test1_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a == b) ? c : d; } \
+mytype test2_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a != b) ? c : d; } \
+mytype test3_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a > b) ? c : d; } \
+mytype test4_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a >= b) ? c : d; } \
+mytype test5_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a < b) ? c : d; } \
+mytype test6_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a <= b) ? c : d; } \
+mytype test7_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a == 1) ? c : d; } \
+mytype test8_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a != 1) ? c : d; } \
+mytype test9_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a > 1) ? c : d; } \
+mytype test10_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a >= 1) ? c : d; } \
+mytype test11_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a < 1) ? c : d; } \
+mytype test12_ ## name (mytype a, mytype b, mytype c, mytype d) { return (a <= 1) ? c : d; }
+
+MYTEST(1, long)
+MYTEST(2, unsigned long)
+MYTEST(3, int)
+MYTEST(4, unsigned int)
+MYTEST(5, short)
+MYTEST(6, unsigned short)
+MYTEST(7, signed char)
+MYTEST(8, unsigned char)
+
+/* { dg-final { scan-assembler-times "mips.ccmov" 96 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/pr118241-b.cc b/gcc/testsuite/gcc.target/riscv/pr118241-b.cc
new file mode 100644
index 0000000..b2cc73f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/pr118241-b.cc
@@ -0,0 +1,33 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=rv64imafdc_zba_zbb_zbs_zicbom_zicbop -mabi=lp64d" } */
+
+/* Reduced from libsanitizer::asan_allocator.  */
+
+enum a { c };
+class d;
+struct e {
+  long count;
+  void *batch[];
+};
+template <typename> class f {
+public:
+  void g() {
+    if (e *b = h->i())
+      for (; b->count;)
+        if (6 < b->count)
+          __builtin_prefetch(b->batch[6]);
+  }
+  d *h;
+};
+class d {
+public:
+  e *i();
+};
+struct j {
+  f<int> k;
+  j(a);
+  void l() { k.g(); }
+} a(c);
+void m() { a.l(); }
+
+/* { dg-final { scan-assembler-times "prefetch.r\t0\\(\[a-x0-9\]+\\)" 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg.h
index 4aeb637..2de7d7c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg.h
@@ -3,6 +3,11 @@
 
 #include <stdint.h>
 
+#if __riscv_xlen == 64
+typedef unsigned __int128 uint128_t;
+typedef signed __int128 int128_t;
+#endif
+
 #define DEF_AVG_0(NT, WT, NAME)                                 \
 __attribute__((noinline))                                       \
 void                                                            \
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i32.c
index 138124c..31d3b43 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i32.c
@@ -6,7 +6,7 @@
 #define NT int16_t
 #define WT int32_t
 
-DEF_AVG_1(NT, WT, avg_ceil)
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i64.c
index 30438c9..7f30b9e 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i16-from-i64.c
@@ -6,7 +6,7 @@
 #define NT int16_t
 #define WT int64_t
 
-DEF_AVG_1(NT, WT, avg_ceil)
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i32-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i32-from-i64.c
index 2e9cfa5..2e06d0a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i32-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i32-from-i64.c
@@ -6,7 +6,7 @@
 #define NT int32_t
 #define WT int64_t
 
-DEF_AVG_1(NT, WT, avg_ceil)
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i64-from-i128.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i64-from-i128.c
new file mode 100644
index 0000000..ca23066
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i64-from-i128.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d" } */
+
+#include "avg.h"
+
+#define NT int64_t
+#define WT int128_t
+
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
+
+/* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i16.c
index 2ebf294..dda84a6 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i16.c
@@ -6,7 +6,7 @@
 #define NT int8_t
 #define WT int16_t
 
-DEF_AVG_1(NT, WT, avg_ceil)
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i32.c
index 64fec913..dfd2bb3 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i32.c
@@ -6,7 +6,7 @@
 #define NT int8_t
 #define WT int32_t
 
-DEF_AVG_1(NT, WT, avg_ceil)
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i64.c
index a72642c..d1060cc 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-1-i8-from-i64.c
@@ -6,7 +6,7 @@
 #define NT int8_t
 #define WT int64_t
 
-DEF_AVG_1(NT, WT, avg_ceil)
+DEF_AVG_1_WRAP(NT, WT, avg_ceil)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*0} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i32.c
index 1fa080b..3d872a8 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i32.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i64.c
index deec763..eda9736 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i16-from-i64.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i32-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i32-from-i64.c
index fa72000..21cbb94 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i32-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i32-from-i64.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i64-from-i128.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i64-from-i128.c
new file mode 100644
index 0000000..ee5330c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i64-from-i128.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
+
+#include "avg.h"
+#include "avg_data.h"
+
+#define WT   int128_t
+#define NT   int64_t
+#define NAME avg_ceil
+
+DEF_AVG_1_WRAP(NT, WT, NAME)
+
+#define TEST_DATA                            TEST_AVG_DATA_WRAP(NT, NAME)
+#define TEST_RUN(NT, WT, NAME, a, b, out, n) RUN_AVG_1_WRAP(NT, WT, NAME, a, b, out, n)
+
+#include "avg_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i16.c
index 6865cf2..fd91b6f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i16.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i32.c
index 78620f4..38f4920 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i32.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i64.c
index b2c763c..f65ee15 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_ceil-run-1-i8-from-i64.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_data.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_data.h
index 12b464a..49103f3 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_data.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_data.h
@@ -169,8 +169,8 @@ int64_t TEST_AVG_DATA(int64_t, avg_floor)[][3][N] =
     },
     {
        9223372036854775806ull,  9223372036854775806ull,  9223372036854775806ull,  9223372036854775806ull,
-       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
                         -2ull,                   -2ull,                   -2ull,                   -2ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
       -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
     },
     {
@@ -345,8 +345,8 @@ int64_t TEST_AVG_DATA(int64_t, avg_ceil)[][3][N] =
     },
     {
        9223372036854775806ull,  9223372036854775806ull,  9223372036854775806ull,  9223372036854775806ull,
-       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
                         -2ull,                   -2ull,                   -2ull,                   -2ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
       -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
     },
     {
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i32.c
index 16ba967..fc7943c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i32.c
@@ -6,7 +6,7 @@
 #define NT int16_t
 #define WT int32_t
 
-DEF_AVG_0(NT, WT, avg_floor)
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i64.c
index b229b4b..e02e5df 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i16-from-i64.c
@@ -6,7 +6,7 @@
 #define NT int16_t
 #define WT int64_t
 
-DEF_AVG_0(NT, WT, avg_floor)
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i32-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i32-from-i64.c
index 5f946bb..e36e424 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i32-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i32-from-i64.c
@@ -6,7 +6,7 @@
 #define NT int32_t
 #define WT int64_t
 
-DEF_AVG_0(NT, WT, avg_floor)
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i64-from-i128.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i64-from-i128.c
new file mode 100644
index 0000000..3e2d97d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i64-from-i128.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d" } */
+
+#include "avg.h"
+
+#define NT int64_t
+#define WT int128_t
+
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
+
+/* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i16.c
index 5d9297a..cdbb299 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i16.c
@@ -6,7 +6,7 @@
 #define NT int8_t
 #define WT int16_t
 
-DEF_AVG_0(NT, WT, avg_floor)
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i32.c
index 5c5d4ea..53508b0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i32.c
@@ -6,7 +6,7 @@
 #define NT int8_t
 #define WT int32_t
 
-DEF_AVG_0(NT, WT, avg_floor)
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i64.c
index f297953..9a6d1a2 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-1-i8-from-i64.c
@@ -6,7 +6,7 @@
 #define NT int8_t
 #define WT int64_t
 
-DEF_AVG_0(NT, WT, avg_floor)
+DEF_AVG_0_WRAP(NT, WT, avg_floor)
 
 /* { dg-final { scan-assembler-times {csrwi\s*vxrm,\s*2} 1 } } */
 /* { dg-final { scan-assembler-times {vaadd.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i32.c
index 9d0dd61..92239a2 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i32.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i64.c
index 2736baa..5716c29 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i16-from-i64.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i32-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i32-from-i64.c
index 2334045..705e091 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i32-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i32-from-i64.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i64-from-i128.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i64-from-i128.c
new file mode 100644
index 0000000..91e9809
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i64-from-i128.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
+
+#include "avg.h"
+#include "avg_data.h"
+
+#define WT   int128_t
+#define NT   int64_t
+#define NAME avg_floor
+
+DEF_AVG_0_WRAP(NT, WT, NAME)
+
+#define TEST_DATA                            TEST_AVG_DATA_WRAP(NT, NAME)
+#define TEST_RUN(NT, WT, NAME, a, b, out, n) RUN_AVG_0_WRAP(NT, WT, NAME, a, b, out, n)
+
+#include "avg_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i16.c
index 8364748..abe5c5b 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i16.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i32.c
index 157c936..355b90f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i32.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i64.c
index 2db0d3c..a9ae96f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/avg_floor-run-1-i8-from-i64.c
@@ -1,5 +1,5 @@
 /* { dg-do run { target { riscv_v } } } */
-/* { dg-additional-options "-std=c99 -O3" } */
+/* { dg-additional-options "-std=c99 -O3 -Wno-pedantic" } */
 
 #include "avg.h"
 #include "avg_data.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_arith.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_arith.h
index f78bdc0..93c29f0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_arith.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_arith.h
@@ -385,6 +385,8 @@ vec_sat_u_sub_##T##_fmt_1 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = (x - y) & (-(T)(x >= y));                             \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_1_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_1(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_2(T)                                   \
 void __attribute__((noinline))                                       \
@@ -398,6 +400,8 @@ vec_sat_u_sub_##T##_fmt_2 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = (x - y) & (-(T)(x > y));                              \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_2_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_2(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_3(T)                                   \
 void __attribute__((noinline))                                       \
@@ -411,6 +415,8 @@ vec_sat_u_sub_##T##_fmt_3 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = x > y ? x - y : 0;                                    \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_3_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_3(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_4(T)                                   \
 void __attribute__((noinline))                                       \
@@ -424,6 +430,8 @@ vec_sat_u_sub_##T##_fmt_4 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = x >= y ? x - y : 0;                                   \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_4_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_4(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_5(T)                                   \
 void __attribute__((noinline))                                       \
@@ -437,6 +445,8 @@ vec_sat_u_sub_##T##_fmt_5 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = x < y ? 0 : x - y;                                    \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_5_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_5(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_6(T)                                   \
 void __attribute__((noinline))                                       \
@@ -450,6 +460,8 @@ vec_sat_u_sub_##T##_fmt_6 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = x <= y ? 0 : x - y;                                   \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_6_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_6(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_7(T)                                   \
 void __attribute__((noinline))                                       \
@@ -465,6 +477,8 @@ vec_sat_u_sub_##T##_fmt_7 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = ret & (T)(overflow - 1);                              \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_7_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_7(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_8(T)                                   \
 void __attribute__((noinline))                                       \
@@ -480,6 +494,8 @@ vec_sat_u_sub_##T##_fmt_8 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = ret & (T)-(!overflow);                                \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_8_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_8(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_9(T)                                   \
 void __attribute__((noinline))                                       \
@@ -495,6 +511,8 @@ vec_sat_u_sub_##T##_fmt_9 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = overflow ? 0 : ret;                                   \
     }                                                                \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_9_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_9(T)
 
 #define DEF_VEC_SAT_U_SUB_FMT_10(T)                                   \
 void __attribute__((noinline))                                        \
@@ -510,6 +528,42 @@ vec_sat_u_sub_##T##_fmt_10 (T *out, T *op_1, T *op_2, unsigned limit) \
       out[i] = !overflow ? ret : 0;                                   \
     }                                                                 \
 }
+#define DEF_VEC_SAT_U_SUB_FMT_10_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_10(T)
+
+#define DEF_VEC_SAT_U_SUB_FMT_11(T)                                   \
+void __attribute__((noinline))                                       \
+vec_sat_u_sub_##T##_fmt_11 (T *out, T *op_1, T *op_2, unsigned limit) \
+{                                                                    \
+  unsigned i;                                                        \
+  for (i = 0; i < limit; i++)                                        \
+    {                                                                \
+      T x = op_1[i];                                                 \
+      T y = op_2[i];                                                 \
+      T ret;                                                         \
+      T overflow = __builtin_sub_overflow (x, y, &ret);           \
+      out[i] = overflow ? 0 : ret;                                   \
+    }                                                                \
+}
+#define DEF_VEC_SAT_U_SUB_FMT_11_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_11(T)
+
+#define DEF_VEC_SAT_U_SUB_FMT_12(T)                                   \
+void __attribute__((noinline))                                        \
+vec_sat_u_sub_##T##_fmt_12 (T *out, T *op_1, T *op_2, unsigned limit) \
+{                                                                     \
+  unsigned i;                                                         \
+  for (i = 0; i < limit; i++)                                         \
+    {                                                                 \
+      T x = op_1[i];                                                  \
+      T y = op_2[i];                                                  \
+      T ret;                                                          \
+      T overflow = __builtin_sub_overflow (x, y, &ret);            \
+      out[i] = !overflow ? ret : 0;                                   \
+    }                                                                 \
+}
+#define DEF_VEC_SAT_U_SUB_FMT_12_WRAP(T) \
+  DEF_VEC_SAT_U_SUB_FMT_12(T)
 
 #define DEF_VEC_SAT_U_SUB_ZIP(T1, T2)                             \
 void __attribute__((noinline))                                    \
@@ -669,33 +723,63 @@ vec_sat_s_sub_##T##_fmt_4 (T *out, T *op_1, T *op_2, unsigned limit) \
 
 #define RUN_VEC_SAT_U_SUB_FMT_1(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_1(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_1_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_1(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_2(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_2(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_2_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_2(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_3(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_3(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_3_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_3(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_4(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_4(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_4_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_4(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_5(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_5(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_5_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_5(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_6(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_6(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_6_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_6(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_7(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_7(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_7_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_7(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_8(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_8(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_8_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_8(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_9(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_9(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_9_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_9(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_10(T, out, op_1, op_2, N) \
   vec_sat_u_sub_##T##_fmt_10(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_10_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_10(T, out, op_1, op_2, N)
+
+#define RUN_VEC_SAT_U_SUB_FMT_11(T, out, op_1, op_2, N) \
+  vec_sat_u_sub_##T##_fmt_11(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_11_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_11(T, out, op_1, op_2, N)
+
+#define RUN_VEC_SAT_U_SUB_FMT_12(T, out, op_1, op_2, N) \
+  vec_sat_u_sub_##T##_fmt_12(out, op_1, op_2, N)
+#define RUN_VEC_SAT_U_SUB_FMT_12_WRAP(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_12(T, out, op_1, op_2, N)
 
 #define RUN_VEC_SAT_U_SUB_FMT_ZIP(T1, T2, x, b, N) \
   vec_sat_u_sub_##T1##_##T2##_fmt_zip(x, b, N)
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_data.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_data.h
index 9f05c0c..7647439 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_data.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_data.h
@@ -744,6 +744,258 @@ uint64_t TEST_UNARY_DATA(uint64_t, sat_u_sub_imm)[][2][N] =
   },
 };
 
+uint8_t TEST_UNARY_DATA(uint8_t, ussub)[][3][N] = {
+  {
+    {
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+    }, /* arg_0 */
+    {
+      0, 1, 2, 3,
+      0, 1, 2, 3,
+      0, 1, 2, 3,
+      0, 1, 2, 3,
+    }, /* arg_1 */
+    {
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+    }, /* expect */
+  },
+  {
+    {
+	0, 255, 255, 255,
+	0, 255, 255, 255,
+	0, 255, 255, 255,
+	0, 255, 255, 255,
+    },
+    {
+	1, 255, 254, 251,
+	1, 255, 254, 251,
+	1, 255, 254, 251,
+	1, 255, 254, 251,
+    },
+    {
+	0,   0,   1,   4,
+	0,   0,   1,   4,
+	0,   0,   1,   4,
+	0,   0,   1,   4,
+    },
+  },
+  {
+    {
+	0,   0,   1,   0,
+	1,   2,   3,   0,
+	1,   2,   3, 255,
+	5, 254, 255,   9,
+    },
+    {
+	0,   1,   0, 254,
+      254, 254, 254, 255,
+      255, 255,   0, 252,
+      255, 255, 255,   1,
+    },
+    {
+	0,   0,   1,   0,
+	0,   0,   0,   0,
+	0,   0,   3,   3,
+	0,   0,   0,   8,
+    },
+  },
+};
+
+uint16_t TEST_UNARY_DATA(uint16_t, ussub)[][3][N] = {
+  {
+    {
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+    }, /* arg_0 */
+    {
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+    }, /* arg_1 */
+    {
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+      0, 0, 0, 0,
+    }, /* expect */
+  },
+  {
+    {
+      65535, 65535, 65535, 65535,
+      65535, 65535, 65535, 65535,
+      65535, 65535, 65535, 65535,
+      65535, 65535, 65535, 65535,
+    },
+    {
+      55535, 45535, 35535, 25535,
+      55535, 45535, 35535, 25535,
+      55535, 45535, 35535, 25535,
+      55535, 45535, 35535, 25535,
+    },
+    {
+      10000, 20000, 30000, 40000,
+      10000, 20000, 30000, 40000,
+      10000, 20000, 30000, 40000,
+      10000, 20000, 30000, 40000,
+    },
+  },
+  {
+    {
+	  0,     0,     1,     0,
+	  1,     2,     3,     0,
+	  1, 65535,     3, 65535,
+	  5, 65534, 65535,     9,
+    },
+    {
+	  0,     1,     1, 65534,
+      65534, 65534,     1, 65535,
+	  0, 65535, 65535,     0,
+      65535, 65535,     1,     2,
+    },
+    {
+	  0,     0,     0,     0,
+	  0,     0,     2,     0,
+	  1,     0,     0, 65535,
+	  0,     0, 65534,     7,
+    },
+  },
+};
+
+uint32_t TEST_UNARY_DATA(uint32_t, ussub)[][3][N] = {
+  {
+    {
+      0, 0, 4, 0,
+      0, 0, 4, 0,
+      0, 0, 4, 0,
+      0, 0, 4, 0,
+    }, /* arg_0 */
+    {
+      0, 1, 2, 3,
+      0, 1, 2, 3,
+      0, 1, 2, 3,
+      0, 1, 2, 3,
+    }, /* arg_1 */
+    {
+      0, 0, 2, 0,
+      0, 0, 2, 0,
+      0, 0, 2, 0,
+      0, 0, 2, 0,
+    }, /* expect */
+  },
+  {
+    {
+      4294967295, 4294967295, 4294967295, 4294967295,
+      4294967295, 4294967295, 4294967295, 4294967295,
+      4294967295, 4294967295, 4294967295, 4294967295,
+      4294967295, 4294967295, 4294967295, 4294967295,
+    },
+    {
+      1294967295, 2294967295, 3294967295, 4294967295,
+      1294967295, 2294967295, 3294967295, 4294967295,
+      1294967295, 2294967295, 3294967295, 4294967295,
+      1294967295, 2294967295, 3294967295, 4294967295,
+    },
+    {
+      3000000000, 2000000000, 1000000000,          0,
+      3000000000, 2000000000, 1000000000,          0,
+      3000000000, 2000000000, 1000000000,          0,
+      3000000000, 2000000000, 1000000000,          0,
+    },
+  },
+  {
+    {
+	       0,          0,          9,          0,
+	       1, 4294967295,          3,          0,
+	       1,          2,          3,          4,
+	       5, 4294967294, 4294967295, 4294967295,
+    },
+    {
+	       0,          1,          1, 4294967294,
+	       1,          2, 4294967294, 4294967295,
+	       1, 4294967295, 4294967295,          1,
+	       1, 4294967295, 4294967290,          9,
+    },
+    {
+	       0,          0,          8,          0,
+	       0, 4294967293,          0,          0,
+	       0,          0,          0,          3,
+	       4,          0,          5, 4294967286,
+    },
+  },
+};
+
+uint64_t TEST_UNARY_DATA(uint64_t, ussub)[][3][N] = {
+  {
+    {
+      0, 9, 0, 0,
+      0, 9, 0, 0,
+      0, 9, 0, 0,
+      0, 9, 0, 0,
+    }, /* arg_0 */
+    {
+      0, 2, 3, 1,
+      0, 2, 3, 1,
+      0, 2, 3, 1,
+      0, 2, 3, 1,
+    }, /* arg_1 */
+    {
+      0, 7, 0, 0,
+      0, 7, 0, 0,
+      0, 7, 0, 0,
+      0, 7, 0, 0,
+    }, /* expect */
+  },
+  {
+    {
+      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
+      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
+      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
+      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
+    },
+    {
+      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
+      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
+      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
+      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
+    },
+    {
+       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
+       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
+       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
+       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
+    },
+  },
+  {
+    {
+			  0, 18446744073709551615u,                     1,                     0,
+			  1, 18446744073709551615u,                     3,                     0,
+			  1, 18446744073709551614u,                     3,                     4,
+			  5, 18446744073709551614u, 18446744073709551615u,                     9,
+    },
+    {
+			  0,                     1,                     1, 18446744073709551614u,
+      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
+      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
+      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
+    },
+    {
+			  0, 18446744073709551614u,                     0,                     0,
+			  0,                     1,                     0,                     0,
+			  0,                     0,                     0,                     0,
+			  0,                     0,                     0,                     8,
+    },
+  },
+};
+
 int8_t TEST_UNARY_DATA(int8_t, sat_s_add_imm)[][2][N] =
 {
   { /* For add imm -128 */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u16.c
new file mode 100644
index 0000000..57da9e8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u16.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_11(uint16_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u32.c
new file mode 100644
index 0000000..b5264a3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u32.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_11(uint32_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u64.c
new file mode 100644
index 0000000..1a68b5c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u64.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_11(uint64_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u8.c
new file mode 100644
index 0000000..a1c5c19
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-11-u8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_11(uint8_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u16.c
new file mode 100644
index 0000000..fd987e9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u16.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_12(uint16_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u32.c
new file mode 100644
index 0000000..bc380fe
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u32.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_12(uint32_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u64.c
new file mode 100644
index 0000000..c03163f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u64.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_12(uint64_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u8.c
new file mode 100644
index 0000000..91e1909
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-12-u8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -ftree-vectorize -fdump-tree-optimized" } */
+
+#include "vec_sat_arith.h"
+
+DEF_VEC_SAT_U_SUB_FMT_12(uint8_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_SUB " 2 "optimized" } } */
+/* { dg-final { scan-assembler-times {vssubu\.vv} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u16.c
index 97e5040..5878c5b6 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_1
 
-DEF_VEC_SAT_U_SUB_FMT_1(T)
+DEF_VEC_SAT_U_SUB_FMT_1_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_1_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u32.c
index a5428c4..f74979f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_1
 
-DEF_VEC_SAT_U_SUB_FMT_1(T)
+DEF_VEC_SAT_U_SUB_FMT_1_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_1_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u64.c
index bdb65d9..1250e5b 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_1
 
-DEF_VEC_SAT_U_SUB_FMT_1(T)
+DEF_VEC_SAT_U_SUB_FMT_1_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_1_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u8.c
index 3fe5fe3..a2a77dd 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-1-u8.c
@@ -2,74 +2,15 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_1
 
-DEF_VEC_SAT_U_SUB_FMT_1(T)
+DEF_VEC_SAT_U_SUB_FMT_1_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_1_WRAP(T, out, op_1, op_2, N)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u16.c
index 0f4129c..19c8fa0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_10
 
-DEF_VEC_SAT_U_SUB_FMT_10(T)
+DEF_VEC_SAT_U_SUB_FMT_10_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_10_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u32.c
index 8b995eb..ada136f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_10
 
-DEF_VEC_SAT_U_SUB_FMT_10(T)
+DEF_VEC_SAT_U_SUB_FMT_10_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_10_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u64.c
index d12d981..488c158 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_10
 
-DEF_VEC_SAT_U_SUB_FMT_10(T)
+DEF_VEC_SAT_U_SUB_FMT_10_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_10_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u8.c
index 384ef3e..127c27a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-10-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_10
 
-DEF_VEC_SAT_U_SUB_FMT_10(T)
+DEF_VEC_SAT_U_SUB_FMT_10_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_10_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u16.c
new file mode 100644
index 0000000..4b49467
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u16.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint16_t
+
+DEF_VEC_SAT_U_SUB_FMT_11_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_11_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u32.c
new file mode 100644
index 0000000..80b55ea
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u32.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint32_t
+
+DEF_VEC_SAT_U_SUB_FMT_11_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_11_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u64.c
new file mode 100644
index 0000000..6a89d0f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u64.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint64_t
+
+DEF_VEC_SAT_U_SUB_FMT_11_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_11_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u8.c
new file mode 100644
index 0000000..974493e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-11-u8.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint8_t
+
+DEF_VEC_SAT_U_SUB_FMT_11_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_11_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u16.c
new file mode 100644
index 0000000..28778b9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u16.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint16_t
+
+DEF_VEC_SAT_U_SUB_FMT_12_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_12_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u32.c
new file mode 100644
index 0000000..936a39a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u32.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint32_t
+
+DEF_VEC_SAT_U_SUB_FMT_12_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_12_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u64.c
new file mode 100644
index 0000000..b8fa65b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u64.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint64_t
+
+DEF_VEC_SAT_U_SUB_FMT_12_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_12_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u8.c
new file mode 100644
index 0000000..6bff1e1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-12-u8.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "vec_sat_arith.h"
+#include "vec_sat_data.h"
+
+#define T                  uint8_t
+
+DEF_VEC_SAT_U_SUB_FMT_12_WRAP(T)
+
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_12_WRAP(T, out, op_1, op_2, N)
+
+#include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u16.c
index 5cf08ac..45bef88 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_2
 
-DEF_VEC_SAT_U_SUB_FMT_2(T)
+DEF_VEC_SAT_U_SUB_FMT_2_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_2_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u32.c
index 85c8454..6d8a653 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_2
 
-DEF_VEC_SAT_U_SUB_FMT_2(T)
+DEF_VEC_SAT_U_SUB_FMT_2_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_2_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u64.c
index 67d5ac5..0132d46 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_2
 
-DEF_VEC_SAT_U_SUB_FMT_2(T)
+DEF_VEC_SAT_U_SUB_FMT_2_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_2_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u8.c
index 809f07f..425f86f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-2-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_2
 
-DEF_VEC_SAT_U_SUB_FMT_2(T)
+DEF_VEC_SAT_U_SUB_FMT_2_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_2_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u16.c
index 57839a9..97a8e08 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_3
 
-DEF_VEC_SAT_U_SUB_FMT_3(T)
+DEF_VEC_SAT_U_SUB_FMT_3_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_3_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u32.c
index ffb0dcc..9124899 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_3
 
-DEF_VEC_SAT_U_SUB_FMT_3(T)
+DEF_VEC_SAT_U_SUB_FMT_3_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_3_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u64.c
index 3966677..1e54ede 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_3
 
-DEF_VEC_SAT_U_SUB_FMT_3(T)
+DEF_VEC_SAT_U_SUB_FMT_3_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_3_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u8.c
index e795f62..d8d53b7 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-3-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_3
 
-DEF_VEC_SAT_U_SUB_FMT_3(T)
+DEF_VEC_SAT_U_SUB_FMT_3_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_3_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u16.c
index 0eecf82..b293823 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_4
 
-DEF_VEC_SAT_U_SUB_FMT_4(T)
+DEF_VEC_SAT_U_SUB_FMT_4_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_4_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u32.c
index 1d0d16b..f0f1c4f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_4
 
-DEF_VEC_SAT_U_SUB_FMT_4(T)
+DEF_VEC_SAT_U_SUB_FMT_4_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_4_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u64.c
index 98fdfa2..27c28e2 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_4
 
-DEF_VEC_SAT_U_SUB_FMT_4(T)
+DEF_VEC_SAT_U_SUB_FMT_4_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_4_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u8.c
index 18a887d..7911825 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-4-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_4
 
-DEF_VEC_SAT_U_SUB_FMT_4(T)
+DEF_VEC_SAT_U_SUB_FMT_4_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_4_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u16.c
index ce44c04..6ae7b36 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_5
 
-DEF_VEC_SAT_U_SUB_FMT_5(T)
+DEF_VEC_SAT_U_SUB_FMT_5_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_5_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u32.c
index 36ae7b3..4e6b9e6 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_5
 
-DEF_VEC_SAT_U_SUB_FMT_5(T)
+DEF_VEC_SAT_U_SUB_FMT_5_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_5_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u64.c
index 7b40ffd..6b26913 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_5
 
-DEF_VEC_SAT_U_SUB_FMT_5(T)
+DEF_VEC_SAT_U_SUB_FMT_5_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_5_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u8.c
index 3b0807f..2bd28cd 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-5-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_5
 
-DEF_VEC_SAT_U_SUB_FMT_5(T)
+DEF_VEC_SAT_U_SUB_FMT_5_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_5_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u16.c
index e972078..69b0be9 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_6
 
-DEF_VEC_SAT_U_SUB_FMT_6(T)
+DEF_VEC_SAT_U_SUB_FMT_6_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_6_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u32.c
index 54e2848..2450586 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_6
 
-DEF_VEC_SAT_U_SUB_FMT_6(T)
+DEF_VEC_SAT_U_SUB_FMT_6_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_6_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u64.c
index 33f3be0..0b97910 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_6
 
-DEF_VEC_SAT_U_SUB_FMT_6(T)
+DEF_VEC_SAT_U_SUB_FMT_6_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_6_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u8.c
index 1376038..afb23f6 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-6-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_6
 
-DEF_VEC_SAT_U_SUB_FMT_6(T)
+DEF_VEC_SAT_U_SUB_FMT_6_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_6_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u16.c
index 83241ef..0466d4c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_7
 
-DEF_VEC_SAT_U_SUB_FMT_7(T)
+DEF_VEC_SAT_U_SUB_FMT_7_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_7_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u32.c
index f20bb21..14b8701 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_7
 
-DEF_VEC_SAT_U_SUB_FMT_7(T)
+DEF_VEC_SAT_U_SUB_FMT_7_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_7_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u64.c
index 4ad0afd..7e0afd8 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_7
 
-DEF_VEC_SAT_U_SUB_FMT_7(T)
+DEF_VEC_SAT_U_SUB_FMT_7_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_7_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u8.c
index 3b33b13..40b1a6a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-7-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_7
 
-DEF_VEC_SAT_U_SUB_FMT_7(T)
+DEF_VEC_SAT_U_SUB_FMT_7_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_7_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u16.c
index b212550..bd33048 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_8
 
-DEF_VEC_SAT_U_SUB_FMT_8(T)
+DEF_VEC_SAT_U_SUB_FMT_8_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_8_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u32.c
index 1fb707c..36f78f5 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_8
 
-DEF_VEC_SAT_U_SUB_FMT_8(T)
+DEF_VEC_SAT_U_SUB_FMT_8_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_8_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u64.c
index da8c09c..3bc5d5d 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_8
 
-DEF_VEC_SAT_U_SUB_FMT_8(T)
+DEF_VEC_SAT_U_SUB_FMT_8_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_8_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u8.c
index 647607f..3964d1b 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-8-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_8
 
-DEF_VEC_SAT_U_SUB_FMT_8(T)
+DEF_VEC_SAT_U_SUB_FMT_8_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_8_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u16.c
index 9bb0664..4c0809a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u16.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint16_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_9
 
-DEF_VEC_SAT_U_SUB_FMT_9(T)
+DEF_VEC_SAT_U_SUB_FMT_9_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-      65535, 65535, 65535, 65535,
-    },
-    {
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-      55535, 45535, 35535, 25535,
-    },
-    {
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-      10000, 20000, 30000, 40000,
-    },
-  },
-  {
-    {
-	  0,     0,     1,     0,
-	  1,     2,     3,     0,
-	  1, 65535,     3, 65535,
-	  5, 65534, 65535,     9,
-    },
-    {
-	  0,     1,     1, 65534,
-      65534, 65534,     1, 65535,
-	  0, 65535, 65535,     0,
-      65535, 65535,     1,     2,
-    },
-    {
-	  0,     0,     0,     0,
-	  0,     0,     2,     0,
-	  1,     0,     0, 65535,
-	  0,     0, 65534,     7,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_9_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u32.c
index f142b8b..3e700bd 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u32.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint32_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_9
 
-DEF_VEC_SAT_U_SUB_FMT_9(T)
+DEF_VEC_SAT_U_SUB_FMT_9_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-      0, 0, 4, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-      0, 0, 2, 0,
-    }, /* expect */
-  },
-  {
-    {
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-      4294967295, 4294967295, 4294967295, 4294967295,
-    },
-    {
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-      1294967295, 2294967295, 3294967295, 4294967295,
-    },
-    {
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-      3000000000, 2000000000, 1000000000,          0,
-    },
-  },
-  {
-    {
-	       0,          0,          9,          0,
-	       1, 4294967295,          3,          0,
-	       1,          2,          3,          4,
-	       5, 4294967294, 4294967295, 4294967295,
-    },
-    {
-	       0,          1,          1, 4294967294,
-	       1,          2, 4294967294, 4294967295,
-	       1, 4294967295, 4294967295,          1,
-	       1, 4294967295, 4294967290,          9,
-    },
-    {
-	       0,          0,          8,          0,
-	       0, 4294967293,          0,          0,
-	       0,          0,          0,          3,
-	       4,          0,          5, 4294967286,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_9_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u64.c
index 574b91a..81b8dc8 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u64.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint64_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_9
 
-DEF_VEC_SAT_U_SUB_FMT_9(T)
+DEF_VEC_SAT_U_SUB_FMT_9_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-      0, 9, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-      0, 2, 3, 1,
-    }, /* arg_1 */
-    {
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-      0, 7, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-    },
-    {
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-      10446744073709551615u, 11446744073709551615u, 12446744073709551615u, 18446744073709551615u,
-    },
-    {
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-       8000000000000000000u,  7000000000000000000u,  6000000000000000000u,                    0u,
-    },
-  },
-  {
-    {
-			  0, 18446744073709551615u,                     1,                     0,
-			  1, 18446744073709551615u,                     3,                     0,
-			  1, 18446744073709551614u,                     3,                     4,
-			  5, 18446744073709551614u, 18446744073709551615u,                     9,
-    },
-    {
-			  0,                     1,                     1, 18446744073709551614u,
-      18446744073709551614u, 18446744073709551614u, 18446744073709551614u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u, 18446744073709551615u,
-      18446744073709551615u, 18446744073709551615u, 18446744073709551615u,                     1,
-    },
-    {
-			  0, 18446744073709551614u,                     0,                     0,
-			  0,                     1,                     0,                     0,
-			  0,                     0,                     0,                     0,
-			  0,                     0,                     0,                     8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_9_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u8.c
index 2c8ee42..8bc52ae 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/sat/vec_sat_u_sub-run-9-u8.c
@@ -2,74 +2,14 @@
 /* { dg-additional-options "-std=c99" } */
 
 #include "vec_sat_arith.h"
+#include "vec_sat_data.h"
 
 #define T                  uint8_t
-#define N                  16
-#define RUN_VEC_SAT_BINARY RUN_VEC_SAT_U_SUB_FMT_9
 
-DEF_VEC_SAT_U_SUB_FMT_9(T)
+DEF_VEC_SAT_U_SUB_FMT_9_WRAP(T)
 
-T test_data[][3][N] = {
-  {
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* arg_0 */
-    {
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-      0, 1, 2, 3,
-    }, /* arg_1 */
-    {
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-      0, 0, 0, 0,
-    }, /* expect */
-  },
-  {
-    {
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-	0, 255, 255, 255,
-    },
-    {
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-	1, 255, 254, 251,
-    },
-    {
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-	0,   0,   1,   4,
-    },
-  },
-  {
-    {
-	0,   0,   1,   0,
-	1,   2,   3,   0,
-	1,   2,   3, 255,
-	5, 254, 255,   9,
-    },
-    {
-	0,   1,   0, 254,
-      254, 254, 254, 255,
-      255, 255,   0, 252,
-      255, 255, 255,   1,
-    },
-    {
-	0,   0,   1,   0,
-	0,   0,   0,   0,
-	0,   0,   3,   3,
-	0,   0,   0,   8,
-    },
-  },
-};
+#define test_data          TEST_UNARY_DATA_WRAP(T, ussub)
+#define RUN_VEC_SAT_BINARY(T, out, op_1, op_2, N) \
+  RUN_VEC_SAT_U_SUB_FMT_9_WRAP(T, out, op_1, op_2, N)
 
 #include "vec_sat_binary_vvv_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c
index 4dc5703..0fa1ea0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vls-vlmax/repeat-6.c
@@ -72,7 +72,7 @@ f_vnx128qi (int8_t *out)
   *(vnx128qi *) out = v;
 }
 
-/* { dg-final { scan-assembler-times {vmv.v.x\tv[0-9]+,\s*[a-x0-9]+} 6 } } */
+/* { dg-final { scan-assembler-times {vmv.v.x\tv[0-9]+,\s*[a-x0-9]+} 7 } } */
 /* { dg-final { scan-assembler-times {slli\t[a-x0-9]+,\s*[a-x0-9]+,\s*8} 6 } } */
 /* { dg-final { scan-assembler-times {or\t[a-x0-9]+,\s*[a-x0-9]+,\s*[a-x0-9]+} 6 } } */
 /* { dg-final { scan-assembler-times {vslide1down\.vx\tv[0-9]+,\s*v[0-9]+,\s*[a-x0-9]+} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f16.c
index 05cf57c..811f26c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f16.c
@@ -11,6 +11,10 @@ DEF_VF_MULOP_ACC_CASE_0 (_Float16, +, +, acc)
 DEF_VF_MULOP_ACC_CASE_0 (_Float16, -, +, sac)
 DEF_VF_MULOP_ACC_CASE_0 (_Float16, +, -, nacc)
 DEF_VF_MULOP_ACC_CASE_0 (_Float16, -, -, nsac)
+DEF_VF_MULOP_WIDEN_CASE_0 (_Float16, float, +, +, acc)
+DEF_VF_MULOP_WIDEN_CASE_0 (_Float16, float, -, +, sac)
+DEF_VF_MULOP_WIDEN_CASE_0 (_Float16, float, +, -, nacc)
+DEF_VF_MULOP_WIDEN_CASE_0 (_Float16, float, -, -, nsac)
 
 /* { dg-final { scan-assembler-times {vfmadd.vf} 1 } } */
 /* { dg-final { scan-assembler-times {vfmsub.vf} 1 } } */
@@ -20,3 +24,7 @@ DEF_VF_MULOP_ACC_CASE_0 (_Float16, -, -, nsac)
 /* { dg-final { scan-assembler-times {vfmsac.vf} 1 } } */
 /* { dg-final { scan-assembler-times {vfnmacc.vf} 1 } } */
 /* { dg-final { scan-assembler-times {vfnmsac.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwmacc.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwmsac.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwnmacc.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwnmsac.vf} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f32.c
index 873e315..ca82ead 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-1-f32.c
@@ -11,6 +11,10 @@ DEF_VF_MULOP_ACC_CASE_0 (float, +, +, acc)
 DEF_VF_MULOP_ACC_CASE_0 (float, -, +, sac)
 DEF_VF_MULOP_ACC_CASE_0 (float, +, -, nacc)
 DEF_VF_MULOP_ACC_CASE_0 (float, -, -, nsac)
+DEF_VF_MULOP_WIDEN_CASE_0 (float, double, +, +, acc)
+DEF_VF_MULOP_WIDEN_CASE_0 (float, double, -, +, sac)
+DEF_VF_MULOP_WIDEN_CASE_0 (float, double, +, -, nacc)
+DEF_VF_MULOP_WIDEN_CASE_0 (float, double, -, -, nsac)
 
 /* { dg-final { scan-assembler-times {vfmadd.vf} 1 } } */
 /* { dg-final { scan-assembler-times {vfmsub.vf} 1 } } */
@@ -20,3 +24,7 @@ DEF_VF_MULOP_ACC_CASE_0 (float, -, -, nsac)
 /* { dg-final { scan-assembler-times {vfmsac.vf} 1 } } */
 /* { dg-final { scan-assembler-times {vfnmacc.vf} 1 } } */
 /* { dg-final { scan-assembler-times {vfnmsac.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwmacc.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwmsac.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwnmacc.vf} 1 } } */
+/* { dg-final { scan-assembler-times {vfwnmsac.vf} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f16.c
index 78127b6..3a39303 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f16.c
@@ -11,3 +11,9 @@
 /* { dg-final { scan-assembler-not {vfmsac.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmacc.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmsac.vf} } } */
+/* { dg-final { scan-assembler-times {fcvt.s.h} 4 } } */
+/* { dg-final { scan-assembler-times {vfmv.v.f} 12 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f32.c
index 30d57e0..b4618bae 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-2-f32.c
@@ -11,3 +11,9 @@
 /* { dg-final { scan-assembler-not {vfmsac.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmacc.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmsac.vf} } } */
+/* { dg-final { scan-assembler-times {fcvt.d.s} 4 } } */
+/* { dg-final { scan-assembler-times {vfmv.v.f} 12 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f16.c
index 8295ffb..58afaa4 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f16.c
@@ -11,6 +11,10 @@ DEF_VF_MULOP_ACC_CASE_1 (_Float16, +, +, acc, VF_MULOP_ACC_BODY_X128)
 DEF_VF_MULOP_ACC_CASE_1 (_Float16, -, +, sac, VF_MULOP_ACC_BODY_X128)
 DEF_VF_MULOP_ACC_CASE_1 (_Float16, +, -, nacc, VF_MULOP_ACC_BODY_X128)
 DEF_VF_MULOP_ACC_CASE_1 (_Float16, -, -, nsac, VF_MULOP_ACC_BODY_X128)
+DEF_VF_MULOP_WIDEN_CASE_1 (_Float16, float, +, +, acc)
+DEF_VF_MULOP_WIDEN_CASE_1 (_Float16, float, -, +, sac)
+DEF_VF_MULOP_WIDEN_CASE_1 (_Float16, float, +, -, nacc)
+DEF_VF_MULOP_WIDEN_CASE_1 (_Float16, float, -, -, nsac)
 
 /* { dg-final { scan-assembler {vfmadd.vf} } } */
 /* { dg-final { scan-assembler {vfmsub.vf} } } */
@@ -20,3 +24,7 @@ DEF_VF_MULOP_ACC_CASE_1 (_Float16, -, -, nsac, VF_MULOP_ACC_BODY_X128)
 /* { dg-final { scan-assembler {vfmsac.vf} } } */
 /* { dg-final { scan-assembler {vfnmacc.vf} } } */
 /* { dg-final { scan-assembler {vfnmsac.vf} } } */
+/* { dg-final { scan-assembler {vfwmacc.vf} } } */
+/* { dg-final { scan-assembler {vfwmsac.vf} } } */
+/* { dg-final { scan-assembler {vfwnmacc.vf} } } */
+/* { dg-final { scan-assembler {vfwnmsac.vf} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f32.c
index f237f84..0e95774 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-3-f32.c
@@ -11,6 +11,10 @@ DEF_VF_MULOP_ACC_CASE_1 (float, +, +, acc, VF_MULOP_ACC_BODY_X128)
 DEF_VF_MULOP_ACC_CASE_1 (float, -, +, sac, VF_MULOP_ACC_BODY_X128)
 DEF_VF_MULOP_ACC_CASE_1 (float, +, -, nacc, VF_MULOP_ACC_BODY_X128)
 DEF_VF_MULOP_ACC_CASE_1 (float, -, -, nsac, VF_MULOP_ACC_BODY_X128)
+DEF_VF_MULOP_WIDEN_CASE_1 (float, double, +, +, acc)
+DEF_VF_MULOP_WIDEN_CASE_1 (float, double, -, +, sac)
+DEF_VF_MULOP_WIDEN_CASE_1 (float, double, +, -, nacc)
+DEF_VF_MULOP_WIDEN_CASE_1 (float, double, -, -, nsac)
 
 /* { dg-final { scan-assembler {vfmadd.vf} } } */
 /* { dg-final { scan-assembler {vfmsub.vf} } } */
@@ -20,3 +24,7 @@ DEF_VF_MULOP_ACC_CASE_1 (float, -, -, nsac, VF_MULOP_ACC_BODY_X128)
 /* { dg-final { scan-assembler {vfmsac.vf} } } */
 /* { dg-final { scan-assembler {vfnmacc.vf} } } */
 /* { dg-final { scan-assembler {vfnmsac.vf} } } */
+/* { dg-final { scan-assembler {vfwmacc.vf} } } */
+/* { dg-final { scan-assembler {vfwmsac.vf} } } */
+/* { dg-final { scan-assembler {vfwnmacc.vf} } } */
+/* { dg-final { scan-assembler {vfwnmsac.vf} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f16.c
index 7a50f67..559df6c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f16.c
@@ -11,3 +11,8 @@
 /* { dg-final { scan-assembler-not {vfmsac.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmacc.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmsac.vf} } } */
+/* { dg-final { scan-assembler {fcvt.s.h} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f32.c
index fb0493e..03f9c5a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf-4-f32.c
@@ -11,3 +11,8 @@
 /* { dg-final { scan-assembler-not {vfmsac.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmacc.vf} } } */
 /* { dg-final { scan-assembler-not {vfnmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwmsac.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmacc.vf} } } */
+/* { dg-final { scan-assembler-not {vfwnmsac.vf} } } */
+/* { dg-final { scan-assembler {fcvt.d.s} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop.h
index 1659f78..b1a324f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop.h
@@ -34,6 +34,21 @@
 #define RUN_VF_MULOP_ACC_CASE_0_WRAP(T, NAME, out, in, x, n)                   \
   RUN_VF_MULOP_ACC_CASE_0 (T, NAME, out, in, x, n)
 
+#define DEF_VF_MULOP_WIDEN_CASE_0(T1, T2, OP, NEG, NAME)                       \
+  void test_vf_mulop_widen_##NAME##_##T1##_case_0 (T2 *restrict out,           \
+						   T1 *restrict in,            \
+						   T1 *restrict f, unsigned n) \
+  {                                                                            \
+    for (unsigned i = 0; i < n; i++)                                           \
+      out[i] = NEG ((T2) * f * (T2) in[i] OP out[i]);                          \
+  }
+#define DEF_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, OP, NEG, NAME)                  \
+  DEF_VF_MULOP_WIDEN_CASE_0 (T1, T2, OP, NEG, NAME)
+#define RUN_VF_MULOP_WIDEN_CASE_0(T1, T2, NAME, out, in, x, n)                 \
+  test_vf_mulop_widen_##NAME##_##T1##_case_0 (out, in, x, n)
+#define RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, x, n)            \
+  RUN_VF_MULOP_WIDEN_CASE_0 (T1, T2, NAME, out, in, x, n)
+
 #define VF_MULOP_BODY(op, neg)                                                 \
   out[k + 0] = neg (tmp * out[k + 0] op in[k + 0]);                            \
   out[k + 1] = neg (tmp * out[k + 1] op in[k + 1]);                            \
@@ -129,4 +144,19 @@
 #define DEF_VF_MULOP_ACC_CASE_1_WRAP(T, OP, NEG, NAME, BODY)                   \
   DEF_VF_MULOP_ACC_CASE_1 (T, OP, NEG, NAME, BODY)
 
+#define DEF_VF_MULOP_WIDEN_CASE_1(TYPE1, TYPE2, OP, NEG, NAME)                 \
+  void test_vf_mulop_widen_##NAME##_##TYPE1##_##TYPE2##_case_1 (               \
+    TYPE2 *__restrict dst, TYPE2 *__restrict dst2, TYPE2 *__restrict dst3,     \
+    TYPE2 *__restrict dst4, TYPE1 *__restrict a, TYPE1 *__restrict b,          \
+    TYPE1 *__restrict a2, TYPE1 *__restrict b2, int n)                         \
+  {                                                                            \
+    for (int i = 0; i < n; i++)                                                \
+      {                                                                        \
+	dst[i] = NEG ((TYPE2) * a * (TYPE2) b[i] OP dst[i]);                   \
+	dst2[i] = NEG ((TYPE2) * a2 * (TYPE2) b[i] OP dst2[i]);                \
+	dst3[i] = NEG ((TYPE2) * a2 * (TYPE2) a[i] OP dst3[i]);                \
+	dst4[i] = NEG ((TYPE2) * a * (TYPE2) b2[i] OP dst4[i]);                \
+      }                                                                        \
+  }
+
 #endif
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_run.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_run.h
index 3dc3999..3dadfab 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_run.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_run.h
@@ -6,6 +6,10 @@
 #define TYPE_FABS(x, T)                                                        \
   (__builtin_types_compatible_p (T, double) ? fabs (x) : fabsf (x))
 
+#define MAX_RELATIVE_DIFF(T)                                                   \
+  (__builtin_types_compatible_p (T, _Float16) ? 0.1f  :                        \
+  (__builtin_types_compatible_p (T, float)    ? 0.01f : 0.01))
+
 int
 main ()
 {
@@ -23,7 +27,8 @@ main ()
       for (k = 0; k < N; k++)
 	{
 	  T diff = expect[k] - TEST_OUT[k];
-	  if (TYPE_FABS (diff, T) > .01 * TYPE_FABS (expect[k], T))
+	  if (TYPE_FABS (diff, T)
+	      > MAX_RELATIVE_DIFF (T) * TYPE_FABS (expect[k], T))
 	    __builtin_abort ();
 	}
     }
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_widen_run.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_widen_run.h
new file mode 100644
index 0000000..9f95fbb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_mulop_widen_run.h
@@ -0,0 +1,32 @@
+#ifndef HAVE_DEFINED_VF_MULOP_WIDEN_RUN_H
+#define HAVE_DEFINED_VF_MULOP_WIDEN_RUN_H
+
+#include <assert.h>
+
+#define N 512
+
+int main ()
+{
+  T1 f[N];
+  T1 in[N];
+  T2 out[N];
+  T2 out2[N];
+
+  for (int i = 0; i < N; i++) 
+    {
+      f[i] = LIMIT + i % 8723;
+      in[i] = LIMIT + i & 1964;
+      out[i] = LIMIT + i & 628;
+      out2[i] = LIMIT + i & 628;
+      asm volatile ("" ::: "memory");
+    }
+
+  TEST_RUN (T1, T2, NAME, out, in, f, N);
+
+  for (int i = 0; i < N; i++)
+    assert (out[i] == NEG(((T2) *f * (T2) in[i]) OP out2[i]));
+
+  return 0;
+}
+
+#endif
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c
index 5bb926d..fd8aa30 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmacc-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c
index be4dc1d..8fd8552 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmadd-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c
index 812e608..e91fd15 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsac-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c
index 48c2880..ca7e0db 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfmsub-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c
index b960e7a..b38e800 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmacc-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c
index 2be8917..fef5d77 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmadd-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c
index 4f10600..7951d40 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsac-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c
index 7bed0ed..d0def86 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfnmsub-run-1-f16.c
@@ -1,4 +1,8 @@
 /* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
 /* { dg-additional-options "--param=fpr2vr-cost=0" } */
 
 #include "vf_mulop.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c
new file mode 100644
index 0000000..d4c527a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f16.c
@@ -0,0 +1,21 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    _Float16
+#define T2    float
+#define NAME acc
+#define OP +
+#define NEG +
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -32768
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f32.c
new file mode 100644
index 0000000..1af5240
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmacc-run-1-f32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    float
+#define T2    double
+#define NAME acc
+#define OP +
+#define NEG +
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -2147483648
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c
new file mode 100644
index 0000000..abce2f2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f16.c
@@ -0,0 +1,21 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    _Float16
+#define T2    float
+#define NAME sac
+#define OP -
+#define NEG +
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -32768
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f32.c
new file mode 100644
index 0000000..13617a0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwmsac-run-1-f32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    float
+#define T2    double
+#define NAME sac
+#define OP -
+#define NEG +
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -2147483648
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c
new file mode 100644
index 0000000..ddf49d5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f16.c
@@ -0,0 +1,21 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    _Float16
+#define T2    float
+#define NAME nacc
+#define OP +
+#define NEG -
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -32768
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f32.c
new file mode 100644
index 0000000..851c335
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmacc-run-1-f32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    float
+#define T2    double
+#define NAME nacc
+#define OP +
+#define NEG -
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -2147483648
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c
new file mode 100644
index 0000000..a874991
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f16.c
@@ -0,0 +1,21 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-require-effective-target riscv_zvfh } */
+/* { dg-add-options "riscv_v" } */
+/* { dg-add-options "riscv_zvfh" } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    _Float16
+#define T2    float
+#define NAME nsac
+#define OP -
+#define NEG -
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -32768
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f32.c
new file mode 100644
index 0000000..9eacace
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vf_vfwnmsac-run-1-f32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "--param=fpr2vr-cost=0" } */
+
+#include "vf_mulop.h"
+
+#define T1    float
+#define T2    double
+#define NAME nsac
+#define OP -
+#define NEG -
+
+DEF_VF_MULOP_WIDEN_CASE_0_WRAP (T1, T2, OP, NEG, NAME)
+
+#define TEST_RUN(T1, T2, NAME, out, in, f, n) RUN_VF_MULOP_WIDEN_CASE_0_WRAP(T1, T2, NAME, out, in, f, n)
+#define LIMIT -2147483648
+
+#include "vf_mulop_widen_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c
index c86d77c..4e1a575 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i16.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-times {vrem.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vmax.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vmin.vx} 2 } } */
+/* { dg-final { scan-assembler-times {vsadd.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vssub.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 2 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c
index f6524cb..4c4f72d 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i32.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-times {vrem.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vmax.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vmin.vx} 2 } } */
+/* { dg-final { scan-assembler-times {vsadd.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vssub.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 2 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c
index f1e8627..abf62c2 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i64.c
@@ -18,3 +18,9 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-times {vrem.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vmax.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vmin.vx} 2 } } */
+/* { dg-final { scan-assembler-times {vsadd.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vssub.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 2 { target { no-opts
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c
index 9b0cbd2..7744bcb 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-i8.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-times {vrem.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vmax.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vmin.vx} 2 } } */
+/* { dg-final { scan-assembler-times {vsadd.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vssub.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 2 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u16.c
index b064748..cb62e0f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u16.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-times {vminu.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vsaddu.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vssubu.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u32.c
index e334bb3..e2a5dbb 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u32.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-times {vminu.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vsaddu.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vssubu.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c
index 3e8ca05..8e7a788 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u64.c
@@ -19,3 +19,7 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-times {vminu.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vsaddu.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vssubu.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 2 { target { no-opts {
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c
index 1f995cd..d213c18 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-1-u8.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-times {vminu.vx} 2 } } */
 /* { dg-final { scan-assembler-times {vsaddu.vx} 1 } } */
 /* { dg-final { scan-assembler-times {vssubu.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 2 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c
index fb1154c..05801a9 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i16.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c
index d4baa4b..f05f091 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i32.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c
index 18c1a78..adf9ccb 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i64.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c
index 5ce3c88..8b3f5bc 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-i8.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u16.c
index f7fae37..365e650 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u16.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u32.c
index b111a4e..c8fd42a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u32.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u64.c
index 4640d16..bdb76b4 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u64.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u8.c
index 58341ad..fc9c101 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-2-u8.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c
index 2965924..741f431 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i16.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c
index e7815e9..1741c22 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i32.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c
index 063a7a1..d326357 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i64.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c
index 0efb60c..3137dc0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-i8.c
@@ -18,3 +18,6 @@ TEST_BINARY_VX_SIGNED_0(T)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u16.c
index 406b999..121daeb 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u16.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u32.c
index 6792b6b..9616e7f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u32.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u64.c
index 692a709..cf985f0 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u64.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u8.c
index 4e30498..3bb382d 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-3-u8.c
@@ -19,3 +19,4 @@ TEST_BINARY_VX_UNSIGNED_0(T)
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c
index 199f8a7..2ae4804 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i16.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,11 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=scalable -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c
index 392f4fe..88cfc72 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i32.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c
index d22c387..6b29a72 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i64.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,9 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { no-opts {
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c
index 9a832a2..f862eb7 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-i8.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c
index afb5a85..3ecfce6 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u16.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c
index a907e9b..7ce1fe8 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u32.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,4 +32,6 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vremu.vx} } } */
 /* { dg-final { scan-assembler {vmaxu.vx} } } */
 /* { dg-final { scan-assembler {vminu.vx} } } */
+/* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c
index efabf99..c84a30c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u64.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,7 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} { target { no-opts {
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-O3 -mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c
index 7b2b088..9f3d7df 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-4-u8.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c
index f15fec5..df6872c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i16.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler-not {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c
index 8d21c47..05ed639 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i32.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { no-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c
index 0660000..6776b1f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i64.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,6 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler-not {vaadd.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c
index ce33461..d3e2785 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-i8.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler-not {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { no-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c
index da1b1be..5497b5a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u16.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c
index b7ec6c9..3a8e85f 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u32.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c
index dce78b1..060d591 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u64.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c
index c5c6fb8..86a6c45 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-5-u8.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c
index 3c9afdd..0bfa2cb 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i16.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler-not {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c
index b80a6b3..3e3acfc 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i32.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c
index 15bfe60..531c119 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i64.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
 
 /* { dg-final { scan-assembler-not {vadd.vx} } } */
 /* { dg-final { scan-assembler-not {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
 /* { dg-final { scan-assembler-not {vrem.vx} } } */
 /* { dg-final { scan-assembler-not {vmax.vx} } } */
 /* { dg-final { scan-assembler-not {vmin.vx} } } */
+/* { dg-final { scan-assembler-not {vsadd.vx} } } */
+/* { dg-final { scan-assembler-not {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c
index 4d529fe..43246bb 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-i8.c
@@ -18,6 +18,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_0_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MAX_FUNC_1_WARP(T), max, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, SAT_S_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler-not {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -30,3 +34,10 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 /* { dg-final { scan-assembler {vrem.vx} } } */
 /* { dg-final { scan-assembler {vmax.vx} } } */
 /* { dg-final { scan-assembler {vmin.vx} } } */
+/* { dg-final { scan-assembler {vsadd.vx} } } */
+/* { dg-final { scan-assembler {vssub.vx} } } */
+/* { dg-final { scan-assembler {vaadd.vx} { target { any-opts {
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m1"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m2"
+     "-mrvv-vector-bits=zvl -mrvv-max-lmul=m4"
+   } } } } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c
index 5952a7c..f51e7a1 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u16.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c
index 5bbc585..79b7477 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u32.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X4)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X4)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X4)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c
index 255ae62..ac5fd69 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u64.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY)
 
 /* { dg-final { scan-assembler-not {vadd.vx} } } */
 /* { dg-final { scan-assembler-not {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler-not {vminu.vx} } } */
 /* { dg-final { scan-assembler-not {vsaddu.vx} } } */
 /* { dg-final { scan-assembler-not {vssubu.vx} } } */
+/* { dg-final { scan-assembler-not {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c
index 63cd449..84aa06b 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-6-u8.c
@@ -19,6 +19,8 @@ DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_0_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, MIN_FUNC_1_WARP(T), min, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_ADD_FUNC_WRAP(T), sat_add, VX_BINARY_FUNC_BODY_X8)
 DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor, VX_BINARY_FUNC_BODY_X8)
+DEF_VX_BINARY_CASE_3_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil, VX_BINARY_FUNC_BODY_X8)
 
 /* { dg-final { scan-assembler {vadd.vx} } } */
 /* { dg-final { scan-assembler {vsub.vx} } } */
@@ -32,3 +34,4 @@ DEF_VX_BINARY_CASE_3_WRAP(T, SAT_U_SUB_FUNC_WRAP(T), sat_sub, VX_BINARY_FUNC_BOD
 /* { dg-final { scan-assembler {vminu.vx} } } */
 /* { dg-final { scan-assembler {vsaddu.vx} } } */
 /* { dg-final { scan-assembler {vssubu.vx} } } */
+/* { dg-final { scan-assembler {vaaddu.vx} } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i16.c
new file mode 100644
index 0000000..2b87321
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i16.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        8
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vint16m1_t
+#define T         int16_t
+#define ELEM_SIZE 16
+#define SUFFIX    i16
+#define FUNC      __riscv_vaadd_vv_i16m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i32.c
new file mode 100644
index 0000000..b95699b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i32.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        4
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vint32m1_t
+#define T         int32_t
+#define ELEM_SIZE 32
+#define SUFFIX    i32
+#define FUNC      __riscv_vaadd_vv_i32m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i64.c
new file mode 100644
index 0000000..48b6010
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i64.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        2
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vint64m1_t
+#define T         int64_t
+#define ELEM_SIZE 64
+#define SUFFIX    i64
+#define FUNC      __riscv_vaadd_vv_i64m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i8.c
new file mode 100644
index 0000000..d07a625
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-i8.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        16
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vint8m1_t
+#define T         int8_t
+#define ELEM_SIZE 8
+#define SUFFIX    i8
+#define FUNC      __riscv_vaadd_vv_i8m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaadd.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u16.c
new file mode 100644
index 0000000..bd36429
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u16.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        8
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vuint16m1_t
+#define T         uint16_t
+#define ELEM_SIZE 16
+#define SUFFIX    u16
+#define FUNC      __riscv_vaaddu_vv_u16m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u32.c
new file mode 100644
index 0000000..f023a76
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u32.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        4
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vuint32m1_t
+#define T         uint32_t
+#define ELEM_SIZE 32
+#define SUFFIX    u32
+#define FUNC      __riscv_vaaddu_vv_u32m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u64.c
new file mode 100644
index 0000000..d9a37ae
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u64.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        2
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vuint64m1_t
+#define T         uint64_t
+#define ELEM_SIZE 64
+#define SUFFIX    u64
+#define FUNC      __riscv_vaaddu_vv_u64m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u8.c
new file mode 100644
index 0000000..328e5d4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm-1-u8.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -mrvv-vector-bits=zvl --param=gpr2vr-cost=0 " } */
+
+#define VL        16
+
+#include "vx-fixed-vxrm.h"
+
+#define VT        vuint8m1_t
+#define T         uint8_t
+#define ELEM_SIZE 8
+#define SUFFIX    u8
+#define FUNC      __riscv_vaaddu_vv_u8m1
+
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNU, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RNE, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_RDN, FUNC)
+DEF_FIXED_BINARY_VX_WRAP(VT, T, ELEM_SIZE, SUFFIX, __RISCV_VXRM_ROD, FUNC)
+
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,0} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,1} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,2} 1 } } */
+/* { dg-final { scan-assembler-times {csrwi\s+vxrm,3} 1 } } */
+/* { dg-final { scan-assembler-times {vaaddu.vx} 4 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm.h
new file mode 100644
index 0000000..438c7ab
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx-fixed-vxrm.h
@@ -0,0 +1,28 @@
+#ifndef HAVE_DEFINED_VX_FIXED_VXRM_H
+#define HAVE_DEFINED_VX_FIXED_VXRM_H
+
+#include <riscv_vector.h>
+
+int64_t go[VL] = {};
+int64_t ga[VL] = {};
+
+#define DEF_FIXED_BINARY_VX(VT, T, ES, SX, VXRM, FUNC)       \
+void __attribute__((noinline))                               \
+test_fixed_binary_##VT##_##VXRM##_##FUNC##_vx () {           \
+  VT a = __riscv_vle##ES##_v_##SX##m1((T *)ga, VL);          \
+  VT b;                                                      \
+  T *bp = (T *)&b;                                           \
+                                                             \
+  for (int i = 0; i < VL; i++) {                             \
+    bp[i] = 123;                                             \
+  }                                                          \
+                                                             \
+  VT d = FUNC (a, b, VXRM, VL);                              \
+                                                             \
+  __riscv_vse##ES##_v_##SX##m1((T *)&go, d, VL);             \
+}
+
+#define DEF_FIXED_BINARY_VX_WRAP(VT, T, ES, SX, VXRM, FUNC) \
+  DEF_FIXED_BINARY_VX(VT, T, ES, SX, VXRM, FUNC)
+
+#endif
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h
index f12d1d1..4a9daff 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary.h
@@ -3,6 +3,14 @@
 
 #include <stdint.h>
 
+#undef HAS_INT128
+
+#if __riscv_xlen == 64
+#define HAS_INT128
+typedef unsigned __int128 uint128_t;
+typedef signed __int128 int128_t;
+#endif
+
 #define DEF_VX_BINARY_CASE_0(T, OP, NAME)                                \
 void                                                                     \
 test_vx_binary_##NAME##_##T##_case_0 (T * restrict out, T * restrict in, \
@@ -294,41 +302,131 @@ DEF_SAT_U_SUB(uint16_t)
 DEF_SAT_U_SUB(uint32_t)
 DEF_SAT_U_SUB(uint64_t)
 
+#define DEF_SAT_S_ADD(T, UT, MIN, MAX) \
+T                                      \
+test_##T##_sat_add (T x, T y)          \
+{                                      \
+  T sum = (UT)x + (UT)y;               \
+  return (x ^ y) < 0                   \
+    ? sum                              \
+    : (sum ^ x) >= 0                   \
+      ? sum                            \
+      : x < 0 ? MIN : MAX;             \
+}
+
+DEF_SAT_S_ADD(int8_t, uint8_t, INT8_MIN, INT8_MAX)
+DEF_SAT_S_ADD(int16_t, uint16_t, INT16_MIN, INT16_MAX)
+DEF_SAT_S_ADD(int32_t, uint32_t, INT32_MIN, INT32_MAX)
+DEF_SAT_S_ADD(int64_t, uint64_t, INT64_MIN, INT64_MAX)
+
+#define DEF_SAT_S_SUB(T, UT, MIN, MAX) \
+T                                      \
+test_##T##_sat_sub (T x, T y)          \
+{                                      \
+  T minus = (UT)x - (UT)y;             \
+  return (x ^ y) >= 0                  \
+    ? minus                            \
+    : (minus ^ x) >= 0                 \
+      ? minus                          \
+      : x < 0 ? MIN : MAX;             \
+}
+
+DEF_SAT_S_SUB(int8_t, uint8_t, INT8_MIN, INT8_MAX)
+DEF_SAT_S_SUB(int16_t, uint16_t, INT16_MIN, INT16_MAX)
+DEF_SAT_S_SUB(int32_t, uint32_t, INT32_MIN, INT32_MAX)
+DEF_SAT_S_SUB(int64_t, uint64_t, INT64_MIN, INT64_MAX)
+
 #define SAT_U_ADD_FUNC(T) test_##T##_sat_add
 #define SAT_U_ADD_FUNC_WRAP(T) SAT_U_ADD_FUNC(T)
 
 #define SAT_U_SUB_FUNC(T) test_##T##_sat_sub
 #define SAT_U_SUB_FUNC_WRAP(T) SAT_U_SUB_FUNC(T)
 
-#define TEST_BINARY_VX_SIGNED_0(T)                      \
-  DEF_VX_BINARY_CASE_0_WRAP(T, +, add)                  \
-  DEF_VX_BINARY_CASE_0_WRAP(T, -, sub)                  \
-  DEF_VX_BINARY_REVERSE_CASE_0_WRAP(T, -, rsub)         \
-  DEF_VX_BINARY_CASE_0_WRAP(T, &, and)                  \
-  DEF_VX_BINARY_CASE_0_WRAP(T, |, or)                   \
-  DEF_VX_BINARY_CASE_0_WRAP(T, ^, xor)                  \
-  DEF_VX_BINARY_CASE_0_WRAP(T, *, mul)                  \
-  DEF_VX_BINARY_CASE_0_WRAP(T, /, div)                  \
-  DEF_VX_BINARY_CASE_0_WRAP(T, %, rem)                  \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_0_WARP(T), max) \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_1_WARP(T), max) \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_0_WARP(T), min) \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_1_WARP(T), min)
-
-#define TEST_BINARY_VX_UNSIGNED_0(T)                       \
-  DEF_VX_BINARY_CASE_0_WRAP(T, +, add)                     \
-  DEF_VX_BINARY_CASE_0_WRAP(T, -, sub)                     \
-  DEF_VX_BINARY_REVERSE_CASE_0_WRAP(T, -, rsub)            \
-  DEF_VX_BINARY_CASE_0_WRAP(T, &, and)                     \
-  DEF_VX_BINARY_CASE_0_WRAP(T, |, or)                      \
-  DEF_VX_BINARY_CASE_0_WRAP(T, ^, xor)                     \
-  DEF_VX_BINARY_CASE_0_WRAP(T, /, div)                     \
-  DEF_VX_BINARY_CASE_0_WRAP(T, %, rem)                     \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_0_WARP(T), max)    \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_1_WARP(T), max)    \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_0_WARP(T), min)    \
-  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_1_WARP(T), min)    \
-  DEF_VX_BINARY_CASE_2_WRAP(T, SAT_U_ADD_FUNC(T), sat_add) \
-  DEF_VX_BINARY_CASE_2_WRAP(T, SAT_U_SUB_FUNC(T), sat_sub) \
+#define SAT_S_ADD_FUNC(T) test_##T##_sat_add
+#define SAT_S_ADD_FUNC_WRAP(T) SAT_S_ADD_FUNC(T)
+
+#define SAT_S_SUB_FUNC(T) test_##T##_sat_sub
+#define SAT_S_SUB_FUNC_WRAP(T) SAT_S_SUB_FUNC(T)
+
+#define DEF_AVG_FLOOR(NT, WT)        \
+NT                                   \
+test_##NT##_avg_floor(NT x, NT y)    \
+{                                    \
+  return (NT)(((WT)x + (WT)y) >> 1); \
+}
+
+DEF_AVG_FLOOR(uint8_t, uint16_t)
+DEF_AVG_FLOOR(uint16_t, uint32_t)
+DEF_AVG_FLOOR(uint32_t, uint64_t)
+
+DEF_AVG_FLOOR(int8_t, int16_t)
+DEF_AVG_FLOOR(int16_t, int32_t)
+DEF_AVG_FLOOR(int32_t, int64_t)
+
+#define DEF_AVG_CEIL(NT, WT)             \
+NT                                       \
+test_##NT##_avg_ceil(NT x, NT y)         \
+{                                        \
+  return (NT)(((WT)x + (WT)y + 1) >> 1); \
+}
+
+DEF_AVG_CEIL(uint8_t, uint16_t)
+DEF_AVG_CEIL(uint16_t, uint32_t)
+DEF_AVG_CEIL(uint32_t, uint64_t)
+
+DEF_AVG_CEIL(int8_t, int16_t)
+DEF_AVG_CEIL(int16_t, int32_t)
+DEF_AVG_CEIL(int32_t, int64_t)
+
+#ifdef HAS_INT128
+  DEF_AVG_FLOOR(uint64_t, uint128_t)
+  DEF_AVG_FLOOR(int64_t, int128_t)
+
+  DEF_AVG_CEIL(uint64_t, uint128_t)
+  DEF_AVG_CEIL(int64_t, int128_t)
+#endif
+
+#define AVG_FLOOR_FUNC(T)      test_##T##_avg_floor
+#define AVG_FLOOR_FUNC_WRAP(T) AVG_FLOOR_FUNC(T)
+
+#define AVG_CEIL_FUNC(T)      test_##T##_avg_ceil
+#define AVG_CEIL_FUNC_WRAP(T) AVG_CEIL_FUNC(T)
+
+#define TEST_BINARY_VX_SIGNED_0(T)                                \
+  DEF_VX_BINARY_CASE_0_WRAP(T, +, add)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, -, sub)                            \
+  DEF_VX_BINARY_REVERSE_CASE_0_WRAP(T, -, rsub)                   \
+  DEF_VX_BINARY_CASE_0_WRAP(T, &, and)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, |, or)                             \
+  DEF_VX_BINARY_CASE_0_WRAP(T, ^, xor)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, *, mul)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, /, div)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, %, rem)                            \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_0_WARP(T), max)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_1_WARP(T), max)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_0_WARP(T), min)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_1_WARP(T), min)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, SAT_S_ADD_FUNC(T), sat_add)        \
+  DEF_VX_BINARY_CASE_2_WRAP(T, SAT_S_SUB_FUNC(T), sat_sub)        \
+  DEF_VX_BINARY_CASE_2_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor) \
+  DEF_VX_BINARY_CASE_2_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil)   \
+
+#define TEST_BINARY_VX_UNSIGNED_0(T)                              \
+  DEF_VX_BINARY_CASE_0_WRAP(T, +, add)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, -, sub)                            \
+  DEF_VX_BINARY_REVERSE_CASE_0_WRAP(T, -, rsub)                   \
+  DEF_VX_BINARY_CASE_0_WRAP(T, &, and)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, |, or)                             \
+  DEF_VX_BINARY_CASE_0_WRAP(T, ^, xor)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, /, div)                            \
+  DEF_VX_BINARY_CASE_0_WRAP(T, %, rem)                            \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_0_WARP(T), max)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MAX_FUNC_1_WARP(T), max)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_0_WARP(T), min)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, MIN_FUNC_1_WARP(T), min)           \
+  DEF_VX_BINARY_CASE_2_WRAP(T, SAT_U_ADD_FUNC(T), sat_add)        \
+  DEF_VX_BINARY_CASE_2_WRAP(T, SAT_U_SUB_FUNC(T), sat_sub)        \
+  DEF_VX_BINARY_CASE_2_WRAP(T, AVG_FLOOR_FUNC_WRAP(T), avg_floor) \
+  DEF_VX_BINARY_CASE_2_WRAP(T, AVG_CEIL_FUNC_WRAP(T), avg_ceil)   \
 
 #endif
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h
index f475e36..626347c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_binary_data.h
@@ -4514,4 +4514,1180 @@ uint64_t TEST_BINARY_DATA(uint64_t, sat_sub)[][3][N] =
   },
 };
 
+int8_t TEST_BINARY_DATA(int8_t, sat_add)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+  },
+  {
+    { 127 },
+    {
+       127,  127,  127,  127,
+      -128, -128, -128, -128,
+      -127, -127, -127, -127,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+        -1,   -1,   -1,   -1,
+         0,    0,    0,    0,
+       127,  127,  127,  127,
+    },
+  },
+  {
+    { -128 },
+    {
+       127,  127,  127,  127,
+        -1,   -1,   -1,   -1,
+      -128, -128, -128, -128,
+         1,    1,    1,    1,
+    },
+    {
+        -1,   -1,   -1,   -1,
+      -128, -128, -128, -128,
+      -128, -128, -128, -128,
+      -127, -127, -127, -127,
+    },
+  },
+};
+
+int16_t TEST_BINARY_DATA(int16_t, sat_add)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+  },
+  {
+    { 32767 },
+    {
+       32767,  32767,  32767,  32767,
+      -32768, -32768, -32768, -32768,
+      -32767, -32767, -32767, -32767,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+          -1,     -1,     -1,     -1,
+           0,      0,      0,      0,
+       32767,  32767,  32767,  32767,
+    },
+  },
+  {
+    { -32768 },
+    {
+       32767,  32767,  32767,  32767,
+          -1,     -1,     -1,     -1,
+      -32768, -32768, -32768, -32768,
+           1,      1,      1,      1,
+    },
+    {
+          -1,     -1,     -1,     -1,
+      -32768, -32768, -32768, -32768,
+      -32768, -32768, -32768, -32768,
+      -32767, -32767, -32767, -32767,
+    },
+  },
+};
+
+int32_t TEST_BINARY_DATA(int32_t, sat_add)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+  },
+  {
+    { 2147483647 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483647, -2147483647, -2147483647, -2147483647,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+               -1,          -1,          -1,          -1,
+                0,           0,           0,           0,
+       2147483647,  2147483647,  2147483647,  2147483647,
+    },
+  },
+  {
+    { -2147483648 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+               -1,          -1,          -1,          -1,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+                1,           1,           1,           1,
+    },
+    {
+               -1,          -1,          -1,          -1,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483647, -2147483647, -2147483647, -2147483647,
+    },
+  },
+};
+
+int64_t TEST_BINARY_DATA(int64_t, sat_add)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+  },
+  {
+    { 9223372036854775807ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
+                            1,                       1,                       1,                       1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+        -1,   -1,   -1,   -1,
+         0,    0,    0,    0,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+    },
+  },
+  {
+    { -9223372036854775808ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+                           -1,                      -1,                      -1,                      -1,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+                            1,                       1,                       1,                       1,
+    },
+    {
+                           -1,                      -1,                      -1,                      -1,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
+    },
+  },
+};
+
+int8_t TEST_BINARY_DATA(int8_t, sat_sub)[][3][N] =
+{
+  {
+    { 1 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+      -1, -1, -1, -1,
+       3,  3,  3,  3,
+    },
+  },
+  {
+    { 127 },
+    {
+       127,  127,  127,  127,
+      -128, -128, -128, -128,
+      -127, -127, -127, -127,
+         1,    1,    1,    1,
+    },
+    {
+         0,    0,    0,    0,
+      -128, -128, -128, -128,
+      -128, -128, -128, -128,
+      -126, -126, -126, -126,
+    },
+  },
+  {
+    { -128 },
+    {
+       127,  127,  127,  127,
+        -1,   -1,   -1,   -1,
+      -128, -128, -128, -128,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+       127,  127,  127,  127,
+         0,    0,    0,    0,
+       127,  127,  127,  127,
+    },
+  },
+};
+
+int16_t TEST_BINARY_DATA(int16_t, sat_sub)[][3][N] =
+{
+  {
+    { 1 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+      -1, -1, -1, -1,
+       3,  3,  3,  3,
+    },
+  },
+  {
+    { 32767 },
+    {
+       32767,  32767,  32767,  32767,
+      -32768, -32768, -32768, -32768,
+      -32767, -32767, -32767, -32767,
+           1,      1,      1,      1,
+    },
+    {
+           0,      0,      0,      0,
+      -32768, -32768, -32768, -32768,
+      -32768, -32768, -32768, -32768,
+      -32766, -32766, -32766, -32766,
+    },
+  },
+  {
+    { -32768 },
+    {
+       32767,  32767,  32767,  32767,
+          -1,     -1,     -1,     -1,
+      -32768, -32768, -32768, -32768,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+       32767,  32767,  32767,  32767,
+           0,      0,      0,      0,
+       32767,  32767,  32767,  32767,
+    },
+  },
+};
+
+int32_t TEST_BINARY_DATA(int32_t, sat_sub)[][3][N] =
+{
+  {
+    { 1 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+      -1, -1, -1, -1,
+       3,  3,  3,  3,
+    },
+  },
+  {
+    { 2147483647 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483647, -2147483647, -2147483647, -2147483647,
+                1,           1,           1,           1,
+    },
+    {
+                0,           0,           0,           0,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483646, -2147483646, -2147483646, -2147483646,
+    },
+  },
+  {
+    { -2147483648 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+               -1,          -1,          -1,          -1,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+       2147483647,  2147483647,  2147483647,  2147483647,
+                0,           0,           0,           0,
+       2147483647,  2147483647,  2147483647,  2147483647,
+    },
+  },
+};
+
+int64_t TEST_BINARY_DATA(int64_t, sat_sub)[][3][N] =
+{
+  {
+    { 1 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+      -1, -1, -1, -1,
+       3,  3,  3,  3,
+    },
+  },
+  {
+    { 9223372036854775807ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
+                            1,                       1,                       1,                       1,
+    },
+    {
+                            0,                       0,                       0,                       0,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775806ull, -9223372036854775806ull, -9223372036854775806ull, -9223372036854775806ull,
+    },
+  },
+  {
+    { -9223372036854775808ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+                           -1,                      -1,                      -1,                      -1,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+                            1,                       1,                       1,                       1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+                            0,                       0,                       0,                       0,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+    },
+  },
+};
+
+uint8_t TEST_BINARY_DATA(uint8_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 127 },
+    {
+       127,  127,  127,  127,
+       128,  128,  128,  128,
+       255,  255,  255,  255,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+       127,  127,  127,  127,
+       191,  191,  191,  191,
+        64,   64,   64,   64,
+    },
+  },
+  {
+    { 255 },
+    {
+         0,    0,    0,    0,
+       255,  255,  255,  255,
+       254,  254,  254,  254,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+       255,  255,  255,  255,
+       254,  254,  254,  254,
+       128,  128,  128,  128,
+    },
+  },
+};
+
+uint16_t TEST_BINARY_DATA(uint16_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 32767 },
+    {
+       32767,  32767,  32767,  32767,
+       32768,  32768,  32768,  32768,
+       65535,  65535,  65535,  65535,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+       32767,  32767,  32767,  32767,
+       49151,  49151,  49151,  49151,
+       16384,  16384,  16384,  16384,
+    },
+  },
+  {
+    { 65535 },
+    {
+           0,      0,      0,      0,
+       65535,  65535,  65535,  65535,
+       65534,  65534,  65534,  65534,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+       65535,  65535,  65535,  65535,
+       65534,  65534,  65534,  65534,
+       32768,  32768,  32768,  32768,
+    },
+  },
+};
+
+uint32_t TEST_BINARY_DATA(uint32_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 2147483647 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+       2147483648,  2147483648,  2147483648,  2147483648,
+       4294967295,  4294967295,  4294967295,  4294967295,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+       2147483647,  2147483647,  2147483647,  2147483647,
+       3221225471,  3221225471,  3221225471,  3221225471,
+       1073741824,  1073741824,  1073741824,  1073741824,
+    },
+  },
+  {
+    { 4294967295 },
+    {
+                0,           0,           0,           0,
+       4294967295,  4294967295,  4294967295,  4294967295,
+       4294967294,  4294967294,  4294967294,  4294967294,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+       4294967295,  4294967295,  4294967295,  4294967295,
+       4294967294,  4294967294,  4294967294,  4294967294,
+       2147483648,  2147483648,  2147483648,  2147483648,
+    },
+  },
+};
+
+uint64_t TEST_BINARY_DATA(uint64_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 9223372036854775807ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+       9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,
+       18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,
+         1,    1,    1,    1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+      13835058055282163711ull, 13835058055282163711ull, 13835058055282163711ull, 13835058055282163711ull,
+       4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,
+    },
+  },
+  {
+    { 18446744073709551615ull },
+    {
+         0,    0,    0,    0,
+       18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,
+       18446744073709551614ull,  18446744073709551614ull,  18446744073709551614ull,  18446744073709551614ull,
+         1,    1,    1,    1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+      18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull,
+      18446744073709551614ull, 18446744073709551614ull, 18446744073709551614ull, 18446744073709551614ull,
+       9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,
+    },
+  },
+};
+
+int8_t TEST_BINARY_DATA(int8_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 127 },
+    {
+       127,  127,  127,  127,
+      -128, -128, -128, -128,
+      -127, -127, -127, -127,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+        -1,   -1,   -1,   -1,
+         0,    0,    0,    0,
+        64,   64,   64,   64,
+    },
+  },
+  {
+    {-128 },
+    {
+         0,    0,    0,    0,
+      -128, -128, -128, -128,
+       126,  126,  126,  126,
+       127,  127,  127,  127,
+    },
+    {
+       -64,  -64,  -64,  -64,
+      -128, -128, -128, -128,
+        -1,   -1,   -1,   -1,
+        -1,   -1,   -1,   -1,
+    },
+  },
+};
+
+int16_t TEST_BINARY_DATA(int16_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 32767 },
+    {
+       32767,  32767,  32767,  32767,
+      -32768, -32768, -32768, -32768,
+      -32767, -32767, -32767, -32767,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+          -1,     -1,     -1,     -1,
+           0,      0,      0,      0,
+       16384,  16384,  16384,  16384,
+    },
+  },
+  {
+    {-32768 },
+    {
+           0,      0,      0,      0,
+      -32768, -32768, -32768, -32768,
+       32766,  32766,  32766,  32766,
+       32767,  32767,  32767,  32767,
+    },
+    {
+       -16384, -16384, -16384, -16384,
+       -32768, -32768, -32768, -32768,
+           -1,     -1,     -1,     -1,
+           -1,     -1,     -1,     -1,
+    },
+  },
+};
+
+int32_t TEST_BINARY_DATA(int32_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 2147483647 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483647, -2147483647, -2147483647, -2147483647,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+               -1,          -1,          -1,          -1,
+                0,           0,           0,           0,
+       1073741824,  1073741824,  1073741824,  1073741824,
+    },
+  },
+  {
+    {-2147483648 },
+    {
+                0,           0,           0,           0,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+       2147483646,  2147483646,  2147483646,  2147483646,
+       2147483647,  2147483647,  2147483647,  2147483647,
+    },
+    {
+      -1073741824, -1073741824, -1073741824, -1073741824,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+               -1,          -1,          -1,          -1,
+               -1,          -1,          -1,          -1,
+    },
+  },
+};
+
+int64_t TEST_BINARY_DATA(int64_t, avg_floor)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 9223372036854775807ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
+                            1,                       1,                       1,                       1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+                           -1,                      -1,                      -1,                      -1,
+                            0,                       0,                       0,                       0,
+      4611686018427387904ull,   4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,
+    },
+  },
+  {
+    {-9223372036854775808ull },
+    {
+                            0,                       0,                       0,                       0,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+    },
+    {
+      -4611686018427387904ull, -4611686018427387904ull, -4611686018427387904ull, -4611686018427387904ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+                           -1,                      -1,                      -1,                      -1,
+                           -1,                      -1,                      -1,                      -1,
+    },
+  },
+};
+
+uint8_t TEST_BINARY_DATA(uint8_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 127 },
+    {
+       127,  127,  127,  127,
+       128,  128,  128,  128,
+       255,  255,  255,  255,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+       128,  128,  128,  128,
+       191,  191,  191,  191,
+        64,   64,   64,   64,
+    },
+  },
+  {
+    { 255 },
+    {
+         0,    0,    0,    0,
+       255,  255,  255,  255,
+       254,  254,  254,  254,
+         1,    1,    1,    1,
+    },
+    {
+       128,  128,  128,  128,
+       255,  255,  255,  255,
+       255,  255,  255,  255,
+       128,  128,  128,  128,
+    },
+  },
+};
+
+uint16_t TEST_BINARY_DATA(uint16_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 32767 },
+    {
+       32767,  32767,  32767,  32767,
+       32768,  32768,  32768,  32768,
+       65535,  65535,  65535,  65535,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+       32768,  32768,  32768,  32768,
+       49151,  49151,  49151,  49151,
+       16384,  16384,  16384,  16384,
+    },
+  },
+  {
+    { 65535 },
+    {
+           0,      0,      0,      0,
+       65535,  65535,  65535,  65535,
+       65534,  65534,  65534,  65534,
+           1,      1,      1,      1,
+    },
+    {
+       32768,  32768,  32768,  32768,
+       65535,  65535,  65535,  65535,
+       65535,  65535,  65535,  65535,
+       32768,  32768,  32768,  32768,
+    },
+  },
+};
+
+uint32_t TEST_BINARY_DATA(uint32_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 2147483647 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+       2147483648,  2147483648,  2147483648,  2147483648,
+       4294967295,  4294967295,  4294967295,  4294967295,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+       2147483648,  2147483648,  2147483648,  2147483648,
+       3221225471,  3221225471,  3221225471,  3221225471,
+       1073741824,  1073741824,  1073741824,  1073741824,
+    },
+  },
+  {
+    { 4294967295 },
+    {
+                0,           0,           0,           0,
+       4294967295,  4294967295,  4294967295,  4294967295,
+       4294967294,  4294967294,  4294967294,  4294967294,
+                1,           1,           1,           1,
+    },
+    {
+       2147483648,  2147483648,  2147483648,  2147483648,
+       4294967295,  4294967295,  4294967295,  4294967295,
+       4294967295,  4294967295,  4294967295,  4294967295,
+       2147483648,  2147483648,  2147483648,  2147483648,
+    },
+  },
+};
+
+uint64_t TEST_BINARY_DATA(uint64_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 9223372036854775807ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+       9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,
+       18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,
+         1,    1,    1,    1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+       9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,
+      13835058055282163711ull, 13835058055282163711ull, 13835058055282163711ull, 13835058055282163711ull,
+       4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,
+    },
+  },
+  {
+    { 18446744073709551615ull },
+    {
+         0,    0,    0,    0,
+       18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,  18446744073709551615ull,
+       18446744073709551614ull,  18446744073709551614ull,  18446744073709551614ull,  18446744073709551614ull,
+         1,    1,    1,    1,
+    },
+    {
+       9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,
+      18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull,
+      18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull,
+       9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,  9223372036854775808ull,
+    },
+  },
+};
+
+int8_t TEST_BINARY_DATA(int8_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 127 },
+    {
+       127,  127,  127,  127,
+      -128, -128, -128, -128,
+      -127, -127, -127, -127,
+         1,    1,    1,    1,
+    },
+    {
+       127,  127,  127,  127,
+         0,    0,    0,    0,
+         0,    0,    0,    0,
+        64,   64,   64,   64,
+    },
+  },
+  {
+    {-128 },
+    {
+         0,    0,    0,    0,
+      -128, -128, -128, -128,
+       126,  126,  126,  126,
+       127,  127,  127,  127,
+    },
+    {
+       -64,  -64,  -64,  -64,
+      -128, -128, -128, -128,
+        -1,   -1,   -1,   -1,
+         0,    0,    0,    0,
+    },
+  },
+};
+
+int16_t TEST_BINARY_DATA(int16_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 32767 },
+    {
+       32767,  32767,  32767,  32767,
+      -32768, -32768, -32768, -32768,
+      -32767, -32767, -32767, -32767,
+           1,      1,      1,      1,
+    },
+    {
+       32767,  32767,  32767,  32767,
+           0,      0,      0,      0,
+           0,      0,      0,      0,
+       16384,  16384,  16384,  16384,
+    },
+  },
+  {
+    {-32768 },
+    {
+           0,      0,      0,      0,
+      -32768, -32768, -32768, -32768,
+       32766,  32766,  32766,  32766,
+       32767,  32767,  32767,  32767,
+    },
+    {
+       -16384, -16384, -16384, -16384,
+       -32768, -32768, -32768, -32768,
+           -1,     -1,     -1,     -1,
+            0,      0,      0,      0,
+    },
+  },
+};
+
+int32_t TEST_BINARY_DATA(int32_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 2147483647 },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+      -2147483647, -2147483647, -2147483647, -2147483647,
+                1,           1,           1,           1,
+    },
+    {
+       2147483647,  2147483647,  2147483647,  2147483647,
+                0,           0,           0,           0,
+                0,           0,           0,           0,
+       1073741824,  1073741824,  1073741824,  1073741824,
+    },
+  },
+  {
+    {-2147483648 },
+    {
+                0,           0,           0,           0,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+       2147483646,  2147483646,  2147483646,  2147483646,
+       2147483647,  2147483647,  2147483647,  2147483647,
+    },
+    {
+      -1073741824, -1073741824, -1073741824, -1073741824,
+      -2147483648, -2147483648, -2147483648, -2147483648,
+               -1,          -1,          -1,          -1,
+                0,           0,           0,           0,
+    },
+  },
+};
+
+int64_t TEST_BINARY_DATA(int64_t, avg_ceil)[][3][N] =
+{
+  {
+    { 0 },
+    {
+       2,  2,  2,  2,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       4,  4,  4,  4,
+    },
+    {
+       1,  1,  1,  1,
+       1,  1,  1,  1,
+       0,  0,  0,  0,
+       2,  2,  2,  2,
+    },
+  },
+  {
+    { 9223372036854775807ull },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+      -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull, -9223372036854775807ull,
+                            1,                       1,                       1,                       1,
+    },
+    {
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+                            0,                       0,                       0,                       0,
+                            0,                       0,                       0,                       0,
+       4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,  4611686018427387904ull,
+    },
+  },
+  {
+    {-9223372036854775808ull },
+    {
+                            0,                       0,                       0,                       0,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+       9223372036854775806ull,  9223372036854775806ull,  9223372036854775806ull,  9223372036854775806ull,
+       9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,  9223372036854775807ull,
+    },
+    {
+      -4611686018427387904ull, -4611686018427387904ull, -4611686018427387904ull, -4611686018427387904ull,
+      -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull, -9223372036854775808ull,
+                           -1,                      -1,                      -1,                      -1,
+                            0,                       0,                       0,                       0,
+    },
+  },
+};
+
 #endif
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i16.c
new file mode 100644
index 0000000..0307b3f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int16_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i32.c
new file mode 100644
index 0000000..d73325b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int32_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i64.c
new file mode 100644
index 0000000..481774b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int64_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i8.c
new file mode 100644
index 0000000..7de89ee
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-i8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int8_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u16.c
new file mode 100644
index 0000000..73d1a57
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint16_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u32.c
new file mode 100644
index 0000000..60a7aa4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint32_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u64.c
new file mode 100644
index 0000000..803bcba
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint64_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u8.c
new file mode 100644
index 0000000..f28147b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-1-u8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint8_t
+#define NAME       avg_floor
+#define FUNC       AVG_FLOOR_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i16.c
new file mode 100644
index 0000000..8def643
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int16_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i32.c
new file mode 100644
index 0000000..d9ca67d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int32_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i64.c
new file mode 100644
index 0000000..313109a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int64_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i8.c
new file mode 100644
index 0000000..47e4a5d7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-i8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int8_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u16.c
new file mode 100644
index 0000000..6297672
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint16_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u32.c
new file mode 100644
index 0000000..30db24b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint32_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u64.c
new file mode 100644
index 0000000..db3c911
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v && rv64 } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint64_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u8.c
new file mode 100644
index 0000000..a7755f0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vaadd-run-2-u8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          uint8_t
+#define NAME       avg_ceil
+#define FUNC       AVG_CEIL_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i16.c
new file mode 100644
index 0000000..1f0fd46
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int16_t
+#define NAME       sat_add
+#define FUNC       SAT_S_ADD_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i32.c
new file mode 100644
index 0000000..4a8df0c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int32_t
+#define NAME       sat_add
+#define FUNC       SAT_S_ADD_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i64.c
new file mode 100644
index 0000000..534cd25
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int64_t
+#define NAME       sat_add
+#define FUNC       SAT_S_ADD_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i8.c
new file mode 100644
index 0000000..de2a9b6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vsadd-run-1-i8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int8_t
+#define NAME       sat_add
+#define FUNC       SAT_S_ADD_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i16.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i16.c
new file mode 100644
index 0000000..bd985c2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i16.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int16_t
+#define NAME       sat_sub
+#define FUNC       SAT_S_SUB_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i32.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i32.c
new file mode 100644
index 0000000..c510ea0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i32.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int32_t
+#define NAME       sat_sub
+#define FUNC       SAT_S_SUB_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i64.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i64.c
new file mode 100644
index 0000000..b82278d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i64.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int64_t
+#define NAME       sat_sub
+#define FUNC       SAT_S_SUB_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i8.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i8.c
new file mode 100644
index 0000000..5fae704
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/vx_vf/vx_vssub-run-1-i8.c
@@ -0,0 +1,17 @@
+/* { dg-do run { target { riscv_v } } } */
+/* { dg-additional-options "-std=c99 --param=gpr2vr-cost=0" } */
+
+#include "vx_binary.h"
+#include "vx_binary_data.h"
+
+#define T          int8_t
+#define NAME       sat_sub
+#define FUNC       SAT_S_SUB_FUNC_WRAP(T)
+#define TEST_DATA  TEST_BINARY_DATA_WRAP(T, NAME)
+
+DEF_VX_BINARY_CASE_2_WRAP(T, FUNC, NAME)
+
+#define TEST_RUN(T, NAME, out, in, x, n) \
+  RUN_VX_BINARY_CASE_2_WRAP(T, NAME, FUNC, out, in, x, n)
+
+#include "vx_binary_run.h"
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/pr113829.c b/gcc/testsuite/gcc.target/riscv/rvv/base/pr113829.c
new file mode 100644
index 0000000..48c291a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/pr113829.c
@@ -0,0 +1,10 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=rv64gcv -mabi=lp64d" } */
+
+#pragma riscv intrinsic "vector"
+void
+foo (void)
+{
+  __riscv_vfredosum_tu (X); /* { dg-error "undeclared" } */
+  /* { dg-error "too many arguments" "" { target *-*-* } .-1 } */
+}
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-5.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-5.c
index 04dec7b..4f6785a 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-5.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-5.c
@@ -6,9 +6,9 @@
 
 /*
 ** foo:
-** addi\t[a-x0-9]+,\s*[a-x0-9]+,100
+** ...
 ** vsetvli\tzero,a2,e64,m2,t[au],m[au]
-** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
+** vmv.s.x\tv[0-9]+.*
 ** vs2r.v\tv[0-9]+,0\([a-x0-9]+\)
 ** ret
 */
@@ -23,7 +23,7 @@ void foo (void *base, void *out, size_t vl)
 ** foo2:
 ** fld\tfa[0-9]+,\s*100\(a0\)
 ** vsetvli\tzero,a2,e64,m2,t[au],m[au]
-** vfmv\.v\.f\tv[0-9]+,\s*fa[0-9]+
+** vfmv\.s\.f\tv[0-9]+,\s*fa[0-9]+
 ** vs2r.v\tv[0-9]+,0\([a-x0-9]+\)
 ** ret
 */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-6.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-6.c
index 0ebb92e..a8c9263c 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-6.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-6.c
@@ -23,7 +23,7 @@ void foo (void *base, void *out, size_t vl)
 ** foo2:
 ** fld\tfa[0-9]+,\s*100\(a0\)
 ** vsetvli\tzero,a2,e64,m2,t[au],m[au]
-** vfmv\.v\.f\tv[0-9]+,\s*fa[0-9]+
+** vfmv\.s\.f\tv[0-9]+,\s*fa[0-9]+
 ** vs2r.v\tv[0-9]+,0\([a-x0-9]+\)
 ** ret
 */
@@ -52,7 +52,7 @@ void foo3 (void *base, void *out, size_t vl)
 /*
 ** foo4:
 ** ...
-** vfmv\.v\.f\tv[0-9]+,\s*fa[0-9]+
+** vfmv\.s\.f\tv[0-9]+,\s*fa[0-9]+
 ** ...
 ** ret
 */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-7.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-7.c
index 512fa62..cf53aca 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-7.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-7.c
@@ -6,9 +6,9 @@
 
 /*
 ** foo:
-** addi\t[a-x0-9]+,\s*[a-x0-9]+,100
+** ...
 ** vsetvli\tzero,a2,e64,m2,t[au],m[au]
-** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
+** vmv\.v\.x\tv[0-9]+,\s*a[0-9]+
 ** vs2r.v\tv[0-9]+,0\([a-x0-9]+\)
 ** ret
 */
@@ -37,7 +37,7 @@ void foo2 (void *base, void *out, size_t vl)
 /*
 ** foo3:
 ** ...
-** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
+** vmv\.v\.x\tv[0-9]+,\s*a[0-9]+
 ** ...
 ** ret
 */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-8.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-8.c
index d9d10f3..fd3b7c5 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-8.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-8.c
@@ -175,9 +175,8 @@ void foo12 (void *base, void *out, size_t vl)
 /*
 ** foo13:
 ** ...
-** vmv.v.x\tv[0-9]+,\s*[a-x0-9]+
+** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
 ** ...
-** ret
 */
 void foo13 (void *base, void *out, size_t vl)
 {
@@ -189,7 +188,7 @@ void foo13 (void *base, void *out, size_t vl)
 /*
 ** foo14:
 ** ...
-** vmv.v.x\tv[0-9]+,\s*[a-x0-9]+
+** vlse64.v\tv[0-9]+,0\([a-x0-9]+\),zero
 ** ...
 */
 void foo14 (void *base, void *out, size_t vl)
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-9.c b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-9.c
index 80ee1b5..64c22dd 100644
--- a/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-9.c
+++ b/gcc/testsuite/gcc.target/riscv/rvv/base/scalar_move-9.c
@@ -23,4 +23,3 @@ vuint64m2_t f3(vuint64m2_t var_17, uint64_t var_60, size_t vl)
 
 /* { dg-final { scan-assembler-times {vsetivli\s+zero,\s*0,\s*e64,\s*m2,\s*t[au],\s*m[au]} 1 } } */
 /* { dg-final { scan-assembler-times {vsetivli\s+zero,\s*1,\s*e64,\s*m2,\s*t[au],\s*m[au]} 1 } } */
-/* { dg-final { scan-assembler-times {sgtu} 1 } } */
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/pr120297.c b/gcc/testsuite/gcc.target/riscv/rvv/pr120297.c
new file mode 100644
index 0000000..3d1845d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/pr120297.c
@@ -0,0 +1,50 @@
+/* { dg-do run } */
+/* { dg-require-effective-target riscv_v_ok } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -O3 -fwhole-program" } */
+
+unsigned a;
+short c;
+char d;
+unsigned long e;
+_Bool f[10][10];
+unsigned g[10];
+long long ak;
+char i = 7;
+long long t[10];
+short x[10][10][10][10];
+short y[10][10][10][10];
+
+void
+h (char i, long long t[], short x[][10][10][10], short y[][10][10][10],
+   _Bool aa)
+{
+  for (int j = 2; j < 8; j += 2)
+    {
+      for (short k = 0; k < 10; k++)
+	{
+	  for (int l = 3; l < 8; l += 2)
+	    a = x[1][j][k][l];
+	  c = x[c][1][1][c];
+	}
+      for (int k = 0; k < 10; k++)
+	{
+	  f[2][k] |= (_Bool) t[c];
+	  g[c] = t[c + 1];
+	  d += y[j][1][k][k];
+	  e = e > i ? e : i;
+	}
+    }
+}
+
+int
+main ()
+{
+  t[c] = 1;
+  h (i, t, x, y, a);
+  for (int j = 0; j < 10; ++j)
+    for (int k = 0; k < 10; ++k)
+      ak ^= f[j][k] + 238516665 + (ak >> 2);
+  ak ^= g[c] + 238516665 + (ak >> 2);
+  if (ak != 234635118ull)
+    __builtin_abort ();
+}
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/pr121073.c b/gcc/testsuite/gcc.target/riscv/rvv/pr121073.c
new file mode 100644
index 0000000..2789d0f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/pr121073.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv -mabi=lp64d -O3 -mrvv-vector-bits=zvl -fsigned-char -fno-strict-aliasing -fwrapv -Wno-stringop-overflow -Wno-aggressive-loop-optimizations" } */
+
+int a;
+unsigned char p[1][21];
+void init() {
+  for (int s = 0; s < 21; ++s)
+    for (int t = 0; t < 21; ++t)
+      p[s][t] = 39;
+  for (short t = 0; t < 9; t += -5077966496202321318LL + 28071)
+    a = p[3][t] && p[2][t];
+}
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/xtheadvector/pr120461.c b/gcc/testsuite/gcc.target/riscv/rvv/xtheadvector/pr120461.c
new file mode 100644
index 0000000..6939157
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/xtheadvector/pr120461.c
@@ -0,0 +1,6 @@
+/* { dg-do compile } */
+/* { dg-options "-mcpu=xt-c920 -mrvv-vector-bits=zvl -fzero-call-used-regs=all" */
+
+void
+foo ()
+{}
diff --git a/gcc/testsuite/gcc.target/riscv/rvv/xtheadvector/pr120642.c b/gcc/testsuite/gcc.target/riscv/rvv/xtheadvector/pr120642.c
new file mode 100644
index 0000000..1a72580
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/xtheadvector/pr120642.c
@@ -0,0 +1,4 @@
+/* { dg-do compile } */
+/* { dg-options "-O -mcpu=xt-c920 -mrvv-vector-bits=zvl" } */
+int __attribute__((__vector_size__(4 * sizeof(int)))) v;
+void foo() { v /= 3; }
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_arith.h b/gcc/testsuite/gcc.target/riscv/sat/sat_arith.h
index 6e97cae..e40902a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_arith.h
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_arith.h
@@ -4,6 +4,10 @@
 #include <stdint-gcc.h>
 #include <stdbool.h>
 
+#if __riscv_xlen == 64
+typedef unsigned __int128 uint128_t;
+#endif
+
 /******************************************************************************/
 /* Saturation Add (unsigned and signed)                                       */
 /******************************************************************************/
@@ -69,6 +73,22 @@ sat_u_add_##WT##_##T##_fmt_7(T x, T y) \
 }
 #define DEF_SAT_U_ADD_FMT_7_WRAP(WT, T) DEF_SAT_U_ADD_FMT_7(WT, T)
 
+#define DEF_SAT_U_ADD_FMT_8(T)                \
+T __attribute__((noinline))                   \
+sat_u_add_##T##_fmt_8(T x, T y)               \
+{                                             \
+  return x <= (T)(x + y) ? (x + y) : -1;      \
+}
+#define DEF_SAT_U_ADD_FMT_8_WRAP(T) DEF_SAT_U_ADD_FMT_8(T)
+
+#define DEF_SAT_U_ADD_FMT_9(T)               \
+T __attribute__((noinline))                  \
+sat_u_add_##T##_fmt_9(T x, T y)              \
+{                                            \
+  return x > (T)(x + y) ? -1 : (x + y);      \
+}
+#define DEF_SAT_U_ADD_FMT_9_WRAP(T) DEF_SAT_U_ADD_FMT_9(T)
+
 #define RUN_SAT_U_ADD_FMT_1(T, x, y) sat_u_add_##T##_fmt_1(x, y)
 #define RUN_SAT_U_ADD_FMT_1_WRAP(T, x, y) RUN_SAT_U_ADD_FMT_1(T, x, y)
 #define RUN_SAT_U_ADD_FMT_2(T, x, y) sat_u_add_##T##_fmt_2(x, y)
@@ -93,6 +113,10 @@ sat_u_add_##WT##_##T##_fmt_7(T x, T y) \
   sat_u_add_uint64_t_##T##_fmt_7(x, y)
 #define RUN_SAT_U_ADD_FMT_7_FROM_U64_WRAP(T, x, y) \
   RUN_SAT_U_ADD_FMT_7_FROM_U64(T, x, y)
+#define RUN_SAT_U_ADD_FMT_8(T, x, y) sat_u_add_##T##_fmt_8(x, y)
+#define RUN_SAT_U_ADD_FMT_8_WRAP(T, x, y) RUN_SAT_U_ADD_FMT_8(T, x, y)
+#define RUN_SAT_U_ADD_FMT_9(T, x, y) sat_u_add_##T##_fmt_9(x, y)
+#define RUN_SAT_U_ADD_FMT_9_WRAP(T, x, y) RUN_SAT_U_ADD_FMT_9(T, x, y)
 
 #define DEF_SAT_U_ADD_IMM_FMT_1(T, IMM)      \
 T __attribute__((noinline))                  \
@@ -227,6 +251,18 @@ sat_s_add_imm_##T##_fmt_1##_##INDEX (T x)             \
 #define RUN_SAT_S_ADD_IMM_FMT_1(INDEX, T, x, expect) \
   if (sat_s_add_imm##_##T##_fmt_1##_##INDEX(x) != expect) __builtin_abort ()
 
+#define DEF_SAT_S_ADD_IMM_FMT_2(INDEX, T, UT, IMM, MIN, MAX) \
+T __attribute__((noinline))                                  \
+sat_s_add_imm_##T##_fmt_2##_##INDEX (T x)                    \
+{                                                            \
+  T sum = (T)((UT)x + (UT)IMM);                                   \
+  return ((x ^ sum) < 0 && (x ^ IMM) >= 0) ?                 \
+    (-(T)(x < 0) ^ MAX) : sum;                         \
+}
+
+#define RUN_SAT_S_ADD_IMM_FMT_2(INDEX, T, x, expect) \
+  if (sat_s_add_imm##_##T##_fmt_2##_##INDEX(x) != expect) __builtin_abort ()
+
 /******************************************************************************/
 /* Saturation Sub (Unsigned and Signed)                                       */
 /******************************************************************************/
@@ -636,4 +672,25 @@ sat_s_trunc_##WT##_to_##NT##_fmt_8 (WT x)             \
 #define RUN_SAT_S_TRUNC_FMT_8(NT, WT, x) sat_s_trunc_##WT##_to_##NT##_fmt_8 (x)
 #define RUN_SAT_S_TRUNC_FMT_8_WRAP(NT, WT, x) RUN_SAT_S_TRUNC_FMT_8(NT, WT, x)
 
+/******************************************************************************/
+/* Saturation Mult (unsigned and signed)                                  */
+/******************************************************************************/
+
+#define DEF_SAT_U_MUL_FMT_1(NT, WT)             \
+NT __attribute__((noinline))                    \
+sat_u_mul_##NT##_from_##WT##_fmt_1 (NT a, NT b) \
+{                                               \
+  WT x = (WT)a * (WT)b;                         \
+  NT max = -1;                                  \
+  if (x > (WT)(max))                            \
+    return max;                                 \
+  else                                          \
+    return (NT)x;                               \
+}
+
+#define DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT) DEF_SAT_U_MUL_FMT_1(NT, WT)
+#define RUN_SAT_U_MUL_FMT_1(NT, WT, a, b) \
+  sat_u_mul_##NT##_from_##WT##_fmt_1 (a, b)
+#define RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, a, b) RUN_SAT_U_MUL_FMT_1(NT, WT, a, b)
+
 #endif
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_arith_data.h b/gcc/testsuite/gcc.target/riscv/sat/sat_arith_data.h
index f100688..bd33ff1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_arith_data.h
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_arith_data.h
@@ -12,6 +12,7 @@
 
 #define TEST_BINARY_STRUCT_NAME(T, NAME) test_##T##_##NAME##_s
 #define TEST_BINARY_STRUCT_DECL(T, NAME) struct TEST_BINARY_STRUCT_NAME(T, NAME)
+#define TEST_BINARY_STRUCT_DECL_WRAP(T, NAME) TEST_BINARY_STRUCT_DECL(T, NAME)
 #define TEST_BINARY_STRUCT(T, NAME)       \
   struct TEST_BINARY_STRUCT_NAME(T, NAME) \
     {                                     \
@@ -37,6 +38,11 @@ TEST_BINARY_STRUCT (uint16_t, usadd)
 TEST_BINARY_STRUCT (uint32_t, usadd)
 TEST_BINARY_STRUCT (uint64_t, usadd)
 
+TEST_BINARY_STRUCT (uint8_t, usmul)
+TEST_BINARY_STRUCT (uint16_t, usmul)
+TEST_BINARY_STRUCT (uint32_t, usmul)
+TEST_BINARY_STRUCT (uint64_t, usmul)
+
 TEST_BINARY_STRUCT (int8_t,  ssadd)
 TEST_BINARY_STRUCT (int16_t, ssadd)
 TEST_BINARY_STRUCT (int32_t, ssadd)
@@ -433,4 +439,60 @@ TEST_BINARY_STRUCT_DECL(int64_t, sssub) TEST_BINARY_DATA(int64_t, sssub)[] =
   {  9223372036854775806ll,   9223372036854775800ll,                       6},
 };
 
+TEST_BINARY_STRUCT_DECL(uint8_t, usmul) TEST_BINARY_DATA(uint8_t, usmul)[] =
+{
+  {      0,     0,      0, },
+  {      0,     1,      0, },
+  {      1,     1,      1, },
+  {      1,   127,    127, },
+  {      2,   127,    254, },
+  {      3,   127,    255, },
+  {    127,   127,    255, },
+  {      1,   255,    255, },
+  {    127,   255,    255, },
+  {    255,   255,    255, },
+};
+
+TEST_BINARY_STRUCT_DECL(uint16_t, usmul) TEST_BINARY_DATA(uint16_t, usmul)[] =
+{
+  {      0,     0,      0, },
+  {      0,     1,      0, },
+  {      1,     1,      1, },
+  {      1, 32767,  32767, },
+  {      2, 32767,  65534, },
+  {      3, 32767,  65535, },
+  {  32767, 32767,  65535, },
+  {      1, 65535,  65535, },
+  {  32767, 65535,  65535, },
+  {  65535, 65535,  65535, },
+};
+
+TEST_BINARY_STRUCT_DECL(uint32_t, usmul) TEST_BINARY_DATA(uint32_t, usmul)[] =
+{
+  {          0,          0,          0, },
+  {          0,          1,          0, },
+  {          1,          1,          1, },
+  {          1, 2147483647, 2147483647, },
+  {          2, 2147483647, 4294967294, },
+  {          3, 2147483647, 4294967295, },
+  { 2147483647, 2147483647, 4294967295, },
+  {          1, 4294967295, 4294967295, },
+  { 2147483647, 4294967295, 4294967295, },
+  { 4294967295, 4294967295, 4294967295, },
+};
+
+TEST_BINARY_STRUCT_DECL(uint64_t, usmul) TEST_BINARY_DATA(uint64_t, usmul)[] =
+{
+  {                       0,                       0,                       0, },
+  {                       0,                       1,                       0, },
+  {                       1,                       1,                       1, },
+  {                       1,  9223372036854775807ull,  9223372036854775807ull, },
+  {                       2,  9223372036854775807ull, 18446744073709551614ull, },
+  {                       3,  9223372036854775807ull, 18446744073709551615ull, },
+  {  9223372036854775807ull,  9223372036854775807ull, 18446744073709551615ull, },
+  {                       1, 18446744073709551615ull, 18446744073709551615ull, },
+  {  9223372036854775807ull, 18446744073709551615ull, 18446744073709551615ull, },
+  { 18446744073709551615ull, 18446744073709551615ull, 18446744073709551615ull, },
+};
+
 #endif
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i16.c
index 55890d8..50f0f1f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i16.c
@@ -1,32 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int16_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_ADD_FMT_1(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i32.c
index 29e843f..dc65817 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i32.c
@@ -1,31 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int32_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_ADD_FMT_1(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i64.c
index 7f29d21..9995bc7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i64.c
@@ -1,29 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int64_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_ADD_FMT_1(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i8.c
index 3ad7bdd..caf745a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-1-i8.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int8_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_ADD_FMT_1(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i16.c
index 07d3101..f19187d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i16.c
@@ -1,32 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int16_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_ADD_FMT_2(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i32.c
index 81b85b4..88dc37d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i32.c
@@ -1,31 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int32_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_ADD_FMT_2(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i64.c
index 9a3d83e..891d6cf 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i64.c
@@ -1,29 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int64_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_ADD_FMT_2(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i8.c
index ecc9a0f..a07172b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-2-i8.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int8_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_ADD_FMT_2(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i16.c
index 7e93385..5077198 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i16.c
@@ -1,32 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int16_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_ADD_FMT_3(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i32.c
index 09bf497..07af4e1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i32.c
@@ -1,31 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int32_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_ADD_FMT_3(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i64.c
index 5652cdb..7c4be5b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i64.c
@@ -1,29 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int64_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_ADD_FMT_3(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i8.c
index 0eb0c84..fc0e1b7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-3-i8.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int8_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_ADD_FMT_3(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i16.c
index 9dfdb9e..4c0b38a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i16.c
@@ -1,32 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int16_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_ADD_FMT_4(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i32.c
index 74df576..45b4638 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i32.c
@@ -1,31 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int32_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_ADD_FMT_4(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i64.c
index 5937699..294eb52 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i64.c
@@ -1,29 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int64_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_ADD_FMT_4(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i8.c
index af850d0..143fa3c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-4-i8.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_int8_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_ADD_FMT_4(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i16.c
index 34459b8..1023934 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i32.c
index 4d4841f..bccb768 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i64.c
index df81887..34de520 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i8.c
index 9a4ce33..6d136ec 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-1-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i16.c
index cdac5bd..ee8e439 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i32.c
index 4ac952e..8996dd2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i64.c
index 4d25e7f..155c8e9 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i8.c
index d57e0a0..4502ed3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-2-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i16.c
index 08b961a..21289c9 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i32.c
index 3611b6e..3d4a6fa 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i64.c
index 3eaa6c2..b55d221 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i8.c
index 6d38e5f..9fef8b0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-3-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i16.c
index 2e73450..fd135e5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i32.c
index ec3022d..38ade40 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i64.c
index 911856e..04ba746 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i8.c
index 94d48ef..32aea5c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add-run-4-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i16.c
index 2e23af5..414cb61 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i16.c
@@ -1,57 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_imm_int16_t_fmt_1_0:
-**	addi\s+[atx][0-9]+,\s*a0,\s*-7
-**	xori\s+[atx][0-9]+,\s*a0,\s*-7
-**	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	srai\s+a0,\s*a0,\s*63
-**	li\s+[atx][0-9]+,\s*32768
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	neg\s+a0,\s*[atx][0-9]+
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	or\s+a0,\s*a0,\s*[atx][0-9]+
-**	slliw\s+a0,\s*a0,\s*16
-**	sraiw\s+a0,\s*a0,\s*16
-**	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(0, int16_t, uint16_t, -7, INT16_MIN, INT16_MAX)
-
-/*
-** sat_s_add_imm_int16_t_fmt_1_1:
-** 	addi\s+[atx][0-9]+,\s*a0,\s*-1
-** 	not\s+[atx][0-9]+,\s*a0
-** 	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** 	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** 	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** 	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** 	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** 	andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** 	srai\s+a0,\s*a0,\s*63
-** 	li\s+[atx][0-9]+,\s*32768
-** 	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** 	xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** 	neg\s+a0,\s*[atx][0-9]+
-** 	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** 	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** 	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** 	or\s+a0,\s*a0,\s*[atx][0-9]+
-** 	slliw\s+a0,\s*a0,\s*16
-** 	sraiw\s+a0,\s*a0,\s*16
-** 	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(1, int16_t, uint16_t, -1, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i32.c
index e63211f..adf5b39 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i32.c
@@ -1,54 +1,11 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_imm_int32_t_fmt_1_0:
-**	addi\s+[atx][0-9]+,\s*a0,\s*10
-**	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-**	srli\s+[atx][0-9]+,\s*a0,\s*31
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	srai\s+a0,\s*a0,\s*63
-**	li\s+[atx][0-9]+,\s*-2147483648
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	neg\s+a0,\s*[atx][0-9]+
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	or\s+a0,a0,\s*[atx][0-9]+
-**	sext.w\s+a0,\s*a0
-**	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(0, int32_t, uint32_t, 10, INT32_MIN, INT32_MAX)
 
-/*
-** sat_s_add_imm_int32_t_fmt_1_1:
-**	addi\s+[atx][0-9]+,\s*a0,\s*-1
-**	not\s+[atx][0-9]+,\s*a0
-**	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	srai\s+a0,\s*a0,\s*63
-**	li\s+[atx][0-9]+,\s*-2147483648
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	neg\s+a0,\s*[atx][0-9]+
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	or\s+a0,\s*a0,\s*[atx][0-9]+
-**	sext.w\s+a0,\s*a0
-**	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(1, int32_t, uint32_t, -1, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i64.c
index 3843b71..b88e064 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i64.c
@@ -1,48 +1,11 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_imm_int64_t_fmt_1_0:
-**	addi\s+[atx][0-9]+,\s*a0,\s*10
-**	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-**	srli\s+[atx][0-9]+,\s*a0,\s*63
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	srai\s+[atx][0-9]+,\s*a0,\s*63
-**	li\s+[atx][0-9]+,\s*-1
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	neg\s+[atx][0-9]+,\s*[atx][0-9]+
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	or\s+a0,\s*a0,\s*[atx][0-9]+
-**	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(0, int64_t, uint64_t, 10, INT64_MIN, INT64_MAX)
 
-/*
-** sat_s_add_imm_int64_t_fmt_1_1:
-**	addi\s+[atx][0-9]+,\s*a0,\s*-1
-**	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-**	slti\s+[atx][0-9]+,\s*a0,\s*0
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	srai\s+[atx][0-9]+,\s*a0,\s*63
-**	li\s+[atx][0-9]+,\s*-1
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	neg\s+[atx][0-9]+,\s*[atx][0-9]+
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	or\s+a0,\s*a0,\s*[atx][0-9]+
-**	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(1, int64_t, uint64_t, -1, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i8.c
index ceae1ea..0e337ef 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-1-i8.c
@@ -1,49 +1,11 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_add_imm_int8_t_fmt_1_0:
-**	addi\s+[atx][0-9]+,\s*a0,\s*9
-**	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-**	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-**	srli\s+[atx][0-9]+,\s*a0,\s*7
-**	xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-**	srai\s+a0,\s*a0,\s*63
-**	xori\s+[atx][0-9]+,\s*a0,\s*127
-**	neg\s+a0,\s*[atx][0-9]+
-**	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-**	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-**	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-**	or\s+a0,\s*a0,\s*[atx][0-9]+
-**	slliw\s+a0,\s*a0,\s*24
-**	sraiw\s+a0,\s*a0,\s*24
-**	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(0, int8_t, uint8_t, 9, INT8_MIN, INT8_MAX)
 
-/*
-** sat_s_add_imm_int8_t_fmt_1_1:
-** 	addi\s+[atx][0-9]+,\s*a0,\s*-1
-** 	xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** 	and\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** 	slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*56
-** 	srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** 	srai\s+a0,\s*a0,\s*63
-** 	xori\s+[atx][0-9]+,\s*a0,\s*127
-** 	neg\s+a0,\s*a5
-** 	and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** 	addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** 	and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** 	or\s+a0,\s*a0,\s*[atx][0-9]+
-** 	slliw\s+a0,\s*a0,\s*24
-** 	sraiw\s+a0,\s*a0,\s*24
-** 	ret
-*/
 DEF_SAT_S_ADD_IMM_FMT_1(1, int8_t, uint8_t, -1, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i16.c
new file mode 100644
index 0000000..f217fe1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i16.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int16_t, uint16_t, -7, INT16_MIN, INT16_MAX)
+
+DEF_SAT_S_ADD_IMM_FMT_2(1, int16_t, uint16_t, -1, INT16_MIN, INT16_MAX)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i32.c
new file mode 100644
index 0000000..4025b5a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i32.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int32_t, uint32_t, 10, INT32_MIN, INT32_MAX)
+
+DEF_SAT_S_ADD_IMM_FMT_2(1, int32_t, uint32_t, -1, INT32_MIN, INT32_MAX)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i64.c
new file mode 100644
index 0000000..3fc2514
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int64_t, uint64_t, 10, INT64_MIN, INT64_MAX)
+
+DEF_SAT_S_ADD_IMM_FMT_2(1, int64_t, uint64_t, -1, INT64_MIN, INT64_MAX)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i8.c
new file mode 100644
index 0000000..a0e15cf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-2-i8.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int8_t, uint8_t, 9, INT8_MIN, INT8_MAX)
+
+DEF_SAT_S_ADD_IMM_FMT_2(1, int8_t, uint8_t, -1, INT8_MIN, INT8_MAX)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 2 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i16.c
new file mode 100644
index 0000000..4f24624
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i16.c
@@ -0,0 +1,48 @@
+/* { dg-do run } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int16_t, uint16_t, -32768, INT16_MIN, INT16_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(1, int16_t, uint16_t, 32767, INT16_MIN, INT16_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(2, int16_t, uint16_t, 100, INT16_MIN, INT16_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(3, int16_t, uint16_t, -100, INT16_MIN, INT16_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(4, int16_t, uint16_t, -1, INT16_MIN, INT16_MAX)
+
+#define T                       int16_t
+#define RUN(INDEX,T, x, expect) RUN_SAT_S_ADD_IMM_FMT_2(INDEX, T, x, expect)
+
+T d[][2] = {
+  /* arg_0,   expect */
+  {     -1,     -32768, },
+  {      2,     -32766, },
+  {      1,      32767, },
+  {    -10,      32757, },
+  {  32669,      32767, },
+  { -32768,     -32668, },
+  { -32768,     -32768, },
+  {      0,       -100, },
+  { -32768,     -32768, },
+  {      0,         -1, },
+};
+
+int
+main ()
+{
+  RUN (0, T, d[0][0], d[0][1]);
+  RUN (0, T, d[1][0], d[1][1]);
+
+  RUN (1, T, d[2][0], d[2][1]);
+  RUN (1, T, d[3][0], d[3][1]);
+
+  RUN (2, T, d[4][0], d[4][1]);
+  RUN (2, T, d[5][0], d[5][1]);
+
+  RUN (3, T, d[6][0], d[6][1]);
+  RUN (3, T, d[7][0], d[7][1]);
+
+  RUN (4, T, d[8][0], d[8][1]);
+  RUN (4, T, d[9][0], d[9][1]);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i32.c
new file mode 100644
index 0000000..8d9ddeb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i32.c
@@ -0,0 +1,48 @@
+/* { dg-do run } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int32_t, uint32_t, -2147483648, INT32_MIN, INT32_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(1, int32_t, uint32_t, 2147483647, INT32_MIN, INT32_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(2, int32_t, uint32_t, 100, INT32_MIN, INT32_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(3, int32_t, uint32_t, -100, INT32_MIN, INT32_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(4, int32_t, uint32_t, -1, INT32_MIN, INT32_MAX)
+
+#define T                       int32_t
+#define RUN(INDEX,T, x, expect) RUN_SAT_S_ADD_IMM_FMT_2(INDEX, T, x, expect)
+
+T d[][2] = {
+  /* arg_0,   expect */
+  {     -1,     -2147483648, },
+  {      2,     -2147483646, },
+  {      1,      2147483647, },
+  {    -10,      2147483637, },
+  {    300,             400, },
+  {   -300,            -200, },
+  {    100,               0, },
+  {      0,            -100, },
+  {    100,              99, },
+  {      0,              -1, },
+};
+
+int
+main ()
+{
+  RUN (0, T, d[0][0], d[0][1]);
+  RUN (0, T, d[1][0], d[1][1]);
+
+  RUN (1, T, d[2][0], d[2][1]);
+  RUN (1, T, d[3][0], d[3][1]);
+
+  RUN (2, T, d[4][0], d[4][1]);
+  RUN (2, T, d[5][0], d[5][1]);
+
+  RUN (3, T, d[6][0], d[6][1]);
+  RUN (3, T, d[7][0], d[7][1]);
+
+  RUN (4, T, d[8][0], d[8][1]);
+  RUN (4, T, d[9][0], d[9][1]);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i64.c
new file mode 100644
index 0000000..4523f9a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i64.c
@@ -0,0 +1,48 @@
+/* { dg-do run } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int64_t, uint64_t, (-9223372036854775807ll - 1), INT64_MIN, INT64_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(1, int64_t, uint64_t, 9223372036854775807ll, INT64_MIN, INT64_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(2, int64_t, uint64_t, 100, INT64_MIN, INT64_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(3, int64_t, uint64_t, -100, INT64_MIN, INT64_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(4, int64_t, uint64_t, -1, INT64_MIN, INT64_MAX)
+
+#define T                       int64_t
+#define RUN(INDEX,T, x, expect) RUN_SAT_S_ADD_IMM_FMT_2(INDEX, T, x, expect)
+
+T d[][2] = {
+  /* arg_0,   expect */
+  {     -1,     (-9223372036854775807ll - 1), },
+  {      2,           -9223372036854775806ll, },
+  {      1,            9223372036854775807ll, },
+  {     -7,            9223372036854775800ll, },
+  {      0,                              100, },
+  {     -1,                               99, },
+  {      0,                             -100, },
+  {    100,                                0, },
+  {      0,                               -1, },
+  {    100,                               99, },
+};
+
+int
+main ()
+{
+  RUN (0, T, d[0][0], d[0][1]);
+  RUN (0, T, d[1][0], d[1][1]);
+
+  RUN (1, T, d[2][0], d[2][1]);
+  RUN (1, T, d[3][0], d[3][1]);
+
+  RUN (2, T, d[4][0], d[4][1]);
+  RUN (2, T, d[5][0], d[5][1]);
+
+  RUN (3, T, d[6][0], d[6][1]);
+  RUN (3, T, d[7][0], d[7][1]);
+
+  RUN (4, T, d[8][0], d[8][1]);
+  RUN (4, T, d[9][0], d[9][1]);
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i8.c
new file mode 100644
index 0000000..96445ae
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm-run-2-i8.c
@@ -0,0 +1,49 @@
+/* { dg-do run } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int8_t, uint8_t, -128, INT8_MIN, INT8_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(1, int8_t, uint8_t, 127, INT8_MIN, INT8_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(2, int8_t, uint8_t, 6, INT8_MIN, INT8_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(3, int8_t, uint8_t, -6, INT8_MIN, INT8_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(4, int8_t, uint8_t, -1, INT8_MIN, INT8_MAX)
+
+#define T                       int8_t
+#define RUN(INDEX,T, x, expect) RUN_SAT_S_ADD_IMM_FMT_2(INDEX, T, x, expect)
+
+T d[][2] = {
+  /* arg_0,   expect */
+  {     -1,     -128, },
+  {      2,     -126, },
+  {      1,      127, },
+  {    -10,      117, },
+  {    122,      127, },
+  {    -10,       -4, },
+  {   -128,     -128, },
+  {    127,      121, },
+  {   -128,     -128, },
+  {    1,          0, },
+};
+
+int
+main ()
+{
+  RUN (0, T, d[0][0], d[0][1]);
+  RUN (0, T, d[1][0], d[1][1]);
+
+  RUN (1, T, d[2][0], d[2][1]);
+  RUN (1, T, d[3][0], d[3][1]);
+
+  RUN (2, T, d[4][0], d[4][1]);
+  RUN (2, T, d[5][0], d[5][1]);
+
+  RUN (3, T, d[6][0], d[6][1]);
+  RUN (3, T, d[7][0], d[7][1]);
+
+  RUN (4, T, d[8][0], d[8][1]);
+  RUN (4, T, d[9][0], d[9][1]);
+
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i16.c
new file mode 100644
index 0000000..a73a77f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i16.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int16_t, uint16_t, -32769, INT16_MIN, INT16_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(1, int16_t, uint16_t, 32768, INT16_MIN, INT16_MAX)
+
+/* { dg-final { scan-tree-dump-not ".SAT_ADD " "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i32.c
new file mode 100644
index 0000000..9dae425
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i32.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_1(0, int32_t, uint32_t, -2147483649, INT32_MIN, INT32_MAX)
+DEF_SAT_S_ADD_IMM_FMT_1(1, int32_t, uint32_t, 2147483648, INT32_MIN, INT32_MAX)
+
+/* { dg-final { scan-tree-dump-not ".SAT_ADD " "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i8.c
new file mode 100644
index 0000000..a9cd4b9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_add_imm_type_check-2-i8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_S_ADD_IMM_FMT_2(0, int8_t, uint8_t, -129, INT8_MIN, INT8_MAX)
+DEF_SAT_S_ADD_IMM_FMT_2(1, int8_t, uint8_t, 128, INT8_MIN, INT8_MAX)
+
+/* { dg-final { scan-tree-dump-not ".SAT_ADD " "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i16.c
index c244eb4..734e8be 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i16.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int16_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_SUB_FMT_1(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i32.c
index 9d8245d..3aa4c58 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i32.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int32_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srliw\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-7]
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_1(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i64.c
index 929de16..4c0caa1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i64.c
@@ -1,27 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int64_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_1(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i8.c
index a918d5c..6c1441b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-1-i8.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int8_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_SUB_FMT_1(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i16.c
index 2da1c0d..57a4327 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i16.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int16_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_SUB_FMT_2(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i32.c
index 20b28e7..28582fb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i32.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int32_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srliw\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-7]
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_2(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i64.c
index a540198..130ca46 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i64.c
@@ -1,27 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int64_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_2(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i8.c
index c54057d..cd407b2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-2-i8.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int8_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_SUB_FMT_2(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i16.c
index 469a113..748d61a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i16.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int16_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_SUB_FMT_3(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i32.c
index b2c03f6..be7869a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i32.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int32_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srliw\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-7]
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_3(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i64.c
index e3fe6c7..d16a7fb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i64.c
@@ -1,27 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int64_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_3(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i8.c
index 150cde1..14a2454 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-3-i8.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int8_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_SUB_FMT_3(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i16.c
index 26d159c..614d1ec 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i16.c
@@ -1,30 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int16_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*15
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_SUB_FMT_4(int16_t, uint16_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i32.c
index d576c38..2f52bd7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i32.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int32_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srliw\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-7]
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_4(int32_t, uint32_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i64.c
index f42ffea..cef478b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i64.c
@@ -1,27 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int64_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_S_SUB_FMT_4(int64_t, uint64_t, INT64_MIN, INT64_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i8.c
index ee510a6..3ed7790 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-4-i8.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_sub_int8_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*a1
-** xor\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*7
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_SUB_FMT_4(int8_t, uint8_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i16.c
index e248b73..b2c5735 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i32.c
index bebb4be..6d1518e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i64.c
index f31eb29..adcd1bb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i8.c
index e165e39..31fa0a6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-1-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i16.c
index 08a9b5c..0c5ad8c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i32.c
index fc79969..5e89539 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i64.c
index 8d5f745..199e204 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i8.c
index 9f6ef30..4cfe787 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-2-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i16.c
index 0523d13..3cf4ecd 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i32.c
index e720964..ce2151c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i64.c
index 49ed051..158eeaa 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i8.c
index 99b413f..8eb7ab5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-3-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i16.c
index c7056ed..339a403 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i32.c
index 7168f94..285733a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i64.c
index 29b2b54..546bac1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i8.c
index 65027b7..dafc86f1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_sub-run-4-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i16-to-i8.c
index 451a375..6d1fbc4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_1:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_1(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i16.c
index 2aafb94..56a6699 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_1(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i8.c
index 6e21ee3..10c3320 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_1:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_1(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i16.c
index 5e971e4..558d704 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_1(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i32.c
index 87e5a52..02bef46 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_1(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i8.c
index 22a0dd4..da04904 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-1-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_1:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_1(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i16-to-i8.c
index cb307ac..41391e2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_2:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_2(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i16.c
index b4bee21..3e5f9e1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_2:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_2(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i8.c
index c467c8d..228eeab 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_2:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_2(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i16.c
index 883b77b..78542ca 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_2:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_2(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i32.c
index bb9ffce..556e8ea 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_2:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_2(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i8.c
index a54db48..918a8c3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-2-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_2:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_2(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i16-to-i8.c
index 219156c..13c0291 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_3:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_3(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i16.c
index 87b8a70..03077b7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_3(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i8.c
index 7acd515..e09a88d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_3:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_3(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i16.c
index 9141f08..ca071d1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_3(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i32.c
index 839a6f7..4acd93c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_3:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_3(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i8.c
index 5d13f09..362970c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-3-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_3:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_3(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i16-to-i8.c
index 34dc804..94d9cc4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_4:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_4(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i16.c
index 89c476e..51a6e7b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_4:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_4(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i8.c
index 03ca7b7..9101b40 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_4:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_4(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i16.c
index aafe167..48452e3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_4:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_4(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i32.c
index 08e5eb3..6757913 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_4:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_4(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i8.c
index b0e71fe..9c65582 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-4-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_4:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_4(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i16-to-i8.c
index b42c759..f02f866 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_5:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_5(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i16.c
index 625372e..6753c03 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_5:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_5(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i8.c
index 250e174..3fd17fa 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_5:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_5(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i16.c
index 4a6ac6d..fba761a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_5:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_5(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i32.c
index 02aa6db..8872f7f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_5:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_5(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i8.c
index ae1bcb9..13539aa 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-5-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_5:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_5(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i16-to-i8.c
index 9a740d7..4aa9a8f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_6:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_6(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i16.c
index 1e42bfd..a772ee8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_6:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_6(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i8.c
index c3bd46d..9c5d88b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_6:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_6(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i16.c
index a6575f5..f9f18e9 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_6:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_6(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i32.c
index fd7b72e..3658fbb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_6:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_6(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i8.c
index 242d2d0..f1a7eb8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-6-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_6:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_6(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i16-to-i8.c
index 3f258b8..50b06d5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_7:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_7(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i16.c
index f37a57e..12be220 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_7:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_7(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i8.c
index 4e4a7eb..cb73531 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_7:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_7(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i16.c
index 29b64b4..d52394c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_7:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_7(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i32.c
index 2bfe898..cf79778 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_7:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_7(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i8.c
index 494a314..67485a3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-7-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_7:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_7(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i16-to-i8.c
index 678dec6..a34bf4a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i16-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int16_t_to_int8_t_fmt_8:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_8(int8_t, int16_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i16.c
index 4acc789..9c25ff0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int16_t_fmt_8:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_8(int16_t, int32_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i8.c
index 34a992b..9ee75e2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i32-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int32_t_to_int8_t_fmt_8:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_8(int8_t, int32_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i16.c
index 1919ba5..8cd361e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i16.c
@@ -1,28 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int16_t_fmt_8:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** li\s+[atx][0-9]+,\s*-32768
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*16
-** sraiw\s+a0,\s*a0,\s*16
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_8(int16_t, int64_t, INT16_MIN, INT16_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i32.c
index 541e55c..ace064b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i32.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int32_t_fmt_8:
-** li\s+[atx][0-9]+,\s*-2147483648
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xor\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_8(int32_t, int64_t, INT32_MIN, INT32_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i8.c
index 36a0085..e9a4d3b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-8-i64-to-i8.c
@@ -1,26 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_s_trunc_int64_t_to_int8_t_fmt_8:
-** slti\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** li\s+[atx][0-9]+,\s*-128
-** slt\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** srai\s+[atx][0-9]+,\s*[atx][0-9]+,\s*63
-** xori\s+[atx][0-9]+,\s*[atx][0-9]+,\s*127
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** and\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slliw\s+a0,\s*a0,\s*24
-** sraiw\s+a0,\s*a0,\s*24
-** ret
-*/
 DEF_SAT_S_TRUNC_FMT_8(int8_t, int64_t, INT8_MIN, INT8_MAX)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i16-to-i8.c
index 1f230c5..7ed6809 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i16.c
index 563760b..82e4201 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i8.c
index af50d3e..78be831 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i16.c
index 4ac7025..e8a497f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i32.c
index ca6d31c..1420541 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i8.c
index 697e1bc..31fecc7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-1-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i16-to-i8.c
index 0d9da40..333bb92 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i16.c
index 2e183ef..f494909 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i8.c
index 1950092..d8a619b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i16.c
index b11b097..348832d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i32.c
index 419e909..fc183cf 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i8.c
index de3d9f1..dec54d3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-2-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i16-to-i8.c
index 032c83b..2b8700a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i16.c
index 51f4946..cf3f763 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i8.c
index b959bce..20a68bb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i16.c
index ddfb522..5159ab1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i32.c
index 22965e2..edeff90 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i8.c
index 7cba408..7a22637 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-3-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i16-to-i8.c
index 6dfdd4b..65f9aea 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i16.c
index fcf8e47..ab32e5d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i8.c
index 9d911a4..eecfc49 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i16.c
index 3cc2498..410d202 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i32.c
index b9abf50..17518ba 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i8.c
index d90682f..bf0c43e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-4-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i16-to-i8.c
index 1911166..bac1fda 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i16.c
index 28116eb..3a82ea0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i8.c
index 54b1ffb..26a89e7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i16.c
index 633417b..a8bfeef 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i32.c
index c5e4e4a..f79a049 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i8.c
index 9acbee0..eea31af 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-5-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i16-to-i8.c
index db1a698..0ea32f0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i16.c
index e6b52d4..39e44d8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i8.c
index d83836d..cb42b7e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i16.c
index e910edf..f64a46b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i32.c
index 98dd0c2..18e9029 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i8.c
index b843300..d8cda79 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-6-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i16-to-i8.c
index ab51ad5..894d5f5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i16.c
index 9b2c525..1ced757 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i8.c
index ab409f2..ab41a84 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i16.c
index 9013952..c078136 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i32.c
index 67e19e7..af86e69 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i8.c
index a573706..4a2532d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-7-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i16-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i16-to-i8.c
index dbd70de..65c82ad 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i16-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i16-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i16.c
index 25bb42f..a8cb8e1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i8.c
index 7c71b3d..5b5f8f4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i32-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i16.c
index 61392b5..f489846 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i32.c
index b47e5da..a3f3ae5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i8.c
index 1cd7f80..aafe96b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_s_trunc-run-8-i64-to-i8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u16.c
index 3c916bc..8f1b5c0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_1(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u32.c
index edded3e..2c66eee 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_fmt_1:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_1(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u64.c
index 821e4bc..28d7b7c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_FMT_1(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u8.c
index fd73c3a..ab18336 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-1-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint8_t_fmt_1:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_1(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u16.c
index a166d28..c03b15d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_2(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u32.c
index c06731b..f753c01 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_fmt_2:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_2(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u64.c
index ae10dff..cad539c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_FMT_2(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u8.c
index f3977be..b595241 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-2-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint8_t_fmt_2:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_2(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u16.c
index 5898c3b..08cd820 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_3(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u32.c
index a1017c9..e0b73748 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_fmt_3:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_3(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u64.c
index 83fcb60..7ce0121 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_FMT_3(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u8.c
index 2c398e0c..48f61c1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-3-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint8_t_fmt_3:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_3(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u16.c
index c18a5d59..49d5af1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_4(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u32.c
index fa2e55d..20ad476 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_fmt_4:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_4(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u64.c
index 6818c0c..6d2c9a7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_FMT_4(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u8.c
index 1096de8..15e613b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-4-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint8_t_fmt_4:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_4(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u16.c
index fd4be5c..225ba0c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_fmt_5:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_5(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u32.c
index 4fbc807..106baf7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_fmt_5:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_5(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u64.c
index 5bc2948..48e84f6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_fmt_5:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_FMT_5(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u8.c
index 74109c3..9c0d42a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-5-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint8_t_fmt_5:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_5(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u16.c
index 3cb9cbe..0b541e0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_fmt_6:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_6(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u32.c
index fd1cb1a..ee79156 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_fmt_6:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_6(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u64.c
index c968f33..fd79139 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_fmt_6:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_FMT_6(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u8.c
index 9cd95ad..f826aa4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-6-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint8_t_fmt_6:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_6(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u32.c
index 527f8de..446a951 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u32.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_uint16_t_fmt_7:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_7(uint32_t, uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u64.c
index e9031de..626effc 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u16-from-u64.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_uint16_t_fmt_7:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_FMT_7(uint64_t, uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u32-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u32-from-u64.c
index a71bd2f..3014634 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u32-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u32-from-u64.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_uint32_t_fmt_7:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** add\s+[atx][0-9]+,\s*a[01],\s*a[01]
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_FMT_7(uint64_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u16.c
index 5892986..541a1d8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint16_t_uint8_t_fmt_7:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_7(uint16_t, uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u32.c
index a42a712..26749a8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u32.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint32_t_uint8_t_fmt_7:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_7(uint32_t, uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u64.c
index f37ef1c..321f662 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-7-u8-from-u64.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_uint64_t_uint8_t_fmt_7:
-** add\s+[atx][0-9]+,\s*a0,\s*a1
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_FMT_7(uint64_t, uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u16.c
new file mode 100644
index 0000000..a7062b5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u16.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_8(uint16_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u32.c
new file mode 100644
index 0000000..2e43c7f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u32.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_8(uint32_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u64.c
new file mode 100644
index 0000000..4ad18c1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u64.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_8(uint64_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u8.c
new file mode 100644
index 0000000..608d31b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-8-u8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_8(uint8_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u16.c
new file mode 100644
index 0000000..b9766d1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u16.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_9(uint16_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u32.c
new file mode 100644
index 0000000..2456d39
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u32.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_9(uint32_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u64.c
new file mode 100644
index 0000000..0a0ff24
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u64.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_9(uint64_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u8.c
new file mode 100644
index 0000000..53879dd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-9-u8.c
@@ -0,0 +1,9 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+DEF_SAT_U_ADD_FMT_9(uint8_t)
+
+/* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u16.c
index fe015cc..548fae3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u32.c
index 8ee6501..e76b636 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u64.c
index d2c6af0..0ea6509 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u8.c
index 154edde..3aa7441 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-1-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u16.c
index 1fc08bd..f6f8b9d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u32.c
index a52a230..da8c3eb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u64.c
index d05ed33..03f5960 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u8.c
index fd39335..af898e55 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-2-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u16.c
index 7084272..7862a48 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u32.c
index 82de67d..d2fbcf2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u64.c
index d73f305..23b5488 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u8.c
index f572c44..b5931d4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-3-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u16.c
index 65c431f..a9937a7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u32.c
index 8a73fcd..966831a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u64.c
index 0903e10..08db7a1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u8.c
index ffdd390..f7bbb5a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-4-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u16.c
index 72ccd2f..da1782d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u32.c
index 34d1a4e..524106a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u64.c
index d502a58..62fdd25 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u8.c
index f611376..334eb04 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-5-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u16.c
index 5ef250d..28a2fb8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u32.c
index ba95dbf..3b19af3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u64.c
index d0e9dfd..f35334a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u8.c
index b3d00df..e04fbf0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-6-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u32.c
index 26c2778..3363220 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u64.c
index 1f3e2f3..bc4ca2f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u16-from-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u32-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u32-from-u64.c
index 558f6ce..04abd95 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u32-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u32-from-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u16.c
index ec5ac70..c514a86 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u32.c
index aa94eef..b1a644b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u64.c
index 6ac38ba..8664ffa 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-7-u8-from-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u16.c
new file mode 100644
index 0000000..aaf13be
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u16.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint16_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_8_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_8_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u32.c
new file mode 100644
index 0000000..0ec8d90
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u32.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint32_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_8_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_8_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u64.c
new file mode 100644
index 0000000..f367f67
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u64.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint64_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_8_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_8_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u8.c
new file mode 100644
index 0000000..0fd4036
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-8-u8.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint8_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_8_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_8_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u16.c
new file mode 100644
index 0000000..4289e2a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u16.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint16_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_9_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_9_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u32.c
new file mode 100644
index 0000000..d3dd52e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u32.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint32_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_9_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_9_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u64.c
new file mode 100644
index 0000000..a9f0964
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u64.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint64_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_9_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_9_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u8.c
new file mode 100644
index 0000000..91cdb7e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add-run-9-u8.c
@@ -0,0 +1,15 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define T1               uint8_t
+#define DATA             TEST_BINARY_DATA_WRAP(T1, usadd)
+#define T                TEST_BINARY_STRUCT_DECL(T1, usadd)
+
+DEF_SAT_U_ADD_FMT_9_WRAP(T1)
+
+#define RUN_BINARY(x, y) RUN_SAT_U_ADD_FMT_9_WRAP(T1, x, y)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u16.c
index 3c31ac3..b6388dc 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm3_uint16_t_fmt_1:
-** addi\s+[atx][0-9]+,\s*a0,\s*3
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_1(uint16_t, 3)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u32.c
index c6b352c..cae6796 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm7_uint32_t_fmt_1:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*a0,\s*7
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_1(uint32_t, 7)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u64.c
index 1d9df3c..f9d6939 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm8_uint64_t_fmt_1:
-** addi\s+[atx][0-9]+,\s*a0,\s*8
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_1(uint64_t, 8)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u8.c
index 101acd8..d90209a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-1-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm9_uint8_t_fmt_1:
-** addi\s+[atx][0-9]+,\s*a0,\s*9
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_1(uint8_t, 9)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u16.c
index ac57cc9..a34194d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm3_uint16_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*3
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_2(uint16_t, 3)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u32.c
index 6aca60c..9a801d2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm7_uint32_t_fmt_2:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*a0,\s*7
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_2(uint32_t, 7)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u64.c
index d041724..2eb57a3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm8_uint64_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*8
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_2(uint64_t, 8)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u8.c
index 7baeb8d..363b2df8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-2-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm9_uint8_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*9
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_2(uint8_t, 9)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u16.c
index 6dbabf6..aaf1209 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm3_uint16_t_fmt_3:
-** addi\s+[atx][0-9]+,\s*a0,\s*3
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_3(uint16_t, 3)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u32.c
index 1c52b21..e430b37 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm7u_uint32_t_fmt_3:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*a0,\s*7
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_3(uint32_t, 7u)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u64.c
index ef60ce2..aef5c58 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm8ull_uint64_t_fmt_3:
-** addi\s+[atx][0-9]+,\s*a0,\s*8
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_3(uint64_t, 8ull)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u8.c
index 81a4b21..039d982 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-3-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm9_uint8_t_fmt_3:
-** addi\s+[atx][0-9]+,\s*a0,\s*9
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_3(uint8_t, 9)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u16.c
index 2f6c0460..baf70c3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u16.c
@@ -1,21 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm3_uint16_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*3
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*48
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_4(uint16_t, 3)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u32.c
index 1fc9a50..a4bfe50 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm7u_uint32_t_fmt_4:
-** slli\s+[atx][0-9]+,\s*a0,\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*a0,\s*7
-** slli\s+[atx][0-9]+,\s*[atx][0-9],\s*32
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_4(uint32_t, 7u)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u64.c
index 0ca423c..f355de6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm8ull_uint64_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*8
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_4(uint64_t, 8ull)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u8.c
index c8a43fa..54880d7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-4-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_add_imm9_uint8_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*9
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** neg\s+[atx][0-9]+,\s*[atx][0-9]+
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_ADD_IMM_FMT_4(uint8_t, 9)
 
 /* { dg-final { scan-tree-dump-times ".SAT_ADD " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u16.c
index 090c765..e715bb0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u32.c
index 8dade74..8b8b475 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u64.c
index ace2df8..f6f6408 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u8.c
index 0ce546f..f2154fc 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-1-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u16.c
index 7b6bd73..8e3aa83 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u32.c
index 8024152..403cf14 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u64.c
index 4a76dbb..17eca5e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u8.c
index 8e8759c..9a277a1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-2-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u16.c
index 64924a6..2068037 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u32.c
index 04f3217..5f8f1e6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u64.c
index 8ef6c14..c574521 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u8.c
index 8867361..6b9439a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-3-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u16.c
index 0b75206..224c3ae 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u32.c
index e548d0c..5c03e1b7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u64.c
index 4335d82..1ceacd2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u8.c
index 872923e..aef253c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_add_imm-run-4-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u128.c
new file mode 100644
index 0000000..cd6f2f8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u128.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint16_t
+#define WT uint128_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u32.c
new file mode 100644
index 0000000..7409232
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u32.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint16_t
+#define WT uint32_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u64.c
new file mode 100644
index 0000000..43ab563
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u16-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv32gc -mabi=ilp32d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint16_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u32-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u32-from-u128.c
new file mode 100644
index 0000000..dea9f6d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u32-from-u128.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint32_t
+#define WT uint128_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u32-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u32-from-u64.c
new file mode 100644
index 0000000..8d5449b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u32-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv32gc -mabi=ilp32d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint32_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u64-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u64-from-u128.c
new file mode 100644
index 0000000..d8a01d1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u64-from-u128.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint64_t
+#define WT uint128_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u128.c
new file mode 100644
index 0000000..dfc9d2e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u128.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint8_t
+#define WT uint128_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u16.c
new file mode 100644
index 0000000..ec79e5d
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u16.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint8_t
+#define WT uint16_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u32.c
new file mode 100644
index 0000000..eb95184
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u32.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint8_t
+#define WT uint32_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u64.c
new file mode 100644
index 0000000..ee41593
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-1-u8-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv32gc -mabi=ilp32d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint8_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u16-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u16-from-u64.c
new file mode 100644
index 0000000..b1d33a9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u16-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint16_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u32-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u32-from-u64.c
new file mode 100644
index 0000000..af5ffecf
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u32-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint32_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u8-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u8-from-u64.c
new file mode 100644
index 0000000..d65cab0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-2-u8-from-u64.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
+
+#include "sat_arith.h"
+
+#define NT uint8_t
+#define WT uint64_t
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+/* { dg-final { scan-tree-dump-times ".SAT_MUL" 1 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u128.c
new file mode 100644
index 0000000..79f6297
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u128.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint16_t
+#define WT               uint128_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u32.c
new file mode 100644
index 0000000..e212391
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u32.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint16_t
+#define WT               uint32_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c
new file mode 100644
index 0000000..79d3fb3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u16-from-u64.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint16_t
+#define WT               uint64_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u128.c
new file mode 100644
index 0000000..e5a9462
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u128.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint32_t
+#define WT               uint128_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c
new file mode 100644
index 0000000..ad63db3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u32-from-u64.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint32_t
+#define WT               uint64_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u64-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u64-from-u128.c
new file mode 100644
index 0000000..cbe2a22
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u64-from-u128.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint64_t
+#define WT               uint128_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u128.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u128.c
new file mode 100644
index 0000000..1f54c30
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u128.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint8_t
+#define WT               uint128_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u16.c
new file mode 100644
index 0000000..f5a0ab5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u16.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint8_t
+#define WT               uint16_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u32.c
new file mode 100644
index 0000000..32074a4
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u32.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint8_t
+#define WT               uint32_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c
new file mode 100644
index 0000000..16ca905
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_mul-run-1-u8-from-u64.c
@@ -0,0 +1,16 @@
+/* { dg-do run { target { rv32 || rv64 } } } */
+/* { dg-additional-options "-std=c99" } */
+
+#include "sat_arith.h"
+#include "sat_arith_data.h"
+
+#define NT               uint8_t
+#define WT               uint64_t
+#define NAME             usmul
+#define DATA             TEST_BINARY_DATA_WRAP(NT, NAME)
+#define T                TEST_BINARY_STRUCT_DECL_WRAP(NT, NAME)
+#define RUN_BINARY(x, y) RUN_SAT_U_MUL_FMT_1_WRAP(NT, WT, x, y)
+
+DEF_SAT_U_MUL_FMT_1_WRAP(NT, WT)
+
+#include "scalar_sat_binary_run_xxx.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u16.c
index eb140ae..66a439e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_1(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u32.c
index 59ad242..6f40907 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_1:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_1(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u64.c
index 47a8382..647fc6d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_1(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u8.c
index f01317b..a344c58 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-1-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_1:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_1(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u16.c
index 4b7bd3a..87fb1fc 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_10:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_10(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u32.c
index a28213f..280236a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_10:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_10(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u64.c
index 432da0c..4b7d339 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_10:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_10(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u8.c
index 0658d38..191c3a5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-10-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_10:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_10(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u16.c
index 2e4b875..9dc41e1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_11:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_11(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u32.c
index 61fb80f..475f944 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_11:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_11(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u64.c
index 2a28b1f..61e3584 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_11:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_11(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u8.c
index 3033844..7a61055 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-11-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_11:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_11(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u16.c
index 9cb86df..c4d21cb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_12:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_12(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u32.c
index babe768..56beb83 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_12:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_12(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u64.c
index 294ef5a..1bef3fe 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_12:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_12(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u8.c
index 8b8f924..9004281 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-12-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_12:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_12(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u16.c
index e724752..7b85582 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_2(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u32.c
index 9240406..cfdf66c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_2:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_2(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u64.c
index 3e1efba..3898817 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_2(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u8.c
index 600688a..3318211 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-2-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_2:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_2(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u16.c
index bb2d0b7..61bb5e5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_3(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u32.c
index 06635df..73bfa99 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_3:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_3(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u64.c
index ac485da..24d1e69 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_3(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u8.c
index cdc8776..5523112 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-3-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_3:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_3(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u16.c
index 407ff8f..fb6a604 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_4(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u32.c
index cb2cd05..0f7e2d3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_4:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_4(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u64.c
index 0ce6269..c762647 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_4(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u8.c
index 302206a..3e5d2e6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-4-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_4:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_4(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u16.c
index ce2758f..ab1b375 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_5:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_5(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u32.c
index d33cef3..1b8ce84 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_5:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_5(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u64.c
index 1bf1e97..3fc4e7a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_5:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_5(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u8.c
index b2ed732..5c34ead 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-5-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_5:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_5(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u16.c
index 20614ec..70dc6ec 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_6:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_6(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u32.c
index 5d7adfd..cc36036 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_6:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_6(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u64.c
index b3c6f8d..ea633ff 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_6:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_6(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u8.c
index a4f92a8..7c4747a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-6-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_6:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_6(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u16.c
index ebfe673..cac8471 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_7:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_7(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u32.c
index 9884123..18b8e5f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_7:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_7(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u64.c
index 67236d5..f5ade61 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_7:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_7(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u8.c
index 549d9d2..9b528a4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-7-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_7:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_7(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u16.c
index aa5aec7..0d093c3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_8:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_8(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u32.c
index 89a8cc9..f04ea1d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_8:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_8(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u64.c
index a52948d..17dd8f3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_8:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_8(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u8.c
index 5606733..b043207 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-8-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_8:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_8(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u16.c
index 984867a..19b1a5b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u16.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint16_t_fmt_9:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_SUB_FMT_9(uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u32.c
index d1109a4..a0026a1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u32.c
@@ -1,22 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint32_t_fmt_9:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** slli\s+a1,\s*a1,\s*32
-** srli\s+a1,\s*a1,\s*32
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_SUB_FMT_9(uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u64.c
index a9acf15..01c155e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint64_t_fmt_9:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*a0,\s*a1
-** addi\s+a0,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*[atx][0-9]+,\s*[atx][0-9]+
-** ret
-*/
 DEF_SAT_U_SUB_FMT_9(uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u8.c
index 47551fa..7b94d40 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-9-u8.c
@@ -1,18 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_uint8_t_fmt_9:
-** sub\s+[atx][0-9]+,\s*a0,\s*a1
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 DEF_SAT_U_SUB_FMT_9(uint8_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u16.c
index 1534cf9..20e14d6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u32.c
index 5c60d28..1a0c394 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u64.c
index 403764c..ee348b3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u8.c
index 931420a..216af86 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-1-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u16.c
index ae87544..109539d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u32.c
index 43414ae..9e35fa2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u64.c
index 3ef70a1..3c7c8db 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u8.c
index 2a157f0..df291e2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-10-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u16.c
index 534795c..88dded4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u32.c
index 4d0a34f..239b422 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u64.c
index d74d10d..9a524fd 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u8.c
index 949bd0d..b9b84ea 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-11-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u16.c
index 80cce95..91bd9de 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u32.c
index 3ecd19c..eaaa256 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u64.c
index 2d7bfc4..04d2a20 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u8.c
index 209965c..caedfe7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-12-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u16.c
index 7deaae9a5..06a44f1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u32.c
index d9b1d5c..9d38c9c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u64.c
index 2774c23..5c10409 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u8.c
index 6fa44ca..0ff9827 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-2-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u16.c
index ea52ff4..aab99ca 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u32.c
index fdea891..5231d6f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u64.c
index 164ee77..d7462a8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u8.c
index 724adf9..5da7838 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-3-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u16.c
index 9b57861..8e69888 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u32.c
index df2eece..9b22dda 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u64.c
index 09e9ac3..abd0a95 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u8.c
index c8ae7a6..d92c0e1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-4-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u16.c
index 9f575a47..b404bfd 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u32.c
index c370455..b746712 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u64.c
index 22d82f9..da90b7a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u8.c
index b282311..38dcabe 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-5-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u16.c
index e0dda45..fd55bec 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u32.c
index dfd95ef..2e810dd 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u64.c
index 7cac446..e86eebc 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u8.c
index 0b4cbdb..e749bb5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-6-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u16.c
index 10c65fe..eb57d55 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u32.c
index e3b4dde..c1a5bcf 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u64.c
index 6e93fcf..27d4b82 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u8.c
index d101d28..feb56e1 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-7-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u16.c
index 4e50e3f..a22f1df 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u32.c
index 3c8f78d..b98931d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u64.c
index 932596a..dff3c0a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u8.c
index 1f74562..d2f3126 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-8-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u16.c
index 66a82f2..3740099 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u32.c
index a54b5c3..b6ae459 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u64.c
index 97943b3e..55198d6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u8.c
index ab8b475..ce73d26 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub-run-9-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-1.c
index 573ef11..475b31e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-1.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm32768_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*32768
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint16_t, 32768)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-2.c
index 0fefbe7..a984f84b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-2.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm65533_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-3
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint16_t, 65533)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-3.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-3.c
index ad6d4f9..b2930d4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-3.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-3.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm65534_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-2
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint16_t, 65534)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-4.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-4.c
index 02dcbc5..362cf48 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-4.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16-4.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint16_t, 1)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16.c
index 7346fbb..9f17082 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u16.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm6_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*6
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint16_t, 6)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-1.c
index c7dac8a..801a86e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-1.c
@@ -1,23 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm2147483648_uint32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint32_t, 2147483648)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-2.c
index 4320db3..e044768 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-2.c
@@ -1,24 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm68719476732_uint32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-4
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint32_t, 68719476732)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-3.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-3.c
index 765d13c..5518064 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-3.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-3.c
@@ -1,24 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm4294967294_uint32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-2
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint32_t, 4294967294)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-4.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-4.c
index ca11cf1..a4cb49b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-4.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32-4.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint32_t, 1)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32.c
index 3711930..64808bf 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u32.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm255_uint32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*255
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint32_t, 255)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-1.c
index 2e490f0..493a14d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-1.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm18446744073709551614u_uint64_t_fmt_1:
-** li\s+[atx][0-9]+,\s*-2
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint64_t, 18446744073709551614u)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-2.c
index 45baa8f..4faae52 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64-2.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint64_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint64_t, 1)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64.c
index a29a6e9..3f993fd 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u64.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm82_uint64_t_fmt_1:
-** li\s+[atx][0-9]+,\s*82
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint64_t, 82)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-1.c
index d1c6e94..a0d9235 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-1.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm128_uint8_t_fmt_1:
-** li\s+[atx][0-9]+,\s*128
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint8_t, 128)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-2.c
index 4c8cf90..67dae03 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-2.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm253_uint8_t_fmt_1:
-** li\s+[atx][0-9]+,\s*253
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint8_t, 253)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-3.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-3.c
index b958f5e..0054532 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-3.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-3.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm254_uint8_t_fmt_1:
-** li\s+[atx][0-9]+,\s*254
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint8_t, 254)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-4.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-4.c
index 1951ec5..c12b560 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-4.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8-4.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint8_t_fmt_1:
-** li\s+[atx][0-9]+,\s*1
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint8_t, 1)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8.c
index 86d0b39..ce9f495 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-1-u8.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm11_uint8_t_fmt_1:
-** li\s+[atx][0-9]+,\s*11
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_1(uint8_t, 11)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-1.c
index 31c1bb8..93d7169 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-1.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm32768_uint16_t_fmt_2:
-** li\s+[atx][0-9]+,\s*32768
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint16_t, 32768)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-2.c
index 68807b9..8ac2ce8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-2.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm65533_uint16_t_fmt_2:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-3
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint16_t, 65533)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-3.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-3.c
index 62deec1..740d6ac 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-3.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16-3.c
@@ -1,18 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint16_t_fmt_2:
-** snez\s+[atx][0-9]+,\s*a0
-** subw\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint16_t, 1)
 
 /* { dg-final { scan-tree-dump-not ".SAT_SUB" "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16.c
index f789fee..c82c478 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u16.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm6_uint16_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*-6
-** sltiu\s+a0,\s*[atx][0-9]+,\s*6
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint16_t, 6)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-1.c
index 2f4a439..b2f690a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-1.c
@@ -1,23 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm2147483648_uint32_t_fmt_2:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint32_t, 2147483648)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-2.c
index dcfba62..e62010b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-2.c
@@ -1,24 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm68719476732_uint32_t_fmt_2:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-4
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint32_t, 68719476732)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-3.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-3.c
index a3f48f7..dd063d8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-3.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32-3.c
@@ -1,16 +1,8 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint32_t_fmt_2:
-** snez\s+[atx][0-9]+,\s*a0
-** subw\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
-
 DEF_SAT_U_SUB_IMM_FMT_2(uint32_t, 1)
 
 /* { dg-final { scan-tree-dump-not ".SAT_SUB" "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32.c
index 0bd8ddc..c0eb8a7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u32.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm255_uint32_t_fmt_2:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** addi\s+[atx][0-9]+,\s*a0,\s*-255
-** sltiu\s+a0,\s*[atx][0-9]+,\s*255
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint32_t, 255)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64-1.c
index 7b6d857..ed69313 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64-1.c
@@ -1,16 +1,8 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint64_t_fmt_2:
-** snez\s+[atx][0-9]+,\s*a0
-** sub\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
-
 DEF_SAT_U_SUB_IMM_FMT_2(uint64_t, 1)
 
 /* { dg-final { scan-tree-dump-not ".SAT_SUB" "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64.c
index c334665..fb7db13 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u64.c
@@ -1,18 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm82_uint64_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-82
-** sltiu\s+a0,\s*[atx][0-9]+,\s*82
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint64_t, 82)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-1.c
index 26e77f0..efe6c00 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-1.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm128_uint8_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*-128
-** sltiu\s+a0,\s*[atx][0-9]+,\s*128
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint8_t, 128)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-2.c
index c5ac1b0..1262648 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-2.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm253_uint8_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*-253
-** sltiu\s+a0,\s*[atx][0-9]+,\s*253
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint8_t, 253)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-3.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-3.c
index ee59b5a..108daf2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-3.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8-3.c
@@ -1,17 +1,8 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm1_uint8_t_fmt_2:
-** snez\s+[atx][0-9]+,\s*a0
-** subw\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
-
 DEF_SAT_U_SUB_IMM_FMT_2(uint8_t, 1)
 
 /* { dg-final { scan-tree-dump-not ".SAT_SUB" "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8.c
index 69dcc2a..784a97b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-2-u8.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm11_uint8_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*a0,\s*-11
-** sltiu\s+a0,\s*[atx][0-9]+,\s*11
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint8_t, 11)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-1.c
index f312362..0f16f9c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-1.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm32769_uint16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*32768
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint16_t, 32769)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-2.c
index fa9a9ef..49daab5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16-2.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm65533_uint16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-3
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint16_t, 65533)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16.c
index b98de41..30fc2bf 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u16.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm6_uint16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*6
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint16_t, 6)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-1.c
index 79457a3..2d3c63d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-1.c
@@ -1,24 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm2147483649_uint32_t_fmt_3:
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*1
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint32_t, 2147483649)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-2.c
index 2e8426e..8d96c00 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32-2.c
@@ -1,24 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm68719476732_uint32_t_fmt_3:
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-4
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint32_t, 68719476732)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32.c
index 845218c..c06c441 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u32.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm255_uint32_t_fmt_3:
-** li\s+[atx][0-9]+,\s*255
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint32_t, 255)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u64.c
index ee2fbf8..4d2b96d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u64.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm82_uint64_t_fmt_3:
-** li\s+[atx][0-9]+,\s*82
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint64_t, 82)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-1.c
index 8cc81e2..8c3eb14 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-1.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm134_uint8_t_fmt_3:
-** li\s+[atx][0-9]+,\s*134
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint8_t, 134)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-2.c
index 8d8c70b..b02d832 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8-2.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm253_uint8_t_fmt_3:
-** li\s+[atx][0-9]+,\s*253
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint8_t, 253)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8.c
index 348d75b..d8e0a69 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-3-u8.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm11_uint8_t_fmt_3:
-** li\s+[atx][0-9]+,\s*11
-** sub\s+[atx][0-9]+,\s*[atx][0-9]+,\s*a0
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_3(uint8_t, 11)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-1.c
index 089c168..8f3726f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-1.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm32768_uint16_t_fmt_4:
-** li\s+[atx][0-9]+,\s*32768
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint16_t, 32768)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-2.c
index b96e3f3..56c377e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16-2.c
@@ -1,22 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm65533_uint16_t_fmt_4:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-3
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint16_t, 65533)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16.c
index 5c209bc..29c6b86 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u16.c
@@ -1,20 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm6_uint16_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*-6
-** sltiu\s+a0,\s*[atx][0-9]+,\s*6
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint16_t, 6)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-1.c
index 2f4a439..b2f690a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-1.c
@@ -1,23 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm2147483648_uint32_t_fmt_2:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*31
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint32_t, 2147483648)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-2.c
index dcfba62..e62010b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32-2.c
@@ -1,24 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm68719476732_uint32_t_fmt_2:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** li\s+[atx][0-9]+,\s*1
-** slli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-4
-** sub\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** sltu\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint32_t, 68719476732)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32.c
index ee1ad9a..6cfb1e4c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u32.c
@@ -1,21 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm255_uint32_t_fmt_4:
-** slli\s+a0,\s*a0,\s*32
-** srli\s+a0,\s*a0,\s*32
-** addi\s+[atx][0-9]+,\s*a0,\s*-255
-** sltiu\s+a0,\s*[atx][0-9]+,\s*255
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** sext\.w\s+a0,\s*a0
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint32_t, 255)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u64.c
index c334665..fb7db13 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u64.c
@@ -1,18 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm82_uint64_t_fmt_2:
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-82
-** sltiu\s+a0,\s*[atx][0-9]+,\s*82
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_2(uint64_t, 82)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-1.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-1.c
index 3fe4103..49a4150 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-1.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-1.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm128_uint8_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*-128
-** sltiu\s+a0,\s*[atx][0-9]+,\s*128
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint8_t, 128)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-2.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-2.c
index 18dc505..1022de2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-2.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8-2.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm253_uint8_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*-253
-** sltiu\s+a0,\s*[atx][0-9]+,\s*253
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint8_t, 253)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8.c
index 5c40f32..48aaeb2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-4-u8.c
@@ -1,19 +1,10 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_sub_imm11_uint8_t_fmt_4:
-** addi\s+[atx][0-9]+,\s*a0,\s*-11
-** sltiu\s+a0,\s*[atx][0-9]+,\s*11
-** addi\s+a0,\s*a0,\s*-1
-** and\s+a0,\s*a0,\s*[atx][0-9]+
-** andi\s+a0,\s*a0,\s*0xff
-** ret
-*/
 
 DEF_SAT_U_SUB_IMM_FMT_4(uint8_t, 11)
 
 /* { dg-final { scan-tree-dump-times ".SAT_SUB " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u16.c
index 2bc3be3..a193d88 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u32.c
index b1d1ee3..e1dd81c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u64.c
index 2539d75..a71526c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u8.c
index 5091872..4fedf96 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-1-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u16.c
index 0f4f9e4..f990c43 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u32.c
index ea15d85..44d5e88 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u64.c
index 612da92..91ea986 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u8.c
index fc38095..7da49eb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-2-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u16.c
index 150ab2a..8c44ee0 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u32.c
index c7d2850..f5c4e5a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u64.c
index 6bf5cd2..393f7f6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u8.c
index dfef1f2..e46463b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-3-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u16.c
index 610e021..3062e0f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u32.c
index 1d9e0cb..e621cd2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u64.c
index f864a67..cfc96bf 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u8.c
index 603f2ee..771ec4a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_sub_imm-run-4-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u16.c
index b73290a..d368621 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u16.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_1(uint16_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u32.c
index 8af803f..02ca992 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u32.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint32_t_fmt_1:
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_1(uint32_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u64.c
index 1c887d4..cc01abd 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint8_t_fmt_1:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_1(uint8_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u8.c
index 6bcf64b..e28ee5c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-1-u8.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint16_t_to_uint8_t_fmt_1:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_1(uint8_t, uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u16.c
index 8a35e72..59302cb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u16.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint16_t_fmt_1:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_1(uint16_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u32.c
index a3b52de..735ea7e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u32.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint16_t_to_uint8_t_fmt_2:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_2(uint8_t, uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u64.c
index b9b43f1..8fd3f43 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u64.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint16_t_fmt_2:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_2(uint16_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u8.c
index 7ed3623..bb4ecc5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-2-u8.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint8_t_fmt_1:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_1(uint8_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u16.c
index 7572c9e..e476897 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u16.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint8_t_fmt_2:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_2(uint8_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u32.c
index d83b5dd..524d625 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u32.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint8_t_fmt_2:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_2(uint8_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u64.c
index b7202f9..ba8b238 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u64.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint16_t_fmt_2:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_2(uint16_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u8.c
index e90b853..cba8573 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-3-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint32_t_fmt_2:
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_2(uint32_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u16.c
index e8655b9..5852028 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u16.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_3(uint16_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u32.c
index 41e676a..5d5cf97 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u32.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint32_t_fmt_3:
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_3(uint32_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u64.c
index 32eeb88..866e240 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u64.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint8_t_fmt_3:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_3(uint8_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u8.c
index 5d043ce..f3adfb6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-4-u8.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint16_t_to_uint8_t_fmt_3:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_3(uint8_t, uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u16.c
index 7e5906b..4e132a9 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u16.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint16_t_fmt_3:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_3(uint16_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u32.c
index e1b0acd..893f43e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u32.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint16_t_to_uint8_t_fmt_4:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_4(uint8_t, uint16_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u64.c
index 618d50bd..5c0c7a7e 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u64.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint16_t_fmt_4:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_4(uint16_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u8.c
index c9a9a4c..395bb1b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-5-u8.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint8_t_fmt_3:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_3(uint8_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u16.c
index 418cdc8..8f20c8f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u16.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint32_t_to_uint8_t_fmt_4:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_4(uint8_t, uint32_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u32.c
index 4903a04..f7e7ff2 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u32.c
@@ -1,17 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint8_t_fmt_4:
-** sltiu\s+[atx][0-9]+,\s*a0,\s*255
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** andi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*0xff
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_4(uint8_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u64.c
index 6f8191c..2d9b6a6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u64.c
@@ -1,20 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint16_t_fmt_4:
-** li\s+[atx][0-9]+,\s*65536
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** slli\s+a0,\s*a0,\s*48
-** srli\s+a0,\s*a0,\s*48
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_4(uint16_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u8.c
index 24bb846..4fa81fe 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-6-u8.c
@@ -1,19 +1,9 @@
 /* { dg-do compile } */
-/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized -fno-schedule-insns -fno-schedule-insns2" } */
-/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-options "-march=rv64gc -mabi=lp64d -fdump-tree-optimized" } */
 
 #include "sat_arith.h"
 
-/*
-** sat_u_trunc_uint64_t_to_uint32_t_fmt_4:
-** li\s+[atx][0-9]+,\s*-1
-** srli\s+[atx][0-9]+,\s*[atx][0-9]+,\s*32
-** sltu\s+[atx][0-9]+,\s*a0,\s*[atx][0-9]+
-** addi\s+[atx][0-9]+,\s*[atx][0-9]+,\s*-1
-** or\s+[atx][0-9]+,\s*[atx][0-9]+,\s*[atx][0-9]+
-** sext.w\s+a0,\s*a0
-** ret
-*/
 DEF_SAT_U_TRUNC_FMT_4(uint32_t, uint64_t)
 
 /* { dg-final { scan-tree-dump-times ".SAT_TRUNC " 1 "optimized" } } */
+/* { dg-final { scan-assembler-not "\.L\[0-9\]+" } } */
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u16.c
index a5f43e9..72c175c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u32.c
index a76ae08..aef195a 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u64.c
index d05ea79..4517418 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u8.c
index adaa421..2e51023 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-1-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u16.c
index 38fcba3..8ea83d6 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u32.c
index 93705f9..1d0dd5b 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u64.c
index c116484..f69968c 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u8.c
index 4fbdc91..dcff0b4 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-2-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u16.c
index 2281610..33f46ec 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u32.c
index 126c97c..b9c4617 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u64.c
index 61ad79d..21755a7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u8.c
index 4142e87..bcf2081 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-3-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u16.c
index 8952c06..69f5352 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u32.c
index 8952c06..69f5352 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u64.c
index 20ceda6..f001c39 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u8.c
index 7011e50..1394d9f 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-4-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u16.c
index e868da1..de5d723 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u32.c
index 7f52283fb..c345bfa 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u64.c
index ee13f0a..8ca8cc7 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u8.c
index 8471c76..54e00e8 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-5-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u16.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u16.c
index f056bd4..a957cc3 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u16.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u16.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u32.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u32.c
index 96c06eb..9691b4d 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u32.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u32.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u64.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u64.c
index 1623e52..ff2c2a5 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u64.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u64.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u8.c b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u8.c
index a1b8a5f..918eabb 100644
--- a/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u8.c
+++ b/gcc/testsuite/gcc.target/riscv/sat/sat_u_trunc-run-6-u8.c
@@ -1,4 +1,4 @@
-/* { dg-do run { target { riscv_v } } } */
+/* { dg-do run { target { rv32 || rv64 } } } */
 /* { dg-additional-options "-std=c99" } */
 
 #include "sat_arith.h"
diff --git a/gcc/testsuite/gcc.target/riscv/xtheadint-push-pop.c b/gcc/testsuite/gcc.target/riscv/xtheadint-push-pop.c
index dc5609c..167fa15 100644
--- a/gcc/testsuite/gcc.target/riscv/xtheadint-push-pop.c
+++ b/gcc/testsuite/gcc.target/riscv/xtheadint-push-pop.c
@@ -20,12 +20,6 @@ void func_machine (void)
 /* { dg-final { scan-assembler-times {\mth\.ipop\M} 2 { target { rv32 } } } } */
 
 
-__attribute__ ((interrupt ("user")))
-void func_usr (void)
-{
-  f ();
-}
-
 __attribute__ ((interrupt ("supervisor")))
 void func_supervisor (void)
 {
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-1.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-1.c
new file mode 100644
index 0000000..671c0ed
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-1.c
@@ -0,0 +1,103 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -march=z13 -mzarch" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/*
+** test_in_1:
+**     foo	%r2
+**     br	%r14
+*/
+
+int
+test_in_1 (int x)
+{
+  asm ("foo	%0" :: "{r2}" (x));
+  return x;
+}
+
+/*
+** test_in_2:
+**     lgr	(%r[0-9]+),%r2
+**     lr	%r2,%r3
+**     foo	%r2
+**     lgr	%r2,\1
+**     br	%r14
+*/
+
+int
+test_in_2 (int x, int y)
+{
+  asm ("foo	%0" :: "{r2}" (y));
+  return x;
+}
+
+/*
+** test_in_3:
+**     stmg	%r12,%r15,96\(%r15\)
+**     lay	%r15,-160\(%r15\)
+**     lgr	(%r[0-9]+),%r2
+**     ahi	%r2,1
+**     lgfr	%r2,%r2
+**     brasl	%r14,foo@PLT
+**     lr	%r3,%r2
+**     lr	%r2,\1
+**     foo	%r3,%r2
+**     lgr	%r2,\1
+**     lmg	%r12,%r15,256\(%r15\)
+**     br	%r14
+*/
+
+extern int foo (int);
+
+int
+test_in_3 (int x)
+{
+  asm ("foo	%0,%1\n" :: "{r3}" (foo (x + 1)), "{r2}" (x));
+  return x;
+}
+
+/*
+** test_out_1:
+**     foo	%r3
+**     lgfr	%r2,%r3
+**     br	%r14
+*/
+
+int
+test_out_1 (void)
+{
+  int x;
+  asm ("foo	%0" : "={r3}" (x));
+  return x;
+}
+
+/*
+** test_out_2:
+**     lgr	(%r[0-9]+),%r2
+**     foo	%r2
+**     ark	(%r[0-9]+),\1,%r2
+**     lgfr	%r2,\2
+**     br	%r14
+*/
+
+int
+test_out_2 (int x)
+{
+  int y;
+  asm ("foo	%0" : "={r2}" (y));
+  return x + y;
+}
+
+/*
+** test_inout_1:
+**     foo	%r2
+**     lgfr	%r2,%r2
+**     br	%r14
+*/
+
+int
+test_inout_1 (int x)
+{
+  asm ("foo	%0" : "+{r2}" (x));
+  return x;
+}
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-2.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-2.c
new file mode 100644
index 0000000..a892fe8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-2.c
@@ -0,0 +1,43 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -march=z13 -mzarch" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+/* { dg-final { scan-assembler {\.LC0:\n\t\.long\t1078523331\n} } } */
+
+
+/*
+** test_float_into_gpr:
+**     lrl	%r4,.LC0
+**     foo	%r4
+**     br	%r14
+*/
+
+void
+test_float_into_gpr (void)
+{
+  // This is the counterpart to
+  //   register float x asm ("r4") = 3.14f;
+  //   asm ("foo	%0" :: "r" (x));
+  // where the bit-pattern of 3.14f is loaded into GPR.
+  asm ("foo	%0" :: "{r4}" (3.14f));
+}
+
+/*
+** test_float:
+** (
+**     ldr	%f4,%f0
+**     ldr	%f5,%f2
+** |
+**     ldr	%f5,%f2
+**     ldr	%f4,%f0
+** )
+**     aebr	%f5,%f4
+**     ldr	%f0,%f5
+**     br	%r14
+*/
+
+float
+test_float (float x, float y)
+{
+  asm ("aebr	%0,%1" : "+{f5}" (y) : "{f4}" (x));
+  return y;
+}
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-3.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-3.c
new file mode 100644
index 0000000..5df37b5
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-3.c
@@ -0,0 +1,42 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -march=z13 -mzarch" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+/* { dg-final { scan-assembler {\.LC0:\n\t\.long\t1074339512\n\t\.long\t1374389535\n} } } */
+
+/*
+** test_double_into_gpr:
+**     lgrl	%r4,.LC0
+**     foo	%r4
+**     br	%r14
+*/
+
+void
+test_double_into_gpr (void)
+{
+  // This is the counterpart to
+  //   register double x asm ("r4") = 3.14;
+  //   asm ("foo	%0" :: "r" (x));
+  // where the bit-pattern of 3.14 is loaded into GPR.
+  asm ("foo	%0" :: "{r4}" (3.14));
+}
+
+/*
+** test_double:
+** (
+**     ldr	%f4,%f0
+**     ldr	%f5,%f2
+** |
+**     ldr	%f5,%f2
+**     ldr	%f4,%f0
+** )
+**     adbr	%f5,%f4
+**     ldr	%f0,%f5
+**     br	%r14
+*/
+
+double
+test_double (double x, double y)
+{
+  asm ("adbr	%0,%1" : "+{f5}" (y) : "{f4}" (x));
+  return y;
+}
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-4.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-4.c
new file mode 100644
index 0000000..29927ce
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-4.c
@@ -0,0 +1,6 @@
+/* { dg-do run { target lp64 } } */
+/* { dg-options "-O2 -march=z13 -mzarch" } */
+
+/* Test TARGET_MD_ASM_ADJUST for z13 and long double.  */
+
+#include "asm-hard-reg-longdouble.h"
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-5.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-5.c
new file mode 100644
index 0000000..eaf34d9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-5.c
@@ -0,0 +1,6 @@
+/* { dg-do run { target lp64 } } */
+/* { dg-options "-O2 -march=z14 -mzarch" } */
+
+/* Test TARGET_MD_ASM_ADJUST for z14 and long double.  */
+
+#include "asm-hard-reg-longdouble.h"
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-6.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-6.c
new file mode 100644
index 0000000..d012966
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-6.c
@@ -0,0 +1,152 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+void
+test (void)
+{
+  // GPRs
+  {
+    int a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p;
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14"
+        : "=r" (a),
+          "=r" (b),
+          "=r" (c),
+          "=r" (d),
+          "=r" (e),
+          "=r" (f),
+          "=r" (g),
+          "=r" (h),
+          "=r" (i),
+          "=r" (j),
+          "=r" (k),
+          "=r" (l),
+          "=r" (m),
+          "=r" (n),
+          "=r" (o));
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14"
+        : "={r0}" (a),
+          "={r1}" (b),
+          "={r2}" (c),
+          "={r3}" (d),
+          "={r4}" (e),
+          "={r5}" (f),
+          "={r6}" (g),
+          "={r7}" (h),
+          "={r8}" (i),
+          "={r9}" (j),
+          "={r10}" (k),
+          "={r11}" (l),
+          "={r12}" (m),
+          "={r13}" (n),
+          "={r14}" (o));
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14 %15" /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+        : "=r" (a),
+          "=r" (b),
+          "=r" (c),
+          "=r" (d),
+          "=r" (e),
+          "=r" (f),
+          "=r" (g),
+          "=r" (h),
+          "=r" (i),
+          "=r" (j),
+          "=r" (k),
+          "=r" (l),
+          "=r" (m),
+          "=r" (n),
+          "=r" (o),
+          "=r" (p));
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14 %15" /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+        : "=r" (a),
+          "=r" (b),
+          "=r" (c),
+          "=r" (d),
+          "=r" (e),
+          "=r" (f),
+          "=r" (g),
+          "=r" (h),
+          "=r" (i),
+          "=r" (j),
+          "=r" (k),
+          "=r" (l),
+          "=r" (m),
+          "=r" (n),
+          "=r" (o),
+          "={r4}" (p));
+  }
+
+  // FPRs
+  {
+    float a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q;
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14 %15"
+        : "=f" (a),
+          "=f" (b),
+          "=f" (c),
+          "=f" (d),
+          "=f" (e),
+          "=f" (f),
+          "=f" (g),
+          "=f" (h),
+          "=f" (i),
+          "=f" (j),
+          "=f" (k),
+          "=f" (l),
+          "=f" (m),
+          "=f" (n),
+          "=f" (o),
+          "=f" (p));
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14 %15"
+        : "={f0}" (a),
+          "={f1}" (b),
+          "={f2}" (c),
+          "={f3}" (d),
+          "={f4}" (e),
+          "={f5}" (f),
+          "={f6}" (g),
+          "={f7}" (h),
+          "={f8}" (i),
+          "={f9}" (j),
+          "={f10}" (k),
+          "={f11}" (l),
+          "={f12}" (m),
+          "={f13}" (n),
+          "={f14}" (o),
+          "={f15}" (p));
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14 %15 %16" /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+        : "=f" (a),
+          "=f" (b),
+          "=f" (c),
+          "=f" (d),
+          "=f" (e),
+          "=f" (f),
+          "=f" (g),
+          "=f" (h),
+          "=f" (i),
+          "=f" (j),
+          "=f" (k),
+          "=f" (l),
+          "=f" (m),
+          "=f" (n),
+          "=f" (o),
+          "=f" (p),
+          "=f" (q));
+    __asm__ __volatile__ ("%0 %1 %2 %3 %4 %5 %6 %7 %8 %9 %10 %11 %12 %13 %14 %15 %16" /* { dg-error "'asm' operand has impossible constraints or there are not enough registers" } */
+        : "=f" (a),
+          "=f" (b),
+          "=f" (c),
+          "=f" (d),
+          "=f" (e),
+          "=f" (f),
+          "=f" (g),
+          "=f" (h),
+          "=f" (i),
+          "=f" (j),
+          "=f" (k),
+          "=f" (l),
+          "=f" (m),
+          "=f" (n),
+          "=f" (o),
+          "=f" (p),
+          "={f4}" (q));
+  }
+}
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-7.c b/gcc/testsuite/gcc.target/s390/asm-hard-reg-7.c
new file mode 100644
index 0000000..923c9d2
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-7.c
@@ -0,0 +1,34 @@
+/* { dg-do compile } */
+/* { dg-options "-march=z13" } */
+
+/* Test register pairs.  */
+
+void
+test (void)
+{
+  register double f0 __asm__ ("f0");
+  register double f2 __asm__ ("f2");
+  register long double f0f2 __asm__ ("f0");
+  double x;
+  long double y;
+
+  /* Outputs */
+  __asm__ __volatile__ ("" : "=r" (f0), "=r" (f0f2));
+  __asm__ __volatile__ ("" : "=r" (f0f2), "={f0}" (y));  /* { dg-error "multiple outputs to hard register: %f0" } */
+  __asm__ __volatile__ ("" : "={f0}" (x), "=r" (f0f2));  /* { dg-error "multiple outputs to hard register: %f0" } */
+
+  __asm__ __volatile__ ("" : "=r" (f2), "=r" (f0f2));
+  __asm__ __volatile__ ("" : "={f2}" (x), "={f0}" (y));  /* { dg-error "multiple outputs to hard register: %f2" } */
+  __asm__ __volatile__ ("" : "=r" (f2), "={f0}" (y));  /* { dg-error "multiple outputs to hard register: %f2" } */
+  __asm__ __volatile__ ("" : "={f2}" (x), "=r" (f0f2));  /* { dg-error "multiple outputs to hard register: %f2" } */
+
+  /* Inputs */
+  __asm__ __volatile__ ("" :: "r" (f0), "r" (f0f2));
+  __asm__ __volatile__ ("" :: "r" (f0f2), "{f0}" (y));  /* { dg-error "multiple inputs to hard register: %f0" } */
+  __asm__ __volatile__ ("" :: "{f0}" (x), "r" (f0f2));  /* { dg-error "multiple inputs to hard register: %f0" } */
+
+  __asm__ __volatile__ ("" :: "r" (f2), "r" (f0f2));
+  __asm__ __volatile__ ("" :: "{f2}" (x), "{f0}" (y));  /* { dg-error "multiple inputs to hard register: %f2" } */
+  __asm__ __volatile__ ("" :: "r" (f2), "{f0}" (y));  /* { dg-error "multiple inputs to hard register: %f2" } */
+  __asm__ __volatile__ ("" :: "{f2}" (x), "r" (f0f2));  /* { dg-error "multiple inputs to hard register: %f2" } */
+}
diff --git a/gcc/testsuite/gcc.target/s390/asm-hard-reg-longdouble.h b/gcc/testsuite/gcc.target/s390/asm-hard-reg-longdouble.h
new file mode 100644
index 0000000..9f4adad
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/asm-hard-reg-longdouble.h
@@ -0,0 +1,18 @@
+__attribute__ ((noipa))
+long double
+test_longdouble (long double x)
+{
+  long double y;
+  asm ("sqxbr\t%0,%1" : "={f4}" (y) : "{f5}" (x));
+  return y;
+}
+
+int
+main (void)
+{
+  long double x = test_longdouble (42.L);
+  long double y = 6.48074069840786023096596743608799656681773277430814773408787249757445105002862106857719481922686100006103515625L;
+  if (x != y)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/s390/fminmax-1.c b/gcc/testsuite/gcc.target/s390/fminmax-1.c
new file mode 100644
index 0000000..df10905
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/fminmax-1.c
@@ -0,0 +1,77 @@
+/* Check fmin/fmax expanders for scalars on VXE targets.  */
+
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=z14 -mzarch" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+/*
+** dofmaxl:
+**	vl	(%v.),0\(%r3\),3
+**	vl	(%v.),0\(%r4\),3
+**	wfmaxxb	(%v.),\1,\2,4
+**	vst	\3,0\(%r2\),3
+**	br	%r14
+*/
+long double
+dofmaxl (long double d1, long double d2)
+{
+  return __builtin_fmaxl (d1, d2);
+}
+
+/*
+** dofminl:
+**	vl	(%v.),0\(%r3\),3
+**	vl	(%v.),0\(%r4\),3
+**	wfminxb	(%v.),\1,\2,4
+**	vst	\3,0\(%r2\),3
+**	br	%r14
+*/
+long double
+dofminl (long double d1, long double d2)
+{
+  return __builtin_fminl (d1, d2);
+}
+
+/*
+** dofmax:
+**	wfmaxdb	%v0,%v0,%v2,4
+**	br	%r14
+*/
+double
+dofmax (double d1, double d2)
+{
+  return __builtin_fmax (d1, d2);
+}
+
+/*
+** dofmin:
+**	wfmindb	%v0,%v0,%v2,4
+**	br	%r14
+*/
+double
+dofmin (double d1, double d2)
+{
+  return __builtin_fmin (d1, d2);
+}
+
+/*
+** dofmaxf:
+**	wfmaxsb	%v0,%v0,%v2,4
+**	br	%r14
+*/
+float
+dofmaxf (float f1, float f2)
+{
+  return __builtin_fmaxf (f1, f2);
+}
+
+/*
+** dofminf:
+**	wfminsb	%v0,%v0,%v2,4
+**	br	%r14
+*/
+float
+dofminf (float f1, float f2)
+{
+  return __builtin_fminf (f1, f2);
+}
diff --git a/gcc/testsuite/gcc.target/s390/fminmax-2.c b/gcc/testsuite/gcc.target/s390/fminmax-2.c
new file mode 100644
index 0000000..ea37a0a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/fminmax-2.c
@@ -0,0 +1,29 @@
+/* Check fmin/fmax expanders for scalars on non-VXE targets.  */
+
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=z13 -mzarch" } */
+/* { dg-final { scan-assembler-times "jg" 4 } } */
+
+double
+dofmax (double d1, double d2)
+{
+  return __builtin_fmax (d1, d2);
+}
+
+double
+dofmin (double d1, double d2)
+{
+  return __builtin_fmin (d1, d2);
+}
+
+float
+dofmaxf (float f1, float f2)
+{
+  return __builtin_fmaxf (f1, f2);
+}
+
+float
+dofminf (float f1, float f2)
+{
+  return __builtin_fminf (f1, f2);
+}
diff --git a/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-2.c b/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-2.c
index 2ff5a37..e1c7806 100644
--- a/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-2.c
+++ b/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-2.c
@@ -3,8 +3,10 @@
 
 #include "isfinite-isinf-isnormal-signbit.h"
 
-/* { dg-final { scan-assembler-times {tcxb\t%f[0-9]+,1365} 1 } } SIGNBIT long double */
-/* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,1365} 1 } } SIGNBIT _Decimal128 */
+/* { dg-final { scan-assembler-times {tcxb\t%f[0-9]+,1365} 0 { target lp64 } } } SIGNBIT long double */
+/* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,1365} 0 { target lp64 } } } SIGNBIT _Decimal128 */
+/* { dg-final { scan-assembler-times {tcxb\t%f[0-9]+,1365} 1 { target { ! lp64 } } } } SIGNBIT long double */
+/* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,1365} 1 { target { ! lp64 } } } } SIGNBIT _Decimal128 */
 /* { dg-final { scan-assembler-times {tcxb\t%f[0-9]+,4032} 1 } } ISFINITE long double */
 /* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,4032} 1 } } ISFINITE _Decimal128 */
 /* { dg-final { scan-assembler-times {tcxb\t%f[0-9]+,48} 1 } } ISINF long double */
diff --git a/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-3.c b/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-3.c
index 8f67553..5c9986d 100644
--- a/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-3.c
+++ b/gcc/testsuite/gcc.target/s390/isfinite-isinf-isnormal-signbit-3.c
@@ -3,8 +3,10 @@
 
 #include "isfinite-isinf-isnormal-signbit.h"
 
-/* { dg-final { scan-assembler-times {wftcixb\t%v[0-9]+,%v[0-9]+,1365} 1 } } */
-/* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,1365} 1 } } */
+/* { dg-final { scan-assembler-times {wftcixb\t%v[0-9]+,%v[0-9]+,1365} 0 { target lp64 } } } */
+/* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,1365} 0 { target lp64 } } } */
+/* { dg-final { scan-assembler-times {wftcixb\t%v[0-9]+,%v[0-9]+,1365} 1 { target { ! lp64 } } } } */
+/* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,1365} 1 { target { ! lp64 } } } } */
 /* { dg-final { scan-assembler-times {wftcixb\t%v[0-9]+,%v[0-9]+,4032} 1 } } */
 /* { dg-final { scan-assembler-times {tdcxt\t%f[0-9]+,4032} 1 } } */
 /* { dg-final { scan-assembler-times {wftcixb\t%v[0-9]+,%v[0-9]+,48} 1 } } */
diff --git a/gcc/testsuite/gcc.target/s390/signbit-1.c b/gcc/testsuite/gcc.target/s390/signbit-1.c
new file mode 100644
index 0000000..45f608a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/signbit-1.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -march=z900 -save-temps" } */
+/* { dg-final { scan-assembler-times {\ttceb\t} 2 } } */
+/* { dg-final { scan-assembler-times {\ttcdb\t} 2 } } */
+/* { dg-final { scan-assembler-times {\ttcxb\t} 2 } } */
+
+/* Binary Floating-Point */
+
+__attribute__ ((noipa))
+int signbit_float_reg (float x) { return __builtin_signbit (x); }
+__attribute__ ((noipa))
+int signbit_float_mem (float *x) { return __builtin_signbit (*x); }
+__attribute__ ((noipa))
+int signbit_double_reg (double x) { return __builtin_signbit (x); }
+__attribute__ ((noipa))
+int signbit_double_mem (double *x) { return __builtin_signbit (*x); }
+
+__attribute__ ((noipa))
+int
+signbit_longdouble_reg (long double x)
+{
+  __asm__ ("" : "+f" (x));
+  return __builtin_signbit (x);
+}
+
+__attribute__ ((noipa))
+int signbit_longdouble_mem (long double *x) { return __builtin_signbit (*x); }
+
+#include "signbit.h"
+TEST (float, float, __builtin_inff(), __builtin_nanf("42"), 0.f, 42.f)
+TEST (double, double, __builtin_inf(), __builtin_nan("42"), 0., 42.)
+TEST (longdouble, long double, __builtin_infl(), __builtin_nanl("42"), 0.L, 42.L)
+
+int
+main (void)
+{
+  test_float ();
+  test_double ();
+  test_longdouble ();
+}
diff --git a/gcc/testsuite/gcc.target/s390/signbit-2.c b/gcc/testsuite/gcc.target/s390/signbit-2.c
new file mode 100644
index 0000000..488c477
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/signbit-2.c
@@ -0,0 +1,40 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -march=z9-ec -mzarch -save-temps" } */
+/* { dg-final { scan-assembler-times {\ttdcet\t} 2 } } */
+/* { dg-final { scan-assembler-times {\ttdcdt\t} 2 } } */
+/* { dg-final { scan-assembler-times {\ttdcxt\t} 2 } } */
+
+/* Decimal Floating-Point */
+
+__attribute__ ((noipa))
+int signbit_dec32_reg (_Decimal32 x) { return __builtin_signbit (x); }
+__attribute__ ((noipa))
+int signbit_dec32_mem (_Decimal32 *x) { return __builtin_signbit (*x); }
+__attribute__ ((noipa))
+int signbit_dec64_reg (_Decimal64 x) { return __builtin_signbit (x); }
+__attribute__ ((noipa))
+int signbit_dec64_mem (_Decimal64 *x) { return __builtin_signbit (*x); }
+
+__attribute__ ((noipa))
+int
+signbit_dec128_reg (_Decimal128 x)
+{
+  __asm__ ("" : "+f" (x));
+  return __builtin_signbit (x);
+}
+
+__attribute__ ((noipa))
+int signbit_dec128_mem (_Decimal128 *x) { return __builtin_signbit (*x); }
+
+#include "signbit.h"
+TEST (dec32, _Decimal32, __builtin_infd32(), __builtin_nand32("42"), 0.df, 42.df)
+TEST (dec64, _Decimal64, __builtin_infd64(), __builtin_nand64("42"), 0.dd, 42.dd)
+TEST (dec128, _Decimal128, __builtin_infd128(), __builtin_nand128("42"), 0.dl, 42.dl)
+
+int
+main (void)
+{
+  test_dec32 ();
+  test_dec64 ();
+  test_dec128 ();
+}
diff --git a/gcc/testsuite/gcc.target/s390/signbit-3.c b/gcc/testsuite/gcc.target/s390/signbit-3.c
new file mode 100644
index 0000000..2fad58b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/signbit-3.c
@@ -0,0 +1,152 @@
+/* { dg-do run { target lp64 } } */
+/* { dg-options "-O2 -march=z10 -save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/* Binary Floating-Point */
+
+/*
+** signbit_float_reg:
+**	lgdr	(%r[0-9]+),%f0
+**	srlg	(%r[0-9]+),\1,63
+**	lgfr	%r2,\2
+**	br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_float_reg (float x) { return __builtin_signbit (x); }
+
+/*
+** signbit_float_mem:
+**    l	(%r[0-9]+),0\(%r2\)
+**    srl	\1,31
+**    lgfr	%r2,\1
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_float_mem (float *x) { return __builtin_signbit (*x); }
+
+/*
+** signbit_double_reg:
+**    lgdr	(%r[0-9]+),%f0
+**    srlg	%r2,\1,63
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_double_reg (double x) { return __builtin_signbit (x); }
+
+/*
+** signbit_double_mem:
+**    lg	(%r[0-9]+),0\(%r2\)
+**    srlg	%r2,\1,63
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_double_mem (double *x) { return __builtin_signbit (*x); }
+
+/*
+** signbit_longdouble_reg:
+**     ld	%f0,0\(%r2\)
+**     ld	%f2,8\(%r2\)
+**     lgdr	(%r[0-9]+),%f0
+**     srlg	%r2,\1,63
+**     br	%r14
+*/
+__attribute__ ((noipa))
+int
+signbit_longdouble_reg (long double x)
+{
+  __asm__ ("" : "+f" (x));
+  return __builtin_signbit (x);
+}
+
+/*
+** signbit_longdouble_mem:
+**      lg	(%r[0-9]+),0\(%r2\)
+**      srlg	%r2,\1,63
+**      br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_longdouble_mem (long double *x) { return __builtin_signbit (*x); }
+
+/* Decimal Floating-Point */
+
+/*
+** signbit_dec32_reg:
+**	lgdr	(%r[0-9]+),%f0
+**	srlg	(%r[0-9]+),\1,63
+**	lgfr	%r2,\2
+**	br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec32_reg (_Decimal32 x) { return __builtin_signbit (x); }
+
+/*
+** signbit_dec32_mem:
+**    l	(%r[0-9]+),0\(%r2\)
+**    srl	\1,31
+**    lgfr	%r2,\1
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec32_mem (_Decimal32 *x) { return __builtin_signbit (*x); }
+
+/*
+** signbit_dec64_reg:
+**    lgdr	(%r[0-9]+),%f0
+**    srlg	%r2,\1,63
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec64_reg (_Decimal64 x) { return __builtin_signbit (x); }
+
+/*
+** signbit_dec64_mem:
+**    lg	(%r[0-9]+),0\(%r2\)
+**    srlg	%r2,\1,63
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec64_mem (_Decimal64 *x) { return __builtin_signbit (*x); }
+
+/*
+** signbit_dec128_reg:
+**     ld	%f0,0\(%r2\)
+**     ld	%f2,8\(%r2\)
+**     lgdr	(%r[0-9]+),%f0
+**     srlg	%r2,\1,63
+**     br	%r14
+*/
+__attribute__ ((noipa))
+int
+signbit_dec128_reg (_Decimal128 x)
+{
+  __asm__ ("" : "+f" (x));
+  return __builtin_signbit (x);
+}
+
+/*
+** signbit_dec128_mem:
+**      lg	(%r[0-9]+),0\(%r2\)
+**      srlg	%r2,\1,63
+**      br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec128_mem (_Decimal128 *x) { return __builtin_signbit (*x); }
+
+#include "signbit.h"
+TEST (float, float, __builtin_inff(), __builtin_nanf("42"), 0.f, 42.f)
+TEST (double, double, __builtin_inf(), __builtin_nan("42"), 0., 42.)
+TEST (longdouble, long double, __builtin_infl(), __builtin_nanl("42"), 0.L, 42.L)
+TEST (dec32, _Decimal32, __builtin_infd32(), __builtin_nand32("42"), 0.df, 42.df)
+TEST (dec64, _Decimal64, __builtin_infd64(), __builtin_nand64("42"), 0.dd, 42.dd)
+TEST (dec128, _Decimal128, __builtin_infd128(), __builtin_nand128("42"), 0.dl, 42.dl)
+
+int
+main (void)
+{
+  test_float ();
+  test_double ();
+  test_longdouble ();
+  test_dec32 ();
+  test_dec64 ();
+  test_dec128 ();
+}
diff --git a/gcc/testsuite/gcc.target/s390/signbit-4.c b/gcc/testsuite/gcc.target/s390/signbit-4.c
new file mode 100644
index 0000000..2cb743e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/signbit-4.c
@@ -0,0 +1,55 @@
+/* { dg-do run { target lp64 } } */
+/* { dg-require-effective-target s390_vx } */
+/* { dg-options "-O2 -march=z13 -save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/* Binary Floating-Point */
+
+/*
+** signbit_float_reg:
+**    vlgvf	(%r[0-9]+),%v0,0
+**    risbgn	%r2,\1,64-1,128\+63,32\+1
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_float_reg (float x) { return __builtin_signbit (x); }
+
+/*
+** signbit_float_mem:
+**    l	(%r[0-9]+),0\(%r2\)
+**    risbgn	%r2,\1,64-1,128\+63,32\+1
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_float_mem (float *x) { return __builtin_signbit (*x); }
+
+/* Decimal Floating-Point */
+
+/*
+** signbit_dec32_reg:
+**    vlgvf	(%r[0-9]+),%v0,0
+**    risbgn	%r2,\1,64-1,128\+63,32\+1
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec32_reg (_Decimal32 x) { return __builtin_signbit (x); }
+
+/*
+** signbit_dec32_mem:
+**    l	(%r[0-9]+),0\(%r2\)
+**    risbgn	%r2,\1,64-1,128\+63,32\+1
+**    br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_dec32_mem (_Decimal32 *x) { return __builtin_signbit (*x); }
+
+#include "signbit.h"
+TEST (float, float, __builtin_inff(), __builtin_nanf("42"), 0.f, 42.f)
+TEST (dec32, _Decimal32, __builtin_infd32(), __builtin_nand32("42"), 0.df, 42.df)
+
+int
+main (void)
+{
+  test_float ();
+  test_dec32 ();
+}
diff --git a/gcc/testsuite/gcc.target/s390/signbit-5.c b/gcc/testsuite/gcc.target/s390/signbit-5.c
new file mode 100644
index 0000000..6840327
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/signbit-5.c
@@ -0,0 +1,35 @@
+/* { dg-do run { target lp64 } } */
+/* { dg-options "-O2 -march=z14 -save-temps" } */
+
+/*
+** signbit_longdouble_reg:
+**      ld	%f0,0(%r2);ld	%f2,8+0(%r2)
+**      lgdr	(%r[0-9]+),%f0
+**      srlg	%r2,\1,63
+**      br	%r14
+*/
+__attribute__ ((noipa))
+int
+signbit_longdouble_reg (long double x)
+{
+  __asm__ ("" : "+f" (x));
+  return __builtin_signbit (x);
+}
+
+/*
+** signbit_longdouble_mem:
+**      lg	(%r[0-9]+),0\(%r2\)
+**      srlg	%r2,\1,63
+**      br	%r14
+*/
+__attribute__ ((noipa))
+int signbit_longdouble_mem (long double *x) { return __builtin_signbit (*x); }
+
+#include "signbit.h"
+TEST (longdouble, long double, __builtin_infl(), __builtin_nanl("42"), 0.L, 42.L)
+
+int
+main (void)
+{
+  test_longdouble ();
+}
diff --git a/gcc/testsuite/gcc.target/s390/signbit.h b/gcc/testsuite/gcc.target/s390/signbit.h
new file mode 100644
index 0000000..730e387
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/signbit.h
@@ -0,0 +1,36 @@
+#define TEST(T, U, I, N, C0, C42)                                              \
+  void test_##T (void)                                                         \
+  {                                                                            \
+    U tmp;                                                                     \
+    int x;                                                                     \
+                                                                               \
+    x = signbit_##T##_reg(C42);                                                \
+    x += signbit_##T##_reg(C0);                                                \
+    x += signbit_##T##_reg(I);                                                 \
+    x += signbit_##T##_reg(N);                                                 \
+    tmp = C42;                                                                 \
+    x += signbit_##T##_mem(&tmp);                                              \
+    tmp = C0;                                                                  \
+    x += signbit_##T##_mem(&tmp);                                              \
+    tmp = I;                                                                   \
+    x += signbit_##T##_mem(&tmp);                                              \
+    tmp = N;                                                                   \
+    x += signbit_##T##_mem(&tmp);                                              \
+    if (x != 0)                                                                \
+      __builtin_abort();                                                       \
+                                                                               \
+    x = signbit_##T##_reg(-C42);                                               \
+    x += signbit_##T##_reg(-C0);                                               \
+    x += signbit_##T##_reg(-I);                                                \
+    x += signbit_##T##_reg(-N);                                                \
+    tmp = -C42;                                                                \
+    x += signbit_##T##_mem(&tmp);                                              \
+    tmp = -C0;                                                                 \
+    x += signbit_##T##_mem(&tmp);                                              \
+    tmp = -I;                                                                  \
+    x += signbit_##T##_mem(&tmp);                                              \
+    tmp = -N;                                                                  \
+    x += signbit_##T##_mem(&tmp);                                              \
+    if (x != 8)                                                                \
+      __builtin_abort();                                                       \
+  }
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-fp-1.c b/gcc/testsuite/gcc.target/s390/spaceship-fp-1.c
new file mode 100644
index 0000000..56c3d77
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-fp-1.c
@@ -0,0 +1,23 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 2\)} 3 optimized } } */
+/* { dg-final { scan-assembler-times {\tk[edx]br\t} 3 } } */
+/* { dg-final { scan-assembler-not {\tbrc} } } */
+/* { dg-final { scan-assembler-not {\tc[edx]br\t} } } */
+
+#define TEST(T, U)		\
+  int test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      return 0;			\
+    else if (x < y)		\
+      return -1;		\
+    else if (x > y)		\
+      return 1;			\
+    else			\
+      return 2;			\
+  }
+
+TEST (float, float)
+TEST (double, double)
+TEST (long double, longdouble)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-fp-2.c b/gcc/testsuite/gcc.target/s390/spaceship-fp-2.c
new file mode 100644
index 0000000..0c6e6b6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-fp-2.c
@@ -0,0 +1,23 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -ffinite-math-only -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 2\)} 3 optimized } } */
+/* { dg-final { scan-assembler-times {\tc[edx]br\t} 3 } } */
+/* { dg-final { scan-assembler-not {\tbrc} } } */
+/* { dg-final { scan-assembler-not {\tk[edx]br\t} } } */
+
+#define TEST(T, U)		\
+  int test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      return 0;			\
+    else if (x < y)		\
+      return -1;		\
+    else if (x > y)		\
+      return 1;			\
+    else			\
+      return 2;			\
+  }
+
+TEST (float, float)
+TEST (double, double)
+TEST (long double, longdouble)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-fp-3.c b/gcc/testsuite/gcc.target/s390/spaceship-fp-3.c
new file mode 100644
index 0000000..2f567d1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-fp-3.c
@@ -0,0 +1,23 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 42\)} 3 optimized } } */
+/* { dg-final { scan-assembler-times {\tk[edx]br\t} 3 } } */
+/* { dg-final { scan-assembler-not {\tbrc} } } */
+/* { dg-final { scan-assembler-not {\tc[edx]br\t} } } */
+
+#define TEST(T, U)		\
+  int test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      return 0;			\
+    else if (x < y)		\
+      return -1;		\
+    else if (x > y)		\
+      return 1;			\
+    else			\
+      return 42;		\
+  }
+
+TEST (float, float)
+TEST (double, double)
+TEST (long double, longdouble)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-fp-4.c b/gcc/testsuite/gcc.target/s390/spaceship-fp-4.c
new file mode 100644
index 0000000..4531ecb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-fp-4.c
@@ -0,0 +1,53 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 0\)} 3 optimized } } */
+/* { dg-final { scan-assembler-times {\tk[edx]br\t} 3 } } */
+/* { dg-final { scan-assembler-not {\tloc} } } */
+/* { dg-final { scan-assembler-not {\tbrc} } } */
+/* { dg-final { scan-assembler-not {\tc[edx]br\t} } } */
+
+/* By time of writing this we emit
+
+	kebr	%f0,%f2
+	jo	.L2
+	je	.L3
+	jnh	.L10
+	jg	f3@PLT
+.L10:
+	jg	f2@PLT
+.L3:
+	jg	f1@PLT
+.L2:
+	jg	f4@PLT
+
+   which is not optimal.  Instead we could fold the conditional branch with the
+   unconditional into something along the lines
+
+	kebr	%f0,%f2
+	jo	f4@PLT
+	je	f1@PLT
+	jnh	f2@PLT
+	jg	f3@PLT
+*/
+
+void f1 (void);
+void f2 (void);
+void f3 (void);
+void f4 (void);
+
+#define TEST(T, U)		\
+  void test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      f1 ();			\
+    else if (x < y)		\
+      f2 ();			\
+    else if (x > y)		\
+      f3 ();			\
+    else			\
+      f4 ();			\
+  }
+
+TEST (float, float)
+TEST (double, double)
+TEST (long double, longdouble)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-int-1.c b/gcc/testsuite/gcc.target/s390/spaceship-int-1.c
new file mode 100644
index 0000000..8ca2677
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-int-1.c
@@ -0,0 +1,30 @@
+/* { dg-do compile { target lp64 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, -1\)} 4 optimized } } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 1\)} 5 optimized } } */
+/* { dg-final { scan-assembler-times {\tlhi} 9 } } */
+/* { dg-final { scan-assembler-times {\tloc} 18 } } */
+
+#define TEST(T, U)		\
+  int test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      return 0;			\
+    else if (x < y)		\
+      return -1;		\
+    else			\
+      return 1;			\
+  }
+
+TEST(signed char, schar)
+TEST(unsigned char, uchar)
+TEST(char, char)
+
+TEST(short, sshort)
+TEST(unsigned short, ushort)
+
+TEST(int, sint)
+TEST(unsigned int, uint)
+
+TEST(long, slong)
+TEST(unsigned long, ulong)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-int-2.c b/gcc/testsuite/gcc.target/s390/spaceship-int-2.c
new file mode 100644
index 0000000..5f7975c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-int-2.c
@@ -0,0 +1,24 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O2 -mzarch -march=z13 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, -1\)} 1 optimized } } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 1\)} 1 optimized } } */
+/* { dg-final { scan-assembler-times {\tvecg} 1 } } */
+/* { dg-final { scan-assembler-times {\tveclg} 1 } } */
+/* { dg-final { scan-assembler-times {\tvchlgs} 2 } } */
+/* { dg-final { scan-assembler-times {\tvceqgs} 2 } } */
+/* { dg-final { scan-assembler-times {\tlhi} 2 } } */
+/* { dg-final { scan-assembler-times {\tloc} 4 } } */
+
+#define TEST(T, U)		\
+  int test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      return 0;			\
+    else if (x < y)		\
+      return -1;		\
+    else			\
+      return 1;			\
+  }
+
+TEST(__int128, sint128)
+TEST(unsigned __int128, uint128)
diff --git a/gcc/testsuite/gcc.target/s390/spaceship-int-3.c b/gcc/testsuite/gcc.target/s390/spaceship-int-3.c
new file mode 100644
index 0000000..46b0e4a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/spaceship-int-3.c
@@ -0,0 +1,21 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O2 -march=z17 -fdump-tree-optimized" } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, -1\)} 1 optimized } } */
+/* { dg-final { scan-tree-dump-times {\.SPACESHIP \([^,]+, [^,]+, 1\)} 1 optimized } } */
+/* { dg-final { scan-assembler-times {\tvecq\t} 1 } } */
+/* { dg-final { scan-assembler-times {\tveclq\t} 1 } } */
+/* { dg-final { scan-assembler-times {\tloc} 4 } } */
+
+#define TEST(T, U)		\
+  int test_##U (T x, T y)	\
+  {				\
+    if (x == y)			\
+      return 0;			\
+    else if (x < y)		\
+      return -1;		\
+    else			\
+      return 1;			\
+  }
+
+TEST(__int128, sint128)
+TEST(unsigned __int128, uint128)
diff --git a/gcc/testsuite/gcc.target/s390/stack-protector-guard-tls-1.c b/gcc/testsuite/gcc.target/s390/stack-protector-guard-tls-1.c
new file mode 100644
index 0000000..1efd245
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/stack-protector-guard-tls-1.c
@@ -0,0 +1,39 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fstack-protector-all" } */
+/* { dg-final { scan-assembler-times {\tear\t%r[0-9]+,%a[01]} 8 { target lp64 } } } */
+/* { dg-final { scan-assembler-times {\tsllg\t%r[0-9]+,%r[0-9]+,32} 4 { target lp64 } } } */
+/* { dg-final { scan-assembler-times {\tear\t%r[0-9]+,%a[01]} 3 { target { ! lp64 } } } } */
+/* { dg-final { scan-assembler-times {\tmvc\t160\(8,%r15\),40\(%r[0-9]+\)} 2 { target lp64 } } } */
+/* { dg-final { scan-assembler-times {\tmvc\t100\(4,%r15\),20\(%r[0-9]+\)} 2 { target { ! lp64 } } } } */
+/* { dg-final { scan-assembler-times {\tclc\t160\(8,%r15\),40\(%r[0-9]+\)} 2 { target lp64 } } } */
+/* { dg-final { scan-assembler-times {\tclc\t100\(4,%r15\),20\(%r[0-9]+\)} 2 { target { ! lp64 } } } } */
+
+/* Computing the address of the thread pointer on s390 involves multiple
+   instructions and therefore bears the risk that the address of the canary or
+   intermediate values of it are spilled and reloaded.  Therefore, as a
+   precaution compute the address always twice, i.e., one time for the prologue
+   and one time for the epilogue.  */
+
+void test_0 (void) { }
+
+void test_1 (void)
+{
+  __asm__ __volatile ("" :::
+      "r0",
+      "r1",
+      "r2",
+      "r3",
+      "r4",
+      "r5",
+      "r6",
+      "r7",
+      "r8",
+      "r9",
+      "r10",
+      "r11",
+#ifndef __PIC__
+      "r12",
+#endif
+      "r13",
+      "r14");
+}
diff --git a/gcc/testsuite/gcc.target/s390/vector/pattern-avg-1.c b/gcc/testsuite/gcc.target/s390/vector/pattern-avg-1.c
new file mode 100644
index 0000000..285ebc9
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/pattern-avg-1.c
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=z16 -ftree-vectorize -fdump-tree-optimized" } */
+
+#define TEST(T1,T2,N)                                                   \
+  void                                                                  \
+  avg##T1 (signed T1 *__restrict res, signed T1 *__restrict a,          \
+           signed T1 *__restrict b)                                     \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = ((signed T2)a[i] + b[i] + 1) >> 1;                       \
+  }                                                                     \
+                                                                        \
+  void                                                                  \
+  uavg##T1 (unsigned T1 *__restrict res, unsigned T1 *__restrict a,     \
+            unsigned T1 *__restrict b)                                  \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = ((unsigned T2)a[i] + b[i] + 1) >> 1;                     \
+  }
+
+TEST(char,short,16)
+TEST(short,int,8)
+TEST(int,long,4)
+
+/* { dg-final { scan-tree-dump-times "\.AVG_CEIL" 6 "optimized" { target lp64 } } } */
+/* { dg-final { scan-tree-dump-times "\.AVG_CEIL" 4 "optimized" { target { ! lp64 } } } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/pattern-avg-2.c b/gcc/testsuite/gcc.target/s390/vector/pattern-avg-2.c
new file mode 100644
index 0000000..1cc614e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/pattern-avg-2.c
@@ -0,0 +1,23 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O3 -mzarch -march=z16 -ftree-vectorize -fdump-tree-optimized" } */
+
+#define TEST(T1,T2,N)                                                   \
+  void                                                                  \
+  avg##T1 (signed T1 *__restrict res, signed T1 *__restrict a,          \
+           signed T1 *__restrict b)                                     \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = ((signed T2)a[i] + b[i] + 1) >> 1;                       \
+  }                                                                     \
+                                                                        \
+  void                                                                  \
+  uavg##T1 (unsigned T1 *__restrict res, unsigned T1 *__restrict a,     \
+            unsigned T1 *__restrict b)                                  \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = ((unsigned T2)a[i] + b[i] + 1) >> 1;                     \
+  }
+
+TEST(long,__int128,2)
+
+/* { dg-final { scan-tree-dump-times "\.AVG_CEIL" 2 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/pattern-mulh-1.c b/gcc/testsuite/gcc.target/s390/vector/pattern-mulh-1.c
new file mode 100644
index 0000000..f0b37d6
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/pattern-mulh-1.c
@@ -0,0 +1,27 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=arch15 -ftree-vectorize -fdump-tree-optimized" } */
+
+#define TEST(T1,T2,N,S)                                                 \
+  void                                                                  \
+  mulh##T1 (signed T1 *__restrict res,                                  \
+            signed T1 *__restrict l,                                    \
+            signed T1 *__restrict r)                                    \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = (signed T1) (((signed T2)l[i] * (signed T2)r[i]) >> S);  \
+  }                                                                     \
+                                                                        \
+  void                                                                  \
+  umulh##T1 (unsigned T1 *__restrict res,                               \
+             unsigned T1 *__restrict l,                                 \
+             unsigned T1 *__restrict r)                                 \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = (unsigned T1)                                            \
+        (((unsigned T2)l[i] * (unsigned T2)r[i]) >> S);                 \
+  }
+
+TEST(char,short,16,8)
+TEST(short,int,8,16)
+
+/* { dg-final { scan-tree-dump-times "\.MULH" 4 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/pattern-mulh-2.c b/gcc/testsuite/gcc.target/s390/vector/pattern-mulh-2.c
new file mode 100644
index 0000000..2ff66b7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/pattern-mulh-2.c
@@ -0,0 +1,27 @@
+/* { dg-do compile { target int128 } } */
+/* { dg-options "-O3 -mzarch -march=arch15 -ftree-vectorize -fdump-tree-optimized" } */
+
+#define TEST(T1,T2,N,S)                                                 \
+  void                                                                  \
+  mulh##T1 (signed T1 *__restrict res,                                  \
+            signed T1 *__restrict l,                                    \
+            signed T1 *__restrict r)                                    \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = (signed T1) (((signed T2)l[i] * (signed T2)r[i]) >> S);  \
+  }                                                                     \
+                                                                        \
+  void                                                                  \
+  umulh##T1 (unsigned T1 *__restrict res,                               \
+             unsigned T1 *__restrict l,                                 \
+             unsigned T1 *__restrict r)                                 \
+  {                                                                     \
+    for (int i = 0; i < N; ++i)                                         \
+      res[i] = (unsigned T1)                                            \
+        (((unsigned T2)l[i] * (unsigned T2)r[i]) >> S);                 \
+  }
+
+TEST(int,long,4,32)
+TEST(long,__int128,2,64)
+
+/* { dg-final { scan-tree-dump-times "\.MULH" 4 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/reduc-binops-1.c b/gcc/testsuite/gcc.target/s390/vector/reduc-binops-1.c
new file mode 100644
index 0000000..efd3294
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/reduc-binops-1.c
@@ -0,0 +1,40 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=z13 -ftree-vectorize -fdump-tree-optimized" } */
+
+#define T(X,N)                                  \
+  unsigned X                                    \
+  reduce_and_##X (unsigned X *in)               \
+  {                                             \
+  unsigned X acc = (unsigned X)-1;              \
+  for (int i = 0; i < N; i++)                   \
+    acc &= in[i];                               \
+  return acc;                                   \
+  }                                             \
+  unsigned X                                    \
+  reduce_ior_##X (unsigned X *in)               \
+  {                                             \
+  unsigned X acc = 0;                           \
+  for (int i = 0; i < N; i++)                   \
+    acc |= in[i];                               \
+  return acc;                                   \
+  }                                             \
+  unsigned X                                    \
+  redue_xor_##X (unsigned X *in)                \
+  {                                             \
+  unsigned X acc = 0;                           \
+  for (int i = 0; i < N; i++)                   \
+    acc ^= in[i];                               \
+  return acc;                                   \
+  }
+
+T(char,16)
+
+T(short, 8)
+
+T(int,4)
+
+T(long,4)
+
+/* { dg-final { scan-tree-dump-times "\.REDUC_AND" 4 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "\.REDUC_IOR" 4 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "\.REDUC_XOR" 4 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/reduc-minmax-1.c b/gcc/testsuite/gcc.target/s390/vector/reduc-minmax-1.c
new file mode 100644
index 0000000..5295250
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/reduc-minmax-1.c
@@ -0,0 +1,234 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=z14 -ftree-vectorize -fdump-tree-optimized" } */
+
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) > (b) ? (b) : (a))
+
+/* unsigned integers */
+
+unsigned char
+reduce_umax_char (unsigned char *p)
+{
+  unsigned char res = p[0];
+  for (int i = 0; i < 16; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+unsigned char
+reduce_umin_char (unsigned char *p)
+{
+  unsigned char res = p[0];
+  for (int i = 0; i < 16; i++)
+    res = MIN (res, p[i]);
+  return res;
+}
+
+unsigned short
+reduce_umax_short (unsigned short *p)
+{
+  unsigned short res = p[0];
+  for (int i = 0; i < 8; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+unsigned short
+reduce_umin_short (unsigned short *p)
+{
+  unsigned short res = p[0];
+  for (int i = 0; i < 8; i++)
+    res = MIN (res, p[i]);
+  return res;
+}
+
+unsigned int
+reduce_umax_int (unsigned int* p)
+{
+  unsigned int res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+unsigned int
+reduce_umin_int (unsigned int* p)
+{
+  unsigned int res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MIN(res, p[i]);
+  return res;
+}
+
+unsigned long
+reduce_umax_long (unsigned long* p)
+{
+  unsigned long res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+unsigned long
+reduce_umin_long (unsigned long* p)
+{
+  unsigned long res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MIN(res, p[i]);
+  return res;
+}
+
+/* signed integers */
+
+signed char
+reduce_smax_char (signed char *p)
+{
+  signed char res = p[0];
+  for (int i = 0; i < 16; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+signed char
+reduce_smin_char (signed char *p)
+{
+  signed char res = p[0];
+  for (int i = 0; i < 16; i++)
+    res = MIN (res, p[i]);
+  return res;
+}
+
+signed short
+reduce_smax_short (signed short *p)
+{
+  signed short res = p[0];
+  for (int i = 0; i < 8; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+signed short
+reduce_smin_short (signed short *p)
+{
+  signed short res = p[0];
+  for (int i = 0; i < 8; i++)
+    res = MIN (res, p[i]);
+  return res;
+}
+
+signed int
+reduce_smax_int (signed int* p)
+{
+  signed int res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+signed int
+reduce_smin_int (signed int* p)
+{
+  signed int res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MIN(res, p[i]);
+  return res;
+}
+
+signed long
+reduce_smax_long (signed long* p)
+{
+  signed long res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+signed long
+reduce_smin_long (signed long* p)
+{
+  signed long res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MIN(res, p[i]);
+  return res;
+}
+
+float
+__attribute__((optimize("Ofast")))
+reduce_smax_float (float* p)
+{
+  float res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+float
+__attribute__((optimize("Ofast")))
+reduce_smin_float (float* p)
+{
+  float res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MIN (res, p[i]);
+  return res;
+}
+
+double
+__attribute__((optimize("Ofast")))
+reduce_smax_double (double* p)
+{
+  double res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MAX (res, p[i]);
+  return res;
+}
+
+double
+__attribute__((optimize("Ofast")))
+reduce_smin_double (double* p)
+{
+  double res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = MIN (res, p[i]);
+  return res;
+}
+
+float
+reduce_fmax_float (float* p)
+{
+  float res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = __builtin_fmaxf (res, p[i]);
+  return res;
+}
+
+float
+reduce_fmin_float (float* p)
+{
+  float res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = __builtin_fminf (res, p[i]);
+  return res;
+}
+
+double
+reduce_fmax_double (double* p)
+{
+  double res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = __builtin_fmax (res, p[i]);
+  return res;
+}
+
+double
+reduce_fmin_double (double* p)
+{
+  double res = p[0];
+  for (int i = 0; i != 4; i++)
+    res = __builtin_fmin (res, p[i]);
+  return res;
+}
+
+/* { dg-final { scan-tree-dump-times "\.REDUC_MAX" 10 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "\.REDUC_MIN" 10 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "\.REDUC_FMAX" 2 "optimized" } } */
+/* { dg-final { scan-tree-dump-times "\.REDUC_FMIN" 2 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/reduc-plus-1.c b/gcc/testsuite/gcc.target/s390/vector/reduc-plus-1.c
new file mode 100644
index 0000000..12cdd5f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/reduc-plus-1.c
@@ -0,0 +1,152 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=z14 -ftree-vectorize -fdump-tree-optimized" } */
+/* { dg-do run { target { s390_z14_hw } } } */
+
+/* signed integers */
+
+signed char
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_char (signed char* p)
+{
+  signed char sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+short
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_short (short* p)
+{
+  short sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+int
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_int (int* p)
+{
+  int sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+long
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_long (long* p)
+{
+  long sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+/* unsigned integers */
+
+unsigned char
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_uchar (unsigned char* p)
+{
+  unsigned char sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+unsigned short
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_ushort (unsigned short* p)
+{
+  unsigned short sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+unsigned int
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_uint (unsigned int* p)
+{
+  unsigned int sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+unsigned long
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_ulong (unsigned long* p)
+{
+  unsigned long sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+/* floating point */
+
+float
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_float (float* p)
+{
+  float sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+double
+__attribute__((noipa, optimize("Ofast")))
+reduce_add_double (double* p)
+{
+  double sum = 0;
+  for (int i = 0; i != 16; i++)
+    sum += p[i];
+  return sum;
+}
+
+int
+main()
+{
+  signed char chararr[] = {-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16};
+  signed short shortarr[] = {-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16};
+  signed int intarr[] = {-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16};
+  signed long longarr[] = {-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16};
+
+  unsigned char uchararr[] = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16};
+  unsigned short ushortarr[] = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16};
+  unsigned int uintarr[] = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16};
+  unsigned long ulongarr[] = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16};
+
+  float floatarr[] = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16};
+  double doublearr[] = {-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16};
+
+  if (reduce_add_char (chararr) != (-136 & 0xff))
+    __builtin_abort();
+  if (reduce_add_short (shortarr) != -136)
+    __builtin_abort();
+  if (reduce_add_int (intarr) != -136)
+    __builtin_abort();
+  if (reduce_add_long (longarr) != -136)
+    __builtin_abort();
+
+  if (reduce_add_uchar (uchararr) != 136)
+    __builtin_abort();
+  if (reduce_add_ushort (ushortarr) != 136)
+    __builtin_abort();
+  if (reduce_add_uint (uintarr) != 136)
+    __builtin_abort();
+  if (reduce_add_ulong (ulongarr) != 136)
+    __builtin_abort();
+
+  if (reduce_add_float (floatarr) != 136)
+    __builtin_abort();
+  if (reduce_add_double (doublearr) != -136)
+    __builtin_abort();
+  return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\.REDUC_PLUS" 10 "optimized" } } */
diff --git a/gcc/testsuite/gcc.target/s390/vector/vec-perm-merge-1.c b/gcc/testsuite/gcc.target/s390/vector/vec-perm-merge-1.c
new file mode 100644
index 0000000..79f8a88
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/vec-perm-merge-1.c
@@ -0,0 +1,242 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=z14 -mzvector --save-temps -fno-stack-protector" } */
+/* { dg-do run { target { s390_z14_hw } } } */
+/* { dg-final {check-function-bodies "**" "" } } */
+
+#include "vec-types.h"
+
+/*
+** qi_via_hi_hi:
+**	vmrhh	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_hi_hi (v16qi a, v16qi b)
+{
+  return (v16qi){a[0], a[1], b[0], b[1], a[2], a[3], b[2], b[3],
+		 a[4], a[5], b[4], b[5], a[6], a[7], b[6], b[7]};
+}
+
+/*
+** qi_via_hi_lo:
+**	vmrlh	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_hi_lo (v16qi a, v16qi b)
+{
+  return (v16qi){a[8], a[9], b[8], b[9], a[10], a[11], b[10], b[11],
+		 a[12], a[13], b[12], b[13], a[14], a[15], b[14], b[15]};
+}
+
+/*
+** qi_via_si_hi:
+**	vmrhf	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_si_hi (v16qi a, v16qi b)
+{
+  return (v16qi){a[0], a[1], a[2], a[3], b[0], b[1], b[2], b[3],
+		 a[4], a[5], a[6], a[7], b[4], b[5], b[6], b[7]};
+}
+
+/*
+** qi_via_si_lo:
+**	vmrlf	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_si_lo (v16qi a, v16qi b)
+{
+  return (v16qi){a[8], a[9], a[10], a[11], b[8], b[9], b[10], b[11],
+		 a[12], a[13], a[14], a[15], b[12], b[13], b[14], b[15]};
+}
+
+/*
+** qi_via_di_hi:
+**	vmrhg	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_di_hi (v16qi a, v16qi b)
+{
+  return (v16qi){a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7],
+		 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]};
+}
+
+/*
+** qi_via_di_lo:
+**	vmrlg	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_di_lo (v16qi a, v16qi b)
+{
+  return (v16qi){a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15],
+		 b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15]};
+}
+
+/*
+** hi_via_si_hi:
+**	vmrhf	%v24,%v24,%v26
+**	br	%r14
+*/
+v8hi __attribute__((noinline,noipa))
+hi_via_si_hi (v8hi a, v8hi b)
+{
+  return (v8hi){a[0], a[1], b[0], b[1], a[2], a[3], b[2], b[3]};
+}
+
+/*
+** hi_via_si_lo:
+**	vmrlf	%v24,%v24,%v26
+**	br	%r14
+*/
+v8hi __attribute__((noinline,noipa))
+hi_via_si_lo (v8hi a, v8hi b)
+{
+  return (v8hi){a[4], a[5], b[4], b[5], a[6], a[7], b[6], b[7]};
+}
+
+/*
+** hi_via_di_hi:
+**	vmrhg	%v24,%v24,%v26
+**	br	%r14
+*/
+v8hi __attribute__((noinline,noipa))
+hi_via_di_hi (v8hi a, v8hi b)
+{
+  return (v8hi){a[0], a[1], a[2], a[3], b[0], b[1], b[2], b[3]};
+}
+
+/*
+** hi_via_di_lo:
+**	vmrlg	%v24,%v24,%v26
+**	br	%r14
+*/
+v8hi __attribute__((noinline,noipa))
+hi_via_di_lo (v8hi a, v8hi b)
+{
+  return (v8hi){a[4], a[5], a[6], a[7], b[4], b[5], b[6], b[7]};
+}
+
+/*
+** si_via_di_hi:
+**	vmrhg	%v24,%v24,%v26
+**	br	%r14
+*/
+v4si __attribute__((noinline,noipa))
+si_via_di_hi (v4si a, v4si b)
+{
+  return (v4si){a[0], a[1], b[0], b[1]};
+}
+
+/*
+** si_via_di_lo:
+**	vmrlg	%v24,%v24,%v26
+**	br	%r14
+*/
+v4si __attribute__((noinline,noipa))
+si_via_di_lo (v4si a, v4si b)
+{
+  return (v4si){a[2], a[3], b[2], b[3]};
+}
+
+int
+main ()
+{
+  static const signed char e_qi_via_hi_hi[16]
+    = {0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23};
+  static const signed char e_qi_via_hi_lo[16]
+    = {8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31};
+  static const signed char e_qi_via_si_hi[16]
+    = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23};
+  static const signed char e_qi_via_si_lo[16]
+    = {8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31};
+  static const signed char e_qi_via_di_hi[16]
+    = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
+  static const signed char e_qi_via_di_lo[16]
+    = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
+
+  static const short e_hi_via_si_hi[8] = {0, 1, 8, 9, 2, 3, 10, 11};
+  static const short e_hi_via_si_lo[8] = {4, 5, 12, 13, 6, 7, 14, 15};
+  static const short e_hi_via_di_hi[8] = {0, 1, 2, 3, 8, 9, 10, 11};
+  static const short e_hi_via_di_lo[8] = {4, 5, 6, 7, 12, 13, 14, 15};
+
+  static const int e_si_via_di_hi[4] = {0, 1, 4, 5};
+  static const int e_si_via_di_lo[4] = {2, 3, 6, 7};
+
+  v16qi a_qi = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+  v16qi b_qi = {16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31};
+  v8hi a_hi = {0, 1, 2, 3, 4, 5, 6, 7};
+  v8hi b_hi = {8, 9, 10, 11, 12, 13, 14, 15};
+  v4si a_si = {0, 1, 2, 3};
+  v4si b_si = {4, 5, 6, 7};
+  v16qi r_qi;
+  v8hi r_hi;
+  v4si r_si;
+  int i;
+
+  r_qi = qi_via_hi_hi (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_hi_hi[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_hi_lo (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_hi_lo[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_si_hi (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_si_hi[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_si_lo (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_si_lo[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_di_hi (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_di_hi[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_di_lo (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_di_lo[i])
+      __builtin_abort ();
+
+  r_hi = hi_via_si_hi (a_hi, b_hi);
+  for (i = 0; i < 8; ++i)
+    if (r_hi[i] != e_hi_via_si_hi[i])
+      __builtin_abort ();
+
+  r_hi = hi_via_si_lo (a_hi, b_hi);
+  for (i = 0; i < 8; ++i)
+    if (r_hi[i] != e_hi_via_si_lo[i])
+      __builtin_abort ();
+
+  r_hi = hi_via_di_hi (a_hi, b_hi);
+  for (i = 0; i < 8; ++i)
+    if (r_hi[i] != e_hi_via_di_hi[i])
+      __builtin_abort ();
+
+  r_hi = hi_via_di_lo (a_hi, b_hi);
+  for (i = 0; i < 8; ++i)
+    if (r_hi[i] != e_hi_via_di_lo[i])
+      __builtin_abort ();
+
+  r_si = si_via_di_hi (a_si, b_si);
+  for (i = 0; i < 4; ++i)
+    if (r_si[i] != e_si_via_di_hi[i])
+      __builtin_abort ();
+
+  r_si = si_via_di_lo (a_si, b_si);
+  for (i = 0; i < 4; ++i)
+    if (r_si[i] != e_si_via_di_lo[i])
+      __builtin_abort ();
+
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/s390/vector/vec-perm-pack-1.c b/gcc/testsuite/gcc.target/s390/vector/vec-perm-pack-1.c
new file mode 100644
index 0000000..6590c92
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/vec-perm-pack-1.c
@@ -0,0 +1,133 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mzarch -march=z14 -mzvector --save-temps -fno-stack-protector" } */
+/* { dg-do run { target { s390_z14_hw } } } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "vec-types.h"
+
+/*
+** qi_via_hi:
+**	vpkh	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_hi (v16qi a, v16qi b)
+{
+  return (v16qi){a[1], a[3], a[5], a[7], a[9], a[11], a[13], a[15],
+		 b[1], b[3], b[5], b[7], b[9], b[11], b[13], b[15]};
+}
+
+/*
+** qi_via_si:
+**	vpkf	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_si (v16qi a, v16qi b)
+{
+  return (v16qi){a[2], a[3], a[6], a[7], a[10], a[11], a[14], a[15],
+		 b[2], b[3], b[6], b[7], b[10], b[11], b[14], b[15]};
+}
+
+/*
+** qi_via_di:
+**	vpkg	%v24,%v24,%v26
+**	br	%r14
+*/
+v16qi __attribute__((noinline,noipa))
+qi_via_di (v16qi a, v16qi b)
+{
+  return (v16qi){a[4], a[5], a[6], a[7], a[12], a[13], a[14], a[15],
+		 b[4], b[5], b[6], b[7], b[12], b[13], b[14], b[15]};
+}
+
+/*
+** hi_via_si:
+**	vpkf	%v24,%v24,%v26
+**	br	%r14
+*/
+v8hi __attribute__((noinline,noipa))
+hi_via_si (v8hi a, v8hi b)
+{
+  return (v8hi){a[1], a[3], a[5], a[7], b[1], b[3], b[5], b[7]};
+}
+
+/*
+** hi_via_di:
+**	vpkg	%v24,%v24,%v26
+**	br	%r14
+*/
+v8hi __attribute__((noinline,noipa))
+hi_via_di (v8hi a, v8hi b)
+{
+  return (v8hi){a[2], a[3], a[6], a[7], b[2], b[3], b[6], b[7]};
+}
+
+/*
+** si_via_di:
+**	vpkg	%v24,%v24,%v26
+**	br	%r14
+*/
+v4si __attribute__((noinline,noipa))
+si_via_di (v4si a, v4si b)
+{
+  return (v4si){a[1], a[3], b[1], b[3]};
+}
+
+int
+main ()
+{
+  static const signed char e_qi_via_hi[16]
+    = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31};
+  static const signed char e_qi_via_si[16]
+    = {2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31};
+  static const signed char e_qi_via_di[16]
+    = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31};
+
+  static const short e_hi_via_si[8] = {1, 3, 5, 7, 9, 11, 13, 15};
+  static const short e_hi_via_di[8] = {2, 3, 6, 7, 10, 11, 14, 15};
+
+  static const int e_si_via_di[4] = {1, 3, 5, 7};
+
+  v16qi a_qi = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+  v16qi b_qi = {16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31};
+  v8hi a_hi = {0, 1, 2, 3, 4, 5, 6, 7};
+  v8hi b_hi = {8, 9, 10, 11, 12, 13, 14, 15};
+  v4si a_si = {0, 1, 2, 3};
+  v4si b_si = {4, 5, 6, 7};
+  v16qi r_qi;
+  v8hi r_hi;
+  v4si r_si;
+  int i;
+
+  r_qi = qi_via_hi (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_hi[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_si (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_si[i])
+      __builtin_abort ();
+
+  r_qi = qi_via_di (a_qi, b_qi);
+  for (i = 0; i < 16; ++i)
+    if (r_qi[i] != e_qi_via_di[i])
+      __builtin_abort ();
+
+  r_hi = hi_via_si (a_hi, b_hi);
+  for (i = 0; i < 8; ++i)
+    if (r_hi[i] != e_hi_via_si[i])
+      __builtin_abort ();
+
+  r_hi = hi_via_di (a_hi, b_hi);
+  for (i = 0; i < 8; ++i)
+    if (r_hi[i] != e_hi_via_di[i])
+      __builtin_abort ();
+
+  r_si = si_via_di (a_si, b_si);
+  for (i = 0; i < 4; ++i)
+    if (r_si[i] != e_si_via_di[i])
+      __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/gcc.target/s390/vector/vlgv-zero-extend-1.c b/gcc/testsuite/gcc.target/s390/vector/vlgv-zero-extend-1.c
new file mode 100644
index 0000000..11df6c1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/s390/vector/vlgv-zero-extend-1.c
@@ -0,0 +1,71 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target s390_vx } */
+/* { dg-additional-options "-O2" } */
+/* { dg-final { scan-assembler-not {\tllg?[fhc]r\t} } } */
+
+typedef unsigned char __attribute__ ((vector_size (1))) V1QI;
+typedef unsigned char __attribute__ ((vector_size (2))) V2QI;
+typedef unsigned char __attribute__ ((vector_size (4))) V4QI;
+typedef unsigned char __attribute__ ((vector_size (8))) V8QI;
+typedef unsigned char __attribute__ ((vector_size (16))) V16QI;
+
+typedef unsigned short __attribute__ ((vector_size (2))) V1HI;
+typedef unsigned short __attribute__ ((vector_size (4))) V2HI;
+typedef unsigned short __attribute__ ((vector_size (8))) V4HI;
+typedef unsigned short __attribute__ ((vector_size (16))) V8HI;
+
+typedef unsigned int __attribute__ ((vector_size (4))) V1SI;
+typedef unsigned int __attribute__ ((vector_size (8))) V2SI;
+typedef unsigned int __attribute__ ((vector_size (16))) V4SI;
+
+unsigned short ushort;
+unsigned int uint;
+
+#define TEST(T, U, I) \
+  unsigned T test_ ## I ## _ ## U (U x) { return x[I]; } \
+  void       test_ ## I ## _ ## U ## _ushort (U x) { ushort = x[I]; } \
+  void       test_ ## I ## _ ## U ## _uint (U x) { uint = x[I]; }
+
+#define TEST1(T, U) \
+  TEST(T, U, 0)
+
+#define TEST2(T, U) \
+  TEST1 (T, U) \
+  TEST(T, U, 1)
+
+#define TEST4(T, U) \
+  TEST2 (T, U) \
+  TEST(T, U, 2) \
+  TEST(T, U, 3)
+
+#define TEST8(T, U) \
+  TEST4 (T, U) \
+  TEST(T, U, 4) \
+  TEST(T, U, 5) \
+  TEST(T, U, 6) \
+  TEST(T, U, 7)
+
+#define TEST16(T, U) \
+  TEST8 (T, U) \
+  TEST(T, U, 9) \
+  TEST(T, U, 10) \
+  TEST(T, U, 11) \
+  TEST(T, U, 12) \
+  TEST(T, U, 13) \
+  TEST(T, U, 14) \
+  TEST(T, U, 15)
+
+TEST1 (char, V1QI)
+TEST2 (char, V2QI)
+TEST4 (char, V4QI)
+TEST8 (char, V8QI)
+TEST16 (char, V16QI)
+
+TEST1 (short, V1HI)
+TEST2 (short, V2HI)
+TEST4 (short, V4HI)
+TEST8 (short, V8HI)
+
+TEST1 (int, V1SI)
+TEST2 (int, V2SI)
+TEST4 (int, V4SI)
diff --git a/gcc/testsuite/gcc.target/sh/pr54236-2.c b/gcc/testsuite/gcc.target/sh/pr54236-2.c
index 1e2f3bb..78befe4 100644
--- a/gcc/testsuite/gcc.target/sh/pr54236-2.c
+++ b/gcc/testsuite/gcc.target/sh/pr54236-2.c
@@ -4,10 +4,10 @@
 /* { dg-do compile }  */
 /* { dg-options "-O1" } */
 
-/* { dg-final { scan-assembler-times "addc" 36 } } */
+/* { dg-final { scan-assembler-times "addc" 32 } } */
 /* { dg-final { scan-assembler-times "shll" 14 } } */
-/* { dg-final { scan-assembler-times "add\tr" 12 } } */
-/* { dg-final { scan-assembler-not "movt" } } */
+/* { dg-final { scan-assembler-times "add\tr" 16 } } */
+/* { dg-final { scan-assembler-times "movt" 4 } } */
 
 /* { dg-final { scan-assembler-times "add\t#1" 1 } } */
 
@@ -184,28 +184,28 @@ test_022 (int a, int b, int c, int d)
 int
 test_023 (int a, int b, int c, int d)
 {
-  // 1x shll, 1x addc
+  // 1x shll, 1x add
   return a + ((b >> 31) & 1);
 }
 
 int
 test_024 (int a, int b, int c, int d)
 {
-  // 1x shll, 1x addc
+  // 1x shll, 1x add
   return ((b >> 31) & 1) + a;
 }
 
 int
 test_025 (int a, int b, int c, int d)
 {
-  // 1x shll, 1x addc
+  // 1x shll, 1x add
   return ((a >> 31) & 1) + a;
 }
 
 int
 test_026 (int a, int b, int c, int d)
 {
-  // 1x shll, 1x addc
+  // 1x shll, 1x add
   return a + ((a >> 31) & 1);
 }
 
diff --git a/gcc/testsuite/gcc.target/xtensa/BGEUI-BLTUI-32k-64k.c b/gcc/testsuite/gcc.target/xtensa/BGEUI-BLTUI-32k-64k.c
new file mode 100644
index 0000000..05873b8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/xtensa/BGEUI-BLTUI-32k-64k.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+extern void foo(void);
+
+void BGEUI_test(unsigned int a)
+{
+  if (a < 32768U)
+    foo();
+}
+
+void BLTUI_test(unsigned int a)
+{
+  if (a >= 65536U)
+    foo();
+}
+
+/* { dg-final { scan-assembler-times "bgeui" 1 } } */
+/* { dg-final { scan-assembler-times "bltui" 1 } } */
diff --git a/gcc/testsuite/gcc.target/xtensa/pr120888-1.c b/gcc/testsuite/gcc.target/xtensa/pr120888-1.c
new file mode 100644
index 0000000..f438e4c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/xtensa/pr120888-1.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O1 -fdump-rtl-expand" } */
+
+void u8(unsigned char c);
+void cu8(unsigned char *p)
+{
+  u8(*p);
+}
+
+/* { dg-final { scan-rtl-dump "zero_extend" "expand" } } */
+/* { dg-final { scan-rtl-dump-not "sign_extend" "expand" } } */
diff --git a/gcc/testsuite/gcc.target/xtensa/pr120888-2.c b/gcc/testsuite/gcc.target/xtensa/pr120888-2.c
new file mode 100644
index 0000000..9b5caad
--- /dev/null
+++ b/gcc/testsuite/gcc.target/xtensa/pr120888-2.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O1 -fdump-rtl-expand" } */
+
+void s8(signed char c);
+void cs8(signed char *p)
+{
+  s8(*p);
+}
+
+/* { dg-final { scan-rtl-dump "sign_extend" "expand" } } */
+/* { dg-final { scan-rtl-dump-not "zero_extend" "expand" } } */
diff --git a/gcc/testsuite/gfortran.dg/array_constructor_58.f90 b/gcc/testsuite/gfortran.dg/array_constructor_58.f90
new file mode 100644
index 0000000..1473be0
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/array_constructor_58.f90
@@ -0,0 +1,17 @@
+!{ dg-do run }
+
+! Contributed by  Federico Perini <federico.perini@gmail.com>
+! Check that PR fortran/119106 is fixed.
+
+program char_param_array
+implicit none
+character, parameter :: p(5) = ['1','2','3','4','5']
+character, save      :: n(5) = ['1','2','3','4','5']
+integer :: i(10), j
+
+i = 4
+if (any([(n(i(j)),j=1,10)] /= '4')) stop 1 ! OK
+if (any([(p(i(j)),j=1,10)] /= '4')) stop 2 ! used to runtime out-of-bounds error
+
+end program char_param_array
+
diff --git a/gcc/testsuite/gfortran.dg/asan/array_constructor_1.f90 b/gcc/testsuite/gfortran.dg/asan/array_constructor_1.f90
index 45eafac..a0c5507 100644
--- a/gcc/testsuite/gfortran.dg/asan/array_constructor_1.f90
+++ b/gcc/testsuite/gfortran.dg/asan/array_constructor_1.f90
@@ -9,6 +9,8 @@ program grow_type_array
     
     type(container), allocatable :: list(:)
 
+    allocate(list(0))
+
     list = [list, new_elem(5)]
 
     deallocate(list)
diff --git a/gcc/testsuite/gfortran.dg/asan/finalize_1.f90 b/gcc/testsuite/gfortran.dg/asan/finalize_1.f90
new file mode 100644
index 0000000..ab53a9e
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/asan/finalize_1.f90
@@ -0,0 +1,67 @@
+!{ dg-do run }
+
+! PR fortran/120637
+
+! Contributed by Antony Lewis  <antony@cosmologist.info>
+! The unused module is needed to trigger the issue of not freeing the
+! memory of second module.
+
+    module MiscUtils
+    implicit none
+
+    contains
+        
+    logical function isFloat0(R)
+    class(*), intent(in) :: R
+
+    select type(R)
+    type is (real)
+        isFloat0 = .true.
+    end select
+    end function isFloat0
+
+    end module MiscUtils
+    
+    module results3
+    implicit none
+    public
+
+    Type ClTransferData2
+      real, dimension(:,:,:), allocatable :: Delta_p_l_k
+    end type ClTransferData2
+
+    type TCLdata2
+       Type(ClTransferData2) :: CTransScal, CTransTens, CTransVec
+    end type TCLdata2 
+
+    type :: CAMBdata2
+        Type(TClData2) :: CLdata2
+    end type  
+
+    end module results3
+
+program driver
+   use results3
+   integer i
+   do i=1, 2
+   call test()   
+   end do
+
+   contains
+
+   subroutine test
+       implicit none
+       class(CAMBdata2), pointer :: Data
+
+       allocate(CAMBdata2::Data)
+
+       allocate(Data%ClData2%CTransScal%Delta_p_l_k(3, 1000, 1000)) 
+       allocate(Data%ClData2%CTransVec%Delta_p_l_k(3, 1000, 1000))
+       deallocate(Data)
+
+   end subroutine test
+
+ end program driver
+
+!{ dg-final { cleanup-modules "miscutils results3" } }
+
diff --git a/gcc/testsuite/gfortran.dg/assign_13.f90 b/gcc/testsuite/gfortran.dg/assign_13.f90
new file mode 100644
index 0000000..262ade0
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/assign_13.f90
@@ -0,0 +1,25 @@
+! { dg-do run }
+!
+! PR fortran/121185
+! The assignment to Y%X in CHECK_T was using a polymorphic array access on the
+! left hand side, using the virtual table of Y.
+
+program p
+  implicit none
+  type t
+     complex, allocatable :: x(:)
+  end type t
+  real :: trace = 2.
+  type(t) :: z
+  z%x = [1,2] * trace
+  call check_t (z)
+contains
+  subroutine check_t (y)
+    class(t) :: y
+    ! print *, y% x
+    if (any(y%x /= [2., 4.])) error stop 11
+    y%x = y%x / trace
+    ! print *, y% x
+    if (any(y%x /= [1., 2.])) error stop 12
+  end subroutine
+end
diff --git a/gcc/testsuite/gfortran.dg/assign_14.f90 b/gcc/testsuite/gfortran.dg/assign_14.f90
new file mode 100644
index 0000000..33b46b9
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/assign_14.f90
@@ -0,0 +1,24 @@
+! { dg-do compile }
+! { dg-additional-options {-fdump-tree-original} }
+!
+! PR fortran/121185
+! Check that an intermediary variable is used to reference component a.
+! { dg-final { scan-tree-dump-not {->b->a} original } }
+
+program p
+  implicit none
+  type t
+     integer, allocatable :: a(:)
+  end type t
+  type u
+     type(t), allocatable :: b
+  end type u
+  type v
+     type(u), allocatable :: c
+  end type v
+  type(v) :: z
+  z%c = u()
+  z%c%b = t()
+  z%c%b%a = [1,2]
+  z%c%b%a = z%c%b%a * 2
+end
diff --git a/gcc/testsuite/gfortran.dg/associate_75.f90 b/gcc/testsuite/gfortran.dg/associate_75.f90
new file mode 100644
index 0000000..c7c461a
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/associate_75.f90
@@ -0,0 +1,50 @@
+! { dg-do run }
+!
+! Test fix for PR121060.
+!
+! Contributed by Damian Rouson  <damian@archaeologic.codes>
+!
+module subdomain_m
+  implicit none
+
+  type subdomain_t 
+    real :: s_ = 99.
+  contains
+    generic :: operator(.laplacian.) => laplacian
+    procedure laplacian
+  end type
+
+contains
+
+  function laplacian(rhs)
+    class(subdomain_t), intent(in) :: rhs
+    type(subdomain_t) laplacian
+    laplacian%s_ = rhs%s_ + 42
+  end function
+
+end module
+
+  use subdomain_m
+  implicit none
+
+  type operands_t
+    real :: s_
+  end type
+
+  type(subdomain_t) phi
+  type(operands_t) operands
+
+  associate(laplacian_phi => .laplacian. phi) ! ICE because specific not found.
+    operands = approximates(laplacian_phi%s_)
+  end associate
+
+  if (int (operands%s_) /= 42) stop 1
+contains
+
+  function approximates(actual)
+    real actual 
+    type(operands_t) approximates
+    approximates%s_ = actual - 99
+  end function
+
+end
diff --git a/gcc/testsuite/gfortran.dg/class_elemental_1.f90 b/gcc/testsuite/gfortran.dg/class_elemental_1.f90
new file mode 100644
index 0000000..547ae98
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/class_elemental_1.f90
@@ -0,0 +1,35 @@
+! { dg-do run }
+!
+! PR fortran/121342
+! The polymorphic function result as actual argument used to force the loop
+! bounds around the elemental call, altering access to the other arrays.
+
+program p
+  implicit none
+  type :: t
+    integer :: i
+  end type
+  type :: u
+    integer :: i, a
+  end type
+  type(u) :: accum(5)
+  integer :: a(3:7), k
+  a = [ (k*k, k=1,5) ]
+  call s(accum, f(), a)
+  ! print *, accum%i
+  ! print *, accum%a
+  if (any(accum%i /= accum%a)) error stop 1
+contains
+  elemental subroutine s(l, c, a)
+    type(u)  , intent(out) :: l
+    class(t) , intent(in)  :: c
+    integer  , intent(in)  :: a
+    l%i = c%i
+    l%a = a
+  end subroutine
+  function f()
+    class(t), allocatable :: f(:)
+    allocate(f(-1:3))
+    f%i = [ (k*k, k=1,5) ]
+  end function
+end program
diff --git a/gcc/testsuite/gfortran.dg/function_charlen_4.f90 b/gcc/testsuite/gfortran.dg/function_charlen_4.f90
new file mode 100644
index 0000000..ed39aca
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/function_charlen_4.f90
@@ -0,0 +1,34 @@
+! { dg-do run }
+! { dg-options "-O2 -std=legacy -fdump-tree-optimized" }
+!
+! PR fortran/121203 - fix passing of character length of function to procedure
+
+program p
+  character(10), external :: f
+  call eval (f,"abc")
+  call eval2(f,"abc")
+contains
+  subroutine eval2(func,c_arg)
+    character(*) c_arg
+    character(*) func
+    external func
+    ! These tests should get optimized:
+    if (len      (c_arg)  /=  3) stop 1
+    if (len (func(c_arg)) /= 10) stop 2
+  end subroutine
+end
+
+character(10) function f(arg)
+  character(*) arg
+  f=arg
+end
+
+subroutine eval(func,c_arg)
+  character(*) c_arg
+  character(*) func
+  external func
+  if (len      (c_arg)  /=  3) error stop 3
+  if (len (func(c_arg)) /= 10) error stop 4
+end subroutine
+
+! { dg-final { scan-tree-dump-not "_gfortran_stop_numeric" "optimized" } }
diff --git a/gcc/testsuite/gfortran.dg/g77/980310-3.f b/gcc/testsuite/gfortran.dg/g77/980310-3.f
index 39bd86c..4bf4d91 100644
--- a/gcc/testsuite/gfortran.dg/g77/980310-3.f
+++ b/gcc/testsuite/gfortran.dg/g77/980310-3.f
@@ -12,7 +12,7 @@ C Date: Wed, 17 Dec 1997 23:20:29 +0000
 C From: Joao Cardoso <jcardoso@inescn.pt>
 C To: egcs-bugs@cygnus.com
 C Subject: egcs-1.0 f77 bug on OSR5
-C When trying to compile the Fortran file that I enclose bellow,
+C When trying to compile the Fortran file that I enclose below,
 C I got an assembler error:
 C 
 C ./g77 -B./ -fpic -O -c scaleg.f
diff --git a/gcc/testsuite/gfortran.dg/goacc/parameter-3.f90 b/gcc/testsuite/gfortran.dg/goacc/parameter-3.f90
new file mode 100644
index 0000000..2c8aa61
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/goacc/parameter-3.f90
@@ -0,0 +1,16 @@
+! { dg-do compile }
+
+subroutine x
+  integer :: var
+  integer, parameter :: ilog = 0
+  integer, parameter :: array(*) = [11,22,33]
+  !$ACC DECLARE COPYIN(ilog, array, var, array) ! { dg-error "Symbol 'array' present on multiple clauses" }
+end subroutine x
+
+integer :: a
+integer, parameter :: b = 4
+integer, parameter :: c(*) = [1,2,3]
+
+!$acc parallel copy(a,c,b,c)  ! { dg-error "Symbol 'c' present on multiple clauses" }
+!$acc end parallel
+end
diff --git a/gcc/testsuite/gfortran.dg/goacc/parameter-4.f90 b/gcc/testsuite/gfortran.dg/goacc/parameter-4.f90
new file mode 100644
index 0000000..aadd7cf
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/goacc/parameter-4.f90
@@ -0,0 +1,26 @@
+! { dg-do compile }
+! { dg-additional-options "-fdump-tree-original" }
+
+subroutine x
+  integer :: var
+  integer, parameter :: ilog = 0
+  integer, parameter :: array(*) = [11,22,33]
+  !$ACC DECLARE COPYIN(ilog, array, var)
+end subroutine x
+
+integer :: a
+integer, parameter :: b = 4
+integer, parameter :: c(*) = [1,2,3]
+
+!$acc parallel copy(a,c,b)
+  a = c(2) + b
+!$acc end parallel
+
+!$acc parallel firstprivate(a,c,b)
+  a = c(2) + b
+!$acc end parallel
+end
+
+! { dg-final { scan-tree-dump-times "#pragma acc data map\\(to:var\\)" 1 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma acc parallel map\\(tofrom:a\\)" 1 "original" } }
+! { dg-final { scan-tree-dump-times "#pragma acc parallel firstprivate\\(a\\)" 1 "original" } }
diff --git a/gcc/testsuite/gfortran.dg/goacc/parameter.f95 b/gcc/testsuite/gfortran.dg/goacc/parameter.f95
index b581338..a9bde4a 100644
--- a/gcc/testsuite/gfortran.dg/goacc/parameter.f95
+++ b/gcc/testsuite/gfortran.dg/goacc/parameter.f95
@@ -1,4 +1,5 @@
 ! { dg-do compile } 
+! { dg-additional-options "-Wsurprising" }
 
 module test
 contains
@@ -6,37 +7,37 @@ contains
     implicit none
     integer :: i
     integer, parameter :: a = 1
-    !$acc declare device_resident (a) ! { dg-error "is not a variable" }
-    !$acc data copy (a) ! { dg-error "not a variable" }
+    !$acc declare device_resident (a) ! (no warning here - for semi-good reasons)
+    !$acc data copy (a)  ! { dg-warning "Clause for object 'a' at .1. is ignored as parameters need not be copied \\\[-Wsurprising\\\]" }
     !$acc end data
-    !$acc data deviceptr (a) ! { dg-error "not a variable" }
+    !$acc data deviceptr (a)  ! { dg-warning "Clause for object 'a' at .1. is ignored as parameters need not be copied \\\[-Wsurprising\\\]" }
     !$acc end data
-    !$acc parallel private (a) ! { dg-error "not a variable" }
+    !$acc parallel private (a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     !$acc end parallel
-    !$acc serial private (a) ! { dg-error "not a variable" }
+    !$acc serial private (a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     !$acc end serial
-    !$acc host_data use_device (a) ! { dg-error "not a variable" }
+    !$acc host_data use_device (a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     !$acc end host_data
-    !$acc parallel loop reduction(+:a) ! { dg-error "not a variable" }
+    !$acc parallel loop reduction(+:a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     do i = 1,5
     enddo
     !$acc end parallel loop
-    !$acc serial loop reduction(+:a) ! { dg-error "not a variable" }
+    !$acc serial loop reduction(+:a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     do i = 1,5
     enddo
     !$acc end serial loop
     !$acc parallel loop
     do i = 1,5
-      !$acc cache (a) ! { dg-error "not a variable" }
+      !$acc cache (a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     enddo
     !$acc end parallel loop
     !$acc serial loop
     do i = 1,5
-      !$acc cache (a) ! { dg-error "not a variable" }
+      !$acc cache (a) ! { dg-warning "Clause for object 'a' at .1. is ignored as it is a parameter \\\[-Wsurprising\\\]" }
     enddo
     !$acc end serial loop
-    !$acc update device (a) ! { dg-error "not a variable" }
-    !$acc update host (a) ! { dg-error "not a variable" }
-    !$acc update self (a) ! { dg-error "not a variable" }
+    !$acc update device (a)  ! { dg-warning "Clause for object 'a' at .1. is ignored as parameters need not be copied \\\[-Wsurprising\\\]" }
+    !$acc update host (a)  ! { dg-warning "Clause for object 'a' at .1. is ignored as parameters need not be copied \\\[-Wsurprising\\\]" }
+    !$acc update self (a)  ! { dg-warning "Clause for object 'a' at .1. is ignored as parameters need not be copied \\\[-Wsurprising\\\]" }
   end subroutine oacc1
 end module test
diff --git a/gcc/testsuite/gfortran.dg/gomp/omp_get_num_devices_initial_device.f90 b/gcc/testsuite/gfortran.dg/gomp/omp_get_num_devices_initial_device.f90
index 5409f12..279656b 100644
--- a/gcc/testsuite/gfortran.dg/gomp/omp_get_num_devices_initial_device.f90
+++ b/gcc/testsuite/gfortran.dg/gomp/omp_get_num_devices_initial_device.f90
@@ -17,8 +17,8 @@ end
 
 ! { dg-final { scan-tree-dump-not "error_stop" "optimized" } }
 
-! { dg-final { scan-tree-dump-not "omp_get_num_devices;" "optimized" { target { ! offloading_enabled } } } }
+! { dg-final { scan-tree-dump-not "omp_get_num_devices" "optimized" { target { ! offloading_enabled } } } }
 ! { dg-final { scan-tree-dump "return 0;" "optimized" { target { ! offloading_enabled } } } }
 
-! { dg-final { scan-tree-dump-times "omp_get_num_devices;" 1 "optimized" { target offloading_enabled } } }
+! { dg-final { scan-tree-dump-times "omp_get_num_devices" 1 "optimized" { target offloading_enabled } } }
 ! { dg-final { scan-tree-dump "_1 = __builtin_omp_get_num_devices \\(\\);\[\\r\\n\]+\[ \]+return _1;" "optimized" { target offloading_enabled } } }
diff --git a/gcc/testsuite/gfortran.dg/gomp/pr104428.f90 b/gcc/testsuite/gfortran.dg/gomp/pr104428.f90
new file mode 100644
index 0000000..639b331
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/gomp/pr104428.f90
@@ -0,0 +1,15 @@
+! { dg-do compile }
+
+program p
+  interface
+    subroutine x
+    end subroutine x
+  end interface
+contains
+  subroutine foo
+    !$omp declare variant(x) match(construct={do})
+  end
+  subroutine bar
+    !$omp declare variant(y) match(construct={do}) ! { dg-error "Cannot find symbol 'y'" }
+  end
+end
diff --git a/gcc/testsuite/gfortran.dg/import12.f90 b/gcc/testsuite/gfortran.dg/import12.f90
new file mode 100644
index 0000000..df1aae6
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/import12.f90
@@ -0,0 +1,302 @@
+! { dg-do compile }
+!
+! Tests the variants of IMPORT introduced in F2018
+!
+! Contributed by Paul Thomas  <pault@gcc.gnu.org>
+!
+MODULE M
+  import, none          ! { dg-error "F2018: C897 IMPORT statement" }
+  IMPLICIT NONE
+  integer :: z
+end module
+
+MODULE N
+  IMPLICIT NONE
+  integer :: z
+end module
+
+! Taken from gfortran.dg/pr103312.f90. These F2008-style invocations should
+! be accepted.
+module example
+  type, abstract :: foo
+    integer :: i
+  contains
+    procedure(foo_size), deferred :: size
+    procedure(foo_func), deferred :: func
+  end type
+  abstract interface
+    pure integer function foo_size (this)
+      import :: foo
+      class(foo), intent(in) :: this
+    end function
+    function foo_func (this) result (string)
+      import :: foo
+      class(foo) :: this
+      character(this%size()) :: string
+    end function
+  end interface
+end module
+
+block data blk
+  import, all          ! { dg-error "F2018: C897 IMPORT statement" }
+  integer a(2)
+  common /my_common/a
+  data a/1,2/
+end
+
+subroutine extern_sub1
+  import               ! { dg-error "F2018: C897 IMPORT statement" }
+end
+
+subroutine extern_sub2 (arg1, arg2, arg3)
+  implicit none
+  integer :: arg1, arg2, arg3
+  arg1 = int_fcn ()
+contains
+  integer function int_fcn () 
+    import, only : arg2, arg3
+    int_fcn = arg2 * arg3
+  end
+end
+
+program p
+  import, all          ! { dg-error "F2018: C897 IMPORT statement" }
+  implicit none
+  integer :: x, y
+  type :: t
+    integer :: i
+  end type
+  type(t) :: progtype
+  type, extends(t) :: s
+    integer :: j
+  end type
+  class(t), allocatable :: progclass
+contains
+
+! OK because arg is just that and x is declared in scope of sub1.
+  subroutine sub1 (arg)
+    import, none
+    implicit none
+    real :: arg, x
+  end
+
+! IMPORT, ALL must be the only IMPORT statement in the scope.
+  subroutine sub2 (arg)
+    import, none
+    import, all         ! { dg-error "F2018: C8100 IMPORT statement" }
+    implicit none
+    real :: arg, x
+  end
+
+! Error message says it all.
+  subroutine sub3 (arg)
+    import, none
+    implicit none
+    integer :: arg
+    print *, arg
+    x = 1              ! { dg-error "F2018: C8102" }
+  end
+
+! Error messages say it all.
+  subroutine sub4 (arg)
+    import, only : y
+    implicit none
+    integer :: arg
+    print *, arg
+    x = 1              ! { dg-error "F2018: C8102" }
+    y = 2
+    print *, x         ! { dg-error "F2018: C8102" }
+  end
+
+! IMPORT eos and IMPORT, ALL must be unique in the scope.
+  subroutine sub5a (arg)
+    import, all
+    import             ! { dg-error "F2018: C8100" }
+    implicit none
+    real :: arg
+    real :: x          ! { dg-error "F2018: C8102" }
+  end
+
+  subroutine sub5b (arg)
+    import, only : x
+    implicit none
+    real :: arg
+    real :: x          ! { dg-error "F2018: C8102" }
+  end
+
+! Error message says it all.
+  integer function func1 ()
+    import, only : x
+    func1 = x * y      ! { dg-error "F2018: C8102" }
+  end
+
+! Error messages say it all.
+  subroutine sub6 (arg)
+    import, only : func1
+    import, only : func2
+    import, only : foobar                ! { dg-error "has no IMPLICIT type" }
+    implicit none
+    integer :: arg
+    arg = func1 () * func2 () * func3 () ! { dg-error "F2018: C8102" }
+  end
+
+! Error message says it all.
+  integer function func2 ()
+    use N
+    import, none
+    implicit none
+    func2 = y          ! { dg-error "F2018: C8102" }
+  end
+
+! OK
+  integer function func3 ()
+    func3 = 42
+  end
+
+  subroutine sub7 (arg)
+    implicit none
+    integer :: arg
+! OK
+    block
+       import, only : arg, func1, func2, func3
+       arg = func1 () * func2 () * func3 ()
+    end block
+    block
+       arg = func1 ()
+       import, only : arg, func1   ! { dg-error "Unexpected IMPORT statement" }
+    end block
+  end
+
+! Error messages say it all.
+  subroutine sub8 (arg)
+    implicit none
+    integer :: arg
+    block
+       import, only : func1
+       import, only : func2
+       import, only : foobar                ! { dg-error "has no IMPLICIT type" }
+       arg = func1 () * func2 () * func3 () ! { dg-error "F2018: C8102" }
+    end block
+  end
+
+! ASSOCIATE does not have a specification part so IMPORT cannot appear.
+  subroutine sub9 (arg)
+    implicit none
+    integer :: arg
+    associate (f3 => func3 ())              ! { dg-error "F2018: C8102" }
+       import, only : arg, func1            ! { dg-error "Unexpected IMPORT statement" }
+       arg = func1 () * func2 () * f3       ! { dg-error "F2018: C8102" }
+    end associate
+  end
+
+! OK
+  subroutine sub10 (arg)
+    import, only : t
+    implicit none
+    type(t) :: arg, mytype
+    mytype%i = 1
+    arg = mytype
+  end
+
+! TYPE t does not appear in the IMPORT list
+  subroutine sub11 (arg)
+    import, only : progtype
+    implicit none
+    type(t) :: arg
+    progtype%i = 1   ! { dg-error "F2018: C8102" }
+    arg = progtype   ! { dg-error "F2018: C8102" }
+  end
+
+! TYPE t is excluded by IMPORT, NONE
+  subroutine sub12 (arg)
+    import, none
+    implicit none
+    type(t) :: arg, mytype
+    mytype%i = 1     ! { dg-error "F2018: C8102" }
+    arg = mytype     ! { dg-error "F2018: C8102" }
+  end
+
+! TYPE t does not appear in the IMPORT list
+  subroutine sub13 (arg)
+    import, only : progclass
+    implicit none
+    class(t) :: arg
+    type(t) :: ca(2) = [t(1), t(2)]  ! { dg-error "F2018: C8102" }
+    progclass%i = t(1) ! { dg-error "F2018: C8102" }
+    arg = progclass    ! { dg-error "F2018: C8102" }
+    ca = [t(1), t(2)]  ! { dg-error "has no IMPLICIT type|F2018: C8102" }
+    arg = ca(2)        ! Note: The preceeding line catches 'ca' having no implicit type.
+  end
+
+! TYPE t is excluded by IMPORT, NONE
+  subroutine sub14 (arg)
+    import, none
+    implicit none
+    class(t) :: arg
+    class(t), allocatable ::  myclass
+    myclass%i =  t(1)  ! { dg-error "F2018: C8102" }
+    arg%i = myclass%i  ! { dg-error "F2018: C8102" }
+    select type (arg)  ! { dg-error "F2018: C8102" }
+      type is (t)
+        arg%i = arg%i + 1
+      type is (s)
+        arg%j = -1
+    end select
+  end
+
+! TYPE s does not appear in the IMPORT, ONLY list
+  subroutine sub15 (arg)
+    import, only : t
+    implicit none
+    class(t) :: arg
+    class(t), allocatable ::  myclass
+    myclass =  t(1)
+    arg%i = myclass%i
+    select type (arg)  ! { dg-error "F2018: C8102" }
+      type is (t)
+        arg%i = arg%i + 1
+      type is (s)
+        arg%j = -1     ! s is caught at the SELECT TYPE statement
+    end select
+  end
+
+! This is OK
+  subroutine sub16 (arg)
+    import, only : t, s
+    implicit none
+    class(t) :: arg
+    class(t), allocatable ::  myclass
+    myclass =  t(1)
+    arg%i = myclass%i
+    select type (arg)
+      type is (t)
+        arg%i = arg%i + 1
+      type is (s)
+        arg%j = -1
+    end select
+  end
+
+  subroutine sub17 (arg)
+    import, only : t
+    implicit none
+    class(t) :: arg
+    call sub16 (arg)  ! { dg-error "F2018: C8102" }
+  end
+
+! Make sure that recursive procedures do not require the procedure itself to be imported.
+  recursive subroutine sub18 (arg)
+    import, none
+    implicit none
+    integer :: arg
+    if (arg <= 0) call sub18 (arg)
+    arg = 1
+  end
+
+  recursive integer function func4 (arg) result (res)
+    import, none
+    implicit none
+    integer :: arg
+    if (arg <= 0) arg = func4 (arg)
+    res = 1
+  end
+end
diff --git a/gcc/testsuite/gfortran.dg/import13.f90 b/gcc/testsuite/gfortran.dg/import13.f90
new file mode 100644
index 0000000..3bcfec3
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/import13.f90
@@ -0,0 +1,21 @@
+! { dg-do compile }
+!
+! Contributed by Steve Kargl  <sgk@troutmask.apl.washington.edu>
+!
+program foo
+   implicit none
+   integer i
+   i = 42
+   if (i /= 42) stop 1
+   call bah
+   contains
+      subroutine bah       ! { dg-error "is already defined at" }
+         i = 43
+         if (i /= 43) stop 2
+      end subroutine bah
+      subroutine bah       ! { dg-error "is already defined at" }
+         ! import statement missing a comma
+         import none       ! { dg-error "Unexpected IMPORT statement" }
+         i = 44            ! { dg-error "Unexpected assignment" }
+      end subroutine bah   ! { dg-error "Expecting END PROGRAM" }
+end program foo
diff --git a/gcc/testsuite/gfortran.dg/import3.f90 b/gcc/testsuite/gfortran.dg/import3.f90
index 74cd527..9288c6b 100644
--- a/gcc/testsuite/gfortran.dg/import3.f90
+++ b/gcc/testsuite/gfortran.dg/import3.f90
@@ -1,6 +1,8 @@
 ! { dg-do compile }
+! { dg-options "-std=f2008" }
 ! { dg-shouldfail "Invalid use of IMPORT" }
 ! Test invalid uses of import
+! Wording of some error messages change for -std>=F2018 but all are caught.
 ! PR fortran/29601
 
 subroutine test()
diff --git a/gcc/testsuite/gfortran.dg/move_alloc_20.f03 b/gcc/testsuite/gfortran.dg/move_alloc_20.f03
new file mode 100644
index 0000000..20403c3
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/move_alloc_20.f03
@@ -0,0 +1,151 @@
+! { dg-do run }
+!
+! Check the presence of the pre and post code of the FROM and TO arguments
+! of the MOVE_ALLOC intrinsic subroutine.
+
+module m
+  implicit none
+  type :: t
+    integer, allocatable :: a(:)
+  end type
+end module 
+
+module pre
+  use m
+  implicit none
+  private
+  public :: check_pre
+
+contains
+
+  subroutine check_pre
+    integer, parameter :: n = 5
+    type(t) :: x(n)
+    integer, allocatable :: tmp(:)
+    integer :: array(4) = [ -1, 0, 1, 2 ]
+    integer :: i
+
+    if (allocated(tmp)) error stop 1
+
+    tmp = [17]
+
+    if (.not. allocated(tmp)) error stop 11
+    if (any(shape(tmp) /= [1])) error stop 12
+    if (any(tmp /= [17])) error stop 13
+    do i=1,n
+      if (allocated(x(i)%a)) error stop 14
+    end do
+
+    ! Check that the index of X is properly computed for the evaluation of TO.
+    call move_alloc(tmp, x(sum(array))%a)
+
+    do i=1,n
+      if (i == 2) cycle
+      if (allocated(x(i)%a)) error stop 21
+    end do
+    if (.not. allocated(x(2)%a)) error stop 22
+    if (any(shape(x(2)%a) /= [1])) error stop 23
+    if (any(x(2)%a /= [17])) error stop 24
+    if (allocated(tmp)) error stop 25
+
+    ! Check that the index of X is properly computed for the evaluation of FROM.
+    call move_alloc(x(sum(array))%a, tmp)
+
+    if (.not. allocated(tmp)) error stop 31
+    if (any(shape(tmp) /= [1])) error stop 32
+    if (any(tmp /= [17])) error stop 33
+    do i=1,n
+      if (allocated(x(i)%a)) error stop 34
+    end do
+  end subroutine
+
+end module
+
+module post
+  use m
+  implicit none
+  private
+  public :: check_post
+  integer, parameter :: n = 5
+  type(t), target :: x(n)
+  type :: u
+    integer :: a
+  contains
+    final :: finalize
+  end type
+  integer :: finalization_count = 0
+
+contains
+
+  function idx(arg)
+    type(u) :: arg
+    integer :: idx
+    idx = mod(arg%a, n)
+  end function
+
+  subroutine check_post
+    type(u) :: y
+    integer, allocatable :: tmp(:)
+    integer, target :: array(4) = [ -1, 0, 1, 2 ]
+    integer :: i
+
+    y%a = 12
+
+    if (allocated(tmp)) error stop 1
+
+    tmp = [37]
+
+    if (.not. allocated(tmp)) error stop 11
+    if (any(shape(tmp) /= [1])) error stop 12
+    if (any(tmp /= [37])) error stop 13
+    if (finalization_count /= 0) error stop 14
+    do i=1,n
+      if (allocated(x(i)%a)) error stop 15
+    end do
+
+    ! Check that the cleanup code for the evaluation of TO is properly
+    ! executed after MOVE_ALLOC: the result of GET_U should be finalized.
+    call move_alloc(tmp, x(idx(get_u(y)))%a)
+
+    do i=1,n
+      if (i == 2) cycle
+      if (allocated(x(i)%a)) error stop 21
+    end do
+    if (.not. allocated(x(2)%a)) error stop 22
+    if (any(shape(x(2)%a) /= [1])) error stop 23
+    if (any(x(2)%a /= [37])) error stop 24
+    if (allocated(tmp)) error stop 25
+    if (finalization_count /= 1) error stop 26
+
+    ! Check that the cleanup code for the evaluation of FROM is properly
+    ! executed after MOVE_ALLOC: the result of GET_U should be finalized.
+    call move_alloc(x(idx(get_u(y)))%a, tmp)
+
+    if (.not. allocated(tmp)) error stop 31
+    if (any(shape(tmp) /= [1])) error stop 32
+    if (any(tmp /= [37])) error stop 33
+    if (finalization_count /= 2) error stop 34
+    do i=1,n
+      if (allocated(x(i)%a)) error stop 35
+    end do
+  end subroutine
+
+  function get_u(arg)
+    type(u) :: arg, get_u
+    get_u = arg
+  end function get_u
+
+  subroutine finalize(obj)
+    type(u) :: obj
+    finalization_count = finalization_count + 1
+  end subroutine
+
+end module
+
+program p
+  use pre
+  use post
+  implicit none
+  call check_pre
+  call check_post
+end program
diff --git a/gcc/testsuite/gfortran.dg/pointer_check_15.f90 b/gcc/testsuite/gfortran.dg/pointer_check_15.f90
new file mode 100644
index 0000000..13c6820
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/pointer_check_15.f90
@@ -0,0 +1,46 @@
+! { dg-do run }
+! { dg-additional-options "-O -fcheck=pointer -fdump-tree-original" }
+!
+! PR fortran/121145
+! Erroneous runtime error: Proc-pointer actual argument 'ptr' is not associated
+!
+! Contributed by Federico Perini.
+
+module m
+  implicit none
+
+  abstract interface
+     subroutine fun(x)
+       real, intent(in) :: x
+     end subroutine fun
+  end interface   
+
+contains
+
+  subroutine with_fun(sub)
+    procedure(fun), optional :: sub
+    if (present(sub)) stop 1
+  end subroutine   
+
+  subroutine with_non_optional(sub)
+    procedure(fun) :: sub
+  end subroutine   
+
+end module m
+
+program p
+  use m
+  implicit none
+
+  procedure(fun), pointer :: ptr1 => null()
+  procedure(fun), pointer :: ptr2 => null()
+  
+  call with_fun()
+  call with_fun(sub=ptr1)               ! no runtime check here
+
+  if (associated (ptr2)) then
+     call with_non_optional(sub=ptr2)   ! runtime check here
+  end if
+end  
+
+! { dg-final { scan-tree-dump-times "Proc-pointer actual argument .'ptr2.'" 1 "original" } }
diff --git a/gcc/testsuite/gfortran.dg/split_1.f90 b/gcc/testsuite/gfortran.dg/split_1.f90
new file mode 100644
index 0000000..21659b0
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/split_1.f90
@@ -0,0 +1,28 @@
+! { dg-do run }
+program b
+  character(len=:), allocatable :: input
+  character(len=2) :: set = ', '
+  integer :: p
+  input = " one,last example,"
+  p = 0
+
+  call split(input, set, p)
+  if (p /= 1) STOP 1
+  call split(input, set, p)
+  if (p /= 5) STOP 2
+  call split(input, set, p)
+  if (p /= 10) STOP 3
+  call split(input, set, p)
+  if (p /= 18) STOP 4
+  call split(input, set, p)
+  if (p /= 19) STOP 5
+
+  call split(input, set, p, .true.)
+  if (p /= 18) STOP 6
+  call split(input, set, p, .true.)
+  if (p /= 10) STOP 7
+  call split(input, set, p, .true.)
+  if (p /= 5) STOP 8
+  call split(input, set, p, .true.)
+  if (p /= 1) STOP 9
+end program b
diff --git a/gcc/testsuite/gfortran.dg/split_2.f90 b/gcc/testsuite/gfortran.dg/split_2.f90
new file mode 100644
index 0000000..9afb30b
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/split_2.f90
@@ -0,0 +1,22 @@
+! { dg-do run }
+program b
+  integer, parameter :: ucs4 = selected_char_kind('ISO_10646')
+  character(kind=ucs4, len=:), allocatable :: input, set
+  integer :: p = 0
+
+  input = char(int(z'4f60'), ucs4) // char(int(z'597d'), ucs4) // char(int(z'4f60'), ucs4) // char(int(z'4e16'), ucs4)
+  set = char(int(z'597d'), ucs4) // char(int(z'4e16'), ucs4)
+
+  call split(input, set, p)
+  if (p /= 2) stop 1
+  call split(input, set, p)
+  if (p /= 4) stop 2
+  call split(input, set, p)
+  if (p /= 5) stop 3
+  call split(input, set, p, .true.)
+  if (p /= 4) stop 4
+  call split(input, set, p, .true.)
+  if (p /= 2) stop 5
+  call split(input, set, p, .true.)
+  if (p /= 0) stop 6
+end program b
diff --git a/gcc/testsuite/gfortran.dg/split_3.f90 b/gcc/testsuite/gfortran.dg/split_3.f90
new file mode 100644
index 0000000..bec3fdc
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/split_3.f90
@@ -0,0 +1,11 @@
+! { dg-do run }
+! { dg-shouldfail "Fortran runtime error" }
+
+program b
+  character(len=:), allocatable :: input
+  character(len=2) :: set = ', '
+  integer :: p
+  input = " one,last example,"
+  p = -1
+  call split(input, set, p)
+end program b
diff --git a/gcc/testsuite/gfortran.dg/split_4.f90 b/gcc/testsuite/gfortran.dg/split_4.f90
new file mode 100644
index 0000000..a3c27bb
--- /dev/null
+++ b/gcc/testsuite/gfortran.dg/split_4.f90
@@ -0,0 +1,11 @@
+! { dg-do run }
+! { dg-shouldfail "Fortran runtime error" }
+
+program b
+  character(len=:), allocatable :: input
+  character(len=2) :: set = ', '
+  integer :: p
+  input = " one,last example,"
+  p = 0
+  call split(input, set, p, .true.)
+end program b
diff --git a/gcc/testsuite/gm2/errors/fail/badindrtype.mod b/gcc/testsuite/gm2/errors/fail/badindrtype.mod
new file mode 100644
index 0000000..b393027
--- /dev/null
+++ b/gcc/testsuite/gm2/errors/fail/badindrtype.mod
@@ -0,0 +1,16 @@
+MODULE badindrtype ;
+
+
+PROCEDURE init (VAR ch: CHAR) ;
+VAR
+   c: CARDINAL ;
+BEGIN
+   ch := c
+END init ;
+
+
+VAR
+   ch: CHAR ;
+BEGIN
+   init (ch)
+END badindrtype.
diff --git a/gcc/testsuite/gm2/errors/fail/badindrtype2.mod b/gcc/testsuite/gm2/errors/fail/badindrtype2.mod
new file mode 100644
index 0000000..a31303b
--- /dev/null
+++ b/gcc/testsuite/gm2/errors/fail/badindrtype2.mod
@@ -0,0 +1,16 @@
+MODULE badindrtype2 ;
+
+
+PROCEDURE init (VAR ch: CHAR) ;
+VAR
+   c: CARDINAL ;
+BEGIN
+   c := ch
+END init ;
+
+
+VAR
+   ch: CHAR ;
+BEGIN
+   init (ch)
+END badindrtype2.
diff --git a/gcc/testsuite/gm2/switches/pedantic-params/fail/arrayofchar.def b/gcc/testsuite/gm2/switches/pedantic-params/fail/arrayofchar.def
new file mode 100644
index 0000000..a24f7d3
--- /dev/null
+++ b/gcc/testsuite/gm2/switches/pedantic-params/fail/arrayofchar.def
@@ -0,0 +1,12 @@
+DEFINITION MODULE arrayofchar ;
+
+FROM FIO IMPORT File ;
+
+(*
+   Description: provides write procedures for ARRAY OF CHAR.
+*)
+
+PROCEDURE Write (f: File; str: ARRAY OF CHAR) ;
+PROCEDURE WriteLn (f: File) ;
+
+END arrayofchar.
diff --git a/gcc/testsuite/gm2/switches/pedantic-params/fail/arrayofchar.mod b/gcc/testsuite/gm2/switches/pedantic-params/fail/arrayofchar.mod
new file mode 100644
index 0000000..4e630a9
--- /dev/null
+++ b/gcc/testsuite/gm2/switches/pedantic-params/fail/arrayofchar.mod
@@ -0,0 +1,30 @@
+IMPLEMENTATION MODULE arrayofchar ;
+
+FROM FIO IMPORT WriteChar, WriteLine ;
+IMPORT StrLib ;
+
+
+(*
+   Write - writes a string to file f.
+*)
+
+PROCEDURE Write (f: File; a: ARRAY OF CHAR) ;
+VAR
+   len, i: CARDINAL ;
+BEGIN
+   len := StrLib.StrLen (a) ;
+   i := 0 ;
+   WHILE i < len DO
+      WriteChar (f, a[i]) ;
+      INC (i)
+   END
+END Write ;
+
+
+PROCEDURE WriteLn (f: File) ;
+BEGIN
+   WriteLine (f)
+END WriteLn ;
+
+
+END arrayofchar.
diff --git a/gcc/testsuite/gm2/warnings/style/fail/badvarname.mod b/gcc/testsuite/gm2/warnings/style/fail/badvarname.mod
new file mode 100644
index 0000000..e589b0d
--- /dev/null
+++ b/gcc/testsuite/gm2/warnings/style/fail/badvarname.mod
@@ -0,0 +1,14 @@
+MODULE badvarname ;
+
+
+PROCEDURE Foo ;
+VAR
+   end: CARDINAL ;
+BEGIN
+   end := 1
+END Foo ;
+
+
+BEGIN
+   Foo
+END badvarname.
diff --git a/gcc/testsuite/gm2/warnings/style/fail/warnings-style-fail.exp b/gcc/testsuite/gm2/warnings/style/fail/warnings-style-fail.exp
new file mode 100644
index 0000000..f44ed80
--- /dev/null
+++ b/gcc/testsuite/gm2/warnings/style/fail/warnings-style-fail.exp
@@ -0,0 +1,44 @@
+# Expect driver script for GCC Regression Tests
+# Copyright (C) 2025 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GCC; see the file COPYING3.  If not see
+# <http://www.gnu.org/licenses/>.
+
+# This file was written by Gaius Mulley (gaius.mulley@southwales.ac.uk)
+# for GNU Modula-2.
+
+if $tracelevel then {
+    strace $tracelevel
+}
+
+# load support procs
+load_lib gm2-torture.exp
+
+gm2_init_pim "${srcdir}/gm2/warnings/style/fail"
+
+global TORTURE_OPTIONS
+
+set old_options $TORTURE_OPTIONS
+set TORTURE_OPTIONS { { -O0 -g -Werror=style } }
+
+foreach testcase [lsort [glob -nocomplain $srcdir/$subdir/*.mod]] {
+    # If we're only testing specific files and this isn't one of them, skip it.
+    if ![runtest_file_p $runtests $testcase] then {
+	continue
+    }
+
+    gm2-torture-fail $testcase
+}
+
+set TORTURE_OPTIONS $old_options
diff --git a/gcc/testsuite/gnat.dg/deref4.adb b/gcc/testsuite/gnat.dg/deref4.adb
new file mode 100644
index 0000000..586a6186
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/deref4.adb
@@ -0,0 +1,9 @@
+-- { dg-do compile }
+-- { dg-options "-gnatX" }
+
+with Deref4_Pkg; use Deref4_Pkg;
+
+procedure Deref4 is
+begin
+  Obj.Proc (null);
+end;
diff --git a/gcc/testsuite/gnat.dg/deref4_pkg.ads b/gcc/testsuite/gnat.dg/deref4_pkg.ads
new file mode 100644
index 0000000..9410d0d
--- /dev/null
+++ b/gcc/testsuite/gnat.dg/deref4_pkg.ads
@@ -0,0 +1,8 @@
+package Deref4_Pkg is
+
+  type A is tagged null record;
+  type A_Ptr is access A;
+  procedure Proc (This : in out A'Class; Some_Parameter : A_Ptr) is null;
+  Obj : A_Ptr;
+
+end Deref4_Pkg;
diff --git a/gcc/testsuite/jit.dg/test-debuginfo.c b/gcc/testsuite/jit.dg/test-debuginfo.c
index 49e8834..e0d6f2d 100644
--- a/gcc/testsuite/jit.dg/test-debuginfo.c
+++ b/gcc/testsuite/jit.dg/test-debuginfo.c
@@ -1,5 +1,5 @@
 /* Essentially this test checks that debug info are generated for globals
-   locals and functions, including type info. The comment bellow is used
+   locals and functions, including type info. The comment below is used
    as fake code (does not affect the test, use for manual debugging). */
 /*
 int a_global_for_test_debuginfo;
diff --git a/gcc/testsuite/lib/gcc-defs.exp b/gcc/testsuite/lib/gcc-defs.exp
index 2f8b7d4..d66c833 100644
--- a/gcc/testsuite/lib/gcc-defs.exp
+++ b/gcc/testsuite/lib/gcc-defs.exp
@@ -599,15 +599,16 @@ proc aarch64-arch-dg-options { args } {
 
     set add_arch 1
     set add_tune 1
+    set add_override 1
     set checks_output [string equal [lindex $do_what 0] "compile"]
     set options [lindex $args 1]
 
     foreach option [split $options] {
 	switch -glob -- $option {
 	    -march=* { set add_arch 0 }
-	    -mcpu=* { set add_arch 0; set add_tune 0 }
-	    -mtune=* { set add_tune 0 }
-	    -moverride=* { set add_tune 0 }
+	    -mcpu=* { set add_arch 0; set add_tune 0; set add_override 0}
+	    -mtune=* { set add_tune 0; set add_override 0 }
+	    -moverride=* { set add_override 0 }
 	    -save-temps { set checks_output 1 }
 	    --save-temps { set checks_output 1 }
 	    -fdump* { set checks_output 1 }
@@ -619,9 +620,14 @@ proc aarch64-arch-dg-options { args } {
 	append options " $aarch64_default_testing_arch"
     }
 
-    if { $add_tune && $checks_output } {
+    if { $checks_output } {
 	# Turn off any default tuning and codegen tweaks.
-	append options " -mtune=generic -moverride=tune=none"
+        if { $add_tune } {
+            append options " -mtune=generic"
+        }
+        if { $add_override } {
+            append options " -moverride=tune=none"
+        }
     }
 
     uplevel 1 aarch64-old-dg-options [lreplace $args 1 1 $options]
diff --git a/gcc/testsuite/lib/gcc-dg.exp b/gcc/testsuite/lib/gcc-dg.exp
index 312c4b8..859a943 100644
--- a/gcc/testsuite/lib/gcc-dg.exp
+++ b/gcc/testsuite/lib/gcc-dg.exp
@@ -1338,8 +1338,8 @@ proc dg-missed { args } {
 }
 
 # Look for messages with 'note: ' prefixes.
-# In addition to standard compiler diagnostics ('DK_NOTE', 'inform' functions,
-# "for additional details on an error message"),
+# In addition to standard compiler diagnostics (diagnostics::kind::note,
+# 'inform' functions, "for additional details on an error message"),
 # this also includes output from '-fopt-info' for 'MSG_NOTE':
 # a general optimization info.
 # By default, any *excess* notes are pruned, meaning their appearance doesn't
diff --git a/gcc/testsuite/lib/profopt.exp b/gcc/testsuite/lib/profopt.exp
index b4d244b..81d86c6 100644
--- a/gcc/testsuite/lib/profopt.exp
+++ b/gcc/testsuite/lib/profopt.exp
@@ -382,6 +382,7 @@ proc profopt-execute { src } {
 	    unsupported "$testcase"
 	    unset testname_with_flags
 	    verbose "$src not supported on this target, skipping it" 3
+	    cleanup-after-saved-dg-test
 	    return
 	}
 
@@ -458,6 +459,7 @@ proc profopt-execute { src } {
 		    unsupported "$testcase -fauto-profile: cannot run create_gcov"
 		    unset testname_with_flags
 		    set status "fail"
+		    cleanup-after-saved-dg-test
 		    return
 		}
 		set status [remote_wait "" 300]
diff --git a/gcc/testsuite/lib/sarif.py b/gcc/testsuite/lib/sarif.py
index 384de2f..d75a87e 100644
--- a/gcc/testsuite/lib/sarif.py
+++ b/gcc/testsuite/lib/sarif.py
@@ -29,10 +29,24 @@ def get_result_by_index(sarif, idx):
     results = run['results']
     return results[idx]
     
-def get_xml_state(events, event_idx):
-    xml_src = events[event_idx]['properties']['gcc/diagnostic_event/xml_state']
+def get_state_graph(events, event_idx):
+    graph = events[event_idx]['properties']['gcc/diagnostics/paths/event/state_graph']
     if 0:
-        print(xml_src)
-    xml = ET.fromstring(xml_src)
-    assert xml.tag == 'state-diagram'
-    return xml    
+        print(graph)
+    assert graph is not None
+    return graph
+
+def get_state_node_attr(obj, attr_name):
+    return obj['properties']['gcc/diagnostic_state_node/%s' % attr_name]
+
+def get_state_node_kind(obj):
+    return get_state_node_attr(obj, 'kind')
+
+def get_state_node_name(obj):
+    return get_state_node_attr(obj, 'name')
+
+def get_state_node_type(obj):
+    return get_state_node_attr(obj, 'type')
+
+def get_state_node_value(obj):
+    return get_state_node_attr(obj, 'value')
diff --git a/gcc/testsuite/lib/scanasm.exp b/gcc/testsuite/lib/scanasm.exp
index 97935cb..51952a6 100644
--- a/gcc/testsuite/lib/scanasm.exp
+++ b/gcc/testsuite/lib/scanasm.exp
@@ -896,6 +896,10 @@ proc configure_check-function-bodies { config } {
 	set up_config(fluff) {^\s*(?://)}
     } elseif { [istarget *-*-darwin*] } {
 	set up_config(fluff) {^\s*(?:\.|//|@)|^L[0-9ABCESV]}
+    } elseif { [istarget s390*-*-*] } {
+	# Additionally to the defaults skip lines beginning with a # resulting
+	# from inline asm.
+	set up_config(fluff) {^\s*(?:\.|//|@|$|#)}
     } else {
 	# Skip lines beginning with labels ('.L[...]:') or other directives
 	# ('.align', '.cfi_startproc', '.quad [...]', '.text', etc.), '//' or
@@ -1109,6 +1113,8 @@ proc check-function-bodies { args } {
 		append function_regexp ".*"
 	    } elseif { [regexp {^\.L} $line] } {
 		append function_regexp $line "\n"
+	    } elseif { [regexp {^[0-9]+:} $line] } {
+		append function_regexp $line "\n"
 	    } else {
 		append function_regexp $config(line_prefix) $line "\n"
 	    }
diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp
index 956bc0b..7435519 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -1017,6 +1017,18 @@ proc check_effective_target_label_values {} {
     return 1
 }
 
+
+# Return 1 if builtin_trap expands not into a call but an instruction,
+# 0 otherwise.
+proc check_effective_target_trap { } {
+    return [check_no_messages_and_pattern trap "!\\(call" rtl-expand {
+	void foo ()
+	  {
+	    return __builtin_trap ();
+	  }
+    } "" ]
+}
+
 # Return 1 if builtin_return_address and builtin_frame_address are
 # supported, 0 otherwise.
 
@@ -2428,7 +2440,7 @@ proc check_effective_target_riscv_v_misalign_ok { } {
 		= {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
 	      asm ("vsetivli zero,7,e8,m1,ta,ma");
 	      asm ("addi a7,%0,1" : : "r" (a) : "a7" );
-	      asm ("vle8.v v8,0(a7)" : : : "v8");
+	      asm ("vle16.v v8,0(a7)" : : : "v8");
 	      return 0; } } "-march=${gcc_march}"] } {
 	return 1
     }
@@ -2685,6 +2697,14 @@ proc remove_options_for_riscv_zvbb { flags } {
     return [add_options_for_riscv_z_ext zvbb $flags]
 }
 
+proc add_options_for_riscv_zvfh { flags } {
+    return [add_options_for_riscv_z_ext zvfh $flags]
+}
+
+proc remove_options_for_riscv_zvfh { flags } {
+    return [add_options_for_riscv_z_ext zvfh $flags]
+}
+
 # Return 1 if the target is ia32 or x86_64.
 
 proc check_effective_target_x86 { } {
@@ -5800,6 +5820,13 @@ proc add_options_for_aarch64_sve { flags } {
     return "$flags -march=armv8.2-a+sve"
 }
 
+proc add_options_for_aarch64_sme { flags } {
+    if { ![istarget aarch64*-*-*] || [check_effective_target_aarch64_sme] } {
+	return "$flags"
+    }
+    return "$flags -march=armv9-a+sme"
+}
+
 # Return 1 if this is an ARM target supporting the FP16 alternative
 # format.  Some multilibs may be incompatible with the options needed.  Also
 # set et_arm_fp16_alternative_flags to the best options to add.
@@ -6491,6 +6518,23 @@ proc check_effective_target_aarch64_sve2_hw { } {
     }]
 }
 
+# Return true if this is an AArch64 target that can run SVE2.1 code.
+
+proc check_effective_target_aarch64_sve2p1_hw { } {
+    if { ![istarget aarch64*-*-*] } {
+	return 0
+    }
+    return [check_runtime aarch64_sve2p1_hw_available {
+	#pragma GCC target "+sve2p1"
+	int
+	main (void)
+	{
+	  asm volatile ("dupq z0.b, z0.b[0]");
+	  return 0;
+	}
+    }]
+}
+
 # Return true if this is an AArch64 target that can run SVE code and
 # if its SVE vectors have exactly BITS bits.
 
@@ -6522,6 +6566,22 @@ foreach N { 128 256 512 1024 2048 } {
     }]
 }
 
+# Return true if this is an AArch64 target that can run SME code.
+
+proc check_effective_target_aarch64_sme_hw { } {
+    if { ![istarget aarch64*-*-*] } {
+	return 0
+    }
+    return [check_runtime aarch64_sme_hw_available {
+	int
+	main (void)
+	{
+	  asm volatile ("rdsvl x0, #1");
+	  return 0;
+	}
+    } [add_options_for_aarch64_sme ""]]
+}
+
 proc check_effective_target_arm_neonv2_hw { } {
     return [check_runtime arm_neon_hwv2_available {
 	#include "arm_neon.h"
@@ -9944,7 +10004,8 @@ proc check_effective_target_vect_logical_reduc { } {
 		   || [istarget amdgcn-*-*]
 		   || [check_effective_target_riscv_v]
 		   || [check_effective_target_loongarch_sx]
-		   || [check_effective_target_x86]}]
+		   || [check_effective_target_x86]
+		   || [check_effective_target_s390_vx]}]
 }
 
 # Return 1 if the target supports the fold_extract_last optab.
@@ -12461,10 +12522,16 @@ proc check_effective_target_aarch64_gas_has_build_attributes { } {
 # various architecture extensions via the .arch_extension pseudo-op.
 
 set exts {
-    "bf16" "cmpbr" "crc" "crypto" "dotprod" "f32mm" "f64mm" "fp" "fp8"
-    "fp8dot2" "fp8dot4" "fp8fma" "i8mm" "ls64" "lse" "lut" "sb" "simd"
-    "sme-b16b16" "sme-f16f16" "sme-i16i64" "sme" "sme2" "sme2p1" "ssve-fp8dot2"
-    "ssve-fp8dot4" "ssve-fp8fma" "sve-b16b16" "sve" "sve2"
+    "bf16" "cmpbr" "crc" "crypto" "dotprod" "f32mm" "f64mm" "faminmax"
+    "fp" "fp8" "fp8dot2" "fp8dot4" "fp8fma" "i8mm" "ls64" "lse" "lut"
+    "sb" "simd" "sve-b16b16" "sve" "sve2"
+}
+
+# We don't support SME without SVE2, so we'll use armv9 as the base
+# archiecture for SME and the features that require it.
+set exts_sve2 {
+    "sme-b16b16" "sme-f16f16" "sme-i16i64" "sme" "sme2" "sme2p1"
+    "ssve-fp8dot2" "ssve-fp8dot4" "ssve-fp8fma"
 }
 
 foreach { aarch64_ext } $exts {
@@ -12481,6 +12548,20 @@ foreach { aarch64_ext } $exts {
     }]
 }
 
+foreach { aarch64_ext } $exts_sve2 {
+    eval [string map [list FUNC $aarch64_ext] {
+	proc check_effective_target_aarch64_asm_FUNC_ok { } {
+	  if { [istarget aarch64*-*-*] } {
+		return [check_no_compiler_messages aarch64_FUNC_assembler object {
+			__asm__ (".arch_extension FUNC");
+		} "-march=armv9-a+FUNC"]
+	  } else {
+		return 0
+	  }
+	}
+    }]
+}
+
 proc check_effective_target_aarch64_asm_sve2p1_ok { } {
     if { [istarget aarch64*-*-*] } {
 	return [check_no_compiler_messages aarch64_sve2p1_assembler object {
@@ -14511,3 +14592,51 @@ proc check_effective_target_foldable_pi_based_trigonometry { } {
 	}
     }]
 }
+#
+# Return 1 if the x86-64 target enables -mfentry by default, 0
+# otherwise.  Cache the result.
+
+proc check_effective_target_fentry { } {
+    global tool
+    global GCC_UNDER_TEST
+
+    if { ![check_effective_target_x86] } {
+	return 0
+    }
+
+    # Need auto-host.h to check linker support.
+    if { ![file exists ../../auto-host.h ] } {
+	return 0
+    }
+
+    return [check_cached_effective_target fentry {
+	# Set up and compile to see if ENABLE_X86_64_MFENTRY is
+	# non-zero.  Include the current process ID in the file
+	# names to prevent conflicts with invocations for multiple
+	# testsuites.
+
+	set src pie[pid].c
+	set obj pie[pid].o
+
+	set f [open $src "w"]
+	puts $f "#include \"../../auto-host.h\""
+	puts $f "#if ENABLE_X86_64_MFENTRY == 0 || !defined __x86_64__"
+	puts $f "# error -mfentry is not enabled by default."
+	puts $f "#endif"
+	close $f
+
+	verbose "check_effective_target_fentry compiling testfile $src" 2
+	set lines [${tool}_target_compile $src $obj object ""]
+
+	file delete $src
+	file delete $obj
+
+	if [string match "" $lines] then {
+	    verbose "check_effective_target_fentry testfile compilation passed" 2
+	    return 1
+	} else {
+	    verbose "check_effective_target_fentry testfile compilation failed" 2
+	    return 0
+	}
+    }]
+}
diff --git a/gcc/testsuite/libgdiagnostics.dg/sarif.py b/gcc/testsuite/libgdiagnostics.dg/sarif.py
deleted file mode 100644
index 7daf35b..0000000
--- a/gcc/testsuite/libgdiagnostics.dg/sarif.py
+++ /dev/null
@@ -1,23 +0,0 @@
-import json
-import os
-
-def sarif_from_env():
-    # return parsed JSON content a SARIF_PATH file
-    json_filename = os.environ['SARIF_PATH']
-    json_filename += '.sarif'
-    print('json_filename: %r' % json_filename)
-    with open(json_filename) as f:
-        json_data = f.read()
-    return json.loads(json_data)
-
-def get_location_artifact_uri(location):
-    return location['physicalLocation']['artifactLocation']['uri']
-
-def get_location_physical_region(location):
-    return location['physicalLocation']['region']
-
-def get_location_snippet_text(location):
-    return location['physicalLocation']['contextRegion']['snippet']['text']
-
-def get_location_relationships(location):
-    return location['relationships']
diff --git a/gcc/testsuite/libgdiagnostics.dg/test-message-buffer-c.py b/gcc/testsuite/libgdiagnostics.dg/test-message-buffer-c.py
new file mode 100644
index 0000000..9d14b9a
--- /dev/null
+++ b/gcc/testsuite/libgdiagnostics.dg/test-message-buffer-c.py
@@ -0,0 +1,12 @@
+from sarif import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def sarif():
+    return sarif_from_env()
+
+def test_message_in_generated_sarif(sarif):
+    result = get_result_by_index(sarif, 0)
+    assert result['level'] == 'error'
+    assert result['message']['text'] == "this is a string; foo; int: 42 str: mostly harmless; [this is a link](https://example.com/) 'this is quoted' highlight A highlight B (1)."
diff --git a/gcc/testsuite/libgdiagnostics.dg/test-message-buffer.c b/gcc/testsuite/libgdiagnostics.dg/test-message-buffer.c
new file mode 100644
index 0000000..a958fc5
--- /dev/null
+++ b/gcc/testsuite/libgdiagnostics.dg/test-message-buffer.c
@@ -0,0 +1,80 @@
+/* Example of using a message buffer to build the text of a diagnostic
+   in pieces before emitting it.  */
+
+#include "libgdiagnostics.h"
+#include "test-helpers.h"
+
+int
+main ()
+{
+  begin_test ("test-message-buffer.c.exe",
+	      "test-message-buffer.c.sarif",
+	      __FILE__, "c");
+
+  diagnostic_event_id event_id = 0;
+
+  /* begin quoted source */
+  diagnostic *d = diagnostic_begin (diag_mgr,
+				    DIAGNOSTIC_LEVEL_ERROR);
+
+  diagnostic_message_buffer *msg_buf = diagnostic_message_buffer_new ();
+
+  /* Add a null-terminated string.  */
+  diagnostic_message_buffer_append_str (msg_buf, "this is a string; ");
+
+  /* Add a length-specified string.  */
+  diagnostic_message_buffer_append_text (msg_buf, "foobar", 3);
+
+  /* "printf"-formatting.  */
+  diagnostic_message_buffer_append_printf (msg_buf,
+					   "; int: %i str: %s; ",
+					   42, "mostly harmless");
+
+  /* Adding a URL.  */
+  diagnostic_message_buffer_begin_url (msg_buf, "https://example.com/");
+  diagnostic_message_buffer_append_str (msg_buf, "this is a link");  
+  diagnostic_message_buffer_end_url (msg_buf);
+
+  diagnostic_message_buffer_append_str (msg_buf, " ");  
+
+  /* Add quoted text.  */
+  diagnostic_message_buffer_begin_quote (msg_buf);
+  diagnostic_message_buffer_append_str (msg_buf, "this is quoted");  
+  diagnostic_message_buffer_end_quote (msg_buf);
+  
+  diagnostic_message_buffer_append_str (msg_buf, " ");  
+
+  /* Add colorized text.  */
+  diagnostic_message_buffer_begin_color (msg_buf, "highlight-a");
+  diagnostic_message_buffer_append_str (msg_buf, "highlight A");  
+  diagnostic_message_buffer_end_color (msg_buf);
+
+  diagnostic_message_buffer_append_str (msg_buf, " ");
+  
+  diagnostic_message_buffer_begin_color (msg_buf, "highlight-b");
+  diagnostic_message_buffer_append_str (msg_buf, "highlight B");  
+  diagnostic_message_buffer_end_color (msg_buf);
+  
+  diagnostic_message_buffer_append_str (msg_buf, " ");
+
+  /* Add an event ID.  This will be printed as "(1)".  */
+  diagnostic_message_buffer_append_event_id (msg_buf, event_id);
+  
+  /* Add an ASCII char.  */
+  diagnostic_message_buffer_append_byte (msg_buf, '.');
+
+  diagnostic_finish_via_msg_buf (d, msg_buf);
+  /* end quoted source */
+
+  return end_test ();
+};
+
+/* Verify the output from the text sink.
+   { dg-regexp "test-message-buffer.c.exe: error: this is a string; foo; int: 42 str: mostly harmless; this is a link 'this is quoted' highlight A highlight B \\(1\\)." } */
+
+/* Verify that some JSON was written to a file with the expected name:
+   { dg-final { verify-sarif-file } } */
+
+/* Use a Python script to verify various properties about the generated
+   .sarif file:
+   { dg-final { run-sarif-pytest test-message-buffer.c "test-message-buffer-c.py" } } */
diff --git a/gcc/testsuite/libgdiagnostics.dg/test-warning-with-path-c.py b/gcc/testsuite/libgdiagnostics.dg/test-warning-with-path-c.py
index af1e7b9..61ccb93 100644
--- a/gcc/testsuite/libgdiagnostics.dg/test-warning-with-path-c.py
+++ b/gcc/testsuite/libgdiagnostics.dg/test-warning-with-path-c.py
@@ -101,7 +101,7 @@ def test_sarif_output_for_warning_with_path(sarif):
         == '    PyList_Append(list, item);\n'
     assert tfl_2_loc['logicalLocations'] == location['logicalLocations']
     assert tfl_2_loc['message']['text'] \
-        == "when calling 'PyList_Append', passing NULL from (1) as argument 1"
+        == "when calling 'PyList_Append', passing NULL from [(1)](sarif:/runs/0/results/0/codeFlows/0/threadFlows/0/locations/0) as argument 1"
     assert tfl_2['nestingLevel'] == 0
     assert tfl_2['executionOrder'] == 3
 
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-invalid/3.40.2-duplicate-node-id.sarif b/gcc/testsuite/sarif-replay.dg/2.1.0-invalid/3.40.2-duplicate-node-id.sarif
new file mode 100644
index 0000000..a4eb9d4
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-invalid/3.40.2-duplicate-node-id.sarif
@@ -0,0 +1,23 @@
+{"$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
+ "version": "2.1.0",
+ "runs": [{"tool": { "driver": { "name": "example" } },
+           "results": [],
+           "graphs": [{"nodes": [{"id": "a", /* { dg-message "'a' already used as node id within graph here" } */
+                                  "children": [{"id": "a"}]}], /* { dg-error "duplicate node id 'a' within graph \\\[SARIF v2.1.0 §3.40.2\\\]" } */
+                       "edges": []}]}]}
+
+/* { dg-begin-multiline-output "" }
+In JSON property '/runs/0/graphs/0/nodes/0/children/0/id':
+   { dg-end-multiline-output "" } */
+/* { dg-begin-multiline-output "" }
+    6 |                                   "children": [{"id": "a"}]}],
+      |                                                       ^~~
+   { dg-end-multiline-output "" }  */
+
+/* { dg-begin-multiline-output "" }
+In JSON property '/runs/0/graphs/0/nodes/0/id':
+   { dg-end-multiline-output "" } */
+/* { dg-begin-multiline-output "" }
+    5 |            "graphs": [{"nodes": [{"id": "a",
+      |                                         ^~~
+   { dg-end-multiline-output "" } */
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-invalid/3.41.4-unrecognized-node-id.sarif b/gcc/testsuite/sarif-replay.dg/2.1.0-invalid/3.41.4-unrecognized-node-id.sarif
new file mode 100644
index 0000000..b483346
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-invalid/3.41.4-unrecognized-node-id.sarif
@@ -0,0 +1,16 @@
+{"$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
+ "version": "2.1.0",
+ "runs": [{"tool": { "driver": { "name": "example" } },
+           "results": [],
+           "graphs": [{"nodes": [],
+                       "edges": [{"id": "edge0",
+                                  "sourceNodeId": "this-does-not-exist", /* { dg-error "no node with id 'this-does-not-exist' in graph \\\[SARIF v2.1.0 §3.41.4\\\]" } */
+                                  "targetNodeId": "neither-does-this"}]}]}]}
+
+/* { dg-begin-multiline-output "" }
+In JSON property '/runs/0/graphs/0/edges/0/sourceNodeId':
+   { dg-end-multiline-output "" } */
+/* { dg-begin-multiline-output "" }
+    7 |                                   "sourceNodeId": "this-does-not-exist",
+      |                                                   ^~~~~~~~~~~~~~~~~~~~~
+   { dg-end-multiline-output "" }  */
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/3.11.6-embedded-links.sarif b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/3.11.6-embedded-links.sarif
index bc64521..cd7b822 100644
--- a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/3.11.6-embedded-links.sarif
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/3.11.6-embedded-links.sarif
@@ -1,3 +1,6 @@
+/* { dg-additional-options "-fdiagnostics-add-output=experimental-html:file=3.11.6-embedded-links.sarif.html,javascript=no" } */
+/* { dg-additional-options "-fdiagnostics-add-output=sarif:file=3.11.6-embedded-links.sarif.roundtrip.sarif" } */
+
 {"$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
  "version": "2.1.0",
  "runs": [{"tool": {"driver": {"name": "hand-written"}},
@@ -16,10 +19,24 @@ hand-written: warning: 002: Prohibited term used in [para\[0\]\\spans\[2\](1).
                        
                        /* With the fix from https://github.com/oasis-tcs/sarif-spec/issues/656 */
                        {"message": {"text": "003: Prohibited term used in [para\\[0\\]\\\\spans\\[2\\]](1)."},
-	   	        "locations": []}
+	   	        "locations": []},
 /* { dg-begin-multiline-output "" }
 hand-written: warning: 003: Prohibited term used in para[0]\spans[2].
    { dg-end-multiline-output "" } */
 
+                       {"message": {"text": "004: This is a [link](http://www.example.com)."},
+	   	        "locations": []}
+/* { dg-begin-multiline-output "" }
+hand-written: warning: 004: This is a link.
+   { dg-end-multiline-output "" } */
+
 ]}]}
 
+/* Use a Python script to verify various properties about the generated
+   .html file:
+   { dg-final { run-html-pytest 3.11.6-embedded-links.sarif "2.1.0-valid/embedded-links-check-html.py" } } */
+
+/* Use a Python script to verify various properties about the *generated*
+   .sarif file:
+   { dg-final { run-sarif-pytest 3.11.6-embedded-links.sarif.roundtrip "2.1.0-valid/embedded-links-check-sarif-roundtrip.py" } } */
+
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/embedded-links-check-html.py b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/embedded-links-check-html.py
new file mode 100644
index 0000000..ff1c2f2
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/embedded-links-check-html.py
@@ -0,0 +1,28 @@
+from htmltest import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def html_tree():
+    return html_tree_from_env()
+
+def test_generated_html(html_tree):
+    root = html_tree.getroot ()
+    assert root.tag == make_tag('html')
+
+    head = root.find('xhtml:head', ns)
+    assert head is not None
+
+    # Get "warning: 004: This is a link."
+    diag = get_diag_by_index(html_tree, 3)
+
+    msg = get_message_within_diag(diag)
+    assert msg is not None
+    
+    assert_tag(msg[0], 'strong')
+    assert msg[0].text == 'warning: '
+    assert msg[0].tail == ' 004: This is a '
+    assert_tag(msg[1], 'a')
+    assert msg[1].text == 'link'
+    assert msg[1].get('href') == 'http://www.example.com'
+    assert msg[1].tail == '. '
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/embedded-links-check-sarif-roundtrip.py b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/embedded-links-check-sarif-roundtrip.py
new file mode 100644
index 0000000..171339e
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/embedded-links-check-sarif-roundtrip.py
@@ -0,0 +1,13 @@
+from sarif import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def sarif():
+    return sarif_from_env()
+
+def test_roundtrip_of_url_in_generated_sarif(sarif):
+    # Get "warning: 004: This is a link."
+    result = get_result_by_index(sarif, 3)
+    assert result['level'] == 'warning'
+    assert result['message']['text'] == "004: This is a [link](http://www.example.com)."
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs-check-html.py b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs-check-html.py
new file mode 100644
index 0000000..63b80c9
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs-check-html.py
@@ -0,0 +1,46 @@
+from htmltest import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def html_tree():
+    return html_tree_from_env()
+
+def test_result_graph(html_tree):
+    root = html_tree.getroot ()
+    assert root.tag == make_tag('html')
+
+    body = root.find('xhtml:body', ns)
+    assert body is not None
+
+    diag_list = body.find('xhtml:div', ns)
+    assert diag_list is not None
+    assert diag_list.attrib['class'] == 'gcc-diagnostic-list'
+
+    diag = diag_list.find('xhtml:div', ns)
+    assert diag is not None
+
+    message = diag.find("./xhtml:div[@class='gcc-message']", ns)
+    assert message.attrib['id'] == 'gcc-diag-0-message'
+
+    assert message[0].tag == make_tag('strong')
+    assert message[0].tail == ' this is a placeholder error, with graphs '
+
+    graph = diag.find("./xhtml:div[@class='gcc-directed-graph']", ns)
+    assert graph is not None
+
+    header = graph.find("./xhtml:h2", ns)
+    assert header.text == 'foo'
+
+def test_run_graph(html_tree):
+    root = html_tree.getroot ()
+    assert root.tag == make_tag('html')
+
+    body = root.find('xhtml:body', ns)
+    assert body is not None
+
+    graph = body.find("./xhtml:div[@class='gcc-directed-graph']", ns)
+    assert graph is not None
+
+    header = graph.find("./xhtml:h2", ns)
+    assert header.text == 'Optimization Passes'
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs-check-sarif-roundtrip.py b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs-check-sarif-roundtrip.py
new file mode 100644
index 0000000..4bb7535
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs-check-sarif-roundtrip.py
@@ -0,0 +1,55 @@
+from sarif import *
+
+import pytest
+
+@pytest.fixture(scope='function', autouse=True)
+def sarif():
+    return sarif_from_env()
+
+def test_basics(sarif):
+    schema = sarif['$schema']
+    assert schema == "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json"
+
+    version = sarif['version']
+    assert version == "2.1.0"
+
+def test_result_graph(sarif):
+    runs = sarif['runs']
+    run = runs[0]
+    results = run['results']
+
+    assert len(results) == 1
+    
+    result = results[0]
+    assert result['level'] == 'error'
+    assert result['message']['text'] == "this is a placeholder error, with graphs"
+
+    assert len(result['graphs']) == 2
+
+    assert result['graphs'][0]['description']['text'] == 'foo'
+
+    assert len(result['graphs'][0]['nodes']) == 2
+    assert result['graphs'][0]['nodes'][0]['id'] == 'a'
+    assert result['graphs'][0]['nodes'][1]['id'] == 'b'
+    assert result['graphs'][0]['nodes'][1]['properties']['/placeholder-prefix/color'] == 'red'
+    assert len(result['graphs'][0]['nodes'][1]['children']) == 1
+    assert result['graphs'][0]['nodes'][1]['children'][0]['id'] == 'c'
+    assert result['graphs'][0]['nodes'][1]['children'][0]['label']['text'] == 'I am a node label'
+
+    assert len(result['graphs'][0]['edges']) == 1
+    result['graphs'][0]['edges'][0]['id'] == 'my-edge'
+    assert result['graphs'][0]['edges'][0]['label']['text'] == 'I am an edge label'
+    assert result['graphs'][0]['edges'][0]['sourceNodeId'] == 'a'
+    assert result['graphs'][0]['edges'][0]['targetNodeId'] == 'c'
+    
+    assert result['graphs'][1]['description']['text'] == 'bar'
+
+def test_run_graph(sarif):
+    runs = sarif['runs']
+    run = runs[0]
+
+    assert len(run['graphs']) == 1
+
+    assert run['graphs'][0]['description']['text'] == 'Optimization Passes'
+    assert run['graphs'][0]['nodes'][0]['id'] == 'all_lowering_passes'
+    assert run['graphs'][0]['edges'][0]['id'] == 'edge0'
diff --git a/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs.sarif b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs.sarif
new file mode 100644
index 0000000..da236ba
--- /dev/null
+++ b/gcc/testsuite/sarif-replay.dg/2.1.0-valid/graphs.sarif
@@ -0,0 +1,2445 @@
+/* Test a replay of a .sarif file generated from GCC testsuite.
+
+   The dg directives were stripped out from the generated .sarif
+   to avoid confusing DejaGnu for this test.   */
+/* { dg-additional-options "-fdiagnostics-add-output=experimental-html:file=graphs.sarif.html,javascript=no" } */
+/* { dg-additional-options "-fdiagnostics-add-output=sarif:file=graphs.roundtrip.sarif" } */
+
+{"$schema": "https://docs.oasis-open.org/sarif/sarif/v2.1.0/errata01/os/schemas/sarif-schema-2.1.0.json",
+ "version": "2.1.0",
+ "runs": [{"tool": {"driver": {"name": "GNU C23",
+                               "fullName": "GNU C23 (GCC) version 16.0.0 20250702 (experimental) (x86_64-pc-linux-gnu)",
+                               "version": "16.0.0 20250702 (experimental)",
+                               "informationUri": "https://gcc.gnu.org/gcc-16/",
+                               "rules": []},
+                    "extensions": [{"name": "diagnostic_plugin_test_graphs",
+                                    "fullName": "./diagnostic_plugin_test_graphs.so"}]},
+           "invocations": [{"arguments": ["/home/david/gcc-newgit-gcc16/build/gcc/cc1",
+                                          "-quiet",
+                                          "-iprefix",
+                                          "/usr/local/lib/gcc/x86_64-pc-linux-gnu/16.0.0/",
+                                          "-isystem",
+                                          "/home/david/gcc-newgit-gcc16/build/gcc/include",
+                                          "-isystem",
+                                          "/home/david/gcc-newgit-gcc16/build/gcc/include-fixed",
+                                          "-iplugindir=/home/david/gcc-newgit-gcc16/build/gcc/plugin",
+                                          "/home/david/gcc-newgit-gcc16/src/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c",
+                                          "-iplugindir=/home/david/gcc-newgit-gcc16/build/gcc/plugin",
+                                          "-quiet",
+                                          "-dumpbase",
+                                          "diagnostic-test-graphs-sarif.c",
+                                          "-dumpbase-ext",
+                                          ".c",
+                                          "-mtune=generic",
+                                          "-march=x86-64",
+                                          "-fdiagnostics-color=never",
+                                          "-fdiagnostics-urls=never",
+                                          "-fno-diagnostics-show-caret",
+                                          "-fno-diagnostics-show-line-numbers",
+                                          "-fdiagnostics-path-format=separate-events",
+                                          "-fdiagnostics-text-art-charset=none",
+                                          "-fno-diagnostics-show-event-links",
+                                          "-fplugin=./diagnostic_plugin_test_graphs.so",
+                                          "-fdiagnostics-add-output=sarif",
+                                          "-o",
+                                          "diagnostic-test-graphs-sarif.s"],
+                            "workingDirectory": {"uri": "/home/david/gcc-newgit-gcc16/build/gcc/testsuite/gcc"},
+                            "startTimeUtc": "2025-07-09T22:43:31Z",
+                            "executionSuccessful": false,
+                            "toolExecutionNotifications": [],
+                            "endTimeUtc": "2025-07-09T22:43:31Z"}],
+           "artifacts": [{"location": {"uri": "/home/david/gcc-newgit-gcc16/src/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c"},
+                          "sourceLanguage": "c",
+                          "roles": ["analysisTarget"]}],
+           "results": [{"ruleId": "error",
+                        "level": "error",
+                        "message": {"text": "this is a placeholder error, with graphs"},
+                        "locations": [{"physicalLocation": {"artifactLocation": {"uri": "/home/david/gcc-newgit-gcc16/src/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c"},
+                                                            "region": {"startLine": 8,
+                                                                       "startColumn": 3,
+                                                                       "endColumn": 10},
+                                                            "contextRegion": {"startLine": 8,
+                                                                              "snippet": {"text": "  here ();"}}},
+                                       "logicalLocations": [{"index": 0,
+                                                             "fullyQualifiedName": "test_graphs"}]}],
+                        "graphs": [{"description": {"text": "foo"},
+                                    "nodes": [{"id": "a"},
+                                              {"id": "b",
+                                               "properties": {"/placeholder-prefix/color": "red"},
+                                               "children": [{"id": "c",
+                                                             "label": {"text": "I am a node label"}}]}],
+                                    "edges": [{"id": "my-edge",
+                                               "label": {"text": "I am an edge label"},
+                                               "sourceNodeId": "a",
+                                               "targetNodeId": "c"}]},
+                                   {"description": {"text": "bar"},
+                                    "nodes": [{"id": "a"},
+                                              {"id": "b",
+                                               "properties": {"/placeholder-prefix/color": "red"},
+                                               "children": [{"id": "c",
+                                                             "label": {"text": "I am a node label"}}]}],
+                                    "edges": [{"id": "my-edge",
+                                               "label": {"text": "I am an edge label"},
+                                               "sourceNodeId": "a",
+                                               "targetNodeId": "c"}]}]}],
+           "logicalLocations": [{"name": "test_graphs",
+                                 "fullyQualifiedName": "test_graphs",
+                                 "decoratedName": "test_graphs",
+                                 "kind": "function",
+                                 "index": 0}],
+           "graphs": [{"description": {"text": "Optimization Passes"},
+                       "nodes": [{"id": "all_lowering_passes",
+                                  "label": {"text": "all_lowering_passes"},
+                                  "children": [{"id": "*warn_unused_result_0x101ef3d0",
+                                                "label": {"text": "*warn_unused_result"}},
+                                               {"id": "*diagnose_omp_blocks_0x101ef430",
+                                                "label": {"text": "*diagnose_omp_blocks"}},
+                                               {"id": "*diagnose_tm_blocks_0x101ef490",
+                                                "label": {"text": "*diagnose_tm_blocks"}},
+                                               {"id": "13_omp_oacc_kernels_decompose",
+                                                "label": {"text": "13_omp_oacc_kernels_decompose"}},
+                                               {"id": "14_omplower",
+                                                "label": {"text": "14_omplower"}},
+                                               {"id": "15_lower",
+                                                "label": {"text": "15_lower"}},
+                                               {"id": "16_tmlower",
+                                                "label": {"text": "16_tmlower"}},
+                                               {"id": "17_ehopt",
+                                                "label": {"text": "17_ehopt"}},
+                                               {"id": "18_eh",
+                                                "label": {"text": "18_eh"}},
+                                               {"id": "19_coro-lower-builtins",
+                                                "label": {"text": "19_coro-lower-builtins"}},
+                                               {"id": "20_cfg",
+                                                "label": {"text": "20_cfg"}},
+                                               {"id": "*warn_function_return_0x101ef7f0",
+                                                "label": {"text": "*warn_function_return"}},
+                                               {"id": "21_coro-early-expand-ifns",
+                                                "label": {"text": "21_coro-early-expand-ifns"}},
+                                               {"id": "22_ompexp",
+                                                "label": {"text": "22_ompexp"}},
+                                               {"id": "*build_cgraph_edges_0x101ef910",
+                                                "label": {"text": "*build_cgraph_edges"}}]},
+                                 {"id": "all_small_ipa_passes",
+                                  "label": {"text": "all_small_ipa_passes"},
+                                  "children": [{"id": "23_afdo_offline",
+                                                "label": {"text": "23_afdo_offline"}},
+                                               {"id": "*free_lang_data_0x101ef9d0",
+                                                "label": {"text": "*free_lang_data"}},
+                                               {"id": "24_visibility",
+                                                "label": {"text": "24_visibility"}},
+                                               {"id": "25_strubm",
+                                                "label": {"text": "25_strubm"}},
+                                               {"id": "26_build_ssa_passes",
+                                                "label": {"text": "26_build_ssa_passes"}},
+                                               {"id": "27_fixup_cfg",
+                                                "label": {"text": "27_fixup_cfg"}},
+                                               {"id": "28_ssa",
+                                                "label": {"text": "28_ssa"}},
+                                               {"id": "376_test_graph_emission",
+                                                "label": {"text": "376_test_graph_emission"}},
+                                               {"id": "29_walloca",
+                                                "label": {"text": "29_walloca"}},
+                                               {"id": "30_warn-printf",
+                                                "label": {"text": "30_warn-printf"}},
+                                               {"id": "*nonnullcmp_0x101efce0",
+                                                "label": {"text": "*nonnullcmp"}},
+                                               {"id": "31_early_uninit",
+                                                "label": {"text": "31_early_uninit"}},
+                                               {"id": "32_waccess",
+                                                "label": {"text": "32_waccess"}},
+                                               {"id": "33_ubsan",
+                                                "label": {"text": "33_ubsan"}},
+                                               {"id": "34_nothrow",
+                                                "label": {"text": "34_nothrow"}},
+                                               {"id": "*rebuild_cgraph_edges_0x101f0020",
+                                                "label": {"text": "*rebuild_cgraph_edges"}},
+                                               {"id": "35_opt_local_passes",
+                                                "label": {"text": "35_opt_local_passes"}},
+                                               {"id": "36_fixup_cfg",
+                                                "label": {"text": "36_fixup_cfg"}},
+                                               {"id": "*rebuild_cgraph_edges_0x101f0140",
+                                                "label": {"text": "*rebuild_cgraph_edges"}},
+                                               {"id": "37_local-fnsummary",
+                                                "label": {"text": "37_local-fnsummary"}},
+                                               {"id": "38_einline",
+                                                "label": {"text": "38_einline"}},
+                                               {"id": "*infinite-recursion_0x101f0260",
+                                                "label": {"text": "*infinite-recursion"}},
+                                               {"id": "39_early_optimizations",
+                                                "label": {"text": "39_early_optimizations"}},
+                                               {"id": "*remove_cgraph_callee_edges_0x101f0340",
+                                                "label": {"text": "*remove_cgraph_callee_edges"}},
+                                               {"id": "40_early_objsz",
+                                                "label": {"text": "40_early_objsz"}},
+                                               {"id": "41_ccp",
+                                                "label": {"text": "41_ccp"}},
+                                               {"id": "42_forwprop",
+                                                "label": {"text": "42_forwprop"}},
+                                               {"id": "43_ethread",
+                                                "label": {"text": "43_ethread"}},
+                                               {"id": "44_esra",
+                                                "label": {"text": "44_esra"}},
+                                               {"id": "45_ealias",
+                                                "label": {"text": "45_ealias"}},
+                                               {"id": "46_phiprop",
+                                                "label": {"text": "46_phiprop"}},
+                                               {"id": "47_fre",
+                                                "label": {"text": "47_fre"}},
+                                               {"id": "48_evrp",
+                                                "label": {"text": "48_evrp"}},
+                                               {"id": "49_mergephi",
+                                                "label": {"text": "49_mergephi"}},
+                                               {"id": "50_dse",
+                                                "label": {"text": "50_dse"}},
+                                               {"id": "51_cddce",
+                                                "label": {"text": "51_cddce"}},
+                                               {"id": "52_phiopt",
+                                                "label": {"text": "52_phiopt"}},
+                                               {"id": "53_tailr",
+                                                "label": {"text": "53_tailr"}},
+                                               {"id": "54_iftoswitch",
+                                                "label": {"text": "54_iftoswitch"}},
+                                               {"id": "55_switchconv",
+                                                "label": {"text": "55_switchconv"}},
+                                               {"id": "56_ehcleanup",
+                                                "label": {"text": "56_ehcleanup"}},
+                                               {"id": "57_sccopy",
+                                                "label": {"text": "57_sccopy"}},
+                                               {"id": "58_profile_estimate",
+                                                "label": {"text": "58_profile_estimate"}},
+                                               {"id": "59_local-pure-const",
+                                                "label": {"text": "59_local-pure-const"}},
+                                               {"id": "60_modref",
+                                                "label": {"text": "60_modref"}},
+                                               {"id": "61_fnsplit",
+                                                "label": {"text": "61_fnsplit"}},
+                                               {"id": "*strip_predict_hints_0x101f0c60",
+                                                "label": {"text": "*strip_predict_hints"}},
+                                               {"id": "62_release_ssa",
+                                                "label": {"text": "62_release_ssa"}},
+                                               {"id": "*rebuild_cgraph_edges_0x101f0d30",
+                                                "label": {"text": "*rebuild_cgraph_edges"}},
+                                               {"id": "63_local-fnsummary",
+                                                "label": {"text": "63_local-fnsummary"}},
+                                               {"id": "64_remove_symbols",
+                                                "label": {"text": "64_remove_symbols"}},
+                                               {"id": "65_strub",
+                                                "label": {"text": "65_strub"}},
+                                               {"id": "66_ipa_oacc",
+                                                "label": {"text": "66_ipa_oacc"}},
+                                               {"id": "67_pta",
+                                                "label": {"text": "67_pta"}},
+                                               {"id": "68_ipa_oacc_kernels",
+                                                "label": {"text": "68_ipa_oacc_kernels"}},
+                                               {"id": "69_oacc_kernels",
+                                                "label": {"text": "69_oacc_kernels"}},
+                                               {"id": "70_ch",
+                                                "label": {"text": "70_ch"}},
+                                               {"id": "71_fre",
+                                                "label": {"text": "71_fre"}},
+                                               {"id": "72_lim",
+                                                "label": {"text": "72_lim"}},
+                                               {"id": "73_dom",
+                                                "label": {"text": "73_dom"}},
+                                               {"id": "74_dce",
+                                                "label": {"text": "74_dce"}},
+                                               {"id": "75_parloops",
+                                                "label": {"text": "75_parloops"}},
+                                               {"id": "76_ompexpssa",
+                                                "label": {"text": "76_ompexpssa"}},
+                                               {"id": "*rebuild_cgraph_edges_0x101f1310",
+                                                "label": {"text": "*rebuild_cgraph_edges"}},
+                                               {"id": "77_targetclone",
+                                                "label": {"text": "77_targetclone"}},
+                                               {"id": "78_afdo",
+                                                "label": {"text": "78_afdo"}},
+                                               {"id": "79_feedback_fnsplit",
+                                                "label": {"text": "79_feedback_fnsplit"}},
+                                               {"id": "80_profile",
+                                                "label": {"text": "80_profile"}},
+                                               {"id": "81_feedback_fnsplit",
+                                                "label": {"text": "81_feedback_fnsplit"}},
+                                               {"id": "82_free-fnsummary",
+                                                "label": {"text": "82_free-fnsummary"}},
+                                               {"id": "83_increase_alignment",
+                                                "label": {"text": "83_increase_alignment"}},
+                                               {"id": "84_tmipa",
+                                                "label": {"text": "84_tmipa"}},
+                                               {"id": "85_emutls",
+                                                "label": {"text": "85_emutls"}}]},
+                                 {"id": "all_regular_ipa_passes",
+                                  "label": {"text": "all_regular_ipa_passes"},
+                                  "children": [{"id": "86_analyzer",
+                                                "label": {"text": "86_analyzer"}},
+                                               {"id": "87_odr",
+                                                "label": {"text": "87_odr"}},
+                                               {"id": "88_whole-program",
+                                                "label": {"text": "88_whole-program"}},
+                                               {"id": "89_profile_estimate",
+                                                "label": {"text": "89_profile_estimate"}},
+                                               {"id": "90_icf",
+                                                "label": {"text": "90_icf"}},
+                                               {"id": "91_devirt",
+                                                "label": {"text": "91_devirt"}},
+                                               {"id": "92_cdtor",
+                                                "label": {"text": "92_cdtor"}},
+                                               {"id": "93_cp",
+                                                "label": {"text": "93_cp"}},
+                                               {"id": "94_sra",
+                                                "label": {"text": "94_sra"}},
+                                               {"id": "95_fnsummary",
+                                                "label": {"text": "95_fnsummary"}},
+                                               {"id": "96_inline",
+                                                "label": {"text": "96_inline"}},
+                                               {"id": "97_locality-clone",
+                                                "label": {"text": "97_locality-clone"}},
+                                               {"id": "98_pure-const",
+                                                "label": {"text": "98_pure-const"}},
+                                               {"id": "99_modref",
+                                                "label": {"text": "99_modref"}},
+                                               {"id": "100_free-fnsummary",
+                                                "label": {"text": "100_free-fnsummary"}},
+                                               {"id": "101_static-var",
+                                                "label": {"text": "101_static-var"}},
+                                               {"id": "102_single-use",
+                                                "label": {"text": "102_single-use"}},
+                                               {"id": "103_comdats",
+                                                "label": {"text": "103_comdats"}}]},
+                                 {"id": "all_late_ipa_passes",
+                                  "label": {"text": "all_late_ipa_passes"},
+                                  "children": [{"id": "104_pta",
+                                                "label": {"text": "104_pta"}},
+                                               {"id": "105_simdclone",
+                                                "label": {"text": "105_simdclone"}}]},
+                                 {"id": "all_passes",
+                                  "label": {"text": "all_passes"},
+                                  "children": [{"id": "106_fixup_cfg",
+                                                "label": {"text": "106_fixup_cfg"}},
+                                               {"id": "107_ehdisp",
+                                                "label": {"text": "107_ehdisp"}},
+                                               {"id": "108_oaccloops",
+                                                "label": {"text": "108_oaccloops"}},
+                                               {"id": "109_omp_oacc_neuter_broadcast",
+                                                "label": {"text": "109_omp_oacc_neuter_broadcast"}},
+                                               {"id": "110_oaccdevlow",
+                                                "label": {"text": "110_oaccdevlow"}},
+                                               {"id": "111_ompdevlow",
+                                                "label": {"text": "111_ompdevlow"}},
+                                               {"id": "112_omptargetlink",
+                                                "label": {"text": "112_omptargetlink"}},
+                                               {"id": "113_adjust_alignment",
+                                                "label": {"text": "113_adjust_alignment"}},
+                                               {"id": "114_hardcfr",
+                                                "label": {"text": "114_hardcfr"}},
+                                               {"id": "*all_optimizations_0x101f2720",
+                                                "label": {"text": "*all_optimizations"}},
+                                               {"id": "*remove_cgraph_callee_edges_0x101f2780",
+                                                "label": {"text": "*remove_cgraph_callee_edges"}},
+                                               {"id": "*strip_predict_hints_0x101f27e0",
+                                                "label": {"text": "*strip_predict_hints"}},
+                                               {"id": "115_ccp",
+                                                "label": {"text": "115_ccp"}},
+                                               {"id": "116_objsz",
+                                                "label": {"text": "116_objsz"}},
+                                               {"id": "117_post_ipa_warn",
+                                                "label": {"text": "117_post_ipa_warn"}},
+                                               {"id": "118_waccess",
+                                                "label": {"text": "118_waccess"}},
+                                               {"id": "119_rebuild_frequencies",
+                                                "label": {"text": "119_rebuild_frequencies"}},
+                                               {"id": "120_cunrolli",
+                                                "label": {"text": "120_cunrolli"}},
+                                               {"id": "121_backprop",
+                                                "label": {"text": "121_backprop"}},
+                                               {"id": "122_phiprop",
+                                                "label": {"text": "122_phiprop"}},
+                                               {"id": "123_forwprop",
+                                                "label": {"text": "123_forwprop"}},
+                                               {"id": "124_alias",
+                                                "label": {"text": "124_alias"}},
+                                               {"id": "125_retslot",
+                                                "label": {"text": "125_retslot"}},
+                                               {"id": "126_fre",
+                                                "label": {"text": "126_fre"}},
+                                               {"id": "127_mergephi",
+                                                "label": {"text": "127_mergephi"}},
+                                               {"id": "128_threadfull",
+                                                "label": {"text": "128_threadfull"}},
+                                               {"id": "129_vrp",
+                                                "label": {"text": "129_vrp"}},
+                                               {"id": "130_bounds",
+                                                "label": {"text": "130_bounds"}},
+                                               {"id": "131_dse",
+                                                "label": {"text": "131_dse"}},
+                                               {"id": "132_dce",
+                                                "label": {"text": "132_dce"}},
+                                               {"id": "133_stdarg",
+                                                "label": {"text": "133_stdarg"}},
+                                               {"id": "134_cdce",
+                                                "label": {"text": "134_cdce"}},
+                                               {"id": "135_cselim",
+                                                "label": {"text": "135_cselim"}},
+                                               {"id": "136_copyprop",
+                                                "label": {"text": "136_copyprop"}},
+                                               {"id": "137_ifcombine",
+                                                "label": {"text": "137_ifcombine"}},
+                                               {"id": "138_mergephi",
+                                                "label": {"text": "138_mergephi"}},
+                                               {"id": "139_phiopt",
+                                                "label": {"text": "139_phiopt"}},
+                                               {"id": "140_tailr",
+                                                "label": {"text": "140_tailr"}},
+                                               {"id": "141_ch",
+                                                "label": {"text": "141_ch"}},
+                                               {"id": "142_cplxlower",
+                                                "label": {"text": "142_cplxlower"}},
+                                               {"id": "143_bitintlower",
+                                                "label": {"text": "143_bitintlower"}},
+                                               {"id": "144_sra",
+                                                "label": {"text": "144_sra"}},
+                                               {"id": "145_thread",
+                                                "label": {"text": "145_thread"}},
+                                               {"id": "146_dom",
+                                                "label": {"text": "146_dom"}},
+                                               {"id": "147_copyprop",
+                                                "label": {"text": "147_copyprop"}},
+                                               {"id": "148_isolate-paths",
+                                                "label": {"text": "148_isolate-paths"}},
+                                               {"id": "149_reassoc",
+                                                "label": {"text": "149_reassoc"}},
+                                               {"id": "150_dce",
+                                                "label": {"text": "150_dce"}},
+                                               {"id": "151_forwprop",
+                                                "label": {"text": "151_forwprop"}},
+                                               {"id": "152_phiopt",
+                                                "label": {"text": "152_phiopt"}},
+                                               {"id": "153_ccp",
+                                                "label": {"text": "153_ccp"}},
+                                               {"id": "154_pow",
+                                                "label": {"text": "154_pow"}},
+                                               {"id": "155_bswap",
+                                                "label": {"text": "155_bswap"}},
+                                               {"id": "156_laddress",
+                                                "label": {"text": "156_laddress"}},
+                                               {"id": "157_lim",
+                                                "label": {"text": "157_lim"}},
+                                               {"id": "158_walloca",
+                                                "label": {"text": "158_walloca"}},
+                                               {"id": "159_pre",
+                                                "label": {"text": "159_pre"}},
+                                               {"id": "160_sink",
+                                                "label": {"text": "160_sink"}},
+                                               {"id": "161_sancov",
+                                                "label": {"text": "161_sancov"}},
+                                               {"id": "162_asan",
+                                                "label": {"text": "162_asan"}},
+                                               {"id": "163_tsan",
+                                                "label": {"text": "163_tsan"}},
+                                               {"id": "164_dse",
+                                                "label": {"text": "164_dse"}},
+                                               {"id": "165_dce",
+                                                "label": {"text": "165_dce"}},
+                                               {"id": "166_fix_loops",
+                                                "label": {"text": "166_fix_loops"}},
+                                               {"id": "167_loop",
+                                                "label": {"text": "167_loop"}},
+                                               {"id": "168_loopinit",
+                                                "label": {"text": "168_loopinit"}},
+                                               {"id": "169_unswitch",
+                                                "label": {"text": "169_unswitch"}},
+                                               {"id": "170_lsplit",
+                                                "label": {"text": "170_lsplit"}},
+                                               {"id": "171_sccp",
+                                                "label": {"text": "171_sccp"}},
+                                               {"id": "172_lversion",
+                                                "label": {"text": "172_lversion"}},
+                                               {"id": "173_unrolljam",
+                                                "label": {"text": "173_unrolljam"}},
+                                               {"id": "174_cddce",
+                                                "label": {"text": "174_cddce"}},
+                                               {"id": "175_ivcanon",
+                                                "label": {"text": "175_ivcanon"}},
+                                               {"id": "176_ldist",
+                                                "label": {"text": "176_ldist"}},
+                                               {"id": "177_crc",
+                                                "label": {"text": "177_crc"}},
+                                               {"id": "178_linterchange",
+                                                "label": {"text": "178_linterchange"}},
+                                               {"id": "179_copyprop",
+                                                "label": {"text": "179_copyprop"}},
+                                               {"id": "180_graphite0",
+                                                "label": {"text": "180_graphite0"}},
+                                               {"id": "181_graphite",
+                                                "label": {"text": "181_graphite"}},
+                                               {"id": "182_lim",
+                                                "label": {"text": "182_lim"}},
+                                               {"id": "183_copyprop",
+                                                "label": {"text": "183_copyprop"}},
+                                               {"id": "184_dce",
+                                                "label": {"text": "184_dce"}},
+                                               {"id": "185_parloops",
+                                                "label": {"text": "185_parloops"}},
+                                               {"id": "186_ompexpssa",
+                                                "label": {"text": "186_ompexpssa"}},
+                                               {"id": "187_ch_vect",
+                                                "label": {"text": "187_ch_vect"}},
+                                               {"id": "188_ifcvt",
+                                                "label": {"text": "188_ifcvt"}},
+                                               {"id": "189_vect",
+                                                "label": {"text": "189_vect"}},
+                                               {"id": "190_dce",
+                                                "label": {"text": "190_dce"}},
+                                               {"id": "191_pcom",
+                                                "label": {"text": "191_pcom"}},
+                                               {"id": "192_cunroll",
+                                                "label": {"text": "192_cunroll"}},
+                                               {"id": "*pre_slp_scalar_cleanup_0x101f4880",
+                                                "label": {"text": "*pre_slp_scalar_cleanup"}},
+                                               {"id": "193_fre",
+                                                "label": {"text": "193_fre"}},
+                                               {"id": "194_dse",
+                                                "label": {"text": "194_dse"}},
+                                               {"id": "195_slp",
+                                                "label": {"text": "195_slp"}},
+                                               {"id": "196_aprefetch",
+                                                "label": {"text": "196_aprefetch"}},
+                                               {"id": "197_ivopts",
+                                                "label": {"text": "197_ivopts"}},
+                                               {"id": "198_lim",
+                                                "label": {"text": "198_lim"}},
+                                               {"id": "199_loopdone",
+                                                "label": {"text": "199_loopdone"}},
+                                               {"id": "200_no_loop",
+                                                "label": {"text": "200_no_loop"}},
+                                               {"id": "201_slp",
+                                                "label": {"text": "201_slp"}},
+                                               {"id": "202_simduid",
+                                                "label": {"text": "202_simduid"}},
+                                               {"id": "203_veclower2",
+                                                "label": {"text": "203_veclower2"}},
+                                               {"id": "204_switchlower",
+                                                "label": {"text": "204_switchlower"}},
+                                               {"id": "205_sincos",
+                                                "label": {"text": "205_sincos"}},
+                                               {"id": "206_recip",
+                                                "label": {"text": "206_recip"}},
+                                               {"id": "207_reassoc",
+                                                "label": {"text": "207_reassoc"}},
+                                               {"id": "208_slsr",
+                                                "label": {"text": "208_slsr"}},
+                                               {"id": "209_split-paths",
+                                                "label": {"text": "209_split-paths"}},
+                                               {"id": "210_tracer",
+                                                "label": {"text": "210_tracer"}},
+                                               {"id": "211_fre",
+                                                "label": {"text": "211_fre"}},
+                                               {"id": "212_thread",
+                                                "label": {"text": "212_thread"}},
+                                               {"id": "213_dom",
+                                                "label": {"text": "213_dom"}},
+                                               {"id": "214_strlen",
+                                                "label": {"text": "214_strlen"}},
+                                               {"id": "215_threadfull",
+                                                "label": {"text": "215_threadfull"}},
+                                               {"id": "216_vrp",
+                                                "label": {"text": "216_vrp"}},
+                                               {"id": "217_ccp",
+                                                "label": {"text": "217_ccp"}},
+                                               {"id": "218_wrestrict",
+                                                "label": {"text": "218_wrestrict"}},
+                                               {"id": "219_dse",
+                                                "label": {"text": "219_dse"}},
+                                               {"id": "220_dce",
+                                                "label": {"text": "220_dce"}},
+                                               {"id": "221_forwprop",
+                                                "label": {"text": "221_forwprop"}},
+                                               {"id": "222_sink",
+                                                "label": {"text": "222_sink"}},
+                                               {"id": "223_phiopt",
+                                                "label": {"text": "223_phiopt"}},
+                                               {"id": "224_fab",
+                                                "label": {"text": "224_fab"}},
+                                               {"id": "225_widening_mul",
+                                                "label": {"text": "225_widening_mul"}},
+                                               {"id": "226_store-merging",
+                                                "label": {"text": "226_store-merging"}},
+                                               {"id": "227_cddce",
+                                                "label": {"text": "227_cddce"}},
+                                               {"id": "228_sccopy",
+                                                "label": {"text": "228_sccopy"}},
+                                               {"id": "229_tailc",
+                                                "label": {"text": "229_tailc"}},
+                                               {"id": "230_crited",
+                                                "label": {"text": "230_crited"}},
+                                               {"id": "231_uninit",
+                                                "label": {"text": "231_uninit"}},
+                                               {"id": "232_local-pure-const",
+                                                "label": {"text": "232_local-pure-const"}},
+                                               {"id": "233_modref",
+                                                "label": {"text": "233_modref"}},
+                                               {"id": "234_uncprop",
+                                                "label": {"text": "234_uncprop"}},
+                                               {"id": "*all_optimizations_g_0x101f5af0",
+                                                "label": {"text": "*all_optimizations_g"}},
+                                               {"id": "*remove_cgraph_callee_edges_0x101f5b50",
+                                                "label": {"text": "*remove_cgraph_callee_edges"}},
+                                               {"id": "*strip_predict_hints_0x101f5bb0",
+                                                "label": {"text": "*strip_predict_hints"}},
+                                               {"id": "235_cplxlower",
+                                                "label": {"text": "235_cplxlower"}},
+                                               {"id": "236_bitintlower",
+                                                "label": {"text": "236_bitintlower"}},
+                                               {"id": "237_veclower2",
+                                                "label": {"text": "237_veclower2"}},
+                                               {"id": "238_switchlower",
+                                                "label": {"text": "238_switchlower"}},
+                                               {"id": "239_ccp",
+                                                "label": {"text": "239_ccp"}},
+                                               {"id": "240_post_ipa_warn",
+                                                "label": {"text": "240_post_ipa_warn"}},
+                                               {"id": "241_objsz",
+                                                "label": {"text": "241_objsz"}},
+                                               {"id": "242_fab",
+                                                "label": {"text": "242_fab"}},
+                                               {"id": "243_strlen",
+                                                "label": {"text": "243_strlen"}},
+                                               {"id": "244_copyprop",
+                                                "label": {"text": "244_copyprop"}},
+                                               {"id": "245_dce",
+                                                "label": {"text": "245_dce"}},
+                                               {"id": "246_rebuild_frequencies",
+                                                "label": {"text": "246_rebuild_frequencies"}},
+                                               {"id": "247_sancov",
+                                                "label": {"text": "247_sancov"}},
+                                               {"id": "248_asan",
+                                                "label": {"text": "248_asan"}},
+                                               {"id": "249_tsan",
+                                                "label": {"text": "249_tsan"}},
+                                               {"id": "250_crited",
+                                                "label": {"text": "250_crited"}},
+                                               {"id": "251_uninit",
+                                                "label": {"text": "251_uninit"}},
+                                               {"id": "252_uncprop",
+                                                "label": {"text": "252_uncprop"}},
+                                               {"id": "253_assumptions",
+                                                "label": {"text": "253_assumptions"}},
+                                               {"id": "*tminit_0x101f6370",
+                                                "label": {"text": "*tminit"}},
+                                               {"id": "254_tmmark",
+                                                "label": {"text": "254_tmmark"}},
+                                               {"id": "255_tmmemopt",
+                                                "label": {"text": "255_tmmemopt"}},
+                                               {"id": "256_tmedge",
+                                                "label": {"text": "256_tmedge"}},
+                                               {"id": "257_simduid",
+                                                "label": {"text": "257_simduid"}},
+                                               {"id": "258_vtable-verify",
+                                                "label": {"text": "258_vtable-verify"}},
+                                               {"id": "259_lower_vaarg",
+                                                "label": {"text": "259_lower_vaarg"}},
+                                               {"id": "260_veclower",
+                                                "label": {"text": "260_veclower"}},
+                                               {"id": "261_cplxlower0",
+                                                "label": {"text": "261_cplxlower0"}},
+                                               {"id": "262_bitintlower0",
+                                                "label": {"text": "262_bitintlower0"}},
+                                               {"id": "263_sancov_O0",
+                                                "label": {"text": "263_sancov_O0"}},
+                                               {"id": "264_switchlower_O0",
+                                                "label": {"text": "264_switchlower_O0"}},
+                                               {"id": "265_asan0",
+                                                "label": {"text": "265_asan0"}},
+                                               {"id": "266_tsan0",
+                                                "label": {"text": "266_tsan0"}},
+                                               {"id": "267_musttail",
+                                                "label": {"text": "267_musttail"}},
+                                               {"id": "268_sanopt",
+                                                "label": {"text": "268_sanopt"}},
+                                               {"id": "269_ehcleanup",
+                                                "label": {"text": "269_ehcleanup"}},
+                                               {"id": "270_resx",
+                                                "label": {"text": "270_resx"}},
+                                               {"id": "271_nrv",
+                                                "label": {"text": "271_nrv"}},
+                                               {"id": "272_isel",
+                                                "label": {"text": "272_isel"}},
+                                               {"id": "273_hardcbr",
+                                                "label": {"text": "273_hardcbr"}},
+                                               {"id": "274_hardcmp",
+                                                "label": {"text": "274_hardcmp"}},
+                                               {"id": "275_waccess",
+                                                "label": {"text": "275_waccess"}},
+                                               {"id": "276_optimized",
+                                                "label": {"text": "276_optimized"}},
+                                               {"id": "*warn_function_noreturn_0x101f6dd0",
+                                                "label": {"text": "*warn_function_noreturn"}},
+                                               {"id": "277_expand",
+                                                "label": {"text": "277_expand"}},
+                                               {"id": "*rest_of_compilation_0x101f6e90",
+                                                "label": {"text": "*rest_of_compilation"}},
+                                               {"id": "278_vregs",
+                                                "label": {"text": "278_vregs"}},
+                                               {"id": "279_into_cfglayout",
+                                                "label": {"text": "279_into_cfglayout"}},
+                                               {"id": "280_jump",
+                                                "label": {"text": "280_jump"}},
+                                               {"id": "281_subreg1",
+                                                "label": {"text": "281_subreg1"}},
+                                               {"id": "282_dfinit",
+                                                "label": {"text": "282_dfinit"}},
+                                               {"id": "283_cse1",
+                                                "label": {"text": "283_cse1"}},
+                                               {"id": "284_fwprop1",
+                                                "label": {"text": "284_fwprop1"}},
+                                               {"id": "285_cprop",
+                                                "label": {"text": "285_cprop"}},
+                                               {"id": "286_rtl pre",
+                                                "label": {"text": "286_rtl pre"}},
+                                               {"id": "287_hoist",
+                                                "label": {"text": "287_hoist"}},
+                                               {"id": "288_hardreg_pre",
+                                                "label": {"text": "288_hardreg_pre"}},
+                                               {"id": "289_cprop",
+                                                "label": {"text": "289_cprop"}},
+                                               {"id": "290_store_motion",
+                                                "label": {"text": "290_store_motion"}},
+                                               {"id": "291_cse_local",
+                                                "label": {"text": "291_cse_local"}},
+                                               {"id": "292_ce1",
+                                                "label": {"text": "292_ce1"}},
+                                               {"id": "293_apx_nfcvt",
+                                                "label": {"text": "293_apx_nfcvt"}},
+                                               {"id": "294_reginfo",
+                                                "label": {"text": "294_reginfo"}},
+                                               {"id": "295_loop2",
+                                                "label": {"text": "295_loop2"}},
+                                               {"id": "296_loop2_init",
+                                                "label": {"text": "296_loop2_init"}},
+                                               {"id": "297_loop2_invariant",
+                                                "label": {"text": "297_loop2_invariant"}},
+                                               {"id": "298_loop2_unroll",
+                                                "label": {"text": "298_loop2_unroll"}},
+                                               {"id": "299_loop2_doloop",
+                                                "label": {"text": "299_loop2_doloop"}},
+                                               {"id": "300_loop2_done",
+                                                "label": {"text": "300_loop2_done"}},
+                                               {"id": "301_subreg2",
+                                                "label": {"text": "301_subreg2"}},
+                                               {"id": "302_web",
+                                                "label": {"text": "302_web"}},
+                                               {"id": "303_cprop",
+                                                "label": {"text": "303_cprop"}},
+                                               {"id": "304_stv",
+                                                "label": {"text": "304_stv"}},
+                                               {"id": "305_cse2",
+                                                "label": {"text": "305_cse2"}},
+                                               {"id": "306_dse1",
+                                                "label": {"text": "306_dse1"}},
+                                               {"id": "307_fwprop2",
+                                                "label": {"text": "307_fwprop2"}},
+                                               {"id": "308_auto_inc_dec",
+                                                "label": {"text": "308_auto_inc_dec"}},
+                                               {"id": "309_init-regs",
+                                                "label": {"text": "309_init-regs"}},
+                                               {"id": "310_ud_dce",
+                                                "label": {"text": "310_ud_dce"}},
+                                               {"id": "311_ext_dce",
+                                                "label": {"text": "311_ext_dce"}},
+                                               {"id": "312_combine",
+                                                "label": {"text": "312_combine"}},
+                                               {"id": "313_late_combine",
+                                                "label": {"text": "313_late_combine"}},
+                                               {"id": "314_rpad",
+                                                "label": {"text": "314_rpad"}},
+                                               {"id": "315_rrvl",
+                                                "label": {"text": "315_rrvl"}},
+                                               {"id": "316_stv",
+                                                "label": {"text": "316_stv"}},
+                                               {"id": "317_ce2",
+                                                "label": {"text": "317_ce2"}},
+                                               {"id": "318_jump_after_combine",
+                                                "label": {"text": "318_jump_after_combine"}},
+                                               {"id": "319_bbpart",
+                                                "label": {"text": "319_bbpart"}},
+                                               {"id": "320_outof_cfglayout",
+                                                "label": {"text": "320_outof_cfglayout"}},
+                                               {"id": "321_split1",
+                                                "label": {"text": "321_split1"}},
+                                               {"id": "322_subreg3",
+                                                "label": {"text": "322_subreg3"}},
+                                               {"id": "323_no-opt dfinit",
+                                                "label": {"text": "323_no-opt dfinit"}},
+                                               {"id": "*stack_ptr_mod_0x101f8050",
+                                                "label": {"text": "*stack_ptr_mod"}},
+                                               {"id": "324_mode_sw",
+                                                "label": {"text": "324_mode_sw"}},
+                                               {"id": "325_asmcons",
+                                                "label": {"text": "325_asmcons"}},
+                                               {"id": "326_sms",
+                                                "label": {"text": "326_sms"}},
+                                               {"id": "327_lr_shrinkage",
+                                                "label": {"text": "327_lr_shrinkage"}},
+                                               {"id": "328_sched1",
+                                                "label": {"text": "328_sched1"}},
+                                               {"id": "329_avoid_store_forwarding",
+                                                "label": {"text": "329_avoid_store_forwarding"}},
+                                               {"id": "330_early_remat",
+                                                "label": {"text": "330_early_remat"}},
+                                               {"id": "331_ira",
+                                                "label": {"text": "331_ira"}},
+                                               {"id": "332_reload",
+                                                "label": {"text": "332_reload"}},
+                                               {"id": "*all-postreload_0x101f8410",
+                                                "label": {"text": "*all-postreload"}},
+                                               {"id": "333_postreload",
+                                                "label": {"text": "333_postreload"}},
+                                               {"id": "334_vzeroupper",
+                                                "label": {"text": "334_vzeroupper"}},
+                                               {"id": "335_late_combine",
+                                                "label": {"text": "335_late_combine"}},
+                                               {"id": "336_gcse2",
+                                                "label": {"text": "336_gcse2"}},
+                                               {"id": "337_split2",
+                                                "label": {"text": "337_split2"}},
+                                               {"id": "338_ree",
+                                                "label": {"text": "338_ree"}},
+                                               {"id": "339_cmpelim",
+                                                "label": {"text": "339_cmpelim"}},
+                                               {"id": "340_pro_and_epilogue",
+                                                "label": {"text": "340_pro_and_epilogue"}},
+                                               {"id": "341_dse2",
+                                                "label": {"text": "341_dse2"}},
+                                               {"id": "342_csa",
+                                                "label": {"text": "342_csa"}},
+                                               {"id": "343_jump2",
+                                                "label": {"text": "343_jump2"}},
+                                               {"id": "344_compgotos",
+                                                "label": {"text": "344_compgotos"}},
+                                               {"id": "345_sched_fusion",
+                                                "label": {"text": "345_sched_fusion"}},
+                                               {"id": "346_peephole2",
+                                                "label": {"text": "346_peephole2"}},
+                                               {"id": "347_ce3",
+                                                "label": {"text": "347_ce3"}},
+                                               {"id": "348_rnreg",
+                                                "label": {"text": "348_rnreg"}},
+                                               {"id": "349_fold_mem_offsets",
+                                                "label": {"text": "349_fold_mem_offsets"}},
+                                               {"id": "350_cprop_hardreg",
+                                                "label": {"text": "350_cprop_hardreg"}},
+                                               {"id": "351_rtl_dce",
+                                                "label": {"text": "351_rtl_dce"}},
+                                               {"id": "352_bbro",
+                                                "label": {"text": "352_bbro"}},
+                                               {"id": "*leaf_regs_0x101f8bf0",
+                                                "label": {"text": "*leaf_regs"}},
+                                               {"id": "353_split3",
+                                                "label": {"text": "353_split3"}},
+                                               {"id": "354_sched2",
+                                                "label": {"text": "354_sched2"}},
+                                               {"id": "*stack_regs_0x101f8d10",
+                                                "label": {"text": "*stack_regs"}},
+                                               {"id": "355_split4",
+                                                "label": {"text": "355_split4"}},
+                                               {"id": "356_stack",
+                                                "label": {"text": "356_stack"}},
+                                               {"id": "357_late_pro_and_epilogue",
+                                                "label": {"text": "357_late_pro_and_epilogue"}},
+                                               {"id": "*all-late_compilation_0x101f8e90",
+                                                "label": {"text": "*all-late_compilation"}},
+                                               {"id": "358_zero_call_used_regs",
+                                                "label": {"text": "358_zero_call_used_regs"}},
+                                               {"id": "359_alignments",
+                                                "label": {"text": "359_alignments"}},
+                                               {"id": "360_vartrack",
+                                                "label": {"text": "360_vartrack"}},
+                                               {"id": "*free_cfg_0x101f9010",
+                                                "label": {"text": "*free_cfg"}},
+                                               {"id": "361_mach",
+                                                "label": {"text": "361_mach"}},
+                                               {"id": "362_barriers",
+                                                "label": {"text": "362_barriers"}},
+                                               {"id": "363_dbr",
+                                                "label": {"text": "363_dbr"}},
+                                               {"id": "364_split5",
+                                                "label": {"text": "364_split5"}},
+                                               {"id": "365_eh_ranges",
+                                                "label": {"text": "365_eh_ranges"}},
+                                               {"id": "366_endbr_and_patchable_area",
+                                                "label": {"text": "366_endbr_and_patchable_area"}},
+                                               {"id": "367_align_tight_loops",
+                                                "label": {"text": "367_align_tight_loops"}},
+                                               {"id": "368_shorten",
+                                                "label": {"text": "368_shorten"}},
+                                               {"id": "369_nothrow",
+                                                "label": {"text": "369_nothrow"}},
+                                               {"id": "370_dwarf2",
+                                                "label": {"text": "370_dwarf2"}},
+                                               {"id": "371_final",
+                                                "label": {"text": "371_final"}},
+                                               {"id": "372_dfinish",
+                                                "label": {"text": "372_dfinish"}},
+                                               {"id": "*clean_state_0x101f9500",
+                                                "label": {"text": "*clean_state"}}]}],
+                       "edges": [{"id": "edge0",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "22_ompexp",
+                                  "targetNodeId": "*build_cgraph_edges_0x101ef910"},
+                                 {"id": "edge1",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "21_coro-early-expand-ifns",
+                                  "targetNodeId": "22_ompexp"},
+                                 {"id": "edge2",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*warn_function_return_0x101ef7f0",
+                                  "targetNodeId": "21_coro-early-expand-ifns"},
+                                 {"id": "edge3",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "20_cfg",
+                                  "targetNodeId": "*warn_function_return_0x101ef7f0"},
+                                 {"id": "edge4",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "19_coro-lower-builtins",
+                                  "targetNodeId": "20_cfg"},
+                                 {"id": "edge5",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "18_eh",
+                                  "targetNodeId": "19_coro-lower-builtins"},
+                                 {"id": "edge6",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "17_ehopt",
+                                  "targetNodeId": "18_eh"},
+                                 {"id": "edge7",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "16_tmlower",
+                                  "targetNodeId": "17_ehopt"},
+                                 {"id": "edge8",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "15_lower",
+                                  "targetNodeId": "16_tmlower"},
+                                 {"id": "edge9",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "14_omplower",
+                                  "targetNodeId": "15_lower"},
+                                 {"id": "edge10",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "13_omp_oacc_kernels_decompose",
+                                  "targetNodeId": "14_omplower"},
+                                 {"id": "edge11",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*diagnose_tm_blocks_0x101ef490",
+                                  "targetNodeId": "13_omp_oacc_kernels_decompose"},
+                                 {"id": "edge12",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*diagnose_omp_blocks_0x101ef430",
+                                  "targetNodeId": "*diagnose_tm_blocks_0x101ef490"},
+                                 {"id": "edge13",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*warn_unused_result_0x101ef3d0",
+                                  "targetNodeId": "*diagnose_omp_blocks_0x101ef430"},
+                                 {"id": "edge14",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "34_nothrow",
+                                  "targetNodeId": "*rebuild_cgraph_edges_0x101f0020"},
+                                 {"id": "edge15",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "33_ubsan",
+                                  "targetNodeId": "34_nothrow"},
+                                 {"id": "edge16",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "32_waccess",
+                                  "targetNodeId": "33_ubsan"},
+                                 {"id": "edge17",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "31_early_uninit",
+                                  "targetNodeId": "32_waccess"},
+                                 {"id": "edge18",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*nonnullcmp_0x101efce0",
+                                  "targetNodeId": "31_early_uninit"},
+                                 {"id": "edge19",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "30_warn-printf",
+                                  "targetNodeId": "*nonnullcmp_0x101efce0"},
+                                 {"id": "edge20",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "29_walloca",
+                                  "targetNodeId": "30_warn-printf"},
+                                 {"id": "edge21",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "376_test_graph_emission",
+                                  "targetNodeId": "29_walloca"},
+                                 {"id": "edge22",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "28_ssa",
+                                  "targetNodeId": "376_test_graph_emission"},
+                                 {"id": "edge23",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "27_fixup_cfg",
+                                  "targetNodeId": "28_ssa"},
+                                 {"id": "edge24",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "26_build_ssa_passes",
+                                  "targetNodeId": "27_fixup_cfg"},
+                                 {"id": "edge25",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "61_fnsplit",
+                                  "targetNodeId": "*strip_predict_hints_0x101f0c60"},
+                                 {"id": "edge26",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "60_modref",
+                                  "targetNodeId": "61_fnsplit"},
+                                 {"id": "edge27",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "59_local-pure-const",
+                                  "targetNodeId": "60_modref"},
+                                 {"id": "edge28",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "58_profile_estimate",
+                                  "targetNodeId": "59_local-pure-const"},
+                                 {"id": "edge29",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "57_sccopy",
+                                  "targetNodeId": "58_profile_estimate"},
+                                 {"id": "edge30",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "56_ehcleanup",
+                                  "targetNodeId": "57_sccopy"},
+                                 {"id": "edge31",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "55_switchconv",
+                                  "targetNodeId": "56_ehcleanup"},
+                                 {"id": "edge32",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "54_iftoswitch",
+                                  "targetNodeId": "55_switchconv"},
+                                 {"id": "edge33",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "53_tailr",
+                                  "targetNodeId": "54_iftoswitch"},
+                                 {"id": "edge34",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "52_phiopt",
+                                  "targetNodeId": "53_tailr"},
+                                 {"id": "edge35",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "51_cddce",
+                                  "targetNodeId": "52_phiopt"},
+                                 {"id": "edge36",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "50_dse",
+                                  "targetNodeId": "51_cddce"},
+                                 {"id": "edge37",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "49_mergephi",
+                                  "targetNodeId": "50_dse"},
+                                 {"id": "edge38",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "48_evrp",
+                                  "targetNodeId": "49_mergephi"},
+                                 {"id": "edge39",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "47_fre",
+                                  "targetNodeId": "48_evrp"},
+                                 {"id": "edge40",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "46_phiprop",
+                                  "targetNodeId": "47_fre"},
+                                 {"id": "edge41",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "45_ealias",
+                                  "targetNodeId": "46_phiprop"},
+                                 {"id": "edge42",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "44_esra",
+                                  "targetNodeId": "45_ealias"},
+                                 {"id": "edge43",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "43_ethread",
+                                  "targetNodeId": "44_esra"},
+                                 {"id": "edge44",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "42_forwprop",
+                                  "targetNodeId": "43_ethread"},
+                                 {"id": "edge45",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "41_ccp",
+                                  "targetNodeId": "42_forwprop"},
+                                 {"id": "edge46",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "40_early_objsz",
+                                  "targetNodeId": "41_ccp"},
+                                 {"id": "edge47",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*remove_cgraph_callee_edges_0x101f0340",
+                                  "targetNodeId": "40_early_objsz"},
+                                 {"id": "edge48",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "39_early_optimizations",
+                                  "targetNodeId": "*remove_cgraph_callee_edges_0x101f0340"},
+                                 {"id": "edge49",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*rebuild_cgraph_edges_0x101f0d30",
+                                  "targetNodeId": "63_local-fnsummary"},
+                                 {"id": "edge50",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "62_release_ssa",
+                                  "targetNodeId": "*rebuild_cgraph_edges_0x101f0d30"},
+                                 {"id": "edge51",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "39_early_optimizations",
+                                  "targetNodeId": "62_release_ssa"},
+                                 {"id": "edge52",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*infinite-recursion_0x101f0260",
+                                  "targetNodeId": "39_early_optimizations"},
+                                 {"id": "edge53",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "38_einline",
+                                  "targetNodeId": "*infinite-recursion_0x101f0260"},
+                                 {"id": "edge54",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "37_local-fnsummary",
+                                  "targetNodeId": "38_einline"},
+                                 {"id": "edge55",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*rebuild_cgraph_edges_0x101f0140",
+                                  "targetNodeId": "37_local-fnsummary"},
+                                 {"id": "edge56",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "36_fixup_cfg",
+                                  "targetNodeId": "*rebuild_cgraph_edges_0x101f0140"},
+                                 {"id": "edge57",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "35_opt_local_passes",
+                                  "targetNodeId": "36_fixup_cfg"},
+                                 {"id": "edge58",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "76_ompexpssa",
+                                  "targetNodeId": "*rebuild_cgraph_edges_0x101f1310"},
+                                 {"id": "edge59",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "75_parloops",
+                                  "targetNodeId": "76_ompexpssa"},
+                                 {"id": "edge60",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "74_dce",
+                                  "targetNodeId": "75_parloops"},
+                                 {"id": "edge61",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "73_dom",
+                                  "targetNodeId": "74_dce"},
+                                 {"id": "edge62",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "72_lim",
+                                  "targetNodeId": "73_dom"},
+                                 {"id": "edge63",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "71_fre",
+                                  "targetNodeId": "72_lim"},
+                                 {"id": "edge64",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "70_ch",
+                                  "targetNodeId": "71_fre"},
+                                 {"id": "edge65",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "69_oacc_kernels",
+                                  "targetNodeId": "70_ch"},
+                                 {"id": "edge66",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "68_ipa_oacc_kernels",
+                                  "targetNodeId": "69_oacc_kernels"},
+                                 {"id": "edge67",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "67_pta",
+                                  "targetNodeId": "68_ipa_oacc_kernels"},
+                                 {"id": "edge68",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "66_ipa_oacc",
+                                  "targetNodeId": "67_pta"},
+                                 {"id": "edge69",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "78_afdo",
+                                  "targetNodeId": "79_feedback_fnsplit"},
+                                 {"id": "edge70",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "80_profile",
+                                  "targetNodeId": "81_feedback_fnsplit"},
+                                 {"id": "edge71",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "84_tmipa",
+                                  "targetNodeId": "85_emutls"},
+                                 {"id": "edge72",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "83_increase_alignment",
+                                  "targetNodeId": "84_tmipa"},
+                                 {"id": "edge73",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "82_free-fnsummary",
+                                  "targetNodeId": "83_increase_alignment"},
+                                 {"id": "edge74",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "80_profile",
+                                  "targetNodeId": "82_free-fnsummary"},
+                                 {"id": "edge75",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "78_afdo",
+                                  "targetNodeId": "80_profile"},
+                                 {"id": "edge76",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "77_targetclone",
+                                  "targetNodeId": "78_afdo"},
+                                 {"id": "edge77",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "66_ipa_oacc",
+                                  "targetNodeId": "77_targetclone"},
+                                 {"id": "edge78",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "65_strub",
+                                  "targetNodeId": "66_ipa_oacc"},
+                                 {"id": "edge79",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "64_remove_symbols",
+                                  "targetNodeId": "65_strub"},
+                                 {"id": "edge80",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "35_opt_local_passes",
+                                  "targetNodeId": "64_remove_symbols"},
+                                 {"id": "edge81",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "26_build_ssa_passes",
+                                  "targetNodeId": "35_opt_local_passes"},
+                                 {"id": "edge82",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "25_strubm",
+                                  "targetNodeId": "26_build_ssa_passes"},
+                                 {"id": "edge83",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "24_visibility",
+                                  "targetNodeId": "25_strubm"},
+                                 {"id": "edge84",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*free_lang_data_0x101ef9d0",
+                                  "targetNodeId": "24_visibility"},
+                                 {"id": "edge85",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "23_afdo_offline",
+                                  "targetNodeId": "*free_lang_data_0x101ef9d0"},
+                                 {"id": "edge86",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "102_single-use",
+                                  "targetNodeId": "103_comdats"},
+                                 {"id": "edge87",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "101_static-var",
+                                  "targetNodeId": "102_single-use"},
+                                 {"id": "edge88",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "100_free-fnsummary",
+                                  "targetNodeId": "101_static-var"},
+                                 {"id": "edge89",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "99_modref",
+                                  "targetNodeId": "100_free-fnsummary"},
+                                 {"id": "edge90",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "98_pure-const",
+                                  "targetNodeId": "99_modref"},
+                                 {"id": "edge91",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "97_locality-clone",
+                                  "targetNodeId": "98_pure-const"},
+                                 {"id": "edge92",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "96_inline",
+                                  "targetNodeId": "97_locality-clone"},
+                                 {"id": "edge93",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "95_fnsummary",
+                                  "targetNodeId": "96_inline"},
+                                 {"id": "edge94",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "94_sra",
+                                  "targetNodeId": "95_fnsummary"},
+                                 {"id": "edge95",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "93_cp",
+                                  "targetNodeId": "94_sra"},
+                                 {"id": "edge96",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "92_cdtor",
+                                  "targetNodeId": "93_cp"},
+                                 {"id": "edge97",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "91_devirt",
+                                  "targetNodeId": "92_cdtor"},
+                                 {"id": "edge98",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "90_icf",
+                                  "targetNodeId": "91_devirt"},
+                                 {"id": "edge99",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "89_profile_estimate",
+                                  "targetNodeId": "90_icf"},
+                                 {"id": "edge100",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "88_whole-program",
+                                  "targetNodeId": "89_profile_estimate"},
+                                 {"id": "edge101",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "87_odr",
+                                  "targetNodeId": "88_whole-program"},
+                                 {"id": "edge102",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "86_analyzer",
+                                  "targetNodeId": "87_odr"},
+                                 {"id": "edge103",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "104_pta",
+                                  "targetNodeId": "105_simdclone"},
+                                 {"id": "edge104",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "183_copyprop",
+                                  "targetNodeId": "184_dce"},
+                                 {"id": "edge105",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "182_lim",
+                                  "targetNodeId": "183_copyprop"},
+                                 {"id": "edge106",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "181_graphite",
+                                  "targetNodeId": "182_lim"},
+                                 {"id": "edge107",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "180_graphite0",
+                                  "targetNodeId": "181_graphite"},
+                                 {"id": "edge108",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "189_vect",
+                                  "targetNodeId": "190_dce"},
+                                 {"id": "edge109",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "193_fre",
+                                  "targetNodeId": "194_dse"},
+                                 {"id": "edge110",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*pre_slp_scalar_cleanup_0x101f4880",
+                                  "targetNodeId": "193_fre"},
+                                 {"id": "edge111",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "198_lim",
+                                  "targetNodeId": "199_loopdone"},
+                                 {"id": "edge112",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "197_ivopts",
+                                  "targetNodeId": "198_lim"},
+                                 {"id": "edge113",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "196_aprefetch",
+                                  "targetNodeId": "197_ivopts"},
+                                 {"id": "edge114",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "195_slp",
+                                  "targetNodeId": "196_aprefetch"},
+                                 {"id": "edge115",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*pre_slp_scalar_cleanup_0x101f4880",
+                                  "targetNodeId": "195_slp"},
+                                 {"id": "edge116",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "192_cunroll",
+                                  "targetNodeId": "*pre_slp_scalar_cleanup_0x101f4880"},
+                                 {"id": "edge117",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "191_pcom",
+                                  "targetNodeId": "192_cunroll"},
+                                 {"id": "edge118",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "189_vect",
+                                  "targetNodeId": "191_pcom"},
+                                 {"id": "edge119",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "188_ifcvt",
+                                  "targetNodeId": "189_vect"},
+                                 {"id": "edge120",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "187_ch_vect",
+                                  "targetNodeId": "188_ifcvt"},
+                                 {"id": "edge121",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "186_ompexpssa",
+                                  "targetNodeId": "187_ch_vect"},
+                                 {"id": "edge122",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "185_parloops",
+                                  "targetNodeId": "186_ompexpssa"},
+                                 {"id": "edge123",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "180_graphite0",
+                                  "targetNodeId": "185_parloops"},
+                                 {"id": "edge124",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "179_copyprop",
+                                  "targetNodeId": "180_graphite0"},
+                                 {"id": "edge125",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "178_linterchange",
+                                  "targetNodeId": "179_copyprop"},
+                                 {"id": "edge126",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "177_crc",
+                                  "targetNodeId": "178_linterchange"},
+                                 {"id": "edge127",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "176_ldist",
+                                  "targetNodeId": "177_crc"},
+                                 {"id": "edge128",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "175_ivcanon",
+                                  "targetNodeId": "176_ldist"},
+                                 {"id": "edge129",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "174_cddce",
+                                  "targetNodeId": "175_ivcanon"},
+                                 {"id": "edge130",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "173_unrolljam",
+                                  "targetNodeId": "174_cddce"},
+                                 {"id": "edge131",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "172_lversion",
+                                  "targetNodeId": "173_unrolljam"},
+                                 {"id": "edge132",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "171_sccp",
+                                  "targetNodeId": "172_lversion"},
+                                 {"id": "edge133",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "170_lsplit",
+                                  "targetNodeId": "171_sccp"},
+                                 {"id": "edge134",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "169_unswitch",
+                                  "targetNodeId": "170_lsplit"},
+                                 {"id": "edge135",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "168_loopinit",
+                                  "targetNodeId": "169_unswitch"},
+                                 {"id": "edge136",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "167_loop",
+                                  "targetNodeId": "168_loopinit"},
+                                 {"id": "edge137",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "200_no_loop",
+                                  "targetNodeId": "201_slp"},
+                                 {"id": "edge138",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "233_modref",
+                                  "targetNodeId": "234_uncprop"},
+                                 {"id": "edge139",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "232_local-pure-const",
+                                  "targetNodeId": "233_modref"},
+                                 {"id": "edge140",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "231_uninit",
+                                  "targetNodeId": "232_local-pure-const"},
+                                 {"id": "edge141",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "230_crited",
+                                  "targetNodeId": "231_uninit"},
+                                 {"id": "edge142",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "229_tailc",
+                                  "targetNodeId": "230_crited"},
+                                 {"id": "edge143",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "228_sccopy",
+                                  "targetNodeId": "229_tailc"},
+                                 {"id": "edge144",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "227_cddce",
+                                  "targetNodeId": "228_sccopy"},
+                                 {"id": "edge145",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "226_store-merging",
+                                  "targetNodeId": "227_cddce"},
+                                 {"id": "edge146",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "225_widening_mul",
+                                  "targetNodeId": "226_store-merging"},
+                                 {"id": "edge147",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "224_fab",
+                                  "targetNodeId": "225_widening_mul"},
+                                 {"id": "edge148",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "223_phiopt",
+                                  "targetNodeId": "224_fab"},
+                                 {"id": "edge149",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "222_sink",
+                                  "targetNodeId": "223_phiopt"},
+                                 {"id": "edge150",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "221_forwprop",
+                                  "targetNodeId": "222_sink"},
+                                 {"id": "edge151",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "220_dce",
+                                  "targetNodeId": "221_forwprop"},
+                                 {"id": "edge152",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "219_dse",
+                                  "targetNodeId": "220_dce"},
+                                 {"id": "edge153",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "218_wrestrict",
+                                  "targetNodeId": "219_dse"},
+                                 {"id": "edge154",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "217_ccp",
+                                  "targetNodeId": "218_wrestrict"},
+                                 {"id": "edge155",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "216_vrp",
+                                  "targetNodeId": "217_ccp"},
+                                 {"id": "edge156",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "215_threadfull",
+                                  "targetNodeId": "216_vrp"},
+                                 {"id": "edge157",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "214_strlen",
+                                  "targetNodeId": "215_threadfull"},
+                                 {"id": "edge158",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "213_dom",
+                                  "targetNodeId": "214_strlen"},
+                                 {"id": "edge159",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "212_thread",
+                                  "targetNodeId": "213_dom"},
+                                 {"id": "edge160",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "211_fre",
+                                  "targetNodeId": "212_thread"},
+                                 {"id": "edge161",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "210_tracer",
+                                  "targetNodeId": "211_fre"},
+                                 {"id": "edge162",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "209_split-paths",
+                                  "targetNodeId": "210_tracer"},
+                                 {"id": "edge163",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "208_slsr",
+                                  "targetNodeId": "209_split-paths"},
+                                 {"id": "edge164",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "207_reassoc",
+                                  "targetNodeId": "208_slsr"},
+                                 {"id": "edge165",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "206_recip",
+                                  "targetNodeId": "207_reassoc"},
+                                 {"id": "edge166",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "205_sincos",
+                                  "targetNodeId": "206_recip"},
+                                 {"id": "edge167",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "204_switchlower",
+                                  "targetNodeId": "205_sincos"},
+                                 {"id": "edge168",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "203_veclower2",
+                                  "targetNodeId": "204_switchlower"},
+                                 {"id": "edge169",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "202_simduid",
+                                  "targetNodeId": "203_veclower2"},
+                                 {"id": "edge170",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "200_no_loop",
+                                  "targetNodeId": "202_simduid"},
+                                 {"id": "edge171",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "167_loop",
+                                  "targetNodeId": "200_no_loop"},
+                                 {"id": "edge172",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "166_fix_loops",
+                                  "targetNodeId": "167_loop"},
+                                 {"id": "edge173",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "165_dce",
+                                  "targetNodeId": "166_fix_loops"},
+                                 {"id": "edge174",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "164_dse",
+                                  "targetNodeId": "165_dce"},
+                                 {"id": "edge175",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "163_tsan",
+                                  "targetNodeId": "164_dse"},
+                                 {"id": "edge176",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "162_asan",
+                                  "targetNodeId": "163_tsan"},
+                                 {"id": "edge177",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "161_sancov",
+                                  "targetNodeId": "162_asan"},
+                                 {"id": "edge178",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "160_sink",
+                                  "targetNodeId": "161_sancov"},
+                                 {"id": "edge179",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "159_pre",
+                                  "targetNodeId": "160_sink"},
+                                 {"id": "edge180",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "158_walloca",
+                                  "targetNodeId": "159_pre"},
+                                 {"id": "edge181",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "157_lim",
+                                  "targetNodeId": "158_walloca"},
+                                 {"id": "edge182",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "156_laddress",
+                                  "targetNodeId": "157_lim"},
+                                 {"id": "edge183",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "155_bswap",
+                                  "targetNodeId": "156_laddress"},
+                                 {"id": "edge184",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "154_pow",
+                                  "targetNodeId": "155_bswap"},
+                                 {"id": "edge185",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "153_ccp",
+                                  "targetNodeId": "154_pow"},
+                                 {"id": "edge186",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "152_phiopt",
+                                  "targetNodeId": "153_ccp"},
+                                 {"id": "edge187",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "151_forwprop",
+                                  "targetNodeId": "152_phiopt"},
+                                 {"id": "edge188",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "150_dce",
+                                  "targetNodeId": "151_forwprop"},
+                                 {"id": "edge189",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "149_reassoc",
+                                  "targetNodeId": "150_dce"},
+                                 {"id": "edge190",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "148_isolate-paths",
+                                  "targetNodeId": "149_reassoc"},
+                                 {"id": "edge191",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "147_copyprop",
+                                  "targetNodeId": "148_isolate-paths"},
+                                 {"id": "edge192",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "146_dom",
+                                  "targetNodeId": "147_copyprop"},
+                                 {"id": "edge193",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "145_thread",
+                                  "targetNodeId": "146_dom"},
+                                 {"id": "edge194",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "144_sra",
+                                  "targetNodeId": "145_thread"},
+                                 {"id": "edge195",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "143_bitintlower",
+                                  "targetNodeId": "144_sra"},
+                                 {"id": "edge196",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "142_cplxlower",
+                                  "targetNodeId": "143_bitintlower"},
+                                 {"id": "edge197",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "141_ch",
+                                  "targetNodeId": "142_cplxlower"},
+                                 {"id": "edge198",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "140_tailr",
+                                  "targetNodeId": "141_ch"},
+                                 {"id": "edge199",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "139_phiopt",
+                                  "targetNodeId": "140_tailr"},
+                                 {"id": "edge200",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "138_mergephi",
+                                  "targetNodeId": "139_phiopt"},
+                                 {"id": "edge201",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "137_ifcombine",
+                                  "targetNodeId": "138_mergephi"},
+                                 {"id": "edge202",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "136_copyprop",
+                                  "targetNodeId": "137_ifcombine"},
+                                 {"id": "edge203",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "135_cselim",
+                                  "targetNodeId": "136_copyprop"},
+                                 {"id": "edge204",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "134_cdce",
+                                  "targetNodeId": "135_cselim"},
+                                 {"id": "edge205",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "133_stdarg",
+                                  "targetNodeId": "134_cdce"},
+                                 {"id": "edge206",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "132_dce",
+                                  "targetNodeId": "133_stdarg"},
+                                 {"id": "edge207",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "131_dse",
+                                  "targetNodeId": "132_dce"},
+                                 {"id": "edge208",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "130_bounds",
+                                  "targetNodeId": "131_dse"},
+                                 {"id": "edge209",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "129_vrp",
+                                  "targetNodeId": "130_bounds"},
+                                 {"id": "edge210",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "128_threadfull",
+                                  "targetNodeId": "129_vrp"},
+                                 {"id": "edge211",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "127_mergephi",
+                                  "targetNodeId": "128_threadfull"},
+                                 {"id": "edge212",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "126_fre",
+                                  "targetNodeId": "127_mergephi"},
+                                 {"id": "edge213",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "125_retslot",
+                                  "targetNodeId": "126_fre"},
+                                 {"id": "edge214",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "124_alias",
+                                  "targetNodeId": "125_retslot"},
+                                 {"id": "edge215",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "123_forwprop",
+                                  "targetNodeId": "124_alias"},
+                                 {"id": "edge216",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "122_phiprop",
+                                  "targetNodeId": "123_forwprop"},
+                                 {"id": "edge217",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "121_backprop",
+                                  "targetNodeId": "122_phiprop"},
+                                 {"id": "edge218",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "120_cunrolli",
+                                  "targetNodeId": "121_backprop"},
+                                 {"id": "edge219",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "119_rebuild_frequencies",
+                                  "targetNodeId": "120_cunrolli"},
+                                 {"id": "edge220",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "118_waccess",
+                                  "targetNodeId": "119_rebuild_frequencies"},
+                                 {"id": "edge221",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "117_post_ipa_warn",
+                                  "targetNodeId": "118_waccess"},
+                                 {"id": "edge222",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "116_objsz",
+                                  "targetNodeId": "117_post_ipa_warn"},
+                                 {"id": "edge223",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "115_ccp",
+                                  "targetNodeId": "116_objsz"},
+                                 {"id": "edge224",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*strip_predict_hints_0x101f27e0",
+                                  "targetNodeId": "115_ccp"},
+                                 {"id": "edge225",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*remove_cgraph_callee_edges_0x101f2780",
+                                  "targetNodeId": "*strip_predict_hints_0x101f27e0"},
+                                 {"id": "edge226",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*all_optimizations_0x101f2720",
+                                  "targetNodeId": "*remove_cgraph_callee_edges_0x101f2780"},
+                                 {"id": "edge227",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "251_uninit",
+                                  "targetNodeId": "252_uncprop"},
+                                 {"id": "edge228",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "250_crited",
+                                  "targetNodeId": "251_uninit"},
+                                 {"id": "edge229",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "249_tsan",
+                                  "targetNodeId": "250_crited"},
+                                 {"id": "edge230",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "248_asan",
+                                  "targetNodeId": "249_tsan"},
+                                 {"id": "edge231",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "247_sancov",
+                                  "targetNodeId": "248_asan"},
+                                 {"id": "edge232",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "246_rebuild_frequencies",
+                                  "targetNodeId": "247_sancov"},
+                                 {"id": "edge233",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "245_dce",
+                                  "targetNodeId": "246_rebuild_frequencies"},
+                                 {"id": "edge234",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "244_copyprop",
+                                  "targetNodeId": "245_dce"},
+                                 {"id": "edge235",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "243_strlen",
+                                  "targetNodeId": "244_copyprop"},
+                                 {"id": "edge236",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "242_fab",
+                                  "targetNodeId": "243_strlen"},
+                                 {"id": "edge237",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "241_objsz",
+                                  "targetNodeId": "242_fab"},
+                                 {"id": "edge238",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "240_post_ipa_warn",
+                                  "targetNodeId": "241_objsz"},
+                                 {"id": "edge239",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "239_ccp",
+                                  "targetNodeId": "240_post_ipa_warn"},
+                                 {"id": "edge240",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "238_switchlower",
+                                  "targetNodeId": "239_ccp"},
+                                 {"id": "edge241",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "237_veclower2",
+                                  "targetNodeId": "238_switchlower"},
+                                 {"id": "edge242",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "236_bitintlower",
+                                  "targetNodeId": "237_veclower2"},
+                                 {"id": "edge243",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "235_cplxlower",
+                                  "targetNodeId": "236_bitintlower"},
+                                 {"id": "edge244",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*strip_predict_hints_0x101f5bb0",
+                                  "targetNodeId": "235_cplxlower"},
+                                 {"id": "edge245",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*remove_cgraph_callee_edges_0x101f5b50",
+                                  "targetNodeId": "*strip_predict_hints_0x101f5bb0"},
+                                 {"id": "edge246",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*all_optimizations_g_0x101f5af0",
+                                  "targetNodeId": "*remove_cgraph_callee_edges_0x101f5b50"},
+                                 {"id": "edge247",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "255_tmmemopt",
+                                  "targetNodeId": "256_tmedge"},
+                                 {"id": "edge248",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "254_tmmark",
+                                  "targetNodeId": "255_tmmemopt"},
+                                 {"id": "edge249",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*tminit_0x101f6370",
+                                  "targetNodeId": "254_tmmark"},
+                                 {"id": "edge250",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "299_loop2_doloop",
+                                  "targetNodeId": "300_loop2_done"},
+                                 {"id": "edge251",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "298_loop2_unroll",
+                                  "targetNodeId": "299_loop2_doloop"},
+                                 {"id": "edge252",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "297_loop2_invariant",
+                                  "targetNodeId": "298_loop2_unroll"},
+                                 {"id": "edge253",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "296_loop2_init",
+                                  "targetNodeId": "297_loop2_invariant"},
+                                 {"id": "edge254",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "295_loop2",
+                                  "targetNodeId": "296_loop2_init"},
+                                 {"id": "edge255",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "355_split4",
+                                  "targetNodeId": "356_stack"},
+                                 {"id": "edge256",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*stack_regs_0x101f8d10",
+                                  "targetNodeId": "355_split4"},
+                                 {"id": "edge257",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "354_sched2",
+                                  "targetNodeId": "*stack_regs_0x101f8d10"},
+                                 {"id": "edge258",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "353_split3",
+                                  "targetNodeId": "354_sched2"},
+                                 {"id": "edge259",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*leaf_regs_0x101f8bf0",
+                                  "targetNodeId": "353_split3"},
+                                 {"id": "edge260",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "352_bbro",
+                                  "targetNodeId": "*leaf_regs_0x101f8bf0"},
+                                 {"id": "edge261",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "351_rtl_dce",
+                                  "targetNodeId": "352_bbro"},
+                                 {"id": "edge262",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "350_cprop_hardreg",
+                                  "targetNodeId": "351_rtl_dce"},
+                                 {"id": "edge263",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "349_fold_mem_offsets",
+                                  "targetNodeId": "350_cprop_hardreg"},
+                                 {"id": "edge264",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "348_rnreg",
+                                  "targetNodeId": "349_fold_mem_offsets"},
+                                 {"id": "edge265",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "347_ce3",
+                                  "targetNodeId": "348_rnreg"},
+                                 {"id": "edge266",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "346_peephole2",
+                                  "targetNodeId": "347_ce3"},
+                                 {"id": "edge267",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "345_sched_fusion",
+                                  "targetNodeId": "346_peephole2"},
+                                 {"id": "edge268",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "344_compgotos",
+                                  "targetNodeId": "345_sched_fusion"},
+                                 {"id": "edge269",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "343_jump2",
+                                  "targetNodeId": "344_compgotos"},
+                                 {"id": "edge270",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "342_csa",
+                                  "targetNodeId": "343_jump2"},
+                                 {"id": "edge271",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "341_dse2",
+                                  "targetNodeId": "342_csa"},
+                                 {"id": "edge272",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "340_pro_and_epilogue",
+                                  "targetNodeId": "341_dse2"},
+                                 {"id": "edge273",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "339_cmpelim",
+                                  "targetNodeId": "340_pro_and_epilogue"},
+                                 {"id": "edge274",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "338_ree",
+                                  "targetNodeId": "339_cmpelim"},
+                                 {"id": "edge275",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "337_split2",
+                                  "targetNodeId": "338_ree"},
+                                 {"id": "edge276",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "336_gcse2",
+                                  "targetNodeId": "337_split2"},
+                                 {"id": "edge277",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "335_late_combine",
+                                  "targetNodeId": "336_gcse2"},
+                                 {"id": "edge278",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "334_vzeroupper",
+                                  "targetNodeId": "335_late_combine"},
+                                 {"id": "edge279",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "333_postreload",
+                                  "targetNodeId": "334_vzeroupper"},
+                                 {"id": "edge280",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*all-postreload_0x101f8410",
+                                  "targetNodeId": "333_postreload"},
+                                 {"id": "edge281",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "370_dwarf2",
+                                  "targetNodeId": "371_final"},
+                                 {"id": "edge282",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "369_nothrow",
+                                  "targetNodeId": "370_dwarf2"},
+                                 {"id": "edge283",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "368_shorten",
+                                  "targetNodeId": "369_nothrow"},
+                                 {"id": "edge284",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "367_align_tight_loops",
+                                  "targetNodeId": "368_shorten"},
+                                 {"id": "edge285",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "366_endbr_and_patchable_area",
+                                  "targetNodeId": "367_align_tight_loops"},
+                                 {"id": "edge286",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "365_eh_ranges",
+                                  "targetNodeId": "366_endbr_and_patchable_area"},
+                                 {"id": "edge287",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "364_split5",
+                                  "targetNodeId": "365_eh_ranges"},
+                                 {"id": "edge288",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "363_dbr",
+                                  "targetNodeId": "364_split5"},
+                                 {"id": "edge289",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "362_barriers",
+                                  "targetNodeId": "363_dbr"},
+                                 {"id": "edge290",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "361_mach",
+                                  "targetNodeId": "362_barriers"},
+                                 {"id": "edge291",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*free_cfg_0x101f9010",
+                                  "targetNodeId": "361_mach"},
+                                 {"id": "edge292",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "360_vartrack",
+                                  "targetNodeId": "*free_cfg_0x101f9010"},
+                                 {"id": "edge293",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "359_alignments",
+                                  "targetNodeId": "360_vartrack"},
+                                 {"id": "edge294",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "358_zero_call_used_regs",
+                                  "targetNodeId": "359_alignments"},
+                                 {"id": "edge295",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*all-late_compilation_0x101f8e90",
+                                  "targetNodeId": "358_zero_call_used_regs"},
+                                 {"id": "edge296",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*all-late_compilation_0x101f8e90",
+                                  "targetNodeId": "372_dfinish"},
+                                 {"id": "edge297",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "357_late_pro_and_epilogue",
+                                  "targetNodeId": "*all-late_compilation_0x101f8e90"},
+                                 {"id": "edge298",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*all-postreload_0x101f8410",
+                                  "targetNodeId": "357_late_pro_and_epilogue"},
+                                 {"id": "edge299",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "332_reload",
+                                  "targetNodeId": "*all-postreload_0x101f8410"},
+                                 {"id": "edge300",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "331_ira",
+                                  "targetNodeId": "332_reload"},
+                                 {"id": "edge301",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "330_early_remat",
+                                  "targetNodeId": "331_ira"},
+                                 {"id": "edge302",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "329_avoid_store_forwarding",
+                                  "targetNodeId": "330_early_remat"},
+                                 {"id": "edge303",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "328_sched1",
+                                  "targetNodeId": "329_avoid_store_forwarding"},
+                                 {"id": "edge304",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "327_lr_shrinkage",
+                                  "targetNodeId": "328_sched1"},
+                                 {"id": "edge305",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "326_sms",
+                                  "targetNodeId": "327_lr_shrinkage"},
+                                 {"id": "edge306",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "325_asmcons",
+                                  "targetNodeId": "326_sms"},
+                                 {"id": "edge307",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "324_mode_sw",
+                                  "targetNodeId": "325_asmcons"},
+                                 {"id": "edge308",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*stack_ptr_mod_0x101f8050",
+                                  "targetNodeId": "324_mode_sw"},
+                                 {"id": "edge309",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "323_no-opt dfinit",
+                                  "targetNodeId": "*stack_ptr_mod_0x101f8050"},
+                                 {"id": "edge310",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "322_subreg3",
+                                  "targetNodeId": "323_no-opt dfinit"},
+                                 {"id": "edge311",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "321_split1",
+                                  "targetNodeId": "322_subreg3"},
+                                 {"id": "edge312",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "320_outof_cfglayout",
+                                  "targetNodeId": "321_split1"},
+                                 {"id": "edge313",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "319_bbpart",
+                                  "targetNodeId": "320_outof_cfglayout"},
+                                 {"id": "edge314",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "318_jump_after_combine",
+                                  "targetNodeId": "319_bbpart"},
+                                 {"id": "edge315",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "317_ce2",
+                                  "targetNodeId": "318_jump_after_combine"},
+                                 {"id": "edge316",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "316_stv",
+                                  "targetNodeId": "317_ce2"},
+                                 {"id": "edge317",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "315_rrvl",
+                                  "targetNodeId": "316_stv"},
+                                 {"id": "edge318",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "314_rpad",
+                                  "targetNodeId": "315_rrvl"},
+                                 {"id": "edge319",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "313_late_combine",
+                                  "targetNodeId": "314_rpad"},
+                                 {"id": "edge320",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "312_combine",
+                                  "targetNodeId": "313_late_combine"},
+                                 {"id": "edge321",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "311_ext_dce",
+                                  "targetNodeId": "312_combine"},
+                                 {"id": "edge322",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "310_ud_dce",
+                                  "targetNodeId": "311_ext_dce"},
+                                 {"id": "edge323",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "309_init-regs",
+                                  "targetNodeId": "310_ud_dce"},
+                                 {"id": "edge324",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "308_auto_inc_dec",
+                                  "targetNodeId": "309_init-regs"},
+                                 {"id": "edge325",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "307_fwprop2",
+                                  "targetNodeId": "308_auto_inc_dec"},
+                                 {"id": "edge326",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "306_dse1",
+                                  "targetNodeId": "307_fwprop2"},
+                                 {"id": "edge327",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "305_cse2",
+                                  "targetNodeId": "306_dse1"},
+                                 {"id": "edge328",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "304_stv",
+                                  "targetNodeId": "305_cse2"},
+                                 {"id": "edge329",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "303_cprop",
+                                  "targetNodeId": "304_stv"},
+                                 {"id": "edge330",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "302_web",
+                                  "targetNodeId": "303_cprop"},
+                                 {"id": "edge331",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "301_subreg2",
+                                  "targetNodeId": "302_web"},
+                                 {"id": "edge332",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "295_loop2",
+                                  "targetNodeId": "301_subreg2"},
+                                 {"id": "edge333",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "294_reginfo",
+                                  "targetNodeId": "295_loop2"},
+                                 {"id": "edge334",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "293_apx_nfcvt",
+                                  "targetNodeId": "294_reginfo"},
+                                 {"id": "edge335",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "292_ce1",
+                                  "targetNodeId": "293_apx_nfcvt"},
+                                 {"id": "edge336",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "291_cse_local",
+                                  "targetNodeId": "292_ce1"},
+                                 {"id": "edge337",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "290_store_motion",
+                                  "targetNodeId": "291_cse_local"},
+                                 {"id": "edge338",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "289_cprop",
+                                  "targetNodeId": "290_store_motion"},
+                                 {"id": "edge339",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "288_hardreg_pre",
+                                  "targetNodeId": "289_cprop"},
+                                 {"id": "edge340",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "287_hoist",
+                                  "targetNodeId": "288_hardreg_pre"},
+                                 {"id": "edge341",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "286_rtl pre",
+                                  "targetNodeId": "287_hoist"},
+                                 {"id": "edge342",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "285_cprop",
+                                  "targetNodeId": "286_rtl pre"},
+                                 {"id": "edge343",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "284_fwprop1",
+                                  "targetNodeId": "285_cprop"},
+                                 {"id": "edge344",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "283_cse1",
+                                  "targetNodeId": "284_fwprop1"},
+                                 {"id": "edge345",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "282_dfinit",
+                                  "targetNodeId": "283_cse1"},
+                                 {"id": "edge346",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "281_subreg1",
+                                  "targetNodeId": "282_dfinit"},
+                                 {"id": "edge347",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "280_jump",
+                                  "targetNodeId": "281_subreg1"},
+                                 {"id": "edge348",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "279_into_cfglayout",
+                                  "targetNodeId": "280_jump"},
+                                 {"id": "edge349",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "278_vregs",
+                                  "targetNodeId": "279_into_cfglayout"},
+                                 {"id": "edge350",
+                                  "label": {"text": "sub"},
+                                  "sourceNodeId": "*rest_of_compilation_0x101f6e90",
+                                  "targetNodeId": "278_vregs"},
+                                 {"id": "edge351",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*rest_of_compilation_0x101f6e90",
+                                  "targetNodeId": "*clean_state_0x101f9500"},
+                                 {"id": "edge352",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "277_expand",
+                                  "targetNodeId": "*rest_of_compilation_0x101f6e90"},
+                                 {"id": "edge353",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*warn_function_noreturn_0x101f6dd0",
+                                  "targetNodeId": "277_expand"},
+                                 {"id": "edge354",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "276_optimized",
+                                  "targetNodeId": "*warn_function_noreturn_0x101f6dd0"},
+                                 {"id": "edge355",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "275_waccess",
+                                  "targetNodeId": "276_optimized"},
+                                 {"id": "edge356",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "274_hardcmp",
+                                  "targetNodeId": "275_waccess"},
+                                 {"id": "edge357",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "273_hardcbr",
+                                  "targetNodeId": "274_hardcmp"},
+                                 {"id": "edge358",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "272_isel",
+                                  "targetNodeId": "273_hardcbr"},
+                                 {"id": "edge359",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "271_nrv",
+                                  "targetNodeId": "272_isel"},
+                                 {"id": "edge360",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "270_resx",
+                                  "targetNodeId": "271_nrv"},
+                                 {"id": "edge361",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "269_ehcleanup",
+                                  "targetNodeId": "270_resx"},
+                                 {"id": "edge362",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "268_sanopt",
+                                  "targetNodeId": "269_ehcleanup"},
+                                 {"id": "edge363",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "267_musttail",
+                                  "targetNodeId": "268_sanopt"},
+                                 {"id": "edge364",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "266_tsan0",
+                                  "targetNodeId": "267_musttail"},
+                                 {"id": "edge365",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "265_asan0",
+                                  "targetNodeId": "266_tsan0"},
+                                 {"id": "edge366",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "264_switchlower_O0",
+                                  "targetNodeId": "265_asan0"},
+                                 {"id": "edge367",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "263_sancov_O0",
+                                  "targetNodeId": "264_switchlower_O0"},
+                                 {"id": "edge368",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "262_bitintlower0",
+                                  "targetNodeId": "263_sancov_O0"},
+                                 {"id": "edge369",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "261_cplxlower0",
+                                  "targetNodeId": "262_bitintlower0"},
+                                 {"id": "edge370",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "260_veclower",
+                                  "targetNodeId": "261_cplxlower0"},
+                                 {"id": "edge371",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "259_lower_vaarg",
+                                  "targetNodeId": "260_veclower"},
+                                 {"id": "edge372",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "258_vtable-verify",
+                                  "targetNodeId": "259_lower_vaarg"},
+                                 {"id": "edge373",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "257_simduid",
+                                  "targetNodeId": "258_vtable-verify"},
+                                 {"id": "edge374",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*tminit_0x101f6370",
+                                  "targetNodeId": "257_simduid"},
+                                 {"id": "edge375",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "253_assumptions",
+                                  "targetNodeId": "*tminit_0x101f6370"},
+                                 {"id": "edge376",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*all_optimizations_g_0x101f5af0",
+                                  "targetNodeId": "253_assumptions"},
+                                 {"id": "edge377",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "*all_optimizations_0x101f2720",
+                                  "targetNodeId": "*all_optimizations_g_0x101f5af0"},
+                                 {"id": "edge378",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "114_hardcfr",
+                                  "targetNodeId": "*all_optimizations_0x101f2720"},
+                                 {"id": "edge379",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "113_adjust_alignment",
+                                  "targetNodeId": "114_hardcfr"},
+                                 {"id": "edge380",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "112_omptargetlink",
+                                  "targetNodeId": "113_adjust_alignment"},
+                                 {"id": "edge381",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "111_ompdevlow",
+                                  "targetNodeId": "112_omptargetlink"},
+                                 {"id": "edge382",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "110_oaccdevlow",
+                                  "targetNodeId": "111_ompdevlow"},
+                                 {"id": "edge383",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "109_omp_oacc_neuter_broadcast",
+                                  "targetNodeId": "110_oaccdevlow"},
+                                 {"id": "edge384",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "108_oaccloops",
+                                  "targetNodeId": "109_omp_oacc_neuter_broadcast"},
+                                 {"id": "edge385",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "107_ehdisp",
+                                  "targetNodeId": "108_oaccloops"},
+                                 {"id": "edge386",
+                                  "label": {"text": "next"},
+                                  "sourceNodeId": "106_fixup_cfg",
+                                  "targetNodeId": "107_ehdisp"}]}]}]}
+
+/* { dg-begin-multiline-output "" }
+In function 'test_graphs':
+/home/david/gcc-newgit-gcc16/src/gcc/testsuite/gcc.dg/plugin/diagnostic-test-graphs-sarif.c:8:3: error: this is a placeholder error, with graphs
+   { dg-end-multiline-output "" } */
+
+/* Use a Python script to verify various properties about the generated
+   .html file:
+   { dg-final { run-html-pytest graphs.sarif "2.1.0-valid/graphs-check-html.py" } } */
+
+/* Use a Python script to verify various properties about the *generated*
+   .sarif file:
+   { dg-final { run-sarif-pytest graphs.roundtrip "2.1.0-valid/graphs-check-sarif-roundtrip.py" } } */
diff --git a/gcc/text-art/style.cc b/gcc/text-art/style.cc
index ffc75b6..40886d6 100644
--- a/gcc/text-art/style.cc
+++ b/gcc/text-art/style.cc
@@ -29,7 +29,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "text-art/selftests.h"
 #include "text-art/types.h"
 #include "color-macros.h"
-#include "diagnostic-color.h"
+#include "diagnostics/color.h"
 
 using namespace text_art;
 
diff --git a/gcc/toplev.cc b/gcc/toplev.cc
index 00a8ccb..70dbb3e 100644
--- a/gcc/toplev.cc
+++ b/gcc/toplev.cc
@@ -81,7 +81,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "ipa-utils.h"
 #include "gcse.h"
 #include "omp-offload.h"
-#include "edit-context.h"
+#include "diagnostics/changes.h"
+#include "diagnostics/file-cache.h"
 #include "tree-pass.h"
 #include "dumpfile.h"
 #include "ipa-fnsummary.h"
@@ -230,7 +231,8 @@ announce_function (tree decl)
 		 identifier_to_locale (lang_hooks.decl_printable_name (decl, 2)));
       fflush (stderr);
       pp_needs_newline (global_dc->get_reference_printer ()) = true;
-      diagnostic_set_last_function (global_dc, (diagnostic_info *) NULL);
+      diagnostic_set_last_function (global_dc,
+				    (diagnostics::diagnostic_info *) nullptr);
     }
 }
 
@@ -1017,7 +1019,7 @@ open_auxiliary_file (const char *ext)
 /* Alternative diagnostics callback for reentered ICE reporting.  */
 
 static void
-internal_error_reentered (diagnostic_context *, const char *, va_list *)
+internal_error_reentered (diagnostics::context *, const char *, va_list *)
 {
   /* Flush the dump file if emergency_dump_function itself caused an ICE.  */
   if (dump_file)
@@ -1027,7 +1029,7 @@ internal_error_reentered (diagnostic_context *, const char *, va_list *)
 /* Auxiliary callback for the diagnostics code.  */
 
 static void
-internal_error_function (diagnostic_context *, const char *, va_list *)
+internal_error_function (diagnostics::context *, const char *, va_list *)
 {
   global_dc->set_internal_error_callback (internal_error_reentered);
   warn_if_plugins ();
@@ -1068,13 +1070,14 @@ general_init (const char *argv0, bool init_signals, unique_argv original_argv)
 
   global_dc->set_original_argv (std::move (original_argv));
 
-  global_dc->m_source_printing.enabled
+  auto &source_printing_opts = global_dc->get_source_printing_options ();
+  source_printing_opts.enabled
     = global_options_init.x_flag_diagnostics_show_caret;
-  global_dc->m_source_printing.show_event_links_p
+  source_printing_opts.show_event_links_p
     = global_options_init.x_flag_diagnostics_show_event_links;
-  global_dc->m_source_printing.show_labels_p
+  source_printing_opts.show_labels_p
     = global_options_init.x_flag_diagnostics_show_labels;
-  global_dc->m_source_printing.show_line_numbers_p
+  source_printing_opts.show_line_numbers_p
     = global_options_init.x_flag_diagnostics_show_line_numbers;
   global_dc->set_show_cwe (global_options_init.x_flag_diagnostics_show_cwe);
   global_dc->set_show_rules (global_options_init.x_flag_diagnostics_show_rules);
@@ -1085,7 +1088,7 @@ general_init (const char *argv0, bool init_signals, unique_argv original_argv)
     (global_options_init.x_flag_diagnostics_show_path_depths);
   global_dc->set_show_option_requested
     (global_options_init.x_flag_diagnostics_show_option);
-  global_dc->m_source_printing.min_margin_width
+  source_printing_opts.min_margin_width
     = global_options_init.x_diagnostics_minimum_margin_width;
   global_dc->m_show_column
     = global_options_init.x_flag_show_column;
@@ -1285,7 +1288,7 @@ process_options ()
   input_location = saved_location;
 
   if (flag_diagnostics_generate_patch)
-    global_dc->create_edit_context ();
+    global_dc->initialize_fixits_change_set ();
 
   /* Avoid any informative notes in the second run of -fcompare-debug.  */
   if (flag_compare_debug)
@@ -1748,11 +1751,13 @@ process_options ()
       if (warn_coverage_mismatch
 	  && option_unspecified_p (OPT_Wcoverage_mismatch))
 	diagnostic_classify_diagnostic (global_dc, OPT_Wcoverage_mismatch,
-					DK_ERROR, UNKNOWN_LOCATION);
+					diagnostics::kind::error,
+					UNKNOWN_LOCATION);
       if (warn_coverage_invalid_linenum
 	  && option_unspecified_p (OPT_Wcoverage_invalid_line_number))
 	diagnostic_classify_diagnostic (global_dc, OPT_Wcoverage_invalid_line_number,
-					DK_ERROR, UNKNOWN_LOCATION);
+					diagnostics::kind::error,
+					UNKNOWN_LOCATION);
     }
 
   /* Save the current optimization options.  */
@@ -1815,6 +1820,10 @@ backend_init_target (void)
 static void
 backend_init (void)
 {
+#if CHECKING_P
+  verify_reg_names_in_constraints ();
+#endif
+
   init_emit_once ();
 
   init_rtlanal ();
@@ -2385,11 +2394,11 @@ toplev::main (int argc, char **argv)
      emit some diagnostics here.  */
   invoke_plugin_callbacks (PLUGIN_FINISH, NULL);
 
-  if (auto edit_context_ptr = global_dc->get_edit_context ())
+  if (auto change_set_ptr = global_dc->get_fixits_change_set ())
     {
       pretty_printer pp;
       pp_show_color (&pp) = pp_show_color (global_dc->get_reference_printer ());
-      edit_context_ptr->print_diff (&pp, true);
+      change_set_ptr->print_diff (&pp, true);
       pp_flush (&pp);
     }
 
diff --git a/gcc/tree-call-cdce.cc b/gcc/tree-call-cdce.cc
index 649c1e2..3edea75 100644
--- a/gcc/tree-call-cdce.cc
+++ b/gcc/tree-call-cdce.cc
@@ -36,6 +36,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "builtins.h"
 #include "internal-fn.h"
 #include "tree-dfa.h"
+#include "tree-eh.h"
 
 
 /* This pass serves two closely-related purposes:
@@ -1222,8 +1223,20 @@ use_internal_fn (gcall *call)
     {
       /* Skip the call if LHS == LHS.  If we reach here, EDOM is the only
 	 valid errno value and it is used iff the result is NaN.  */
-      conds.quick_push (gimple_build_cond (EQ_EXPR, lhs, lhs,
-					   NULL_TREE, NULL_TREE));
+      /* In the case of non call exceptions, with signaling NaNs, EQ_EXPR
+	 can throw an exception and that can't be part of the GIMPLE_COND. */
+      if (flag_exceptions
+	  && cfun->can_throw_non_call_exceptions
+	  && operation_could_trap_p (EQ_EXPR, true, false, NULL_TREE))
+	{
+	  tree b = make_ssa_name (boolean_type_node);
+	  conds.quick_push (gimple_build_assign (b, EQ_EXPR, lhs, lhs));
+	  conds.quick_push (gimple_build_cond (NE_EXPR, b, boolean_false_node,
+					       NULL_TREE, NULL_TREE));
+	}
+      else
+	conds.quick_push (gimple_build_cond (EQ_EXPR, lhs, lhs,
+					     NULL_TREE, NULL_TREE));
       nconds++;
 
       /* Try replacing the original call with a direct assignment to
diff --git a/gcc/tree-cfg.cc b/gcc/tree-cfg.cc
index 72763fd..275fc43 100644
--- a/gcc/tree-cfg.cc
+++ b/gcc/tree-cfg.cc
@@ -114,43 +114,6 @@ struct replace_decls_d
   tree to_context;
 };
 
-/* Hash table to store last discriminator assigned for each locus.  */
-struct locus_discrim_map
-{
-  int location_line;
-  int discriminator;
-};
-
-/* Hashtable helpers.  */
-
-struct locus_discrim_hasher : free_ptr_hash <locus_discrim_map>
-{
-  static inline hashval_t hash (const locus_discrim_map *);
-  static inline bool equal (const locus_discrim_map *,
-			    const locus_discrim_map *);
-};
-
-/* Trivial hash function for a location_t.  ITEM is a pointer to
-   a hash table entry that maps a location_t to a discriminator.  */
-
-inline hashval_t
-locus_discrim_hasher::hash (const locus_discrim_map *item)
-{
-  return item->location_line;
-}
-
-/* Equality function for the locus-to-discriminator map.  A and B
-   point to the two hash table entries to compare.  */
-
-inline bool
-locus_discrim_hasher::equal (const locus_discrim_map *a,
-			     const locus_discrim_map *b)
-{
-  return a->location_line == b->location_line;
-}
-
-static hash_table<locus_discrim_hasher> *discriminator_per_locus;
-
 /* Basic blocks and flowgraphs.  */
 static void make_blocks (gimple_seq);
 
@@ -168,7 +131,6 @@ static edge gimple_try_redirect_by_replacing_jump (edge, basic_block);
 static inline bool stmt_starts_bb_p (gimple *, gimple *);
 static bool gimple_verify_flow_info (void);
 static void gimple_make_forwarder_block (edge);
-static gimple *first_non_label_nondebug_stmt (basic_block);
 static bool verify_gimple_transaction (gtransaction *);
 static bool call_can_make_abnormal_goto (gimple *);
 
@@ -247,12 +209,9 @@ build_gimple_cfg (gimple_seq seq)
   group_case_labels ();
 
   /* Create the edges of the flowgraph.  */
-  discriminator_per_locus = new hash_table<locus_discrim_hasher> (13);
   make_edges ();
   assign_discriminators ();
   cleanup_dead_labels ();
-  delete discriminator_per_locus;
-  discriminator_per_locus = NULL;
 }
 
 /* Look for ANNOTATE calls with loop annotation kind in BB; if found, remove
@@ -1120,77 +1079,41 @@ gimple_find_sub_bbs (gimple_seq seq, gimple_stmt_iterator *gsi)
   return true;
 }
 
-/* Find the next available discriminator value for LOCUS.  The
-   discriminator distinguishes among several basic blocks that
-   share a common locus, allowing for more accurate sample-based
-   profiling.  */
-
-static int
-next_discriminator_for_locus (int line)
+/* Auto-profile needs discriminator to distinguish statements with same line
+   number (file name is ignored) which are in different basic block.  This
+   map keeps track of current discriminator for a given line number.  */
+struct discrim_entry
 {
-  struct locus_discrim_map item;
-  struct locus_discrim_map **slot;
-
-  item.location_line = line;
-  item.discriminator = 0;
-  slot = discriminator_per_locus->find_slot_with_hash (&item, line, INSERT);
-  gcc_assert (slot);
-  if (*slot == HTAB_EMPTY_ENTRY)
-    {
-      *slot = XNEW (struct locus_discrim_map);
-      gcc_assert (*slot);
-      (*slot)->location_line = line;
-      (*slot)->discriminator = 0;
-    }
-  (*slot)->discriminator++;
-  return (*slot)->discriminator;
-}
-
-/* Return TRUE if LOCUS1 and LOCUS2 refer to the same source line.  */
-
-static bool
-same_line_p (location_t locus1, expanded_location *from, location_t locus2)
-{
-  expanded_location to;
-
-  if (locus1 == locus2)
-    return true;
-
-  to = expand_location (locus2);
-
-  if (from->line != to.line)
-    return false;
-  if (from->file == to.file)
-    return true;
-  return (from->file != NULL
-          && to.file != NULL
-          && filename_cmp (from->file, to.file) == 0);
-}
+  /* ID of basic block we saw line number last time.  */
+  unsigned int bb_id;
+  /* Discriminator we used.  */
+  unsigned int discrim;
+};
 
-/* Assign a unique discriminator value to all statements in block bb that
-   have the same line number as locus. */
+/* Return updated LOC with discriminator for use in basic block BB_ID.
+   MAP keeps track of current values.  */
 
-static void
-assign_discriminator (location_t locus, basic_block bb)
+location_t
+assign_discriminator (location_t loc, unsigned int bb_id,
+		      hash_map<int_hash <int64_t, -1, -2>, discrim_entry> &map)
 {
-  gimple_stmt_iterator gsi;
-  int discriminator;
-
-  if (locus == UNKNOWN_LOCATION)
-    return;
-
-  expanded_location locus_e = expand_location (locus);
-
-  discriminator = next_discriminator_for_locus (locus_e.line);
-
-  for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+  bool existed;
+  discrim_entry &e = map.get_or_insert (LOCATION_LINE (loc), &existed);
+  gcc_checking_assert (!has_discriminator (loc));
+  if (!existed)
     {
-      gimple *stmt = gsi_stmt (gsi);
-      location_t stmt_locus = gimple_location (stmt);
-      if (same_line_p (locus, &locus_e, stmt_locus))
-	gimple_set_location (stmt,
-	    location_with_discriminator (stmt_locus, discriminator));
+      e.bb_id = bb_id;
+      e.discrim = 0;
+      return loc;
     }
+  if (e.bb_id != bb_id)
+    {
+      e.bb_id = bb_id;
+      e.discrim++;
+    }
+  if (e.discrim)
+    return location_with_discriminator (loc, e.discrim);
+  return loc;
 }
 
 /* Assign discriminators to statement locations.  */
@@ -1198,92 +1121,54 @@ assign_discriminator (location_t locus, basic_block bb)
 static void
 assign_discriminators (void)
 {
+  hash_map<int_hash <int64_t, -1, -2>, discrim_entry> map (13);
+  unsigned int bb_id = 0;
   basic_block bb;
-
   FOR_EACH_BB_FN (bb, cfun)
     {
-      edge e;
-      edge_iterator ei;
-      gimple_stmt_iterator gsi;
-      location_t curr_locus = UNKNOWN_LOCATION;
-      expanded_location curr_locus_e = {};
-      int curr_discr = 0;
-
+      location_t prev_loc = UNKNOWN_LOCATION, prev_replacement = UNKNOWN_LOCATION;
       /* Traverse the basic block, if two function calls within a basic block
 	are mapped to the same line, assign a new discriminator because a call
 	stmt could be a split point of a basic block.  */
-      for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+      for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
+	   !gsi_end_p (gsi); gsi_next (&gsi))
 	{
 	  gimple *stmt = gsi_stmt (gsi);
-
-	  /* Don't allow debug stmts to affect discriminators, but
-	     allow them to take discriminators when they're on the
-	     same line as the preceding nondebug stmt.  */
-	  if (is_gimple_debug (stmt))
-	    {
-	      if (curr_locus != UNKNOWN_LOCATION
-		  && same_line_p (curr_locus, &curr_locus_e,
-				  gimple_location (stmt)))
-		{
-		  location_t loc = gimple_location (stmt);
-		  location_t dloc = location_with_discriminator (loc,
-								 curr_discr);
-		  gimple_set_location (stmt, dloc);
-		}
-	      continue;
-	    }
-	  if (curr_locus == UNKNOWN_LOCATION)
-	    {
-	      curr_locus = gimple_location (stmt);
-	      curr_locus_e = expand_location (curr_locus);
-	    }
-	  else if (!same_line_p (curr_locus, &curr_locus_e, gimple_location (stmt)))
-	    {
-	      curr_locus = gimple_location (stmt);
-	      curr_locus_e = expand_location (curr_locus);
-	      curr_discr = 0;
-	    }
-	  else if (curr_discr != 0)
+	  location_t loc = gimple_location (stmt);
+	  if (loc == UNKNOWN_LOCATION)
+	    continue;
+	  if (loc == prev_loc)
+	    gimple_set_location (stmt, prev_replacement);
+	  else
 	    {
-	      location_t loc = gimple_location (stmt);
-	      location_t dloc = location_with_discriminator (loc, curr_discr);
-	      gimple_set_location (stmt, dloc);
+	      prev_loc = loc;
+	      prev_replacement = assign_discriminator (loc, bb_id, map);
+	      gimple_set_location (stmt, prev_replacement);
 	    }
-	  /* Allocate a new discriminator for CALL stmt.  */
+	  /* Break basic blocks after each call.  This is requires so each
+	     call site has unque discriminator.
+	     More correctly, we can break after each statement that can possibly
+	     terinate execution of the basic block, but for auto-profile this
+	     precision is probably not useful.  */
 	  if (gimple_code (stmt) == GIMPLE_CALL)
-	    curr_discr = next_discriminator_for_locus (curr_locus_e.line);
-	}
-
-      gimple *last = last_nondebug_stmt (bb);
-      location_t locus = last ? gimple_location (last) : UNKNOWN_LOCATION;
-      if (locus == UNKNOWN_LOCATION)
-	continue;
-
-      expanded_location locus_e = expand_location (locus);
-
-      FOR_EACH_EDGE (e, ei, bb->succs)
-	{
-	  gimple *first = first_non_label_nondebug_stmt (e->dest);
-	  gimple *last = last_nondebug_stmt (e->dest);
-
-	  gimple *stmt_on_same_line = NULL;
-	  if (first && same_line_p (locus, &locus_e,
-				     gimple_location (first)))
-	    stmt_on_same_line = first;
-	  else if (last && same_line_p (locus, &locus_e,
-					gimple_location (last)))
-	    stmt_on_same_line = last;
-
-	  if (stmt_on_same_line)
 	    {
-	      if (has_discriminator (gimple_location (stmt_on_same_line))
-		  && !has_discriminator (locus))
-		assign_discriminator (locus, bb);
-	      else
-		assign_discriminator (locus, e->dest);
+	      prev_loc = UNKNOWN_LOCATION;
+	      bb_id++;
 	    }
 	}
+      /* IF basic block has multiple sucessors, consdier every edge as a separate
+	 block.  */
+      if (!single_succ_p (bb))
+	bb_id++;
+      for (edge e : bb->succs)
+	if (e->goto_locus != UNKNOWN_LOCATION)
+	  {
+	    e->goto_locus = assign_discriminator (e->goto_locus, bb_id, map);
+	    bb_id++;
+	  }
+      bb_id++;
     }
+
 }
 
 /* Create the edges for a GIMPLE_COND starting at block BB.  */
@@ -2948,16 +2833,6 @@ first_stmt (basic_block bb)
   return stmt;
 }
 
-/* Return the first non-label/non-debug statement in basic block BB.  */
-
-static gimple *
-first_non_label_nondebug_stmt (basic_block bb)
-{
-  gimple_stmt_iterator i;
-  i = gsi_start_nondebug_after_labels_bb (bb);
-  return !gsi_end_p (i) ? gsi_stmt (i) : NULL;
-}
-
 /* Return the last statement in basic block BB.  */
 
 gimple *
@@ -4623,6 +4498,14 @@ verify_gimple_assign_single (gassign *stmt)
       return true;
     }
 
+  /* LHS can't be a constant or an address expression. */
+  if (CONSTANT_CLASS_P (lhs)|| TREE_CODE (lhs) == ADDR_EXPR)
+    {
+      error ("invalid LHS (%qs) for assignment: %qs",
+	     get_tree_code_name (TREE_CODE (lhs)), code_name);
+      return true;
+    }
+
   if (gimple_clobber_p (stmt)
       && !(DECL_P (lhs) || TREE_CODE (lhs) == MEM_REF))
     {
@@ -4745,6 +4628,11 @@ verify_gimple_assign_single (gassign *stmt)
 
 	  if (CONSTRUCTOR_NELTS (rhs1) == 0)
 	    return res;
+	  if (!is_gimple_reg (lhs))
+	    {
+	      error ("non-register as LHS with vector constructor");
+	      return true;
+	    }
 	  /* For vector CONSTRUCTORs we require that either it is empty
 	     CONSTRUCTOR, or it is a CONSTRUCTOR of smaller vector elements
 	     (then the element count must be correct to cover the whole
@@ -9961,6 +9849,8 @@ do_warn_unused_result (gimple_seq seq)
 	    break;
 	  if (gimple_call_internal_p (g))
 	    break;
+	  if (warning_suppressed_p (g, OPT_Wunused_result))
+	    break;
 
 	  /* This is a naked call, as opposed to a GIMPLE_CALL with an
 	     LHS.  All calls whose value is ignored should be
diff --git a/gcc/tree-cfgcleanup.cc b/gcc/tree-cfgcleanup.cc
index a34a51e..5aaa18d 100644
--- a/gcc/tree-cfgcleanup.cc
+++ b/gcc/tree-cfgcleanup.cc
@@ -47,6 +47,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-into-ssa.h"
 #include "tree-cfgcleanup.h"
 #include "gimple-pretty-print.h"
+#include "target.h"
 
 
 /* The set of blocks in that at least one of the following changes happened:
@@ -1569,6 +1570,29 @@ execute_cleanup_cfg_post_optimizing (void)
   cleanup_dead_labels ();
   if (group_case_labels ())
     todo |= TODO_cleanup_cfg;
+
+  basic_block bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
+  gimple_stmt_iterator gsi = gsi_start_nondebug_after_labels_bb (bb);
+  /* If the first (and only) bb and the only non debug
+     statement is __builtin_unreachable call, then replace it with a trap
+     so the function is at least one instruction in size.  */
+  if (!gsi_end_p (gsi)
+      && gimple_call_builtin_p (gsi_stmt (gsi), BUILT_IN_UNREACHABLE))
+    {
+      if (targetm.have_trap ())
+	{
+	  gimple_call_set_fndecl (gsi_stmt (gsi), builtin_decl_implicit (BUILT_IN_UNREACHABLE_TRAP));
+	  update_stmt (gsi_stmt (gsi));
+	}
+      /* If the target does not have a trap, convert it into an infinite loop. */
+      else
+	{
+	  gsi_remove (&gsi, true);
+	  make_single_succ_edge (bb, bb, EDGE_FALLTHRU);
+	  fix_loop_structure (NULL);
+	}
+    }
+
   if ((flag_compare_debug_opt || flag_compare_debug)
       && flag_dump_final_insns)
     {
diff --git a/gcc/tree-diagnostic-client-data-hooks.cc b/gcc/tree-diagnostic-client-data-hooks.cc
index 84bd4b5..77eb292 100644
--- a/gcc/tree-diagnostic-client-data-hooks.cc
+++ b/gcc/tree-diagnostic-client-data-hooks.cc
@@ -1,4 +1,4 @@
-/* Implementation of diagnostic_client_data_hooks for the compilers
+/* Implementation of diagnostics::client_data_hooks for the compilers
    (e.g. with knowledge of "tree", lang_hooks, and timevars).
    Copyright (C) 2022-2025 Free Software Foundation, Inc.
    Contributed by David Malcolm <dmalcolm@redhat.com>.
@@ -26,18 +26,18 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree.h"
 #include "diagnostic.h"
 #include "tree-logical-location.h"
-#include "diagnostic-client-data-hooks.h"
-#include "diagnostic-format-sarif.h"
+#include "diagnostics/client-data-hooks.h"
+#include "diagnostics/sarif-sink.h"
 #include "langhooks.h"
 #include "plugin.h"
 #include "timevar.h"
 
-/* Concrete class for supplying a diagnostic_context with information
+/* Concrete class for supplying a diagnostics::context with information
    about a specific plugin within the client, when the client is the
    compiler (i.e. a GCC plugin).  */
 
 class compiler_diagnostic_client_plugin_info
-  : public diagnostic_client_plugin_info
+  : public diagnostics::client_plugin_info
 {
 public:
   compiler_diagnostic_client_plugin_info (const plugin_name_args *args)
@@ -67,7 +67,7 @@ private:
 /* Concrete subclass of client_version_info for use by compilers proper,
    (i.e. using lang_hooks, and with knowledge of GCC plugins).  */
 
-class compiler_version_info : public client_version_info
+class compiler_version_info : public diagnostics::client_version_info
 {
 public:
   const char *get_tool_name () const final override
@@ -112,23 +112,26 @@ private:
   }
 };
 
-/* Subclass of diagnostic_client_data_hooks for use by compilers proper
+/* Subclass of diagnostics::client_data_hooks for use by compilers proper
    i.e. with knowledge of "tree", access to langhooks, timevars etc.  */
 
-class compiler_data_hooks : public diagnostic_client_data_hooks
+class compiler_data_hooks : public diagnostics::client_data_hooks
 {
 public:
-  const client_version_info *get_any_version_info () const final override
+  const diagnostics::client_version_info *
+  get_any_version_info () const final override
   {
     return &m_version_info;
   }
 
-  const logical_location_manager *get_logical_location_manager () const final override
+  const diagnostics::logical_locations::manager *
+  get_logical_location_manager () const final override
   {
     return &m_logical_location_manager;
   }
 
-  logical_location_manager::key get_current_logical_location () const final override
+  diagnostics::logical_locations::key
+  get_current_logical_location () const final override
   {
     return m_logical_location_manager.key_from_tree (current_function_decl);
   }
@@ -140,13 +143,13 @@ public:
   }
 
   void
-  add_sarif_invocation_properties (sarif_object &invocation_obj)
+  add_sarif_invocation_properties (diagnostics::sarif_object &invocation_obj)
     const final override
   {
     if (g_timer)
       if (auto timereport_val = g_timer->make_json ())
 	{
-	  sarif_property_bag &bag_obj
+	  auto &bag_obj
 	    = invocation_obj.get_or_create_properties ();
 	  bag_obj.set ("gcc/timeReport", std::move (timereport_val));
 
@@ -167,7 +170,7 @@ private:
 /* Create a compiler_data_hooks (so that the class can be local
    to this file).  */
 
-std::unique_ptr<diagnostic_client_data_hooks>
+std::unique_ptr<diagnostics::client_data_hooks>
 make_compiler_data_hooks ()
 {
   return std::make_unique<compiler_data_hooks> ();
diff --git a/gcc/tree-diagnostic.cc b/gcc/tree-diagnostic.cc
index 57c6715..3761fc0 100644
--- a/gcc/tree-diagnostic.cc
+++ b/gcc/tree-diagnostic.cc
@@ -27,29 +27,29 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-pretty-print.h"
 #include "gimple-pretty-print.h"
 #include "tree-diagnostic.h"
-#include "diagnostic-client-data-hooks.h"
+#include "diagnostics/client-data-hooks.h"
 #include "langhooks.h"
 #include "intl.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 
 /* Prints out, if necessary, the name of the current function
    that caused an error.  */
 void
-diagnostic_report_current_function (diagnostic_text_output_format &text_output,
-				    const diagnostic_info *diagnostic)
+diagnostic_report_current_function (diagnostics::text_sink &text_output,
+				    const diagnostics::diagnostic_info *diag)
 {
-  location_t loc = diagnostic_location (diagnostic);
+  location_t loc = diagnostic_location (diag);
   text_output.report_current_module (loc);
-  lang_hooks.print_error_function (text_output, LOCATION_FILE (loc), diagnostic);
+  lang_hooks.print_error_function (text_output, LOCATION_FILE (loc), diag);
 }
 
 static void
-default_tree_diagnostic_text_starter (diagnostic_text_output_format &text_output,
-				      const diagnostic_info *diagnostic)
+default_tree_diagnostic_text_starter (diagnostics::text_sink &text_output,
+				      const diagnostics::diagnostic_info *diag)
 {
   pretty_printer *const pp = text_output.get_printer ();
-  diagnostic_report_current_function (text_output, diagnostic);
-  pp_set_prefix (pp, text_output.build_prefix (*diagnostic));
+  diagnostic_report_current_function (text_output, diag);
+  pp_set_prefix (pp, text_output.build_prefix (*diag));
 }
 
 /* Default tree printer.   Handles declarations only.  */
@@ -110,8 +110,8 @@ default_tree_printer (pretty_printer *pp, text_info *text, const char *spec,
    to the DIAGNOSTIC location.  */
 
 static void
-set_inlining_locations (diagnostic_context *,
-			diagnostic_info *diagnostic)
+set_inlining_locations (diagnostics::context *,
+			diagnostics::diagnostic_info *diagnostic)
 {
   location_t loc = diagnostic_location (diagnostic);
   tree block = LOCATION_BLOCK (loc);
@@ -169,16 +169,16 @@ set_inlining_locations (diagnostic_context *,
   /* Set if all locations are in a system header.  */
   diagnostic->m_iinfo.m_allsyslocs = nsyslocs == ilocs.length ();
 
-  if (tree *ao = pp_ti_abstract_origin (&diagnostic->message))
+  if (tree *ao = pp_ti_abstract_origin (&diagnostic->m_message))
     *ao = (tree)diagnostic->m_iinfo.m_ao;
 }
 
 /* Sets CONTEXT to use language independent diagnostics.  */
 void
-tree_diagnostics_defaults (diagnostic_context *context)
+tree_diagnostics_defaults (diagnostics::context *context)
 {
-  diagnostic_text_starter (context) = default_tree_diagnostic_text_starter;
-  diagnostic_text_finalizer (context) = default_diagnostic_text_finalizer;
+  diagnostics::text_starter (context) = default_tree_diagnostic_text_starter;
+  diagnostics::text_finalizer (context) = diagnostics::default_text_finalizer;
   context->set_format_decoder (default_tree_printer);
   context->set_set_locations_callback (set_inlining_locations);
   context->set_client_data_hooks (make_compiler_data_hooks ());
diff --git a/gcc/tree-diagnostic.h b/gcc/tree-diagnostic.h
index b138b8b..9315d35 100644
--- a/gcc/tree-diagnostic.h
+++ b/gcc/tree-diagnostic.h
@@ -48,10 +48,10 @@ along with GCC; see the file COPYING3.  If not see
        ? diagnostic_abstract_origin (DI)		\
        : current_function_decl)
 
-void diagnostic_report_current_function (diagnostic_context *,
-					 const diagnostic_info *);
+void diagnostic_report_current_function (diagnostics::context *,
+					 const diagnostics::diagnostic_info *);
 
-void tree_diagnostics_defaults (diagnostic_context *context);
+void tree_diagnostics_defaults (diagnostics::context *);
 bool default_tree_printer (pretty_printer *, text_info *, const char *,
 			   int, bool, bool, bool, bool *, pp_token_list &);
 
diff --git a/gcc/tree-if-conv.cc b/gcc/tree-if-conv.cc
index 636361e..b7ce072 100644
--- a/gcc/tree-if-conv.cc
+++ b/gcc/tree-if-conv.cc
@@ -494,6 +494,10 @@ fold_or_predicates (location_t loc, tree c1, tree c2)
 static tree
 fold_build_cond_expr (tree type, tree cond, tree rhs, tree lhs)
 {
+  /* Short cut the case where both rhs and lhs are the same. */
+  if (operand_equal_p (rhs, lhs))
+    return rhs;
+
   /* If COND is comparison r != 0 and r has boolean type, convert COND
      to SSA_NAME to accept by vect bool pattern.  */
   if (TREE_CODE (cond) == NE_EXPR)
@@ -2110,6 +2114,194 @@ gen_phi_arg_condition (gphi *phi, ifcvt_arg_entry_t &arg,
   return cond;
 }
 
+/* Find the operand which is different between ARG0_OP and ARG1_OP.
+   Returns the operand num where the difference is.
+   Set NEWARG0 and NEWARG1 from the different argument.
+   Returns -1 if none is found.
+   If ARG0_OP/ARG1_OP is commutative also try swapping the
+   two commutative operands and return the operand number where
+   the difference happens in ARG0_OP. */
+
+static int
+find_different_opnum (const gimple_match_op &arg0_op,
+		      const gimple_match_op &arg1_op,
+		      tree *new_arg0, tree *new_arg1)
+{
+  unsigned opnum = -1;
+  unsigned first;
+  first = first_commutative_argument (arg1_op.code, arg1_op.type);
+  for (unsigned i = 0; i < arg0_op.num_ops; i++)
+    {
+      if (!operand_equal_for_phi_arg_p (arg0_op.ops[i],
+					arg1_op.ops[i]))
+	{
+	  /* Can handle only one non equal operand. */
+	  if (opnum != -1u)
+	    {
+	      /* Though if opnum is right before i and opnum is equal
+		 to the first communtative argument, handle communtative
+		 specially. */
+	      if (i == opnum + 1 && opnum == first)
+		goto commutative;
+	      return -1;
+	    }
+	  opnum = i;
+	}
+  }
+  /* If all operands are equal only do this is there was single
+     operand.  */
+  if (opnum == -1u)
+    {
+      if (arg0_op.num_ops != 1)
+	return -1;
+      opnum = 0;
+    }
+  *new_arg0 = arg0_op.ops[opnum];
+  *new_arg1 = arg1_op.ops[opnum];
+  return opnum;
+
+/* Handle commutative operations. */
+commutative:
+  gcc_assert (first != -1u);
+
+  /* Check the rest of the arguments to make sure they are the same. */
+  for (unsigned i = first + 2; i < arg0_op.num_ops; i++)
+    if (!operand_equal_for_phi_arg_p (arg0_op.ops[i],
+				      arg1_op.ops[i]))
+      return -1;
+
+  /* If the arg0[first+1] and arg1[first] are the same
+     then the one which is different is arg0[first] and arg1[first+1]
+     return first since this is based on arg0.  */
+  if (operand_equal_for_phi_arg_p (arg0_op.ops[first + 1],
+				   arg1_op.ops[first]))
+    {
+       *new_arg0 = arg0_op.ops[first];
+       *new_arg1 = arg1_op.ops[first + 1];
+       return first;
+    }
+  /* If the arg0[first] and arg1[first+1] are the same
+     then the one which is different is arg0[first+1] and arg1[first]
+     return first+1 since this is based on arg0.  */
+  if (operand_equal_for_phi_arg_p (arg0_op.ops[first],
+				   arg1_op.ops[first + 1]))
+    {
+       *new_arg0 = arg0_op.ops[first + 1];
+       *new_arg1 = arg1_op.ops[first];
+       return first + 1;
+    }
+  return -1;
+}
+
+/* Factors out an operation from *ARG0 and *ARG1 and
+   create the new statement at GSI. *RES is the
+   result of that new statement. Update *ARG0 and *ARG1
+   and *RES to the new values if the factoring happened.
+   Loops until all of the factoring is completed.  */
+
+static void
+factor_out_operators (tree *res, gimple_stmt_iterator *gsi,
+		      tree *arg0, tree *arg1, gphi *phi)
+{
+  gimple_match_op arg0_op, arg1_op;
+  bool repeated = false;
+
+again:
+  if (TREE_CODE (*arg0) != SSA_NAME || TREE_CODE (*arg1) != SSA_NAME)
+    return;
+
+  if (operand_equal_p (*arg0, *arg1))
+    return;
+
+  /* If either args have > 1 use, then this transformation actually
+     increases the number of expressions evaluated at runtime.  */
+  if (repeated
+      ? (!has_zero_uses (*arg0) || !has_zero_uses (*arg1))
+      : (!has_single_use (*arg0) || !has_single_use (*arg1)))
+    return;
+
+  gimple *arg0_def_stmt = SSA_NAME_DEF_STMT (*arg0);
+  if (!gimple_extract_op (arg0_def_stmt, &arg0_op))
+    return;
+
+  /* At this point there should be no ssa names occuring in abnormals.  */
+  gcc_assert (!arg0_op.operands_occurs_in_abnormal_phi ());
+
+  gimple *arg1_def_stmt = SSA_NAME_DEF_STMT (*arg1);
+  if (!gimple_extract_op (arg1_def_stmt, &arg1_op))
+    return;
+
+  /* At this point there should be no ssa names occuring in abnormals.  */
+  gcc_assert (!arg1_op.operands_occurs_in_abnormal_phi ());
+
+  /* No factoring can happen if the codes are different
+     or the number operands.  */
+  if (arg1_op.code != arg0_op.code
+      || arg1_op.num_ops != arg0_op.num_ops)
+    return;
+
+  tree new_arg0, new_arg1;
+  int opnum = find_different_opnum (arg0_op, arg1_op, &new_arg0, &new_arg1);
+  if (opnum == -1)
+    return;
+
+  if (!types_compatible_p (TREE_TYPE (new_arg0), TREE_TYPE (new_arg1)))
+    return;
+  tree new_res = make_ssa_name (TREE_TYPE (new_arg0), NULL);
+
+  /* Create the operation stmt if possible and insert it.  */
+
+  gimple_match_op new_op = arg0_op;
+  new_op.ops[opnum] = new_res;
+  gimple_seq seq = NULL;
+  tree result = *res;
+  result = maybe_push_res_to_seq (&new_op, &seq, result);
+
+  /* If we can't create the new statement, release the temp name
+     and return back.  */
+  if (!result)
+    {
+      release_ssa_name (new_res);
+      return;
+    }
+  gsi_insert_seq_before (gsi, seq, GSI_CONTINUE_LINKING);
+
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, "PHI ");
+      print_generic_expr (dump_file, gimple_phi_result (phi));
+      fprintf (dump_file,
+	       " changed to factor operation out from COND_EXPR.\n");
+      fprintf (dump_file, "New stmt with OPERATION that defines ");
+      print_generic_expr (dump_file, result);
+      fprintf (dump_file, ".\n");
+    }
+
+  /* Remove the old operation(s) that has single use.  */
+  gimple_stmt_iterator gsi_for_def;
+
+  gsi_for_def = gsi_for_stmt (arg0_def_stmt);
+  gsi_remove (&gsi_for_def, true);
+  release_defs (arg0_def_stmt);
+  gsi_for_def = gsi_for_stmt (arg1_def_stmt);
+  gsi_remove (&gsi_for_def, true);
+  release_defs (arg1_def_stmt);
+
+  /* Update the arguments and try again.  */
+  *arg0 = new_arg0;
+  *arg1 = new_arg1;
+  *res = new_res;
+
+  /* Update the phi node too. */
+  gimple_phi_set_result (phi, new_res);
+  gimple_phi_arg (phi, 0)->def = new_arg0;
+  gimple_phi_arg (phi, 0)->def = new_arg1;
+  update_stmt (phi);
+
+  repeated = true;
+  goto again;
+}
+
 /* Create the smallest nested conditional possible.  On pre-order we record
    which conditionals are live, and on post-order rewrite the chain by removing
    already active conditions.
@@ -2289,6 +2481,11 @@ predicate_scalar_phi (gphi *phi, gimple_stmt_iterator *gsi, bool loop_versioned)
 	  arg0 = gimple_phi_arg_def (phi, 0);
 	  arg1 = gimple_phi_arg_def (phi, 1);
 	}
+
+      /* Factor out operand if possible. This can only be done easily
+	 for PHI with 2 elements.  */
+      factor_out_operators (&res, gsi, &arg0, &arg1, phi);
+
       if (is_cond_scalar_reduction (phi, &reduc, arg0, arg1,
 				    &op0, &op1, false, &has_nop,
 				    &nop_reduc))
@@ -3004,12 +3201,10 @@ combine_blocks (class loop *loop, bool loop_versioned)
 
   /* Reset flow-sensitive info before predicating stmts or PHIs we
      might fold.  */
-  bool *predicated = XNEWVEC (bool, orig_loop_num_nodes);
   for (i = 0; i < orig_loop_num_nodes; i++)
     {
       bb = ifc_bbs[i];
-      predicated[i] = is_predicated (bb);
-      if (predicated[i])
+      if (is_predicated (bb))
 	{
 	  for (auto gsi = gsi_start_phis (bb);
 	       !gsi_end_p (gsi); gsi_next (&gsi))
@@ -3211,7 +3406,6 @@ combine_blocks (class loop *loop, bool loop_versioned)
 
   free (ifc_bbs);
   ifc_bbs = NULL;
-  free (predicated);
 }
 
 /* Version LOOP before if-converting it; the original loop
diff --git a/gcc/tree-inline.cc b/gcc/tree-inline.cc
index 7e0ac69..08e6421 100644
--- a/gcc/tree-inline.cc
+++ b/gcc/tree-inline.cc
@@ -1460,10 +1460,7 @@ copy_tree_body_r (tree *tp, int *walk_subtrees, void *data)
 		   || OMP_CLAUSE_CODE (*tp) == OMP_CLAUSE_DEPEND))
 	{
 	  tree t = OMP_CLAUSE_DECL (*tp);
-	  if (t
-	      && TREE_CODE (t) == TREE_LIST
-	      && TREE_PURPOSE (t)
-	      && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	  if (t && OMP_ITERATOR_DECL_P (t))
 	    {
 	      *walk_subtrees = 0;
 	      OMP_CLAUSE_DECL (*tp) = copy_node (t);
@@ -2888,11 +2885,9 @@ initialize_cfun (tree new_fndecl, tree callee_fndecl, profile_count count)
   profile_count::adjust_for_ipa_scaling (&num, &den);
 
   ENTRY_BLOCK_PTR_FOR_FN (cfun)->count =
-    ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (count,
-				ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count);
+    ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (num, den);
   EXIT_BLOCK_PTR_FOR_FN (cfun)->count =
-    EXIT_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (count,
-				ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count);
+    EXIT_BLOCK_PTR_FOR_FN (src_cfun)->count.apply_scale (num, den);
   if (src_cfun->eh)
     init_eh_for_function ();
 
diff --git a/gcc/tree-logical-location.cc b/gcc/tree-logical-location.cc
index 1d0cc58..19bccd1 100644
--- a/gcc/tree-logical-location.cc
+++ b/gcc/tree-logical-location.cc
@@ -27,6 +27,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "langhooks.h"
 #include "intl.h"
 
+using namespace diagnostics::logical_locations;
+
 static void
 assert_valid_tree (const_tree node)
 {
@@ -35,7 +37,8 @@ assert_valid_tree (const_tree node)
   gcc_assert (TREE_CODE (node) != TRANSLATION_UNIT_DECL);
 }
 
-/* class tree_logical_location_manager : public logical_location_manager.  */
+/* class tree_logical_location_manager
+   : public diagnostics::logical_locations::manager.  */
 
 const char *
 tree_logical_location_manager::get_short_name (key k) const
@@ -81,7 +84,7 @@ tree_logical_location_manager::get_internal_name (key k) const
   return NULL;
 }
 
-enum logical_location_kind
+enum kind
 tree_logical_location_manager::get_kind (key k) const
 {
   tree node = tree_from_key (k);
@@ -90,18 +93,18 @@ tree_logical_location_manager::get_kind (key k) const
   switch (TREE_CODE (node))
     {
     default:
-      return logical_location_kind::unknown;
+      return kind::unknown;
     case FUNCTION_DECL:
-      return logical_location_kind::function;
+      return kind::function;
     case PARM_DECL:
-      return logical_location_kind::parameter;
+      return kind::parameter;
     case VAR_DECL:
-      return logical_location_kind::variable;
+      return kind::variable;
     case NAMESPACE_DECL:
-      return logical_location_kind::namespace_;
+      return kind::namespace_;
 
     case RECORD_TYPE:
-      return logical_location_kind::type;
+      return kind::type;
     }
 }
 
@@ -123,7 +126,7 @@ tree_logical_location_manager::get_name_for_path_output (key k) const
   return label_text ();
 }
 
-logical_location
+key
 tree_logical_location_manager::get_parent (key k) const
 {
   tree node = tree_from_key (k);
@@ -132,16 +135,16 @@ tree_logical_location_manager::get_parent (key k) const
   if (DECL_P (node))
     {
       if (!DECL_CONTEXT (node))
-	return logical_location ();
+	return key ();
       if (TREE_CODE (DECL_CONTEXT (node)) == TRANSLATION_UNIT_DECL)
-	return logical_location ();
+	return key ();
       return key_from_tree (DECL_CONTEXT (node));
     }
   else if (TYPE_P (node))
     {
       if (!TYPE_CONTEXT (node))
-	return logical_location ();
+	return key ();
       return key_from_tree (TYPE_CONTEXT (node));
     }
-  return logical_location ();
+  return key ();
 }
diff --git a/gcc/tree-logical-location.h b/gcc/tree-logical-location.h
index 0d57888..2a7de37 100644
--- a/gcc/tree-logical-location.h
+++ b/gcc/tree-logical-location.h
@@ -21,29 +21,33 @@ along with GCC; see the file COPYING3.  If not see
 #ifndef GCC_TREE_LOGICAL_LOCATION_H
 #define GCC_TREE_LOGICAL_LOCATION_H
 
-#include "logical-location.h"
+#include "diagnostics/logical-locations.h"
 
-/* A subclass of logical_location_manager in which the keys are
-   "tree".
+/* A subclass of diagnostics::logical_locations::manager in which the
+   keys are "tree".
    Note that there is no integration with the garbage collector,
-   and so logical_location instances can only be short-lived.  */
-class tree_logical_location_manager : public logical_location_manager
+   and so key instances can only be short-lived.  */
+class tree_logical_location_manager
+  : public diagnostics::logical_locations::manager
 {
 public:
+  using key = diagnostics::logical_locations::key;
+  using kind = diagnostics::logical_locations::kind;
+
   const char *get_short_name (key) const final override;
   const char *get_name_with_scope (key) const final override;
   const char *get_internal_name (key) const final override;
-  enum logical_location_kind get_kind (key) const final override;
+  kind get_kind (key) const final override;
   label_text get_name_for_path_output (key) const final override;
   key get_parent (key) const final override;
 
-  static tree tree_from_key (logical_location k)
+  static tree tree_from_key (key k)
   {
     return const_cast<tree> (k.cast_to<const_tree> ());
   }
-  static logical_location key_from_tree (tree node)
+  static key key_from_tree (tree node)
   {
-    return logical_location::from_ptr (node);
+    return key::from_ptr (node);
   }
 };
 
diff --git a/gcc/tree-object-size.cc b/gcc/tree-object-size.cc
index a551f2b..8545eff 100644
--- a/gcc/tree-object-size.cc
+++ b/gcc/tree-object-size.cc
@@ -851,21 +851,17 @@ addr_object_size (struct object_size_info *osi, const_tree ptr,
 
 /* Compute __builtin_object_size for a CALL to .ACCESS_WITH_SIZE,
    OBJECT_SIZE_TYPE is the second argument from __builtin_object_size.
-   The 2nd, 3rd, and the 4th parameters of the call determine the size of
+
+   The 2nd, 3rd, and 4th parameters of the call determine the size of
    the CALL:
 
    2nd argument REF_TO_SIZE: The reference to the size of the object,
-   3rd argument CLASS_OF_SIZE: The size referenced by the REF_TO_SIZE represents
-     0: the number of bytes;
-     1: the number of the elements of the object type;
-   4th argument TYPE_OF_SIZE: A constant 0 with its TYPE being the same as the TYPE
-    of the object referenced by REF_TO_SIZE
-   6th argument: A constant 0 with the pointer TYPE to the original flexible
-     array type or pointer field type.
-
-   The size of the element can be retrived from the TYPE of the 6th argument
-   of the call, which is the pointer to the original flexible array type or
-   the type of the original pointer field.  */
+   3rd argument TYPE_OF_SIZE + ACCESS_MODE: An integer constant with a pointer
+     TYPE.
+     The pointee TYPE of this pointer TYPE is the TYPE of the object referenced
+	by REF_TO_SIZE.
+
+   4th argument:  The TYPE_SIZE_UNIT of the element TYPE of the array.  */
 
 static tree
 access_with_size_object_size (const gcall *call, int object_size_type)
@@ -873,17 +869,14 @@ access_with_size_object_size (const gcall *call, int object_size_type)
   /* If not for dynamic object size, return.  */
   if ((object_size_type & OST_DYNAMIC) == 0)
     return size_unknown (object_size_type);
-
   gcc_assert (gimple_call_internal_p (call, IFN_ACCESS_WITH_SIZE));
-  /* The type of the 6th argument type is the pointer TYPE to the original
-     flexible array type or to the original pointer type.  */
-  tree pointer_to_array_type = TREE_TYPE (gimple_call_arg (call, 5));
-  gcc_assert (POINTER_TYPE_P (pointer_to_array_type));
-  tree element_type = TREE_TYPE (TREE_TYPE (pointer_to_array_type));
-  tree element_size = TYPE_SIZE_UNIT (element_type);
+
   tree ref_to_size = gimple_call_arg (call, 1);
-  unsigned int class_of_size = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
-  tree type = TREE_TYPE (gimple_call_arg (call, 3));
+  tree type = TREE_TYPE (TREE_TYPE (gimple_call_arg (call, 2)));
+
+  /* The 4th argument is the TYPE_SIZE_UNIT for the element of the original
+     flexible array.  */
+  tree element_size = gimple_call_arg (call, 3);
 
   tree size = fold_build2 (MEM_REF, type, ref_to_size,
 			   build_int_cst (ptr_type_node, 0));
@@ -897,12 +890,9 @@ access_with_size_object_size (const gcall *call, int object_size_type)
 			build_zero_cst (type), size);
   }
 
-  if (class_of_size == 1)
-    size = size_binop (MULT_EXPR,
-		       fold_convert (sizetype, size),
-		       fold_convert (sizetype, element_size));
-  else
-    size = fold_convert (sizetype, size);
+  size = size_binop (MULT_EXPR,
+		     fold_convert (sizetype, size),
+		     fold_convert (sizetype, element_size));
 
   if (!todo)
     todo = TODO_update_ssa_only_virtuals;
@@ -1944,17 +1934,6 @@ collect_object_sizes_for (struct object_size_info *osi, tree var)
             if (TREE_CODE (rhs) == SSA_NAME
                 && POINTER_TYPE_P (TREE_TYPE (rhs)))
 	      reexamine = merge_object_sizes (osi, var, rhs);
-	    /* Handle the following stmt #2 to propagate the size from the
-	       stmt #1 to #3:
-		1  _1 = .ACCESS_WITH_SIZE (_3, _4, 1, 0, -1, 0B);
-		2  _5 = *_1;
-		3  _6 = __builtin_dynamic_object_size (_5, 1);
-	     */
-	    else if (TREE_CODE (rhs) == MEM_REF
-		     && POINTER_TYPE_P (TREE_TYPE (rhs))
-		     && TREE_CODE (TREE_OPERAND (rhs, 0)) == SSA_NAME
-		     && integer_zerop (TREE_OPERAND (rhs, 1)))
-	      reexamine = merge_object_sizes (osi, var, TREE_OPERAND (rhs, 0));
             else
               expr_object_size (osi, var, rhs);
           }
diff --git a/gcc/tree-pretty-print.cc b/gcc/tree-pretty-print.cc
index fadafd6..50d0851 100644
--- a/gcc/tree-pretty-print.cc
+++ b/gcc/tree-pretty-print.cc
@@ -902,9 +902,7 @@ dump_omp_clause (pretty_printer *pp, tree clause, int spc, dump_flags_t flags)
       pp_string (pp, "affinity(");
       {
 	tree t = OMP_CLAUSE_DECL (clause);
-	if (TREE_CODE (t) == TREE_LIST
-	    && TREE_PURPOSE (t)
-	    && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	if (OMP_ITERATOR_DECL_P (t))
 	  {
 	    dump_omp_iterators (pp, TREE_PURPOSE (t), spc, flags);
 	    pp_colon (pp);
@@ -944,9 +942,7 @@ dump_omp_clause (pretty_printer *pp, tree clause, int spc, dump_flags_t flags)
 	}
       {
 	tree t = OMP_CLAUSE_DECL (clause);
-	if (TREE_CODE (t) == TREE_LIST
-	    && TREE_PURPOSE (t)
-	    && TREE_CODE (TREE_PURPOSE (t)) == TREE_VEC)
+	if (OMP_ITERATOR_DECL_P (t))
 	  {
 	    dump_omp_iterators (pp, TREE_PURPOSE (t), spc, flags);
 	    pp_colon (pp);
diff --git a/gcc/tree-scalar-evolution.cc b/gcc/tree-scalar-evolution.cc
index 43311e5..413ca49 100644
--- a/gcc/tree-scalar-evolution.cc
+++ b/gcc/tree-scalar-evolution.cc
@@ -3088,7 +3088,7 @@ iv_can_overflow_p (class loop *loop, tree type, tree base, tree step)
   type_max = wi::max_value (type);
 
   /* Just sanity check that we don't see values out of the range of the type.
-     In this case the arithmetics bellow would overflow.  */
+     In this case the arithmetics below would overflow.  */
   gcc_checking_assert (wi::ge_p (base_min, type_min, sgn)
 		       && wi::le_p (base_max, type_max, sgn));
 
diff --git a/gcc/tree-sra.cc b/gcc/tree-sra.cc
index 23236fc..032f277 100644
--- a/gcc/tree-sra.cc
+++ b/gcc/tree-sra.cc
@@ -2504,6 +2504,12 @@ sort_and_splice_var_accesses (tree var)
 		}
 	      unscalarizable_region = true;
 	    }
+	  /* If there the same place is accessed with two incompatible
+	     aggregate types, trying to base total scalarization on either of
+	     them can be wrong.  */
+	  if (!first_scalar && !types_compatible_p (access->type, ac2->type))
+	    bitmap_set_bit (cannot_scalarize_away_bitmap,
+			    DECL_UID (access->base));
 
 	  if (grp_same_access_path
 	      && (!ac2->grp_same_access_path
@@ -2889,7 +2895,10 @@ analyze_access_subtree (struct access *root, struct access *parent,
 
   for (child = root->first_child; child; child = child->next_sibling)
     {
-      hole |= covered_to < child->offset;
+      if (totally)
+	covered_to = child->offset;
+      else
+	hole |= covered_to < child->offset;
       sth_created |= analyze_access_subtree (child, root,
 					     allow_replacements && !scalar
 					     && !root->grp_partial_lhs,
@@ -2900,6 +2909,8 @@ analyze_access_subtree (struct access *root, struct access *parent,
 	covered_to += child->size;
       else
 	hole = true;
+      if (totally && !hole)
+	covered_to = limit;
     }
 
   if (allow_replacements && scalar && !root->first_child
@@ -2972,7 +2983,7 @@ analyze_access_subtree (struct access *root, struct access *parent,
 	root->grp_total_scalarization = 0;
     }
 
-  if (!hole || totally)
+  if (!hole)
     root->grp_covered = 1;
   else if (root->grp_write || comes_initialized_p (root->base))
     root->grp_unscalarized_data = 1; /* not covered and written to */
diff --git a/gcc/tree-ssa-alias.cc b/gcc/tree-ssa-alias.cc
index 9dd1780..9b028e0 100644
--- a/gcc/tree-ssa-alias.cc
+++ b/gcc/tree-ssa-alias.cc
@@ -901,7 +901,9 @@ ao_ref_init_from_ptr_and_range (ao_ref *ref, tree ptr,
   if (TREE_CODE (ptr) == ADDR_EXPR)
     {
       ref->base = get_addr_base_and_unit_offset (TREE_OPERAND (ptr, 0), &t);
-      if (ref->base)
+      if (ref->base
+	  && coeffs_in_range_p (t, -HOST_WIDE_INT_MAX / BITS_PER_UNIT,
+				HOST_WIDE_INT_MAX / BITS_PER_UNIT))
 	ref->offset = BITS_PER_UNIT * t;
       else
 	{
@@ -4376,7 +4378,7 @@ ao_compare::compare_ao_refs (ao_ref *ref1, ao_ref *ref2,
       i++;
     }
 
-  /* For variable accesses we can not rely on offset match bellow.
+  /* For variable accesses we can not rely on offset match below.
      We know that paths are struturally same, so only check that
      starts of TBAA paths did not diverge.  */
   if (!known_eq (ref1->size, ref1->max_size)
diff --git a/gcc/tree-ssa-dse.cc b/gcc/tree-ssa-dse.cc
index 5ac4280..51a5723 100644
--- a/gcc/tree-ssa-dse.cc
+++ b/gcc/tree-ssa-dse.cc
@@ -181,10 +181,10 @@ initialize_ao_ref_for_dse (gimple *stmt, ao_ref *write, bool may_def_ok = false)
 	       can provide a may-def variant.  */
 	    if (may_def_ok)
 	      {
-		ao_ref_init_from_ptr_and_size (
-		  write, gimple_call_arg (stmt, 0),
-		  TYPE_SIZE_UNIT (
-		    TREE_TYPE (gimple_call_arg (stmt, stored_value_index))));
+		ao_ref_init_from_ptr_and_range (
+		  write, gimple_call_arg (stmt, 0), true, 0, -1,
+		  tree_to_poly_int64 (TYPE_SIZE (
+		    TREE_TYPE (gimple_call_arg (stmt, stored_value_index)))));
 		return true;
 	      }
 	    break;
diff --git a/gcc/tree-ssa-live.cc b/gcc/tree-ssa-live.cc
index 5b8bfd0..5e08913 100644
--- a/gcc/tree-ssa-live.cc
+++ b/gcc/tree-ssa-live.cc
@@ -702,7 +702,10 @@ dump_scope_block (FILE *file, int indent, tree scope, dump_flags_t flags)
   if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (scope)) != UNKNOWN_LOCATION)
     {
       expanded_location s = expand_location (BLOCK_SOURCE_LOCATION (scope));
-      fprintf (file, " %s:%i", s.file, s.line);
+      fprintf (file, " %s:%i:%i", s.file, s.line, s.column);
+      if (has_discriminator (BLOCK_SOURCE_LOCATION (scope)))
+	fprintf (file, " discrim %i",
+		 get_discriminator_from_loc (BLOCK_SOURCE_LOCATION (scope)));
     }
   if (BLOCK_ABSTRACT_ORIGIN (scope))
     {
diff --git a/gcc/tree-ssa-loop-ivopts.cc b/gcc/tree-ssa-loop-ivopts.cc
index 544a946..879668c 100644
--- a/gcc/tree-ssa-loop-ivopts.cc
+++ b/gcc/tree-ssa-loop-ivopts.cc
@@ -147,7 +147,7 @@ along with GCC; see the file COPYING3.  If not see
    The average trip count is computed from profile data if it
    exists. */
 
-static inline HOST_WIDE_INT
+static inline unsigned HOST_WIDE_INT
 avg_loop_niter (class loop *loop)
 {
   HOST_WIDE_INT niter = estimated_stmt_executions_int (loop);
@@ -4213,7 +4213,9 @@ adjust_setup_cost (struct ivopts_data *data, int64_t cost,
     return cost;
   else if (optimize_loop_for_speed_p (data->current_loop))
     {
-      int64_t niters = (int64_t) avg_loop_niter (data->current_loop);
+      uint64_t niters = avg_loop_niter (data->current_loop);
+      if (niters > (uint64_t) cost)
+	return (round_up_p && cost != 0) ? 1 : 0;
       return (cost + (round_up_p ? niters - 1 : 0)) / niters;
     }
   else
@@ -7277,7 +7279,7 @@ create_new_ivs (struct ivopts_data *data, class iv_ca *set)
       if (data->loop_loc != UNKNOWN_LOCATION)
 	fprintf (dump_file, " at %s:%d", LOCATION_FILE (data->loop_loc),
 		 LOCATION_LINE (data->loop_loc));
-      fprintf (dump_file, ", " HOST_WIDE_INT_PRINT_DEC " avg niters",
+      fprintf (dump_file, ", " HOST_WIDE_INT_PRINT_UNSIGNED " avg niters",
 	       avg_loop_niter (data->current_loop));
       fprintf (dump_file, ", %lu IVs:\n", bitmap_count_bits (set->cands));
       EXECUTE_IF_SET_IN_BITMAP (set->cands, 0, i, bi)
diff --git a/gcc/tree-ssa-loop-niter.cc b/gcc/tree-ssa-loop-niter.cc
index 9ce8813..6e13086 100644
--- a/gcc/tree-ssa-loop-niter.cc
+++ b/gcc/tree-ssa-loop-niter.cc
@@ -4701,7 +4701,7 @@ maybe_lower_iteration_bound (class loop *loop)
 
      TODO: Due to the way record_estimate choose estimates to store, the bounds
      will be always nb_iterations_upper_bound-1.  We can change this to record
-     also statements not dominating the loop latch and update the walk bellow
+     also statements not dominating the loop latch and update the walk below
      to the shortest path algorithm.  */
   for (elt = loop->bounds; elt; elt = elt->next)
     {
diff --git a/gcc/tree-ssa-math-opts.cc b/gcc/tree-ssa-math-opts.cc
index 4cfcc42..ca98205 100644
--- a/gcc/tree-ssa-math-opts.cc
+++ b/gcc/tree-ssa-math-opts.cc
@@ -4064,6 +4064,7 @@ arith_overflow_check_p (gimple *stmt, gimple *cast_stmt, gimple *&use_stmt,
 extern bool gimple_unsigned_integer_sat_add (tree, tree*, tree (*)(tree));
 extern bool gimple_unsigned_integer_sat_sub (tree, tree*, tree (*)(tree));
 extern bool gimple_unsigned_integer_sat_trunc (tree, tree*, tree (*)(tree));
+extern bool gimple_unsigned_integer_sat_mul (tree, tree*, tree (*)(tree));
 
 extern bool gimple_signed_integer_sat_add (tree, tree*, tree (*)(tree));
 extern bool gimple_signed_integer_sat_sub (tree, tree*, tree (*)(tree));
@@ -4217,6 +4218,30 @@ match_unsigned_saturation_sub (gimple_stmt_iterator *gsi, gassign *stmt)
 }
 
 /*
+ * Try to match saturation unsigned mul.
+ *   _1 = (unsigned int) a_6(D);
+ *   _2 = (unsigned int) b_7(D);
+ *   x_8 = _1 * _2;
+ *   overflow_9 = x_8 > 255;
+ *   _3 = (unsigned char) overflow_9;
+ *   _4 = -_3;
+ *   _5 = (unsigned char) x_8;
+ *   _10 = _4 | _5;
+ *   =>
+ *   _10 = .SAT_SUB (a_6, b_7);  */
+
+static void
+match_unsigned_saturation_mul (gimple_stmt_iterator *gsi, gassign *stmt)
+{
+  tree ops[2];
+  tree lhs = gimple_assign_lhs (stmt);
+
+  if (gimple_unsigned_integer_sat_mul (lhs, ops, NULL))
+    build_saturation_binary_arith_call_and_replace (gsi, IFN_SAT_MUL, lhs,
+						    ops[0], ops[1]);
+}
+
+/*
  * Try to match saturation unsigned sub.
  *  <bb 2> [local count: 1073741824]:
  *  if (x_2(D) > y_3(D))
@@ -6469,6 +6494,7 @@ math_opts_dom_walker::after_dom_children (basic_block bb)
 	      break;
 
 	    case NOP_EXPR:
+	      match_unsigned_saturation_mul (&gsi, as_a<gassign *> (stmt));
 	      match_unsigned_saturation_trunc (&gsi, as_a<gassign *> (stmt));
 	      match_saturation_add_with_assign (&gsi, as_a<gassign *> (stmt));
 	      break;
diff --git a/gcc/tree-ssa-operands.cc b/gcc/tree-ssa-operands.cc
index 4eb9673..a5970ac 100644
--- a/gcc/tree-ssa-operands.cc
+++ b/gcc/tree-ssa-operands.cc
@@ -721,7 +721,7 @@ operands_scanner::get_asm_stmt_operands (gasm *stmt)
       constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
       oconstraints[i] = constraint;
       parse_output_constraint (&constraint, i, 0, 0, &allows_mem,
-	                       &allows_reg, &is_inout);
+			       &allows_reg, &is_inout, nullptr);
 
       /* This should have been split in gimplify_asm_expr.  */
       gcc_assert (!allows_reg || !is_inout);
@@ -740,7 +740,7 @@ operands_scanner::get_asm_stmt_operands (gasm *stmt)
       tree link = gimple_asm_input_op (stmt, i);
       constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
       parse_input_constraint (&constraint, 0, 0, noutputs, 0, oconstraints,
-	                      &allows_mem, &allows_reg);
+			      &allows_mem, &allows_reg, nullptr);
 
       /* Memory operands are addressable.  Note that STMT needs the
 	 address of this operand.  */
diff --git a/gcc/tree-ssa-pre.cc b/gcc/tree-ssa-pre.cc
index f6c531e..9933173 100644
--- a/gcc/tree-ssa-pre.cc
+++ b/gcc/tree-ssa-pre.cc
@@ -2773,16 +2773,17 @@ find_or_generate_expression (basic_block block, tree op, gimple_seq *stmts)
   bitmap exprset = value_expressions[lookfor];
   bitmap_iterator bi;
   unsigned int i;
-  EXECUTE_IF_SET_IN_BITMAP (exprset, 0, i, bi)
-    {
-      pre_expr temp = expression_for_id (i);
-      /* We cannot insert random REFERENCE expressions at arbitrary
-	 places.  We can insert NARYs which eventually re-materializes
-	 its operand values.  */
-      if (temp->kind == NARY)
-	return create_expression_by_pieces (block, temp, stmts,
-					    TREE_TYPE (op));
-    }
+  if (exprset)
+    EXECUTE_IF_SET_IN_BITMAP (exprset, 0, i, bi)
+      {
+	pre_expr temp = expression_for_id (i);
+	/* We cannot insert random REFERENCE expressions at arbitrary
+	   places.  We can insert NARYs which eventually re-materializes
+	   its operand values.  */
+	if (temp->kind == NARY)
+	  return create_expression_by_pieces (block, temp, stmts,
+					      TREE_TYPE (op));
+      }
 
   /* Defer.  */
   return NULL_TREE;
diff --git a/gcc/tree-ssa-reassoc.cc b/gcc/tree-ssa-reassoc.cc
index 3c38f3d..c140f76 100644
--- a/gcc/tree-ssa-reassoc.cc
+++ b/gcc/tree-ssa-reassoc.cc
@@ -7167,9 +7167,10 @@ reassociate_bb (basic_block bb)
 
 		  /* If the target support FMA, rank_ops_for_fma will detect if
 		     the chain has fmas and rearrange the ops if so.  */
-		  if (direct_internal_fn_supported_p (IFN_FMA,
-						      TREE_TYPE (lhs),
-						      opt_type)
+		  if (!reassoc_insert_powi_p
+		      && direct_internal_fn_supported_p (IFN_FMA,
+							 TREE_TYPE (lhs),
+							 opt_type)
 		      && (rhs_code == PLUS_EXPR || rhs_code == MINUS_EXPR))
 		    {
 		      mult_num = rank_ops_for_fma (&ops);
@@ -7200,7 +7201,8 @@ reassociate_bb (basic_block bb)
 			 to make sure the ones that get the double
 			 binary op are chosen wisely.  */
 		      int len = ops.length ();
-		      if (len >= 3
+		      if (!reassoc_insert_powi_p
+			  && len >= 3
 			  && (!has_fma
 			      /* width > 1 means ranking ops results in better
 				 parallelism.  Check current value to avoid
diff --git a/gcc/tree-ssa-sccvn.cc b/gcc/tree-ssa-sccvn.cc
index 9cdbf3d..3974c4d 100644
--- a/gcc/tree-ssa-sccvn.cc
+++ b/gcc/tree-ssa-sccvn.cc
@@ -717,7 +717,7 @@ vn_reference_compute_hash (const vn_reference_t vr1)
   hashval_t result;
   int i;
   vn_reference_op_t vro;
-  poly_int64 off = -1;
+  poly_offset_int off = -1;
   bool deref = false;
 
   FOR_EACH_VEC_ELT (vr1->operands, i, vro)
@@ -736,7 +736,7 @@ vn_reference_compute_hash (const vn_reference_t vr1)
 	{
 	  if (maybe_ne (off, -1)
 	      && maybe_ne (off, 0))
-	    hstate.add_poly_int (off);
+	    hstate.add_poly_hwi (off.force_shwi ());
 	  off = -1;
 	  if (deref
 	      && vro->opcode == ADDR_EXPR)
@@ -850,7 +850,7 @@ vn_reference_eq (const_vn_reference_t const vr1, const_vn_reference_t const vr2)
   j = 0;
   do
     {
-      poly_int64 off1 = 0, off2 = 0;
+      poly_offset_int off1 = 0, off2 = 0;
       vn_reference_op_t vro1, vro2;
       vn_reference_op_s tem1, tem2;
       bool deref1 = false, deref2 = false;
@@ -1219,7 +1219,7 @@ ao_ref_init_from_vn_reference (ao_ref *ref,
 		  offset = 0;
 		}
 	      else
-		offset += pop->off * BITS_PER_UNIT;
+		offset += poly_offset_int (pop->off) * BITS_PER_UNIT;
 	      op0_p = NULL;
 	      break;
 	    }
@@ -1270,7 +1270,7 @@ ao_ref_init_from_vn_reference (ao_ref *ref,
 	  if (maybe_eq (op->off, -1))
 	    max_size = -1;
 	  else
-	    offset += op->off * BITS_PER_UNIT;
+	    offset += poly_offset_int (op->off) * BITS_PER_UNIT;
 	  break;
 
 	case REALPART_EXPR:
@@ -2809,7 +2809,8 @@ vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
          we find a VN result with exactly the same value as the
 	 possible clobber.  In this case we can ignore the clobber
 	 and return the found value.  */
-      if (is_gimple_reg_type (TREE_TYPE (lhs))
+      if (!gimple_has_volatile_ops (def_stmt)
+	  && is_gimple_reg_type (TREE_TYPE (lhs))
 	  && types_compatible_p (TREE_TYPE (lhs), vr->type)
 	  && (ref->ref || data->orig_ref.ref)
 	  && !data->mask
@@ -3093,7 +3094,8 @@ vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
   else if (is_gimple_reg_type (vr->type)
 	   && gimple_assign_single_p (def_stmt)
 	   && gimple_assign_rhs_code (def_stmt) == CONSTRUCTOR
-	   && CONSTRUCTOR_NELTS (gimple_assign_rhs1 (def_stmt)) == 0)
+	   && CONSTRUCTOR_NELTS (gimple_assign_rhs1 (def_stmt)) == 0
+	   && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt)))
     {
       tree base2;
       poly_int64 offset2, size2, maxsize2;
@@ -3149,6 +3151,7 @@ vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
 	   && !reverse_storage_order_for_component_p (vr->operands)
 	   && !contains_storage_order_barrier_p (vr->operands)
 	   && gimple_assign_single_p (def_stmt)
+	   && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt))
 	   && CHAR_BIT == 8
 	   && BITS_PER_UNIT == 8
 	   && BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN
@@ -3307,6 +3310,7 @@ vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
 	   && !reverse_storage_order_for_component_p (vr->operands)
 	   && !contains_storage_order_barrier_p (vr->operands)
 	   && gimple_assign_single_p (def_stmt)
+	   && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt))
 	   && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME)
     {
       tree lhs = gimple_assign_lhs (def_stmt);
@@ -3518,6 +3522,7 @@ vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
      the copy kills ref.  */
   else if (data->vn_walk_kind == VN_WALKREWRITE
 	   && gimple_assign_single_p (def_stmt)
+	   && !gimple_has_volatile_ops (def_stmt)
 	   && (DECL_P (gimple_assign_rhs1 (def_stmt))
 	       || TREE_CODE (gimple_assign_rhs1 (def_stmt)) == MEM_REF
 	       || handled_component_p (gimple_assign_rhs1 (def_stmt))))
diff --git a/gcc/tree-ssa-sink.cc b/gcc/tree-ssa-sink.cc
index 959e0d5..2244e89f 100644
--- a/gcc/tree-ssa-sink.cc
+++ b/gcc/tree-ssa-sink.cc
@@ -356,37 +356,54 @@ statement_sink_location (gimple *stmt, basic_block frombb,
 
   use = NULL;
 
-  /* If stmt is a store the one and only use needs to be the VOP
-     merging PHI node.  */
+  /* If stmt is a store the one and only use needs to be a VUSE on
+     the live path.  */
   if (virtual_operand_p (DEF_FROM_PTR (def_p)))
     {
+      tree lhs = gimple_get_lhs (stmt);
+      ao_ref ref;
+      ao_ref_init (&ref, lhs);
       FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
 	{
 	  gimple *use_stmt = USE_STMT (use_p);
 
 	  /* A killing definition is not a use.  */
-	  if ((gimple_has_lhs (use_stmt)
-	       && operand_equal_p (gimple_get_lhs (stmt),
-				   gimple_get_lhs (use_stmt), 0))
-	      || stmt_kills_ref_p (use_stmt, gimple_get_lhs (stmt)))
+	  if (gimple_vdef (use_stmt)
+	      && ((gimple_has_lhs (use_stmt)
+		   && operand_equal_p (lhs,
+				       gimple_get_lhs (use_stmt), 0))
+		  || stmt_kills_ref_p (use_stmt, &ref)))
 	    {
 	      /* If use_stmt is or might be a nop assignment then USE_STMT
 	         acts as a use as well as definition.  */
 	      if (stmt != use_stmt
-		  && ref_maybe_used_by_stmt_p (use_stmt,
-					       gimple_get_lhs (stmt)))
-		return false;
+		  && ref_maybe_used_by_stmt_p (use_stmt, &ref))
+		{
+		  if (use && use != use_stmt)
+		    return false;
+		  use = use_stmt;
+		}
 	      continue;
 	    }
 
-	  if (gimple_code (use_stmt) != GIMPLE_PHI)
-	    return false;
-
-	  if (use
-	      && use != use_stmt)
-	    return false;
+	  if (is_a <gphi *> (use_stmt)
+	      || ref_maybe_used_by_stmt_p (use_stmt, &ref))
+	    {
+	      if (use && use != use_stmt)
+		return false;
+	      use = use_stmt;
+	      continue;
+	    }
 
-	  use = use_stmt;
+	  if (gimple_vdef (use_stmt))
+	    {
+	      if (stmt_may_clobber_ref_p_1 (use_stmt, &ref, false))
+		return false;
+	      /* We do not look past VDEFs, so treat them as sink location.  */
+	      if (use && use != use_stmt)
+		return false;
+	      use = use_stmt;
+	    }
 	}
       if (!use)
 	return false;
@@ -448,18 +465,26 @@ statement_sink_location (gimple *stmt, basic_block frombb,
 	  break;
 	}
       use = USE_STMT (one_use);
+    }
 
-      if (gimple_code (use) != GIMPLE_PHI)
-	{
-	  sinkbb = select_best_block (frombb, gimple_bb (use), stmt);
+  if (gimple_code (use) != GIMPLE_PHI)
+    {
+      sinkbb = select_best_block (frombb, gimple_bb (use), stmt);
 
-	  if (sinkbb == frombb)
-	    return false;
+      if (sinkbb == frombb)
+	return false;
 
-	  *togsi = gsi_after_labels (sinkbb);
+      /* The SSA update for sinking of stores cannot insert PHIs, the
+	 sink location has to lead to exit without crossing any CFG
+	 merge points to paths not dominated by the sink location.  */
+      if (gimple_vdef (stmt)
+	  && (!single_succ_p (sinkbb)
+	      || single_succ (sinkbb)->index != EXIT_BLOCK))
+	return false;
 
-	  return true;
-	}
+      *togsi = gsi_after_labels (sinkbb);
+
+      return true;
     }
 
   sinkbb = find_bb_for_arg (as_a <gphi *> (use), DEF_FROM_PTR (def_p));
diff --git a/gcc/tree-ssa-structalias.cc b/gcc/tree-ssa-structalias.cc
index deca44a..0035e50 100644
--- a/gcc/tree-ssa-structalias.cc
+++ b/gcc/tree-ssa-structalias.cc
@@ -48,6 +48,9 @@
 #include "ipa-modref.h"
 #include "attr-fnspec.h"
 
+#include "tree-ssa-structalias.h"
+#include "pta-andersen.h"
+
 /* The idea behind this analyzer is to generate set constraints from the
    program, then solve the resulting constraints in order to generate the
    points-to sets.
@@ -201,172 +204,357 @@
 
    And probably more.  */
 
-static bool use_field_sensitive = true;
-static int in_ipa_mode = 0;
-
-/* Used for predecessor bitmaps. */
-static bitmap_obstack predbitmap_obstack;
+namespace pointer_analysis {
 
 /* Used for points-to sets.  */
-static bitmap_obstack pta_obstack;
+bitmap_obstack pta_obstack;
 
-/* Used for oldsolution members of variables. */
-static bitmap_obstack oldpta_obstack;
+/* Used for oldsolution members of variables.  */
+bitmap_obstack oldpta_obstack;
 
-/* Used for per-solver-iteration bitmaps.  */
-static bitmap_obstack iteration_obstack;
+/* Table of variable info structures for constraint variables.
+   Indexed directly by variable info id.  */
+vec<varinfo_t> varmap;
 
-static unsigned int create_variable_info_for (tree, const char *, bool);
-typedef struct constraint_graph *constraint_graph_t;
-static void unify_nodes (constraint_graph_t, unsigned int, unsigned int, bool);
+/* List of constraints that we use to build the constraint graph from.  */
+vec<constraint_t> constraints;
+
+/* Map from trees to variable infos.  */
+static hash_map<tree, varinfo_t> *vi_for_tree;
 
-struct constraint;
-typedef struct constraint *constraint_t;
+/* The representative variable for a variable.  The points-to solution for a
+   var can be found in its rep.  Trivially, a var can be its own rep.
 
+   The solver provides this array once it is done solving.  */
+unsigned int *var_rep;
 
-#define EXECUTE_IF_IN_NONNULL_BITMAP(a, b, c, d)	\
-  if (a)						\
-    EXECUTE_IF_SET_IN_BITMAP (a, b, c, d)
+struct constraint_stats stats;
 
-static struct constraint_stats
+/* Find the first varinfo in the same variable as START that overlaps with
+   OFFSET.  Return NULL if we can't find one.  */
+
+varinfo_t
+first_vi_for_offset (varinfo_t start, unsigned HOST_WIDE_INT offset)
 {
-  unsigned int total_vars;
-  unsigned int nonpointer_vars;
-  unsigned int unified_vars_static;
-  unsigned int unified_vars_dynamic;
-  unsigned int iterations;
-  unsigned int num_edges;
-  unsigned int num_implicit_edges;
-  unsigned int num_avoided_edges;
-  unsigned int points_to_sets_created;
-} stats;
-
-struct variable_info
+  /* If the offset is outside of the variable, bail out.  */
+  if (offset >= start->fullsize)
+    return NULL;
+
+  /* If we cannot reach offset from start, lookup the first field
+     and start from there.  */
+  if (start->offset > offset)
+    start = get_varinfo (start->head);
+
+  while (start)
+    {
+      /* We may not find a variable in the field list with the actual
+	 offset when we have glommed a structure to a variable.
+	 In that case, however, offset should still be within the size
+	 of the variable.  */
+      if (offset >= start->offset
+	  && (offset - start->offset) < start->size)
+	return start;
+
+      start = vi_next (start);
+    }
+
+  return NULL;
+}
+
+/* Find the first varinfo in the same variable as START that overlaps with
+   OFFSET.  If there is no such varinfo the varinfo directly preceding
+   OFFSET is returned.  */
+
+varinfo_t
+first_or_preceding_vi_for_offset (varinfo_t start,
+				  unsigned HOST_WIDE_INT offset)
 {
-  /* ID of this variable  */
-  unsigned int id;
+  /* If we cannot reach offset from start, lookup the first field
+     and start from there.  */
+  if (start->offset > offset)
+    start = get_varinfo (start->head);
 
-  /* True if this is a variable created by the constraint analysis, such as
-     heap variables and constraints we had to break up.  */
-  unsigned int is_artificial_var : 1;
+  /* We may not find a variable in the field list with the actual
+     offset when we have glommed a structure to a variable.
+     In that case, however, offset should still be within the size
+     of the variable.
+     If we got beyond the offset we look for return the field
+     directly preceding offset which may be the last field.  */
+  while (start->next
+	 && offset >= start->offset
+	 && !((offset - start->offset) < start->size))
+    start = vi_next (start);
 
-  /* True if this is a special variable whose solution set should not be
-     changed.  */
-  unsigned int is_special_var : 1;
+  return start;
+}
 
-  /* True for variables whose size is not known or variable.  */
-  unsigned int is_unknown_size_var : 1;
+/* Print out constraint C to FILE.  */
 
-  /* True for (sub-)fields that represent a whole variable.  */
-  unsigned int is_full_var : 1;
+void
+dump_constraint (FILE *file, constraint_t c)
+{
+  if (c->lhs.type == ADDRESSOF)
+    fprintf (file, "&");
+  else if (c->lhs.type == DEREF)
+    fprintf (file, "*");
+  if (dump_file)
+    fprintf (file, "%s", get_varinfo (c->lhs.var)->name);
+  else
+    fprintf (file, "V%d", c->lhs.var);
+  if (c->lhs.offset == UNKNOWN_OFFSET)
+    fprintf (file, " + UNKNOWN");
+  else if (c->lhs.offset != 0)
+    fprintf (file, " + " HOST_WIDE_INT_PRINT_DEC, c->lhs.offset);
+  fprintf (file, " = ");
+  if (c->rhs.type == ADDRESSOF)
+    fprintf (file, "&");
+  else if (c->rhs.type == DEREF)
+    fprintf (file, "*");
+  if (dump_file)
+    fprintf (file, "%s", get_varinfo (c->rhs.var)->name);
+  else
+    fprintf (file, "V%d", c->rhs.var);
+  if (c->rhs.offset == UNKNOWN_OFFSET)
+    fprintf (file, " + UNKNOWN");
+  else if (c->rhs.offset != 0)
+    fprintf (file, " + " HOST_WIDE_INT_PRINT_DEC, c->rhs.offset);
+}
 
-  /* True if this is a heap variable.  */
-  unsigned int is_heap_var : 1;
+/* Print out constraint C to stderr.  */
 
-  /* True if this is a register variable.  */
-  unsigned int is_reg_var : 1;
+DEBUG_FUNCTION void
+debug_constraint (constraint_t c)
+{
+  dump_constraint (stderr, c);
+  fprintf (stderr, "\n");
+}
 
-  /* True if this field may contain pointers.  */
-  unsigned int may_have_pointers : 1;
+/* Print out all constraints to FILE.  */
 
-  /* True if this field has only restrict qualified pointers.  */
-  unsigned int only_restrict_pointers : 1;
+void
+dump_constraints (FILE *file, int from)
+{
+  int i;
+  constraint_t c;
+  for (i = from; constraints.iterate (i, &c); i++)
+    if (c)
+      {
+	dump_constraint (file, c);
+	fprintf (file, "\n");
+      }
+}
 
-  /* True if this represents a heap var created for a restrict qualified
-     pointer.  */
-  unsigned int is_restrict_var : 1;
+/* Print out all constraints to stderr.  */
 
-  /* True if this represents a global variable.  */
-  unsigned int is_global_var : 1;
+DEBUG_FUNCTION void
+debug_constraints (void)
+{
+  dump_constraints (stderr, 0);
+}
 
-  /* True if this represents a module escape point for IPA analysis.  */
-  unsigned int is_ipa_escape_point : 1;
+/* Print out the points-to solution for VAR to FILE.  */
 
-  /* True if this represents a IPA function info.  */
-  unsigned int is_fn_info : 1;
+void
+dump_solution_for_var (FILE *file, unsigned int var)
+{
+  varinfo_t vi = get_varinfo (var);
+  unsigned int i;
+  bitmap_iterator bi;
 
-  /* True if this appears as RHS in a ADDRESSOF constraint.  */
-  unsigned int address_taken : 1;
+  /* Dump the solution for unified vars anyway, this avoids difficulties
+     in scanning dumps in the testsuite.  */
+  fprintf (file, "%s = { ", vi->name);
+  vi = get_varinfo (var_rep[var]);
+  EXECUTE_IF_SET_IN_BITMAP (vi->solution, 0, i, bi)
+    fprintf (file, "%s ", get_varinfo (i)->name);
+  fprintf (file, "}");
 
-  /* ???  Store somewhere better.  */
-  unsigned short ruid;
+  /* But note when the variable was unified.  */
+  if (vi->id != var)
+    fprintf (file, " same as %s", vi->name);
 
-  /* The ID of the variable for the next field in this structure
-     or zero for the last field in this structure.  */
-  unsigned next;
+  fprintf (file, "\n");
+}
 
-  /* The ID of the variable for the first field in this structure.  */
-  unsigned head;
+/* Print the points-to solution for VAR to stderr.  */
 
-  /* Offset of this variable, in bits, from the base variable  */
-  unsigned HOST_WIDE_INT offset;
+DEBUG_FUNCTION void
+debug_solution_for_var (unsigned int var)
+{
+  dump_solution_for_var (stderr, var);
+}
 
-  /* Size of the variable, in bits.  */
-  unsigned HOST_WIDE_INT size;
+/* Dump stats information to OUTFILE.  */
 
-  /* Full size of the base variable, in bits.  */
-  unsigned HOST_WIDE_INT fullsize;
+void
+dump_sa_stats (FILE *outfile)
+{
+  fprintf (outfile, "Points-to Stats:\n");
+  fprintf (outfile, "Total vars:               %d\n", stats.total_vars);
+  fprintf (outfile, "Non-pointer vars:          %d\n",
+	   stats.nonpointer_vars);
+  fprintf (outfile, "Statically unified vars:  %d\n",
+	   stats.unified_vars_static);
+  fprintf (outfile, "Dynamically unified vars: %d\n",
+	   stats.unified_vars_dynamic);
+  fprintf (outfile, "Iterations:               %d\n", stats.iterations);
+  fprintf (outfile, "Number of edges:          %d\n", stats.num_edges);
+  fprintf (outfile, "Number of implicit edges: %d\n",
+	   stats.num_implicit_edges);
+  fprintf (outfile, "Number of avoided edges: %d\n",
+	   stats.num_avoided_edges);
+}
 
-  /* In IPA mode the shadow UID in case the variable needs to be duplicated in
-     the final points-to solution because it reaches its containing
-     function recursively.  Zero if none is needed.  */
-  unsigned int shadow_var_uid;
+/* Dump points-to information to OUTFILE.  */
 
-  /* Name of this variable */
-  const char *name;
+void
+dump_sa_points_to_info (FILE *outfile)
+{
+  fprintf (outfile, "\nPoints-to sets\n\n");
 
-  /* Tree that this variable is associated with.  */
-  tree decl;
+  for (unsigned i = 1; i < varmap.length (); i++)
+    {
+      varinfo_t vi = get_varinfo (i);
+      if (!vi->may_have_pointers)
+	continue;
+      dump_solution_for_var (outfile, i);
+    }
+}
 
-  /* Points-to set for this variable.  */
-  bitmap solution;
 
-  /* Old points-to set for this variable.  */
-  bitmap oldsolution;
-};
-typedef struct variable_info *varinfo_t;
+/* Debug points-to information to stderr.  */
 
-static varinfo_t first_vi_for_offset (varinfo_t, unsigned HOST_WIDE_INT);
-static varinfo_t first_or_preceding_vi_for_offset (varinfo_t,
-						   unsigned HOST_WIDE_INT);
-static varinfo_t lookup_vi_for_tree (tree);
-static inline bool type_can_have_subvars (const_tree);
-static void make_param_constraints (varinfo_t);
+DEBUG_FUNCTION void
+debug_sa_points_to_info (void)
+{
+  dump_sa_points_to_info (stderr);
+}
 
-/* Pool of variable info structures.  */
-static object_allocator<variable_info> variable_info_pool
-  ("Variable info pool");
+/* Dump varinfo VI to FILE.  */
 
-/* Map varinfo to final pt_solution.  */
-static hash_map<varinfo_t, pt_solution *> *final_solutions;
-struct obstack final_solutions_obstack;
+void
+dump_varinfo (FILE *file, varinfo_t vi)
+{
+  if (vi == NULL)
+    return;
 
-/* Table of variable info structures for constraint variables.
-   Indexed directly by variable info id.  */
-static vec<varinfo_t> varmap;
+  fprintf (file, "%u: %s\n", vi->id, vi->name);
 
-/* Return the varmap element N */
+  const char *sep = " ";
+  if (vi->is_artificial_var)
+    fprintf (file, "%sartificial", sep);
+  if (vi->is_special_var)
+    fprintf (file, "%sspecial", sep);
+  if (vi->is_unknown_size_var)
+    fprintf (file, "%sunknown-size", sep);
+  if (vi->is_full_var)
+    fprintf (file, "%sfull", sep);
+  if (vi->is_heap_var)
+    fprintf (file, "%sheap", sep);
+  if (vi->may_have_pointers)
+    fprintf (file, "%smay-have-pointers", sep);
+  if (vi->only_restrict_pointers)
+    fprintf (file, "%sonly-restrict-pointers", sep);
+  if (vi->is_restrict_var)
+    fprintf (file, "%sis-restrict-var", sep);
+  if (vi->is_global_var)
+    fprintf (file, "%sglobal", sep);
+  if (vi->is_ipa_escape_point)
+    fprintf (file, "%sipa-escape-point", sep);
+  if (vi->is_fn_info)
+    fprintf (file, "%sfn-info", sep);
+  if (vi->ruid)
+    fprintf (file, "%srestrict-uid:%u", sep, vi->ruid);
+  if (vi->next)
+    fprintf (file, "%snext:%u", sep, vi->next);
+  if (vi->head != vi->id)
+    fprintf (file, "%shead:%u", sep, vi->head);
+  if (vi->offset)
+    fprintf (file, "%soffset:" HOST_WIDE_INT_PRINT_DEC, sep, vi->offset);
+  if (vi->size != ~HOST_WIDE_INT_0U)
+    fprintf (file, "%ssize:" HOST_WIDE_INT_PRINT_DEC, sep, vi->size);
+  if (vi->fullsize != ~HOST_WIDE_INT_0U && vi->fullsize != vi->size)
+    fprintf (file, "%sfullsize:" HOST_WIDE_INT_PRINT_DEC, sep,
+	     vi->fullsize);
+  fprintf (file, "\n");
 
-static inline varinfo_t
-get_varinfo (unsigned int n)
+  if (vi->solution && !bitmap_empty_p (vi->solution))
+    {
+      bitmap_iterator bi;
+      unsigned i;
+      fprintf (file, " solution: {");
+      EXECUTE_IF_SET_IN_BITMAP (vi->solution, 0, i, bi)
+	fprintf (file, " %u", i);
+      fprintf (file, " }\n");
+    }
+
+  if (vi->oldsolution && !bitmap_empty_p (vi->oldsolution)
+      && !bitmap_equal_p (vi->solution, vi->oldsolution))
+    {
+      bitmap_iterator bi;
+      unsigned i;
+      fprintf (file, " oldsolution: {");
+      EXECUTE_IF_SET_IN_BITMAP (vi->oldsolution, 0, i, bi)
+	fprintf (file, " %u", i);
+      fprintf (file, " }\n");
+    }
+}
+
+/* Dump varinfo VI to stderr.  */
+
+DEBUG_FUNCTION void
+debug_varinfo (varinfo_t vi)
+{
+  dump_varinfo (stderr, vi);
+}
+
+/* Dump varmap to FILE.  */
+
+void
+dump_varmap (FILE *file)
 {
-  return varmap[n];
+  if (varmap.length () == 0)
+    return;
+
+  fprintf (file, "variables:\n");
+
+  for (unsigned int i = 0; i < varmap.length (); ++i)
+    {
+      varinfo_t vi = get_varinfo (i);
+      dump_varinfo (file, vi);
+    }
+
+  fprintf (file, "\n");
 }
 
-/* Return the next variable in the list of sub-variables of VI
-   or NULL if VI is the last sub-variable.  */
+/* Dump varmap to stderr.  */
 
-static inline varinfo_t
-vi_next (varinfo_t vi)
+DEBUG_FUNCTION void
+debug_varmap (void)
 {
-  return get_varinfo (vi->next);
+  dump_varmap (stderr);
 }
 
-/* Static IDs for the special variables.  Variable ID zero is unused
-   and used as terminator for the sub-variable chain.  */
-enum { nothing_id = 1, anything_id = 2, string_id = 3,
-       escaped_id = 4, nonlocal_id = 5, escaped_return_id = 6,
-       storedanything_id = 7, integer_id = 8 };
+} // namespace pointer_analysis
+
+
+using namespace pointer_analysis;
+
+static bool use_field_sensitive = true;
+static int in_ipa_mode = 0;
+
+static unsigned int create_variable_info_for (tree, const char *, bool);
+static varinfo_t lookup_vi_for_tree (tree);
+static inline bool type_can_have_subvars (const_tree);
+static void make_param_constraints (varinfo_t);
+
+/* Pool of variable info structures.  */
+static object_allocator<variable_info> variable_info_pool
+  ("Variable info pool");
+
+/* Map varinfo to final pt_solution.  */
+static hash_map<varinfo_t, pt_solution *> *final_solutions;
+static struct obstack final_solutions_obstack;
 
 /* Return a new variable info structure consisting for a variable
    named NAME, and using constraint graph node NODE.  Append it
@@ -502,166 +690,15 @@ get_call_clobber_vi (gcall *call)
 }
 
 
-enum constraint_expr_type {SCALAR, DEREF, ADDRESSOF};
-
-/* An expression that appears in a constraint.  */
-
-struct constraint_expr
-{
-  /* Constraint type.  */
-  constraint_expr_type type;
-
-  /* Variable we are referring to in the constraint.  */
-  unsigned int var;
-
-  /* Offset, in bits, of this constraint from the beginning of
-     variables it ends up referring to.
-
-     IOW, in a deref constraint, we would deref, get the result set,
-     then add OFFSET to each member.   */
-  HOST_WIDE_INT offset;
-};
-
-/* Use 0x8000... as special unknown offset.  */
-#define UNKNOWN_OFFSET HOST_WIDE_INT_MIN
-
-typedef struct constraint_expr ce_s;
 static void get_constraint_for_1 (tree, vec<ce_s> *, bool, bool);
 static void get_constraint_for (tree, vec<ce_s> *);
 static void get_constraint_for_rhs (tree, vec<ce_s> *);
 static void do_deref (vec<ce_s> *);
 
-/* Our set constraints are made up of two constraint expressions, one
-   LHS, and one RHS.
-
-   As described in the introduction, our set constraints each represent an
-   operation between set valued variables.
-*/
-struct constraint
-{
-  struct constraint_expr lhs;
-  struct constraint_expr rhs;
-};
+/* Allocator for 'constraints' vector.  */
 
-/* List of constraints that we use to build the constraint graph from.  */
-
-static vec<constraint_t> constraints;
 static object_allocator<constraint> constraint_pool ("Constraint pool");
 
-/* The constraint graph is represented as an array of bitmaps
-   containing successor nodes.  */
-
-struct constraint_graph
-{
-  /* Size of this graph, which may be different than the number of
-     nodes in the variable map.  */
-  unsigned int size;
-
-  /* Explicit successors of each node. */
-  bitmap *succs;
-
-  /* Implicit predecessors of each node (Used for variable
-     substitution). */
-  bitmap *implicit_preds;
-
-  /* Explicit predecessors of each node (Used for variable substitution).  */
-  bitmap *preds;
-
-  /* Indirect cycle representatives, or -1 if the node has no indirect
-     cycles.  */
-  int *indirect_cycles;
-
-  /* Representative node for a node.  rep[a] == a unless the node has
-     been unified. */
-  unsigned int *rep;
-
-  /* Equivalence class representative for a label.  This is used for
-     variable substitution.  */
-  int *eq_rep;
-
-  /* Pointer equivalence label for a node.  All nodes with the same
-     pointer equivalence label can be unified together at some point
-     (either during constraint optimization or after the constraint
-     graph is built).  */
-  unsigned int *pe;
-
-  /* Pointer equivalence representative for a label.  This is used to
-     handle nodes that are pointer equivalent but not location
-     equivalent.  We can unite these once the addressof constraints
-     are transformed into initial points-to sets.  */
-  int *pe_rep;
-
-  /* Pointer equivalence label for each node, used during variable
-     substitution.  */
-  unsigned int *pointer_label;
-
-  /* Location equivalence label for each node, used during location
-     equivalence finding.  */
-  unsigned int *loc_label;
-
-  /* Pointed-by set for each node, used during location equivalence
-     finding.  This is pointed-by rather than pointed-to, because it
-     is constructed using the predecessor graph.  */
-  bitmap *pointed_by;
-
-  /* Points to sets for pointer equivalence.  This is *not* the actual
-     points-to sets for nodes.  */
-  bitmap *points_to;
-
-  /* Bitmap of nodes where the bit is set if the node is a direct
-     node.  Used for variable substitution.  */
-  sbitmap direct_nodes;
-
-  /* Bitmap of nodes where the bit is set if the node is address
-     taken.  Used for variable substitution.  */
-  bitmap address_taken;
-
-  /* Vector of complex constraints for each graph node.  Complex
-     constraints are those involving dereferences or offsets that are
-     not 0.  */
-  vec<constraint_t> *complex;
-};
-
-static constraint_graph_t graph;
-
-/* During variable substitution and the offline version of indirect
-   cycle finding, we create nodes to represent dereferences and
-   address taken constraints.  These represent where these start and
-   end.  */
-#define FIRST_REF_NODE (varmap).length ()
-#define LAST_REF_NODE (FIRST_REF_NODE + (FIRST_REF_NODE - 1))
-
-/* Return the representative node for NODE, if NODE has been unioned
-   with another NODE.
-   This function performs path compression along the way to finding
-   the representative.  */
-
-static unsigned int
-find (unsigned int node)
-{
-  gcc_checking_assert (node < graph->size);
-  if (graph->rep[node] != node)
-    return graph->rep[node] = find (graph->rep[node]);
-  return node;
-}
-
-/* Union the TO and FROM nodes to the TO nodes.
-   Note that at some point in the future, we may want to do
-   union-by-rank, in which case we are going to have to return the
-   node we unified to.  */
-
-static bool
-unite (unsigned int to, unsigned int from)
-{
-  gcc_checking_assert (to < graph->size && from < graph->size);
-  if (to != from && graph->rep[from] != to)
-    {
-      graph->rep[from] = to;
-      return true;
-    }
-  return false;
-}
-
 /* Create a new constraint consisting of LHS and RHS expressions.  */
 
 static constraint_t
@@ -674,2312 +711,6 @@ new_constraint (const struct constraint_expr lhs,
   return ret;
 }
 
-/* Print out constraint C to FILE.  */
-
-static void
-dump_constraint (FILE *file, constraint_t c)
-{
-  if (c->lhs.type == ADDRESSOF)
-    fprintf (file, "&");
-  else if (c->lhs.type == DEREF)
-    fprintf (file, "*");
-  if (dump_file)
-    fprintf (file, "%s", get_varinfo (c->lhs.var)->name);
-  else
-    fprintf (file, "V%d", c->lhs.var);
-  if (c->lhs.offset == UNKNOWN_OFFSET)
-    fprintf (file, " + UNKNOWN");
-  else if (c->lhs.offset != 0)
-    fprintf (file, " + " HOST_WIDE_INT_PRINT_DEC, c->lhs.offset);
-  fprintf (file, " = ");
-  if (c->rhs.type == ADDRESSOF)
-    fprintf (file, "&");
-  else if (c->rhs.type == DEREF)
-    fprintf (file, "*");
-  if (dump_file)
-    fprintf (file, "%s", get_varinfo (c->rhs.var)->name);
-  else
-    fprintf (file, "V%d", c->rhs.var);
-  if (c->rhs.offset == UNKNOWN_OFFSET)
-    fprintf (file, " + UNKNOWN");
-  else if (c->rhs.offset != 0)
-    fprintf (file, " + " HOST_WIDE_INT_PRINT_DEC, c->rhs.offset);
-}
-
-
-void debug_constraint (constraint_t);
-void debug_constraints (void);
-void debug_constraint_graph (void);
-void debug_solution_for_var (unsigned int);
-void debug_sa_points_to_info (void);
-void debug_varinfo (varinfo_t);
-void debug_varmap (void);
-
-/* Print out constraint C to stderr.  */
-
-DEBUG_FUNCTION void
-debug_constraint (constraint_t c)
-{
-  dump_constraint (stderr, c);
-  fprintf (stderr, "\n");
-}
-
-/* Print out all constraints to FILE */
-
-static void
-dump_constraints (FILE *file, int from)
-{
-  int i;
-  constraint_t c;
-  for (i = from; constraints.iterate (i, &c); i++)
-    if (c)
-      {
-	dump_constraint (file, c);
-	fprintf (file, "\n");
-      }
-}
-
-/* Print out all constraints to stderr.  */
-
-DEBUG_FUNCTION void
-debug_constraints (void)
-{
-  dump_constraints (stderr, 0);
-}
-
-/* Print the constraint graph in dot format.  */
-
-static void
-dump_constraint_graph (FILE *file)
-{
-  unsigned int i;
-
-  /* Only print the graph if it has already been initialized:  */
-  if (!graph)
-    return;
-
-  /* Prints the header of the dot file:  */
-  fprintf (file, "strict digraph {\n");
-  fprintf (file, "  node [\n    shape = box\n  ]\n");
-  fprintf (file, "  edge [\n    fontsize = \"12\"\n  ]\n");
-  fprintf (file, "\n  // List of nodes and complex constraints in "
-	   "the constraint graph:\n");
-
-  /* The next lines print the nodes in the graph together with the
-     complex constraints attached to them.  */
-  for (i = 1; i < graph->size; i++)
-    {
-      if (i == FIRST_REF_NODE)
-	continue;
-      if (find (i) != i)
-	continue;
-      if (i < FIRST_REF_NODE)
-	fprintf (file, "\"%s\"", get_varinfo (i)->name);
-      else
-	fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
-      if (graph->complex[i].exists ())
-	{
-	  unsigned j;
-	  constraint_t c;
-	  fprintf (file, " [label=\"\\N\\n");
-	  for (j = 0; graph->complex[i].iterate (j, &c); ++j)
-	    {
-	      dump_constraint (file, c);
-	      fprintf (file, "\\l");
-	    }
-	  fprintf (file, "\"]");
-	}
-      fprintf (file, ";\n");
-    }
-
-  /* Go over the edges.  */
-  fprintf (file, "\n  // Edges in the constraint graph:\n");
-  for (i = 1; i < graph->size; i++)
-    {
-      unsigned j;
-      bitmap_iterator bi;
-      if (find (i) != i)
-	continue;
-      EXECUTE_IF_IN_NONNULL_BITMAP (graph->succs[i], 0, j, bi)
-	{
-	  unsigned to = find (j);
-	  if (i == to)
-	    continue;
-	  if (i < FIRST_REF_NODE)
-	    fprintf (file, "\"%s\"", get_varinfo (i)->name);
-	  else
-	    fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
-	  fprintf (file, " -> ");
-	  if (to < FIRST_REF_NODE)
-	    fprintf (file, "\"%s\"", get_varinfo (to)->name);
-	  else
-	    fprintf (file, "\"*%s\"", get_varinfo (to - FIRST_REF_NODE)->name);
-	  fprintf (file, ";\n");
-	}
-    }
-
-  /* Prints the tail of the dot file.  */
-  fprintf (file, "}\n");
-}
-
-/* Print out the constraint graph to stderr.  */
-
-DEBUG_FUNCTION void
-debug_constraint_graph (void)
-{
-  dump_constraint_graph (stderr);
-}
-
-/* SOLVER FUNCTIONS
-
-   The solver is a simple worklist solver, that works on the following
-   algorithm:
-
-   sbitmap changed_nodes = all zeroes;
-   changed_count = 0;
-   For each node that is not already collapsed:
-       changed_count++;
-       set bit in changed nodes
-
-   while (changed_count > 0)
-   {
-     compute topological ordering for constraint graph
-
-     find and collapse cycles in the constraint graph (updating
-     changed if necessary)
-
-     for each node (n) in the graph in topological order:
-       changed_count--;
-
-       Process each complex constraint associated with the node,
-       updating changed if necessary.
-
-       For each outgoing edge from n, propagate the solution from n to
-       the destination of the edge, updating changed as necessary.
-
-   }  */
-
-/* Return true if two constraint expressions A and B are equal.  */
-
-static bool
-constraint_expr_equal (struct constraint_expr a, struct constraint_expr b)
-{
-  return a.type == b.type && a.var == b.var && a.offset == b.offset;
-}
-
-/* Return true if constraint expression A is less than constraint expression
-   B.  This is just arbitrary, but consistent, in order to give them an
-   ordering.  */
-
-static bool
-constraint_expr_less (struct constraint_expr a, struct constraint_expr b)
-{
-  if (a.type == b.type)
-    {
-      if (a.var == b.var)
-	return a.offset < b.offset;
-      else
-	return a.var < b.var;
-    }
-  else
-    return a.type < b.type;
-}
-
-/* Return true if constraint A is less than constraint B.  This is just
-   arbitrary, but consistent, in order to give them an ordering.  */
-
-static bool
-constraint_less (const constraint_t &a, const constraint_t &b)
-{
-  if (constraint_expr_less (a->lhs, b->lhs))
-    return true;
-  else if (constraint_expr_less (b->lhs, a->lhs))
-    return false;
-  else
-    return constraint_expr_less (a->rhs, b->rhs);
-}
-
-/* Return true if two constraints A and B are equal.  */
-
-static bool
-constraint_equal (const constraint &a, const constraint &b)
-{
-  return constraint_expr_equal (a.lhs, b.lhs)
-    && constraint_expr_equal (a.rhs, b.rhs);
-}
-
-
-/* Find a constraint LOOKFOR in the sorted constraint vector VEC */
-
-static constraint_t
-constraint_vec_find (vec<constraint_t> vec,
-		     constraint &lookfor)
-{
-  unsigned int place;
-  constraint_t found;
-
-  if (!vec.exists ())
-    return NULL;
-
-  place = vec.lower_bound (&lookfor, constraint_less);
-  if (place >= vec.length ())
-    return NULL;
-  found = vec[place];
-  if (!constraint_equal (*found, lookfor))
-    return NULL;
-  return found;
-}
-
-/* Union two constraint vectors, TO and FROM.  Put the result in TO.
-   Returns true of TO set is changed.  */
-
-static bool
-constraint_set_union (vec<constraint_t> *to,
-		      vec<constraint_t> *from)
-{
-  int i;
-  constraint_t c;
-  bool any_change = false;
-
-  FOR_EACH_VEC_ELT (*from, i, c)
-    {
-      if (constraint_vec_find (*to, *c) == NULL)
-	{
-	  unsigned int place = to->lower_bound (c, constraint_less);
-	  to->safe_insert (place, c);
-          any_change = true;
-	}
-    }
-  return any_change;
-}
-
-/* Expands the solution in SET to all sub-fields of variables included.  */
-
-static bitmap
-solution_set_expand (bitmap set, bitmap *expanded)
-{
-  bitmap_iterator bi;
-  unsigned j;
-
-  if (*expanded)
-    return *expanded;
-
-  *expanded = BITMAP_ALLOC (&iteration_obstack);
-
-  /* In a first pass expand variables, once for each head to avoid
-     quadratic behavior, to include all sub-fields.  */
-  unsigned prev_head = 0;
-  EXECUTE_IF_SET_IN_BITMAP (set, 0, j, bi)
-    {
-      varinfo_t v = get_varinfo (j);
-      if (v->is_artificial_var
-	  || v->is_full_var)
-	continue;
-      if (v->head != prev_head)
-	{
-	  varinfo_t head = get_varinfo (v->head);
-	  unsigned num = 1;
-	  for (varinfo_t n = vi_next (head); n != NULL; n = vi_next (n))
-	    {
-	      if (n->id != head->id + num)
-		{
-		  /* Usually sub variables are adjacent but since we
-		     create pointed-to restrict representatives there
-		     can be gaps as well.  */
-		  bitmap_set_range (*expanded, head->id, num);
-		  head = n;
-		  num = 1;
-		}
-	      else
-		num++;
-	    }
-
-	  bitmap_set_range (*expanded, head->id, num);
-	  prev_head = v->head;
-	}
-    }
-
-  /* And finally set the rest of the bits from SET in an efficient way.  */
-  bitmap_ior_into (*expanded, set);
-
-  return *expanded;
-}
-
-/* Union solution sets TO and DELTA, and add INC to each member of DELTA in the
-   process.  */
-
-static bool
-set_union_with_increment  (bitmap to, bitmap delta, HOST_WIDE_INT inc,
-			   bitmap *expanded_delta)
-{
-  bool changed = false;
-  bitmap_iterator bi;
-  unsigned int i;
-
-  /* If the solution of DELTA contains anything it is good enough to transfer
-     this to TO.  */
-  if (bitmap_bit_p (delta, anything_id))
-    return bitmap_set_bit (to, anything_id);
-
-  /* If the offset is unknown we have to expand the solution to
-     all subfields.  */
-  if (inc == UNKNOWN_OFFSET)
-    {
-      delta = solution_set_expand (delta, expanded_delta);
-      changed |= bitmap_ior_into (to, delta);
-      return changed;
-    }
-
-  /* For non-zero offset union the offsetted solution into the destination.  */
-  EXECUTE_IF_SET_IN_BITMAP (delta, 0, i, bi)
-    {
-      varinfo_t vi = get_varinfo (i);
-
-      /* If this is a variable with just one field just set its bit
-         in the result.  */
-      if (vi->is_artificial_var
-	  || vi->is_unknown_size_var
-	  || vi->is_full_var)
-	changed |= bitmap_set_bit (to, i);
-      else
-	{
-	  HOST_WIDE_INT fieldoffset = vi->offset + inc;
-	  unsigned HOST_WIDE_INT size = vi->size;
-
-	  /* If the offset makes the pointer point to before the
-	     variable use offset zero for the field lookup.  */
-	  if (fieldoffset < 0)
-	    vi = get_varinfo (vi->head);
-	  else
-	    vi = first_or_preceding_vi_for_offset (vi, fieldoffset);
-
-	  do
-	    {
-	      changed |= bitmap_set_bit (to, vi->id);
-	      if (vi->is_full_var
-		  || vi->next == 0)
-		break;
-
-	      /* We have to include all fields that overlap the current field
-	         shifted by inc.  */
-	      vi = vi_next (vi);
-	    }
-	  while (vi->offset < fieldoffset + size);
-	}
-    }
-
-  return changed;
-}
-
-/* Insert constraint C into the list of complex constraints for graph
-   node VAR.  */
-
-static void
-insert_into_complex (constraint_graph_t graph,
-		     unsigned int var, constraint_t c)
-{
-  vec<constraint_t> complex = graph->complex[var];
-  unsigned int place = complex.lower_bound (c, constraint_less);
-
-  /* Only insert constraints that do not already exist.  */
-  if (place >= complex.length ()
-      || !constraint_equal (*c, *complex[place]))
-    graph->complex[var].safe_insert (place, c);
-}
-
-
-/* Condense two variable nodes into a single variable node, by moving
-   all associated info from FROM to TO. Returns true if TO node's
-   constraint set changes after the merge.  */
-
-static bool
-merge_node_constraints (constraint_graph_t graph, unsigned int to,
-			unsigned int from)
-{
-  unsigned int i;
-  constraint_t c;
-  bool any_change = false;
-
-  gcc_checking_assert (find (from) == to);
-
-  /* Move all complex constraints from src node into to node  */
-  FOR_EACH_VEC_ELT (graph->complex[from], i, c)
-    {
-      /* In complex constraints for node FROM, we may have either
-	 a = *FROM, and *FROM = a, or an offseted constraint which are
-	 always added to the rhs node's constraints.  */
-
-      if (c->rhs.type == DEREF)
-	c->rhs.var = to;
-      else if (c->lhs.type == DEREF)
-	c->lhs.var = to;
-      else
-	c->rhs.var = to;
-
-    }
-  any_change = constraint_set_union (&graph->complex[to],
-				     &graph->complex[from]);
-  graph->complex[from].release ();
-  return any_change;
-}
-
-
-/* Remove edges involving NODE from GRAPH.  */
-
-static void
-clear_edges_for_node (constraint_graph_t graph, unsigned int node)
-{
-  if (graph->succs[node])
-    BITMAP_FREE (graph->succs[node]);
-}
-
-/* Merge GRAPH nodes FROM and TO into node TO.  */
-
-static void
-merge_graph_nodes (constraint_graph_t graph, unsigned int to,
-		   unsigned int from)
-{
-  if (graph->indirect_cycles[from] != -1)
-    {
-      /* If we have indirect cycles with the from node, and we have
-	 none on the to node, the to node has indirect cycles from the
-	 from node now that they are unified.
-	 If indirect cycles exist on both, unify the nodes that they
-	 are in a cycle with, since we know they are in a cycle with
-	 each other.  */
-      if (graph->indirect_cycles[to] == -1)
-	graph->indirect_cycles[to] = graph->indirect_cycles[from];
-    }
-
-  /* Merge all the successor edges.  */
-  if (graph->succs[from])
-    {
-      if (!graph->succs[to])
-	graph->succs[to] = BITMAP_ALLOC (&pta_obstack);
-      bitmap_ior_into (graph->succs[to],
-		       graph->succs[from]);
-    }
-
-  clear_edges_for_node (graph, from);
-}
-
-
-/* Add an indirect graph edge to GRAPH, going from TO to FROM if
-   it doesn't exist in the graph already.  */
-
-static void
-add_implicit_graph_edge (constraint_graph_t graph, unsigned int to,
-			 unsigned int from)
-{
-  if (to == from)
-    return;
-
-  if (!graph->implicit_preds[to])
-    graph->implicit_preds[to] = BITMAP_ALLOC (&predbitmap_obstack);
-
-  if (bitmap_set_bit (graph->implicit_preds[to], from))
-    stats.num_implicit_edges++;
-}
-
-/* Add a predecessor graph edge to GRAPH, going from TO to FROM if
-   it doesn't exist in the graph already.
-   Return false if the edge already existed, true otherwise.  */
-
-static void
-add_pred_graph_edge (constraint_graph_t graph, unsigned int to,
-		     unsigned int from)
-{
-  if (!graph->preds[to])
-    graph->preds[to] = BITMAP_ALLOC (&predbitmap_obstack);
-  bitmap_set_bit (graph->preds[to], from);
-}
-
-/* Add a graph edge to GRAPH, going from FROM to TO if
-   it doesn't exist in the graph already.
-   Return false if the edge already existed, true otherwise.  */
-
-static bool
-add_graph_edge (constraint_graph_t graph, unsigned int to,
-		unsigned int from)
-{
-  if (to == from)
-    {
-      return false;
-    }
-  else
-    {
-      bool r = false;
-
-      if (!graph->succs[from])
-	graph->succs[from] = BITMAP_ALLOC (&pta_obstack);
-
-      /* The graph solving process does not avoid "triangles", thus
-	 there can be multiple paths from a node to another involving
-	 intermediate other nodes.  That causes extra copying which is
-	 most difficult to avoid when the intermediate node is ESCAPED
-	 because there are no edges added from ESCAPED.  Avoid
-	 adding the direct edge FROM -> TO when we have FROM -> ESCAPED
-	 and TO contains ESCAPED.
-	 ???  Note this is only a heuristic, it does not prevent the
-	 situation from occuring.  The heuristic helps PR38474 and
-	 PR99912 significantly.  */
-      if (to < FIRST_REF_NODE
-	  && bitmap_bit_p (graph->succs[from], find (escaped_id))
-	  && bitmap_bit_p (get_varinfo (find (to))->solution, escaped_id))
-	{
-	  stats.num_avoided_edges++;
-	  return false;
-	}
-
-      if (bitmap_set_bit (graph->succs[from], to))
-	{
-	  r = true;
-	  if (to < FIRST_REF_NODE && from < FIRST_REF_NODE)
-	    stats.num_edges++;
-	}
-      return r;
-    }
-}
-
-
-/* Initialize the constraint graph structure to contain SIZE nodes.  */
-
-static void
-init_graph (unsigned int size)
-{
-  unsigned int j;
-
-  graph = XCNEW (struct constraint_graph);
-  graph->size = size;
-  graph->succs = XCNEWVEC (bitmap, graph->size);
-  graph->indirect_cycles = XNEWVEC (int, graph->size);
-  graph->rep = XNEWVEC (unsigned int, graph->size);
-  /* ??? Macros do not support template types with multiple arguments,
-     so we use a typedef to work around it.  */
-  typedef vec<constraint_t> vec_constraint_t_heap;
-  graph->complex = XCNEWVEC (vec_constraint_t_heap, size);
-  graph->pe = XCNEWVEC (unsigned int, graph->size);
-  graph->pe_rep = XNEWVEC (int, graph->size);
-
-  for (j = 0; j < graph->size; j++)
-    {
-      graph->rep[j] = j;
-      graph->pe_rep[j] = -1;
-      graph->indirect_cycles[j] = -1;
-    }
-}
-
-/* Build the constraint graph, adding only predecessor edges right now.  */
-
-static void
-build_pred_graph (void)
-{
-  int i;
-  constraint_t c;
-  unsigned int j;
-
-  graph->implicit_preds = XCNEWVEC (bitmap, graph->size);
-  graph->preds = XCNEWVEC (bitmap, graph->size);
-  graph->pointer_label = XCNEWVEC (unsigned int, graph->size);
-  graph->loc_label = XCNEWVEC (unsigned int, graph->size);
-  graph->pointed_by = XCNEWVEC (bitmap, graph->size);
-  graph->points_to = XCNEWVEC (bitmap, graph->size);
-  graph->eq_rep = XNEWVEC (int, graph->size);
-  graph->direct_nodes = sbitmap_alloc (graph->size);
-  graph->address_taken = BITMAP_ALLOC (&predbitmap_obstack);
-  bitmap_clear (graph->direct_nodes);
-
-  for (j = 1; j < FIRST_REF_NODE; j++)
-    {
-      if (!get_varinfo (j)->is_special_var)
-	bitmap_set_bit (graph->direct_nodes, j);
-    }
-
-  for (j = 0; j < graph->size; j++)
-    graph->eq_rep[j] = -1;
-
-  for (j = 0; j < varmap.length (); j++)
-    graph->indirect_cycles[j] = -1;
-
-  FOR_EACH_VEC_ELT (constraints, i, c)
-    {
-      struct constraint_expr lhs = c->lhs;
-      struct constraint_expr rhs = c->rhs;
-      unsigned int lhsvar = lhs.var;
-      unsigned int rhsvar = rhs.var;
-
-      if (lhs.type == DEREF)
-	{
-	  /* *x = y.  */
-	  if (rhs.offset == 0 && lhs.offset == 0 && rhs.type == SCALAR)
-	    {
-	      if (lhs.var == anything_id)
-		add_pred_graph_edge (graph, storedanything_id, rhsvar);
-	      else
-		add_pred_graph_edge (graph, FIRST_REF_NODE + lhsvar, rhsvar);
-	    }
-	}
-      else if (rhs.type == DEREF)
-	{
-	  /* x = *y */
-	  if (rhs.offset == 0 && lhs.offset == 0 && lhs.type == SCALAR)
-	    add_pred_graph_edge (graph, lhsvar, FIRST_REF_NODE + rhsvar);
-	  else
-	    bitmap_clear_bit (graph->direct_nodes, lhsvar);
-	}
-      else if (rhs.type == ADDRESSOF)
-	{
-	  varinfo_t v;
-
-	  /* x = &y */
-	  if (graph->points_to[lhsvar] == NULL)
-	    graph->points_to[lhsvar] = BITMAP_ALLOC (&predbitmap_obstack);
-	  bitmap_set_bit (graph->points_to[lhsvar], rhsvar);
-
-	  if (graph->pointed_by[rhsvar] == NULL)
-	    graph->pointed_by[rhsvar] = BITMAP_ALLOC (&predbitmap_obstack);
-	  bitmap_set_bit (graph->pointed_by[rhsvar], lhsvar);
-
-	  /* Implicitly, *x = y */
-	  add_implicit_graph_edge (graph, FIRST_REF_NODE + lhsvar, rhsvar);
-
-	  /* All related variables are no longer direct nodes.  */
-	  bitmap_clear_bit (graph->direct_nodes, rhsvar);
-          v = get_varinfo (rhsvar);
-          if (!v->is_full_var)
-            {
-              v = get_varinfo (v->head);
-              do
-                {
-                  bitmap_clear_bit (graph->direct_nodes, v->id);
-                  v = vi_next (v);
-                }
-              while (v != NULL);
-            }
-	  bitmap_set_bit (graph->address_taken, rhsvar);
-	}
-      else if (lhsvar > anything_id
-	       && lhsvar != rhsvar && lhs.offset == 0 && rhs.offset == 0)
-	{
-	  /* x = y */
-	  add_pred_graph_edge (graph, lhsvar, rhsvar);
-	  /* Implicitly, *x = *y */
-	  add_implicit_graph_edge (graph, FIRST_REF_NODE + lhsvar,
-				   FIRST_REF_NODE + rhsvar);
-	}
-      else if (lhs.offset != 0 || rhs.offset != 0)
-	{
-	  if (rhs.offset != 0)
-	    bitmap_clear_bit (graph->direct_nodes, lhs.var);
-	  else if (lhs.offset != 0)
-	    bitmap_clear_bit (graph->direct_nodes, rhs.var);
-	}
-    }
-}
-
-/* Build the constraint graph, adding successor edges.  */
-
-static void
-build_succ_graph (void)
-{
-  unsigned i, t;
-  constraint_t c;
-
-  FOR_EACH_VEC_ELT (constraints, i, c)
-    {
-      struct constraint_expr lhs;
-      struct constraint_expr rhs;
-      unsigned int lhsvar;
-      unsigned int rhsvar;
-
-      if (!c)
-	continue;
-
-      lhs = c->lhs;
-      rhs = c->rhs;
-      lhsvar = find (lhs.var);
-      rhsvar = find (rhs.var);
-
-      if (lhs.type == DEREF)
-	{
-	  if (rhs.offset == 0 && lhs.offset == 0 && rhs.type == SCALAR)
-	    {
-	      if (lhs.var == anything_id)
-		add_graph_edge (graph, storedanything_id, rhsvar);
-	      else
-		add_graph_edge (graph, FIRST_REF_NODE + lhsvar, rhsvar);
-	    }
-	}
-      else if (rhs.type == DEREF)
-	{
-	  if (rhs.offset == 0 && lhs.offset == 0 && lhs.type == SCALAR)
-	    add_graph_edge (graph, lhsvar, FIRST_REF_NODE + rhsvar);
-	}
-      else if (rhs.type == ADDRESSOF)
-	{
-	  /* x = &y */
-	  gcc_checking_assert (find (rhs.var) == rhs.var);
-	  bitmap_set_bit (get_varinfo (lhsvar)->solution, rhsvar);
-	}
-      else if (lhsvar > anything_id
-	       && lhsvar != rhsvar && lhs.offset == 0 && rhs.offset == 0)
-	{
-	  add_graph_edge (graph, lhsvar, rhsvar);
-	}
-    }
-
-  /* Add edges from STOREDANYTHING to all nodes that can receive pointers.  */
-  t = find (storedanything_id);
-  for (i = integer_id + 1; i < FIRST_REF_NODE; ++i)
-    {
-      if (get_varinfo (i)->may_have_pointers)
-	add_graph_edge (graph, find (i), t);
-    }
-
-  /* Everything stored to ANYTHING also potentially escapes.  */
-  add_graph_edge (graph, find (escaped_id), t);
-}
-
-
-/* Changed variables on the last iteration.  */
-static bitmap changed;
-
-/* Strongly Connected Component visitation info.  */
-
-class scc_info
-{
-public:
-  scc_info (size_t size);
-  ~scc_info ();
-
-  auto_sbitmap visited;
-  auto_sbitmap deleted;
-  unsigned int *dfs;
-  unsigned int *node_mapping;
-  int current_index;
-  auto_vec<unsigned> scc_stack;
-};
-
-
-/* Recursive routine to find strongly connected components in GRAPH.
-   SI is the SCC info to store the information in, and N is the id of current
-   graph node we are processing.
-
-   This is Tarjan's strongly connected component finding algorithm, as
-   modified by Nuutila to keep only non-root nodes on the stack.
-   The algorithm can be found in "On finding the strongly connected
-   connected components in a directed graph" by Esko Nuutila and Eljas
-   Soisalon-Soininen, in Information Processing Letters volume 49,
-   number 1, pages 9-14.  */
-
-static void
-scc_visit (constraint_graph_t graph, class scc_info *si, unsigned int n)
-{
-  unsigned int i;
-  bitmap_iterator bi;
-  unsigned int my_dfs;
-
-  bitmap_set_bit (si->visited, n);
-  si->dfs[n] = si->current_index ++;
-  my_dfs = si->dfs[n];
-
-  /* Visit all the successors.  */
-  EXECUTE_IF_IN_NONNULL_BITMAP (graph->succs[n], 0, i, bi)
-    {
-      unsigned int w;
-
-      if (i > LAST_REF_NODE)
-	break;
-
-      w = find (i);
-      if (bitmap_bit_p (si->deleted, w))
-	continue;
-
-      if (!bitmap_bit_p (si->visited, w))
-	scc_visit (graph, si, w);
-
-      unsigned int t = find (w);
-      gcc_checking_assert (find (n) == n);
-      if (si->dfs[t] < si->dfs[n])
-	si->dfs[n] = si->dfs[t];
-    }
-
-  /* See if any components have been identified.  */
-  if (si->dfs[n] == my_dfs)
-    {
-      if (si->scc_stack.length () > 0
-	  && si->dfs[si->scc_stack.last ()] >= my_dfs)
-	{
-	  bitmap scc = BITMAP_ALLOC (NULL);
-	  unsigned int lowest_node;
-	  bitmap_iterator bi;
-
-	  bitmap_set_bit (scc, n);
-
-	  while (si->scc_stack.length () != 0
-		 && si->dfs[si->scc_stack.last ()] >= my_dfs)
-	    {
-	      unsigned int w = si->scc_stack.pop ();
-
-	      bitmap_set_bit (scc, w);
-	    }
-
-	  lowest_node = bitmap_first_set_bit (scc);
-	  gcc_assert (lowest_node < FIRST_REF_NODE);
-
-	  /* Collapse the SCC nodes into a single node, and mark the
-	     indirect cycles.  */
-	  EXECUTE_IF_SET_IN_BITMAP (scc, 0, i, bi)
-	    {
-	      if (i < FIRST_REF_NODE)
-		{
-		  if (unite (lowest_node, i))
-		    unify_nodes (graph, lowest_node, i, false);
-		}
-	      else
-		{
-		  unite (lowest_node, i);
-		  graph->indirect_cycles[i - FIRST_REF_NODE] = lowest_node;
-		}
-	    }
-	  bitmap_set_bit (si->deleted, lowest_node);
-	}
-      else
-	bitmap_set_bit (si->deleted, n);
-    }
-  else
-    si->scc_stack.safe_push (n);
-}
-
-/* Unify node FROM into node TO, updating the changed count if
-   necessary when UPDATE_CHANGED is true.  */
-
-static void
-unify_nodes (constraint_graph_t graph, unsigned int to, unsigned int from,
-	     bool update_changed)
-{
-  gcc_checking_assert (to != from && find (to) == to);
-
-  if (dump_file && (dump_flags & TDF_DETAILS))
-    fprintf (dump_file, "Unifying %s to %s\n",
-	     get_varinfo (from)->name,
-	     get_varinfo (to)->name);
-
-  if (update_changed)
-    stats.unified_vars_dynamic++;
-  else
-    stats.unified_vars_static++;
-
-  merge_graph_nodes (graph, to, from);
-  if (merge_node_constraints (graph, to, from))
-    {
-      if (update_changed)
-	bitmap_set_bit (changed, to);
-    }
-
-  /* Mark TO as changed if FROM was changed. If TO was already marked
-     as changed, decrease the changed count.  */
-
-  if (update_changed
-      && bitmap_clear_bit (changed, from))
-    bitmap_set_bit (changed, to);
-  varinfo_t fromvi = get_varinfo (from);
-  if (fromvi->solution)
-    {
-      /* If the solution changes because of the merging, we need to mark
-	 the variable as changed.  */
-      varinfo_t tovi = get_varinfo (to);
-      if (bitmap_ior_into (tovi->solution, fromvi->solution))
-	{
-	  if (update_changed)
-	    bitmap_set_bit (changed, to);
-	}
-
-      BITMAP_FREE (fromvi->solution);
-      if (fromvi->oldsolution)
-	BITMAP_FREE (fromvi->oldsolution);
-
-      if (stats.iterations > 0
-	  && tovi->oldsolution)
-	BITMAP_FREE (tovi->oldsolution);
-    }
-  if (graph->succs[to])
-    bitmap_clear_bit (graph->succs[to], to);
-}
-
-/* Add a copy edge FROM -> TO, optimizing special cases.  Returns TRUE
-   if the solution of TO changed.  */
-
-static bool
-solve_add_graph_edge (constraint_graph_t graph, unsigned int to,
-		      unsigned int from)
-{
-  /* Adding edges from the special vars is pointless.
-     They don't have sets that can change.  */
-  if (get_varinfo (from)->is_special_var)
-    return bitmap_ior_into (get_varinfo (to)->solution,
-			    get_varinfo (from)->solution);
-  /* Merging the solution from ESCAPED needlessly increases
-     the set.  Use ESCAPED as representative instead.  */
-  else if (from == find (escaped_id))
-    return bitmap_set_bit (get_varinfo (to)->solution, escaped_id);
-  else if (get_varinfo (from)->may_have_pointers
-	   && add_graph_edge (graph, to, from))
-    return bitmap_ior_into (get_varinfo (to)->solution,
-			    get_varinfo (from)->solution);
-  return false;
-}
-
-/* Process a constraint C that represents x = *(y + off), using DELTA as the
-   starting solution for y.  */
-
-static void
-do_sd_constraint (constraint_graph_t graph, constraint_t c,
-		  bitmap delta, bitmap *expanded_delta)
-{
-  unsigned int lhs = c->lhs.var;
-  bool flag = false;
-  bitmap sol = get_varinfo (lhs)->solution;
-  unsigned int j;
-  bitmap_iterator bi;
-  HOST_WIDE_INT roffset = c->rhs.offset;
-
-  /* Our IL does not allow this.  */
-  gcc_checking_assert (c->lhs.offset == 0);
-
-  /* If the solution of Y contains anything it is good enough to transfer
-     this to the LHS.  */
-  if (bitmap_bit_p (delta, anything_id))
-    {
-      flag |= bitmap_set_bit (sol, anything_id);
-      goto done;
-    }
-
-  /* If we do not know at with offset the rhs is dereferenced compute
-     the reachability set of DELTA, conservatively assuming it is
-     dereferenced at all valid offsets.  */
-  if (roffset == UNKNOWN_OFFSET)
-    {
-      delta = solution_set_expand (delta, expanded_delta);
-      /* No further offset processing is necessary.  */
-      roffset = 0;
-    }
-
-  /* For each variable j in delta (Sol(y)), add
-     an edge in the graph from j to x, and union Sol(j) into Sol(x).  */
-  EXECUTE_IF_SET_IN_BITMAP (delta, 0, j, bi)
-    {
-      varinfo_t v = get_varinfo (j);
-      HOST_WIDE_INT fieldoffset = v->offset + roffset;
-      unsigned HOST_WIDE_INT size = v->size;
-      unsigned int t;
-
-      if (v->is_full_var)
-	;
-      else if (roffset != 0)
-	{
-	  if (fieldoffset < 0)
-	    v = get_varinfo (v->head);
-	  else
-	    v = first_or_preceding_vi_for_offset (v, fieldoffset);
-	}
-
-      /* We have to include all fields that overlap the current field
-	 shifted by roffset.  */
-      do
-	{
-	  t = find (v->id);
-
-	  flag |= solve_add_graph_edge (graph, lhs, t);
-
-	  if (v->is_full_var
-	      || v->next == 0)
-	    break;
-
-	  v = vi_next (v);
-	}
-      while (v->offset < fieldoffset + size);
-    }
-
-done:
-  /* If the LHS solution changed, mark the var as changed.  */
-  if (flag)
-    bitmap_set_bit (changed, lhs);
-}
-
-/* Process a constraint C that represents *(x + off) = y using DELTA
-   as the starting solution for x.  */
-
-static void
-do_ds_constraint (constraint_t c, bitmap delta, bitmap *expanded_delta)
-{
-  unsigned int rhs = c->rhs.var;
-  bitmap sol = get_varinfo (rhs)->solution;
-  unsigned int j;
-  bitmap_iterator bi;
-  HOST_WIDE_INT loff = c->lhs.offset;
-  bool escaped_p = false;
-
-  /* Our IL does not allow this.  */
-  gcc_checking_assert (c->rhs.offset == 0);
-
-  /* If the solution of y contains ANYTHING simply use the ANYTHING
-     solution.  This avoids needlessly increasing the points-to sets.  */
-  if (bitmap_bit_p (sol, anything_id))
-    sol = get_varinfo (find (anything_id))->solution;
-
-  /* If the solution for x contains ANYTHING we have to merge the
-     solution of y into all pointer variables which we do via
-     STOREDANYTHING.  */
-  if (bitmap_bit_p (delta, anything_id))
-    {
-      unsigned t = find (storedanything_id);
-      if (solve_add_graph_edge (graph, t, rhs))
-	bitmap_set_bit (changed, t);
-      return;
-    }
-
-  /* If we do not know at with offset the rhs is dereferenced compute
-     the reachability set of DELTA, conservatively assuming it is
-     dereferenced at all valid offsets.  */
-  if (loff == UNKNOWN_OFFSET)
-    {
-      delta = solution_set_expand (delta, expanded_delta);
-      loff = 0;
-    }
-
-  /* For each member j of delta (Sol(x)), add an edge from y to j and
-     union Sol(y) into Sol(j) */
-  EXECUTE_IF_SET_IN_BITMAP (delta, 0, j, bi)
-    {
-      varinfo_t v = get_varinfo (j);
-      unsigned int t;
-      HOST_WIDE_INT fieldoffset = v->offset + loff;
-      unsigned HOST_WIDE_INT size = v->size;
-
-      if (v->is_full_var)
-	;
-      else if (loff != 0)
-	{
-	  if (fieldoffset < 0)
-	    v = get_varinfo (v->head);
-	  else
-	    v = first_or_preceding_vi_for_offset (v, fieldoffset);
-	}
-
-      /* We have to include all fields that overlap the current field
-	 shifted by loff.  */
-      do
-	{
-	  if (v->may_have_pointers)
-	    {
-	      /* If v is a global variable then this is an escape point.  */
-	      if (v->is_global_var
-		  && !escaped_p)
-		{
-		  t = find (escaped_id);
-		  if (add_graph_edge (graph, t, rhs)
-		      && bitmap_ior_into (get_varinfo (t)->solution, sol))
-		    bitmap_set_bit (changed, t);
-		  /* Enough to let rhs escape once.  */
-		  escaped_p = true;
-		}
-
-	      if (v->is_special_var)
-		break;
-
-	      t = find (v->id);
-
-	      if (solve_add_graph_edge  (graph, t, rhs))
-		bitmap_set_bit (changed, t);
-	    }
-
-	  if (v->is_full_var
-	      || v->next == 0)
-	    break;
-
-	  v = vi_next (v);
-	}
-      while (v->offset < fieldoffset + size);
-    }
-}
-
-/* Handle a non-simple (simple meaning requires no iteration),
-   constraint (IE *x = &y, x = *y, *x = y, and x = y with offsets involved).  */
-
-static void
-do_complex_constraint (constraint_graph_t graph, constraint_t c, bitmap delta,
-		       bitmap *expanded_delta)
-{
-  if (c->lhs.type == DEREF)
-    {
-      if (c->rhs.type == ADDRESSOF)
-	{
-	  gcc_unreachable ();
-	}
-      else
-	{
-	  /* *x = y */
-	  do_ds_constraint (c, delta, expanded_delta);
-	}
-    }
-  else if (c->rhs.type == DEREF)
-    {
-      /* x = *y */
-      if (!(get_varinfo (c->lhs.var)->is_special_var))
-	do_sd_constraint (graph, c, delta, expanded_delta);
-    }
-  else
-    {
-      bitmap tmp;
-      bool flag = false;
-
-      gcc_checking_assert (c->rhs.type == SCALAR && c->lhs.type == SCALAR
-			   && c->rhs.offset != 0 && c->lhs.offset == 0);
-      tmp = get_varinfo (c->lhs.var)->solution;
-
-      flag = set_union_with_increment (tmp, delta, c->rhs.offset,
-				       expanded_delta);
-
-      if (flag)
-	bitmap_set_bit (changed, c->lhs.var);
-    }
-}
-
-/* Initialize and return a new SCC info structure.  */
-
-scc_info::scc_info (size_t size) :
-  visited (size), deleted (size), current_index (0), scc_stack (1)
-{
-  bitmap_clear (visited);
-  bitmap_clear (deleted);
-  node_mapping = XNEWVEC (unsigned int, size);
-  dfs = XCNEWVEC (unsigned int, size);
-
-  for (size_t i = 0; i < size; i++)
-    node_mapping[i] = i;
-}
-
-/* Free an SCC info structure pointed to by SI */
-
-scc_info::~scc_info ()
-{
-  free (node_mapping);
-  free (dfs);
-}
-
-
-/* Find indirect cycles in GRAPH that occur, using strongly connected
-   components, and note them in the indirect cycles map.
-
-   This technique comes from Ben Hardekopf and Calvin Lin,
-   "It Pays to be Lazy: Fast and Accurate Pointer Analysis for Millions of
-   Lines of Code", submitted to PLDI 2007.  */
-
-static void
-find_indirect_cycles (constraint_graph_t graph)
-{
-  unsigned int i;
-  unsigned int size = graph->size;
-  scc_info si (size);
-
-  for (i = 0; i < MIN (LAST_REF_NODE, size); i ++ )
-    if (!bitmap_bit_p (si.visited, i) && find (i) == i)
-      scc_visit (graph, &si, i);
-}
-
-/* Visit the graph in topological order starting at node N, and store the
-   order in TOPO_ORDER using VISITED to indicate visited nodes.  */
-
-static void
-topo_visit (constraint_graph_t graph, vec<unsigned> &topo_order,
-	    sbitmap visited, unsigned int n)
-{
-  bitmap_iterator bi;
-  unsigned int j;
-
-  bitmap_set_bit (visited, n);
-
-  if (graph->succs[n])
-    EXECUTE_IF_SET_IN_BITMAP (graph->succs[n], 0, j, bi)
-      {
-	unsigned k = find (j);
-	if (!bitmap_bit_p (visited, k))
-	  topo_visit (graph, topo_order, visited, k);
-      }
-
-  /* Also consider copy with offset complex constraints as implicit edges.  */
-  for (auto c : graph->complex[n])
-    {
-      /* Constraints are ordered so that SCALAR = SCALAR appear first.  */
-      if (c->lhs.type != SCALAR || c->rhs.type != SCALAR)
-	break;
-      gcc_checking_assert (c->rhs.var == n);
-      unsigned k = find (c->lhs.var);
-      if (!bitmap_bit_p (visited, k))
-	topo_visit (graph, topo_order, visited, k);
-    }
-
-  topo_order.quick_push (n);
-}
-
-/* Compute a topological ordering for GRAPH, and return the result.  */
-
-static auto_vec<unsigned>
-compute_topo_order (constraint_graph_t graph)
-{
-  unsigned int i;
-  unsigned int size = graph->size;
-
-  auto_sbitmap visited (size);
-  bitmap_clear (visited);
-
-  /* For the heuristic in add_graph_edge to work optimally make sure to
-     first visit the connected component of the graph containing
-     ESCAPED.  Do this by extracting the connected component
-     with ESCAPED and append that to all other components as solve_graph
-     pops from the order.  */
-  auto_vec<unsigned> tail (size);
-  topo_visit (graph, tail, visited, find (escaped_id));
-
-  auto_vec<unsigned> topo_order (size);
-
-  for (i = 0; i != size; ++i)
-    if (!bitmap_bit_p (visited, i) && find (i) == i)
-      topo_visit (graph, topo_order, visited, i);
-
-  topo_order.splice (tail);
-  return topo_order;
-}
-
-/* Structure used to for hash value numbering of pointer equivalence
-   classes.  */
-
-typedef struct equiv_class_label
-{
-  hashval_t hashcode;
-  unsigned int equivalence_class;
-  bitmap labels;
-} *equiv_class_label_t;
-typedef const struct equiv_class_label *const_equiv_class_label_t;
-
-/* Equiv_class_label hashtable helpers.  */
-
-struct equiv_class_hasher : nofree_ptr_hash <equiv_class_label>
-{
-  static inline hashval_t hash (const equiv_class_label *);
-  static inline bool equal (const equiv_class_label *,
-			    const equiv_class_label *);
-};
-
-/* Hash function for a equiv_class_label_t */
-
-inline hashval_t
-equiv_class_hasher::hash (const equiv_class_label *ecl)
-{
-  return ecl->hashcode;
-}
-
-/* Equality function for two equiv_class_label_t's.  */
-
-inline bool
-equiv_class_hasher::equal (const equiv_class_label *eql1,
-			   const equiv_class_label *eql2)
-{
-  return (eql1->hashcode == eql2->hashcode
-	  && bitmap_equal_p (eql1->labels, eql2->labels));
-}
-
-/* A hashtable for mapping a bitmap of labels->pointer equivalence
-   classes.  */
-static hash_table<equiv_class_hasher> *pointer_equiv_class_table;
-
-/* A hashtable for mapping a bitmap of labels->location equivalence
-   classes.  */
-static hash_table<equiv_class_hasher> *location_equiv_class_table;
-
-struct obstack equiv_class_obstack;
-
-/* Lookup a equivalence class in TABLE by the bitmap of LABELS with
-   hash HAS it contains.  Sets *REF_LABELS to the bitmap LABELS
-   is equivalent to.  */
-
-static equiv_class_label *
-equiv_class_lookup_or_add (hash_table<equiv_class_hasher> *table,
-			   bitmap labels)
-{
-  equiv_class_label **slot;
-  equiv_class_label ecl;
-
-  ecl.labels = labels;
-  ecl.hashcode = bitmap_hash (labels);
-  slot = table->find_slot (&ecl, INSERT);
-  if (!*slot)
-    {
-      *slot = XOBNEW (&equiv_class_obstack, struct equiv_class_label);
-      (*slot)->labels = labels;
-      (*slot)->hashcode = ecl.hashcode;
-      (*slot)->equivalence_class = 0;
-    }
-
-  return *slot;
-}
-
-/* Perform offline variable substitution.
-
-   This is a worst case quadratic time way of identifying variables
-   that must have equivalent points-to sets, including those caused by
-   static cycles, and single entry subgraphs, in the constraint graph.
-
-   The technique is described in "Exploiting Pointer and Location
-   Equivalence to Optimize Pointer Analysis. In the 14th International
-   Static Analysis Symposium (SAS), August 2007."  It is known as the
-   "HU" algorithm, and is equivalent to value numbering the collapsed
-   constraint graph including evaluating unions.
-
-   The general method of finding equivalence classes is as follows:
-   Add fake nodes (REF nodes) and edges for *a = b and a = *b constraints.
-   Initialize all non-REF nodes to be direct nodes.
-   For each constraint a = a U {b}, we set pts(a) = pts(a) u {fresh
-   variable}
-   For each constraint containing the dereference, we also do the same
-   thing.
-
-   We then compute SCC's in the graph and unify nodes in the same SCC,
-   including pts sets.
-
-   For each non-collapsed node x:
-    Visit all unvisited explicit incoming edges.
-    Ignoring all non-pointers, set pts(x) = Union of pts(a) for y
-    where y->x.
-    Lookup the equivalence class for pts(x).
-     If we found one, equivalence_class(x) = found class.
-     Otherwise, equivalence_class(x) = new class, and new_class is
-    added to the lookup table.
-
-   All direct nodes with the same equivalence class can be replaced
-   with a single representative node.
-   All unlabeled nodes (label == 0) are not pointers and all edges
-   involving them can be eliminated.
-   We perform these optimizations during rewrite_constraints
-
-   In addition to pointer equivalence class finding, we also perform
-   location equivalence class finding.  This is the set of variables
-   that always appear together in points-to sets.  We use this to
-   compress the size of the points-to sets.  */
-
-/* Current maximum pointer equivalence class id.  */
-static int pointer_equiv_class;
-
-/* Current maximum location equivalence class id.  */
-static int location_equiv_class;
-
-/* Recursive routine to find strongly connected components in GRAPH,
-   and label it's nodes with DFS numbers.  */
-
-static void
-condense_visit (constraint_graph_t graph, class scc_info *si, unsigned int n)
-{
-  unsigned int i;
-  bitmap_iterator bi;
-  unsigned int my_dfs;
-
-  gcc_checking_assert (si->node_mapping[n] == n);
-  bitmap_set_bit (si->visited, n);
-  si->dfs[n] = si->current_index ++;
-  my_dfs = si->dfs[n];
-
-  /* Visit all the successors.  */
-  EXECUTE_IF_IN_NONNULL_BITMAP (graph->preds[n], 0, i, bi)
-    {
-      unsigned int w = si->node_mapping[i];
-
-      if (bitmap_bit_p (si->deleted, w))
-	continue;
-
-      if (!bitmap_bit_p (si->visited, w))
-	condense_visit (graph, si, w);
-
-      unsigned int t = si->node_mapping[w];
-      gcc_checking_assert (si->node_mapping[n] == n);
-      if (si->dfs[t] < si->dfs[n])
-	si->dfs[n] = si->dfs[t];
-    }
-
-  /* Visit all the implicit predecessors.  */
-  EXECUTE_IF_IN_NONNULL_BITMAP (graph->implicit_preds[n], 0, i, bi)
-    {
-      unsigned int w = si->node_mapping[i];
-
-      if (bitmap_bit_p (si->deleted, w))
-	continue;
-
-      if (!bitmap_bit_p (si->visited, w))
-	condense_visit (graph, si, w);
-
-      unsigned int t = si->node_mapping[w];
-      gcc_assert (si->node_mapping[n] == n);
-      if (si->dfs[t] < si->dfs[n])
-	si->dfs[n] = si->dfs[t];
-    }
-
-  /* See if any components have been identified.  */
-  if (si->dfs[n] == my_dfs)
-    {
-      if (si->scc_stack.length () != 0
-	  && si->dfs[si->scc_stack.last ()] >= my_dfs)
-	{
-	  /* Find the first node of the SCC and do non-bitmap work.  */
-	  bool direct_p = true;
-	  unsigned first = si->scc_stack.length ();
-	  do
-	    {
-	      --first;
-	      unsigned int w = si->scc_stack[first];
-	      si->node_mapping[w] = n;
-	      if (!bitmap_bit_p (graph->direct_nodes, w))
-		direct_p = false;
-	    }
-	  while (first > 0
-		 && si->dfs[si->scc_stack[first - 1]] >= my_dfs);
-	  if (!direct_p)
-	    bitmap_clear_bit (graph->direct_nodes, n);
-
-	  /* Want to reduce to node n, push that first.  */
-	  si->scc_stack.reserve (1);
-	  si->scc_stack.quick_push (si->scc_stack[first]);
-	  si->scc_stack[first] = n;
-
-	  unsigned scc_size = si->scc_stack.length () - first;
-	  unsigned split = scc_size / 2;
-	  unsigned carry = scc_size - split * 2;
-	  while (split > 0)
-	    {
-	      for (unsigned i = 0; i < split; ++i)
-		{
-		  unsigned a = si->scc_stack[first + i];
-		  unsigned b = si->scc_stack[first + split + carry + i];
-
-		  /* Unify our nodes.  */
-		  if (graph->preds[b])
-		    {
-		      if (!graph->preds[a])
-			std::swap (graph->preds[a], graph->preds[b]);
-		      else
-			bitmap_ior_into_and_free (graph->preds[a],
-						  &graph->preds[b]);
-		    }
-		  if (graph->implicit_preds[b])
-		    {
-		      if (!graph->implicit_preds[a])
-			std::swap (graph->implicit_preds[a],
-				   graph->implicit_preds[b]);
-		      else
-			bitmap_ior_into_and_free (graph->implicit_preds[a],
-						  &graph->implicit_preds[b]);
-		    }
-		  if (graph->points_to[b])
-		    {
-		      if (!graph->points_to[a])
-			std::swap (graph->points_to[a], graph->points_to[b]);
-		      else
-			bitmap_ior_into_and_free (graph->points_to[a],
-						  &graph->points_to[b]);
-		    }
-		}
-	      unsigned remain = split + carry;
-	      split = remain / 2;
-	      carry = remain - split * 2;
-	    }
-	  /* Actually pop the SCC.  */
-	  si->scc_stack.truncate (first);
-	}
-      bitmap_set_bit (si->deleted, n);
-    }
-  else
-    si->scc_stack.safe_push (n);
-}
-
-/* Label pointer equivalences.
-
-   This performs a value numbering of the constraint graph to
-   discover which variables will always have the same points-to sets
-   under the current set of constraints.
-
-   The way it value numbers is to store the set of points-to bits
-   generated by the constraints and graph edges.  This is just used as a
-   hash and equality comparison.  The *actual set of points-to bits* is
-   completely irrelevant, in that we don't care about being able to
-   extract them later.
-
-   The equality values (currently bitmaps) just have to satisfy a few
-   constraints, the main ones being:
-   1. The combining operation must be order independent.
-   2. The end result of a given set of operations must be unique iff the
-      combination of input values is unique
-   3. Hashable.  */
-
-static void
-label_visit (constraint_graph_t graph, class scc_info *si, unsigned int n)
-{
-  unsigned int i, first_pred;
-  bitmap_iterator bi;
-
-  bitmap_set_bit (si->visited, n);
-
-  /* Label and union our incoming edges's points to sets.  */
-  first_pred = -1U;
-  EXECUTE_IF_IN_NONNULL_BITMAP (graph->preds[n], 0, i, bi)
-    {
-      unsigned int w = si->node_mapping[i];
-      if (!bitmap_bit_p (si->visited, w))
-	label_visit (graph, si, w);
-
-      /* Skip unused edges  */
-      if (w == n || graph->pointer_label[w] == 0)
-	continue;
-
-      if (graph->points_to[w])
-	{
-	  if (!graph->points_to[n])
-	    {
-	      if (first_pred == -1U)
-		first_pred = w;
-	      else
-		{
-		  graph->points_to[n] = BITMAP_ALLOC (&predbitmap_obstack);
-		  bitmap_ior (graph->points_to[n],
-			      graph->points_to[first_pred],
-			      graph->points_to[w]);
-		}
-	    }
-	  else
-	    bitmap_ior_into (graph->points_to[n], graph->points_to[w]);
-	}
-    }
-
-  /* Indirect nodes get fresh variables and a new pointer equiv class.  */
-  if (!bitmap_bit_p (graph->direct_nodes, n))
-    {
-      if (!graph->points_to[n])
-	{
-	  graph->points_to[n] = BITMAP_ALLOC (&predbitmap_obstack);
-	  if (first_pred != -1U)
-	    bitmap_copy (graph->points_to[n], graph->points_to[first_pred]);
-	}
-      bitmap_set_bit (graph->points_to[n], FIRST_REF_NODE + n);
-      graph->pointer_label[n] = pointer_equiv_class++;
-      equiv_class_label_t ecl;
-      ecl = equiv_class_lookup_or_add (pointer_equiv_class_table,
-				       graph->points_to[n]);
-      ecl->equivalence_class = graph->pointer_label[n];
-      return;
-    }
-
-  /* If there was only a single non-empty predecessor the pointer equiv
-     class is the same.  */
-  if (!graph->points_to[n])
-    {
-      if (first_pred != -1U)
-	{
-	  graph->pointer_label[n] = graph->pointer_label[first_pred];
-	  graph->points_to[n] = graph->points_to[first_pred];
-	}
-      return;
-    }
-
-  if (!bitmap_empty_p (graph->points_to[n]))
-    {
-      equiv_class_label_t ecl;
-      ecl = equiv_class_lookup_or_add (pointer_equiv_class_table,
-				       graph->points_to[n]);
-      if (ecl->equivalence_class == 0)
-	ecl->equivalence_class = pointer_equiv_class++;
-      else
-	{
-	  BITMAP_FREE (graph->points_to[n]);
-	  graph->points_to[n] = ecl->labels;
-	}
-      graph->pointer_label[n] = ecl->equivalence_class;
-    }
-}
-
-/* Print the pred graph in dot format.  */
-
-static void
-dump_pred_graph (class scc_info *si, FILE *file)
-{
-  unsigned int i;
-
-  /* Only print the graph if it has already been initialized:  */
-  if (!graph)
-    return;
-
-  /* Prints the header of the dot file:  */
-  fprintf (file, "strict digraph {\n");
-  fprintf (file, "  node [\n    shape = box\n  ]\n");
-  fprintf (file, "  edge [\n    fontsize = \"12\"\n  ]\n");
-  fprintf (file, "\n  // List of nodes and complex constraints in "
-	   "the constraint graph:\n");
-
-  /* The next lines print the nodes in the graph together with the
-     complex constraints attached to them.  */
-  for (i = 1; i < graph->size; i++)
-    {
-      if (i == FIRST_REF_NODE)
-	continue;
-      if (si->node_mapping[i] != i)
-	continue;
-      if (i < FIRST_REF_NODE)
-	fprintf (file, "\"%s\"", get_varinfo (i)->name);
-      else
-	fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
-      if (graph->points_to[i]
-	  && !bitmap_empty_p (graph->points_to[i]))
-	{
-	  if (i < FIRST_REF_NODE)
-	    fprintf (file, "[label=\"%s = {", get_varinfo (i)->name);
-	  else
-	    fprintf (file, "[label=\"*%s = {",
-		     get_varinfo (i - FIRST_REF_NODE)->name);
-	  unsigned j;
-	  bitmap_iterator bi;
-	  EXECUTE_IF_SET_IN_BITMAP (graph->points_to[i], 0, j, bi)
-	    fprintf (file, " %d", j);
-	  fprintf (file, " }\"]");
-	}
-      fprintf (file, ";\n");
-    }
-
-  /* Go over the edges.  */
-  fprintf (file, "\n  // Edges in the constraint graph:\n");
-  for (i = 1; i < graph->size; i++)
-    {
-      unsigned j;
-      bitmap_iterator bi;
-      if (si->node_mapping[i] != i)
-	continue;
-      EXECUTE_IF_IN_NONNULL_BITMAP (graph->preds[i], 0, j, bi)
-	{
-	  unsigned from = si->node_mapping[j];
-	  if (from < FIRST_REF_NODE)
-	    fprintf (file, "\"%s\"", get_varinfo (from)->name);
-	  else
-	    fprintf (file, "\"*%s\"", get_varinfo (from - FIRST_REF_NODE)->name);
-	  fprintf (file, " -> ");
-	  if (i < FIRST_REF_NODE)
-	    fprintf (file, "\"%s\"", get_varinfo (i)->name);
-	  else
-	    fprintf (file, "\"*%s\"", get_varinfo (i - FIRST_REF_NODE)->name);
-	  fprintf (file, ";\n");
-	}
-    }
-
-  /* Prints the tail of the dot file.  */
-  fprintf (file, "}\n");
-}
-
-/* Perform offline variable substitution, discovering equivalence
-   classes, and eliminating non-pointer variables.  */
-
-static class scc_info *
-perform_var_substitution (constraint_graph_t graph)
-{
-  unsigned int i;
-  unsigned int size = graph->size;
-  scc_info *si = new scc_info (size);
-
-  bitmap_obstack_initialize (&iteration_obstack);
-  gcc_obstack_init (&equiv_class_obstack);
-  pointer_equiv_class_table = new hash_table<equiv_class_hasher> (511);
-  location_equiv_class_table
-    = new hash_table<equiv_class_hasher> (511);
-  pointer_equiv_class = 1;
-  location_equiv_class = 1;
-
-  /* Condense the nodes, which means to find SCC's, count incoming
-     predecessors, and unite nodes in SCC's.  */
-  for (i = 1; i < FIRST_REF_NODE; i++)
-    if (!bitmap_bit_p (si->visited, si->node_mapping[i]))
-      condense_visit (graph, si, si->node_mapping[i]);
-
-  if (dump_file && (dump_flags & TDF_GRAPH))
-    {
-      fprintf (dump_file, "\n\n// The constraint graph before var-substitution "
-	       "in dot format:\n");
-      dump_pred_graph (si, dump_file);
-      fprintf (dump_file, "\n\n");
-    }
-
-  bitmap_clear (si->visited);
-  /* Actually the label the nodes for pointer equivalences  */
-  for (i = 1; i < FIRST_REF_NODE; i++)
-    if (!bitmap_bit_p (si->visited, si->node_mapping[i]))
-      label_visit (graph, si, si->node_mapping[i]);
-
-  /* Calculate location equivalence labels.  */
-  for (i = 1; i < FIRST_REF_NODE; i++)
-    {
-      bitmap pointed_by;
-      bitmap_iterator bi;
-      unsigned int j;
-
-      if (!graph->pointed_by[i])
-	continue;
-      pointed_by = BITMAP_ALLOC (&iteration_obstack);
-
-      /* Translate the pointed-by mapping for pointer equivalence
-	 labels.  */
-      EXECUTE_IF_SET_IN_BITMAP (graph->pointed_by[i], 0, j, bi)
-	{
-	  bitmap_set_bit (pointed_by,
-			  graph->pointer_label[si->node_mapping[j]]);
-	}
-      /* The original pointed_by is now dead.  */
-      BITMAP_FREE (graph->pointed_by[i]);
-
-      /* Look up the location equivalence label if one exists, or make
-	 one otherwise.  */
-      equiv_class_label_t ecl;
-      ecl = equiv_class_lookup_or_add (location_equiv_class_table, pointed_by);
-      if (ecl->equivalence_class == 0)
-	ecl->equivalence_class = location_equiv_class++;
-      else
-	{
-	  if (dump_file && (dump_flags & TDF_DETAILS))
-	    fprintf (dump_file, "Found location equivalence for node %s\n",
-		     get_varinfo (i)->name);
-	  BITMAP_FREE (pointed_by);
-	}
-      graph->loc_label[i] = ecl->equivalence_class;
-
-    }
-
-  if (dump_file && (dump_flags & TDF_DETAILS))
-    for (i = 1; i < FIRST_REF_NODE; i++)
-      {
-	unsigned j = si->node_mapping[i];
-	if (j != i)
-	  {
-	    fprintf (dump_file, "%s node id %d ",
-		     bitmap_bit_p (graph->direct_nodes, i)
-		     ? "Direct" : "Indirect", i);
-	    if (i < FIRST_REF_NODE)
-	      fprintf (dump_file, "\"%s\"", get_varinfo (i)->name);
-	    else
-	      fprintf (dump_file, "\"*%s\"",
-		       get_varinfo (i - FIRST_REF_NODE)->name);
-	    fprintf (dump_file, " mapped to SCC leader node id %d ", j);
-	    if (j < FIRST_REF_NODE)
-	      fprintf (dump_file, "\"%s\"\n", get_varinfo (j)->name);
-	    else
-	      fprintf (dump_file, "\"*%s\"\n",
-		       get_varinfo (j - FIRST_REF_NODE)->name);
-	  }
-	else
-	  {
-	    fprintf (dump_file,
-		     "Equivalence classes for %s node id %d ",
-		     bitmap_bit_p (graph->direct_nodes, i)
-		     ? "direct" : "indirect", i);
-	    if (i < FIRST_REF_NODE)
-	      fprintf (dump_file, "\"%s\"", get_varinfo (i)->name);
-	    else
-	      fprintf (dump_file, "\"*%s\"",
-		       get_varinfo (i - FIRST_REF_NODE)->name);
-	    fprintf (dump_file,
-		     ": pointer %d, location %d\n",
-		     graph->pointer_label[i], graph->loc_label[i]);
-	  }
-      }
-
-  /* Quickly eliminate our non-pointer variables.  */
-
-  for (i = 1; i < FIRST_REF_NODE; i++)
-    {
-      unsigned int node = si->node_mapping[i];
-
-      if (graph->pointer_label[node] == 0)
-	{
-	  if (dump_file && (dump_flags & TDF_DETAILS))
-	    fprintf (dump_file,
-		     "%s is a non-pointer variable, eliminating edges.\n",
-		     get_varinfo (node)->name);
-	  stats.nonpointer_vars++;
-	  clear_edges_for_node (graph, node);
-	}
-    }
-
-  return si;
-}
-
-/* Free information that was only necessary for variable
-   substitution.  */
-
-static void
-free_var_substitution_info (class scc_info *si)
-{
-  delete si;
-  free (graph->pointer_label);
-  free (graph->loc_label);
-  free (graph->pointed_by);
-  free (graph->points_to);
-  free (graph->eq_rep);
-  sbitmap_free (graph->direct_nodes);
-  delete pointer_equiv_class_table;
-  pointer_equiv_class_table = NULL;
-  delete location_equiv_class_table;
-  location_equiv_class_table = NULL;
-  obstack_free (&equiv_class_obstack, NULL);
-  bitmap_obstack_release (&iteration_obstack);
-}
-
-/* Return an existing node that is equivalent to NODE, which has
-   equivalence class LABEL, if one exists.  Return NODE otherwise.  */
-
-static unsigned int
-find_equivalent_node (constraint_graph_t graph,
-		      unsigned int node, unsigned int label)
-{
-  /* If the address version of this variable is unused, we can
-     substitute it for anything else with the same label.
-     Otherwise, we know the pointers are equivalent, but not the
-     locations, and we can unite them later.  */
-
-  if (!bitmap_bit_p (graph->address_taken, node))
-    {
-      gcc_checking_assert (label < graph->size);
-
-      if (graph->eq_rep[label] != -1)
-	{
-	  /* Unify the two variables since we know they are equivalent.  */
-	  if (unite (graph->eq_rep[label], node))
-	    unify_nodes (graph, graph->eq_rep[label], node, false);
-	  return graph->eq_rep[label];
-	}
-      else
-	{
-	  graph->eq_rep[label] = node;
-	  graph->pe_rep[label] = node;
-	}
-    }
-  else
-    {
-      gcc_checking_assert (label < graph->size);
-      graph->pe[node] = label;
-      if (graph->pe_rep[label] == -1)
-	graph->pe_rep[label] = node;
-    }
-
-  return node;
-}
-
-/* Unite pointer equivalent but not location equivalent nodes in
-   GRAPH.  This may only be performed once variable substitution is
-   finished.  */
-
-static void
-unite_pointer_equivalences (constraint_graph_t graph)
-{
-  unsigned int i;
-
-  /* Go through the pointer equivalences and unite them to their
-     representative, if they aren't already.  */
-  for (i = 1; i < FIRST_REF_NODE; i++)
-    {
-      unsigned int label = graph->pe[i];
-      if (label)
-	{
-	  int label_rep = graph->pe_rep[label];
-
-	  if (label_rep == -1)
-	    continue;
-
-	  label_rep = find (label_rep);
-	  if (label_rep >= 0 && unite (label_rep, find (i)))
-	    unify_nodes (graph, label_rep, i, false);
-	}
-    }
-}
-
-/* Move complex constraints to the GRAPH nodes they belong to.  */
-
-static void
-move_complex_constraints (constraint_graph_t graph)
-{
-  int i;
-  constraint_t c;
-
-  FOR_EACH_VEC_ELT (constraints, i, c)
-    {
-      if (c)
-	{
-	  struct constraint_expr lhs = c->lhs;
-	  struct constraint_expr rhs = c->rhs;
-
-	  if (lhs.type == DEREF)
-	    {
-	      insert_into_complex (graph, lhs.var, c);
-	    }
-	  else if (rhs.type == DEREF)
-	    {
-	      if (!(get_varinfo (lhs.var)->is_special_var))
-		insert_into_complex (graph, rhs.var, c);
-	    }
-	  else if (rhs.type != ADDRESSOF && lhs.var > anything_id
-		   && (lhs.offset != 0 || rhs.offset != 0))
-	    {
-	      insert_into_complex (graph, rhs.var, c);
-	    }
-	}
-    }
-}
-
-
-/* Optimize and rewrite complex constraints while performing
-   collapsing of equivalent nodes.  SI is the SCC_INFO that is the
-   result of perform_variable_substitution.  */
-
-static void
-rewrite_constraints (constraint_graph_t graph,
-		     class scc_info *si)
-{
-  int i;
-  constraint_t c;
-
-  if (flag_checking)
-    {
-      for (unsigned int j = 0; j < graph->size; j++)
-	gcc_assert (find (j) == j);
-    }
-
-  FOR_EACH_VEC_ELT (constraints, i, c)
-    {
-      struct constraint_expr lhs = c->lhs;
-      struct constraint_expr rhs = c->rhs;
-      unsigned int lhsvar = find (lhs.var);
-      unsigned int rhsvar = find (rhs.var);
-      unsigned int lhsnode, rhsnode;
-      unsigned int lhslabel, rhslabel;
-
-      lhsnode = si->node_mapping[lhsvar];
-      rhsnode = si->node_mapping[rhsvar];
-      lhslabel = graph->pointer_label[lhsnode];
-      rhslabel = graph->pointer_label[rhsnode];
-
-      /* See if it is really a non-pointer variable, and if so, ignore
-	 the constraint.  */
-      if (lhslabel == 0)
-	{
-	  if (dump_file && (dump_flags & TDF_DETAILS))
-	    {
-
-	      fprintf (dump_file, "%s is a non-pointer variable, "
-		       "ignoring constraint:",
-		       get_varinfo (lhs.var)->name);
-	      dump_constraint (dump_file, c);
-	      fprintf (dump_file, "\n");
-	    }
-	  constraints[i] = NULL;
-	  continue;
-	}
-
-      if (rhslabel == 0)
-	{
-	  if (dump_file && (dump_flags & TDF_DETAILS))
-	    {
-
-	      fprintf (dump_file, "%s is a non-pointer variable, "
-		       "ignoring constraint:",
-		       get_varinfo (rhs.var)->name);
-	      dump_constraint (dump_file, c);
-	      fprintf (dump_file, "\n");
-	    }
-	  constraints[i] = NULL;
-	  continue;
-	}
-
-      lhsvar = find_equivalent_node (graph, lhsvar, lhslabel);
-      rhsvar = find_equivalent_node (graph, rhsvar, rhslabel);
-      c->lhs.var = lhsvar;
-      c->rhs.var = rhsvar;
-    }
-}
-
-/* Eliminate indirect cycles involving NODE.  Return true if NODE was
-   part of an SCC, false otherwise.  */
-
-static bool
-eliminate_indirect_cycles (unsigned int node)
-{
-  if (graph->indirect_cycles[node] != -1
-      && !bitmap_empty_p (get_varinfo (node)->solution))
-    {
-      unsigned int i;
-      auto_vec<unsigned> queue;
-      int queuepos;
-      unsigned int to = find (graph->indirect_cycles[node]);
-      bitmap_iterator bi;
-
-      /* We can't touch the solution set and call unify_nodes
-	 at the same time, because unify_nodes is going to do
-	 bitmap unions into it. */
-
-      EXECUTE_IF_SET_IN_BITMAP (get_varinfo (node)->solution, 0, i, bi)
-	{
-	  if (find (i) == i && i != to)
-	    {
-	      if (unite (to, i))
-		queue.safe_push (i);
-	    }
-	}
-
-      for (queuepos = 0;
-	   queue.iterate (queuepos, &i);
-	   queuepos++)
-	{
-	  unify_nodes (graph, to, i, true);
-	}
-      return true;
-    }
-  return false;
-}
-
-/* Solve the constraint graph GRAPH using our worklist solver.
-   This is based on the PW* family of solvers from the "Efficient Field
-   Sensitive Pointer Analysis for C" paper.
-   It works by iterating over all the graph nodes, processing the complex
-   constraints and propagating the copy constraints, until everything stops
-   changed.  This corresponds to steps 6-8 in the solving list given above.  */
-
-static void
-solve_graph (constraint_graph_t graph)
-{
-  unsigned int size = graph->size;
-  unsigned int i;
-  bitmap pts;
-
-  changed = BITMAP_ALLOC (NULL);
-
-  /* Mark all initial non-collapsed nodes as changed.  */
-  for (i = 1; i < size; i++)
-    {
-      varinfo_t ivi = get_varinfo (i);
-      if (find (i) == i && !bitmap_empty_p (ivi->solution)
-	  && ((graph->succs[i] && !bitmap_empty_p (graph->succs[i]))
-	      || graph->complex[i].length () > 0))
-	bitmap_set_bit (changed, i);
-    }
-
-  /* Allocate a bitmap to be used to store the changed bits.  */
-  pts = BITMAP_ALLOC (&pta_obstack);
-
-  while (!bitmap_empty_p (changed))
-    {
-      unsigned int i;
-      stats.iterations++;
-
-      bitmap_obstack_initialize (&iteration_obstack);
-
-      auto_vec<unsigned> topo_order = compute_topo_order (graph);
-      while (topo_order.length () != 0)
-	{
-	  i = topo_order.pop ();
-
-	  /* If this variable is not a representative, skip it.  */
-	  if (find (i) != i)
-	    continue;
-
-	  /* In certain indirect cycle cases, we may merge this
-	     variable to another.  */
-	  if (eliminate_indirect_cycles (i) && find (i) != i)
-	    continue;
-
-	  /* If the node has changed, we need to process the
-	     complex constraints and outgoing edges again.  For complex
-	     constraints that modify i itself, like the common group of
-	       callarg = callarg + UNKNOWN;
-	       callarg = *callarg + UNKNOWN;
-	       *callarg = callescape;
-	     make sure to iterate immediately because that maximizes
-	     cache reuse and expands the graph quickest, leading to
-	     better visitation order in the next iteration.  */
-	  while (bitmap_clear_bit (changed, i))
-	    {
-	      bitmap solution;
-	      vec<constraint_t> &complex = graph->complex[i];
-	      varinfo_t vi = get_varinfo (i);
-	      bool solution_empty;
-
-	      /* Compute the changed set of solution bits.  If anything
-	         is in the solution just propagate that.  */
-	      if (bitmap_bit_p (vi->solution, anything_id))
-		{
-		  /* If anything is also in the old solution there is
-		     nothing to do.
-		     ???  But we shouldn't ended up with "changed" set ...  */
-		  if (vi->oldsolution
-		      && bitmap_bit_p (vi->oldsolution, anything_id))
-		    break;
-		  bitmap_copy (pts, get_varinfo (find (anything_id))->solution);
-		}
-	      else if (vi->oldsolution)
-		bitmap_and_compl (pts, vi->solution, vi->oldsolution);
-	      else
-		bitmap_copy (pts, vi->solution);
-
-	      if (bitmap_empty_p (pts))
-		break;
-
-	      if (vi->oldsolution)
-		bitmap_ior_into (vi->oldsolution, pts);
-	      else
-		{
-		  vi->oldsolution = BITMAP_ALLOC (&oldpta_obstack);
-		  bitmap_copy (vi->oldsolution, pts);
-		}
-
-	      solution = vi->solution;
-	      solution_empty = bitmap_empty_p (solution);
-
-	      /* Process the complex constraints */
-	      hash_set<constraint_t> *cvisited = nullptr;
-	      if (flag_checking)
-		cvisited = new hash_set<constraint_t>;
-	      bitmap expanded_pts = NULL;
-	      for (unsigned j = 0; j < complex.length (); ++j)
-		{
-		  constraint_t c = complex[j];
-		  /* At unification time only the directly involved nodes
-		     will get their complex constraints updated.  Update
-		     our complex constraints now but keep the constraint
-		     vector sorted and clear of duplicates.  Also make
-		     sure to evaluate each prevailing constraint only once.  */
-		  unsigned int new_lhs = find (c->lhs.var);
-		  unsigned int new_rhs = find (c->rhs.var);
-		  if (c->lhs.var != new_lhs || c->rhs.var != new_rhs)
-		    {
-		      constraint tem = *c;
-		      tem.lhs.var = new_lhs;
-		      tem.rhs.var = new_rhs;
-		      unsigned int place
-			= complex.lower_bound (&tem, constraint_less);
-		      c->lhs.var = new_lhs;
-		      c->rhs.var = new_rhs;
-		      if (place != j)
-			{
-			  complex.ordered_remove (j);
-			  if (j < place)
-			    --place;
-			  if (place < complex.length ())
-			    {
-			      if (constraint_equal (*complex[place], *c))
-				{
-				  j--;
-				  continue;
-				}
-			      else
-				complex.safe_insert (place, c);
-			    }
-			  else
-			    complex.quick_push (c);
-			  if (place > j)
-			    {
-			      j--;
-			      continue;
-			    }
-			}
-		    }
-
-		  /* The only complex constraint that can change our
-		     solution to non-empty, given an empty solution,
-		     is a constraint where the lhs side is receiving
-		     some set from elsewhere.  */
-		  if (cvisited && cvisited->add (c))
-		    gcc_unreachable ();
-		  if (!solution_empty || c->lhs.type != DEREF)
-		    do_complex_constraint (graph, c, pts, &expanded_pts);
-		}
-	      if (cvisited)
-		{
-		  /* When checking, verify the order of constraints is
-		     maintained and each constraint is evaluated exactly
-		     once.  */
-		  for (unsigned j = 1; j < complex.length (); ++j)
-		    gcc_assert (constraint_less (complex[j-1], complex[j]));
-		  gcc_assert (cvisited->elements () == complex.length ());
-		  delete cvisited;
-		}
-	      BITMAP_FREE (expanded_pts);
-
-	      solution_empty = bitmap_empty_p (solution);
-
-	      if (!solution_empty)
-		{
-		  bitmap_iterator bi;
-		  unsigned eff_escaped_id = find (escaped_id);
-		  unsigned j;
-
-		  /* Propagate solution to all successors.  */
-		  unsigned to_remove = ~0U;
-		  EXECUTE_IF_IN_NONNULL_BITMAP (graph->succs[i],
-						0, j, bi)
-		    {
-		      if (to_remove != ~0U)
-			{
-			  bitmap_clear_bit (graph->succs[i], to_remove);
-			  to_remove = ~0U;
-			}
-		      unsigned int to = find (j);
-		      if (to != j)
-			{
-			  /* Update the succ graph, avoiding duplicate
-			     work.  */
-			  to_remove = j;
-			  if (! bitmap_set_bit (graph->succs[i], to))
-			    continue;
-			  /* We eventually end up processing 'to' twice
-			     as it is undefined whether bitmap iteration
-			     iterates over bits set during iteration.
-			     Play safe instead of doing tricks.  */
-			}
-		      /* Don't try to propagate to ourselves.  */
-		      if (to == i)
-			{
-			  to_remove = j;
-			  continue;
-			}
-		      /* Early node unification can lead to edges from
-			 escaped - remove them.  */
-		      if (i == eff_escaped_id)
-			{
-			  to_remove = j;
-			  if (bitmap_set_bit (get_varinfo (to)->solution,
-					      escaped_id))
-			    bitmap_set_bit (changed, to);
-			  continue;
-			}
-
-		      if (bitmap_ior_into (get_varinfo (to)->solution, pts))
-			bitmap_set_bit (changed, to);
-		    }
-		  if (to_remove != ~0U)
-		    bitmap_clear_bit (graph->succs[i], to_remove);
-		}
-	    }
-	}
-      bitmap_obstack_release (&iteration_obstack);
-    }
-
-  BITMAP_FREE (pts);
-  BITMAP_FREE (changed);
-  bitmap_obstack_release (&oldpta_obstack);
-}
-
-/* Map from trees to variable infos.  */
-static hash_map<tree, varinfo_t> *vi_for_tree;
-
-
 /* Insert ID as the variable id for tree T in the vi_for_tree map.  */
 
 static void
@@ -3003,7 +734,7 @@ lookup_vi_for_tree (tree t)
   return *slot;
 }
 
-/* Return a printable name for DECL  */
+/* Return a printable name for DECL.  */
 
 static const char *
 alias_get_name (tree decl)
@@ -3191,10 +922,10 @@ process_constraint (constraint_t t)
   if (!get_varinfo (lhs.var)->may_have_pointers)
     return;
 
-  /* This can happen in our IR with things like n->a = *p */
+  /* This can happen in our IR with things like n->a = *p.  */
   if (rhs.type == DEREF && lhs.type == DEREF && rhs.var != anything_id)
     {
-      /* Split into tmp = *rhs, *lhs = tmp */
+      /* Split into tmp = *rhs, *lhs = tmp.  */
       struct constraint_expr tmplhs;
       tmplhs = new_scalar_tmp_constraint_exp ("doubledereftmp", true);
       process_constraint (new_constraint (tmplhs, rhs));
@@ -3202,7 +933,7 @@ process_constraint (constraint_t t)
     }
   else if ((rhs.type != SCALAR || rhs.offset != 0) && lhs.type == DEREF)
     {
-      /* Split into tmp = &rhs, *lhs = tmp */
+      /* Split into tmp = &rhs, *lhs = tmp.  */
       struct constraint_expr tmplhs;
       tmplhs = new_scalar_tmp_constraint_exp ("derefaddrtmp", true);
       process_constraint (new_constraint (tmplhs, rhs));
@@ -3370,7 +1101,7 @@ get_constraint_for_component_ref (tree t, vec<ce_s> *results,
   tree forzero;
 
   /* Some people like to do cute things like take the address of
-     &0->a.b */
+     &0->a.b.  */
   forzero = t;
   while (handled_component_p (forzero)
 	 || INDIRECT_REF_P (forzero)
@@ -3444,7 +1175,7 @@ get_constraint_for_component_ref (tree t, vec<ce_s> *results,
     {
       /* In languages like C, you can access one past the end of an
 	 array.  You aren't allowed to dereference it, so we can
-	 ignore this constraint. When we handle pointer subtraction,
+	 ignore this constraint.  When we handle pointer subtraction,
 	 we may have to do something cute here.  */
 
       if (maybe_lt (poly_uint64 (bitpos), get_varinfo (result.var)->fullsize)
@@ -3481,7 +1212,7 @@ get_constraint_for_component_ref (tree t, vec<ce_s> *results,
 	      results->safe_push (cexpr);
 	    }
 	  else if (results->length () == 0)
-	    /* Assert that we found *some* field there. The user couldn't be
+	    /* Assert that we found *some* field there.  The user couldn't be
 	       accessing *only* padding.  */
 	    /* Still the user could access one past the end of an array
 	       embedded in a struct resulting in accessing *only* padding.  */
@@ -3535,7 +1266,7 @@ get_constraint_for_component_ref (tree t, vec<ce_s> *results,
 /* Dereference the constraint expression CONS, and return the result.
    DEREF (ADDRESSOF) = SCALAR
    DEREF (SCALAR) = DEREF
-   DEREF (DEREF) = (temp = DEREF1; result = DEREF(temp))
+   DEREF (DEREF) = (temp = DEREF1; result = DEREF (temp))
    This is needed so that we can handle dereferencing DEREF constraints.  */
 
 static void
@@ -3594,7 +1325,7 @@ get_constraint_for_1 (tree t, vec<ce_s> *results, bool address_p,
      point to anything by itself.  That is, of course, unless it is an
      integer constant being treated as a pointer, in which case, we
      will return that this is really the addressof anything.  This
-     happens below, since it will fall into the default case. The only
+     happens below, since it will fall into the default case.  The only
      case we know something about an integer treated like a pointer is
      when it is the NULL pointer, and then we just say it points to
      NULL.
@@ -3690,7 +1421,10 @@ get_constraint_for_1 (tree t, vec<ce_s> *results, bool address_p,
 		    size = -1;
 		  for (; curr; curr = vi_next (curr))
 		    {
-		      if (curr->offset - vi->offset < size)
+		      /* The start of the access might happen anywhere
+			 within vi, so conservatively assume it was
+			 at its end.  */
+		      if (curr->offset - (vi->offset + vi->size - 1) < size)
 			{
 			  cs.var = curr->id;
 			  results->safe_push (cs);
@@ -3742,7 +1476,7 @@ get_constraint_for_1 (tree t, vec<ce_s> *results, bool address_p,
 		  tmp.truncate (0);
 		}
 	      /* We do not know whether the constructor was complete,
-	         so technically we have to add &NOTHING or &ANYTHING
+		 so technically we have to add &NOTHING or &ANYTHING
 		 like we do for an empty constructor as well.  */
 	      return;
 	    }
@@ -4141,7 +1875,7 @@ get_function_part_constraint (varinfo_t fi, unsigned part)
 /* Produce constraints for argument ARG of call STMT with eaf flags
    FLAGS.  RESULTS is array holding constraints for return value.
    CALLESCAPE_ID is variable where call loocal escapes are added.
-   WRITES_GLOVEL_MEMORY is true if callee may write global memory. */
+   WRITES_GLOVEL_MEMORY is true if callee may write global memory.  */
 
 static void
 handle_call_arg (gcall *stmt, tree arg, vec<ce_s> *results, int flags,
@@ -4507,7 +2241,7 @@ handle_lhs_call (gcall *stmt, tree lhs, int flags, vec<ce_s> &rhsc,
       rhsc.truncate (0);
       vi = make_heapvar ("HEAP", true);
       /* We are marking allocated storage local, we deal with it becoming
-         global by escaping and setting of vars_contains_escaped_heap.  */
+	 global by escaping and setting of vars_contains_escaped_heap.  */
       DECL_EXTERNAL (vi->decl) = 0;
       vi->is_global_var = 0;
       /* If this is not a real malloc call assume the memory was
@@ -4695,8 +2429,8 @@ find_func_aliases_for_builtin_call (struct function *fn, gcall *t)
 	}
       case BUILT_IN_STACK_SAVE:
       case BUILT_IN_STACK_RESTORE:
-        /* Nothing interesting happens.  */
-        return true;
+	/* Nothing interesting happens.  */
+	return true;
       case BUILT_IN_ALLOCA:
       case BUILT_IN_ALLOCA_WITH_ALIGN:
       case BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX:
@@ -4720,7 +2454,7 @@ find_func_aliases_for_builtin_call (struct function *fn, gcall *t)
 	  return true;
 	}
       case BUILT_IN_POSIX_MEMALIGN:
-        {
+	{
 	  tree ptrptr = gimple_call_arg (t, 0);
 	  get_constraint_for (ptrptr, &lhsc);
 	  do_deref (&lhsc);
@@ -4803,7 +2537,7 @@ find_func_aliases_for_builtin_call (struct function *fn, gcall *t)
 	  }
 	break;
       /* String / character search functions return a pointer into the
-         source string or NULL.  */
+	 source string or NULL.  */
       case BUILT_IN_INDEX:
       case BUILT_IN_STRCHR:
       case BUILT_IN_STRRCHR:
@@ -4824,7 +2558,7 @@ find_func_aliases_for_builtin_call (struct function *fn, gcall *t)
 	  }
 	return true;
       /* Pure functions that return something not based on any object and
-         that use the memory pointed to by their arguments (but not
+	 that use the memory pointed to by their arguments (but not
 	 transitively).  */
       case BUILT_IN_STRCMP:
       case BUILT_IN_STRCMP_EQ:
@@ -5189,7 +2923,7 @@ find_func_aliases (struct function *fn, gimple *origt)
 	}
     }
   /* In IPA mode, we need to generate constraints to pass call
-     arguments through their calls.   There are two cases,
+     arguments through their calls.  There are two cases,
      either a GIMPLE_CALL returning a value, or just a plain
      GIMPLE_CALL when we are not.
 
@@ -5334,7 +3068,7 @@ find_func_aliases (struct function *fn, gimple *origt)
 	  constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
 	  oconstraints[i] = constraint;
 	  parse_output_constraint (&constraint, i, 0, 0, &allows_mem,
-				   &allows_reg, &is_inout);
+				   &allows_reg, &is_inout, nullptr);
 
 	  /* A memory constraint makes the address of the operand escape.  */
 	  if (!allows_reg && allows_mem)
@@ -5367,7 +3101,7 @@ find_func_aliases (struct function *fn, gimple *origt)
 	  constraint = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (link)));
 
 	  parse_input_constraint (&constraint, 0, 0, noutputs, 0, oconstraints,
-				  &allows_mem, &allows_reg);
+				  &allows_mem, &allows_reg, nullptr);
 
 	  /* A memory constraint makes the address of the operand escape.  */
 	  if (!allows_reg && allows_mem)
@@ -5442,7 +3176,7 @@ find_func_clobbers (struct function *fn, gimple *origt)
 	  || (TREE_CODE (tem) == MEM_REF
 	      && !(TREE_CODE (TREE_OPERAND (tem, 0)) == ADDR_EXPR
 		   && auto_var_in_fn_p
-		        (TREE_OPERAND (TREE_OPERAND (tem, 0), 0), fn->decl))))
+			(TREE_OPERAND (TREE_OPERAND (tem, 0), 0), fn->decl))))
 	{
 	  struct constraint_expr lhsc, *rhsp;
 	  unsigned i;
@@ -5471,7 +3205,7 @@ find_func_clobbers (struct function *fn, gimple *origt)
 	  || (TREE_CODE (tem) == MEM_REF
 	      && !(TREE_CODE (TREE_OPERAND (tem, 0)) == ADDR_EXPR
 		   && auto_var_in_fn_p
-		        (TREE_OPERAND (TREE_OPERAND (tem, 0), 0), fn->decl))))
+			(TREE_OPERAND (TREE_OPERAND (tem, 0), 0), fn->decl))))
 	{
 	  struct constraint_expr lhs, *rhsp;
 	  unsigned i;
@@ -5783,65 +3517,6 @@ find_func_clobbers (struct function *fn, gimple *origt)
 }
 
 
-/* Find the first varinfo in the same variable as START that overlaps with
-   OFFSET.  Return NULL if we can't find one.  */
-
-static varinfo_t
-first_vi_for_offset (varinfo_t start, unsigned HOST_WIDE_INT offset)
-{
-  /* If the offset is outside of the variable, bail out.  */
-  if (offset >= start->fullsize)
-    return NULL;
-
-  /* If we cannot reach offset from start, lookup the first field
-     and start from there.  */
-  if (start->offset > offset)
-    start = get_varinfo (start->head);
-
-  while (start)
-    {
-      /* We may not find a variable in the field list with the actual
-	 offset when we have glommed a structure to a variable.
-	 In that case, however, offset should still be within the size
-	 of the variable. */
-      if (offset >= start->offset
-	  && (offset - start->offset) < start->size)
-	return start;
-
-      start = vi_next (start);
-    }
-
-  return NULL;
-}
-
-/* Find the first varinfo in the same variable as START that overlaps with
-   OFFSET.  If there is no such varinfo the varinfo directly preceding
-   OFFSET is returned.  */
-
-static varinfo_t
-first_or_preceding_vi_for_offset (varinfo_t start,
-				  unsigned HOST_WIDE_INT offset)
-{
-  /* If we cannot reach offset from start, lookup the first field
-     and start from there.  */
-  if (start->offset > offset)
-    start = get_varinfo (start->head);
-
-  /* We may not find a variable in the field list with the actual
-     offset when we have glommed a structure to a variable.
-     In that case, however, offset should still be within the size
-     of the variable.
-     If we got beyond the offset we look for return the field
-     directly preceding offset which may be the last field.  */
-  while (start->next
-	 && offset >= start->offset
-	 && !((offset - start->offset) < start->size))
-    start = vi_next (start);
-
-  return start;
-}
-
-
 /* This structure is used during pushing fields onto the fieldstack
    to track the offset of the field, since bitpos_of_field gives it
    relative to its immediate containing type, and we want it relative
@@ -5868,7 +3543,7 @@ struct fieldoff
 typedef struct fieldoff fieldoff_s;
 
 
-/* qsort comparison function for two fieldoff's PA and PB */
+/* qsort comparison function for two fieldoff's PA and PB.  */
 
 static int
 fieldoff_compare (const void *pa, const void *pb)
@@ -6596,38 +4271,6 @@ create_variable_info_for (tree decl, const char *name, bool add_id)
   return id;
 }
 
-/* Print out the points-to solution for VAR to FILE.  */
-
-static void
-dump_solution_for_var (FILE *file, unsigned int var)
-{
-  varinfo_t vi = get_varinfo (var);
-  unsigned int i;
-  bitmap_iterator bi;
-
-  /* Dump the solution for unified vars anyway, this avoids difficulties
-     in scanning dumps in the testsuite.  */
-  fprintf (file, "%s = { ", vi->name);
-  vi = get_varinfo (find (var));
-  EXECUTE_IF_SET_IN_BITMAP (vi->solution, 0, i, bi)
-    fprintf (file, "%s ", get_varinfo (i)->name);
-  fprintf (file, "}");
-
-  /* But note when the variable was unified.  */
-  if (vi->id != var)
-    fprintf (file, " same as %s", vi->name);
-
-  fprintf (file, "\n");
-}
-
-/* Print the points-to solution for VAR to stderr.  */
-
-DEBUG_FUNCTION void
-debug_solution_for_var (unsigned int var)
-{
-  dump_solution_for_var (stderr, var);
-}
-
 /* Register the constraints for function parameter related VI.  */
 
 static void
@@ -6715,7 +4358,7 @@ struct shared_bitmap_hasher : free_ptr_hash <shared_bitmap_info>
 			    const shared_bitmap_info *);
 };
 
-/* Hash function for a shared_bitmap_info_t */
+/* Hash function for a shared_bitmap_info_t.  */
 
 inline hashval_t
 shared_bitmap_hasher::hash (const shared_bitmap_info *bi)
@@ -6723,7 +4366,7 @@ shared_bitmap_hasher::hash (const shared_bitmap_info *bi)
   return bi->hashcode;
 }
 
-/* Equality function for two shared_bitmap_info_t's. */
+/* Equality function for two shared_bitmap_info_t's.  */
 
 inline bool
 shared_bitmap_hasher::equal (const shared_bitmap_info *sbi1,
@@ -6781,8 +4424,8 @@ set_uids_in_ptset (bitmap into, bitmap from, struct pt_solution *pt,
 {
   unsigned int i;
   bitmap_iterator bi;
-  varinfo_t escaped_vi = get_varinfo (find (escaped_id));
-  varinfo_t escaped_return_vi = get_varinfo (find (escaped_return_id));
+  varinfo_t escaped_vi = get_varinfo (var_rep[escaped_id]);
+  varinfo_t escaped_return_vi = get_varinfo (var_rep[escaped_return_id]);
   bool everything_escaped
     = escaped_vi->solution && bitmap_bit_p (escaped_vi->solution, anything_id);
 
@@ -6880,7 +4523,7 @@ find_what_var_points_to (tree fndecl, varinfo_t orig_vi)
 
   /* This variable may have been collapsed, let's get the real
      variable.  */
-  vi = get_varinfo (find (orig_vi->id));
+  vi = get_varinfo (var_rep[orig_vi->id]);
 
   /* See if we have already computed the solution and return it.  */
   pt_solution **slot = &final_solutions->get_or_insert (vi);
@@ -6907,7 +4550,7 @@ find_what_var_points_to (tree fndecl, varinfo_t orig_vi)
 	      else
 		pt->escaped = 1;
 	      /* Expand some special vars of ESCAPED in-place here.  */
-	      varinfo_t evi = get_varinfo (find (escaped_id));
+	      varinfo_t evi = get_varinfo (var_rep[escaped_id]);
 	      if (bitmap_bit_p (evi->solution, nonlocal_id))
 		pt->nonlocal = 1;
 	    }
@@ -7134,7 +4777,7 @@ pt_solution_includes_global (struct pt_solution *pt, bool escaped_local_p)
       || pt->nonlocal
       || pt->vars_contains_nonlocal
       /* The following is a hack to make the malloc escape hack work.
-         In reality we'd need different sets for escaped-through-return
+	 In reality we'd need different sets for escaped-through-return
 	 and escaped-to-callees and passes would need to be updated.  */
       || pt->vars_contains_escaped_heap)
     return true;
@@ -7270,52 +4913,6 @@ pt_solutions_intersect (struct pt_solution *pt1, struct pt_solution *pt2)
   return res;
 }
 
-/* Dump stats information to OUTFILE.  */
-
-static void
-dump_sa_stats (FILE *outfile)
-{
-  fprintf (outfile, "Points-to Stats:\n");
-  fprintf (outfile, "Total vars:               %d\n", stats.total_vars);
-  fprintf (outfile, "Non-pointer vars:          %d\n",
-	   stats.nonpointer_vars);
-  fprintf (outfile, "Statically unified vars:  %d\n",
-	   stats.unified_vars_static);
-  fprintf (outfile, "Dynamically unified vars: %d\n",
-	   stats.unified_vars_dynamic);
-  fprintf (outfile, "Iterations:               %d\n", stats.iterations);
-  fprintf (outfile, "Number of edges:          %d\n", stats.num_edges);
-  fprintf (outfile, "Number of implicit edges: %d\n",
-	   stats.num_implicit_edges);
-  fprintf (outfile, "Number of avoided edges: %d\n",
-	   stats.num_avoided_edges);
-}
-
-/* Dump points-to information to OUTFILE.  */
-
-static void
-dump_sa_points_to_info (FILE *outfile)
-{
-  fprintf (outfile, "\nPoints-to sets\n\n");
-
-  for (unsigned i = 1; i < varmap.length (); i++)
-    {
-      varinfo_t vi = get_varinfo (i);
-      if (!vi->may_have_pointers)
-	continue;
-      dump_solution_for_var (outfile, i);
-    }
-}
-
-
-/* Debug points-to information to stderr.  */
-
-DEBUG_FUNCTION void
-debug_sa_points_to_info (void)
-{
-  dump_sa_points_to_info (stderr);
-}
-
 
 /* Initialize the always-existing constraint variables for NULL
    ANYTHING, READONLY, and INTEGER */
@@ -7508,7 +5105,7 @@ init_base_vars (void)
   process_constraint (new_constraint (lhs, rhs));
 }
 
-/* Initialize things necessary to perform PTA */
+/* Initialize things necessary to perform PTA.  */
 
 static void
 init_alias_vars (void)
@@ -7517,7 +5114,6 @@ init_alias_vars (void)
 
   bitmap_obstack_initialize (&pta_obstack);
   bitmap_obstack_initialize (&oldpta_obstack);
-  bitmap_obstack_initialize (&predbitmap_obstack);
 
   constraints.create (8);
   varmap.create (8);
@@ -7534,144 +5130,6 @@ init_alias_vars (void)
   gcc_obstack_init (&final_solutions_obstack);
 }
 
-/* Remove the REF and ADDRESS edges from GRAPH, as well as all the
-   predecessor edges.  */
-
-static void
-remove_preds_and_fake_succs (constraint_graph_t graph)
-{
-  unsigned int i;
-
-  /* Clear the implicit ref and address nodes from the successor
-     lists.  */
-  for (i = 1; i < FIRST_REF_NODE; i++)
-    {
-      if (graph->succs[i])
-	bitmap_clear_range (graph->succs[i], FIRST_REF_NODE,
-			    FIRST_REF_NODE * 2);
-    }
-
-  /* Free the successor list for the non-ref nodes.  */
-  for (i = FIRST_REF_NODE + 1; i < graph->size; i++)
-    {
-      if (graph->succs[i])
-	BITMAP_FREE (graph->succs[i]);
-    }
-
-  /* Now reallocate the size of the successor list as, and blow away
-     the predecessor bitmaps.  */
-  graph->size = varmap.length ();
-  graph->succs = XRESIZEVEC (bitmap, graph->succs, graph->size);
-
-  free (graph->implicit_preds);
-  graph->implicit_preds = NULL;
-  free (graph->preds);
-  graph->preds = NULL;
-  bitmap_obstack_release (&predbitmap_obstack);
-}
-
-/* Solve the constraint set.  */
-
-static void
-solve_constraints (void)
-{
-  class scc_info *si;
-
-  /* Sort varinfos so that ones that cannot be pointed to are last.
-     This makes bitmaps more efficient.  */
-  unsigned int *map = XNEWVEC (unsigned int, varmap.length ());
-  for (unsigned i = 0; i < integer_id + 1; ++i)
-    map[i] = i;
-  /* Start with address-taken vars, followed by not address-taken vars
-     to move vars never appearing in the points-to solution bitmaps last.  */
-  unsigned j = integer_id + 1;
-  for (unsigned i = integer_id + 1; i < varmap.length (); ++i)
-    if (varmap[varmap[i]->head]->address_taken)
-      map[i] = j++;
-  for (unsigned i = integer_id + 1; i < varmap.length (); ++i)
-    if (! varmap[varmap[i]->head]->address_taken)
-      map[i] = j++;
-  /* Shuffle varmap according to map.  */
-  for (unsigned i = integer_id + 1; i < varmap.length (); ++i)
-    {
-      while (map[varmap[i]->id] != i)
-	std::swap (varmap[i], varmap[map[varmap[i]->id]]);
-      gcc_assert (bitmap_empty_p (varmap[i]->solution));
-      varmap[i]->id = i;
-      varmap[i]->next = map[varmap[i]->next];
-      varmap[i]->head = map[varmap[i]->head];
-    }
-  /* Finally rewrite constraints.  */
-  for (unsigned i = 0; i < constraints.length (); ++i)
-    {
-      constraints[i]->lhs.var = map[constraints[i]->lhs.var];
-      constraints[i]->rhs.var = map[constraints[i]->rhs.var];
-    }
-  free (map);
-
-  if (dump_file)
-    fprintf (dump_file,
-	     "\nCollapsing static cycles and doing variable "
-	     "substitution\n");
-
-  init_graph (varmap.length () * 2);
-
-  if (dump_file)
-    fprintf (dump_file, "Building predecessor graph\n");
-  build_pred_graph ();
-
-  if (dump_file)
-    fprintf (dump_file, "Detecting pointer and location "
-	     "equivalences\n");
-  si = perform_var_substitution (graph);
-
-  if (dump_file)
-    fprintf (dump_file, "Rewriting constraints and unifying "
-	     "variables\n");
-  rewrite_constraints (graph, si);
-
-  build_succ_graph ();
-
-  free_var_substitution_info (si);
-
-  /* Attach complex constraints to graph nodes.  */
-  move_complex_constraints (graph);
-
-  if (dump_file)
-    fprintf (dump_file, "Uniting pointer but not location equivalent "
-	     "variables\n");
-  unite_pointer_equivalences (graph);
-
-  if (dump_file)
-    fprintf (dump_file, "Finding indirect cycles\n");
-  find_indirect_cycles (graph);
-
-  /* Implicit nodes and predecessors are no longer necessary at this
-     point. */
-  remove_preds_and_fake_succs (graph);
-
-  if (dump_file && (dump_flags & TDF_GRAPH))
-    {
-      fprintf (dump_file, "\n\n// The constraint graph before solve-graph "
-	       "in dot format:\n");
-      dump_constraint_graph (dump_file);
-      fprintf (dump_file, "\n\n");
-    }
-
-  if (dump_file)
-    fprintf (dump_file, "Solving graph\n");
-
-  solve_graph (graph);
-
-  if (dump_file && (dump_flags & TDF_GRAPH))
-    {
-      fprintf (dump_file, "\n\n// The constraint graph after solve-graph "
-	       "in dot format:\n");
-      dump_constraint_graph (dump_file);
-      fprintf (dump_file, "\n\n");
-    }
-}
-
 /* Create points-to sets for the current function.  See the comments
    at the start of the file for an algorithmic overview.  */
 
@@ -7842,8 +5300,6 @@ compute_points_to_sets (void)
 static void
 delete_points_to_sets (void)
 {
-  unsigned int i;
-
   delete shared_bitmap_table;
   shared_bitmap_table = NULL;
   if (dump_file && (dump_flags & TDF_STATS))
@@ -7855,16 +5311,7 @@ delete_points_to_sets (void)
   bitmap_obstack_release (&pta_obstack);
   constraints.release ();
 
-  for (i = 0; i < graph->size; i++)
-    graph->complex[i].release ();
-  free (graph->complex);
-
-  free (graph->rep);
-  free (graph->succs);
-  free (graph->pe);
-  free (graph->pe_rep);
-  free (graph->indirect_cycles);
-  free (graph);
+  free (var_rep);
 
   varmap.release ();
   variable_info_pool.release ();
@@ -7911,14 +5358,14 @@ visit_loadstore (gimple *, tree base, tree ref, void *data)
 	  if (! vi)
 	    return false;
 
-	  vi = get_varinfo (find (vi->id));
+	  vi = get_varinfo (var_rep[vi->id]);
 	  if (bitmap_intersect_p (rvars, vi->solution)
 	      || (escaped_p && bitmap_bit_p (vi->solution, escaped_id)))
 	    return false;
 	}
 
       /* Do not overwrite existing cliques (that includes clique, base
-         pairs we just set).  */
+	 pairs we just set).  */
       if (MR_DEPENDENCE_CLIQUE (base) == 0)
 	{
 	  MR_DEPENDENCE_CLIQUE (base) = clique;
@@ -8047,7 +5494,7 @@ compute_dependence_clique (void)
       varinfo_t vi = lookup_vi_for_tree (p);
       if (!vi)
 	continue;
-      vi = get_varinfo (find (vi->id));
+      vi = get_varinfo (var_rep[vi->id]);
       bitmap_iterator bi;
       unsigned j;
       varinfo_t restrict_var = NULL;
@@ -8098,11 +5545,11 @@ compute_dependence_clique (void)
 					      maybe_set_dependence_info);
 	  if (used)
 	    {
-	      /* Add all subvars to the set of restrict pointed-to set. */
+	      /* Add all subvars to the set of restrict pointed-to set.  */
 	      for (unsigned sv = restrict_var->head; sv != 0;
 		   sv = get_varinfo (sv)->next)
 		bitmap_set_bit (rvars, sv);
-	      varinfo_t escaped = get_varinfo (find (escaped_id));
+	      varinfo_t escaped = get_varinfo (var_rep[escaped_id]);
 	      if (bitmap_bit_p (escaped->solution, restrict_var->id))
 		escaped_p = true;
 	    }
@@ -8258,7 +5705,7 @@ struct pt_solution ipa_escaped_pt
   = { true, false, false, false, false, false,
       false, false, false, false, false, NULL };
 
-/* Associate node with varinfo DATA. Worker for
+/* Associate node with varinfo DATA.  Worker for
    cgraph_for_symbol_thunks_and_aliases.  */
 static bool
 associate_varinfo_to_alias (struct cgraph_node *node, void *data)
@@ -8272,111 +5719,6 @@ associate_varinfo_to_alias (struct cgraph_node *node, void *data)
   return false;
 }
 
-/* Dump varinfo VI to FILE.  */
-
-static void
-dump_varinfo (FILE *file, varinfo_t vi)
-{
-  if (vi == NULL)
-    return;
-
-  fprintf (file, "%u: %s\n", vi->id, vi->name);
-
-  const char *sep = " ";
-  if (vi->is_artificial_var)
-    fprintf (file, "%sartificial", sep);
-  if (vi->is_special_var)
-    fprintf (file, "%sspecial", sep);
-  if (vi->is_unknown_size_var)
-    fprintf (file, "%sunknown-size", sep);
-  if (vi->is_full_var)
-    fprintf (file, "%sfull", sep);
-  if (vi->is_heap_var)
-    fprintf (file, "%sheap", sep);
-  if (vi->may_have_pointers)
-    fprintf (file, "%smay-have-pointers", sep);
-  if (vi->only_restrict_pointers)
-    fprintf (file, "%sonly-restrict-pointers", sep);
-  if (vi->is_restrict_var)
-    fprintf (file, "%sis-restrict-var", sep);
-  if (vi->is_global_var)
-    fprintf (file, "%sglobal", sep);
-  if (vi->is_ipa_escape_point)
-    fprintf (file, "%sipa-escape-point", sep);
-  if (vi->is_fn_info)
-    fprintf (file, "%sfn-info", sep);
-  if (vi->ruid)
-    fprintf (file, "%srestrict-uid:%u", sep, vi->ruid);
-  if (vi->next)
-    fprintf (file, "%snext:%u", sep, vi->next);
-  if (vi->head != vi->id)
-    fprintf (file, "%shead:%u", sep, vi->head);
-  if (vi->offset)
-    fprintf (file, "%soffset:" HOST_WIDE_INT_PRINT_DEC, sep, vi->offset);
-  if (vi->size != ~HOST_WIDE_INT_0U)
-    fprintf (file, "%ssize:" HOST_WIDE_INT_PRINT_DEC, sep, vi->size);
-  if (vi->fullsize != ~HOST_WIDE_INT_0U && vi->fullsize != vi->size)
-    fprintf (file, "%sfullsize:" HOST_WIDE_INT_PRINT_DEC, sep,
-	     vi->fullsize);
-  fprintf (file, "\n");
-
-  if (vi->solution && !bitmap_empty_p (vi->solution))
-    {
-      bitmap_iterator bi;
-      unsigned i;
-      fprintf (file, " solution: {");
-      EXECUTE_IF_SET_IN_BITMAP (vi->solution, 0, i, bi)
-	fprintf (file, " %u", i);
-      fprintf (file, " }\n");
-    }
-
-  if (vi->oldsolution && !bitmap_empty_p (vi->oldsolution)
-      && !bitmap_equal_p (vi->solution, vi->oldsolution))
-    {
-      bitmap_iterator bi;
-      unsigned i;
-      fprintf (file, " oldsolution: {");
-      EXECUTE_IF_SET_IN_BITMAP (vi->oldsolution, 0, i, bi)
-	fprintf (file, " %u", i);
-      fprintf (file, " }\n");
-    }
-}
-
-/* Dump varinfo VI to stderr.  */
-
-DEBUG_FUNCTION void
-debug_varinfo (varinfo_t vi)
-{
-  dump_varinfo (stderr, vi);
-}
-
-/* Dump varmap to FILE.  */
-
-static void
-dump_varmap (FILE *file)
-{
-  if (varmap.length () == 0)
-    return;
-
-  fprintf (file, "variables:\n");
-
-  for (unsigned int i = 0; i < varmap.length (); ++i)
-    {
-      varinfo_t vi = get_varinfo (i);
-      dump_varinfo (file, vi);
-    }
-
-  fprintf (file, "\n");
-}
-
-/* Dump varmap to stderr.  */
-
-DEBUG_FUNCTION void
-debug_varmap (void)
-{
-  dump_varmap (stderr);
-}
-
 /* Compute whether node is refered to non-locally.  Worker for
    cgraph_for_symbol_thunks_and_aliases.  */
 static bool
@@ -8490,7 +5832,7 @@ ipa_pta_execute (void)
       varinfo_t vi = get_vi_for_tree (var->decl);
 
       /* For the purpose of IPA PTA unit-local globals are not
-         escape points.  */
+	 escape points.  */
       bool nonlocal_p = (DECL_EXTERNAL (var->decl)
 			 || TREE_PUBLIC (var->decl)
 			 || var->used_from_other_partition
@@ -8589,7 +5931,7 @@ ipa_pta_execute (void)
 	for (varinfo_t ai = first_vi_for_offset (fi, fi_parm_base);
 	     ai; ai = vi_next (ai))
 	  {
-	    varinfo_t vi = get_varinfo (find (ai->id));
+	    varinfo_t vi = get_varinfo (var_rep[ai->id]);
 	    bitmap_iterator bi;
 	    unsigned j;
 	    EXECUTE_IF_SET_IN_BITMAP (vi->solution, 0, j, bi)
@@ -8605,12 +5947,12 @@ ipa_pta_execute (void)
 	      }
 	  }
       /* As well as global variables which are another way of passing
-         arguments to recursive invocations.  */
+	 arguments to recursive invocations.  */
       else if (fi->is_global_var)
 	{
 	  for (varinfo_t ai = fi; ai; ai = vi_next (ai))
 	    {
-	      varinfo_t vi = get_varinfo (find (ai->id));
+	      varinfo_t vi = get_varinfo (var_rep[ai->id]);
 	      bitmap_iterator bi;
 	      unsigned j;
 	      EXECUTE_IF_SET_IN_BITMAP (vi->solution, 0, j, bi)
@@ -8705,10 +6047,10 @@ ipa_pta_execute (void)
 		{
 		  *gimple_call_clobber_set (stmt)
 		     = find_what_var_points_to
-		         (node->decl, first_vi_for_offset (fi, fi_clobbers));
+			 (node->decl, first_vi_for_offset (fi, fi_clobbers));
 		  *gimple_call_use_set (stmt)
 		     = find_what_var_points_to
-		         (node->decl, first_vi_for_offset (fi, fi_uses));
+			 (node->decl, first_vi_for_offset (fi, fi_uses));
 		}
 	      /* Handle direct calls to external functions.  */
 	      else if (decl && (!fi || fi->decl))
@@ -8735,7 +6077,8 @@ ipa_pta_execute (void)
 		    }
 
 		  pt = gimple_call_clobber_set (stmt);
-		  if (gimple_call_flags (stmt) & (ECF_CONST|ECF_PURE|ECF_NOVOPS))
+		  if (gimple_call_flags (stmt) &
+		      (ECF_CONST|ECF_PURE|ECF_NOVOPS))
 		    memset (pt, 0, sizeof (struct pt_solution));
 		  else if ((vi = lookup_call_clobber_vi (stmt)) != NULL)
 		    {
@@ -8760,7 +6103,7 @@ ipa_pta_execute (void)
 		{
 		  /* We need to accumulate all clobbers/uses of all possible
 		     callees.  */
-		  fi = get_varinfo (find (fi->id));
+		  fi = get_varinfo (var_rep[fi->id]);
 		  /* If we cannot constrain the set of functions we'll end up
 		     calling we end up using/clobbering everything.  */
 		  if (bitmap_bit_p (fi->solution, anything_id)
@@ -8820,7 +6163,7 @@ ipa_pta_execute (void)
       fn->gimple_df->ipa_pta = true;
 
       /* We have to re-set the final-solution cache after each function
-         because what is a "global" is dependent on function context.  */
+	 because what is a "global" is dependent on function context.  */
       final_solutions->empty ();
       obstack_free (&final_solutions_obstack, NULL);
       gcc_obstack_init (&final_solutions_obstack);
diff --git a/gcc/tree-ssa-structalias.h b/gcc/tree-ssa-structalias.h
new file mode 100644
index 0000000..4104bad
--- /dev/null
+++ b/gcc/tree-ssa-structalias.h
@@ -0,0 +1,217 @@
+/* Tree based points-to analysis
+   Copyright (C) 2005-2025 Free Software Foundation, Inc.
+   Contributed by Daniel Berlin <dberlin@dberlin.org>
+
+   This file is part of GCC.
+
+   GCC is free software; you can redistribute it and/or modify
+   under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   GCC is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GCC; see the file COPYING3.  If not see
+   <http://www.gnu.org/licenses/>.  */
+
+/* NOTE: This file declares the internal interface of the points-to analyzer.
+   Outward-facing function declarations can be found in tree-ssa-alias.h.  */
+
+#ifndef TREE_SSA_STRUCTALIAS_H
+#define TREE_SSA_STRUCTALIAS_H
+
+namespace pointer_analysis {
+
+enum constraint_expr_type {SCALAR, DEREF, ADDRESSOF};
+
+/* Static IDs for the special variables.  Variable ID zero is unused
+   and used as terminator for the sub-variable chain.  */
+enum { nothing_id = 1, anything_id = 2, string_id = 3,
+       escaped_id = 4, nonlocal_id = 5, escaped_return_id = 6,
+       storedanything_id = 7, integer_id = 8 };
+
+/* Use 0x8000... as special unknown offset.  */
+#define UNKNOWN_OFFSET HOST_WIDE_INT_MIN
+
+/* An expression that appears in a constraint.  */
+
+struct constraint_expr
+{
+  /* Constraint type.  */
+  constraint_expr_type type;
+
+  /* Variable we are referring to in the constraint.  */
+  unsigned int var;
+
+  /* Offset, in bits, of this constraint from the beginning of
+     variables it ends up referring to.
+
+     IOW, in a deref constraint, we would deref, get the result set,
+     then add OFFSET to each member.  */
+  HOST_WIDE_INT offset;
+};
+typedef struct constraint_expr ce_s;
+
+/* Our set constraints are made up of two constraint expressions, one
+   LHS, and one RHS.
+
+   As described in the introduction in tree-ssa-structalias.cc, our set
+   constraints each represent an operation between set valued variables.
+*/
+struct constraint
+{
+  struct constraint_expr lhs;
+  struct constraint_expr rhs;
+};
+typedef struct constraint *constraint_t;
+
+struct variable_info
+{
+  /* ID of this variable.  */
+  unsigned int id;
+
+  /* True if this is a variable created by the constraint analysis, such as
+     heap variables and constraints we had to break up.  */
+  unsigned int is_artificial_var : 1;
+
+  /* True if this is a special variable whose solution set should not be
+     changed.  */
+  unsigned int is_special_var : 1;
+
+  /* True for variables whose size is not known or variable.  */
+  unsigned int is_unknown_size_var : 1;
+
+  /* True for (sub-)fields that represent a whole variable.  */
+  unsigned int is_full_var : 1;
+
+  /* True if this is a heap variable.  */
+  unsigned int is_heap_var : 1;
+
+  /* True if this is a register variable.  */
+  unsigned int is_reg_var : 1;
+
+  /* True if this field may contain pointers.  */
+  unsigned int may_have_pointers : 1;
+
+  /* True if this field has only restrict qualified pointers.  */
+  unsigned int only_restrict_pointers : 1;
+
+  /* True if this represents a heap var created for a restrict qualified
+     pointer.  */
+  unsigned int is_restrict_var : 1;
+
+  /* True if this represents a global variable.  */
+  unsigned int is_global_var : 1;
+
+  /* True if this represents a module escape point for IPA analysis.  */
+  unsigned int is_ipa_escape_point : 1;
+
+  /* True if this represents a IPA function info.  */
+  unsigned int is_fn_info : 1;
+
+  /* True if this appears as RHS in a ADDRESSOF constraint.  */
+  unsigned int address_taken : 1;
+
+  /* ???  Store somewhere better.  */
+  unsigned short ruid;
+
+  /* The ID of the variable for the next field in this structure
+     or zero for the last field in this structure.  */
+  unsigned next;
+
+  /* The ID of the variable for the first field in this structure.  */
+  unsigned head;
+
+  /* Offset of this variable, in bits, from the base variable.  */
+  unsigned HOST_WIDE_INT offset;
+
+  /* Size of the variable, in bits.  */
+  unsigned HOST_WIDE_INT size;
+
+  /* Full size of the base variable, in bits.  */
+  unsigned HOST_WIDE_INT fullsize;
+
+  /* In IPA mode the shadow UID in case the variable needs to be duplicated in
+     the final points-to solution because it reaches its containing
+     function recursively.  Zero if none is needed.  */
+  unsigned int shadow_var_uid;
+
+  /* Name of this variable.  */
+  const char *name;
+
+  /* Tree that this variable is associated with.  */
+  tree decl;
+
+  /* Points-to set for this variable.  */
+  bitmap solution;
+
+  /* Old points-to set for this variable.  */
+  bitmap oldsolution;
+};
+typedef struct variable_info *varinfo_t;
+
+struct constraint_stats
+{
+  unsigned int total_vars;
+  unsigned int nonpointer_vars;
+  unsigned int unified_vars_static;
+  unsigned int unified_vars_dynamic;
+  unsigned int iterations;
+  unsigned int num_edges;
+  unsigned int num_implicit_edges;
+  unsigned int num_avoided_edges;
+  unsigned int points_to_sets_created;
+};
+
+extern struct constraint_stats stats;
+
+extern bitmap_obstack pta_obstack;
+extern bitmap_obstack oldpta_obstack;
+
+extern vec<varinfo_t> varmap;
+extern vec<constraint_t> constraints;
+extern unsigned int *var_rep;
+
+
+/* Return the varmap element N.  */
+
+inline varinfo_t
+get_varinfo (unsigned int n)
+{
+  return varmap[n];
+}
+
+/* Return the next variable in the list of sub-variables of VI
+   or NULL if VI is the last sub-variable.  */
+
+inline varinfo_t
+vi_next (varinfo_t vi)
+{
+  return get_varinfo (vi->next);
+}
+
+varinfo_t first_vi_for_offset (varinfo_t start,
+			       unsigned HOST_WIDE_INT offset);
+varinfo_t first_or_preceding_vi_for_offset (varinfo_t start,
+					    unsigned HOST_WIDE_INT offset);
+void dump_constraint (FILE *file, constraint_t c);
+void dump_constraints (FILE *file, int from);
+void dump_solution_for_var (FILE *file, unsigned int var);
+void dump_sa_stats (FILE *outfile);
+void dump_sa_points_to_info (FILE *outfile);
+void dump_varinfo (FILE *file, varinfo_t vi);
+void dump_varmap (FILE *file);
+void debug_constraint (constraint_t);
+void debug_constraints (void);
+void debug_solution_for_var (unsigned int);
+void debug_sa_points_to_info (void);
+void debug_varinfo (varinfo_t);
+void debug_varmap (void);
+
+} // namespace pointer_analysis
+
+#endif /* TREE_SSA_STRUCTALIAS_H  */
diff --git a/gcc/tree-switch-conversion.cc b/gcc/tree-switch-conversion.cc
index d088287..04b357f 100644
--- a/gcc/tree-switch-conversion.cc
+++ b/gcc/tree-switch-conversion.cc
@@ -55,6 +55,7 @@ Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA
 #include "hwint.h"
 #include "internal-fn.h"
 #include "diagnostic-core.h"
+#include "output.h"
 
 /* ??? For lang_hooks.types.type_for_mode, but is there a word_mode
    type in the GIMPLE type system that is language-independent?  */
@@ -1033,6 +1034,16 @@ switch_conversion::build_one_array (int num, tree arr_index_type,
       /* The decl is mergable since we don't take the address ever and
 	 just reading from it. */
       DECL_MERGEABLE (decl) = 1;
+
+      /* Increase the alignments as needed. */
+      if (tree_to_uhwi (DECL_SIZE (decl)) > DECL_ALIGN (decl))
+	{
+	  unsigned HOST_WIDE_INT s = tree_to_uhwi (DECL_SIZE (decl));
+	  /* Only support up to the max supported for merging. */
+	  if (s <= MAX_ALIGN_MERGABLE)
+	    SET_DECL_ALIGN (decl, HOST_WIDE_INT_1U << ceil_log2 (s));
+	}
+
       if (offloading_function_p (cfun->decl))
 	DECL_ATTRIBUTES (decl)
 	  = tree_cons (get_identifier ("omp declare target"), NULL_TREE,
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index ee040eb..da700cd 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -1501,10 +1501,17 @@ vect_compute_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info,
       /* We can only use base and misalignment information relative to
 	 an innermost loop if the misalignment stays the same throughout the
 	 execution of the loop.  As above, this is the case if the stride of
-	 the dataref evenly divides by the alignment.  */
-      poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
-      step_preserves_misalignment_p
-	= multiple_p (drb->step_alignment * vf, vect_align_c);
+	 the dataref evenly divides by the alignment.  Make sure to check
+	 previous epilogues and the main loop.  */
+      step_preserves_misalignment_p = true;
+      auto lvinfo = loop_vinfo;
+      while (lvinfo)
+	{
+	  poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (lvinfo);
+	  step_preserves_misalignment_p
+	    &= multiple_p (drb->step_alignment * vf, vect_align_c);
+	  lvinfo = LOOP_VINFO_ORIG_LOOP_INFO (lvinfo);
+	}
 
       if (!step_preserves_misalignment_p && dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -1571,6 +1578,7 @@ vect_compute_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info,
       unsigned int max_alignment;
       tree base = get_base_for_alignment (drb->base_address, &max_alignment);
       if (max_alignment < vect_align_c
+	  || (loop_vinfo && LOOP_VINFO_EPILOGUE_P (loop_vinfo))
 	  || !vect_can_force_dr_alignment_p (base,
 					     vect_align_c * BITS_PER_UNIT))
 	{
@@ -2910,12 +2918,14 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
      2) there is at least one unsupported misaligned data ref with an unknown
         misalignment, and
      3) all misaligned data refs with a known misalignment are supported, and
-     4) the number of runtime alignment checks is within reason.  */
+     4) the number of runtime alignment checks is within reason.
+     5) the vectorization factor is a constant.  */
 
   do_versioning
     = (optimize_loop_nest_for_speed_p (loop)
        && !loop->inner /* FORNOW */
-       && loop_cost_model (loop) > VECT_COST_MODEL_CHEAP);
+       && loop_cost_model (loop) > VECT_COST_MODEL_CHEAP)
+       && LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant ();
 
   if (do_versioning)
     {
@@ -2956,17 +2966,6 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
                   break;
                 }
 
-	      /* At present we don't support versioning for alignment
-		 with variable VF, since there's no guarantee that the
-		 VF is a power of two.  We could relax this if we added
-		 a way of enforcing a power-of-two size.  */
-	      unsigned HOST_WIDE_INT size;
-	      if (!GET_MODE_SIZE (TYPE_MODE (vectype)).is_constant (&size))
-		{
-		  do_versioning = false;
-		  break;
-		}
-
 	      /* Forcing alignment in the first iteration is no good if
 		 we don't keep it across iterations.  For now, just disable
 		 versioning in this case.
@@ -2985,7 +2984,8 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
                  Construct the mask needed for this test.  For example,
                  GET_MODE_SIZE for the vector mode V4SI is 16 bytes so the
                  mask must be 15 = 0xf. */
-	      int mask = size - 1;
+	      gcc_assert (DR_TARGET_ALIGNMENT (dr_info).is_constant ());
+	      int mask = DR_TARGET_ALIGNMENT (dr_info).to_constant () - 1;
 
 	      /* FORNOW: use the same mask to test all potentially unaligned
 		 references in the loop.  */
@@ -3948,8 +3948,7 @@ vect_vfa_access_size (vec_info *vinfo, dr_vec_info *dr_info)
     }
   tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
   int misalignment;
-  if (STMT_VINFO_VEC_STMTS (stmt_vinfo).exists ()
-      && ((misalignment = dr_misalignment (dr_info, vectype)), true)
+  if (((misalignment = dr_misalignment (dr_info, vectype)), true)
       && (vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment)
 	  == dr_explicit_realign_optimized))
     {
@@ -4531,10 +4530,13 @@ vect_describe_gather_scatter_call (stmt_vec_info stmt_info,
   info->ifn = gimple_call_internal_fn (call);
   info->decl = NULL_TREE;
   info->base = gimple_call_arg (call, 0);
-  info->offset = gimple_call_arg (call, 1);
-  info->offset_dt = vect_unknown_def_type;
+  info->alias_ptr = gimple_call_arg
+		     (call, internal_fn_alias_ptr_index (info->ifn));
+  info->offset = gimple_call_arg
+		  (call, internal_fn_offset_index (info->ifn));
   info->offset_vectype = NULL_TREE;
-  info->scale = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
+  info->scale = TREE_INT_CST_LOW (gimple_call_arg
+				  (call, internal_fn_scale_index (info->ifn)));
   info->element_type = TREE_TYPE (vectype);
   info->memory_type = TREE_TYPE (DR_REF (dr));
 }
@@ -4659,26 +4661,19 @@ vect_check_gather_scatter (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
   if (off == NULL_TREE)
     off = size_zero_node;
 
-  /* If base is not loop invariant, either off is 0, then we start with just
-     the constant offset in the loop invariant BASE and continue with base
-     as OFF, otherwise give up.
-     We could handle that case by gimplifying the addition of base + off
-     into some SSA_NAME and use that as off, but for now punt.  */
+  /* BASE must be loop invariant.  If it is not invariant, but OFF is, then we
+   * can fix that by swapping BASE and OFF.  */
   if (!expr_invariant_in_loop_p (loop, base))
     {
-      if (!integer_zerop (off))
+      if (!expr_invariant_in_loop_p (loop, off))
 	return false;
-      off = base;
-      base = size_int (pbytepos);
-    }
-  /* Otherwise put base + constant offset into the loop invariant BASE
-     and continue with OFF.  */
-  else
-    {
-      base = fold_convert (sizetype, base);
-      base = size_binop (PLUS_EXPR, base, size_int (pbytepos));
+
+      std::swap (base, off);
     }
 
+  base = fold_convert (sizetype, base);
+  base = size_binop (PLUS_EXPR, base, size_int (pbytepos));
+
   /* OFF at this point may be either a SSA_NAME or some tree expression
      from get_inner_reference.  Try to peel off loop invariants from it
      into BASE as long as possible.  */
@@ -4856,11 +4851,18 @@ vect_check_gather_scatter (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
       offset_vectype = NULL_TREE;
     }
 
+  gcc_checking_assert (expr_invariant_in_loop_p (loop, base));
+  gcc_checking_assert (!expr_invariant_in_loop_p (loop, off));
+
   info->ifn = ifn;
   info->decl = decl;
   info->base = base;
+
+  info->alias_ptr = build_int_cst
+    (reference_alias_ptr_type (DR_REF (dr)),
+     get_object_alignment (DR_REF (dr)));
+
   info->offset = off;
-  info->offset_dt = vect_unknown_def_type;
   info->offset_vectype = offset_vectype;
   info->scale = scale;
   info->element_type = TREE_TYPE (vectype);
@@ -5048,7 +5050,7 @@ vect_find_stmt_data_reference (loop_p loop, gimple *stmt,
 */
 
 opt_result
-vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf, bool *fatal)
+vect_analyze_data_refs (vec_info *vinfo, bool *fatal)
 {
   class loop *loop = NULL;
   unsigned int i;
@@ -5067,7 +5069,6 @@ vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf, bool *fatal)
   FOR_EACH_VEC_ELT (datarefs, i, dr)
     {
       enum { SG_NONE, GATHER, SCATTER } gatherscatter = SG_NONE;
-      poly_uint64 vf;
 
       gcc_assert (DR_REF (dr));
       stmt_vec_info stmt_info = vinfo->lookup_stmt (DR_STMT (dr));
@@ -5259,11 +5260,6 @@ vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf, bool *fatal)
 			     stmt_info->stmt, vectype);
 	}
 
-      /* Adjust the minimal vectorization factor according to the
-	 vector type.  */
-      vf = TYPE_VECTOR_SUBPARTS (vectype);
-      *min_vf = upper_bound (*min_vf, vf);
-
       /* Leave the BB vectorizer to pick the vector type later, based on
 	 the final dataref group size and SLP node size.  */
       if (is_a <loop_vec_info> (vinfo))
@@ -5720,8 +5716,7 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
       standard_iv_increment_position (loop, &incr_gsi, &insert_after);
 
       create_iv (aggr_ptr_init, PLUS_EXPR,
-		 fold_convert (aggr_ptr_type, iv_step),
-		 aggr_ptr, loop, &incr_gsi, insert_after,
+		 iv_step, aggr_ptr, loop, &incr_gsi, insert_after,
 		 &indx_before_incr, &indx_after_incr);
       incr = gsi_stmt (incr_gsi);
 
@@ -5749,7 +5744,7 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
     {
       standard_iv_increment_position (containing_loop, &incr_gsi,
 				      &insert_after);
-      create_iv (aptr, PLUS_EXPR, fold_convert (aggr_ptr_type, DR_STEP (dr)),
+      create_iv (aptr, PLUS_EXPR, DR_STEP (dr),
 		 aggr_ptr, containing_loop, &incr_gsi, insert_after,
 		 &indx_before_incr, &indx_after_incr);
       incr = gsi_stmt (incr_gsi);
@@ -6072,204 +6067,6 @@ vect_store_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
 }
 
 
-/* Function vect_permute_store_chain.
-
-   Given a chain of interleaved stores in DR_CHAIN of LENGTH that must be
-   a power of 2 or equal to 3, generate interleave_high/low stmts to reorder
-   the data correctly for the stores.  Return the final references for stores
-   in RESULT_CHAIN.
-
-   E.g., LENGTH is 4 and the scalar type is short, i.e., VF is 8.
-   The input is 4 vectors each containing 8 elements.  We assign a number to
-   each element, the input sequence is:
-
-   1st vec:   0  1  2  3  4  5  6  7
-   2nd vec:   8  9 10 11 12 13 14 15
-   3rd vec:  16 17 18 19 20 21 22 23
-   4th vec:  24 25 26 27 28 29 30 31
-
-   The output sequence should be:
-
-   1st vec:  0  8 16 24  1  9 17 25
-   2nd vec:  2 10 18 26  3 11 19 27
-   3rd vec:  4 12 20 28  5 13 21 30
-   4th vec:  6 14 22 30  7 15 23 31
-
-   i.e., we interleave the contents of the four vectors in their order.
-
-   We use interleave_high/low instructions to create such output.  The input of
-   each interleave_high/low operation is two vectors:
-   1st vec    2nd vec
-   0 1 2 3    4 5 6 7
-   the even elements of the result vector are obtained left-to-right from the
-   high/low elements of the first vector.  The odd elements of the result are
-   obtained left-to-right from the high/low elements of the second vector.
-   The output of interleave_high will be:   0 4 1 5
-   and of interleave_low:                   2 6 3 7
-
-
-   The permutation is done in log LENGTH stages.  In each stage interleave_high
-   and interleave_low stmts are created for each pair of vectors in DR_CHAIN,
-   where the first argument is taken from the first half of DR_CHAIN and the
-   second argument from it's second half.
-   In our example,
-
-   I1: interleave_high (1st vec, 3rd vec)
-   I2: interleave_low (1st vec, 3rd vec)
-   I3: interleave_high (2nd vec, 4th vec)
-   I4: interleave_low (2nd vec, 4th vec)
-
-   The output for the first stage is:
-
-   I1:  0 16  1 17  2 18  3 19
-   I2:  4 20  5 21  6 22  7 23
-   I3:  8 24  9 25 10 26 11 27
-   I4: 12 28 13 29 14 30 15 31
-
-   The output of the second stage, i.e. the final result is:
-
-   I1:  0  8 16 24  1  9 17 25
-   I2:  2 10 18 26  3 11 19 27
-   I3:  4 12 20 28  5 13 21 30
-   I4:  6 14 22 30  7 15 23 31.  */
-
-void
-vect_permute_store_chain (vec_info *vinfo, vec<tree> &dr_chain,
-			  unsigned int length,
-			  stmt_vec_info stmt_info,
-			  gimple_stmt_iterator *gsi,
-			  vec<tree> *result_chain)
-{
-  tree vect1, vect2, high, low;
-  gimple *perm_stmt;
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-  tree perm_mask_low, perm_mask_high;
-  tree data_ref;
-  tree perm3_mask_low, perm3_mask_high;
-  unsigned int i, j, n, log_length = exact_log2 (length);
-
-  result_chain->quick_grow (length);
-  memcpy (result_chain->address (), dr_chain.address (),
-	  length * sizeof (tree));
-
-  if (length == 3)
-    {
-      /* vect_grouped_store_supported ensures that this is constant.  */
-      unsigned int nelt = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
-      unsigned int j0 = 0, j1 = 0, j2 = 0;
-
-      vec_perm_builder sel (nelt, nelt, 1);
-      sel.quick_grow (nelt);
-      vec_perm_indices indices;
-      for (j = 0; j < 3; j++)
-        {
-	  int nelt0 = ((3 - j) * nelt) % 3;
-	  int nelt1 = ((3 - j) * nelt + 1) % 3;
-	  int nelt2 = ((3 - j) * nelt + 2) % 3;
-
-	  for (i = 0; i < nelt; i++)
-	    {
-	      if (3 * i + nelt0 < nelt)
-		sel[3 * i + nelt0] = j0++;
-	      if (3 * i + nelt1 < nelt)
-		sel[3 * i + nelt1] = nelt + j1++;
-	      if (3 * i + nelt2 < nelt)
-		sel[3 * i + nelt2] = 0;
-	    }
-	  indices.new_vector (sel, 2, nelt);
-	  perm3_mask_low = vect_gen_perm_mask_checked (vectype, indices);
-
-	  for (i = 0; i < nelt; i++)
-	    {
-	      if (3 * i + nelt0 < nelt)
-		sel[3 * i + nelt0] = 3 * i + nelt0;
-	      if (3 * i + nelt1 < nelt)
-		sel[3 * i + nelt1] = 3 * i + nelt1;
-	      if (3 * i + nelt2 < nelt)
-		sel[3 * i + nelt2] = nelt + j2++;
-	    }
-	  indices.new_vector (sel, 2, nelt);
-	  perm3_mask_high = vect_gen_perm_mask_checked (vectype, indices);
-
-	  vect1 = dr_chain[0];
-	  vect2 = dr_chain[1];
-
-	  /* Create interleaving stmt:
-	     low = VEC_PERM_EXPR <vect1, vect2,
-				  {j, nelt, *, j + 1, nelt + j + 1, *,
-				   j + 2, nelt + j + 2, *, ...}>  */
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_low");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect1,
-					   vect2, perm3_mask_low);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-
-	  vect1 = data_ref;
-	  vect2 = dr_chain[2];
-	  /* Create interleaving stmt:
-	     low = VEC_PERM_EXPR <vect1, vect2,
-				  {0, 1, nelt + j, 3, 4, nelt + j + 1,
-				   6, 7, nelt + j + 2, ...}>  */
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_high");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect1,
-					   vect2, perm3_mask_high);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	  (*result_chain)[j] = data_ref;
-	}
-    }
-  else
-    {
-      /* If length is not equal to 3 then only power of 2 is supported.  */
-      gcc_assert (pow2p_hwi (length));
-
-      /* The encoding has 2 interleaved stepped patterns.  */
-      poly_uint64 nelt = TYPE_VECTOR_SUBPARTS (vectype);
-      vec_perm_builder sel (nelt, 2, 3);
-      sel.quick_grow (6);
-      for (i = 0; i < 3; i++)
-	{
-	  sel[i * 2] = i;
-	  sel[i * 2 + 1] = i + nelt;
-	}
-	vec_perm_indices indices (sel, 2, nelt);
-	perm_mask_high = vect_gen_perm_mask_checked (vectype, indices);
-
-	for (i = 0; i < 6; i++)
-	  sel[i] += exact_div (nelt, 2);
-	indices.new_vector (sel, 2, nelt);
-	perm_mask_low = vect_gen_perm_mask_checked (vectype, indices);
-
-	for (i = 0, n = log_length; i < n; i++)
-	  {
-	    for (j = 0; j < length/2; j++)
-	      {
-		vect1 = dr_chain[j];
-		vect2 = dr_chain[j+length/2];
-
-		/* Create interleaving stmt:
-		   high = VEC_PERM_EXPR <vect1, vect2, {0, nelt, 1, nelt+1,
-							...}>  */
-		high = make_temp_ssa_name (vectype, NULL, "vect_inter_high");
-		perm_stmt = gimple_build_assign (high, VEC_PERM_EXPR, vect1,
-						 vect2, perm_mask_high);
-		vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-		(*result_chain)[2*j] = high;
-
-		/* Create interleaving stmt:
-		   low = VEC_PERM_EXPR <vect1, vect2,
-					{nelt/2, nelt*3/2, nelt/2+1, nelt*3/2+1,
-					 ...}>  */
-		low = make_temp_ssa_name (vectype, NULL, "vect_inter_low");
-		perm_stmt = gimple_build_assign (low, VEC_PERM_EXPR, vect1,
-						 vect2, perm_mask_low);
-		vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-		(*result_chain)[2*j+1] = low;
-	      }
-	    memcpy (dr_chain.address (), result_chain->address (),
-		    length * sizeof (tree));
-	  }
-    }
-}
-
 /* Function vect_setup_realignment
 
    This function is called when vectorizing an unaligned load using
@@ -6700,633 +6497,6 @@ vect_load_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
   return IFN_LAST;
 }
 
-/* Function vect_permute_load_chain.
-
-   Given a chain of interleaved loads in DR_CHAIN of LENGTH that must be
-   a power of 2 or equal to 3, generate extract_even/odd stmts to reorder
-   the input data correctly.  Return the final references for loads in
-   RESULT_CHAIN.
-
-   E.g., LENGTH is 4 and the scalar type is short, i.e., VF is 8.
-   The input is 4 vectors each containing 8 elements. We assign a number to each
-   element, the input sequence is:
-
-   1st vec:   0  1  2  3  4  5  6  7
-   2nd vec:   8  9 10 11 12 13 14 15
-   3rd vec:  16 17 18 19 20 21 22 23
-   4th vec:  24 25 26 27 28 29 30 31
-
-   The output sequence should be:
-
-   1st vec:  0 4  8 12 16 20 24 28
-   2nd vec:  1 5  9 13 17 21 25 29
-   3rd vec:  2 6 10 14 18 22 26 30
-   4th vec:  3 7 11 15 19 23 27 31
-
-   i.e., the first output vector should contain the first elements of each
-   interleaving group, etc.
-
-   We use extract_even/odd instructions to create such output.  The input of
-   each extract_even/odd operation is two vectors
-   1st vec    2nd vec
-   0 1 2 3    4 5 6 7
-
-   and the output is the vector of extracted even/odd elements.  The output of
-   extract_even will be:   0 2 4 6
-   and of extract_odd:     1 3 5 7
-
-
-   The permutation is done in log LENGTH stages.  In each stage extract_even
-   and extract_odd stmts are created for each pair of vectors in DR_CHAIN in
-   their order.  In our example,
-
-   E1: extract_even (1st vec, 2nd vec)
-   E2: extract_odd (1st vec, 2nd vec)
-   E3: extract_even (3rd vec, 4th vec)
-   E4: extract_odd (3rd vec, 4th vec)
-
-   The output for the first stage will be:
-
-   E1:  0  2  4  6  8 10 12 14
-   E2:  1  3  5  7  9 11 13 15
-   E3: 16 18 20 22 24 26 28 30
-   E4: 17 19 21 23 25 27 29 31
-
-   In order to proceed and create the correct sequence for the next stage (or
-   for the correct output, if the second stage is the last one, as in our
-   example), we first put the output of extract_even operation and then the
-   output of extract_odd in RESULT_CHAIN (which is then copied to DR_CHAIN).
-   The input for the second stage is:
-
-   1st vec (E1):  0  2  4  6  8 10 12 14
-   2nd vec (E3): 16 18 20 22 24 26 28 30
-   3rd vec (E2):  1  3  5  7  9 11 13 15
-   4th vec (E4): 17 19 21 23 25 27 29 31
-
-   The output of the second stage:
-
-   E1: 0 4  8 12 16 20 24 28
-   E2: 2 6 10 14 18 22 26 30
-   E3: 1 5  9 13 17 21 25 29
-   E4: 3 7 11 15 19 23 27 31
-
-   And RESULT_CHAIN after reordering:
-
-   1st vec (E1):  0 4  8 12 16 20 24 28
-   2nd vec (E3):  1 5  9 13 17 21 25 29
-   3rd vec (E2):  2 6 10 14 18 22 26 30
-   4th vec (E4):  3 7 11 15 19 23 27 31.  */
-
-static void
-vect_permute_load_chain (vec_info *vinfo, vec<tree> dr_chain,
-			 unsigned int length,
-			 stmt_vec_info stmt_info,
-			 gimple_stmt_iterator *gsi,
-			 vec<tree> *result_chain)
-{
-  tree data_ref, first_vect, second_vect;
-  tree perm_mask_even, perm_mask_odd;
-  tree perm3_mask_low, perm3_mask_high;
-  gimple *perm_stmt;
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-  unsigned int i, j, log_length = exact_log2 (length);
-
-  result_chain->quick_grow (length);
-  memcpy (result_chain->address (), dr_chain.address (),
-	  length * sizeof (tree));
-
-  if (length == 3)
-    {
-      /* vect_grouped_load_supported ensures that this is constant.  */
-      unsigned nelt = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
-      unsigned int k;
-
-      vec_perm_builder sel (nelt, nelt, 1);
-      sel.quick_grow (nelt);
-      vec_perm_indices indices;
-      for (k = 0; k < 3; k++)
-	{
-	  for (i = 0; i < nelt; i++)
-	    if (3 * i + k < 2 * nelt)
-	      sel[i] = 3 * i + k;
-	    else
-	      sel[i] = 0;
-	  indices.new_vector (sel, 2, nelt);
-	  perm3_mask_low = vect_gen_perm_mask_checked (vectype, indices);
-
-	  for (i = 0, j = 0; i < nelt; i++)
-	    if (3 * i + k < 2 * nelt)
-	      sel[i] = i;
-	    else
-	      sel[i] = nelt + ((nelt + k) % 3) + 3 * (j++);
-	  indices.new_vector (sel, 2, nelt);
-	  perm3_mask_high = vect_gen_perm_mask_checked (vectype, indices);
-
-	  first_vect = dr_chain[0];
-	  second_vect = dr_chain[1];
-
-	  /* Create interleaving stmt (low part of):
-	     low = VEC_PERM_EXPR <first_vect, second_vect2, {k, 3 + k, 6 + k,
-							     ...}>  */
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_low");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, first_vect,
-					   second_vect, perm3_mask_low);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-
-	  /* Create interleaving stmt (high part of):
-	     high = VEC_PERM_EXPR <first_vect, second_vect2, {k, 3 + k, 6 + k,
-							      ...}>  */
-	  first_vect = data_ref;
-	  second_vect = dr_chain[2];
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_high");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, first_vect,
-					   second_vect, perm3_mask_high);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	  (*result_chain)[k] = data_ref;
-	}
-    }
-  else
-    {
-      /* If length is not equal to 3 then only power of 2 is supported.  */
-      gcc_assert (pow2p_hwi (length));
-
-      /* The encoding has a single stepped pattern.  */
-      poly_uint64 nelt = TYPE_VECTOR_SUBPARTS (vectype);
-      vec_perm_builder sel (nelt, 1, 3);
-      sel.quick_grow (3);
-      for (i = 0; i < 3; ++i)
-	sel[i] = i * 2;
-      vec_perm_indices indices (sel, 2, nelt);
-      perm_mask_even = vect_gen_perm_mask_checked (vectype, indices);
-
-      for (i = 0; i < 3; ++i)
-	sel[i] = i * 2 + 1;
-      indices.new_vector (sel, 2, nelt);
-      perm_mask_odd = vect_gen_perm_mask_checked (vectype, indices);
-
-      for (i = 0; i < log_length; i++)
-	{
-	  for (j = 0; j < length; j += 2)
-	    {
-	      first_vect = dr_chain[j];
-	      second_vect = dr_chain[j+1];
-
-	      /* data_ref = permute_even (first_data_ref, second_data_ref);  */
-	      data_ref = make_temp_ssa_name (vectype, NULL, "vect_perm_even");
-	      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					       first_vect, second_vect,
-					       perm_mask_even);
-	      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	      (*result_chain)[j/2] = data_ref;
-
-	      /* data_ref = permute_odd (first_data_ref, second_data_ref);  */
-	      data_ref = make_temp_ssa_name (vectype, NULL, "vect_perm_odd");
-	      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					       first_vect, second_vect,
-					       perm_mask_odd);
-	      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	      (*result_chain)[j/2+length/2] = data_ref;
-	    }
-	  memcpy (dr_chain.address (), result_chain->address (),
-		  length * sizeof (tree));
-	}
-    }
-}
-
-/* Function vect_shift_permute_load_chain.
-
-   Given a chain of loads in DR_CHAIN of LENGTH 2 or 3, generate
-   sequence of stmts to reorder the input data accordingly.
-   Return the final references for loads in RESULT_CHAIN.
-   Return true if successed, false otherwise.
-
-   E.g., LENGTH is 3 and the scalar type is short, i.e., VF is 8.
-   The input is 3 vectors each containing 8 elements.  We assign a
-   number to each element, the input sequence is:
-
-   1st vec:   0  1  2  3  4  5  6  7
-   2nd vec:   8  9 10 11 12 13 14 15
-   3rd vec:  16 17 18 19 20 21 22 23
-
-   The output sequence should be:
-
-   1st vec:  0 3 6  9 12 15 18 21
-   2nd vec:  1 4 7 10 13 16 19 22
-   3rd vec:  2 5 8 11 14 17 20 23
-
-   We use 3 shuffle instructions and 3 * 3 - 1 shifts to create such output.
-
-   First we shuffle all 3 vectors to get correct elements order:
-
-   1st vec:  ( 0  3  6) ( 1  4  7) ( 2  5)
-   2nd vec:  ( 8 11 14) ( 9 12 15) (10 13)
-   3rd vec:  (16 19 22) (17 20 23) (18 21)
-
-   Next we unite and shift vector 3 times:
-
-   1st step:
-     shift right by 6 the concatenation of:
-     "1st vec" and  "2nd vec"
-       ( 0  3  6) ( 1  4  7) |( 2  5) _ ( 8 11 14) ( 9 12 15)| (10 13)
-     "2nd vec" and  "3rd vec"
-       ( 8 11 14) ( 9 12 15) |(10 13) _ (16 19 22) (17 20 23)| (18 21)
-     "3rd vec" and  "1st vec"
-       (16 19 22) (17 20 23) |(18 21) _ ( 0  3  6) ( 1  4  7)| ( 2  5)
-			     | New vectors                   |
-
-     So that now new vectors are:
-
-     1st vec:  ( 2  5) ( 8 11 14) ( 9 12 15)
-     2nd vec:  (10 13) (16 19 22) (17 20 23)
-     3rd vec:  (18 21) ( 0  3  6) ( 1  4  7)
-
-   2nd step:
-     shift right by 5 the concatenation of:
-     "1st vec" and  "3rd vec"
-       ( 2  5) ( 8 11 14) |( 9 12 15) _ (18 21) ( 0  3  6)| ( 1  4  7)
-     "2nd vec" and  "1st vec"
-       (10 13) (16 19 22) |(17 20 23) _ ( 2  5) ( 8 11 14)| ( 9 12 15)
-     "3rd vec" and  "2nd vec"
-       (18 21) ( 0  3  6) |( 1  4  7) _ (10 13) (16 19 22)| (17 20 23)
-			  | New vectors                   |
-
-     So that now new vectors are:
-
-     1st vec:  ( 9 12 15) (18 21) ( 0  3  6)
-     2nd vec:  (17 20 23) ( 2  5) ( 8 11 14)
-     3rd vec:  ( 1  4  7) (10 13) (16 19 22) READY
-
-   3rd step:
-     shift right by 5 the concatenation of:
-     "1st vec" and  "1st vec"
-       ( 9 12 15) (18 21) |( 0  3  6) _ ( 9 12 15) (18 21)| ( 0  3  6)
-     shift right by 3 the concatenation of:
-     "2nd vec" and  "2nd vec"
-               (17 20 23) |( 2  5) ( 8 11 14) _ (17 20 23)| ( 2  5) ( 8 11 14)
-			  | New vectors                   |
-
-     So that now all vectors are READY:
-     1st vec:  ( 0  3  6) ( 9 12 15) (18 21)
-     2nd vec:  ( 2  5) ( 8 11 14) (17 20 23)
-     3rd vec:  ( 1  4  7) (10 13) (16 19 22)
-
-   This algorithm is faster than one in vect_permute_load_chain if:
-     1.  "shift of a concatination" is faster than general permutation.
-	 This is usually so.
-     2.  The TARGET machine can't execute vector instructions in parallel.
-	 This is because each step of the algorithm depends on previous.
-	 The algorithm in vect_permute_load_chain is much more parallel.
-
-   The algorithm is applicable only for LOAD CHAIN LENGTH less than VF.
-*/
-
-static bool
-vect_shift_permute_load_chain (vec_info *vinfo, vec<tree> dr_chain,
-			       unsigned int length,
-			       stmt_vec_info stmt_info,
-			       gimple_stmt_iterator *gsi,
-			       vec<tree> *result_chain)
-{
-  tree vect[3], vect_shift[3], data_ref, first_vect, second_vect;
-  tree perm2_mask1, perm2_mask2, perm3_mask;
-  tree select_mask, shift1_mask, shift2_mask, shift3_mask, shift4_mask;
-  gimple *perm_stmt;
-
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-  machine_mode vmode = TYPE_MODE (vectype);
-  unsigned int i;
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-
-  unsigned HOST_WIDE_INT nelt, vf;
-  if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nelt)
-      || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&vf))
-    /* Not supported for variable-length vectors.  */
-    return false;
-
-  vec_perm_builder sel (nelt, nelt, 1);
-  sel.quick_grow (nelt);
-
-  result_chain->quick_grow (length);
-  memcpy (result_chain->address (), dr_chain.address (),
-	  length * sizeof (tree));
-
-  if (pow2p_hwi (length) && vf > 4)
-    {
-      unsigned int j, log_length = exact_log2 (length);
-      for (i = 0; i < nelt / 2; ++i)
-	sel[i] = i * 2;
-      for (i = 0; i < nelt / 2; ++i)
-	sel[nelt / 2 + i] = i * 2 + 1;
-      vec_perm_indices indices (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shuffle of 2 fields structure is not \
-			      supported by target\n");
-	  return false;
-	}
-      perm2_mask1 = vect_gen_perm_mask_checked (vectype, indices);
-
-      for (i = 0; i < nelt / 2; ++i)
-	sel[i] = i * 2 + 1;
-      for (i = 0; i < nelt / 2; ++i)
-	sel[nelt / 2 + i] = i * 2;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shuffle of 2 fields structure is not \
-			      supported by target\n");
-	  return false;
-	}
-      perm2_mask2 = vect_gen_perm_mask_checked (vectype, indices);
-
-      /* Generating permutation constant to shift all elements.
-	 For vector length 8 it is {4 5 6 7 8 9 10 11}.  */
-      for (i = 0; i < nelt; i++)
-	sel[i] = nelt / 2 + i;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shift permutation is not supported by target\n");
-	  return false;
-	}
-      shift1_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      /* Generating permutation constant to select vector from 2.
-	 For vector length 8 it is {0 1 2 3 12 13 14 15}.  */
-      for (i = 0; i < nelt / 2; i++)
-	sel[i] = i;
-      for (i = nelt / 2; i < nelt; i++)
-	sel[i] = nelt + i;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "select is not supported by target\n");
-	  return false;
-	}
-      select_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      for (i = 0; i < log_length; i++)
-	{
-	  for (j = 0; j < length; j += 2)
-	    {
-	      first_vect = dr_chain[j];
-	      second_vect = dr_chain[j + 1];
-
-	      data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle2");
-	      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					       first_vect, first_vect,
-					       perm2_mask1);
-	      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	      vect[0] = data_ref;
-
-	      data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle2");
-	      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					       second_vect, second_vect,
-					       perm2_mask2);
-	      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	      vect[1] = data_ref;
-
-	      data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift");
-	      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					       vect[0], vect[1], shift1_mask);
-	      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	      (*result_chain)[j/2 + length/2] = data_ref;
-
-	      data_ref = make_temp_ssa_name (vectype, NULL, "vect_select");
-	      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					       vect[0], vect[1], select_mask);
-	      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	      (*result_chain)[j/2] = data_ref;
-	    }
-	  memcpy (dr_chain.address (), result_chain->address (),
-		  length * sizeof (tree));
-	}
-      return true;
-    }
-  if (length == 3 && vf > 2)
-    {
-      unsigned int k = 0, l = 0;
-
-      /* Generating permutation constant to get all elements in rigth order.
-	 For vector length 8 it is {0 3 6 1 4 7 2 5}.  */
-      for (i = 0; i < nelt; i++)
-	{
-	  if (3 * k + (l % 3) >= nelt)
-	    {
-	      k = 0;
-	      l += (3 - (nelt % 3));
-	    }
-	  sel[i] = 3 * k + (l % 3);
-	  k++;
-	}
-      vec_perm_indices indices (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shuffle of 3 fields structure is not \
-			      supported by target\n");
-	  return false;
-	}
-      perm3_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      /* Generating permutation constant to shift all elements.
-	 For vector length 8 it is {6 7 8 9 10 11 12 13}.  */
-      for (i = 0; i < nelt; i++)
-	sel[i] = 2 * (nelt / 3) + (nelt % 3) + i;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shift permutation is not supported by target\n");
-	  return false;
-	}
-      shift1_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      /* Generating permutation constant to shift all elements.
-	 For vector length 8 it is {5 6 7 8 9 10 11 12}.  */
-      for (i = 0; i < nelt; i++)
-	sel[i] = 2 * (nelt / 3) + 1 + i;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shift permutation is not supported by target\n");
-	  return false;
-	}
-      shift2_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      /* Generating permutation constant to shift all elements.
-	 For vector length 8 it is {3 4 5 6 7 8 9 10}.  */
-      for (i = 0; i < nelt; i++)
-	sel[i] = (nelt / 3) + (nelt % 3) / 2 + i;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shift permutation is not supported by target\n");
-	  return false;
-	}
-      shift3_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      /* Generating permutation constant to shift all elements.
-	 For vector length 8 it is {5 6 7 8 9 10 11 12}.  */
-      for (i = 0; i < nelt; i++)
-	sel[i] = 2 * (nelt / 3) + (nelt % 3) / 2 + i;
-      indices.new_vector (sel, 2, nelt);
-      if (!can_vec_perm_const_p (vmode, vmode, indices))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "shift permutation is not supported by target\n");
-	  return false;
-	}
-      shift4_mask = vect_gen_perm_mask_checked (vectype, indices);
-
-      for (k = 0; k < 3; k++)
-	{
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					   dr_chain[k], dr_chain[k],
-					   perm3_mask);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	  vect[k] = data_ref;
-	}
-
-      for (k = 0; k < 3; k++)
-	{
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift1");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					   vect[k % 3], vect[(k + 1) % 3],
-					   shift1_mask);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	  vect_shift[k] = data_ref;
-	}
-
-      for (k = 0; k < 3; k++)
-	{
-	  data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift2");
-	  perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
-					   vect_shift[(4 - k) % 3],
-					   vect_shift[(3 - k) % 3],
-					   shift2_mask);
-	  vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-	  vect[k] = data_ref;
-	}
-
-      (*result_chain)[3 - (nelt % 3)] = vect[2];
-
-      data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift3");
-      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect[0],
-				       vect[0], shift3_mask);
-      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-      (*result_chain)[nelt % 3] = data_ref;
-
-      data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift4");
-      perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect[1],
-				       vect[1], shift4_mask);
-      vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
-      (*result_chain)[0] = data_ref;
-      return true;
-    }
-  return false;
-}
-
-/* Function vect_transform_grouped_load.
-
-   Given a chain of input interleaved data-refs (in DR_CHAIN), build statements
-   to perform their permutation and ascribe the result vectorized statements to
-   the scalar statements.
-*/
-
-void
-vect_transform_grouped_load (vec_info *vinfo, stmt_vec_info stmt_info,
-			     vec<tree> dr_chain,
-			     int size, gimple_stmt_iterator *gsi)
-{
-  machine_mode mode;
-  vec<tree> result_chain = vNULL;
-
-  /* DR_CHAIN contains input data-refs that are a part of the interleaving.
-     RESULT_CHAIN is the output of vect_permute_load_chain, it contains permuted
-     vectors, that are ready for vector computation.  */
-  result_chain.create (size);
-
-  /* If reassociation width for vector type is 2 or greater target machine can
-     execute 2 or more vector instructions in parallel.  Otherwise try to
-     get chain for loads group using vect_shift_permute_load_chain.  */
-  mode = TYPE_MODE (STMT_VINFO_VECTYPE (stmt_info));
-  if (targetm.sched.reassociation_width (VEC_PERM_EXPR, mode) > 1
-      || pow2p_hwi (size)
-      || !vect_shift_permute_load_chain (vinfo, dr_chain, size, stmt_info,
-					 gsi, &result_chain))
-    vect_permute_load_chain (vinfo, dr_chain,
-			     size, stmt_info, gsi, &result_chain);
-  vect_record_grouped_load_vectors (vinfo, stmt_info, result_chain);
-  result_chain.release ();
-}
-
-/* RESULT_CHAIN contains the output of a group of grouped loads that were
-   generated as part of the vectorization of STMT_INFO.  Assign the statement
-   for each vector to the associated scalar statement.  */
-
-void
-vect_record_grouped_load_vectors (vec_info *, stmt_vec_info stmt_info,
-				  vec<tree> result_chain)
-{
-  stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
-  unsigned int i, gap_count;
-  tree tmp_data_ref;
-
-  /* Put a permuted data-ref in the VECTORIZED_STMT field.
-     Since we scan the chain starting from it's first node, their order
-     corresponds the order of data-refs in RESULT_CHAIN.  */
-  stmt_vec_info next_stmt_info = first_stmt_info;
-  gap_count = 1;
-  FOR_EACH_VEC_ELT (result_chain, i, tmp_data_ref)
-    {
-      if (!next_stmt_info)
-	break;
-
-      /* Skip the gaps.  Loads created for the gaps will be removed by dead
-       code elimination pass later.  No need to check for the first stmt in
-       the group, since it always exists.
-       DR_GROUP_GAP is the number of steps in elements from the previous
-       access (if there is no gap DR_GROUP_GAP is 1).  We skip loads that
-       correspond to the gaps.  */
-      if (next_stmt_info != first_stmt_info
-	  && gap_count < DR_GROUP_GAP (next_stmt_info))
-	{
-	  gap_count++;
-	  continue;
-	}
-
-      /* ???  The following needs cleanup after the removal of
-         DR_GROUP_SAME_DR_STMT.  */
-      if (next_stmt_info)
-        {
-	  gimple *new_stmt = SSA_NAME_DEF_STMT (tmp_data_ref);
-	  /* We assume that if VEC_STMT is not NULL, this is a case of multiple
-	     copies, and we put the new vector statement last.  */
-	  STMT_VINFO_VEC_STMTS (next_stmt_info).safe_push (new_stmt);
-
-	  next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
-	  gap_count = 1;
-        }
-    }
-}
-
 /* Function vect_force_dr_alignment_p.
 
    Returns whether the alignment of a DECL can be forced to be aligned
@@ -7354,13 +6524,14 @@ vect_can_force_dr_alignment_p (const_tree decl, poly_uint64 alignment)
    alignment.
    If CHECK_ALIGNED_ACCESSES is TRUE, check if the access is supported even
    it is aligned, i.e., check if it is possible to vectorize it with different
-   alignment.  */
+   alignment.  If GS_INFO is passed we are dealing with a gather/scatter.  */
 
 enum dr_alignment_support
 vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
-			       tree vectype, int misalignment)
+			       tree vectype, int misalignment,
+			       gather_scatter_info *gs_info)
 {
-  data_reference *dr = dr_info->dr;
+  data_reference *dr = dr_info ? dr_info->dr : nullptr;
   stmt_vec_info stmt_info = dr_info->stmt;
   machine_mode mode = TYPE_MODE (vectype);
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
@@ -7372,14 +6543,6 @@ vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
   else if (dr_safe_speculative_read_required (stmt_info))
     return dr_unaligned_unsupported;
 
-  /* For now assume all conditional loads/stores support unaligned
-     access without any special code.  */
-  if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
-    if (gimple_call_internal_p (stmt)
-	&& (gimple_call_internal_fn (stmt) == IFN_MASK_LOAD
-	    || gimple_call_internal_fn (stmt) == IFN_MASK_STORE))
-      return dr_unaligned_supported;
-
   if (loop_vinfo)
     {
       vect_loop = LOOP_VINFO_LOOP (loop_vinfo);
@@ -7449,7 +6612,7 @@ vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
         }
     } */
 
-  if (DR_IS_READ (dr))
+  if (dr && DR_IS_READ (dr))
     {
       if (can_implement_p (vec_realign_load_optab, mode)
 	  && (!targetm.vectorize.builtin_mask_for_load
@@ -7477,10 +6640,43 @@ vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
 
   bool is_packed = false;
   tree type = TREE_TYPE (DR_REF (dr));
+  bool is_gather_scatter = gs_info != nullptr;
   if (misalignment == DR_MISALIGNMENT_UNKNOWN)
-    is_packed = not_size_aligned (DR_REF (dr));
+    {
+      if (!is_gather_scatter || dr != nullptr)
+	is_packed = not_size_aligned (DR_REF (dr));
+      else
+	{
+	  /* Gather-scatter accesses normally perform only component accesses
+	     so alignment is irrelevant for them.  Targets like riscv do care
+	     about scalar alignment in vector accesses, though, so check scalar
+	     alignment here.  We determined the alias pointer as well as the
+	     base alignment during pattern recognition and can re-use it here.
+
+	     As we do not have an analyzed dataref we only know the alignment
+	     of the reference itself and nothing about init, steps, etc.
+	     For now don't try harder to determine misalignment and
+	     just assume it is unknown.  We consider the type packed if its
+	     scalar alignment is lower than the natural alignment of a vector
+	     element's type.  */
+
+	  gcc_assert (!GATHER_SCATTER_LEGACY_P (*gs_info));
+	  gcc_assert (dr == nullptr);
+
+	  tree inner_vectype = TREE_TYPE (vectype);
+
+	  unsigned HOST_WIDE_INT scalar_align
+	    = tree_to_uhwi (gs_info->alias_ptr);
+	  unsigned HOST_WIDE_INT inner_vectype_sz
+	    = tree_to_uhwi (TYPE_SIZE (inner_vectype));
+
+	  bool is_misaligned = scalar_align < inner_vectype_sz;
+	  is_packed = scalar_align > 1 && is_misaligned;
+	}
+    }
   if (targetm.vectorize.support_vector_misalignment (mode, type, misalignment,
-						     is_packed))
+						     is_packed,
+						     is_gather_scatter))
     return dr_unaligned_supported;
 
   /* Unsupported.  */
diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index 4696943..6c1b26a 100644
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -2857,9 +2857,21 @@ vect_gen_vector_loop_niters (loop_vec_info loop_vinfo, tree niters,
 	 we set range information to make niters analyzer's life easier.
 	 Note the number of latch iteration value can be TYPE_MAX_VALUE so
 	 we have to represent the vector niter TYPE_MAX_VALUE + 1 / vf.  */
-      if (stmts != NULL && const_vf > 0)
+      if (stmts != NULL && const_vf > 0 && !LOOP_VINFO_EPILOGUE_P (loop_vinfo))
 	{
-	  if (niters_no_overflow)
+	  if (niters_no_overflow
+	      && LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo))
+	    {
+	      int_range<1> vr (type,
+			       wi::one (TYPE_PRECISION (type)),
+			       wi::div_ceil (wi::max_value
+							(TYPE_PRECISION (type),
+							 TYPE_SIGN (type)),
+					     const_vf,
+					     TYPE_SIGN (type)));
+	      set_range_info (niters_vector, vr);
+	    }
+	  else if (niters_no_overflow)
 	    {
 	      int_range<1> vr (type,
 			       wi::one (TYPE_PRECISION (type)),
@@ -3283,7 +3295,7 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
   bool skip_vector = (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
 		      ? maybe_lt (LOOP_VINFO_INT_NITERS (loop_vinfo),
 				  bound_prolog + bound_epilog)
-		      : (!LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo)
+		      : (!LOOP_VINFO_USE_VERSIONING_WITHOUT_PEELING (loop_vinfo)
 			 || vect_epilogues));
 
   /* Epilog loop must be executed if the number of iterations for epilog
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 2782d61..460de57 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -58,6 +58,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-eh.h"
 #include "case-cfn-macros.h"
 #include "langhooks.h"
+#include "opts.h"
 
 /* Loop Vectorization Pass.
 
@@ -167,9 +168,8 @@ static stmt_vec_info vect_is_simple_reduction (loop_vec_info, stmt_vec_info,
    may already be set for general statements (not just data refs).  */
 
 static opt_result
-vect_determine_vf_for_stmt_1 (vec_info *vinfo, stmt_vec_info stmt_info,
-			      bool vectype_maybe_set_p,
-			      poly_uint64 *vf)
+vect_determine_vectype_for_stmt_1 (vec_info *vinfo, stmt_vec_info stmt_info,
+				   bool vectype_maybe_set_p)
 {
   gimple *stmt = stmt_info->stmt;
 
@@ -202,9 +202,6 @@ vect_determine_vf_for_stmt_1 (vec_info *vinfo, stmt_vec_info stmt_info,
 	STMT_VINFO_VECTYPE (stmt_info) = stmt_vectype;
     }
 
-  if (nunits_vectype)
-    vect_update_max_nunits (vf, nunits_vectype);
-
   return opt_result::success ();
 }
 
@@ -214,13 +211,12 @@ vect_determine_vf_for_stmt_1 (vec_info *vinfo, stmt_vec_info stmt_info,
    or false if something prevented vectorization.  */
 
 static opt_result
-vect_determine_vf_for_stmt (vec_info *vinfo,
-			    stmt_vec_info stmt_info, poly_uint64 *vf)
+vect_determine_vectype_for_stmt (vec_info *vinfo, stmt_vec_info stmt_info)
 {
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G",
 		     stmt_info->stmt);
-  opt_result res = vect_determine_vf_for_stmt_1 (vinfo, stmt_info, false, vf);
+  opt_result res = vect_determine_vectype_for_stmt_1 (vinfo, stmt_info, false);
   if (!res)
     return res;
 
@@ -239,7 +235,7 @@ vect_determine_vf_for_stmt (vec_info *vinfo,
 	    dump_printf_loc (MSG_NOTE, vect_location,
 			     "==> examining pattern def stmt: %G",
 			     def_stmt_info->stmt);
-	  res = vect_determine_vf_for_stmt_1 (vinfo, def_stmt_info, true, vf);
+	  res = vect_determine_vectype_for_stmt_1 (vinfo, def_stmt_info, true);
 	  if (!res)
 	    return res;
 	}
@@ -248,7 +244,7 @@ vect_determine_vf_for_stmt (vec_info *vinfo,
 	dump_printf_loc (MSG_NOTE, vect_location,
 			 "==> examining pattern statement: %G",
 			 stmt_info->stmt);
-      res = vect_determine_vf_for_stmt_1 (vinfo, stmt_info, true, vf);
+      res = vect_determine_vectype_for_stmt_1 (vinfo, stmt_info, true);
       if (!res)
 	return res;
     }
@@ -256,45 +252,23 @@ vect_determine_vf_for_stmt (vec_info *vinfo,
   return opt_result::success ();
 }
 
-/* Function vect_determine_vectorization_factor
-
-   Determine the vectorization factor (VF).  VF is the number of data elements
-   that are operated upon in parallel in a single iteration of the vectorized
-   loop.  For example, when vectorizing a loop that operates on 4byte elements,
-   on a target with vector size (VS) 16byte, the VF is set to 4, since 4
-   elements can fit in a single vector register.
-
-   We currently support vectorization of loops in which all types operated upon
-   are of the same size.  Therefore this function currently sets VF according to
-   the size of the types operated upon, and fails if there are multiple sizes
-   in the loop.
+/* Function vect_set_stmts_vectype
 
-   VF is also the factor by which the loop iterations are strip-mined, e.g.:
-   original loop:
-        for (i=0; i<N; i++){
-          a[i] = b[i] + c[i];
-        }
-
-   vectorized loop:
-        for (i=0; i<N; i+=VF){
-          a[i:VF] = b[i:VF] + c[i:VF];
-        }
-*/
+   Set STMT_VINFO_VECTYPE of all stmts.  */
 
 static opt_result
-vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
+vect_set_stmts_vectype (loop_vec_info loop_vinfo)
 {
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
   unsigned nbbs = loop->num_nodes;
-  poly_uint64 vectorization_factor = 1;
   tree scalar_type = NULL_TREE;
   gphi *phi;
   tree vectype;
   stmt_vec_info stmt_info;
   unsigned i;
 
-  DUMP_VECT_SCOPE ("vect_determine_vectorization_factor");
+  DUMP_VECT_SCOPE ("vect_set_stmts_vectype");
 
   for (i = 0; i < nbbs; i++)
     {
@@ -333,15 +307,6 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
 	      if (dump_enabled_p ())
 		dump_printf_loc (MSG_NOTE, vect_location, "vectype: %T\n",
 				 vectype);
-
-	      if (dump_enabled_p ())
-		{
-		  dump_printf_loc (MSG_NOTE, vect_location, "nunits = ");
-		  dump_dec (MSG_NOTE, TYPE_VECTOR_SUBPARTS (vectype));
-		  dump_printf (MSG_NOTE, "\n");
-		}
-
-	      vect_update_max_nunits (&vectorization_factor, vectype);
 	    }
 	}
 
@@ -352,25 +317,12 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
 	    continue;
 	  stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
 	  opt_result res
-	    = vect_determine_vf_for_stmt (loop_vinfo,
-					  stmt_info, &vectorization_factor);
+	    = vect_determine_vectype_for_stmt (loop_vinfo, stmt_info);
 	  if (!res)
 	    return res;
         }
     }
 
-  /* TODO: Analyze cost. Decide if worth while to vectorize.  */
-  if (dump_enabled_p ())
-    {
-      dump_printf_loc (MSG_NOTE, vect_location, "vectorization factor = ");
-      dump_dec (MSG_NOTE, vectorization_factor);
-      dump_printf (MSG_NOTE, "\n");
-    }
-
-  if (known_le (vectorization_factor, 1U))
-    return opt_result::failure_at (vect_location,
-				   "not vectorized: unsupported data-type\n");
-  LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
   return opt_result::success ();
 }
 
@@ -1967,7 +1919,6 @@ vect_create_loop_vinfo (class loop *loop, vec_info_shared *shared,
   for (gcond *cond : info->conds)
     {
       stmt_vec_info loop_cond_info = loop_vinfo->lookup_stmt (cond);
-      STMT_VINFO_TYPE (loop_cond_info) = loop_exit_ctrl_vec_info_type;
       /* Mark the statement as a condition.  */
       STMT_VINFO_DEF_TYPE (loop_cond_info) = vect_condition_def;
     }
@@ -1984,9 +1935,6 @@ vect_create_loop_vinfo (class loop *loop, vec_info_shared *shared,
 
   if (info->inner_loop_cond)
     {
-      stmt_vec_info inner_loop_cond_info
-	= loop_vinfo->lookup_stmt (info->inner_loop_cond);
-      STMT_VINFO_TYPE (inner_loop_cond_info) = loop_exit_ctrl_vec_info_type;
       /* If we have an estimate on the number of iterations of the inner
 	 loop use that to limit the scale for costing, otherwise use
 	 --param vect-inner-loop-cost-factor literally.  */
@@ -2001,234 +1949,6 @@ vect_create_loop_vinfo (class loop *loop, vec_info_shared *shared,
 
 
 
-/* Scan the loop stmts and dependent on whether there are any (non-)SLP
-   statements update the vectorization factor.  */
-
-static void
-vect_update_vf_for_slp (loop_vec_info loop_vinfo)
-{
-  class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
-  basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
-  int nbbs = loop->num_nodes;
-  poly_uint64 vectorization_factor;
-  int i;
-
-  DUMP_VECT_SCOPE ("vect_update_vf_for_slp");
-
-  vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
-  gcc_assert (known_ne (vectorization_factor, 0U));
-
-  /* If all the stmts in the loop can be SLPed, we perform only SLP, and
-     vectorization factor of the loop is the unrolling factor required by
-     the SLP instances.  If that unrolling factor is 1, we say, that we
-     perform pure SLP on loop - cross iteration parallelism is not
-     exploited.  */
-  bool only_slp_in_loop = true;
-  for (i = 0; i < nbbs; i++)
-    {
-      basic_block bb = bbs[i];
-      for (gphi_iterator si = gsi_start_phis (bb); !gsi_end_p (si);
-	   gsi_next (&si))
-	{
-	  stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (si.phi ());
-	  if (!stmt_info)
-	    continue;
-	  if ((STMT_VINFO_RELEVANT_P (stmt_info)
-	       || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
-	      && !PURE_SLP_STMT (stmt_info))
-	    /* STMT needs both SLP and loop-based vectorization.  */
-	    only_slp_in_loop = false;
-	}
-      for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);
-	   gsi_next (&si))
-	{
-	  if (is_gimple_debug (gsi_stmt (si)))
-	    continue;
-	  stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
-	  stmt_info = vect_stmt_to_vectorize (stmt_info);
-	  if ((STMT_VINFO_RELEVANT_P (stmt_info)
-	       || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
-	      && !PURE_SLP_STMT (stmt_info))
-	    /* STMT needs both SLP and loop-based vectorization.  */
-	    only_slp_in_loop = false;
-	}
-    }
-
-  if (only_slp_in_loop)
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "Loop contains only SLP stmts\n");
-      vectorization_factor = LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo);
-    }
-  else
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "Loop contains SLP and non-SLP stmts\n");
-      /* Both the vectorization factor and unroll factor have the form
-	 GET_MODE_SIZE (loop_vinfo->vector_mode) * X for some rational X,
-	 so they must have a common multiple.  */
-      vectorization_factor
-	= force_common_multiple (vectorization_factor,
-				 LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo));
-    }
-
-  LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
-  if (dump_enabled_p ())
-    {
-      dump_printf_loc (MSG_NOTE, vect_location,
-		       "Updating vectorization factor to ");
-      dump_dec (MSG_NOTE, vectorization_factor);
-      dump_printf (MSG_NOTE, ".\n");
-    }
-}
-
-/* Return true if STMT_INFO describes a double reduction phi and if
-   the other phi in the reduction is also relevant for vectorization.
-   This rejects cases such as:
-
-      outer1:
-	x_1 = PHI <x_3(outer2), ...>;
-	...
-
-      inner:
-	x_2 = ...;
-	...
-
-      outer2:
-	x_3 = PHI <x_2(inner)>;
-
-   if nothing in x_2 or elsewhere makes x_1 relevant.  */
-
-static bool
-vect_active_double_reduction_p (stmt_vec_info stmt_info)
-{
-  if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def)
-    return false;
-
-  return STMT_VINFO_RELEVANT_P (STMT_VINFO_REDUC_DEF (stmt_info));
-}
-
-/* Function vect_analyze_loop_operations.
-
-   Scan the loop stmts and make sure they are all vectorizable.  */
-
-static opt_result
-vect_analyze_loop_operations (loop_vec_info loop_vinfo)
-{
-  class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
-  basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
-  int nbbs = loop->num_nodes;
-  int i;
-  stmt_vec_info stmt_info;
-
-  DUMP_VECT_SCOPE ("vect_analyze_loop_operations");
-
-  for (i = 0; i < nbbs; i++)
-    {
-      basic_block bb = bbs[i];
-
-      for (gphi_iterator si = gsi_start_phis (bb); !gsi_end_p (si);
-	   gsi_next (&si))
-        {
-          gphi *phi = si.phi ();
-
-	  stmt_info = loop_vinfo->lookup_stmt (phi);
-          if (dump_enabled_p ())
-	    dump_printf_loc (MSG_NOTE, vect_location, "examining phi: %G",
-			     (gimple *) phi);
-	  if (virtual_operand_p (gimple_phi_result (phi)))
-	    continue;
-
-	  /* ???  All of the below unconditional FAILs should be in
-	     done earlier after analyzing cycles, possibly when
-	     determining stmt relevancy?  */
-
-          /* Inner-loop loop-closed exit phi in outer-loop vectorization
-             (i.e., a phi in the tail of the outer-loop).  */
-          if (! is_loop_header_bb_p (bb))
-            {
-              /* FORNOW: we currently don't support the case that these phis
-                 are not used in the outerloop (unless it is double reduction,
-                 i.e., this phi is vect_reduction_def), cause this case
-                 requires to actually do something here.  */
-              if (STMT_VINFO_LIVE_P (stmt_info)
-		  && !vect_active_double_reduction_p (stmt_info))
-		return opt_result::failure_at (phi,
-					       "Unsupported loop-closed phi"
-					       " in outer-loop.\n");
-
-              /* If PHI is used in the outer loop, we check that its operand
-                 is defined in the inner loop.  */
-              if (STMT_VINFO_RELEVANT_P (stmt_info))
-                {
-                  tree phi_op;
-
-                  if (gimple_phi_num_args (phi) != 1)
-                    return opt_result::failure_at (phi, "unsupported phi");
-
-                  phi_op = PHI_ARG_DEF (phi, 0);
-		  stmt_vec_info op_def_info = loop_vinfo->lookup_def (phi_op);
-		  if (!op_def_info)
-		    return opt_result::failure_at (phi, "unsupported phi\n");
-
-		  if (STMT_VINFO_RELEVANT (op_def_info) != vect_used_in_outer
-		      && (STMT_VINFO_RELEVANT (op_def_info)
-			  != vect_used_in_outer_by_reduction))
-		    return opt_result::failure_at (phi, "unsupported phi\n");
-
-		  if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def
-		       || (STMT_VINFO_DEF_TYPE (stmt_info)
-			   == vect_double_reduction_def))
-		      && ! PURE_SLP_STMT (stmt_info))
-		    return opt_result::failure_at (phi, "unsupported phi\n");
-                }
-
-              continue;
-            }
-
-          gcc_assert (stmt_info);
-
-          if ((STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope
-               || STMT_VINFO_LIVE_P (stmt_info))
-	      && STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def
-	      && STMT_VINFO_DEF_TYPE (stmt_info) != vect_first_order_recurrence)
-	    /* A scalar-dependence cycle that we don't support.  */
-	    return opt_result::failure_at (phi,
-					   "not vectorized:"
-					   " scalar dependence cycle.\n");
-
-	  if (STMT_VINFO_RELEVANT_P (stmt_info)
-	      && ! PURE_SLP_STMT (stmt_info))
-	    return opt_result::failure_at (phi,
-					   "not vectorized: relevant phi not "
-					   "supported: %G",
-					   static_cast <gimple *> (phi));
-        }
-
-      for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);
-	   gsi_next (&si))
-        {
-	  gimple *stmt = gsi_stmt (si);
-	  if (!gimple_clobber_p (stmt)
-	      && !is_gimple_debug (stmt))
-	    {
-	      bool need_to_vectorize = false;
-	      opt_result res
-		= vect_analyze_stmt (loop_vinfo,
-				     loop_vinfo->lookup_stmt (stmt),
-				     &need_to_vectorize,
-				     NULL, NULL, NULL);
-	      if (!res)
-		return res;
-	    }
-        }
-    } /* bbs */
-
-  return opt_result::success ();
-}
-
 /* Return true if we know that the iteration count is smaller than the
    vectorization factor.  Return false if it isn't, or if we can't be sure
    either way.  */
@@ -2529,78 +2249,6 @@ vect_get_datarefs_in_loop (loop_p loop, basic_block *bbs,
   return opt_result::success ();
 }
 
-/* Look for SLP-only access groups and turn each individual access into its own
-   group.  */
-static void
-vect_dissolve_slp_only_groups (loop_vec_info loop_vinfo)
-{
-  unsigned int i;
-  struct data_reference *dr;
-
-  DUMP_VECT_SCOPE ("vect_dissolve_slp_only_groups");
-
-  vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
-  FOR_EACH_VEC_ELT (datarefs, i, dr)
-    {
-      gcc_assert (DR_REF (dr));
-      stmt_vec_info stmt_info
-	= vect_stmt_to_vectorize (loop_vinfo->lookup_stmt (DR_STMT (dr)));
-
-      /* Check if the load is a part of an interleaving chain.  */
-      if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
-	{
-	  stmt_vec_info first_element = DR_GROUP_FIRST_ELEMENT (stmt_info);
-	  dr_vec_info *dr_info = STMT_VINFO_DR_INFO (first_element);
-	  unsigned int group_size = DR_GROUP_SIZE (first_element);
-
-	  /* Check if SLP-only groups.  */
-	  if (!STMT_SLP_TYPE (stmt_info)
-	      && STMT_VINFO_SLP_VECT_ONLY (first_element))
-	    {
-	      /* Dissolve the group.  */
-	      STMT_VINFO_SLP_VECT_ONLY (first_element) = false;
-
-	      stmt_vec_info vinfo = first_element;
-	      while (vinfo)
-		{
-		  stmt_vec_info next = DR_GROUP_NEXT_ELEMENT (vinfo);
-		  DR_GROUP_FIRST_ELEMENT (vinfo) = vinfo;
-		  DR_GROUP_NEXT_ELEMENT (vinfo) = NULL;
-		  DR_GROUP_SIZE (vinfo) = 1;
-		  if (STMT_VINFO_STRIDED_P (first_element)
-		      /* We cannot handle stores with gaps.  */
-		      || DR_IS_WRITE (dr_info->dr))
-		    {
-		      STMT_VINFO_STRIDED_P (vinfo) = true;
-		      DR_GROUP_GAP (vinfo) = 0;
-		    }
-		  else
-		    DR_GROUP_GAP (vinfo) = group_size - 1;
-		  /* Duplicate and adjust alignment info, it needs to
-		     be present on each group leader, see dr_misalignment.  */
-		  if (vinfo != first_element)
-		    {
-		      dr_vec_info *dr_info2 = STMT_VINFO_DR_INFO (vinfo);
-		      dr_info2->target_alignment = dr_info->target_alignment;
-		      int misalignment = dr_info->misalignment;
-		      if (misalignment != DR_MISALIGNMENT_UNKNOWN)
-			{
-			  HOST_WIDE_INT diff
-			    = (TREE_INT_CST_LOW (DR_INIT (dr_info2->dr))
-			       - TREE_INT_CST_LOW (DR_INIT (dr_info->dr)));
-			  unsigned HOST_WIDE_INT align_c
-			    = dr_info->target_alignment.to_constant ();
-			  misalignment = (misalignment + diff) % align_c;
-			}
-		      dr_info2->misalignment = misalignment;
-		    }
-		  vinfo = next;
-		}
-	    }
-	}
-    }
-}
-
 /* Determine if operating on full vectors for LOOP_VINFO might leave
    some scalar iterations still to do.  If so, decide how we should
    handle those scalar iterations.  The possibilities are:
@@ -2721,7 +2369,6 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal,
   opt_result ok = opt_result::success ();
   int res;
   unsigned int max_vf = MAX_VECTORIZATION_FACTOR;
-  poly_uint64 min_vf = 2;
   loop_vec_info orig_loop_vinfo = NULL;
 
   /* If we are dealing with an epilogue then orig_loop_vinfo points to the
@@ -2768,7 +2415,7 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal,
   /* Analyze the data references and also adjust the minimal
      vectorization factor according to the loads and stores.  */
 
-  ok = vect_analyze_data_refs (loop_vinfo, &min_vf, &fatal);
+  ok = vect_analyze_data_refs (loop_vinfo, &fatal);
   if (!ok)
     {
       if (dump_enabled_p ())
@@ -2833,24 +2480,20 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal,
 			 "bad data dependence.\n");
       return ok;
     }
-  if (max_vf != MAX_VECTORIZATION_FACTOR
-      && maybe_lt (max_vf, min_vf))
-    return opt_result::failure_at (vect_location, "bad data dependence.\n");
   LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo) = max_vf;
 
-  ok = vect_determine_vectorization_factor (loop_vinfo);
+  ok = vect_set_stmts_vectype (loop_vinfo);
   if (!ok)
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			 "can't determine vectorization factor.\n");
+			 "cannot determine vector types.\n");
       return ok;
     }
 
   /* Compute the scalar iteration cost.  */
   vect_compute_single_scalar_iteration_cost (loop_vinfo);
 
-  poly_uint64 saved_vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
   bool saved_can_use_partial_vectors_p
     = LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo);
 
@@ -2866,21 +2509,29 @@ start_over:
     return ok;
 
   /* If there are any SLP instances mark them as pure_slp.  */
-  if (vect_make_slp_decision (loop_vinfo))
-    {
-      /* Find stmts that need to be both vectorized and SLPed.  */
-      vect_detect_hybrid_slp (loop_vinfo);
+  if (!vect_make_slp_decision (loop_vinfo))
+    return opt_result::failure_at (vect_location, "no stmts to vectorize.\n");
 
-      /* Update the vectorization factor based on the SLP decision.  */
-      vect_update_vf_for_slp (loop_vinfo);
+  /* Find stmts that need to be both vectorized and SLPed.  */
+  if (!vect_detect_hybrid_slp (loop_vinfo))
+    return opt_result::failure_at (vect_location, "needs non-SLP handling\n");
 
-      /* Optimize the SLP graph with the vectorization factor fixed.  */
-      vect_optimize_slp (loop_vinfo);
-
-      /* Gather the loads reachable from the SLP graph entries.  */
-      vect_gather_slp_loads (loop_vinfo);
+  /* Determine the vectorization factor from the SLP decision.  */
+  LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+    = LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo);
+  if (dump_enabled_p ())
+    {
+      dump_printf_loc (MSG_NOTE, vect_location, "vectorization factor = ");
+      dump_dec (MSG_NOTE, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
+      dump_printf (MSG_NOTE, "\n");
     }
 
+  /* Optimize the SLP graph with the vectorization factor fixed.  */
+  vect_optimize_slp (loop_vinfo);
+
+  /* Gather the loads reachable from the SLP graph entries.  */
+  vect_gather_slp_loads (loop_vinfo);
+
   /* We don't expect to have to roll back to anything other than an empty
      set of rgroups.  */
   gcc_assert (LOOP_VINFO_MASKS (loop_vinfo).is_empty ());
@@ -2949,19 +2600,6 @@ start_over:
       goto again;
     }
 
-  /* Dissolve SLP-only groups.  */
-  vect_dissolve_slp_only_groups (loop_vinfo);
-
-  /* Scan all the remaining operations in the loop that we did not catch
-     during SLP build and make sure we fail.  */
-  ok = vect_analyze_loop_operations (loop_vinfo);
-  if (!ok)
-    {
-      ok = opt_result::failure_at (vect_location,
-				   "bad operation or unsupported loop bound\n");
-      goto again;
-    }
-
   /* For now, we don't expect to mix both masking and length approaches for one
      loop, disable it if both are recorded.  */
   if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
@@ -3241,7 +2879,7 @@ again:
 	continue;
       vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
       unsigned int size = DR_GROUP_SIZE (vinfo);
-      tree vectype = STMT_VINFO_VECTYPE (vinfo);
+      tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
       if (vect_store_lanes_supported (vectype, size, false) == IFN_LAST
 	 && ! known_eq (TYPE_VECTOR_SUBPARTS (vectype), 1U)
 	 && ! vect_grouped_store_supported (vectype, size))
@@ -3255,7 +2893,7 @@ again:
 	      vinfo = DR_GROUP_FIRST_ELEMENT (vinfo);
 	      bool single_element_p = !DR_GROUP_NEXT_ELEMENT (vinfo);
 	      size = DR_GROUP_SIZE (vinfo);
-	      vectype = STMT_VINFO_VECTYPE (vinfo);
+	      vectype = SLP_TREE_VECTYPE (node);
 	      if (vect_load_lanes_supported (vectype, size, false) == IFN_LAST
 		  && ! vect_grouped_load_supported (vectype, single_element_p,
 						    size))
@@ -3271,8 +2909,8 @@ again:
     dump_printf_loc (MSG_NOTE, vect_location,
 		     "re-trying with single-lane SLP\n");
 
-  /* Restore vectorization factor as it were without SLP.  */
-  LOOP_VINFO_VECT_FACTOR (loop_vinfo) = saved_vectorization_factor;
+  /* Reset the vectorization factor.  */
+  LOOP_VINFO_VECT_FACTOR (loop_vinfo) = 0;
   /* Free the SLP instances.  */
   FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), j, instance)
     vect_free_slp_instance (instance);
@@ -3400,8 +3038,10 @@ vect_joust_loop_vinfos (loop_vec_info new_loop_vinfo,
 }
 
 /* Analyze LOOP with VECTOR_MODES[MODE_I] and as epilogue if ORIG_LOOP_VINFO is
-   not NULL.  Set AUTODETECTED_VECTOR_MODE if VOIDmode and advance
-   MODE_I to the next mode useful to analyze.
+   not NULL.  When MASKED_P is not -1 override the default
+   LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P with it.
+   Set AUTODETECTED_VECTOR_MODE if VOIDmode and advance MODE_I to the next
+   mode useful to analyze.
    Return the loop_vinfo on success and wrapped null on failure.  */
 
 static opt_loop_vec_info
@@ -3409,6 +3049,7 @@ vect_analyze_loop_1 (class loop *loop, vec_info_shared *shared,
 		     const vect_loop_form_info *loop_form_info,
 		     loop_vec_info orig_loop_vinfo,
 		     const vector_modes &vector_modes, unsigned &mode_i,
+		     int masked_p,
 		     machine_mode &autodetected_vector_mode,
 		     bool &fatal)
 {
@@ -3417,6 +3058,8 @@ vect_analyze_loop_1 (class loop *loop, vec_info_shared *shared,
 
   machine_mode vector_mode = vector_modes[mode_i];
   loop_vinfo->vector_mode = vector_mode;
+  if (masked_p != -1)
+    loop_vinfo->can_use_partial_vectors_p = masked_p;
   unsigned int suggested_unroll_factor = 1;
   unsigned slp_done_for_suggested_uf = 0;
 
@@ -3600,7 +3243,7 @@ vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
       cached_vf_per_mode[last_mode_i] = -1;
       opt_loop_vec_info loop_vinfo
 	= vect_analyze_loop_1 (loop, shared, &loop_form_info,
-			       NULL, vector_modes, mode_i,
+			       NULL, vector_modes, mode_i, -1,
 			       autodetected_vector_mode, fatal);
       if (fatal)
 	break;
@@ -3685,18 +3328,21 @@ vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
      array may contain length-agnostic and length-specific modes.  Their
      ordering is not guaranteed, so we could end up picking a mode for the main
      loop that is after the epilogue's optimal mode.  */
+  int masked_p = -1;
   if (!unlimited_cost_model (loop)
-      && first_loop_vinfo->vector_costs->suggested_epilogue_mode () != VOIDmode)
+      && (first_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p)
+	  != VOIDmode))
     {
       vector_modes[0]
-	= first_loop_vinfo->vector_costs->suggested_epilogue_mode ();
+	= first_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p);
       cached_vf_per_mode[0] = 0;
     }
   else
     vector_modes[0] = autodetected_vector_mode;
   mode_i = 0;
 
-  bool supports_partial_vectors = param_vect_partial_vector_usage != 0;
+  bool supports_partial_vectors = (param_vect_partial_vector_usage != 0
+				   || masked_p == 1);
   machine_mode mask_mode;
   if (supports_partial_vectors
       && !partial_vectors_supported_p ()
@@ -3710,6 +3356,10 @@ vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
   loop_vec_info orig_loop_vinfo = first_loop_vinfo;
   do
     {
+      /* Let the user override what the target suggests.  */
+      if (OPTION_SET_P (param_vect_partial_vector_usage))
+	masked_p = -1;
+
       while (1)
 	{
 	  /* If the target does not support partial vectors we can shorten the
@@ -3750,7 +3400,7 @@ vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
 	  opt_loop_vec_info loop_vinfo
 	    = vect_analyze_loop_1 (loop, shared, &loop_form_info,
 				   orig_loop_vinfo,
-				   vector_modes, mode_i,
+				   vector_modes, mode_i, masked_p,
 				   autodetected_vector_mode, fatal);
 	  if (fatal)
 	    break;
@@ -3781,6 +3431,9 @@ vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
 		break;
 	    }
 
+	  /* Revert back to the default from the suggested prefered
+	     epilogue vectorization mode.  */
+	  masked_p = -1;
 	  if (mode_i == vector_modes.length ())
 	    break;
 	}
@@ -3791,13 +3444,14 @@ vect_analyze_loop (class loop *loop, gimple *loop_vectorized_call,
 
       /* When we selected a first vectorized epilogue, see if the target
 	 suggests to have another one.  */
+      masked_p = -1;
       if (!unlimited_cost_model (loop)
 	  && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (orig_loop_vinfo)
-	  && (orig_loop_vinfo->vector_costs->suggested_epilogue_mode ()
+	  && (orig_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p)
 	      != VOIDmode))
 	{
 	  vector_modes[0]
-	    = orig_loop_vinfo->vector_costs->suggested_epilogue_mode ();
+	    = orig_loop_vinfo->vector_costs->suggested_epilogue_mode (masked_p);
 	  cached_vf_per_mode[0] = 0;
 	  mode_i = 0;
 	}
@@ -5332,7 +4986,7 @@ vect_is_emulated_mixed_dot_prod (stmt_vec_info stmt_info)
 
 static void
 vect_model_reduction_cost (loop_vec_info loop_vinfo,
-			   stmt_vec_info stmt_info, internal_fn reduc_fn,
+			   slp_tree node, internal_fn reduc_fn,
 			   vect_reduction_type reduction_type,
 			   int ncopies, stmt_vector_for_cost *cost_vec)
 {
@@ -5348,9 +5002,10 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
   if (reduction_type == COND_REDUCTION)
     ncopies *= 2;
 
-  vectype = STMT_VINFO_VECTYPE (stmt_info);
+  vectype = SLP_TREE_VECTYPE (node);
   mode = TYPE_MODE (vectype);
-  stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
+  stmt_vec_info orig_stmt_info
+    = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (node));
 
   gimple_match_op op;
   if (!gimple_extract_op (orig_stmt_info->stmt, &op))
@@ -5368,16 +5023,16 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
       if (reduc_fn != IFN_LAST)
 	/* Count one reduction-like operation per vector.  */
 	inside_cost = record_stmt_cost (cost_vec, ncopies, vec_to_scalar,
-					stmt_info, 0, vect_body);
+					node, 0, vect_body);
       else
 	{
 	  /* Use NELEMENTS extracts and NELEMENTS scalar ops.  */
 	  unsigned int nelements = ncopies * vect_nunits_for_cost (vectype);
 	  inside_cost = record_stmt_cost (cost_vec, nelements,
-					  vec_to_scalar, stmt_info, 0,
+					  vec_to_scalar, node, 0,
 					  vect_body);
 	  inside_cost += record_stmt_cost (cost_vec, nelements,
-					   scalar_stmt, stmt_info, 0,
+					   scalar_stmt, node, 0,
 					   vect_body);
 	}
     }
@@ -5394,7 +5049,7 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
 	/* We need the initial reduction value.  */
 	prologue_stmts = 1;
       prologue_cost += record_stmt_cost (cost_vec, prologue_stmts,
-					 scalar_to_vec, stmt_info, 0,
+					 scalar_to_vec, node, 0,
 					 vect_prologue);
     }
 
@@ -5411,24 +5066,24 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
 	    {
 	      /* An EQ stmt and an COND_EXPR stmt.  */
 	      epilogue_cost += record_stmt_cost (cost_vec, 2,
-						 vector_stmt, stmt_info, 0,
+						 vector_stmt, node, 0,
 						 vect_epilogue);
 	      /* Reduction of the max index and a reduction of the found
 		 values.  */
 	      epilogue_cost += record_stmt_cost (cost_vec, 2,
-						 vec_to_scalar, stmt_info, 0,
+						 vec_to_scalar, node, 0,
 						 vect_epilogue);
 	      /* A broadcast of the max value.  */
 	      epilogue_cost += record_stmt_cost (cost_vec, 1,
-						 scalar_to_vec, stmt_info, 0,
+						 scalar_to_vec, node, 0,
 						 vect_epilogue);
 	    }
 	  else
 	    {
 	      epilogue_cost += record_stmt_cost (cost_vec, 1, vector_stmt,
-						 stmt_info, 0, vect_epilogue);
+						 node, 0, vect_epilogue);
 	      epilogue_cost += record_stmt_cost (cost_vec, 1,
-						 vec_to_scalar, stmt_info, 0,
+						 vec_to_scalar, node, 0,
 						 vect_epilogue);
 	    }
 	}
@@ -5438,12 +5093,12 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
 	  /* Extraction of scalar elements.  */
 	  epilogue_cost += record_stmt_cost (cost_vec,
 					     2 * estimated_nunits,
-					     vec_to_scalar, stmt_info, 0,
+					     vec_to_scalar, node, 0,
 					     vect_epilogue);
 	  /* Scalar max reductions via COND_EXPR / MAX_EXPR.  */
 	  epilogue_cost += record_stmt_cost (cost_vec,
 					     2 * estimated_nunits - 3,
-					     scalar_stmt, stmt_info, 0,
+					     scalar_stmt, node, 0,
 					     vect_epilogue);
 	}
       else if (reduction_type == EXTRACT_LAST_REDUCTION
@@ -5469,10 +5124,10 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
 		 Also requires scalar extract.  */
 	      epilogue_cost += record_stmt_cost (cost_vec,
 						 exact_log2 (nelements) * 2,
-						 vector_stmt, stmt_info, 0,
+						 vector_stmt, node, 0,
 						 vect_epilogue);
 	      epilogue_cost += record_stmt_cost (cost_vec, 1,
-						 vec_to_scalar, stmt_info, 0,
+						 vec_to_scalar, node, 0,
 						 vect_epilogue);
 	    }
 	  else
@@ -5480,7 +5135,7 @@ vect_model_reduction_cost (loop_vec_info loop_vinfo,
 	       elements, we have N extracts and N-1 reduction ops.  */
 	    epilogue_cost += record_stmt_cost (cost_vec,
 					       nelements + nelements - 1,
-					       vector_stmt, stmt_info, 0,
+					       vector_stmt, node, 0,
 					       vect_epilogue);
 	}
     }
@@ -7030,7 +6685,7 @@ vectorize_fold_left_reduction (loop_vec_info loop_vinfo,
 			       vec_loop_lens *lens)
 {
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
-  tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype_out = SLP_TREE_VECTYPE (slp_node);
   internal_fn mask_reduc_fn = get_masked_reduction_fn (reduc_fn, vectype_in);
 
   gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
@@ -7417,14 +7072,13 @@ vectorizable_lane_reducing (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
 
   for (int i = 0; i < (int) gimple_num_ops (stmt) - 1; i++)
     {
-      stmt_vec_info def_stmt_info;
       slp_tree slp_op;
       tree op;
       tree vectype;
       enum vect_def_type dt;
 
-      if (!vect_is_simple_use (loop_vinfo, stmt_info, slp_node, i, &op,
-			       &slp_op, &dt, &vectype, &def_stmt_info))
+      if (!vect_is_simple_use (loop_vinfo, slp_node, i, &op,
+			       &slp_op, &dt, &vectype))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -7493,7 +7147,7 @@ vectorizable_lane_reducing (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
     }
 
   /* Transform via vect_transform_reduction.  */
-  STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
+  SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
   return true;
 }
 
@@ -7595,18 +7249,17 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
 	      }
 	  /* Analysis for double-reduction is done on the outer
 	     loop PHI, nested cycles have no further restrictions.  */
-	  STMT_VINFO_TYPE (stmt_info) = cycle_phi_info_type;
+	  SLP_TREE_TYPE (slp_node) = cycle_phi_info_type;
 	}
       else
-	STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
+	SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
       return true;
     }
 
-  stmt_vec_info orig_stmt_of_analysis = stmt_info;
   stmt_vec_info phi_info = stmt_info;
   if (!is_a <gphi *> (stmt_info->stmt))
     {
-      STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = reduc_vec_info_type;
       return true;
     }
   if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
@@ -7646,17 +7299,17 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
   unsigned reduc_chain_length = 0;
   bool only_slp_reduc_chain = true;
   stmt_info = NULL;
-  slp_tree slp_for_stmt_info = slp_node_instance->root;
+  slp_tree slp_for_stmt_info = NULL;
+  slp_tree vdef_slp = slp_node_instance->root;
   /* For double-reductions we start SLP analysis at the inner loop LC PHI
      which is the def of the outer loop live stmt.  */
   if (STMT_VINFO_DEF_TYPE (reduc_info) == vect_double_reduction_def)
-    slp_for_stmt_info = SLP_TREE_CHILDREN (slp_for_stmt_info)[0];
+    vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[0];
   while (reduc_def != PHI_RESULT (reduc_def_phi))
     {
       stmt_vec_info def = loop_vinfo->lookup_def (reduc_def);
       stmt_vec_info vdef = vect_stmt_to_vectorize (def);
       int reduc_idx = STMT_VINFO_REDUC_IDX (vdef);
-
       if (reduc_idx == -1)
 	{
 	  if (dump_enabled_p ())
@@ -7673,14 +7326,9 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
 	 the SLP node with live lane zero the other live lanes also
 	 need to be identified as part of a reduction to be able
 	 to skip code generation for them.  */
-      if (slp_for_stmt_info)
-	{
-	  for (auto s : SLP_TREE_SCALAR_STMTS (slp_for_stmt_info))
-	    if (STMT_VINFO_LIVE_P (s))
-	      STMT_VINFO_REDUC_DEF (vect_orig_stmt (s)) = phi_info;
-	}
-      else if (STMT_VINFO_LIVE_P (vdef))
-	STMT_VINFO_REDUC_DEF (def) = phi_info;
+      for (auto s : SLP_TREE_SCALAR_STMTS (vdef_slp))
+	if (STMT_VINFO_LIVE_P (s))
+	  STMT_VINFO_REDUC_DEF (vect_orig_stmt (s)) = phi_info;
       gimple_match_op op;
       if (!gimple_extract_op (vdef->stmt, &op))
 	{
@@ -7699,32 +7347,33 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
 				 "conversion in the reduction chain.\n");
 	      return false;
 	    }
+	  vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[0];
 	}
       else
 	{
 	  /* First non-conversion stmt.  */
 	  if (!stmt_info)
-	    stmt_info = vdef;
+	    {
+	      stmt_info = vdef;
+	      slp_for_stmt_info = vdef_slp;
+	    }
 
 	  if (lane_reducing_op_p (op.code))
 	    {
-	      enum vect_def_type dt;
-	      tree vectype_op;
-
 	      /* The last operand of lane-reducing operation is for
 		 reduction.  */
 	      gcc_assert (reduc_idx > 0 && reduc_idx == (int) op.num_ops - 1);
 
-	      if (!vect_is_simple_use (op.ops[0], loop_vinfo, &dt, &vectype_op))
-		return false;
-
+	      slp_tree op_node = SLP_TREE_CHILDREN (vdef_slp)[0];
+	      tree vectype_op = SLP_TREE_VECTYPE (op_node);
 	      tree type_op = TREE_TYPE (op.ops[0]);
-
 	      if (!vectype_op)
 		{
 		  vectype_op = get_vectype_for_scalar_type (loop_vinfo,
 							    type_op);
-		  if (!vectype_op)
+		  if (!vectype_op
+		      || !vect_maybe_update_slp_op_vectype (op_node,
+							    vectype_op))
 		    return false;
 		}
 
@@ -7746,14 +7395,14 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
 		       < GET_MODE_SIZE (SCALAR_TYPE_MODE (type_op))))
 		vectype_in = vectype_op;
 	    }
-	  else
+	  else if (!vectype_in)
 	    vectype_in = STMT_VINFO_VECTYPE (phi_info);
+	  if (!REDUC_GROUP_FIRST_ELEMENT (vdef))
+	    vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[reduc_idx];
 	}
 
       reduc_def = op.ops[reduc_idx];
       reduc_chain_length++;
-      if (!stmt_info)
-	slp_for_stmt_info = SLP_TREE_CHILDREN (slp_for_stmt_info)[0];
     }
   /* PHIs should not participate in patterns.  */
   gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info));
@@ -7805,7 +7454,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
         inside the loop body. The last operand is the reduction variable,
         which is defined by the loop-header-phi.  */
 
-  tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype_out = SLP_TREE_VECTYPE (slp_for_stmt_info);
   STMT_VINFO_REDUC_VECTYPE (reduc_info) = vectype_out;
   STMT_VINFO_REDUC_VECTYPE_IN (reduc_info) = vectype_in;
 
@@ -7855,7 +7504,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
 
       stmt_vec_info def_stmt_info;
       enum vect_def_type dt;
-      if (!vect_is_simple_use (loop_vinfo, stmt_info, slp_for_stmt_info,
+      if (!vect_is_simple_use (loop_vinfo, slp_for_stmt_info,
 			       i + opno_adjust, &op.ops[i], &slp_op[i], &dt,
 			       &vectype_op[i], &def_stmt_info))
 	{
@@ -8408,7 +8057,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
 	  return false;
 	}
 
-  vect_model_reduction_cost (loop_vinfo, stmt_info, reduc_fn,
+  vect_model_reduction_cost (loop_vinfo, slp_for_stmt_info, reduc_fn,
 			     reduction_type, ncopies, cost_vec);
   /* Cost the reduction op inside the loop if transformed via
      vect_transform_reduction for non-lane-reducing operation.  Otherwise
@@ -8420,7 +8069,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo,
       && reduction_type == FOLD_LEFT_REDUCTION)
     dump_printf_loc (MSG_NOTE, vect_location,
 		     "using an in-order (fold-left) reduction.\n");
-  STMT_VINFO_TYPE (orig_stmt_of_analysis) = cycle_phi_info_type;
+  SLP_TREE_TYPE (slp_node) = cycle_phi_info_type;
 
   /* All but single defuse-cycle optimized and fold-left reductions go
      through their own vectorizable_* routines.  */
@@ -8526,7 +8175,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
 			  stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
 			  slp_tree slp_node)
 {
-  tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype_out = SLP_TREE_VECTYPE (slp_node);
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   unsigned vec_num;
 
@@ -8553,7 +8202,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
   tree vectype_in = STMT_VINFO_REDUC_VECTYPE_IN (stmt_info);
 
   if (!vectype_in)
-    vectype_in = STMT_VINFO_VECTYPE (stmt_info);
+    vectype_in = SLP_TREE_VECTYPE (slp_node);
 
   vec_num = vect_get_num_copies (loop_vinfo, slp_node, vectype_in);
 
@@ -8618,7 +8267,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
   if (!cond_fn_p)
     {
       gcc_assert (reduc_index >= 0 && reduc_index <= 2);
-      vect_get_vec_defs (loop_vinfo, stmt_info, slp_node, 1,
+      vect_get_vec_defs (loop_vinfo, slp_node,
 			 single_defuse_cycle && reduc_index == 0
 			 ? NULL_TREE : op.ops[0], &vec_oprnds[0],
 			 single_defuse_cycle && reduc_index == 1
@@ -8633,19 +8282,19 @@ vect_transform_reduction (loop_vec_info loop_vinfo,
 	 vectype.  */
       gcc_assert (single_defuse_cycle
 		  && (reduc_index == 1 || reduc_index == 2));
-      vect_get_vec_defs (loop_vinfo, stmt_info, slp_node, 1, op.ops[0],
-			 truth_type_for (vectype_in), &vec_oprnds[0],
+      vect_get_vec_defs (loop_vinfo, slp_node, op.ops[0],
+			 &vec_oprnds[0],
 			 reduc_index == 1 ? NULL_TREE : op.ops[1],
-			 NULL_TREE, &vec_oprnds[1],
+			 &vec_oprnds[1],
 			 reduc_index == 2 ? NULL_TREE : op.ops[2],
-			 NULL_TREE, &vec_oprnds[2]);
+			 &vec_oprnds[2]);
     }
 
   /* For single def-use cycles get one copy of the vectorized reduction
      definition.  */
   if (single_defuse_cycle)
     {
-      vect_get_vec_defs (loop_vinfo, stmt_info, slp_node, 1,
+      vect_get_vec_defs (loop_vinfo, slp_node,
 			 reduc_index == 0 ? op.ops[0] : NULL_TREE,
 			 &vec_oprnds[0],
 			 reduc_index == 1 ? op.ops[1] : NULL_TREE,
@@ -8873,7 +8522,7 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo,
 			  stmt_vec_info stmt_info,
 			  slp_tree slp_node, slp_instance slp_node_instance)
 {
-  tree vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype_out = SLP_TREE_VECTYPE (slp_node);
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   int i;
   bool nested_cycle = false;
@@ -9104,7 +8753,19 @@ vectorizable_lc_phi (loop_vec_info loop_vinfo,
 			 "incompatible vector types for invariants\n");
       return false;
     }
-  STMT_VINFO_TYPE (stmt_info) = lc_phi_info_type;
+
+  /* ???  This can happen with data vs. mask uses of boolean.  */
+  if (!useless_type_conversion_p (SLP_TREE_VECTYPE (slp_node),
+				  SLP_TREE_VECTYPE
+				    (SLP_TREE_CHILDREN (slp_node)[0])))
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "missed mask promotion\n");
+      return false;
+    }
+
+  SLP_TREE_TYPE (slp_node) = lc_phi_info_type;
   return true;
 }
 
@@ -9114,13 +8775,13 @@ vect_transform_lc_phi (loop_vec_info loop_vinfo,
 		       slp_tree slp_node)
 {
 
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   tree scalar_dest = gimple_phi_result (stmt_info->stmt);
   basic_block bb = gimple_bb (stmt_info->stmt);
   edge e = single_pred_edge (bb);
   tree vec_dest = vect_create_destination_var (scalar_dest, vectype);
   auto_vec<tree> vec_oprnds;
-  vect_get_vec_defs (loop_vinfo, stmt_info, slp_node, 1,
+  vect_get_vec_defs (loop_vinfo, slp_node,
 		     gimple_phi_arg_def (stmt_info->stmt, 0), &vec_oprnds);
   for (unsigned i = 0; i < vec_oprnds.length (); i++)
     {
@@ -9137,7 +8798,7 @@ vect_transform_lc_phi (loop_vec_info loop_vinfo,
 
 bool
 vectorizable_phi (vec_info *,
-		  stmt_vec_info stmt_info, gimple **vec_stmt,
+		  stmt_vec_info stmt_info,
 		  slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   if (!is_a <gphi *> (stmt_info->stmt) || !slp_node)
@@ -9148,7 +8809,7 @@ vectorizable_phi (vec_info *,
 
   tree vectype = SLP_TREE_VECTYPE (slp_node);
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
       slp_tree child;
       unsigned i;
@@ -9189,7 +8850,7 @@ vectorizable_phi (vec_info *,
       if (gimple_phi_num_args (as_a <gphi *> (stmt_info->stmt)) > 1)
 	record_stmt_cost (cost_vec, SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node),
 			  vector_stmt, stmt_info, vectype, 0, vect_body);
-      STMT_VINFO_TYPE (stmt_info) = phi_info_type;
+      SLP_TREE_TYPE (slp_node) = phi_info_type;
       return true;
     }
 
@@ -9279,8 +8940,7 @@ vectorizable_phi (vec_info *,
 
 bool
 vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
-		     gimple **vec_stmt, slp_tree slp_node,
-		     stmt_vector_for_cost *cost_vec)
+		     slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   if (!loop_vinfo || !is_a<gphi *> (stmt_info->stmt))
     return false;
@@ -9291,14 +8951,10 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_first_order_recurrence)
     return false;
 
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-  unsigned ncopies;
-  if (slp_node)
-    ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
+  unsigned ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
   poly_int64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
-  unsigned dist = slp_node ? SLP_TREE_LANES (slp_node) : 1;
+  unsigned dist = SLP_TREE_LANES (slp_node);
   /* We need to be able to make progress with a single vector.  */
   if (maybe_gt (dist * 2, nunits))
     {
@@ -9309,6 +8965,33 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
       return false;
     }
 
+  /* We need to be able to build a { ..., a, b } init vector with
+     dist number of distinct trailing values.  Always possible
+     when dist == 1 or when nunits is constant or when the initializations
+     are uniform.  */
+  tree uniform_initval = NULL_TREE;
+  edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
+  for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
+    {
+      gphi *phi = as_a <gphi *> (s->stmt);
+      if (! uniform_initval)
+	uniform_initval = PHI_ARG_DEF_FROM_EDGE (phi, pe);
+      else if (! operand_equal_p (uniform_initval,
+				  PHI_ARG_DEF_FROM_EDGE (phi, pe)))
+	{
+	  uniform_initval = NULL_TREE;
+	  break;
+	}
+    }
+  if (!uniform_initval && !nunits.is_constant ())
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "cannot build initialization vector for "
+			 "first order recurrence\n");
+      return false;
+    }
+
   /* First-order recurrence autovectorization needs to handle permutation
      with indices = [nunits-1, nunits, nunits+1, ...].  */
   vec_perm_builder sel (nunits, 1, 3);
@@ -9316,48 +8999,30 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
     sel.quick_push (nunits - dist + i);
   vec_perm_indices indices (sel, 2, nunits);
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
       if (!can_vec_perm_const_p (TYPE_MODE (vectype), TYPE_MODE (vectype),
 				 indices))
 	return false;
 
-      if (slp_node)
-	{
-	  /* We eventually need to set a vector type on invariant
-	     arguments.  */
-	  unsigned j;
-	  slp_tree child;
-	  FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
-	    if (!vect_maybe_update_slp_op_vectype
-		  (child, SLP_TREE_VECTYPE (slp_node)))
-	      {
-		if (dump_enabled_p ())
-		  dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-				   "incompatible vector types for "
-				   "invariants\n");
-		return false;
-	      }
-	}
+      /* We eventually need to set a vector type on invariant
+	 arguments.  */
+      unsigned j;
+      slp_tree child;
+      FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node), j, child)
+	if (!vect_maybe_update_slp_op_vectype (child, vectype))
+	  {
+	    if (dump_enabled_p ())
+	      dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			       "incompatible vector types for "
+			       "invariants\n");
+	    return false;
+	  }
 
       /* Verify we have set up compatible types.  */
       edge le = loop_latch_edge (LOOP_VINFO_LOOP (loop_vinfo));
-      tree latch_vectype = NULL_TREE;
-      if (slp_node)
-	{
-	  slp_tree latch_def = SLP_TREE_CHILDREN (slp_node)[le->dest_idx];
-	  latch_vectype = SLP_TREE_VECTYPE (latch_def);
-	}
-      else
-	{
-	  tree latch_def = PHI_ARG_DEF_FROM_EDGE (phi, le);
-	  if (TREE_CODE (latch_def) == SSA_NAME)
-	    {
-	      stmt_vec_info latch_def_info = loop_vinfo->lookup_def (latch_def);
-	      latch_def_info = vect_stmt_to_vectorize (latch_def_info);
-	      latch_vectype = STMT_VINFO_VECTYPE (latch_def_info);
-	    }
-	}
+      slp_tree latch_def = SLP_TREE_CHILDREN (slp_node)[le->dest_idx];
+      tree latch_vectype = SLP_TREE_VECTYPE (latch_def);
       if (!types_compatible_p (latch_vectype, vectype))
 	return false;
 
@@ -9365,9 +9030,6 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
 	 for each copy.  With SLP the prologue value is explicitly
 	 represented and costed separately.  */
       unsigned prologue_cost = 0;
-      if (!slp_node)
-	prologue_cost = record_stmt_cost (cost_vec, 1, scalar_to_vec,
-					  stmt_info, 0, vect_prologue);
       unsigned inside_cost = record_stmt_cost (cost_vec, ncopies, vector_stmt,
 					       stmt_info, 0, vect_body);
       if (dump_enabled_p ())
@@ -9376,25 +9038,42 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
 			 "prologue_cost = %d .\n", inside_cost,
 			 prologue_cost);
 
-      STMT_VINFO_TYPE (stmt_info) = recurr_info_type;
+      SLP_TREE_TYPE (slp_node) = recurr_info_type;
       return true;
     }
 
-  edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
-  basic_block bb = gimple_bb (phi);
-  tree preheader = PHI_ARG_DEF_FROM_EDGE (phi, pe);
-  if (!useless_type_conversion_p (TREE_TYPE (vectype), TREE_TYPE (preheader)))
+  tree vec_init;
+  if (! uniform_initval)
     {
-      gimple_seq stmts = NULL;
-      preheader = gimple_convert (&stmts, TREE_TYPE (vectype), preheader);
-      gsi_insert_seq_on_edge_immediate (pe, stmts);
+      vec<constructor_elt, va_gc> *v = NULL;
+      vec_alloc (v, nunits.to_constant ());
+      for (unsigned i = 0; i < nunits.to_constant () - dist; ++i)
+	CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
+				build_zero_cst (TREE_TYPE (vectype)));
+      for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
+	{
+	  gphi *phi = as_a <gphi *> (s->stmt);
+	  tree preheader = PHI_ARG_DEF_FROM_EDGE (phi, pe);
+	  if (!useless_type_conversion_p (TREE_TYPE (vectype),
+					  TREE_TYPE (preheader)))
+	    {
+	      gimple_seq stmts = NULL;
+	      preheader = gimple_convert (&stmts,
+					  TREE_TYPE (vectype), preheader);
+	      gsi_insert_seq_on_edge_immediate (pe, stmts);
+	    }
+	  CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, preheader);
+	}
+      vec_init = build_constructor (vectype, v);
     }
-  tree vec_init = build_vector_from_val (vectype, preheader);
+  else
+    vec_init = uniform_initval;
   vec_init = vect_init_vector (loop_vinfo, stmt_info, vec_init, vectype, NULL);
 
   /* Create the vectorized first-order PHI node.  */
   tree vec_dest = vect_get_new_vect_var (vectype,
 					 vect_simple_var, "vec_recur_");
+  basic_block bb = gimple_bb (phi);
   gphi *new_phi = create_phi_node (vec_dest, bb);
   add_phi_arg (new_phi, vec_init, pe, UNKNOWN_LOCATION);
 
@@ -9419,14 +9098,9 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
 				 NULL, perm);
       vect_finish_stmt_generation (loop_vinfo, stmt_info, vperm, &gsi2);
 
-      if (slp_node)
-	slp_node->push_vec_def (vperm);
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (vperm);
+      slp_node->push_vec_def (vperm);
     }
 
-  if (!slp_node)
-    *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
   return true;
 }
 
@@ -9738,7 +9412,7 @@ vect_update_nonlinear_iv (gimple_seq* stmts, tree vectype,
 static bool
 vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
 				  stmt_vec_info stmt_info,
-				  gimple **vec_stmt, slp_tree slp_node,
+				  slp_tree slp_node,
 				  stmt_vector_for_cost *cost_vec)
 {
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
@@ -9894,7 +9568,7 @@ vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
       gcc_unreachable ();
     }
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
       unsigned inside_cost = 0, prologue_cost = 0;
       /* loop cost for vec_loop. Neg induction doesn't have any
@@ -9917,7 +9591,7 @@ vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
 			 "prologue_cost = %d. \n", inside_cost,
 			 prologue_cost);
 
-      STMT_VINFO_TYPE (stmt_info) = induc_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = induc_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_nonlinear_induction");
       return true;
     }
@@ -10049,8 +9723,7 @@ vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
 bool
 vectorizable_induction (loop_vec_info loop_vinfo,
 			stmt_vec_info stmt_info,
-			gimple **vec_stmt, slp_tree slp_node,
-			stmt_vector_for_cost *cost_vec)
+			slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
   bool nested_in_vect_loop = false;
@@ -10084,7 +9757,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
   /* Handle nonlinear induction in a separate place.  */
   if (induction_type != vect_step_op_add)
     return vectorizable_nonlinear_induction (loop_vinfo, stmt_info,
-					     vec_stmt, slp_node, cost_vec);
+					     slp_node, cost_vec);
 
   tree vectype = SLP_TREE_VECTYPE (slp_node);
   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
@@ -10165,6 +9838,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
     }
   tree stept = TREE_TYPE (step_expr);
   tree step_vectype = get_same_sized_vectype (stept, vectype);
+  stept = TREE_TYPE (step_vectype);
 
   /* Check for target support of the vectorized arithmetic used here.  */
   if (!target_supports_op_p (step_vectype, PLUS_EXPR, optab_default)
@@ -10187,7 +9861,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
 	}
     }
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
       unsigned inside_cost = 0, prologue_cost = 0;
       /* We eventually need to set a vector type on invariant
@@ -10218,7 +9892,7 @@ vectorizable_induction (loop_vec_info loop_vinfo,
 			 "prologue_cost = %d .\n", inside_cost,
 			 prologue_cost);
 
-      STMT_VINFO_TYPE (stmt_info) = induc_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = induc_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_induction");
       return true;
     }
@@ -10619,9 +10293,8 @@ vectorizable_induction (loop_vec_info loop_vinfo,
    helper function for vectorizable_live_operation.  */
 
 static tree
-vectorizable_live_operation_1 (loop_vec_info loop_vinfo,
-			       stmt_vec_info stmt_info, basic_block exit_bb,
-			       tree vectype, int ncopies, slp_tree slp_node,
+vectorizable_live_operation_1 (loop_vec_info loop_vinfo, basic_block exit_bb,
+			       tree vectype, slp_tree slp_node,
 			       tree bitsize, tree bitstart, tree vec_lhs,
 			       tree lhs_type, gimple_stmt_iterator *exit_gsi)
 {
@@ -10652,8 +10325,7 @@ vectorizable_live_operation_1 (loop_vec_info loop_vinfo,
 
 	 where VEC_LHS is the vectorized live-out result and MASK is
 	 the loop mask for the final iteration.  */
-      gcc_assert (ncopies == 1
-		  && (!slp_node || SLP_TREE_LANES (slp_node) == 1));
+      gcc_assert (SLP_TREE_LANES (slp_node) == 1);
       gimple_seq tem = NULL;
       gimple_stmt_iterator gsi = gsi_last (tem);
       tree len = vect_get_loop_len (loop_vinfo, &gsi,
@@ -10688,8 +10360,8 @@ vectorizable_live_operation_1 (loop_vec_info loop_vinfo,
 
 	 where VEC_LHS is the vectorized live-out result and MASK is
 	 the loop mask for the final iteration.  */
-      gcc_assert (!slp_node || SLP_TREE_LANES (slp_node) == 1);
-      tree scalar_type = TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info));
+      gcc_assert (SLP_TREE_LANES (slp_node) == 1);
+      tree scalar_type = TREE_TYPE (vectype);
       gimple_seq tem = NULL;
       gimple_stmt_iterator gsi = gsi_last (tem);
       tree mask = vect_get_loop_mask (loop_vinfo, &gsi,
@@ -10735,11 +10407,8 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   imm_use_iterator imm_iter;
   tree lhs, lhs_type, bitsize;
-  tree vectype = (slp_node
-		  ? SLP_TREE_VECTYPE (slp_node)
-		  : STMT_VINFO_VECTYPE (stmt_info));
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
-  int ncopies;
   gimple *use_stmt;
   use_operand_p use_p;
   auto_vec<tree> vec_oprnds;
@@ -10758,7 +10427,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 	return true;
       /* For SLP reductions we vectorize the epilogue for all involved stmts
 	 together.  */
-      if (slp_node && !REDUC_GROUP_FIRST_ELEMENT (stmt_info) && slp_index != 0)
+      if (!REDUC_GROUP_FIRST_ELEMENT (stmt_info) && slp_index != 0)
 	return true;
       stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
       gcc_assert (reduc_info->is_reduc_info);
@@ -10776,7 +10445,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 	 block, but we have to find an alternate exit first.  */
       if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
 	{
-	  slp_tree phis_node = slp_node ? slp_node_instance->reduc_phis : NULL;
+	  slp_tree phis_node = slp_node_instance->reduc_phis;
 	  for (auto exit : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
 	    if (exit != LOOP_VINFO_IV_EXIT (loop_vinfo))
 	      {
@@ -10807,32 +10476,24 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
       return true;
     }
 
-  if (slp_node)
-    ncopies = 1;
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype);
-
-  if (slp_node)
-    {
-      gcc_assert (slp_index >= 0);
+  gcc_assert (slp_index >= 0);
 
-      /* Get the last occurrence of the scalar index from the concatenation of
-	 all the slp vectors. Calculate which slp vector it is and the index
-	 within.  */
-      int num_scalar = SLP_TREE_LANES (slp_node);
-      int num_vec = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
-      poly_uint64 pos = (num_vec * nunits) - num_scalar + slp_index;
+  /* Get the last occurrence of the scalar index from the concatenation of
+     all the slp vectors. Calculate which slp vector it is and the index
+     within.  */
+  int num_scalar = SLP_TREE_LANES (slp_node);
+  int num_vec = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
+  poly_uint64 pos = (num_vec * nunits) - num_scalar + slp_index;
 
-      /* Calculate which vector contains the result, and which lane of
-	 that vector we need.  */
-      if (!can_div_trunc_p (pos, nunits, &vec_entry, &vec_index))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "Cannot determine which vector holds the"
-			     " final result.\n");
-	  return false;
-	}
+  /* Calculate which vector contains the result, and which lane of
+     that vector we need.  */
+  if (!can_div_trunc_p (pos, nunits, &vec_entry, &vec_index))
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "Cannot determine which vector holds the"
+			 " final result.\n");
+      return false;
     }
 
   if (!vec_stmt_p)
@@ -10840,7 +10501,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
       /* No transformation required.  */
       if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
 	{
-	  if (slp_node && SLP_TREE_LANES (slp_node) != 1)
+	  if (SLP_TREE_LANES (slp_node) != 1)
 	    {
 	      if (dump_enabled_p ())
 		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -10849,8 +10510,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 				 "the loop.\n");
 	      LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
 	    }
-	  else if (ncopies > 1
-		   || (slp_node && SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) > 1))
+	  else if (SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) > 1)
 	    {
 	      if (dump_enabled_p ())
 		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -10860,8 +10520,6 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 	    }
 	  else
 	    {
-	      gcc_assert (ncopies == 1
-			  && (!slp_node || SLP_TREE_LANES (slp_node) == 1));
 	      if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
 						  OPTIMIZE_FOR_SPEED))
 		vect_record_loop_mask (loop_vinfo,
@@ -10903,40 +10561,21 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
   bitsize = vector_element_bits_tree (vectype);
 
   /* Get the vectorized lhs of STMT and the lane to use (counted in bits).  */
-  tree vec_lhs, vec_lhs0, bitstart;
-  gimple *vec_stmt, *vec_stmt0;
-  if (slp_node)
-    {
-      gcc_assert (!loop_vinfo
-		  || ((!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
-		       && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
-		      || SLP_TREE_LANES (slp_node) == 1));
+  gcc_assert (!loop_vinfo
+	      || ((!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
+		   && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
+		  || SLP_TREE_LANES (slp_node) == 1));
 
-      /* Get the correct slp vectorized stmt.  */
-      vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
-      vec_stmt = SSA_NAME_DEF_STMT (vec_lhs);
+  /* Get the correct slp vectorized stmt.  */
+  tree vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
+  gimple *vec_stmt = SSA_NAME_DEF_STMT (vec_lhs);
 
-      /* In case we need to early break vectorize also get the first stmt.  */
-      vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0];
-      vec_stmt0 = SSA_NAME_DEF_STMT (vec_lhs0);
+  /* In case we need to early break vectorize also get the first stmt.  */
+  tree vec_lhs0 = SLP_TREE_VEC_DEFS (slp_node)[0];
 
-      /* Get entry to use.  */
-      bitstart = bitsize_int (vec_index);
-      bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart);
-    }
-  else
-    {
-      /* For multiple copies, get the last copy.  */
-      vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info).last ();
-      vec_lhs = gimple_get_lhs (vec_stmt);
-
-      /* In case we need to early break vectorize also get the first stmt.  */
-      vec_stmt0 = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-      vec_lhs0 = gimple_get_lhs (vec_stmt0);
-
-      /* Get the last lane in the vector.  */
-      bitstart = int_const_binop (MULT_EXPR, bitsize, bitsize_int (nunits - 1));
-    }
+  /* Get entry to use.  */
+  tree bitstart = bitsize_int (vec_index);
+  bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart);
 
   if (loop_vinfo)
     {
@@ -10985,8 +10624,8 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 
 	      gimple_stmt_iterator exit_gsi;
 	      tree new_tree
-		= vectorizable_live_operation_1 (loop_vinfo, stmt_info,
-						 e->dest, vectype, ncopies,
+		= vectorizable_live_operation_1 (loop_vinfo,
+						 e->dest, vectype,
 						 slp_node, bitsize,
 						 tmp_bitstart, tmp_vec_lhs,
 						 lhs_type, &exit_gsi);
@@ -11412,7 +11051,7 @@ vect_get_loop_len (loop_vec_info loop_vinfo, gimple_stmt_iterator *gsi,
 	  factor = exact_div (nunits1, nunits2).to_constant ();
 	  tree iv_type = LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo);
 	  gimple_seq seq = NULL;
-	  loop_len = gimple_build (&seq, RDIV_EXPR, iv_type, loop_len,
+	  loop_len = gimple_build (&seq, EXACT_DIV_EXPR, iv_type, loop_len,
 				   build_int_cst (iv_type, factor));
 	  if (seq)
 	    gsi_insert_seq_before (gsi, seq, GSI_SAME_STMT);
@@ -11472,7 +11111,7 @@ scale_profile_for_vect_loop (class loop *loop, edge exit_e, unsigned vf, bool fl
   profile_count entry_count = loop_preheader_edge (loop)->count ();
 
   /* If we have unreliable loop profile avoid dropping entry
-     count bellow header count.  This can happen since loops
+     count below header count.  This can happen since loops
      has unrealistically low trip counts.  */
   while (vf > 1
 	 && loop->header->count > entry_count
@@ -11653,11 +11292,9 @@ update_epilogue_loop_vinfo (class loop *epilogue, tree advance)
 	 updated offset we set using ADVANCE.  Instead we have to make sure the
 	 reference in the data references point to the corresponding copy of
 	 the original in the epilogue.  Make sure to update both
-	 gather/scatters recognized by dataref analysis and also other
-	 refs that get_load_store_type classified as VMAT_GATHER_SCATTER.  */
+	 gather/scatters recognized by dataref analysis.  */
       auto vstmt_vinfo = vect_stmt_to_vectorize (stmt_vinfo);
-      if (STMT_VINFO_MEMORY_ACCESS_TYPE (vstmt_vinfo) == VMAT_GATHER_SCATTER
-	  || STMT_VINFO_STRIDED_P (vstmt_vinfo)
+      if (STMT_VINFO_STRIDED_P (vstmt_vinfo)
 	  || STMT_VINFO_GATHER_SCATTER_P (vstmt_vinfo))
 	{
 	  /* ???  As we copy epilogues from the main loop incremental
@@ -11679,9 +11316,6 @@ update_epilogue_loop_vinfo (class loop *epilogue, tree advance)
 
   /* Remember the advancement made.  */
   LOOP_VINFO_DRS_ADVANCED_BY (epilogue_vinfo) = advance;
-
-  epilogue_vinfo->shared->datarefs_copy.release ();
-  epilogue_vinfo->shared->save_datarefs ();
 }
 
 /*  When vectorizing early break statements instructions that happen before
@@ -11787,7 +11421,8 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
 
   DUMP_VECT_SCOPE ("vec_transform_loop");
 
-  loop_vinfo->shared->check_datarefs ();
+  if (! LOOP_VINFO_EPILOGUE_P (loop_vinfo))
+    loop_vinfo->shared->check_datarefs ();
 
   /* Use the more conservative vectorization threshold.  If the number
      of iterations is constant assume the cost check has been performed
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index 0f6d6b7..ffb320f 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -130,7 +130,6 @@ vect_init_pattern_stmt (vec_info *vinfo, gimple *pattern_stmt,
   STMT_VINFO_RELATED_STMT (pattern_stmt_info) = orig_stmt_info;
   STMT_VINFO_DEF_TYPE (pattern_stmt_info)
     = STMT_VINFO_DEF_TYPE (orig_stmt_info);
-  STMT_VINFO_TYPE (pattern_stmt_info) = STMT_VINFO_TYPE (orig_stmt_info);
   if (!STMT_VINFO_VECTYPE (pattern_stmt_info))
     {
       gcc_assert (!vectype
@@ -6042,12 +6041,14 @@ vect_recog_gather_scatter_pattern (vec_info *vinfo,
 
 	  tree vec_els
 	    = vect_get_mask_load_else (elsval, TREE_TYPE (gs_vectype));
-	  pattern_stmt = gimple_build_call_internal (gs_info.ifn, 6, base,
+	  pattern_stmt = gimple_build_call_internal (gs_info.ifn, 7, base,
+						     gs_info.alias_ptr,
 						     offset, scale, zero, mask,
 						     vec_els);
 	}
       else
-	pattern_stmt = gimple_build_call_internal (gs_info.ifn, 4, base,
+	pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5, base,
+						   gs_info.alias_ptr,
 						   offset, scale, zero);
       tree lhs = gimple_get_lhs (stmt_info->stmt);
       tree load_lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
@@ -6057,12 +6058,13 @@ vect_recog_gather_scatter_pattern (vec_info *vinfo,
     {
       tree rhs = vect_get_store_rhs (stmt_info);
       if (mask != NULL)
-	pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5,
-						   base, offset, scale, rhs,
-						   mask);
+	pattern_stmt = gimple_build_call_internal (gs_info.ifn, 6,
+						   base, gs_info.alias_ptr,
+						   offset, scale, rhs, mask);
       else
-	pattern_stmt = gimple_build_call_internal (gs_info.ifn, 4,
-						   base, offset, scale, rhs);
+	pattern_stmt = gimple_build_call_internal (gs_info.ifn, 5,
+						   base, gs_info.alias_ptr,
+						   offset, scale, rhs);
     }
   gimple_call_set_nothrow (pattern_stmt, true);
 
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 7a828ca..ca14a2d 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -118,18 +118,19 @@ _slp_tree::_slp_tree ()
   SLP_TREE_CHILDREN (this) = vNULL;
   SLP_TREE_LOAD_PERMUTATION (this) = vNULL;
   SLP_TREE_LANE_PERMUTATION (this) = vNULL;
-  SLP_TREE_SIMD_CLONE_INFO (this) = vNULL;
   SLP_TREE_DEF_TYPE (this) = vect_uninitialized_def;
   SLP_TREE_CODE (this) = ERROR_MARK;
   this->ldst_lanes = false;
   this->avoid_stlf_fail = false;
   SLP_TREE_VECTYPE (this) = NULL_TREE;
   SLP_TREE_REPRESENTATIVE (this) = NULL;
-  SLP_TREE_MEMORY_ACCESS_TYPE (this) = VMAT_INVARIANT;
+  SLP_TREE_MEMORY_ACCESS_TYPE (this) = VMAT_UNINITIALIZED;
   SLP_TREE_REF_COUNT (this) = 1;
   this->failed = NULL;
   this->max_nunits = 1;
   this->lanes = 0;
+  SLP_TREE_TYPE (this) = undef_vec_info_type;
+  this->data = NULL;
 }
 
 /* Tear down a SLP node.  */
@@ -148,9 +149,10 @@ _slp_tree::~_slp_tree ()
   SLP_TREE_VEC_DEFS (this).release ();
   SLP_TREE_LOAD_PERMUTATION (this).release ();
   SLP_TREE_LANE_PERMUTATION (this).release ();
-  SLP_TREE_SIMD_CLONE_INFO (this).release ();
   if (this->failed)
     free (failed);
+  if (this->data)
+    delete this->data;
 }
 
 /* Push the single SSA definition in DEF to the vector of vector defs.  */
@@ -507,19 +509,21 @@ vect_def_types_match (enum vect_def_type dta, enum vect_def_type dtb)
 	      && (dtb == vect_external_def || dtb == vect_constant_def)));
 }
 
+#define GATHER_SCATTER_OFFSET (-3)
+
 static const int no_arg_map[] = { 0 };
 static const int arg0_map[] = { 1, 0 };
-static const int arg1_map[] = { 1, 1 };
+static const int arg2_map[] = { 1, 2 };
 static const int arg2_arg3_map[] = { 2, 2, 3 };
-static const int arg1_arg3_map[] = { 2, 1, 3 };
-static const int arg1_arg4_arg5_map[] = { 3, 1, 4, 5 };
-static const int arg1_arg3_arg4_map[] = { 3, 1, 3, 4 };
+static const int arg2_arg4_map[] = { 2, 2, 4 };
+static const int arg2_arg5_arg6_map[] = { 3, 2, 5, 6 };
+static const int arg2_arg4_arg5_map[] = { 3, 2, 4, 5 };
 static const int arg3_arg2_map[] = { 2, 3, 2 };
 static const int op1_op0_map[] = { 2, 1, 0 };
-static const int off_map[] = { 1, -3 };
-static const int off_op0_map[] = { 2, -3, 0 };
-static const int off_arg2_arg3_map[] = { 3, -3, 2, 3 };
-static const int off_arg3_arg2_map[] = { 3, -3, 3, 2 };
+static const int off_map[] = { 1, GATHER_SCATTER_OFFSET };
+static const int off_op0_map[] = { 2, GATHER_SCATTER_OFFSET, 0 };
+static const int off_arg2_arg3_map[] = { 3, GATHER_SCATTER_OFFSET, 2, 3 };
+static const int off_arg3_arg2_map[] = { 3, GATHER_SCATTER_OFFSET, 3, 2 };
 static const int mask_call_maps[6][7] = {
   { 1, 1, },
   { 2, 1, 2, },
@@ -568,18 +572,18 @@ vect_get_operand_map (const gimple *stmt, bool gather_scatter_p = false,
 	    return gather_scatter_p ? off_arg2_arg3_map : arg2_arg3_map;
 
 	  case IFN_GATHER_LOAD:
-	    return arg1_map;
+	    return arg2_map;
 
 	  case IFN_MASK_GATHER_LOAD:
 	  case IFN_MASK_LEN_GATHER_LOAD:
-	    return arg1_arg4_arg5_map;
+	    return arg2_arg5_arg6_map;
 
 	  case IFN_SCATTER_STORE:
-	    return arg1_arg3_map;
+	    return arg2_arg4_map;
 
 	  case IFN_MASK_SCATTER_STORE:
 	  case IFN_MASK_LEN_SCATTER_STORE:
-	    return arg1_arg3_arg4_map;
+	    return arg2_arg4_arg5_map;
 
 	  case IFN_MASK_STORE:
 	    return gather_scatter_p ? off_arg3_arg2_map : arg3_arg2_map;
@@ -691,7 +695,7 @@ vect_get_and_check_slp_defs (vec_info *vinfo, unsigned char swap,
     {
       oprnd_info = (*oprnds_info)[i];
       int opno = map ? map[i] : int (i);
-      if (opno == -3)
+      if (opno == GATHER_SCATTER_OFFSET)
 	{
 	  gcc_assert (STMT_VINFO_GATHER_SCATTER_P (stmt_info));
 	  if (!is_a <loop_vec_info> (vinfo)
@@ -1114,6 +1118,16 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
       matches[0] = false;
       return false;
     }
+  if (is_a <bb_vec_info> (vinfo)
+      && known_le (TYPE_VECTOR_SUBPARTS (vectype), 1U))
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "Build SLP failed: not using single lane "
+			 "vector type %T\n", vectype);
+      matches[0] = false;
+      return false;
+    }
   /* Record nunits required but continue analysis, producing matches[]
      as if nunits was not an issue.  This allows splitting of groups
      to happen.  */
@@ -4067,7 +4081,12 @@ vect_build_slp_instance (vec_info *vinfo,
 		  for (unsigned i = 0; i < group_size; ++i)
 		    scalar_stmts.quick_push (next_info);
 		  slp_tree conv = vect_create_new_slp_node (scalar_stmts, 1);
-		  SLP_TREE_VECTYPE (conv) = STMT_VINFO_VECTYPE (next_info);
+		  SLP_TREE_VECTYPE (conv)
+		    = get_vectype_for_scalar_type (vinfo,
+						   TREE_TYPE
+						     (gimple_assign_lhs
+						       (scalar_def)),
+						   group_size);
 		  SLP_TREE_CHILDREN (conv).quick_push (node);
 		  SLP_INSTANCE_TREE (new_instance) = conv;
 		  /* We also have to fake this conversion stmt as SLP reduction
@@ -4931,6 +4950,9 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 						 max_tree_size, &limit,
 						 force_single_lane))
 	  {
+	    if (dump_enabled_p ())
+	      dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			       "SLP discovery of reduction chain failed\n");
 	    /* Dissolve reduction chain group.  */
 	    stmt_vec_info vinfo = first_element;
 	    stmt_vec_info last = NULL;
@@ -5063,9 +5085,15 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 	  tree args0 = gimple_cond_lhs (stmt);
 	  tree args1 = gimple_cond_rhs (stmt);
 
-	  /* These should be enforced by cond lowering.  */
-	  gcc_assert (gimple_cond_code (stmt) == NE_EXPR);
-	  gcc_assert (zerop (args1));
+	  /* These should be enforced by cond lowering, but if it failed
+	     bail.  */
+	  if (gimple_cond_code (stmt) != NE_EXPR
+	      || TREE_TYPE (args0) != boolean_type_node
+	      || !integer_zerop (args1))
+	    {
+	      roots.release ();
+	      continue;
+	    }
 
 	  /* An argument without a loop def will be codegened from vectorizing the
 	     root gcond itself.  As such we don't need to try to build an SLP tree
@@ -5218,7 +5246,7 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 		    if (STMT_VINFO_STRIDED_P (stmt_vinfo)
 			|| compare_step_with_zero (vinfo, stmt_vinfo) <= 0
 			|| vect_load_lanes_supported
-			     (STMT_VINFO_VECTYPE (stmt_vinfo),
+			     (SLP_TREE_VECTYPE (load_node),
 			      DR_GROUP_SIZE (stmt_vinfo), masked) == IFN_LAST
 			/* ???  During SLP re-discovery with a single lane
 			   a masked grouped load will appear permuted and
@@ -5239,7 +5267,7 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
 			 || SLP_TREE_LANES (load_node) == group_size
 			 || (vect_slp_prefer_store_lanes_p
 			     (vinfo, stmt_vinfo,
-			      STMT_VINFO_VECTYPE (stmt_vinfo), masked,
+			      SLP_TREE_VECTYPE (load_node), masked,
 			      group_size, SLP_TREE_LANES (load_node))));
 		  }
 
@@ -7565,20 +7593,25 @@ vect_make_slp_decision (loop_vec_info loop_vinfo)
   hash_set<slp_tree> visited;
   FOR_EACH_VEC_ELT (slp_instances, i, instance)
     {
-      /* FORNOW: SLP if you can.  */
+      slp_tree root = SLP_INSTANCE_TREE (instance);
+
       /* All unroll factors have the form:
 
 	   GET_MODE_SIZE (vinfo->vector_mode) * X
 
 	 for some rational X, so they must have a common multiple.  */
-      vect_update_slp_vf_for_node (SLP_INSTANCE_TREE (instance),
-				   unrolling_factor, visited);
+      vect_update_slp_vf_for_node (root, unrolling_factor, visited);
 
       /* Mark all the stmts that belong to INSTANCE as PURE_SLP stmts.  Later we
 	 call vect_detect_hybrid_slp () to find stmts that need hybrid SLP and
 	 loop-based vectorization.  Such stmts will be marked as HYBRID.  */
-      vect_mark_slp_stmts (loop_vinfo, SLP_INSTANCE_TREE (instance));
-      decided_to_slp++;
+      vect_mark_slp_stmts (loop_vinfo, root);
+
+      /* If all instances ended up with vector(1) T roots make sure to
+	 not vectorize.  RVV for example relies on loop vectorization
+	 when some instances are essentially kept scalar.  See PR121048.  */
+      if (known_gt (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (root)), 1U))
+	decided_to_slp++;
     }
 
   LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo) = unrolling_factor;
@@ -7691,7 +7724,7 @@ maybe_push_to_hybrid_worklist (vec_info *vinfo,
 
 /* Find stmts that must be both vectorized and SLPed.  */
 
-void
+bool
 vect_detect_hybrid_slp (loop_vec_info loop_vinfo)
 {
   DUMP_VECT_SCOPE ("vect_detect_hybrid_slp");
@@ -7772,6 +7805,52 @@ vect_detect_hybrid_slp (loop_vec_info loop_vinfo)
 	  vect_detect_hybrid_slp (&gs_info.offset, &dummy, &wi);
 	}
     }
+
+  /* Determine if all the stmts in the loop can be SLPed.  */
+  for (unsigned i = 0; i < LOOP_VINFO_LOOP (loop_vinfo)->num_nodes; i++)
+    {
+      basic_block bb = LOOP_VINFO_BBS (loop_vinfo)[i];
+      for (gphi_iterator si = gsi_start_phis (bb); !gsi_end_p (si);
+	   gsi_next (&si))
+	{
+	  stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (si.phi ());
+	  if (!stmt_info)
+	    continue;
+	  if ((STMT_VINFO_RELEVANT_P (stmt_info)
+	       || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
+	      && !PURE_SLP_STMT (stmt_info))
+	    {
+	      /* STMT needs both SLP and loop-based vectorization.  */
+	      if (dump_enabled_p ())
+		dump_printf_loc (MSG_NOTE, vect_location,
+				 "Loop contains SLP and non-SLP stmts\n");
+	      return false;
+	    }
+	}
+      for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);
+	   gsi_next (&si))
+	{
+	  if (is_gimple_debug (gsi_stmt (si)))
+	    continue;
+	  stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
+	  stmt_info = vect_stmt_to_vectorize (stmt_info);
+	  if ((STMT_VINFO_RELEVANT_P (stmt_info)
+	       || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
+	      && !PURE_SLP_STMT (stmt_info))
+	    {
+	      /* STMT needs both SLP and loop-based vectorization.  */
+	      if (dump_enabled_p ())
+		dump_printf_loc (MSG_NOTE, vect_location,
+				 "Loop contains SLP and non-SLP stmts\n");
+	      return false;
+	    }
+	}
+    }
+
+  if (dump_enabled_p ())
+    dump_printf_loc (MSG_NOTE, vect_location,
+		     "Loop contains only SLP stmts\n");
+  return true;
 }
 
 
@@ -7852,8 +7931,6 @@ vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
 				    slp_instance node_instance,
 				    stmt_vector_for_cost *cost_vec)
 {
-  stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
-
   /* Calculate the number of vector statements to be created for the scalar
      stmts in this node.  It is the number of scalar elements in one scalar
      iteration (DR_GROUP_SIZE) multiplied by VF divided by the number of
@@ -7882,9 +7959,7 @@ vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
       return true;
     }
 
-  bool dummy;
-  return vect_analyze_stmt (vinfo, stmt_info, &dummy,
-			    node, node_instance, cost_vec);
+  return vect_analyze_stmt (vinfo, node, node_instance, cost_vec);
 }
 
 static int
@@ -8189,8 +8264,7 @@ vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node,
 	      /* Masked loads can have an undefined (default SSA definition)
 		 else operand.  We do not need to cost it.  */
 	      vec<tree> ops = SLP_TREE_SCALAR_OPS (child);
-	      if ((STMT_VINFO_TYPE (SLP_TREE_REPRESENTATIVE (node))
-		   == load_vec_info_type)
+	      if (SLP_TREE_TYPE (node) == load_vec_info_type
 		  && ((ops.length ()
 		       && TREE_CODE (ops[0]) == SSA_NAME
 		       && SSA_NAME_IS_DEFAULT_DEF (ops[0])
@@ -8201,8 +8275,7 @@ vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node,
 	      /* For shifts with a scalar argument we don't need
 		 to cost or code-generate anything.
 		 ???  Represent this more explicitely.  */
-	      gcc_assert ((STMT_VINFO_TYPE (SLP_TREE_REPRESENTATIVE (node))
-			   == shift_vec_info_type)
+	      gcc_assert (SLP_TREE_TYPE (node) == shift_vec_info_type
 			  && j == 1);
 	      continue;
 	    }
@@ -8580,7 +8653,7 @@ vect_slp_analyze_operations (vec_info *vinfo)
 	  || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_gcond
 	      && !vectorizable_early_exit (vinfo,
 					   SLP_INSTANCE_ROOT_STMTS (instance)[0],
-					   NULL, NULL,
+					   NULL,
 					   SLP_INSTANCE_TREE (instance),
 					   &cost_vec)))
         {
@@ -9502,14 +9575,13 @@ vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal,
 
   slp_instance instance;
   int i;
-  poly_uint64 min_vf = 2;
 
   /* The first group of checks is independent of the vector size.  */
   fatal = true;
 
   /* Analyze the data references.  */
 
-  if (!vect_analyze_data_refs (bb_vinfo, &min_vf, NULL))
+  if (!vect_analyze_data_refs (bb_vinfo, NULL))
     {
       if (dump_enabled_p ())
         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -11239,9 +11311,9 @@ vect_schedule_slp_node (vec_info *vinfo,
       si = gsi_for_stmt (last_stmt_info->stmt);
     }
   else if (SLP_TREE_CODE (node) != VEC_PERM_EXPR
-	   && (STMT_VINFO_TYPE (stmt_info) == cycle_phi_info_type
-	       || STMT_VINFO_TYPE (stmt_info) == induc_vec_info_type
-	       || STMT_VINFO_TYPE (stmt_info) == phi_info_type))
+	   && (SLP_TREE_TYPE (node) == cycle_phi_info_type
+	       || SLP_TREE_TYPE (node) == induc_vec_info_type
+	       || SLP_TREE_TYPE (node) == phi_info_type))
     {
       /* For PHI node vectorization we do not use the insertion iterator.  */
       si = gsi_none ();
@@ -11261,8 +11333,7 @@ vect_schedule_slp_node (vec_info *vinfo,
 	       last scalar def here.  */
 	    if (SLP_TREE_VEC_DEFS (child).is_empty ())
 	      {
-		gcc_assert (STMT_VINFO_TYPE (SLP_TREE_REPRESENTATIVE (child))
-			    == cycle_phi_info_type);
+		gcc_assert (SLP_TREE_TYPE (child) == cycle_phi_info_type);
 		gphi *phi = as_a <gphi *>
 			      (vect_find_last_scalar_stmt_in_slp (child)->stmt);
 		if (!last_stmt)
@@ -11303,7 +11374,11 @@ vect_schedule_slp_node (vec_info *vinfo,
 		  && !SSA_NAME_IS_DEFAULT_DEF (def))
 		{
 		  gimple *stmt = SSA_NAME_DEF_STMT (def);
-		  if (!last_stmt)
+		  if (gimple_uid (stmt) == -1u)
+		    /* If the stmt is not inside the region do not
+		       use it as possible insertion point.  */
+		    ;
+		  else if (!last_stmt)
 		    last_stmt = stmt;
 		  else if (vect_stmt_dominates_stmt_p (last_stmt, stmt))
 		    last_stmt = stmt;
@@ -11409,7 +11484,7 @@ vect_schedule_slp_node (vec_info *vinfo,
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_NOTE, vect_location,
 			 "------>vectorizing SLP permutation node\n");
-      /* ???  the transform kind is stored to STMT_VINFO_TYPE which might
+      /* ???  the transform kind was stored to STMT_VINFO_TYPE which might
 	 be shared with different SLP nodes (but usually it's the same
 	 operation apart from the case the stmt is only there for denoting
 	 the actual scalar lane defs ...).  So do not call vect_transform_stmt
@@ -11608,10 +11683,9 @@ vectorize_slp_instance_root_stmt (vec_info *vinfo, slp_tree node, slp_instance i
       auto root_stmt_info = instance->root_stmts[0];
       auto last_stmt = STMT_VINFO_STMT (vect_orig_stmt (root_stmt_info));
       gimple_stmt_iterator rgsi = gsi_for_stmt (last_stmt);
-      gimple *vec_stmt = NULL;
       gcc_assert (!SLP_TREE_VEC_DEFS (node).is_empty ());
       bool res = vectorizable_early_exit (vinfo, root_stmt_info, &rgsi,
-					  &vec_stmt, node, NULL);
+					  node, NULL);
       gcc_assert (res);
       return;
     }
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 95406b4..97222f6 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -62,14 +62,6 @@ along with GCC; see the file COPYING3.  If not see
 /* For lang_hooks.types.type_for_mode.  */
 #include "langhooks.h"
 
-/* Return the vectorized type for the given statement.  */
-
-tree
-stmt_vectype (class _stmt_vec_info *stmt_info)
-{
-  return STMT_VINFO_VECTYPE (stmt_info);
-}
-
 /* Return TRUE iff the given statement is in an inner loop relative to
    the loop being vectorized.  */
 bool
@@ -901,13 +893,11 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo, bool *fatal)
 
 /* Function vect_model_simple_cost.
 
-   Models cost for simple operations, i.e. those that only emit ncopies of a
-   single op.  Right now, this does not account for multiple insns that could
-   be generated for the single vector op.  We will handle that shortly.  */
+   Models cost for simple operations, i.e. those that only emit N operations
+   of the same KIND.  */
 
 static void
-vect_model_simple_cost (vec_info *, int ncopies, enum vect_def_type *dt,
-			int ndts, slp_tree node,
+vect_model_simple_cost (vec_info *, int n, slp_tree node,
 			stmt_vector_for_cost *cost_vec,
 			vect_cost_for_stmt kind = vector_stmt)
 {
@@ -915,22 +905,10 @@ vect_model_simple_cost (vec_info *, int ncopies, enum vect_def_type *dt,
 
   gcc_assert (cost_vec != NULL);
 
-  /* ???  Somehow we need to fix this at the callers.  */
-  if (node)
-    ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (node);
-
-  if (!node)
-    /* Cost the "broadcast" of a scalar operand in to a vector operand.
-       Use scalar_to_vec to cost the broadcast, as elsewhere in the vector
-       cost model.  */
-    for (int i = 0; i < ndts; i++)
-      if (dt[i] == vect_constant_def || dt[i] == vect_external_def)
-	prologue_cost += record_stmt_cost (cost_vec, 1, scalar_to_vec,
-					   node, 0, vect_prologue);
+  n *= SLP_TREE_NUMBER_OF_VEC_STMTS (node);
 
   /* Pass the inside-of-loop statements to the target-specific cost model.  */
-  inside_cost += record_stmt_cost (cost_vec, ncopies, kind,
-				   node, 0, vect_body);
+  inside_cost += record_stmt_cost (cost_vec, n, kind, node, 0, vect_body);
 
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location,
@@ -1269,130 +1247,23 @@ vect_init_vector (vec_info *vinfo, stmt_vec_info stmt_info, tree val, tree type,
 }
 
 
-/* Function vect_get_vec_defs_for_operand.
-
-   OP is an operand in STMT_VINFO.  This function returns a vector of
-   NCOPIES defs that will be used in the vectorized stmts for STMT_VINFO.
-
-   In the case that OP is an SSA_NAME which is defined in the loop, then
-   STMT_VINFO_VEC_STMTS of the defining stmt holds the relevant defs.
-
-   In case OP is an invariant or constant, a new stmt that creates a vector def
-   needs to be introduced.  VECTYPE may be used to specify a required type for
-   vector invariant.  */
-
-void
-vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info stmt_vinfo,
-			       unsigned ncopies,
-			       tree op, vec<tree> *vec_oprnds, tree vectype)
-{
-  gimple *def_stmt;
-  enum vect_def_type dt;
-  bool is_simple_use;
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-
-  if (dump_enabled_p ())
-    dump_printf_loc (MSG_NOTE, vect_location,
-		     "vect_get_vec_defs_for_operand: %T\n", op);
-
-  stmt_vec_info def_stmt_info;
-  is_simple_use = vect_is_simple_use (op, loop_vinfo, &dt,
-				      &def_stmt_info, &def_stmt);
-  gcc_assert (is_simple_use);
-  if (def_stmt && dump_enabled_p ())
-    dump_printf_loc (MSG_NOTE, vect_location, "  def_stmt =  %G", def_stmt);
-
-  vec_oprnds->create (ncopies);
-  if (dt == vect_constant_def || dt == vect_external_def)
-    {
-      tree stmt_vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
-      tree vector_type;
-
-      if (vectype)
-	vector_type = vectype;
-      else if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op))
-	       && VECTOR_BOOLEAN_TYPE_P (stmt_vectype))
-	vector_type = truth_type_for (stmt_vectype);
-      else
-	vector_type = get_vectype_for_scalar_type (loop_vinfo, TREE_TYPE (op));
-
-      gcc_assert (vector_type);
-      /* A masked load can have a default SSA definition as else operand.
-	 We should "vectorize" this instead of creating a duplicate from the
-	 scalar default.  */
-      tree vop;
-      if (TREE_CODE (op) == SSA_NAME
-	  && SSA_NAME_IS_DEFAULT_DEF (op)
-	  && VAR_P (SSA_NAME_VAR (op)))
-	vop = get_or_create_ssa_default_def (cfun,
-					     create_tmp_var (vector_type));
-      else
-	vop = vect_init_vector (vinfo, stmt_vinfo, op, vector_type, NULL);
-      while (ncopies--)
-	vec_oprnds->quick_push (vop);
-    }
-  else
-    {
-      def_stmt_info = vect_stmt_to_vectorize (def_stmt_info);
-      gcc_assert (STMT_VINFO_VEC_STMTS (def_stmt_info).length () == ncopies);
-      for (unsigned i = 0; i < ncopies; ++i)
-	vec_oprnds->quick_push (gimple_get_lhs
-				  (STMT_VINFO_VEC_STMTS (def_stmt_info)[i]));
-    }
-}
-
-
 /* Get vectorized definitions for OP0 and OP1.  */
 
 void
-vect_get_vec_defs (vec_info *vinfo, stmt_vec_info stmt_info, slp_tree slp_node,
-		   unsigned ncopies,
-		   tree op0, tree vectype0, vec<tree> *vec_oprnds0,
-		   tree op1, tree vectype1, vec<tree> *vec_oprnds1,
-		   tree op2, tree vectype2, vec<tree> *vec_oprnds2,
-		   tree op3, tree vectype3, vec<tree> *vec_oprnds3)
-{
-  if (slp_node)
-    {
-      if (op0)
-	vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_oprnds0);
-      if (op1)
-	vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[1], vec_oprnds1);
-      if (op2)
-	vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[2], vec_oprnds2);
-      if (op3)
-	vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[3], vec_oprnds3);
-    }
-  else
-    {
-      if (op0)
-	vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
-				       op0, vec_oprnds0, vectype0);
-      if (op1)
-	vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
-				       op1, vec_oprnds1, vectype1);
-      if (op2)
-	vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
-				       op2, vec_oprnds2, vectype2);
-      if (op3)
-	vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
-				       op3, vec_oprnds3, vectype3);
-    }
-}
-
-void
-vect_get_vec_defs (vec_info *vinfo, stmt_vec_info stmt_info, slp_tree slp_node,
-		   unsigned ncopies,
+vect_get_vec_defs (vec_info *, slp_tree slp_node,
 		   tree op0, vec<tree> *vec_oprnds0,
 		   tree op1, vec<tree> *vec_oprnds1,
 		   tree op2, vec<tree> *vec_oprnds2,
 		   tree op3, vec<tree> *vec_oprnds3)
 {
-  vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
-		     op0, NULL_TREE, vec_oprnds0,
-		     op1, NULL_TREE, vec_oprnds1,
-		     op2, NULL_TREE, vec_oprnds2,
-		     op3, NULL_TREE, vec_oprnds3);
+  if (op0)
+    vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_oprnds0);
+  if (op1)
+    vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[1], vec_oprnds1);
+  if (op2)
+    vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[2], vec_oprnds2);
+  if (op3)
+    vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[3], vec_oprnds3);
 }
 
 /* Helper function called by vect_finish_replace_stmt and
@@ -1551,14 +1422,31 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
 				      int group_size,
 				      vect_memory_access_type
 				      memory_access_type,
-				      gather_scatter_info *gs_info,
-				      tree scalar_mask,
+				      const gather_scatter_info *gs_info,
+				      slp_tree mask_node,
 				      vec<int> *elsvals = nullptr)
 {
   /* Invariant loads need no special support.  */
   if (memory_access_type == VMAT_INVARIANT)
     return;
 
+  /* Figure whether the mask is uniform.  scalar_mask is used to
+     populate the scalar_cond_masked_set.  */
+  tree scalar_mask = NULL_TREE;
+  if (mask_node)
+    for (unsigned i = 0; i < SLP_TREE_LANES (mask_node); ++i)
+      {
+	tree def = vect_get_slp_scalar_def (mask_node, i);
+	if (!def
+	    || (scalar_mask && def != scalar_mask))
+	  {
+	    scalar_mask = NULL;
+	    break;
+	  }
+	else
+	  scalar_mask = def;
+      }
+
   unsigned int nvectors = vect_get_num_copies (loop_vinfo, slp_node, vectype);
   vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
   vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
@@ -1566,8 +1454,7 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
   bool is_load = (vls_type == VLS_LOAD);
   if (memory_access_type == VMAT_LOAD_STORE_LANES)
     {
-      if (slp_node)
-	nvectors /= group_size;
+      nvectors /= group_size;
       internal_fn ifn
 	= (is_load ? vect_load_lanes_supported (vectype, group_size, true,
 						elsvals)
@@ -1622,8 +1509,7 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
       return;
     }
 
-  if (memory_access_type != VMAT_CONTIGUOUS
-      && memory_access_type != VMAT_CONTIGUOUS_PERMUTE)
+  if (memory_access_type != VMAT_CONTIGUOUS)
     {
       /* Element X of the data must come from iteration i * VF + X of the
 	 scalar loop.  We need more work to support other mappings.  */
@@ -1646,7 +1532,7 @@ check_load_store_for_partial_vectors (loop_vec_info loop_vinfo, tree vectype,
     }
 
   /* We might load more scalars than we need for permuting SLP loads.
-     We checked in get_group_load_store_type that the extra elements
+     We checked in get_load_store_type that the extra elements
      don't leak into a new vector.  */
   auto group_memory_nvectors = [](poly_uint64 size, poly_uint64 nunits)
   {
@@ -1734,7 +1620,7 @@ prepare_vec_mask (loop_vec_info loop_vinfo, tree mask_type, tree loop_mask,
    else values will be stored in the vector ELSVALS points to.  */
 
 static bool
-vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info,
+vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info, tree vectype,
 				     loop_vec_info loop_vinfo, bool masked_p,
 				     gather_scatter_info *gs_info,
 				     vec<int> *elsvals)
@@ -1752,7 +1638,6 @@ vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info,
     }
 
   /* Get the number of bits in an element.  */
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
   scalar_mode element_mode = SCALAR_TYPE_MODE (TREE_TYPE (vectype));
   unsigned int element_bits = GET_MODE_BITSIZE (element_mode);
 
@@ -1803,9 +1688,11 @@ vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info,
       /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
 	 but we don't need to store that here.  */
       gs_info->base = NULL_TREE;
+      gs_info->alias_ptr = build_int_cst
+	(reference_alias_ptr_type (DR_REF (dr)),
+	 get_object_alignment (DR_REF (dr)));
       gs_info->element_type = TREE_TYPE (vectype);
       gs_info->offset = fold_convert (offset_type, step);
-      gs_info->offset_dt = vect_constant_def;
       gs_info->scale = scale;
       gs_info->memory_type = memory_type;
       return true;
@@ -1829,22 +1716,35 @@ vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info,
    else values will be stored in the vector ELSVALS points to.  */
 
 static bool
-vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info,
+vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info, tree vectype,
 				    loop_vec_info loop_vinfo, bool masked_p,
 				    gather_scatter_info *gs_info,
-				    vec<int> *elsvals)
+				    vec<int> *elsvals,
+				    unsigned int group_size,
+				    bool single_element_p)
 {
   if (!vect_check_gather_scatter (stmt_info, loop_vinfo, gs_info, elsvals)
       || gs_info->ifn == IFN_LAST)
-    return vect_truncate_gather_scatter_offset (stmt_info, loop_vinfo,
-						masked_p, gs_info, elsvals);
+    {
+      if (!vect_truncate_gather_scatter_offset (stmt_info, vectype, loop_vinfo,
+						masked_p, gs_info, elsvals))
+	return false;
+    }
+  else
+    {
+      tree old_offset_type = TREE_TYPE (gs_info->offset);
+      tree new_offset_type = TREE_TYPE (gs_info->offset_vectype);
 
-  tree old_offset_type = TREE_TYPE (gs_info->offset);
-  tree new_offset_type = TREE_TYPE (gs_info->offset_vectype);
+      gcc_assert (TYPE_PRECISION (new_offset_type)
+		  >= TYPE_PRECISION (old_offset_type));
+      gs_info->offset = fold_convert (new_offset_type, gs_info->offset);
+    }
 
-  gcc_assert (TYPE_PRECISION (new_offset_type)
-	      >= TYPE_PRECISION (old_offset_type));
-  gs_info->offset = fold_convert (new_offset_type, gs_info->offset);
+  if (!single_element_p
+      && !targetm.vectorize.prefer_gather_scatter (TYPE_MODE (vectype),
+						   gs_info->scale,
+						   group_size))
+    return false;
 
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location,
@@ -2032,38 +1932,45 @@ vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype)
   return NULL_TREE;
 }
 
-/* A subroutine of get_load_store_type, with a subset of the same
-   arguments.  Handle the case where STMT_INFO is part of a grouped load
-   or store.
+/* Analyze load or store SLP_NODE of type VLS_TYPE.  Return true
+   if there is a memory access type that the vectorized form can use,
+   storing it in *MEMORY_ACCESS_TYPE if so.  If we decide to use gathers
+   or scatters, fill in GS_INFO accordingly.  In addition
+   *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
+   the target does not support the alignment scheme.  *MISALIGNMENT
+   is set according to the alignment of the access (including
+   DR_MISALIGNMENT_UNKNOWN when it is unknown).
 
-   For stores, the statements in the group are all consecutive
-   and there is no gap at the end.  For loads, the statements in the
-   group might not be consecutive; there can be gaps between statements
-   as well as at the end.
+   MASKED_P is true if the statement is conditional on a vectorized mask.
+   VECTYPE is the vector type that the vectorized statements will use.
 
-   If we can use gather/scatter and ELSVALS is nonzero the supported
-   else values will be stored in the vector ELSVALS points to.
-*/
+   If ELSVALS is nonzero the supported else values will be stored in the
+   vector ELSVALS points to.  */
 
 static bool
-get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
-			   tree vectype, slp_tree slp_node,
-			   bool masked_p, vec_load_store_type vls_type,
-			   vect_memory_access_type *memory_access_type,
-			   poly_int64 *poffset,
-			   dr_alignment_support *alignment_support_scheme,
-			   int *misalignment,
-			   gather_scatter_info *gs_info,
-			   internal_fn *lanes_ifn,
-			   vec<int> *elsvals)
+get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
+		     tree vectype, slp_tree slp_node,
+		     bool masked_p, vec_load_store_type vls_type,
+		     vect_memory_access_type *memory_access_type,
+		     poly_int64 *poffset,
+		     dr_alignment_support *alignment_support_scheme,
+		     int *misalignment,
+		     gather_scatter_info *gs_info,
+		     internal_fn *lanes_ifn,
+		     vec<int> *elsvals = nullptr)
 {
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+  poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
   class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
   stmt_vec_info first_stmt_info;
   unsigned int group_size;
   unsigned HOST_WIDE_INT gap;
   bool single_element_p;
   poly_int64 neg_ldst_offset = 0;
+
+  *misalignment = DR_MISALIGNMENT_UNKNOWN;
+  *poffset = 0;
+
   if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
     {
       first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
@@ -2080,7 +1987,6 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
       single_element_p = true;
     }
   dr_vec_info *first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
-  poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
 
   /* True if the vectorized statements would access beyond the last
      statement in the group.  */
@@ -2106,6 +2012,48 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
        separated by the stride, until we have a complete vector.
        Fall back to scalar accesses if that isn't possible.  */
     *memory_access_type = VMAT_STRIDED_SLP;
+  else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
+    {
+      *memory_access_type = VMAT_GATHER_SCATTER;
+      if (!vect_check_gather_scatter (stmt_info, loop_vinfo, gs_info,
+				      elsvals))
+	gcc_unreachable ();
+      slp_tree offset_node = SLP_TREE_CHILDREN (slp_node)[0];
+      tree offset_vectype = SLP_TREE_VECTYPE (offset_node);
+      gs_info->offset_vectype = offset_vectype;
+      /* When using internal functions, we rely on pattern recognition
+	 to convert the type of the offset to the type that the target
+	 requires, with the result being a call to an internal function.
+	 If that failed for some reason (e.g. because another pattern
+	 took priority), just handle cases in which the offset already
+	 has the right type.  */
+      if (GATHER_SCATTER_IFN_P (*gs_info)
+	  && !is_gimple_call (stmt_info->stmt)
+	  && !tree_nop_conversion_p (TREE_TYPE (gs_info->offset),
+				     TREE_TYPE (offset_vectype)))
+	{
+	  if (dump_enabled_p ())
+	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			     "%s offset requires a conversion\n",
+			     vls_type == VLS_LOAD ? "gather" : "scatter");
+	  return false;
+	}
+      else if (GATHER_SCATTER_EMULATED_P (*gs_info))
+	{
+	  if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant ()
+	      || !TYPE_VECTOR_SUBPARTS (offset_vectype).is_constant ()
+	      || VECTOR_BOOLEAN_TYPE_P (offset_vectype)
+	      || !constant_multiple_p (TYPE_VECTOR_SUBPARTS (offset_vectype),
+				       TYPE_VECTOR_SUBPARTS (vectype)))
+	    {
+	      if (dump_enabled_p ())
+		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+				 "unsupported vector types for emulated "
+				 "gather.\n");
+	      return false;
+	    }
+	}
+    }
   else
     {
       int cmp = compare_step_with_zero (vinfo, stmt_info);
@@ -2349,19 +2297,21 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
      allows us to use contiguous accesses.  */
   if ((*memory_access_type == VMAT_ELEMENTWISE
        || *memory_access_type == VMAT_STRIDED_SLP)
-      && single_element_p
+      && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)
       && SLP_TREE_LANES (slp_node) == 1
       && loop_vinfo
-      && vect_use_strided_gather_scatters_p (stmt_info, loop_vinfo,
-					     masked_p, gs_info, elsvals))
+      && vect_use_strided_gather_scatters_p (stmt_info, vectype, loop_vinfo,
+					     masked_p, gs_info, elsvals,
+					     group_size, single_element_p))
     *memory_access_type = VMAT_GATHER_SCATTER;
 
   if (*memory_access_type == VMAT_CONTIGUOUS_DOWN
       || *memory_access_type == VMAT_CONTIGUOUS_REVERSE)
     *poffset = neg_ldst_offset;
 
-  if (*memory_access_type == VMAT_GATHER_SCATTER
-      || *memory_access_type == VMAT_ELEMENTWISE
+  if (*memory_access_type == VMAT_ELEMENTWISE
+      || (*memory_access_type == VMAT_GATHER_SCATTER
+	  && GATHER_SCATTER_LEGACY_P (*gs_info))
       || *memory_access_type == VMAT_STRIDED_SLP
       || *memory_access_type == VMAT_INVARIANT)
     {
@@ -2370,30 +2320,15 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
     }
   else
     {
-      *misalignment = dr_misalignment (first_dr_info, vectype, *poffset);
+      if (*memory_access_type == VMAT_GATHER_SCATTER
+	  && !first_dr_info)
+	*misalignment = DR_MISALIGNMENT_UNKNOWN;
+      else
+	*misalignment = dr_misalignment (first_dr_info, vectype, *poffset);
       *alignment_support_scheme
-	= vect_supportable_dr_alignment (vinfo, first_dr_info, vectype,
-					 *misalignment);
-    }
-
-  if (vls_type != VLS_LOAD && first_stmt_info == stmt_info)
-    {
-      /* STMT is the leader of the group. Check the operands of all the
-	 stmts of the group.  */
-      stmt_vec_info next_stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info);
-      while (next_stmt_info)
-	{
-	  tree op = vect_get_store_rhs (next_stmt_info);
-	  enum vect_def_type dt;
-	  if (!vect_is_simple_use (op, vinfo, &dt))
-	    {
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-				 "use not simple.\n");
-	      return false;
-	    }
-	  next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
-	}
+	= vect_supportable_dr_alignment
+	   (vinfo, first_dr_info, vectype, *misalignment,
+	    *memory_access_type == VMAT_GATHER_SCATTER ? gs_info : nullptr);
     }
 
   if (overrun_p)
@@ -2406,104 +2341,6 @@ get_group_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info,
       LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
     }
 
-  return true;
-}
-
-/* Analyze load or store statement STMT_INFO of type VLS_TYPE.  Return true
-   if there is a memory access type that the vectorized form can use,
-   storing it in *MEMORY_ACCESS_TYPE if so.  If we decide to use gathers
-   or scatters, fill in GS_INFO accordingly.  In addition
-   *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
-   the target does not support the alignment scheme.  *MISALIGNMENT
-   is set according to the alignment of the access (including
-   DR_MISALIGNMENT_UNKNOWN when it is unknown).
-
-   SLP says whether we're performing SLP rather than loop vectorization.
-   MASKED_P is true if the statement is conditional on a vectorized mask.
-   VECTYPE is the vector type that the vectorized statements will use.
-   NCOPIES is the number of vector statements that will be needed.
-
-   If ELSVALS is nonzero the supported else values will be stored in the
-   vector ELSVALS points to.  */
-
-static bool
-get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
-		     tree vectype, slp_tree slp_node,
-		     bool masked_p, vec_load_store_type vls_type,
-		     unsigned int,
-		     vect_memory_access_type *memory_access_type,
-		     poly_int64 *poffset,
-		     dr_alignment_support *alignment_support_scheme,
-		     int *misalignment,
-		     gather_scatter_info *gs_info,
-		     internal_fn *lanes_ifn,
-		     vec<int> *elsvals = nullptr)
-{
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-  poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
-  *misalignment = DR_MISALIGNMENT_UNKNOWN;
-  *poffset = 0;
-  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
-    {
-      *memory_access_type = VMAT_GATHER_SCATTER;
-      if (!vect_check_gather_scatter (stmt_info, loop_vinfo, gs_info,
-				      elsvals))
-	gcc_unreachable ();
-      /* When using internal functions, we rely on pattern recognition
-	 to convert the type of the offset to the type that the target
-	 requires, with the result being a call to an internal function.
-	 If that failed for some reason (e.g. because another pattern
-	 took priority), just handle cases in which the offset already
-	 has the right type.  */
-      else if (gs_info->ifn != IFN_LAST
-	       && !is_gimple_call (stmt_info->stmt)
-	       && !tree_nop_conversion_p (TREE_TYPE (gs_info->offset),
-					  TREE_TYPE (gs_info->offset_vectype)))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "%s offset requires a conversion\n",
-			     vls_type == VLS_LOAD ? "gather" : "scatter");
-	  return false;
-	}
-      else if (!vect_is_simple_use (gs_info->offset, vinfo,
-				    &gs_info->offset_dt,
-				    &gs_info->offset_vectype))
-	{
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			     "%s index use not simple.\n",
-			     vls_type == VLS_LOAD ? "gather" : "scatter");
-	  return false;
-	}
-      else if (gs_info->ifn == IFN_LAST && !gs_info->decl)
-	{
-	  if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant ()
-	      || !TYPE_VECTOR_SUBPARTS (gs_info->offset_vectype).is_constant ()
-	      || VECTOR_BOOLEAN_TYPE_P (gs_info->offset_vectype)
-	      || !constant_multiple_p (TYPE_VECTOR_SUBPARTS
-					 (gs_info->offset_vectype),
-				       TYPE_VECTOR_SUBPARTS (vectype)))
-	    {
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-				 "unsupported vector types for emulated "
-				 "gather.\n");
-	      return false;
-	    }
-	}
-      /* Gather-scatter accesses perform only component accesses, alignment
-	 is irrelevant for them.  */
-      *alignment_support_scheme = dr_unaligned_supported;
-    }
-  else if (!get_group_load_store_type (vinfo, stmt_info, vectype, slp_node,
-				       masked_p,
-				       vls_type, memory_access_type, poffset,
-				       alignment_support_scheme,
-				       misalignment, gs_info, lanes_ifn,
-				       elsvals))
-    return false;
-
   if ((*memory_access_type == VMAT_ELEMENTWISE
        || *memory_access_type == VMAT_STRIDED_SLP)
       && !nunits.is_constant ())
@@ -2515,7 +2352,6 @@ get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
       return false;
     }
 
-
   /* Checks if all scalar iterations are known to be inbounds.  */
   bool inbounds = DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (stmt_info));
 
@@ -2535,17 +2371,18 @@ get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
 			   "alignment. With non-contiguous memory vectorization"
 			   " could read out of bounds at %G ",
 			   STMT_VINFO_STMT (stmt_info));
-	if (inbounds)
-	  LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
-	else
-	  return false;
+      if (inbounds)
+	LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
+      else
+	return false;
     }
 
   /* If this DR needs alignment for correctness, we must ensure the target
      alignment is a constant power-of-two multiple of the amount read per
      vector iteration or force masking.  */
   if (dr_safe_speculative_read_required (stmt_info)
-      && *alignment_support_scheme == dr_aligned)
+      && (*alignment_support_scheme == dr_aligned
+	  && *memory_access_type != VMAT_GATHER_SCATTER))
     {
       /* We can only peel for loops, of course.  */
       gcc_checking_assert (loop_vinfo);
@@ -2648,9 +2485,6 @@ get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
   /* FIXME: At the moment the cost model seems to underestimate the
      cost of using elementwise accesses.  This check preserves the
      traditional behavior until that can be fixed.  */
-  stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
-  if (!first_stmt_info)
-    first_stmt_info = stmt_info;
   if (*memory_access_type == VMAT_ELEMENTWISE
       && !STMT_VINFO_STRIDED_P (first_stmt_info)
       && !(stmt_info == DR_GROUP_FIRST_ELEMENT (stmt_info)
@@ -2667,21 +2501,21 @@ get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
 
 /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
    conditional operation STMT_INFO.  When returning true, store the mask
-   in *MASK, the type of its definition in *MASK_DT_OUT, the type of the
-   vectorized mask in *MASK_VECTYPE_OUT and the SLP node corresponding
-   to the mask in *MASK_NODE if MASK_NODE is not NULL.  */
+   in *MASK_NODE, the type of its definition in *MASK_DT_OUT and the type of
+   the vectorized mask in *MASK_VECTYPE_OUT.  */
 
 static bool
-vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info,
+vect_check_scalar_mask (vec_info *vinfo,
 			slp_tree slp_node, unsigned mask_index,
-			tree *mask, slp_tree *mask_node,
+			slp_tree *mask_node,
 			vect_def_type *mask_dt_out, tree *mask_vectype_out)
 {
   enum vect_def_type mask_dt;
   tree mask_vectype;
   slp_tree mask_node_1;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node, mask_index,
-			   mask, &mask_node_1, &mask_dt, &mask_vectype))
+  tree mask_;
+  if (!vect_is_simple_use (vinfo, slp_node, mask_index,
+			   &mask_, &mask_node_1, &mask_dt, &mask_vectype))
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -2690,7 +2524,7 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info,
     }
 
   if ((mask_dt == vect_constant_def || mask_dt == vect_external_def)
-      && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (*mask)))
+      && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask_)))
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -2698,19 +2532,7 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info,
       return false;
     }
 
-  /* If the caller is not prepared for adjusting an external/constant
-     SLP mask vector type fail.  */
-  if (slp_node
-      && !mask_node
-      && SLP_TREE_DEF_TYPE (mask_node_1) != vect_internal_def)
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			 "SLP mask argument is not vectorized.\n");
-      return false;
-    }
-
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   if (!mask_vectype)
     mask_vectype = get_mask_type_for_scalar_type (vinfo, TREE_TYPE (vectype),
 						  mask_node_1);
@@ -2737,11 +2559,11 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info,
 
   *mask_dt_out = mask_dt;
   *mask_vectype_out = mask_vectype;
-  if (mask_node)
-    *mask_node = mask_node_1;
+  *mask_node = mask_node_1;
   return true;
 }
 
+
 /* Return true if stored value is suitable for vectorizing store
    statement STMT_INFO.  When returning true, store the scalar stored
    in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
@@ -2750,7 +2572,7 @@ vect_check_scalar_mask (vec_info *vinfo, stmt_vec_info stmt_info,
 
 static bool
 vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
-		      slp_tree slp_node, tree *rhs, slp_tree *rhs_node,
+		      slp_tree slp_node, slp_tree *rhs_node,
 		      vect_def_type *rhs_dt_out, tree *rhs_vectype_out,
 		      vec_load_store_type *vls_type_out)
 {
@@ -2761,14 +2583,14 @@ vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
 	  && internal_store_fn_p (gimple_call_internal_fn (call)))
 	op_no = internal_fn_stored_value_index (gimple_call_internal_fn (call));
     }
-  if (slp_node)
-    op_no = vect_slp_child_index_for_operand
-	      (stmt_info->stmt, op_no, STMT_VINFO_GATHER_SCATTER_P (stmt_info));
+  op_no = vect_slp_child_index_for_operand
+	    (stmt_info->stmt, op_no, STMT_VINFO_GATHER_SCATTER_P (stmt_info));
 
   enum vect_def_type rhs_dt;
   tree rhs_vectype;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node, op_no,
-			   rhs, rhs_node, &rhs_dt, &rhs_vectype))
+  tree rhs;
+  if (!vect_is_simple_use (vinfo, slp_node, op_no,
+			   &rhs, rhs_node, &rhs_dt, &rhs_vectype))
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -2779,7 +2601,7 @@ vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
   /* In the case this is a store from a constant make sure
      native_encode_expr can handle it.  */
   if (rhs_dt == vect_constant_def
-      && CONSTANT_CLASS_P (*rhs) && native_encode_expr (*rhs, NULL, 64) == 0)
+      && CONSTANT_CLASS_P (rhs) && native_encode_expr (rhs, NULL, 64) == 0)
     {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -2787,7 +2609,7 @@ vect_check_store_rhs (vec_info *vinfo, stmt_vec_info stmt_info,
       return false;
     }
 
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   if (rhs_vectype && !useless_type_conversion_p (vectype, rhs_vectype))
     {
       if (dump_enabled_p ())
@@ -2893,11 +2715,11 @@ vect_get_mask_load_else (int elsval, tree type)
 
 static gimple *
 vect_build_one_gather_load_call (vec_info *vinfo, stmt_vec_info stmt_info,
+				 tree vectype,
 				 gimple_stmt_iterator *gsi,
-				 gather_scatter_info *gs_info,
+				 const gather_scatter_info *gs_info,
 				 tree ptr, tree offset, tree mask)
 {
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
   tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info->decl));
   tree rettype = TREE_TYPE (TREE_TYPE (gs_info->decl));
   tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
@@ -2993,7 +2815,7 @@ vect_build_one_gather_load_call (vec_info *vinfo, stmt_vec_info stmt_info,
 static gimple *
 vect_build_one_scatter_store_call (vec_info *vinfo, stmt_vec_info stmt_info,
 				   gimple_stmt_iterator *gsi,
-				   gather_scatter_info *gs_info,
+				   const gather_scatter_info *gs_info,
 				   tree ptr, tree offset, tree oprnd, tree mask)
 {
   tree rettype = TREE_TYPE (TREE_TYPE (gs_info->decl));
@@ -3074,9 +2896,8 @@ vect_build_one_scatter_store_call (vec_info *vinfo, stmt_vec_info stmt_info,
    containing loop.  */
 
 static void
-vect_get_gather_scatter_ops (loop_vec_info loop_vinfo,
-			     class loop *loop, stmt_vec_info stmt_info,
-			     slp_tree slp_node, gather_scatter_info *gs_info,
+vect_get_gather_scatter_ops (class loop *loop, slp_tree slp_node,
+			     const gather_scatter_info *gs_info,
 			     tree *dataref_ptr, vec<tree> *vec_offset)
 {
   gimple_seq stmts = NULL;
@@ -3088,16 +2909,7 @@ vect_get_gather_scatter_ops (loop_vec_info loop_vinfo,
       new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
       gcc_assert (!new_bb);
     }
-  if (slp_node)
-    vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_offset);
-  else
-    {
-      unsigned ncopies
-	= vect_get_num_copies (loop_vinfo, gs_info->offset_vectype);
-      vect_get_vec_defs_for_operand (loop_vinfo, stmt_info, ncopies,
-				     gs_info->offset, vec_offset,
-				     gs_info->offset_vectype);
-    }
+  vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[0], vec_offset);
 }
 
 /* Prepare to implement a grouped or strided load or store using
@@ -3110,15 +2922,14 @@ vect_get_gather_scatter_ops (loop_vec_info loop_vinfo,
    I * DR_STEP / SCALE.  */
 
 static void
-vect_get_strided_load_store_ops (stmt_vec_info stmt_info,
+vect_get_strided_load_store_ops (stmt_vec_info stmt_info, tree vectype,
 				 loop_vec_info loop_vinfo,
 				 gimple_stmt_iterator *gsi,
-				 gather_scatter_info *gs_info,
+				 const gather_scatter_info *gs_info,
 				 tree *dataref_bump, tree *vec_offset,
 				 vec_loop_lens *loop_lens)
 {
   struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
 
   if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo))
     {
@@ -3243,29 +3054,17 @@ vect_get_data_ptr_increment (vec_info *vinfo, gimple_stmt_iterator *gsi,
 static bool
 vectorizable_bswap (vec_info *vinfo,
 		    stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-		    gimple **vec_stmt, slp_tree slp_node,
+		    slp_tree slp_node,
 		    slp_tree *slp_op,
 		    tree vectype_in, stmt_vector_for_cost *cost_vec)
 {
   tree op, vectype;
   gcall *stmt = as_a <gcall *> (stmt_info->stmt);
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-  unsigned ncopies;
 
   op = gimple_call_arg (stmt, 0);
-  vectype = STMT_VINFO_VECTYPE (stmt_info);
+  vectype = SLP_TREE_VECTYPE (slp_node);
   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
 
-  /* Multiple types in SLP are handled by creating the appropriate number of
-     vectorized stmts for each SLP node.  Hence, NCOPIES is always 1 in
-     case of SLP.  */
-  if (slp_node)
-    ncopies = 1;
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype);
-
-  gcc_assert (ncopies >= 1);
-
   if (TYPE_SIZE (vectype_in) != TYPE_SIZE (vectype))
     {
       if (dump_enabled_p ())
@@ -3295,10 +3094,9 @@ vectorizable_bswap (vec_info *vinfo,
   if (!can_vec_perm_const_p (vmode, vmode, indices))
     return false;
 
-  if (! vec_stmt)
+  if (cost_vec)
     {
-      if (slp_node
-	  && !vect_maybe_update_slp_op_vectype (slp_op[0], vectype_in))
+      if (!vect_maybe_update_slp_op_vectype (slp_op[0], vectype_in))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -3306,13 +3104,12 @@ vectorizable_bswap (vec_info *vinfo,
 	  return false;
 	}
 
-      STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = call_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_bswap");
       record_stmt_cost (cost_vec,
 			1, vector_stmt, stmt_info, 0, vect_prologue);
       record_stmt_cost (cost_vec,
-			slp_node
-			? SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) : ncopies,
+			SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node),
 			vec_perm, stmt_info, 0, vect_body);
       return true;
     }
@@ -3321,8 +3118,7 @@ vectorizable_bswap (vec_info *vinfo,
 
   /* Transform.  */
   vec<tree> vec_oprnds = vNULL;
-  vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
-		     op, &vec_oprnds);
+  vect_get_vec_defs (vinfo, slp_node, op, &vec_oprnds);
   /* Arguments are ready. create the new vector stmt.  */
   unsigned i;
   tree vop;
@@ -3341,15 +3137,9 @@ vectorizable_bswap (vec_info *vinfo,
       new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
 						   vectype, tem2));
       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-      if (slp_node)
-	slp_node->push_vec_def (new_stmt);
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+      slp_node->push_vec_def (new_stmt);
     }
 
-  if (!slp_node)
-    *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-
   vec_oprnds.release ();
   return true;
 }
@@ -3389,14 +3179,14 @@ simple_integer_narrowing (tree vectype_out, tree vectype_in,
 static bool
 vectorizable_call (vec_info *vinfo,
 		   stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-		   gimple **vec_stmt, slp_tree slp_node,
+		   slp_tree slp_node,
 		   stmt_vector_for_cost *cost_vec)
 {
   gcall *stmt;
   tree vec_dest;
   tree scalar_dest;
   tree op;
-  tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
+  tree vec_oprnd0 = NULL_TREE;
   tree vectype_out, vectype_in;
   poly_uint64 nunits_in;
   poly_uint64 nunits_out;
@@ -3408,19 +3198,16 @@ vectorizable_call (vec_info *vinfo,
 	vect_unknown_def_type };
   tree vectypes[ARRAY_SIZE (dt)] = {};
   slp_tree slp_op[ARRAY_SIZE (dt)] = {};
-  int ndts = ARRAY_SIZE (dt);
-  int ncopies, j;
   auto_vec<tree, 8> vargs;
   enum { NARROW, NONE, WIDEN } modifier;
   size_t i, nargs;
-  tree lhs;
   tree clz_ctz_arg1 = NULL_TREE;
 
   if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   /* Is STMT_INFO a vectorizable call?   */
@@ -3440,7 +3227,7 @@ vectorizable_call (vec_info *vinfo,
 
   gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
 
-  vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+  vectype_out = SLP_TREE_VECTYPE (slp_node);
 
   /* Process function arguments.  */
   rhs_type = NULL_TREE;
@@ -3471,19 +3258,25 @@ vectorizable_call (vec_info *vinfo,
 
   int mask_opno = -1;
   if (internal_fn_p (cfn))
-    mask_opno = internal_fn_mask_index (as_internal_fn (cfn));
+    {
+      /* We can only handle direct internal masked calls here,
+	 vectorizable_simd_clone_call is for the rest.  */
+      if (cfn == CFN_MASK_CALL)
+	return false;
+      mask_opno = internal_fn_mask_index (as_internal_fn (cfn));
+    }
 
   for (i = 0; i < nargs; i++)
     {
       if ((int) i == mask_opno)
 	{
-	  if (!vect_check_scalar_mask (vinfo, stmt_info, slp_node, mask_opno,
-				       &op, &slp_op[i], &dt[i], &vectypes[i]))
+	  if (!vect_check_scalar_mask (vinfo, slp_node, mask_opno,
+				       &slp_op[i], &dt[i], &vectypes[i]))
 	    return false;
 	  continue;
 	}
 
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+      if (!vect_is_simple_use (vinfo, slp_node,
 			       i, &op, &slp_op[i], &dt[i], &vectypes[i]))
 	{
 	  if (dump_enabled_p ())
@@ -3519,7 +3312,7 @@ vectorizable_call (vec_info *vinfo,
      from the scalar type.  */
   if (!vectype_in)
     vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
-  if (vec_stmt)
+  if (!cost_vec)
     gcc_assert (vectype_in);
   if (!vectype_in)
     {
@@ -3539,13 +3332,14 @@ vectorizable_call (vec_info *vinfo,
       return false;
     }
 
-  if (vect_emulated_vector_p (vectype_in) || vect_emulated_vector_p (vectype_out))
-  {
+  if (vect_emulated_vector_p (vectype_in)
+      || vect_emulated_vector_p (vectype_out))
+    {
       if (dump_enabled_p ())
 	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
 			 "use emulated vector type for call\n");
       return false;
-  }
+    }
 
   /* FORNOW */
   nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
@@ -3600,7 +3394,7 @@ vectorizable_call (vec_info *vinfo,
   if (ifn == IFN_LAST && !fndecl)
     {
       if (cfn == CFN_GOMP_SIMD_LANE
-	  && (!slp_node || SLP_TREE_LANES (slp_node) == 1)
+	  && SLP_TREE_LANES (slp_node) == 1
 	  && loop_vinfo
 	  && LOOP_VINFO_LOOP (loop_vinfo)->simduid
 	  && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
@@ -3616,7 +3410,7 @@ vectorizable_call (vec_info *vinfo,
 		   || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP32)
 		   || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP64)
 		   || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP128)))
-	return vectorizable_bswap (vinfo, stmt_info, gsi, vec_stmt, slp_node,
+	return vectorizable_bswap (vinfo, stmt_info, gsi, slp_node,
 				   slp_op, vectype_in, cost_vec);
       else
 	{
@@ -3627,42 +3421,27 @@ vectorizable_call (vec_info *vinfo,
 	}
     }
 
-  if (slp_node)
-    ncopies = 1;
-  else if (modifier == NARROW && ifn == IFN_LAST)
-    ncopies = vect_get_num_copies (loop_vinfo, vectype_out);
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
-
-  /* Sanity check: make sure that at least one copy of the vectorized stmt
-     needs to be generated.  */
-  gcc_assert (ncopies >= 1);
-
   int reduc_idx = STMT_VINFO_REDUC_IDX (stmt_info);
   internal_fn cond_fn = get_conditional_internal_fn (ifn);
   internal_fn cond_len_fn = get_len_internal_fn (ifn);
   int len_opno = internal_fn_len_index (cond_len_fn);
   vec_loop_masks *masks = (loop_vinfo ? &LOOP_VINFO_MASKS (loop_vinfo) : NULL);
   vec_loop_lens *lens = (loop_vinfo ? &LOOP_VINFO_LENS (loop_vinfo) : NULL);
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
-      if (slp_node)
-	for (i = 0; i < nargs; ++i)
-	  if (!vect_maybe_update_slp_op_vectype (slp_op[i],
-						 vectypes[i]
-						 ? vectypes[i] : vectype_in))
-	    {
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-				 "incompatible vector types for invariants\n");
-	      return false;
-	    }
-      STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
+      for (i = 0; i < nargs; ++i)
+	if (!vect_maybe_update_slp_op_vectype (slp_op[i],
+					       vectypes[i]
+					       ? vectypes[i] : vectype_in))
+	  {
+	    if (dump_enabled_p ())
+	      dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			       "incompatible vector types for invariants\n");
+	    return false;
+	  }
+      SLP_TREE_TYPE (slp_node) = call_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_call");
-      vect_model_simple_cost (vinfo, ncopies, dt, ndts, slp_node, cost_vec);
-      if (ifn != IFN_LAST && modifier == NARROW && !slp_node)
-	record_stmt_cost (cost_vec, ncopies / 2,
-			  vec_promote_demote, stmt_info, 0, vect_body);
+      vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
 
       if (loop_vinfo
 	  && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
@@ -3684,10 +3463,7 @@ vectorizable_call (vec_info *vinfo,
 	    }
 	  else
 	    {
-	      unsigned int nvectors
-		= (slp_node
-		   ? SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node)
-		   : ncopies);
+	      unsigned int nvectors = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
 	      tree scalar_mask = NULL_TREE;
 	      if (mask_opno >= 0)
 		scalar_mask = gimple_call_arg (stmt_info->stmt, mask_opno);
@@ -3740,220 +3516,112 @@ vectorizable_call (vec_info *vinfo,
       tree prev_res = NULL_TREE;
       vargs.safe_grow (vect_nargs, true);
       auto_vec<vec<tree> > vec_defs (nargs);
-      for (j = 0; j < ncopies; ++j)
+
+      /* Build argument list for the vectorized call.  */
+      if (cfn == CFN_GOMP_SIMD_LANE)
+	{
+	  for (i = 0; i < SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); ++i)
+	    {
+	      /* ???  For multi-lane SLP we'd need to build
+		 { 0, 0, .., 1, 1, ... }.  */
+	      tree cst = build_index_vector (vectype_out,
+					     i * nunits_out, 1);
+	      tree new_var
+		= vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
+	      gimple *init_stmt = gimple_build_assign (new_var, cst);
+	      vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
+	      new_temp = make_ssa_name (vec_dest);
+	      gimple *new_stmt = gimple_build_assign (new_temp, new_var);
+	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
+	      slp_node->push_vec_def (new_stmt);
+	    }
+	}
+      else
 	{
-	  /* Build argument list for the vectorized call.  */
-	  if (slp_node)
+	  vec<tree> vec_oprnds0;
+	  vect_get_slp_defs (vinfo, slp_node, &vec_defs);
+	  vec_oprnds0 = vec_defs[0];
+
+	  /* Arguments are ready.  Create the new vector stmt.  */
+	  FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
 	    {
-	      if (cfn == CFN_GOMP_SIMD_LANE)
+	      int varg = 0;
+	      if (masked_loop_p && reduc_idx >= 0)
+		{
+		  unsigned int vec_num = vec_oprnds0.length ();
+		  vargs[varg++] = vect_get_loop_mask (loop_vinfo, gsi, masks,
+						      vec_num, vectype_out, i);
+		}
+	      size_t k;
+	      for (k = 0; k < nargs; k++)
 		{
-		  for (i = 0; i < SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); ++i)
+		  vec<tree> vec_oprndsk = vec_defs[k];
+		  vargs[varg++] = vec_oprndsk[i];
+		}
+	      if (masked_loop_p && reduc_idx >= 0)
+		vargs[varg++] = vargs[reduc_idx + 1];
+	      if (clz_ctz_arg1)
+		vargs[varg++] = clz_ctz_arg1;
+
+	      gimple *new_stmt;
+	      if (modifier == NARROW)
+		{
+		  /* We don't define any narrowing conditional functions
+		     at present.  */
+		  gcc_assert (mask_opno < 0);
+		  tree half_res = make_ssa_name (vectype_in);
+		  gcall *call = gimple_build_call_internal_vec (ifn, vargs);
+		  gimple_call_set_lhs (call, half_res);
+		  gimple_call_set_nothrow (call, true);
+		  vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+		  if ((i & 1) == 0)
 		    {
-		      /* ???  For multi-lane SLP we'd need to build
-			 { 0, 0, .., 1, 1, ... }.  */
-		      tree cst = build_index_vector (vectype_out,
-						     i * nunits_out, 1);
-		      tree new_var
-			  = vect_get_new_ssa_name (vectype_out, vect_simple_var,
-						   "cst_");
-		      gimple *init_stmt = gimple_build_assign (new_var, cst);
-		      vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
-		      new_temp = make_ssa_name (vec_dest);
-		      gimple *new_stmt
-			= gimple_build_assign (new_temp, new_var);
-		      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
-						   gsi);
-		      slp_node->push_vec_def (new_stmt);
+		      prev_res = half_res;
+		      continue;
 		    }
-		  continue;
+		  new_temp = make_ssa_name (vec_dest);
+		  new_stmt = vect_gimple_build (new_temp, convert_code,
+						prev_res, half_res);
+		  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 		}
-
-	      vec<tree> vec_oprnds0;
-	      vect_get_slp_defs (vinfo, slp_node, &vec_defs);
-	      vec_oprnds0 = vec_defs[0];
-
-	      /* Arguments are ready.  Create the new vector stmt.  */
-	      FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
+	      else
 		{
-		  int varg = 0;
-		  if (masked_loop_p && reduc_idx >= 0)
+		  if (len_opno >= 0 && len_loop_p)
 		    {
 		      unsigned int vec_num = vec_oprnds0.length ();
-		      /* Always true for SLP.  */
-		      gcc_assert (ncopies == 1);
-		      vargs[varg++] = vect_get_loop_mask (loop_vinfo,
-							  gsi, masks, vec_num,
-							  vectype_out, i);
+		      tree len = vect_get_loop_len (loop_vinfo, gsi, lens,
+						    vec_num, vectype_out, i, 1);
+		      signed char biasval
+			= LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+		      tree bias = build_int_cst (intQI_type_node, biasval);
+		      vargs[len_opno] = len;
+		      vargs[len_opno + 1] = bias;
 		    }
-		  size_t k;
-		  for (k = 0; k < nargs; k++)
+		  else if (mask_opno >= 0 && masked_loop_p)
 		    {
-		      vec<tree> vec_oprndsk = vec_defs[k];
-		      vargs[varg++] = vec_oprndsk[i];
+		      unsigned int vec_num = vec_oprnds0.length ();
+		      tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
+						      vec_num, vectype_out, i);
+		      vargs[mask_opno]
+			= prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
+					    vargs[mask_opno], gsi);
 		    }
-		  if (masked_loop_p && reduc_idx >= 0)
-		    vargs[varg++] = vargs[reduc_idx + 1];
-		  if (clz_ctz_arg1)
-		    vargs[varg++] = clz_ctz_arg1;
 
-		  gimple *new_stmt;
-		  if (modifier == NARROW)
-		    {
-		      /* We don't define any narrowing conditional functions
-			 at present.  */
-		      gcc_assert (mask_opno < 0);
-		      tree half_res = make_ssa_name (vectype_in);
-		      gcall *call
-			= gimple_build_call_internal_vec (ifn, vargs);
-		      gimple_call_set_lhs (call, half_res);
-		      gimple_call_set_nothrow (call, true);
-		      vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
-		      if ((i & 1) == 0)
-			{
-			  prev_res = half_res;
-			  continue;
-			}
-		      new_temp = make_ssa_name (vec_dest);
-		      new_stmt = vect_gimple_build (new_temp, convert_code,
-						    prev_res, half_res);
-		      vect_finish_stmt_generation (vinfo, stmt_info,
-						   new_stmt, gsi);
-		    }
+		  gcall *call;
+		  if (ifn != IFN_LAST)
+		    call = gimple_build_call_internal_vec (ifn, vargs);
 		  else
-		    {
-		      if (len_opno >= 0 && len_loop_p)
-			{
-			  unsigned int vec_num = vec_oprnds0.length ();
-			  /* Always true for SLP.  */
-			  gcc_assert (ncopies == 1);
-			  tree len
-			    = vect_get_loop_len (loop_vinfo, gsi, lens, vec_num,
-						 vectype_out, i, 1);
-			  signed char biasval
-			    = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
-			  tree bias = build_int_cst (intQI_type_node, biasval);
-			  vargs[len_opno] = len;
-			  vargs[len_opno + 1] = bias;
-			}
-		      else if (mask_opno >= 0 && masked_loop_p)
-			{
-			  unsigned int vec_num = vec_oprnds0.length ();
-			  /* Always true for SLP.  */
-			  gcc_assert (ncopies == 1);
-			  tree mask = vect_get_loop_mask (loop_vinfo,
-							  gsi, masks, vec_num,
-							  vectype_out, i);
-			  vargs[mask_opno] = prepare_vec_mask
-			    (loop_vinfo, TREE_TYPE (mask), mask,
-			     vargs[mask_opno], gsi);
-			}
-
-		      gcall *call;
-		      if (ifn != IFN_LAST)
-			call = gimple_build_call_internal_vec (ifn, vargs);
-		      else
-			call = gimple_build_call_vec (fndecl, vargs);
-		      new_temp = make_ssa_name (vec_dest, call);
-		      gimple_call_set_lhs (call, new_temp);
-		      gimple_call_set_nothrow (call, true);
-		      vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
-		      new_stmt = call;
-		    }
-		  slp_node->push_vec_def (new_stmt);
-		}
-	      continue;
-	    }
-
-	  int varg = 0;
-	  if (masked_loop_p && reduc_idx >= 0)
-	    vargs[varg++] = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies,
-						vectype_out, j);
-	  for (i = 0; i < nargs; i++)
-	    {
-	      op = gimple_call_arg (stmt, i);
-	      if (j == 0)
-		{
-		  vec_defs.quick_push (vNULL);
-		  vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
-						 op, &vec_defs[i],
-						 vectypes[i]);
-		}
-	      vargs[varg++] = vec_defs[i][j];
-	    }
-	  if (masked_loop_p && reduc_idx >= 0)
-	    vargs[varg++] = vargs[reduc_idx + 1];
-	  if (clz_ctz_arg1)
-	    vargs[varg++] = clz_ctz_arg1;
-
-	  if (len_opno >= 0 && len_loop_p)
-	    {
-	      tree len = vect_get_loop_len (loop_vinfo, gsi, lens, ncopies,
-					    vectype_out, j, 1);
-	      signed char biasval
-		= LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
-	      tree bias = build_int_cst (intQI_type_node, biasval);
-	      vargs[len_opno] = len;
-	      vargs[len_opno + 1] = bias;
-	    }
-	  else if (mask_opno >= 0 && masked_loop_p)
-	    {
-	      tree mask = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies,
-					      vectype_out, j);
-	      vargs[mask_opno]
-		= prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
-				    vargs[mask_opno], gsi);
-	    }
-
-	  gimple *new_stmt;
-	  if (cfn == CFN_GOMP_SIMD_LANE)
-	    {
-	      tree cst = build_index_vector (vectype_out, j * nunits_out, 1);
-	      tree new_var
-		= vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
-	      gimple *init_stmt = gimple_build_assign (new_var, cst);
-	      vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL);
-	      new_temp = make_ssa_name (vec_dest);
-	      new_stmt = gimple_build_assign (new_temp, new_var);
-	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-	    }
-	  else if (modifier == NARROW)
-	    {
-	      /* We don't define any narrowing conditional functions at
-		 present.  */
-	      gcc_assert (mask_opno < 0);
-	      tree half_res = make_ssa_name (vectype_in);
-	      gcall *call = gimple_build_call_internal_vec (ifn, vargs);
-	      gimple_call_set_lhs (call, half_res);
-	      gimple_call_set_nothrow (call, true);
-	      vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
-	      if ((j & 1) == 0)
-		{
-		  prev_res = half_res;
-		  continue;
+		    call = gimple_build_call_vec (fndecl, vargs);
+		  new_temp = make_ssa_name (vec_dest, call);
+		  gimple_call_set_lhs (call, new_temp);
+		  gimple_call_set_nothrow (call, true);
+		  vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+		  new_stmt = call;
 		}
-	      new_temp = make_ssa_name (vec_dest);
-	      new_stmt = vect_gimple_build (new_temp, convert_code, prev_res,
-					    half_res);
-	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-	    }
-	  else
-	    {
-	      gcall *call;
-	      if (ifn != IFN_LAST)
-		call = gimple_build_call_internal_vec (ifn, vargs);
-	      else
-		call = gimple_build_call_vec (fndecl, vargs);
-	      new_temp = make_ssa_name (vec_dest, call);
-	      gimple_call_set_lhs (call, new_temp);
-	      gimple_call_set_nothrow (call, true);
-	      vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
-	      new_stmt = call;
+	      slp_node->push_vec_def (new_stmt);
 	    }
-
-	  if (j == (modifier == NARROW ? 1 : 0))
-	    *vec_stmt = new_stmt;
-	  STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
 	}
+
       for (i = 0; i < nargs; i++)
 	{
 	  vec<tree> vec_oprndsi = vec_defs[i];
@@ -3965,73 +3633,36 @@ vectorizable_call (vec_info *vinfo,
       auto_vec<vec<tree> > vec_defs (nargs);
       /* We don't define any narrowing conditional functions at present.  */
       gcc_assert (mask_opno < 0);
-      for (j = 0; j < ncopies; ++j)
-	{
-	  /* Build argument list for the vectorized call.  */
-	  if (j == 0)
-	    vargs.create (nargs * 2);
-	  else
-	    vargs.truncate (0);
 
-	  if (slp_node)
-	    {
-	      vec<tree> vec_oprnds0;
+      /* Build argument list for the vectorized call.  */
+      vargs.create (nargs * 2);
 
-	      vect_get_slp_defs (vinfo, slp_node, &vec_defs);
-	      vec_oprnds0 = vec_defs[0];
+      vect_get_slp_defs (vinfo, slp_node, &vec_defs);
+      vec<tree> vec_oprnds0 = vec_defs[0];
 
-	      /* Arguments are ready.  Create the new vector stmt.  */
-	      for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2)
-		{
-		  size_t k;
-		  vargs.truncate (0);
-		  for (k = 0; k < nargs; k++)
-		    {
-		      vec<tree> vec_oprndsk = vec_defs[k];
-		      vargs.quick_push (vec_oprndsk[i]);
-		      vargs.quick_push (vec_oprndsk[i + 1]);
-		    }
-		  gcall *call;
-		  if (ifn != IFN_LAST)
-		    call = gimple_build_call_internal_vec (ifn, vargs);
-		  else
-		    call = gimple_build_call_vec (fndecl, vargs);
-		  new_temp = make_ssa_name (vec_dest, call);
-		  gimple_call_set_lhs (call, new_temp);
-		  gimple_call_set_nothrow (call, true);
-		  vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
-		  slp_node->push_vec_def (call);
-		}
-	      continue;
-	    }
-
-	  for (i = 0; i < nargs; i++)
+      /* Arguments are ready.  Create the new vector stmt.  */
+      for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2)
+	{
+	  size_t k;
+	  vargs.truncate (0);
+	  for (k = 0; k < nargs; k++)
 	    {
-	      op = gimple_call_arg (stmt, i);
-	      if (j == 0)
-		{
-		  vec_defs.quick_push (vNULL);
-		  vect_get_vec_defs_for_operand (vinfo, stmt_info, 2 * ncopies,
-						 op, &vec_defs[i], vectypes[i]);
-		}
-	      vec_oprnd0 = vec_defs[i][2*j];
-	      vec_oprnd1 = vec_defs[i][2*j+1];
-
-	      vargs.quick_push (vec_oprnd0);
-	      vargs.quick_push (vec_oprnd1);
+	      vec<tree> vec_oprndsk = vec_defs[k];
+	      vargs.quick_push (vec_oprndsk[i]);
+	      vargs.quick_push (vec_oprndsk[i + 1]);
 	    }
-
-	  gcall *new_stmt = gimple_build_call_vec (fndecl, vargs);
-	  new_temp = make_ssa_name (vec_dest, new_stmt);
-	  gimple_call_set_lhs (new_stmt, new_temp);
-	  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-
-	  STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+	  gcall *call;
+	  if (ifn != IFN_LAST)
+	    call = gimple_build_call_internal_vec (ifn, vargs);
+	  else
+	    call = gimple_build_call_vec (fndecl, vargs);
+	  new_temp = make_ssa_name (vec_dest, call);
+	  gimple_call_set_lhs (call, new_temp);
+	  gimple_call_set_nothrow (call, true);
+	  vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
+	  slp_node->push_vec_def (call);
 	}
 
-      if (!slp_node)
-	*vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-
       for (i = 0; i < nargs; i++)
 	{
 	  vec<tree> vec_oprndsi = vec_defs[i];
@@ -4044,21 +3675,6 @@ vectorizable_call (vec_info *vinfo,
 
   vargs.release ();
 
-  /* The call in STMT might prevent it from being removed in dce.
-     We however cannot remove it here, due to the way the ssa name
-     it defines is mapped to the new definition.  So just replace
-     rhs of the statement with something harmless.  */
-
-  if (slp_node)
-    return true;
-
-  stmt_info = vect_orig_stmt (stmt_info);
-  lhs = gimple_get_lhs (stmt_info->stmt);
-
-  gassign *new_stmt
-    = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
-  vinfo->replace_stmt (gsi, stmt_info, new_stmt);
-
   return true;
 }
 
@@ -4152,12 +3768,12 @@ vect_simd_lane_linear (tree op, class loop *loop,
 static bool
 vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 			      gimple_stmt_iterator *gsi,
-			      gimple **vec_stmt, slp_tree slp_node,
-			      stmt_vector_for_cost *)
+			      slp_tree slp_node,
+			      stmt_vector_for_cost *cost_vec)
 {
   tree vec_dest;
   tree scalar_dest;
-  tree op, type;
+  tree op;
   tree vec_oprnd0 = NULL_TREE;
   tree vectype;
   poly_uint64 nunits;
@@ -4169,7 +3785,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
   auto_vec<simd_call_arg_info> arginfo;
   vec<tree> vargs = vNULL;
   size_t i, nargs;
-  tree lhs, rtype, ratype;
+  tree rtype, ratype;
   vec<constructor_elt, va_gc> *ret_ctor_elts = NULL;
   int masked_call_offset = 0;
 
@@ -4199,7 +3815,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   if (gimple_call_lhs (stmt)
@@ -4208,7 +3824,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 
   gcc_checking_assert (!stmt_can_throw_internal (cfun, stmt));
 
-  vectype = STMT_VINFO_VECTYPE (stmt_info);
+  vectype = SLP_TREE_VECTYPE (slp_node);
 
   if (loop_vinfo && nested_in_vect_loop_p (loop, stmt_info))
     return false;
@@ -4220,10 +3836,9 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
   if (nargs == 0)
     return false;
 
-  vec<tree>& simd_clone_info = (slp_node ? SLP_TREE_SIMD_CLONE_INFO (slp_node)
-				: STMT_VINFO_SIMD_CLONE_INFO (stmt_info));
-  if (!vec_stmt)
-    simd_clone_info.truncate (0);
+  vect_simd_clone_data _data;
+  vect_simd_clone_data &data = slp_node->get_data (_data);
+  vec<tree>& simd_clone_info = data.simd_clone_info;
   arginfo.reserve (nargs, true);
   auto_vec<slp_tree> slp_op;
   slp_op.safe_grow_cleared (nargs);
@@ -4238,10 +3853,10 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
       thisarginfo.op = NULL_TREE;
       thisarginfo.simd_lane_linear = false;
 
-      int op_no = i + masked_call_offset;
-      if (slp_node)
-	op_no = vect_slp_child_index_for_operand (stmt, op_no, false);
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+      int op_no = vect_slp_child_index_for_operand (stmt,
+						    i + masked_call_offset,
+						    false);
+      if (!vect_is_simple_use (vinfo, slp_node,
 			       op_no, &op, &slp_op[i],
 			       &thisarginfo.dt, &thisarginfo.vectype)
 	  || thisarginfo.dt == vect_uninitialized_def)
@@ -4259,10 +3874,9 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 	     at analysis time, handling conflicts via
 	     vect_maybe_update_slp_op_vectype.  At transform time
 	     we have a vector type recorded for SLP.  */
-	  gcc_assert (!vec_stmt
-		      || !slp_node
+	  gcc_assert (cost_vec
 		      || thisarginfo.vectype != NULL_TREE);
-	  if (!vec_stmt)
+	  if (cost_vec)
 	    thisarginfo.vectype = get_vectype_for_scalar_type (vinfo,
 							       TREE_TYPE (op),
 							       slp_node);
@@ -4272,7 +3886,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 
       /* For linear arguments, the analyze phase should have saved
 	 the base and step in {STMT_VINFO,SLP_TREE}_SIMD_CLONE_INFO.  */
-      if (vec_stmt
+      if (!cost_vec
 	  && i * 3 + 4 <= simd_clone_info.length ()
 	  && simd_clone_info[i * 3 + 2])
 	{
@@ -4296,7 +3910,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 			       thisarginfo.op, bias);
 	    }
 	}
-      else if (!vec_stmt
+      else if (cost_vec
 	       && thisarginfo.dt != vect_constant_def
 	       && thisarginfo.dt != vect_external_def
 	       && loop_vinfo
@@ -4316,7 +3930,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 	 linear too.  */
       if (POINTER_TYPE_P (TREE_TYPE (op))
 	  && !thisarginfo.linear_step
-	  && !vec_stmt
+	  && cost_vec
 	  && thisarginfo.dt != vect_constant_def
 	  && thisarginfo.dt != vect_external_def
 	  && loop_vinfo
@@ -4327,10 +3941,10 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
     }
 
   poly_uint64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1;
-  unsigned group_size = slp_node ? SLP_TREE_LANES (slp_node) : 1;
+  unsigned group_size = SLP_TREE_LANES (slp_node);
   unsigned int badness = 0;
   struct cgraph_node *bestn = NULL;
-  if (vec_stmt)
+  if (!cost_vec)
     bestn = cgraph_node::get (simd_clone_info[0]);
   else
     for (struct cgraph_node *n = node->simd_clones; n != NULL;
@@ -4357,7 +3971,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 	if (n->simdclone->inbranch)
 	  this_badness += 8192;
 
-	/* If STMT_VINFO_VECTYPE has not been set yet pass the general vector
+	/* If SLP_TREE_VECTYPE has not been set yet pass the general vector
 	   mode,  which for targets that use it will determine what ISA we can
 	   vectorize this code with.  */
 	machine_mode vector_mode = vinfo->vector_mode;
@@ -4542,10 +4156,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 
   fndecl = bestn->decl;
   nunits = bestn->simdclone->simdlen;
-  if (slp_node)
-    ncopies = vector_unroll_factor (vf * group_size, nunits);
-  else
-    ncopies = vector_unroll_factor (vf, nunits);
+  ncopies = vector_unroll_factor (vf * group_size, nunits);
 
   /* If the function isn't const, only allow it in simd loops where user
      has asserted that at least nunits consecutive iterations can be
@@ -4558,17 +4169,16 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
      needs to be generated.  */
   gcc_assert (ncopies >= 1);
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
-      if (slp_node)
-	for (unsigned i = 0; i < nargs; ++i)
-	  if (!vect_maybe_update_slp_op_vectype (slp_op[i], arginfo[i].vectype))
-	    {
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-				 "incompatible vector types for invariants\n");
-	      return false;
-	    }
+      for (unsigned i = 0; i < nargs; ++i)
+	if (!vect_maybe_update_slp_op_vectype (slp_op[i], arginfo[i].vectype))
+	  {
+	    if (dump_enabled_p ())
+	      dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			       "incompatible vector types for invariants\n");
+	    return false;
+	  }
       /* When the original call is pure or const but the SIMD ABI dictates
 	 an aggregate return we will have to use a virtual definition and
 	 in a loop eventually even need to add a virtual PHI.  That's
@@ -4580,7 +4190,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
       /* ???  For SLP code-gen we end up inserting after the last
 	 vector argument def rather than at the original call position
 	 so automagic virtual operand updating doesn't work.  */
-      if (gimple_vuse (stmt) && slp_node)
+      if (gimple_vuse (stmt))
 	vinfo->any_known_not_updated_vssa = true;
       simd_clone_info.safe_push (bestn->decl);
       for (i = 0; i < bestn->simdclone->nargs; i++)
@@ -4624,9 +4234,10 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 	  LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
 	}
 
-      STMT_VINFO_TYPE (stmt_info) = call_simd_clone_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = call_simd_clone_vec_info_type;
+      slp_node->data = new vect_simd_clone_data (std::move (_data));
       DUMP_VECT_SCOPE ("vectorizable_simd_clone_call");
-/*      vect_model_simple_cost (vinfo, ncopies, dt, slp_node, cost_vec); */
+/*      vect_model_simple_cost (vinfo, 1, slp_node, cost_vec); */
       return true;
     }
 
@@ -4654,13 +4265,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
   auto_vec<vec<tree> > vec_oprnds;
   auto_vec<unsigned> vec_oprnds_i;
   vec_oprnds_i.safe_grow_cleared (nargs, true);
-  if (slp_node)
-    {
-      vec_oprnds.reserve_exact (nargs);
-      vect_get_slp_defs (vinfo, slp_node, &vec_oprnds);
-    }
-  else
-    vec_oprnds.safe_grow_cleared (nargs, true);
+  vec_oprnds.reserve_exact (nargs);
+  vect_get_slp_defs (vinfo, slp_node, &vec_oprnds);
   for (j = 0; j < ncopies; ++j)
     {
       poly_uint64 callee_nelements;
@@ -4695,10 +4301,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 		      gcc_assert ((k & (k - 1)) == 0);
 		      if (m == 0)
 			{
-			  if (!slp_node)
-			    vect_get_vec_defs_for_operand (vinfo, stmt_info,
-							   ncopies * o / k, op,
-							   &vec_oprnds[i]);
 			  vec_oprnds_i[i] = 0;
 			  vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
 			}
@@ -4735,11 +4337,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 			{
 			  if (m == 0 && l == 0)
 			    {
-			      if (!slp_node)
-				vect_get_vec_defs_for_operand (vinfo, stmt_info,
-							       k * o * ncopies,
-							       op,
-							       &vec_oprnds[i]);
 			      vec_oprnds_i[i] = 0;
 			      vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
 			    }
@@ -4803,14 +4400,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 			  /* The SIMD clone function has the same number of
 			     elements as the current function.  */
 			  if (m == 0)
-			    {
-			      if (!slp_node)
-				vect_get_vec_defs_for_operand (vinfo, stmt_info,
-							       o * ncopies,
-							       op,
-							       &vec_oprnds[i]);
-			      vec_oprnds_i[i] = 0;
-			    }
+			    vec_oprnds_i[i] = 0;
 			  vec_oprnd0 = vec_oprnds[i][vec_oprnds_i[i]++];
 			  if (loop_vinfo
 			      && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
@@ -4861,14 +4451,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 		  for (m = j * o; m < (j + 1) * o; m++)
 		    {
 		      if (m == 0)
-			{
-			  if (!slp_node)
-			    vect_get_vec_defs_for_operand (vinfo, stmt_info,
-							   o * ncopies,
-							   op,
-							   &vec_oprnds[i]);
-			  vec_oprnds_i[i] = 0;
-			}
+			vec_oprnds_i[i] = 0;
 		      if (maybe_lt (atype_subparts,
 				    TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)))
 			{
@@ -5103,13 +4686,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 		  new_stmt = gimple_build_assign (make_ssa_name (vectype), t);
 		  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 
-		  if (j == 0 && l == 0)
-		    *vec_stmt = new_stmt;
-		  if (slp_node)
-		    SLP_TREE_VEC_DEFS (slp_node)
-		      .quick_push (gimple_assign_lhs (new_stmt));
-		  else
-		    STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+		  SLP_TREE_VEC_DEFS (slp_node)
+		    .quick_push (gimple_assign_lhs (new_stmt));
 		}
 
 	      if (ratype)
@@ -5152,13 +4730,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 		= gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
 	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 
-	      if ((unsigned) j == k - 1)
-		*vec_stmt = new_stmt;
-	      if (slp_node)
-		SLP_TREE_VEC_DEFS (slp_node)
-		  .quick_push (gimple_assign_lhs (new_stmt));
-	      else
-		STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+	      SLP_TREE_VEC_DEFS (slp_node)
+		.quick_push (gimple_assign_lhs (new_stmt));
 	      continue;
 	    }
 	  else if (ratype)
@@ -5179,12 +4752,7 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
 	    }
 	}
 
-      if (j == 0)
-	*vec_stmt = new_stmt;
-      if (slp_node)
-	SLP_TREE_VEC_DEFS (slp_node).quick_push (gimple_get_lhs (new_stmt));
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+      SLP_TREE_VEC_DEFS (slp_node).quick_push (gimple_get_lhs (new_stmt));
     }
 
   for (i = 0; i < nargs; ++i)
@@ -5197,26 +4765,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info,
   /* Mark the clone as no longer being a candidate for GC.  */
   bestn->gc_candidate = false;
 
-  /* The call in STMT might prevent it from being removed in dce.
-     We however cannot remove it here, due to the way the ssa name
-     it defines is mapped to the new definition.  So just replace
-     rhs of the statement with something harmless.  */
-
-  if (slp_node)
-    return true;
-
-  gimple *new_stmt;
-  if (scalar_dest)
-    {
-      type = TREE_TYPE (scalar_dest);
-      lhs = gimple_call_lhs (vect_orig_stmt (stmt_info)->stmt);
-      new_stmt = gimple_build_assign (lhs, build_zero_cst (type));
-    }
-  else
-    new_stmt = gimple_build_nop ();
-  vinfo->replace_stmt (gsi, vect_orig_stmt (stmt_info), new_stmt);
-  unlink_stmt_vdef (stmt);
-
   return true;
 }
 
@@ -5287,12 +4835,8 @@ vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec<tree> *vec_oprnds,
       else
 	{
 	  /* This is the last step of the conversion sequence. Store the
-	     vectors in SLP_NODE or in vector info of the scalar statement
-	     (or in STMT_VINFO_RELATED_STMT chain).  */
-	  if (slp_node)
-	    slp_node->push_vec_def (new_stmt);
-	  else
-	    STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+	     vectors in SLP_NODE.  */
+	  slp_node->push_vec_def (new_stmt);
 	}
     }
 
@@ -5437,7 +4981,7 @@ vect_create_half_widening_stmts (vec_info *vinfo,
 static bool
 vectorizable_conversion (vec_info *vinfo,
 			 stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-			 gimple **vec_stmt, slp_tree slp_node,
+			 slp_tree slp_node,
 			 stmt_vector_for_cost *cost_vec)
 {
   tree vec_dest, cvt_op = NULL_TREE;
@@ -5448,7 +4992,6 @@ vectorizable_conversion (vec_info *vinfo,
   code_helper codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
   tree new_temp;
   enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
-  int ndts = 2;
   poly_uint64 nunits_in;
   poly_uint64 nunits_out;
   tree vectype_out, vectype_in;
@@ -5467,6 +5010,7 @@ vectorizable_conversion (vec_info *vinfo,
   vec<tree> vec_oprnds1 = vNULL;
   tree vop0;
   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
+  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   int multi_step_cvt = 0;
   vec<tree> interm_types = vNULL;
   tree intermediate_type, cvt_type = NULL_TREE;
@@ -5479,7 +5023,7 @@ vectorizable_conversion (vec_info *vinfo,
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   gimple* stmt = stmt_info->stmt;
@@ -5523,7 +5067,7 @@ vectorizable_conversion (vec_info *vinfo,
 
   /* Check the operands of the operation.  */
   slp_tree slp_op0, slp_op1 = NULL;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+  if (!vect_is_simple_use (vinfo, slp_node,
 			   0, &op0, &slp_op0, &dt[0], &vectype_in))
     {
       if (dump_enabled_p ())
@@ -5562,7 +5106,7 @@ vectorizable_conversion (vec_info *vinfo,
       op1 = is_gimple_assign (stmt) ? gimple_assign_rhs2 (stmt) :
 				     gimple_call_arg (stmt, 0);
       tree vectype1_in;
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1,
+      if (!vect_is_simple_use (vinfo, slp_node, 1,
 			       &op1, &slp_op1, &dt[1], &vectype1_in))
 	{
           if (dump_enabled_p ())
@@ -5580,7 +5124,7 @@ vectorizable_conversion (vec_info *vinfo,
      from the scalar type.  */
   if (!vectype_in)
     vectype_in = get_vectype_for_scalar_type (vinfo, rhs_type, slp_node);
-  if (vec_stmt)
+  if (!cost_vec)
     gcc_assert (vectype_in);
   if (!vectype_in)
     {
@@ -5809,7 +5353,21 @@ vectorizable_conversion (vec_info *vinfo,
       gcc_unreachable ();
     }
 
-  if (!vec_stmt)		/* transformation not required.  */
+  if (modifier == WIDEN
+      && loop_vinfo
+      && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
+      && (code1 == VEC_WIDEN_MULT_EVEN_EXPR
+	  || widening_evenodd_fn_p (code1)))
+    {
+      if (dump_enabled_p ())
+	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+			 "can't use a fully-masked loop because"
+			 " widening operation on even/odd elements"
+			 " mixes up lanes.\n");
+      LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+    }
+
+  if (cost_vec)		/* transformation not required.  */
     {
       if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype_in)
 	  || !vect_maybe_update_slp_op_vectype (slp_op1, vectype_in))
@@ -5822,13 +5380,13 @@ vectorizable_conversion (vec_info *vinfo,
       DUMP_VECT_SCOPE ("vectorizable_conversion");
       if (modifier == NONE)
         {
-	  STMT_VINFO_TYPE (stmt_info) = type_conversion_vec_info_type;
+	  SLP_TREE_TYPE (slp_node) = type_conversion_vec_info_type;
 	  vect_model_simple_cost (vinfo, (1 + multi_step_cvt),
-				  dt, ndts, slp_node, cost_vec);
+				  slp_node, cost_vec);
 	}
       else if (modifier == NARROW_SRC || modifier == NARROW_DST)
 	{
-	  STMT_VINFO_TYPE (stmt_info) = type_demotion_vec_info_type;
+	  SLP_TREE_TYPE (slp_node) = type_demotion_vec_info_type;
 	  /* The final packing step produces one vector result per copy.  */
 	  unsigned int nvectors = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
 	  vect_model_promotion_demotion_cost (stmt_info, dt, nvectors,
@@ -5837,7 +5395,7 @@ vectorizable_conversion (vec_info *vinfo,
 	}
       else
 	{
-	  STMT_VINFO_TYPE (stmt_info) = type_promotion_vec_info_type;
+	  SLP_TREE_TYPE (slp_node) = type_promotion_vec_info_type;
 	  /* The initial unpacking step produces two vector results
 	     per copy.  MULTI_STEP_CVT is 0 for a single conversion,
 	     so >> MULTI_STEP_CVT divides by 2^(number of steps - 1).  */
@@ -5895,8 +5453,7 @@ vectorizable_conversion (vec_info *vinfo,
   switch (modifier)
     {
     case NONE:
-      vect_get_vec_defs (vinfo, stmt_info, slp_node, 1,
-			 op0, vectype_in, &vec_oprnds0);
+      vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0);
       /* vec_dest is intermediate type operand when multi_step_cvt.  */
       if (multi_step_cvt)
 	{
@@ -5931,10 +5488,9 @@ vectorizable_conversion (vec_info *vinfo,
 	 of elements that we can fit in a vectype (nunits), we have to
 	 generate more than one vector stmt - i.e - we need to "unroll"
 	 the vector stmt by a factor VF/nunits.  */
-      vect_get_vec_defs (vinfo, stmt_info, slp_node, 1,
-			 op0, vectype_in, &vec_oprnds0,
+      vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0,
 			 code == WIDEN_LSHIFT_EXPR ? NULL_TREE : op1,
-			 vectype_in, &vec_oprnds1);
+			 &vec_oprnds1);
       if (code == WIDEN_LSHIFT_EXPR)
 	{
 	  int oprnds_size = vec_oprnds0.length ();
@@ -5985,8 +5541,7 @@ vectorizable_conversion (vec_info *vinfo,
 	 of elements that we can fit in a vectype (nunits), we have to
 	 generate more than one vector stmt - i.e - we need to "unroll"
 	 the vector stmt by a factor VF/nunits.  */
-      vect_get_vec_defs (vinfo, stmt_info, slp_node, 1,
-			 op0, vectype_in, &vec_oprnds0);
+      vect_get_vec_defs (vinfo, slp_node, op0, &vec_oprnds0);
       /* Arguments are ready.  Create the new vector stmts.  */
       if (cvt_type && modifier == NARROW_DST)
 	FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
@@ -6065,17 +5620,14 @@ vect_nop_conversion_p (stmt_vec_info stmt_info)
 static bool
 vectorizable_assignment (vec_info *vinfo,
 			 stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-			 gimple **vec_stmt, slp_tree slp_node,
+			 slp_tree slp_node,
 			 stmt_vector_for_cost *cost_vec)
 {
   tree vec_dest;
   tree scalar_dest;
   tree op;
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   tree new_temp;
   enum vect_def_type dt[1] = {vect_unknown_def_type};
-  int ndts = 1;
-  int ncopies;
   int i;
   vec<tree> vec_oprnds = vNULL;
   tree vop;
@@ -6087,7 +5639,7 @@ vectorizable_assignment (vec_info *vinfo,
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   /* Is vectorizable assignment?  */
@@ -6108,21 +5660,11 @@ vectorizable_assignment (vec_info *vinfo,
 	|| CONVERT_EXPR_CODE_P (code)))
     return false;
 
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
 
-  /* Multiple types in SLP are handled by creating the appropriate number of
-     vectorized stmts for each SLP node.  Hence, NCOPIES is always 1 in
-     case of SLP.  */
-  if (slp_node)
-    ncopies = 1;
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype);
-
-  gcc_assert (ncopies >= 1);
-
   slp_tree slp_op;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 0, &op, &slp_op,
+  if (!vect_is_simple_use (vinfo, slp_node, 0, &op, &slp_op,
 			   &dt[0], &vectype_in))
     {
       if (dump_enabled_p ())
@@ -6179,20 +5721,19 @@ vectorizable_assignment (vec_info *vinfo,
       return false;
     }
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
-      if (slp_node
-	  && !vect_maybe_update_slp_op_vectype (slp_op, vectype_in))
+      if (!vect_maybe_update_slp_op_vectype (slp_op, vectype_in))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
 			     "incompatible vector types for invariants\n");
 	  return false;
 	}
-      STMT_VINFO_TYPE (stmt_info) = assignment_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = assignment_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_assignment");
       if (!vect_nop_conversion_p (stmt_info))
-	vect_model_simple_cost (vinfo, ncopies, dt, ndts, slp_node, cost_vec);
+	vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
       return true;
     }
 
@@ -6204,7 +5745,7 @@ vectorizable_assignment (vec_info *vinfo,
   vec_dest = vect_create_destination_var (scalar_dest, vectype);
 
   /* Handle use.  */
-  vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, op, &vec_oprnds);
+  vect_get_vec_defs (vinfo, slp_node, op, &vec_oprnds);
 
   /* Arguments are ready. create the new vector stmt.  */
   FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
@@ -6216,13 +5757,8 @@ vectorizable_assignment (vec_info *vinfo,
       new_temp = make_ssa_name (vec_dest, new_stmt);
       gimple_assign_set_lhs (new_stmt, new_temp);
       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-      if (slp_node)
-	slp_node->push_vec_def (new_stmt);
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+      slp_node->push_vec_def (new_stmt);
     }
-  if (!slp_node)
-    *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
 
   vec_oprnds.release ();
   return true;
@@ -6264,7 +5800,7 @@ vect_supportable_shift (vec_info *vinfo, enum tree_code code, tree scalar_type)
 static bool
 vectorizable_shift (vec_info *vinfo,
 		    stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-		    gimple **vec_stmt, slp_tree slp_node,
+		    slp_tree slp_node,
 		    stmt_vector_for_cost *cost_vec)
 {
   tree vec_dest;
@@ -6272,7 +5808,6 @@ vectorizable_shift (vec_info *vinfo,
   tree op0, op1 = NULL;
   tree vec_oprnd1 = NULL_TREE;
   tree vectype;
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   enum tree_code code;
   machine_mode vec_mode;
   tree new_temp;
@@ -6280,12 +5815,10 @@ vectorizable_shift (vec_info *vinfo,
   int icode;
   machine_mode optab_op2_mode;
   enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
-  int ndts = 2;
   poly_uint64 nunits_in;
   poly_uint64 nunits_out;
   tree vectype_out;
   tree op1_vectype;
-  int ncopies;
   int i;
   vec<tree> vec_oprnds0 = vNULL;
   vec<tree> vec_oprnds1 = vNULL;
@@ -6300,7 +5833,7 @@ vectorizable_shift (vec_info *vinfo,
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
       && STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   /* Is STMT a vectorizable binary/unary operation?   */
@@ -6318,7 +5851,7 @@ vectorizable_shift (vec_info *vinfo,
     return false;
 
   scalar_dest = gimple_assign_lhs (stmt);
-  vectype_out = STMT_VINFO_VECTYPE (stmt_info);
+  vectype_out = SLP_TREE_VECTYPE (slp_node);
   if (!type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
     {
       if (dump_enabled_p ())
@@ -6328,7 +5861,7 @@ vectorizable_shift (vec_info *vinfo,
     }
 
   slp_tree slp_op0;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+  if (!vect_is_simple_use (vinfo, slp_node,
 			   0, &op0, &slp_op0, &dt[0], &vectype))
     {
       if (dump_enabled_p ())
@@ -6340,7 +5873,7 @@ vectorizable_shift (vec_info *vinfo,
      from the scalar type.  */
   if (!vectype)
     vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0), slp_node);
-  if (vec_stmt)
+  if (!cost_vec)
     gcc_assert (vectype);
   if (!vectype)
     {
@@ -6357,7 +5890,7 @@ vectorizable_shift (vec_info *vinfo,
 
   stmt_vec_info op1_def_stmt_info;
   slp_tree slp_op1;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1, &op1, &slp_op1,
+  if (!vect_is_simple_use (vinfo, slp_node, 1, &op1, &slp_op1,
 			   &dt[1], &op1_vectype, &op1_def_stmt_info))
     {
       if (dump_enabled_p ())
@@ -6366,23 +5899,13 @@ vectorizable_shift (vec_info *vinfo,
       return false;
     }
 
-  /* Multiple types in SLP are handled by creating the appropriate number of
-     vectorized stmts for each SLP node.  Hence, NCOPIES is always 1 in
-     case of SLP.  */
-  if (slp_node)
-    ncopies = 1;
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype);
-
-  gcc_assert (ncopies >= 1);
-
   /* Determine whether the shift amount is a vector, or scalar.  If the
      shift/rotate amount is a vector, use the vector/vector shift optabs.  */
 
   if ((dt[1] == vect_internal_def
        || dt[1] == vect_induction_def
        || dt[1] == vect_nested_cycle)
-      && (!slp_node || SLP_TREE_LANES (slp_node) == 1))
+      && SLP_TREE_LANES (slp_node) == 1)
     scalar_shift_arg = false;
   else if (dt[1] == vect_constant_def
 	   || dt[1] == vect_external_def
@@ -6391,29 +5914,26 @@ vectorizable_shift (vec_info *vinfo,
       /* In SLP, need to check whether the shift count is the same,
 	 in loops if it is a constant or invariant, it is always
 	 a scalar shift.  */
-      if (slp_node)
-	{
-	  vec<stmt_vec_info> stmts = SLP_TREE_SCALAR_STMTS (slp_node);
-	  stmt_vec_info slpstmt_info;
+      vec<stmt_vec_info> stmts = SLP_TREE_SCALAR_STMTS (slp_node);
+      stmt_vec_info slpstmt_info;
 
-	  FOR_EACH_VEC_ELT (stmts, k, slpstmt_info)
-	    if (slpstmt_info)
-	      {
-		gassign *slpstmt = as_a <gassign *> (slpstmt_info->stmt);
-		if (!operand_equal_p (gimple_assign_rhs2 (slpstmt), op1, 0))
-		  scalar_shift_arg = false;
-	      }
+      FOR_EACH_VEC_ELT (stmts, k, slpstmt_info)
+	if (slpstmt_info)
+	  {
+	    gassign *slpstmt = as_a <gassign *> (slpstmt_info->stmt);
+	    if (!operand_equal_p (gimple_assign_rhs2 (slpstmt), op1, 0))
+	      scalar_shift_arg = false;
+	  }
 
-	  /* For internal SLP defs we have to make sure we see scalar stmts
-	     for all vector elements.
-	     ???  For different vectors we could resort to a different
-	     scalar shift operand but code-generation below simply always
-	     takes the first.  */
-	  if (dt[1] == vect_internal_def
-	      && maybe_ne (nunits_out * SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node),
-			   stmts.length ()))
-	    scalar_shift_arg = false;
-	}
+      /* For internal SLP defs we have to make sure we see scalar stmts
+	 for all vector elements.
+	 ???  For different vectors we could resort to a different
+	 scalar shift operand but code-generation below simply always
+	 takes the first.  */
+      if (dt[1] == vect_internal_def
+	  && maybe_ne (nunits_out * SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node),
+		       stmts.length ()))
+	scalar_shift_arg = false;
 
       /* If the shift amount is computed by a pattern stmt we cannot
          use the scalar amount directly thus give up and use a vector
@@ -6447,8 +5967,7 @@ vectorizable_shift (vec_info *vinfo,
 			TYPE_VECTOR_SUBPARTS (vectype))
 	   || TYPE_MODE (op1_vectype) != TYPE_MODE (vectype));
       if (incompatible_op1_vectype_p
-	  && (!slp_node
-	      || SLP_TREE_DEF_TYPE (slp_op1) != vect_constant_def
+	  && (SLP_TREE_DEF_TYPE (slp_op1) != vect_constant_def
 	      || slp_op1->refcnt != 1))
 	{
 	  if (dump_enabled_p ())
@@ -6529,16 +6048,15 @@ vectorizable_shift (vec_info *vinfo,
   if (vect_emulated_vector_p (vectype))
     return false;
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
-      if (slp_node
-	  && (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
-	      || ((!scalar_shift_arg || dt[1] == vect_internal_def)
-		  && (!incompatible_op1_vectype_p
-		      || dt[1] == vect_constant_def)
-		  && !vect_maybe_update_slp_op_vectype
-			(slp_op1,
-			 incompatible_op1_vectype_p ? vectype : op1_vectype))))
+      if (!vect_maybe_update_slp_op_vectype (slp_op0, vectype)
+	  || ((!scalar_shift_arg || dt[1] == vect_internal_def)
+	      && (!incompatible_op1_vectype_p
+		  || dt[1] == vect_constant_def)
+	      && !vect_maybe_update_slp_op_vectype
+			 (slp_op1,
+			  incompatible_op1_vectype_p ? vectype : op1_vectype)))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -6546,24 +6064,20 @@ vectorizable_shift (vec_info *vinfo,
 	  return false;
 	}
       /* Now adjust the constant shift amount in place.  */
-      if (slp_node
-	  && incompatible_op1_vectype_p
+      if (incompatible_op1_vectype_p
 	  && dt[1] == vect_constant_def)
-	{
-	  for (unsigned i = 0;
-	       i < SLP_TREE_SCALAR_OPS (slp_op1).length (); ++i)
-	    {
-	      SLP_TREE_SCALAR_OPS (slp_op1)[i]
-		= fold_convert (TREE_TYPE (vectype),
-				SLP_TREE_SCALAR_OPS (slp_op1)[i]);
-	      gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
-			   == INTEGER_CST));
-	    }
-	}
-      STMT_VINFO_TYPE (stmt_info) = shift_vec_info_type;
+	for (unsigned i = 0;
+	     i < SLP_TREE_SCALAR_OPS (slp_op1).length (); ++i)
+	  {
+	    SLP_TREE_SCALAR_OPS (slp_op1)[i]
+	      = fold_convert (TREE_TYPE (vectype),
+			      SLP_TREE_SCALAR_OPS (slp_op1)[i]);
+	    gcc_assert ((TREE_CODE (SLP_TREE_SCALAR_OPS (slp_op1)[i])
+			 == INTEGER_CST));
+	  }
+      SLP_TREE_TYPE (slp_node) = shift_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_shift");
-      vect_model_simple_cost (vinfo, ncopies, dt,
-			      scalar_shift_arg ? 1 : ndts, slp_node, cost_vec);
+      vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
       return true;
     }
 
@@ -6573,15 +6087,6 @@ vectorizable_shift (vec_info *vinfo,
     dump_printf_loc (MSG_NOTE, vect_location,
                      "transform binary/unary operation.\n");
 
-  if (incompatible_op1_vectype_p && !slp_node)
-    {
-      gcc_assert (!scalar_shift_arg && was_scalar_shift_arg);
-      op1 = fold_convert (TREE_TYPE (vectype), op1);
-      if (dt[1] != vect_constant_def)
-	op1 = vect_init_vector (vinfo, stmt_info, op1,
-				TREE_TYPE (vectype), NULL);
-    }
-
   /* Handle def.  */
   vec_dest = vect_create_destination_var (scalar_dest, vectype);
 
@@ -6598,7 +6103,7 @@ vectorizable_shift (vec_info *vinfo,
 	    dump_printf_loc (MSG_NOTE, vect_location,
 			     "operand 1 using scalar mode.\n");
 	  vec_oprnd1 = op1;
-	  vec_oprnds1.create (slp_node ? slp_node->vec_stmts_size : ncopies);
+	  vec_oprnds1.create (SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node));
 	  vec_oprnds1.quick_push (vec_oprnd1);
 	      /* Store vec_oprnd1 for every vector stmt to be created.
 		 We check during the analysis that all the shift arguments
@@ -6606,11 +6111,11 @@ vectorizable_shift (vec_info *vinfo,
 		 TODO: Allow different constants for different vector
 		 stmts generated for an SLP instance.  */
 	  for (k = 0;
-	       k < (slp_node ? slp_node->vec_stmts_size - 1 : ncopies - 1); k++)
+	       k < SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node) - 1; k++)
 	    vec_oprnds1.quick_push (vec_oprnd1);
 	}
     }
-  else if (!scalar_shift_arg && slp_node && incompatible_op1_vectype_p)
+  else if (!scalar_shift_arg && incompatible_op1_vectype_p)
     {
       if (was_scalar_shift_arg)
 	{
@@ -6638,7 +6143,7 @@ vectorizable_shift (vec_info *vinfo,
   /* vec_oprnd1 is available if operand 1 should be of a scalar-type
      (a special case for certain kind of vector shifts); otherwise,
      operand 1 should be of a vector type (the usual case).  */
-  vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
+  vect_get_vec_defs (vinfo, slp_node,
 		     op0, &vec_oprnds0,
 		     vec_oprnd1 ? NULL_TREE : op1, &vec_oprnds1);
 
@@ -6666,15 +6171,9 @@ vectorizable_shift (vec_info *vinfo,
       new_temp = make_ssa_name (vec_dest, new_stmt);
       gimple_assign_set_lhs (new_stmt, new_temp);
       vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
-      if (slp_node)
-	slp_node->push_vec_def (new_stmt);
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+      slp_node->push_vec_def (new_stmt);
     }
 
-  if (!slp_node)
-    *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-
   vec_oprnds0.release ();
   vec_oprnds1.release ();
 
@@ -6692,7 +6191,7 @@ vectorizable_shift (vec_info *vinfo,
 static bool
 vectorizable_operation (vec_info *vinfo,
 			stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-			gimple **vec_stmt, slp_tree slp_node,
+			slp_tree slp_node,
 			stmt_vector_for_cost *cost_vec)
 {
   tree vec_dest;
@@ -6708,11 +6207,9 @@ vectorizable_operation (vec_info *vinfo,
   bool target_support_p;
   enum vect_def_type dt[3]
     = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
-  int ndts = 3;
   poly_uint64 nunits_in;
   poly_uint64 nunits_out;
   tree vectype_out;
-  int vec_num;
   int i;
   vec<tree> vec_oprnds0 = vNULL;
   vec<tree> vec_oprnds1 = vNULL;
@@ -6724,7 +6221,7 @@ vectorizable_operation (vec_info *vinfo,
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   /* Is STMT a vectorizable binary/unary operation?   */
@@ -6791,7 +6288,7 @@ vectorizable_operation (vec_info *vinfo,
     }
 
   slp_tree slp_op0;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+  if (!vect_is_simple_use (vinfo, slp_node,
 			   0, &op0, &slp_op0, &dt[0], &vectype))
     {
       if (dump_enabled_p ())
@@ -6825,7 +6322,7 @@ vectorizable_operation (vec_info *vinfo,
 	vectype = get_vectype_for_scalar_type (vinfo, TREE_TYPE (op0),
 					       slp_node);
     }
-  if (vec_stmt)
+  if (!cost_vec)
     gcc_assert (vectype);
   if (!vectype)
     {
@@ -6847,7 +6344,7 @@ vectorizable_operation (vec_info *vinfo,
   slp_tree slp_op1 = NULL, slp_op2 = NULL;
   if (op_type == binary_op || op_type == ternary_op)
     {
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+      if (!vect_is_simple_use (vinfo, slp_node,
 			       1, &op1, &slp_op1, &dt[1], &vectype2))
 	{
 	  if (dump_enabled_p ())
@@ -6865,7 +6362,7 @@ vectorizable_operation (vec_info *vinfo,
     }
   if (op_type == ternary_op)
     {
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+      if (!vect_is_simple_use (vinfo, slp_node,
 			       2, &op2, &slp_op2, &dt[2], &vectype3))
 	{
 	  if (dump_enabled_p ())
@@ -6884,7 +6381,7 @@ vectorizable_operation (vec_info *vinfo,
 
   /* Multiple types in SLP are handled by creating the appropriate number of
      vectorized stmts for each SLP node.  */
-  vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
+  auto vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
 
   /* Reject attempts to combine mask types with nonmask types, e.g. if
      we have an AND between a (nonmask) boolean loaded from memory and
@@ -6931,7 +6428,7 @@ vectorizable_operation (vec_info *vinfo,
 	       || !target_support_p)
 	      && maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD))
 	  /* Check only during analysis.  */
-	  || (!vec_stmt && !vect_can_vectorize_without_simd_p (code)))
+	  || (cost_vec && !vect_can_vectorize_without_simd_p (code)))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf (MSG_NOTE, "using word mode not possible.\n");
@@ -6960,7 +6457,7 @@ vectorizable_operation (vec_info *vinfo,
   bool mask_out_inactive = ((!is_invariant && gimple_could_trap_p (stmt))
 			    || reduc_idx >= 0);
 
-  if (!vec_stmt) /* transformation not required.  */
+  if (cost_vec) /* transformation not required.  */
     {
       if (loop_vinfo
 	  && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)
@@ -6997,9 +6494,9 @@ vectorizable_operation (vec_info *vinfo,
 	  return false;
 	}
 
-      STMT_VINFO_TYPE (stmt_info) = op_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = op_vec_info_type;
       DUMP_VECT_SCOPE ("vectorizable_operation");
-      vect_model_simple_cost (vinfo, 1, dt, ndts, slp_node, cost_vec);
+      vect_model_simple_cost (vinfo, 1, slp_node, cost_vec);
       if (using_emulated_vectors_p)
 	{
 	  /* The above vect_model_simple_cost call handles constants
@@ -7059,7 +6556,7 @@ vectorizable_operation (vec_info *vinfo,
   else
     vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
 
-  vect_get_vec_defs (vinfo, stmt_info, slp_node, 1,
+  vect_get_vec_defs (vinfo, slp_node,
 		     op0, &vec_oprnds0, op1, &vec_oprnds1, op2, &vec_oprnds2);
   /* Arguments are ready.  Create the new vector stmt.  */
   FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
@@ -7236,7 +6733,7 @@ vectorizable_operation (vec_info *vinfo,
 	      && code == BIT_AND_EXPR
 	      && VECTOR_BOOLEAN_TYPE_P (vectype))
 	    {
-	      if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, 1 }))
+	      if (loop_vinfo->scalar_cond_masked_set.contains ({ op0, vec_num }))
 		{
 		  mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
 					     vec_num, vectype, i);
@@ -7245,7 +6742,7 @@ vectorizable_operation (vec_info *vinfo,
 					   vop0, gsi);
 		}
 
-	      if (loop_vinfo->scalar_cond_masked_set.contains ({ op1, 1 }))
+	      if (loop_vinfo->scalar_cond_masked_set.contains ({ op1, vec_num }))
 		{
 		  mask = vect_get_loop_mask (loop_vinfo, gsi, masks,
 					     vec_num, vectype, i);
@@ -7538,7 +7035,8 @@ scan_store_can_perm_p (tree vectype, tree init,
 
 static bool
 check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
-		  enum vect_def_type rhs_dt, slp_tree slp_node, tree mask,
+		  enum vect_def_type rhs_dt, slp_tree slp_node,
+		  slp_tree mask_node,
 		  vect_memory_access_type memory_access_type)
 {
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
@@ -7546,13 +7044,14 @@ check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
   tree ref_type;
 
   gcc_assert (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1);
-  if ((slp_node && SLP_TREE_LANES (slp_node) > 1)
-      || mask
+  if (SLP_TREE_LANES (slp_node) > 1
+      || mask_node
       || memory_access_type != VMAT_CONTIGUOUS
       || TREE_CODE (DR_BASE_ADDRESS (dr_info->dr)) != ADDR_EXPR
       || !VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (dr_info->dr), 0))
       || loop_vinfo == NULL
       || LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
+      || LOOP_VINFO_EPILOGUE_P (loop_vinfo)
       || STMT_VINFO_GROUPED_ACCESS (stmt_info)
       || !integer_zerop (get_dr_vinfo_offset (vinfo, dr_info))
       || !integer_zerop (DR_INIT (dr_info->dr))
@@ -7942,17 +7441,16 @@ check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
 
 static bool
 vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
-			 slp_tree slp_node, gimple_stmt_iterator *gsi,
-			 gimple **vec_stmt, int ncopies)
+			 slp_tree slp_node, gimple_stmt_iterator *gsi)
 {
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
   tree ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
 
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location,
-		     "transform scan store. ncopies = %d\n", ncopies);
+		     "transform scan store.\n");
 
   gimple *stmt = STMT_VINFO_STMT (stmt_info);
   tree rhs = gimple_assign_rhs1 (stmt);
@@ -8063,8 +7561,8 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
       /* We want to lookup the vector operands of the reduction, not those
 	 of the store - for SLP we have to use the proper SLP node for the
 	 lookup, which should be the single child of the scan store.  */
-      vect_get_vec_defs (vinfo, stmt_info, SLP_TREE_CHILDREN (slp_node)[0],
-			 ncopies, rhs1, &vec_oprnds2, rhs2, &vec_oprnds3);
+      vect_get_vec_defs (vinfo, SLP_TREE_CHILDREN (slp_node)[0],
+			 rhs1, &vec_oprnds2, rhs2, &vec_oprnds3);
       /* ???  For SLP we do not key the def on 'rhs1' or 'rhs2' but get
 	 them in SLP child order.  So we have to swap here with logic
 	 similar to above.  */
@@ -8078,7 +7576,7 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 	  std::swap (vec_oprnds2[i], vec_oprnds3[i]);;
     }
   else
-    vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
+    vect_get_vec_defs (vinfo, slp_node,
 		       rhs2, &vec_oprnds3);
   for (unsigned j = 0; j < vec_oprnds3.length (); j++)
     {
@@ -8099,11 +7597,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 	  vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr));
 	  gimple *g = gimple_build_assign (vec_oprnd2, data_ref);
 	  vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-	  if (! slp_node)
-	    {
-	      STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
-	      *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-	    }
 	}
 
       tree v = vec_oprnd2;
@@ -8117,11 +7610,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 					   ? zero_vec : vec_oprnd1, v,
 					   perms[i]);
 	  vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-	  if (! slp_node)
-	    {
-	      STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
-	      *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-	    }
 
 	  if (zero_vec && use_whole_vector[i] == scan_store_kind_lshift_cond)
 	    {
@@ -8138,8 +7626,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 				       new_temp, vec_oprnd1);
 	      vect_finish_stmt_generation (vinfo, stmt_info,
 							   g, gsi);
-	      if (! slp_node)
-		STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
 	      new_temp = new_temp2;
 	    }
 
@@ -8157,8 +7643,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 	  tree new_temp2 = make_ssa_name (vectype);
 	  g = gimple_build_assign (new_temp2, code, v, new_temp);
 	  vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-	  if (! slp_node)
-	    STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
 
 	  v = new_temp2;
 	}
@@ -8166,8 +7650,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
       tree new_temp = make_ssa_name (vectype);
       gimple *g = gimple_build_assign (new_temp, code, orig, v);
       vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-      if (! slp_node)
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
 
       tree last_perm_arg = new_temp;
       /* For exclusive scan, new_temp computed above is the exclusive scan
@@ -8178,16 +7660,12 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 	  last_perm_arg = make_ssa_name (vectype);
 	  g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2);
 	  vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-	  if (! slp_node)
-	    STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
 	}
 
       orig = make_ssa_name (vectype);
       g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg,
 			       last_perm_arg, perms[units_log2]);
       vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-      if (! slp_node)
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
 
       if (!inscan_var_store)
 	{
@@ -8197,8 +7675,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 	  vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
 	  g = gimple_build_assign (data_ref, new_temp);
 	  vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-	  if (! slp_node)
-	    STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
 	}
     }
 
@@ -8214,8 +7690,6 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 	vect_copy_ref_info (data_ref, DR_REF (dr_info->dr));
 	gimple *g = gimple_build_assign (data_ref, orig);
 	vect_finish_stmt_generation (vinfo, stmt_info, g, gsi);
-	if (! slp_node)
-	  STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g);
       }
   return true;
 }
@@ -8232,7 +7706,7 @@ vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info,
 static bool
 vectorizable_store (vec_info *vinfo,
 		    stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-		    gimple **vec_stmt, slp_tree slp_node,
+		    slp_tree slp_node,
 		    stmt_vector_for_cost *cost_vec)
 {
   tree data_ref;
@@ -8263,12 +7737,12 @@ vectorizable_store (vec_info *vinfo,
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   /* Is vectorizable store? */
 
-  tree mask = NULL_TREE, mask_vectype = NULL_TREE;
+  tree mask_vectype = NULL_TREE;
   slp_tree mask_node = NULL;
   if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
     {
@@ -8300,8 +7774,8 @@ vectorizable_store (vec_info *vinfo,
 	mask_index = vect_slp_child_index_for_operand
 		    (call, mask_index, STMT_VINFO_GATHER_SCATTER_P (stmt_info));
       if (mask_index >= 0
-	  && !vect_check_scalar_mask (vinfo, stmt_info, slp_node, mask_index,
-				      &mask, &mask_node, &mask_dt,
+	  && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
+				      &mask_node, &mask_dt,
 				      &mask_vectype))
 	return false;
     }
@@ -8332,10 +7806,9 @@ vectorizable_store (vec_info *vinfo,
       return false;
     }
 
-  tree op;
   slp_tree op_node;
   if (!vect_check_store_rhs (vinfo, stmt_info, slp_node,
-			     &op, &op_node, &rhs_dt, &rhs_vectype, &vls_type))
+			     &op_node, &rhs_dt, &rhs_vectype, &vls_type))
     return false;
 
   elem_type = TREE_TYPE (vectype);
@@ -8349,8 +7822,8 @@ vectorizable_store (vec_info *vinfo,
   int misalignment;
   poly_int64 poffset;
   internal_fn lanes_ifn;
-  if (!get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask, vls_type,
-			    1, &memory_access_type, &poffset,
+  if (!get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
+			    vls_type, &memory_access_type, &poffset,
 			    &alignment_support_scheme, &misalignment, &gs_info,
 			    &lanes_ifn))
     return false;
@@ -8364,7 +7837,7 @@ vectorizable_store (vec_info *vinfo,
       return false;
     }
 
-  if (mask)
+  if (mask_node)
     {
       if (memory_access_type == VMAT_CONTIGUOUS)
 	{
@@ -8375,7 +7848,8 @@ vectorizable_store (vec_info *vinfo,
 	}
       else if (memory_access_type != VMAT_LOAD_STORE_LANES
 	       && (memory_access_type != VMAT_GATHER_SCATTER
-		   || (gs_info.decl && !VECTOR_BOOLEAN_TYPE_P (mask_vectype))))
+		   || (GATHER_SCATTER_LEGACY_P (gs_info)
+		       && !VECTOR_BOOLEAN_TYPE_P (mask_vectype))))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -8383,8 +7857,7 @@ vectorizable_store (vec_info *vinfo,
 	  return false;
 	}
       else if (memory_access_type == VMAT_GATHER_SCATTER
-	       && gs_info.ifn == IFN_LAST
-	       && !gs_info.decl)
+	       && GATHER_SCATTER_EMULATED_P (gs_info))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -8416,17 +7889,16 @@ vectorizable_store (vec_info *vinfo,
       group_size = vec_num = 1;
     }
 
-  if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && !vec_stmt)
+  if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) > 1 && cost_vec)
     {
-      if (!check_scan_store (vinfo, stmt_info, vectype, rhs_dt, slp_node, mask,
-			     memory_access_type))
+      if (!check_scan_store (vinfo, stmt_info, vectype, rhs_dt, slp_node,
+			     mask_node, memory_access_type))
 	return false;
     }
 
-  bool costing_p = !vec_stmt;
+  bool costing_p = cost_vec;
   if (costing_p) /* transformation not required.  */
     {
-      STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type;
       SLP_TREE_MEMORY_ACCESS_TYPE (slp_node) = memory_access_type;
 
       if (loop_vinfo
@@ -8434,10 +7906,10 @@ vectorizable_store (vec_info *vinfo,
 	check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
 					      vls_type, group_size,
 					      memory_access_type, &gs_info,
-					      mask);
+					      mask_node);
 
       if (!vect_maybe_update_slp_op_vectype (op_node, vectype)
-	  || (mask
+	  || (mask_node
 	      && !vect_maybe_update_slp_op_vectype (mask_node,
 						    mask_vectype)))
 	{
@@ -8449,16 +7921,15 @@ vectorizable_store (vec_info *vinfo,
 
       if (dump_enabled_p ()
 	  && memory_access_type != VMAT_ELEMENTWISE
-	  && memory_access_type != VMAT_GATHER_SCATTER
 	  && memory_access_type != VMAT_STRIDED_SLP
 	  && memory_access_type != VMAT_INVARIANT
 	  && alignment_support_scheme != dr_aligned)
 	dump_printf_loc (MSG_NOTE, vect_location,
 			 "Vectorizing an unaligned access.\n");
 
-      STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = store_vec_info_type;
     }
-  gcc_assert (memory_access_type == SLP_TREE_MEMORY_ACCESS_TYPE (stmt_info));
+  gcc_assert (memory_access_type == SLP_TREE_MEMORY_ACCESS_TYPE (slp_node));
 
   /* Transform.  */
 
@@ -8486,8 +7957,7 @@ vectorizable_store (vec_info *vinfo,
 
 	  return true;
 	}
-      return vectorizable_scan_store (vinfo, stmt_info, slp_node,
-				      gsi, vec_stmt, 1);
+      return vectorizable_scan_store (vinfo, stmt_info, slp_node, gsi);
     }
 
   /* FORNOW */
@@ -8503,7 +7973,6 @@ vectorizable_store (vec_info *vinfo,
   gcc_assert (!STMT_VINFO_GROUPED_ACCESS (first_stmt_info)
 	      || (DR_GROUP_FIRST_ELEMENT (first_stmt_info) == first_stmt_info));
   first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
-  op = vect_get_store_rhs (first_stmt_info);
 
   ref_type = get_group_alias_ptr_type (first_stmt_info);
 
@@ -8683,15 +8152,13 @@ vectorizable_store (vec_info *vinfo,
 	}
 
       alias_off = build_int_cst (ref_type, 0);
-      stmt_vec_info next_stmt_info = first_stmt_info;
       auto_vec<tree> vec_oprnds;
       /* For costing some adjacent vector stores, we'd like to cost with
 	 the total number of them once instead of cost each one by one. */
       unsigned int n_adjacent_stores = 0;
       running_off = offvar;
       if (!costing_p)
-	vect_get_vec_defs (vinfo, next_stmt_info, slp_node, ncopies, op,
-			   &vec_oprnds);
+	vect_get_slp_defs (op_node, &vec_oprnds);
       unsigned int group_el = 0;
       unsigned HOST_WIDE_INT elsz
 	= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
@@ -8766,7 +8233,7 @@ vectorizable_store (vec_info *vinfo,
 	      else
 		inside_cost
 		  += record_stmt_cost (cost_vec, n_adjacent_stores,
-				       scalar_store, stmt_info, 0, vect_body);
+				       scalar_store, slp_node, 0, vect_body);
 	      /* Only need vector extracting when there are more
 		 than one stores.  */
 	      if (nstores > 1)
@@ -8806,7 +8273,7 @@ vectorizable_store (vec_info *vinfo,
      realignment.  vect_supportable_dr_alignment always returns either
      dr_aligned or dr_unaligned_supported for masked operations.  */
   gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
-	       && !mask
+	       && !mask_node
 	       && !loop_masks)
 	      || alignment_support_scheme == dr_aligned
 	      || alignment_support_scheme == dr_unaligned_supported);
@@ -8826,7 +8293,8 @@ vectorizable_store (vec_info *vinfo,
     {
       aggr_type = elem_type;
       if (!costing_p)
-	vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
+	vect_get_strided_load_store_ops (stmt_info, vectype, loop_vinfo,
+					 gsi, &gs_info,
 					 &bump, &vec_offset, loop_lens);
     }
   else
@@ -8840,7 +8308,7 @@ vectorizable_store (vec_info *vinfo,
 					    memory_access_type, loop_lens);
     }
 
-  if (mask && !costing_p)
+  if (mask_node && !costing_p)
     LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
 
   /* In case the vectorization factor (VF) is bigger than the number
@@ -8848,39 +8316,6 @@ vectorizable_store (vec_info *vinfo,
      more than one vector stmt - i.e - we need to "unroll" the
      vector stmt by a factor VF/nunits.  */
 
-  /* In case of interleaving (non-unit grouped access):
-
-        S1:  &base + 2 = x2
-        S2:  &base = x0
-        S3:  &base + 1 = x1
-        S4:  &base + 3 = x3
-
-     We create vectorized stores starting from base address (the access of the
-     first stmt in the chain (S2 in the above example), when the last store stmt
-     of the chain (S4) is reached:
-
-        VS1: &base = vx2
-	VS2: &base + vec_size*1 = vx0
-	VS3: &base + vec_size*2 = vx1
-	VS4: &base + vec_size*3 = vx3
-
-     Then permutation statements are generated:
-
-	VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
-	VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
-	...
-
-     And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
-     (the order of the data-refs in the output of vect_permute_store_chain
-     corresponds to the order of scalar stmts in the interleaving chain - see
-     the documentation of vect_permute_store_chain()).
-
-     In case of both multiple types and interleaving, above vector stores and
-     permutation stmts are created for every copy.  The result vector stmts are
-     put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
-     STMT_VINFO_RELATED_STMT for the next copies.
-  */
-
   auto_vec<tree> dr_chain (group_size);
   auto_vec<tree> vec_masks;
   tree vec_mask = NULL;
@@ -8913,7 +8348,7 @@ vectorizable_store (vec_info *vinfo,
 	    {
 	      if (!costing_p)
 		{
-		  if (mask)
+		  if (mask_node)
 		    {
 		      vect_get_slp_defs (mask_node, &vec_masks);
 		      vec_mask = vec_masks[0];
@@ -8927,7 +8362,7 @@ vectorizable_store (vec_info *vinfo,
 	  else if (!costing_p)
 	    {
 	      gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
-	      if (mask)
+	      if (mask_node)
 		vec_mask = vec_masks[j];
 	      dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
 					     stmt_info, bump);
@@ -9062,12 +8497,11 @@ vectorizable_store (vec_info *vinfo,
 		     DR_CHAIN is of size 1.  */
 		  gcc_assert (group_size == 1);
 		  vect_get_slp_defs (op_node, gvec_oprnds[0]);
-		  if (mask)
+		  if (mask_node)
 		    vect_get_slp_defs (mask_node, &vec_masks);
 
 		  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
-		    vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info,
-						 slp_node, &gs_info,
+		    vect_get_gather_scatter_ops (loop, slp_node, &gs_info,
 						 &dataref_ptr, &vec_offsets);
 		  else
 		    dataref_ptr
@@ -9089,7 +8523,7 @@ vectorizable_store (vec_info *vinfo,
 	  if (!costing_p)
 	    {
 	      vec_oprnd = (*gvec_oprnds[0])[j];
-	      if (mask)
+	      if (mask_node)
 		vec_mask = vec_masks[j];
 	      /* We should have catched mismatched types earlier.  */
 	      gcc_assert (useless_type_conversion_p (vectype,
@@ -9110,7 +8544,7 @@ vectorizable_store (vec_info *vinfo,
 					       final_mask, vec_mask, gsi);
 	    }
 
-	  if (gs_info.ifn != IFN_LAST)
+	  if (GATHER_SCATTER_IFN_P (gs_info))
 	    {
 	      if (costing_p)
 		{
@@ -9150,30 +8584,37 @@ vectorizable_store (vec_info *vinfo,
 		{
 		  if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
 		    call = gimple_build_call_internal (
-			    IFN_MASK_LEN_SCATTER_STORE, 7, dataref_ptr,
+			    IFN_MASK_LEN_SCATTER_STORE, 8, dataref_ptr,
+			    gs_info.alias_ptr,
 			    vec_offset, scale, vec_oprnd, final_mask, final_len,
 			    bias);
 		  else
 		    /* Non-vector offset indicates that prefer to take
 		       MASK_LEN_STRIDED_STORE instead of the
-		       IFN_MASK_SCATTER_STORE with direct stride arg.  */
+		       IFN_MASK_SCATTER_STORE with direct stride arg.
+		       Similar to the gather case we have checked the
+		       alignment for a scatter already and assume
+		       that the strided store has the same requirements.  */
 		    call = gimple_build_call_internal (
 			    IFN_MASK_LEN_STRIDED_STORE, 6, dataref_ptr,
 			    vec_offset, vec_oprnd, final_mask, final_len, bias);
 		}
 	      else if (final_mask)
 		call = gimple_build_call_internal
-			     (IFN_MASK_SCATTER_STORE, 5, dataref_ptr,
+			     (IFN_MASK_SCATTER_STORE, 6, dataref_ptr,
+			      gs_info.alias_ptr,
 			      vec_offset, scale, vec_oprnd, final_mask);
 	      else
-		call = gimple_build_call_internal (IFN_SCATTER_STORE, 4,
-						   dataref_ptr, vec_offset,
+		call = gimple_build_call_internal (IFN_SCATTER_STORE, 5,
+						   dataref_ptr,
+						   gs_info.alias_ptr,
+						   vec_offset,
 						   scale, vec_oprnd);
 	      gimple_call_set_nothrow (call, true);
 	      vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
 	      new_stmt = call;
 	    }
-	   else if (gs_info.decl)
+	  else if (GATHER_SCATTER_LEGACY_P (gs_info))
 	    {
 	      /* The builtin decls path for scatter is legacy, x86 only.  */
 	      gcc_assert (nunits.is_constant ()
@@ -9370,7 +8811,6 @@ vectorizable_store (vec_info *vinfo,
 
   gcc_assert (memory_access_type == VMAT_CONTIGUOUS
 	      || memory_access_type == VMAT_CONTIGUOUS_DOWN
-	      || memory_access_type == VMAT_CONTIGUOUS_PERMUTE
 	      || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
 
   unsigned inside_cost = 0, prologue_cost = 0;
@@ -9383,42 +8823,11 @@ vectorizable_store (vec_info *vinfo,
   if (!costing_p)
     {
       /* Get vectorized arguments for SLP_NODE.  */
-      vect_get_vec_defs (vinfo, stmt_info, slp_node, 1, op,
-			 &vec_oprnds, mask, &vec_masks);
+      vect_get_slp_defs (op_node, &vec_oprnds);
       vec_oprnd = vec_oprnds[0];
-      if (mask)
-	vec_mask = vec_masks[0];
-    }
-  else
-    {
-      /* For interleaved stores we collect vectorized defs for all the
-	 stores in the group in DR_CHAIN. DR_CHAIN is then used as an
-	 input to vect_permute_store_chain().
-
-	 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
-	 is of size 1.  */
-      stmt_vec_info next_stmt_info = first_stmt_info;
-      for (i = 0; i < group_size; i++)
-	{
-	  /* Since gaps are not supported for interleaved stores,
-	     DR_GROUP_SIZE is the exact number of stmts in the chain.
-	     Therefore, NEXT_STMT_INFO can't be NULL_TREE.  In case
-	     that there is no interleaving, DR_GROUP_SIZE is 1,
-	     and only one iteration of the loop will be executed.  */
-	  op = vect_get_store_rhs (next_stmt_info);
-	  if (!costing_p)
-	    {
-	      vect_get_vec_defs_for_operand (vinfo, next_stmt_info,
-					     1, op, gvec_oprnds[i]);
-	      vec_oprnd = (*gvec_oprnds[i])[0];
-	      dr_chain.quick_push (vec_oprnd);
-	    }
-	  next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
-	}
-      if (mask && !costing_p)
+      if (mask_node)
 	{
-	  vect_get_vec_defs_for_operand (vinfo, stmt_info, 1,
-					 mask, &vec_masks, mask_vectype);
+	  vect_get_slp_defs (mask_node, &vec_masks);
 	  vec_mask = vec_masks[0];
 	}
     }
@@ -9448,25 +8857,7 @@ vectorizable_store (vec_info *vinfo,
 					    simd_lane_access_p, bump);
 
   new_stmt = NULL;
-  if (grouped_store)
-    {
-      /* Permute.  */
-      gcc_assert (memory_access_type == VMAT_CONTIGUOUS_PERMUTE);
-      if (costing_p)
-	{
-	  int group_size = DR_GROUP_SIZE (first_stmt_info);
-	  int nstmts = ceil_log2 (group_size) * group_size;
-	  inside_cost += record_stmt_cost (cost_vec, nstmts, vec_perm,
-					   slp_node, 0, vect_body);
-	  if (dump_enabled_p ())
-	    dump_printf_loc (MSG_NOTE, vect_location, "vect_model_store_cost: "
-			     "strided group_size = %d .\n", group_size);
-	}
-      else
-	vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info,
-				  gsi, &result_chain);
-    }
-
+  gcc_assert (!grouped_store);
   for (i = 0; i < vec_num; i++)
     {
       if (!costing_p)
@@ -9480,10 +8871,7 @@ vectorizable_store (vec_info *vinfo,
 	  else
 	    {
 	      tree perm_mask = perm_mask_for_reverse (vectype);
-	      tree perm_dest
-		= vect_create_destination_var (vect_get_store_rhs (stmt_info),
-					       vectype);
-	      tree new_temp = make_ssa_name (perm_dest);
+	      tree new_temp = make_ssa_name (vectype);
 
 	      /* Generate the permute statement.  */
 	      gimple *perm_stmt
@@ -9819,7 +9207,7 @@ hoist_defs_of_uses (gimple *stmt, class loop *loop, bool hoist_p)
 static bool
 vectorizable_load (vec_info *vinfo,
 		   stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-		   gimple **vec_stmt, slp_tree slp_node,
+		   slp_tree slp_node,
 		   stmt_vector_for_cost *cost_vec)
 {
   tree scalar_dest;
@@ -9863,18 +9251,18 @@ vectorizable_load (vec_info *vinfo,
     return false;
 
   if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
-      && ! vec_stmt)
+      && cost_vec)
     return false;
 
   if (!STMT_VINFO_DATA_REF (stmt_info))
     return false;
 
-  tree mask = NULL_TREE, mask_vectype = NULL_TREE;
+  tree mask_vectype = NULL_TREE;
   tree els = NULL_TREE; tree els_vectype = NULL_TREE;
 
   int mask_index = -1;
   int els_index = -1;
-  slp_tree slp_op = NULL;
+  slp_tree mask_node = NULL;
   slp_tree els_op = NULL;
   if (gassign *assign = dyn_cast <gassign *> (stmt_info->stmt))
     {
@@ -9912,8 +9300,8 @@ vectorizable_load (vec_info *vinfo,
 	mask_index = vect_slp_child_index_for_operand
 		    (call, mask_index, STMT_VINFO_GATHER_SCATTER_P (stmt_info));
       if (mask_index >= 0
-	  && !vect_check_scalar_mask (vinfo, stmt_info, slp_node, mask_index,
-				      &mask, &slp_op, &mask_dt, &mask_vectype))
+	  && !vect_check_scalar_mask (vinfo, slp_node, mask_index,
+				      &mask_node, &mask_dt, &mask_vectype))
 	return false;
 
       els_index = internal_fn_else_index (ifn);
@@ -9921,12 +9309,12 @@ vectorizable_load (vec_info *vinfo,
 	els_index = vect_slp_child_index_for_operand
 	  (call, els_index, STMT_VINFO_GATHER_SCATTER_P (stmt_info));
       if (els_index >= 0
-	  && !vect_is_simple_use (vinfo, stmt_info, slp_node, els_index,
+	  && !vect_is_simple_use (vinfo, slp_node, els_index,
 				  &els, &els_op, &els_dt, &els_vectype))
 	return false;
     }
 
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
 
   if (loop_vinfo)
@@ -9996,8 +9384,8 @@ vectorizable_load (vec_info *vinfo,
   auto_vec<int> elsvals;
   int maskload_elsval = 0;
   bool need_zeroing = false;
-  if (!get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask, VLS_LOAD,
-			    1, &memory_access_type, &poffset,
+  if (!get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node,
+			    VLS_LOAD, &memory_access_type, &poffset,
 			    &alignment_support_scheme, &misalignment, &gs_info,
 			    &lanes_ifn, &elsvals))
     return false;
@@ -10011,7 +9399,7 @@ vectorizable_load (vec_info *vinfo,
     = TYPE_PRECISION (scalar_type) < GET_MODE_PRECISION (GET_MODE_INNER (mode));
 
   /* ???  The following checks should really be part of
-     get_group_load_store_type.  */
+     get_load_store_type.  */
   if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
       && !((memory_access_type == VMAT_ELEMENTWISE
 	    || memory_access_type == VMAT_GATHER_SCATTER)
@@ -10064,7 +9452,7 @@ vectorizable_load (vec_info *vinfo,
       return false;
     }
 
-  if (mask)
+  if (mask_node)
     {
       if (memory_access_type == VMAT_CONTIGUOUS)
 	{
@@ -10084,8 +9472,7 @@ vectorizable_load (vec_info *vinfo,
 	  return false;
 	}
       else if (memory_access_type == VMAT_GATHER_SCATTER
-	       && gs_info.ifn == IFN_LAST
-	       && !gs_info.decl)
+	       && GATHER_SCATTER_EMULATED_P (gs_info))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -10102,12 +9489,12 @@ vectorizable_load (vec_info *vinfo,
 	}
     }
 
-  bool costing_p = !vec_stmt;
+  bool costing_p = cost_vec;
 
   if (costing_p) /* transformation not required.  */
     {
-      if (mask
-	  && !vect_maybe_update_slp_op_vectype (slp_op,
+      if (mask_node
+	  && !vect_maybe_update_slp_op_vectype (mask_node,
 						mask_vectype))
 	{
 	  if (dump_enabled_p ())
@@ -10123,7 +9510,7 @@ vectorizable_load (vec_info *vinfo,
 	check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
 					      VLS_LOAD, group_size,
 					      memory_access_type, &gs_info,
-					      mask, &elsvals);
+					      mask_node, &elsvals);
 
       if (dump_enabled_p ()
 	  && memory_access_type != VMAT_ELEMENTWISE
@@ -10137,7 +9524,7 @@ vectorizable_load (vec_info *vinfo,
       if (memory_access_type == VMAT_LOAD_STORE_LANES)
 	vinfo->any_known_not_updated_vssa = true;
 
-      STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
+      SLP_TREE_TYPE (slp_node) = load_vec_info_type;
     }
   else
     {
@@ -10147,7 +9534,7 @@ vectorizable_load (vec_info *vinfo,
 	check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
 					      VLS_LOAD, group_size,
 					      memory_access_type, &gs_info,
-					      mask, &elsvals);
+					      mask_node, &elsvals);
     }
 
   /* If the type needs padding we must zero inactive elements.
@@ -10182,7 +9569,7 @@ vectorizable_load (vec_info *vinfo,
 
   if (memory_access_type == VMAT_INVARIANT)
     {
-      gcc_assert (!grouped_load && !mask && !bb_vinfo);
+      gcc_assert (!grouped_load && !mask_node && !bb_vinfo);
       /* If we have versioned for aliasing or the loop doesn't
 	 have any data dependencies that would preclude this,
 	 then we are sure this is a loop invariant load and
@@ -10635,7 +10022,6 @@ vectorizable_load (vec_info *vinfo,
       vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
     }
 
-  gcc_assert (alignment_support_scheme);
   vec_loop_masks *loop_masks
     = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
        ? &LOOP_VINFO_MASKS (loop_vinfo)
@@ -10655,10 +10041,12 @@ vectorizable_load (vec_info *vinfo,
 
   /* Targets with store-lane instructions must not require explicit
      realignment.  vect_supportable_dr_alignment always returns either
-     dr_aligned or dr_unaligned_supported for masked operations.  */
+     dr_aligned or dr_unaligned_supported for (non-length) masked
+     operations.  */
   gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
-	       && !mask
+	       && !mask_node
 	       && !loop_masks)
+	      || memory_access_type == VMAT_GATHER_SCATTER
 	      || alignment_support_scheme == dr_aligned
 	      || alignment_support_scheme == dr_unaligned_supported);
 
@@ -10694,39 +10082,6 @@ vectorizable_load (vec_info *vinfo,
         S2:     z = x + 1       -               -
   */
 
-  /* In case of interleaving (non-unit grouped access):
-
-     S1:  x2 = &base + 2
-     S2:  x0 = &base
-     S3:  x1 = &base + 1
-     S4:  x3 = &base + 3
-
-     Vectorized loads are created in the order of memory accesses
-     starting from the access of the first stmt of the chain:
-
-     VS1: vx0 = &base
-     VS2: vx1 = &base + vec_size*1
-     VS3: vx3 = &base + vec_size*2
-     VS4: vx4 = &base + vec_size*3
-
-     Then permutation statements are generated:
-
-     VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
-     VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
-       ...
-
-     And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
-     (the order of the data-refs in the output of vect_permute_load_chain
-     corresponds to the order of scalar stmts in the interleaving chain - see
-     the documentation of vect_permute_load_chain()).
-     The generation of permutation stmts and recording them in
-     STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
-
-     In case of both multiple types and interleaving, the vector loads and
-     permutation stmts above are created for every copy.  The result vector
-     stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
-     corresponding STMT_VINFO_RELATED_STMT for the next copies.  */
-
   /* If the data reference is aligned (dr_aligned) or potentially unaligned
      on a target that supports unaligned accesses (dr_unaligned_supported)
      we generate the following code:
@@ -10808,32 +10163,10 @@ vectorizable_load (vec_info *vinfo,
 
   tree bump;
   tree vec_offset = NULL_TREE;
-  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
-    {
-      aggr_type = NULL_TREE;
-      bump = NULL_TREE;
-    }
-  else if (memory_access_type == VMAT_GATHER_SCATTER)
-    {
-      aggr_type = elem_type;
-      if (!costing_p)
-	vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
-					 &bump, &vec_offset, loop_lens);
-    }
-  else
-    {
-      if (memory_access_type == VMAT_LOAD_STORE_LANES)
-	aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
-      else
-	aggr_type = vectype;
-      if (!costing_p)
-	bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type,
-					    memory_access_type, loop_lens);
-    }
 
   auto_vec<tree> vec_offsets;
   auto_vec<tree> vec_masks;
-  if (mask && !costing_p)
+  if (mask_node && !costing_p)
     vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node)[mask_index],
 		       &vec_masks);
 
@@ -10844,6 +10177,11 @@ vectorizable_load (vec_info *vinfo,
       gcc_assert (alignment_support_scheme == dr_aligned
 		  || alignment_support_scheme == dr_unaligned_supported);
 
+      aggr_type = build_array_type_nelts (elem_type, group_size * nunits);
+      if (!costing_p)
+	bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type,
+					    memory_access_type, loop_lens);
+
       unsigned int inside_cost = 0, prologue_cost = 0;
       /* For costing some adjacent vector loads, we'd like to cost with
 	 the total number of them once instead of cost each one by one. */
@@ -10896,7 +10234,7 @@ vectorizable_load (vec_info *vinfo,
 	      dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
 					     stmt_info, bump);
 	    }
-	  if (mask)
+	  if (mask_node)
 	    vec_mask = vec_masks[j];
 
 	  tree vec_array = create_vector_array (vectype, group_size);
@@ -11003,25 +10341,33 @@ vectorizable_load (vec_info *vinfo,
 
   if (memory_access_type == VMAT_GATHER_SCATTER)
     {
-      gcc_assert (alignment_support_scheme == dr_aligned
-		  || alignment_support_scheme == dr_unaligned_supported);
       gcc_assert (!grouped_load && !slp_perm);
 
-      unsigned int inside_cost = 0, prologue_cost = 0;
-
       /* 1. Create the vector or array pointer update chain.  */
-      if (!costing_p)
+      if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
 	{
-	  if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
-	    vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info,
-					 slp_node, &gs_info, &dataref_ptr,
+	  aggr_type = NULL_TREE;
+	  bump = NULL_TREE;
+	  if (!costing_p)
+	    vect_get_gather_scatter_ops (loop, slp_node, &gs_info, &dataref_ptr,
 					 &vec_offsets);
-	  else
-	    dataref_ptr
-	      = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
-					  at_loop, offset, &dummy, gsi,
-					  &ptr_incr, false, bump);
 	}
+      else
+	{
+	  aggr_type = elem_type;
+	  if (!costing_p)
+	    {
+	      vect_get_strided_load_store_ops (stmt_info, vectype, loop_vinfo,
+					       gsi, &gs_info,
+					       &bump, &vec_offset, loop_lens);
+	      dataref_ptr
+		  = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
+					      at_loop, offset, &dummy, gsi,
+					      &ptr_incr, false, bump);
+	    }
+	}
+
+      unsigned int inside_cost = 0, prologue_cost = 0;
 
       gimple *new_stmt = NULL;
       for (i = 0; i < vec_num; i++)
@@ -11031,7 +10377,7 @@ vectorizable_load (vec_info *vinfo,
 	  tree bias = NULL_TREE;
 	  if (!costing_p)
 	    {
-	      if (mask)
+	      if (mask_node)
 		vec_mask = vec_masks[i];
 	      if (loop_masks)
 		final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
@@ -11047,7 +10393,7 @@ vectorizable_load (vec_info *vinfo,
 
 	  /* 2. Create the vector-load in the loop.  */
 	  unsigned HOST_WIDE_INT align;
-	  if (gs_info.ifn != IFN_LAST)
+	  if (GATHER_SCATTER_IFN_P (gs_info))
 	    {
 	      if (costing_p)
 		{
@@ -11093,7 +10439,8 @@ vectorizable_load (vec_info *vinfo,
 		{
 		  if (VECTOR_TYPE_P (TREE_TYPE (vec_offset)))
 		    call = gimple_build_call_internal (IFN_MASK_LEN_GATHER_LOAD,
-						       8, dataref_ptr,
+						       9, dataref_ptr,
+						       gs_info.alias_ptr,
 						       vec_offset, scale, zero,
 						       final_mask, vec_els,
 						       final_len, bias);
@@ -11108,18 +10455,20 @@ vectorizable_load (vec_info *vinfo,
 		}
 	      else if (final_mask)
 		call = gimple_build_call_internal (IFN_MASK_GATHER_LOAD,
-						   6, dataref_ptr,
+						   7, dataref_ptr,
+						   gs_info.alias_ptr,
 						   vec_offset, scale,
 						   zero, final_mask, vec_els);
 	      else
-		call = gimple_build_call_internal (IFN_GATHER_LOAD, 4,
-						   dataref_ptr, vec_offset,
-						   scale, zero);
+		call = gimple_build_call_internal (IFN_GATHER_LOAD, 5,
+						   dataref_ptr,
+						   gs_info.alias_ptr,
+						   vec_offset, scale, zero);
 	      gimple_call_set_nothrow (call, true);
 	      new_stmt = call;
 	      data_ref = NULL_TREE;
 	    }
-	  else if (gs_info.decl)
+	  else if (GATHER_SCATTER_LEGACY_P (gs_info))
 	    {
 	      /* The builtin decls path for gather is legacy, x86 only.  */
 	      gcc_assert (!final_len && nunits.is_constant ());
@@ -11136,7 +10485,7 @@ vectorizable_load (vec_info *vinfo,
 	      if (known_eq (nunits, offset_nunits))
 		{
 		  new_stmt = vect_build_one_gather_load_call
-			       (vinfo, stmt_info, gsi, &gs_info,
+			       (vinfo, stmt_info, vectype, gsi, &gs_info,
 				dataref_ptr, vec_offsets[i], final_mask);
 		  data_ref = NULL_TREE;
 		}
@@ -11146,7 +10495,7 @@ vectorizable_load (vec_info *vinfo,
 		     lanes but the builtins will produce full vectype
 		     data with just the lower lanes filled.  */
 		  new_stmt = vect_build_one_gather_load_call
-			       (vinfo, stmt_info, gsi, &gs_info,
+			       (vinfo, stmt_info, vectype, gsi, &gs_info,
 				dataref_ptr, vec_offsets[2 * i], final_mask);
 		  tree low = make_ssa_name (vectype);
 		  gimple_set_lhs (new_stmt, low);
@@ -11185,7 +10534,8 @@ vectorizable_load (vec_info *vinfo,
 		    }
 
 		  new_stmt = vect_build_one_gather_load_call
-			       (vinfo, stmt_info, gsi, &gs_info, dataref_ptr,
+			       (vinfo, stmt_info, vectype, gsi, &gs_info,
+				dataref_ptr,
 				vec_offsets[2 * i + 1], final_mask);
 		  tree high = make_ssa_name (vectype);
 		  gimple_set_lhs (new_stmt, high);
@@ -11228,7 +10578,7 @@ vectorizable_load (vec_info *vinfo,
 						   new_stmt, gsi);
 		    }
 		  new_stmt = vect_build_one_gather_load_call
-			       (vinfo, stmt_info, gsi, &gs_info,
+			       (vinfo, stmt_info, vectype, gsi, &gs_info,
 				dataref_ptr, vec_offset, final_mask);
 		  data_ref = NULL_TREE;
 		}
@@ -11339,6 +10689,11 @@ vectorizable_load (vec_info *vinfo,
       return true;
     }
 
+  aggr_type = vectype;
+  if (!costing_p)
+    bump = vect_get_data_ptr_increment (vinfo, gsi, dr_info, aggr_type,
+					memory_access_type, loop_lens);
+
   poly_uint64 group_elt = 0;
   unsigned int inside_cost = 0, prologue_cost = 0;
   /* For costing some adjacent vector loads, we'd like to cost with
@@ -11418,7 +10773,7 @@ vectorizable_load (vec_info *vinfo,
 
       if (!costing_p)
 	{
-	  if (mask)
+	  if (mask_node)
 	    vec_mask = vec_masks[i];
 	  if (loop_masks)
 	    final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
@@ -11809,18 +11164,12 @@ vectorizable_load (vec_info *vinfo,
 	 alignment support schemes.  */
       if (costing_p)
 	{
-	  /* For VMAT_CONTIGUOUS_PERMUTE if it's grouped load, we
-	     only need to take care of the first stmt, whose
-	     stmt_info is first_stmt_info, vec_num iterating on it
-	     will cover the cost for the remaining, it's consistent
-	     with transforming.  For the prologue cost for realign,
+	  /* For the prologue cost for realign,
 	     we only need to count it once for the whole group.  */
 	  bool first_stmt_info_p = first_stmt_info == stmt_info;
 	  bool add_realign_cost = first_stmt_info_p && i == 0;
 	  if (memory_access_type == VMAT_CONTIGUOUS
-	      || memory_access_type == VMAT_CONTIGUOUS_REVERSE
-	      || (memory_access_type == VMAT_CONTIGUOUS_PERMUTE
-		  && (!grouped_load || first_stmt_info_p)))
+	      || memory_access_type == VMAT_CONTIGUOUS_REVERSE)
 	    {
 	      /* Leave realign cases alone to keep them simple.  */
 	      if (alignment_support_scheme == dr_explicit_realign_optimized
@@ -11908,7 +11257,7 @@ vectorizable_load (vec_info *vinfo,
 	}
 
       /* Collect vector loads and later create their permutation in
-	 vect_transform_grouped_load ().  */
+	 vect_transform_slp_perm_load.  */
       if (!costing_p && (grouped_load || slp_perm))
 	dr_chain.quick_push (new_temp);
 
@@ -11977,8 +11326,7 @@ vectorizable_load (vec_info *vinfo,
   if (costing_p)
     {
       gcc_assert (memory_access_type == VMAT_CONTIGUOUS
-		  || memory_access_type == VMAT_CONTIGUOUS_REVERSE
-		  || memory_access_type == VMAT_CONTIGUOUS_PERMUTE);
+		  || memory_access_type == VMAT_CONTIGUOUS_REVERSE);
       if (n_adjacent_loads > 0)
 	vect_get_load_cost (vinfo, stmt_info, slp_node, n_adjacent_loads,
 			    alignment_support_scheme, misalignment, false,
@@ -12008,7 +11356,7 @@ vectorizable_load (vec_info *vinfo,
    condition operands are supportable using vec_is_simple_use.  */
 
 static bool
-vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info,
+vect_is_simple_cond (tree cond, vec_info *vinfo,
 		     slp_tree slp_node, tree *comp_vectype,
 		     enum vect_def_type *dts, tree vectype)
 {
@@ -12020,7 +11368,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info,
   if (TREE_CODE (cond) == SSA_NAME
       && VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (cond)))
     {
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 0, &cond,
+      if (!vect_is_simple_use (vinfo, slp_node, 0, &cond,
 			       &slp_op, &dts[0], comp_vectype)
 	  || !*comp_vectype
 	  || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype))
@@ -12036,7 +11384,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info,
 
   if (TREE_CODE (lhs) == SSA_NAME)
     {
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 0,
+      if (!vect_is_simple_use (vinfo, slp_node, 0,
 			       &lhs, &slp_op, &dts[0], &vectype1))
 	return false;
     }
@@ -12048,7 +11396,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info,
 
   if (TREE_CODE (rhs) == SSA_NAME)
     {
-      if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1,
+      if (!vect_is_simple_use (vinfo, slp_node, 1,
 			       &rhs, &slp_op, &dts[1], &vectype2))
 	return false;
     }
@@ -12101,7 +11449,6 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info,
 static bool
 vectorizable_condition (vec_info *vinfo,
 			stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-			gimple **vec_stmt,
 			slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   tree scalar_dest = NULL_TREE;
@@ -12117,9 +11464,6 @@ vectorizable_condition (vec_info *vinfo,
   enum vect_def_type dts[4]
     = {vect_unknown_def_type, vect_unknown_def_type,
        vect_unknown_def_type, vect_unknown_def_type};
-  int ndts = 4;
-  int ncopies;
-  int vec_num;
   enum tree_code code, cond_code, bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
   int i;
   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
@@ -12149,7 +11493,7 @@ vectorizable_condition (vec_info *vinfo,
     = STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info)) != NULL;
   if (for_reduction)
     {
-      if (slp_node && SLP_TREE_LANES (slp_node) > 1)
+      if (SLP_TREE_LANES (slp_node) > 1)
 	return false;
       reduc_info = info_for_reduction (vinfo, stmt_info);
       reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info);
@@ -12163,38 +11507,25 @@ vectorizable_condition (vec_info *vinfo,
 	return false;
     }
 
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
 
-  if (slp_node)
-    {
-      ncopies = 1;
-      vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
-    }
-  else
-    {
-      ncopies = vect_get_num_copies (loop_vinfo, vectype);
-      vec_num = 1;
-    }
-
-  gcc_assert (ncopies >= 1);
-  if (for_reduction && ncopies > 1)
-    return false; /* FORNOW */
+  int vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
 
   cond_expr = gimple_assign_rhs1 (stmt);
   gcc_assert (! COMPARISON_CLASS_P (cond_expr));
 
-  if (!vect_is_simple_cond (cond_expr, vinfo, stmt_info, slp_node,
+  if (!vect_is_simple_cond (cond_expr, vinfo, slp_node,
 			    &comp_vectype, &dts[0], vectype)
       || !comp_vectype)
     return false;
 
   unsigned op_adjust = COMPARISON_CLASS_P (cond_expr) ? 1 : 0;
   slp_tree then_slp_node, else_slp_node;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 1 + op_adjust,
+  if (!vect_is_simple_use (vinfo, slp_node, 1 + op_adjust,
 			   &then_clause, &then_slp_node, &dts[2], &vectype1))
     return false;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node, 2 + op_adjust,
+  if (!vect_is_simple_use (vinfo, slp_node, 2 + op_adjust,
 			   &else_clause, &else_slp_node, &dts[3], &vectype2))
     return false;
 
@@ -12242,9 +11573,8 @@ vectorizable_condition (vec_info *vinfo,
 	}
       /* ???  The vectorized operand query below doesn't allow swapping
 	 this way for SLP.  */
-      if (slp_node)
-	return false;
-      std::swap (then_clause, else_clause);
+      return false;
+      /* std::swap (then_clause, else_clause); */
     }
 
   if (!masked && VECTOR_BOOLEAN_TYPE_P (comp_vectype))
@@ -12298,7 +11628,7 @@ vectorizable_condition (vec_info *vinfo,
       return false;
     }
 
-  if (!vec_stmt)
+  if (cost_vec)
     {
       if (bitop1 != NOP_EXPR)
 	{
@@ -12329,14 +11659,13 @@ vectorizable_condition (vec_info *vinfo,
 		       || !expand_vec_cond_expr_p (vectype, vec_cmp_type))))
 	return false;
 
-      if (slp_node
-	  && (!vect_maybe_update_slp_op_vectype
-		 (SLP_TREE_CHILDREN (slp_node)[0], comp_vectype)
-	      || (op_adjust == 1
-		  && !vect_maybe_update_slp_op_vectype
-			(SLP_TREE_CHILDREN (slp_node)[1], comp_vectype))
-	      || !vect_maybe_update_slp_op_vectype (then_slp_node, vectype)
-	      || !vect_maybe_update_slp_op_vectype (else_slp_node, vectype)))
+      if (!vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node)[0],
+					     comp_vectype)
+	  || (op_adjust == 1
+	      && !vect_maybe_update_slp_op_vectype
+			      (SLP_TREE_CHILDREN (slp_node)[1], comp_vectype))
+	  || !vect_maybe_update_slp_op_vectype (then_slp_node, vectype)
+	  || !vect_maybe_update_slp_op_vectype (else_slp_node, vectype))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -12353,11 +11682,11 @@ vectorizable_condition (vec_info *vinfo,
 						  vectype, OPTIMIZE_FOR_SPEED))
 		vect_record_loop_len (loop_vinfo,
 				      &LOOP_VINFO_LENS (loop_vinfo),
-				      ncopies * vec_num, vectype, 1);
+				      vec_num, vectype, 1);
 	      else
 		vect_record_loop_mask (loop_vinfo,
 				       &LOOP_VINFO_MASKS (loop_vinfo),
-				       ncopies * vec_num, vectype, NULL);
+				       vec_num, vectype, NULL);
 	    }
 	  /* Extra inactive lanes should be safe for vect_nested_cycle.  */
 	  else if (STMT_VINFO_DEF_TYPE (reduc_info) != vect_nested_cycle)
@@ -12370,9 +11699,8 @@ vectorizable_condition (vec_info *vinfo,
 	    }
 	}
 
-      STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
-      vect_model_simple_cost (vinfo, ncopies, dts, ndts, slp_node,
-			      cost_vec, kind);
+      SLP_TREE_TYPE (slp_node) = condition_vec_info_type;
+      vect_model_simple_cost (vinfo, 1, slp_node, cost_vec, kind);
       return true;
     }
 
@@ -12401,7 +11729,7 @@ vectorizable_condition (vec_info *vinfo,
 	masks = &LOOP_VINFO_MASKS (loop_vinfo);
       else
 	{
-	  scalar_cond_masked_key cond (cond_expr, ncopies);
+	  scalar_cond_masked_key cond (cond_expr, 1);
 	  if (loop_vinfo->scalar_cond_masked_set.contains (cond))
 	    masks = &LOOP_VINFO_MASKS (loop_vinfo);
 	  else
@@ -12436,18 +11764,18 @@ vectorizable_condition (vec_info *vinfo,
 
   /* Handle cond expr.  */
   if (masked)
-    vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
-		       cond_expr, comp_vectype, &vec_oprnds0,
-		       then_clause, vectype, &vec_oprnds2,
+    vect_get_vec_defs (vinfo, slp_node,
+		       cond_expr, &vec_oprnds0,
+		       then_clause, &vec_oprnds2,
 		       reduction_type != EXTRACT_LAST_REDUCTION
-		       ? else_clause : NULL, vectype, &vec_oprnds3);
+		       ? else_clause : NULL, &vec_oprnds3);
   else
-    vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
-		       cond_expr0, comp_vectype, &vec_oprnds0,
-		       cond_expr1, comp_vectype, &vec_oprnds1,
-		       then_clause, vectype, &vec_oprnds2,
+    vect_get_vec_defs (vinfo, slp_node,
+		       cond_expr0, &vec_oprnds0,
+		       cond_expr1, &vec_oprnds1,
+		       then_clause, &vec_oprnds2,
 		       reduction_type != EXTRACT_LAST_REDUCTION
-		       ? else_clause : NULL, vectype, &vec_oprnds3);
+		       ? else_clause : NULL, &vec_oprnds3);
 
   if (reduction_type == EXTRACT_LAST_REDUCTION)
     vec_else_clause = else_clause;
@@ -12564,7 +11892,7 @@ vectorizable_condition (vec_info *vinfo,
 	      if (lens)
 		{
 		  len = vect_get_loop_len (loop_vinfo, gsi, lens,
-					   vec_num * ncopies, vectype, i, 1);
+					   vec_num, vectype, i, 1);
 		  signed char biasval
 		    = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
 		  bias = build_int_cst (intQI_type_node, biasval);
@@ -12578,7 +11906,7 @@ vectorizable_condition (vec_info *vinfo,
 	  if (masks)
 	    {
 	      tree loop_mask
-		= vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num * ncopies,
+		= vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
 				      vectype, i);
 	      tree tmp2 = make_ssa_name (vec_cmp_type);
 	      gassign *g
@@ -12630,15 +11958,9 @@ vectorizable_condition (vec_info *vinfo,
 					  vec_then_clause, vec_else_clause);
 	  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 	}
-      if (slp_node)
-	slp_node->push_vec_def (new_stmt);
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+      slp_node->push_vec_def (new_stmt);
     }
 
-  if (!slp_node)
-    *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-
   vec_oprnds0.release ();
   vec_oprnds1.release ();
   vec_oprnds2.release ();
@@ -12658,18 +11980,15 @@ vectorizable_condition (vec_info *vinfo,
 static bool
 vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
 			   stmt_vec_info stmt_info, tree_code code,
-			   gimple_stmt_iterator *gsi, gimple **vec_stmt,
+			   gimple_stmt_iterator *gsi,
 			   slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   tree lhs, rhs1, rhs2;
   tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
   tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE;
   tree new_temp;
-  loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
   enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type};
-  int ndts = 2;
   poly_uint64 nunits;
-  int ncopies;
   enum tree_code bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
   int i;
   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
@@ -12687,22 +12006,15 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
   mask_type = vectype;
   nunits = TYPE_VECTOR_SUBPARTS (vectype);
 
-  if (slp_node)
-    ncopies = 1;
-  else
-    ncopies = vect_get_num_copies (loop_vinfo, vectype);
-
-  gcc_assert (ncopies >= 1);
-
   if (TREE_CODE_CLASS (code) != tcc_comparison)
     return false;
 
   slp_tree slp_rhs1, slp_rhs2;
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+  if (!vect_is_simple_use (vinfo, slp_node,
 			   0, &rhs1, &slp_rhs1, &dts[0], &vectype1))
     return false;
 
-  if (!vect_is_simple_use (vinfo, stmt_info, slp_node,
+  if (!vect_is_simple_use (vinfo, slp_node,
 			   1, &rhs2, &slp_rhs2, &dts[1], &vectype2))
     return false;
 
@@ -12768,7 +12080,7 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
 	}
     }
 
-  if (!vec_stmt)
+  if (cost_vec)
     {
       if (bitop1 == NOP_EXPR)
 	{
@@ -12793,9 +12105,8 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
 	}
 
       /* Put types on constant and invariant SLP children.  */
-      if (slp_node
-	  && (!vect_maybe_update_slp_op_vectype (slp_rhs1, vectype)
-	      || !vect_maybe_update_slp_op_vectype (slp_rhs2, vectype)))
+      if (!vect_maybe_update_slp_op_vectype (slp_rhs1, vectype)
+	  || !vect_maybe_update_slp_op_vectype (slp_rhs2, vectype))
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -12803,8 +12114,8 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
 	  return false;
 	}
 
-      vect_model_simple_cost (vinfo, ncopies * (1 + (bitop2 != NOP_EXPR)),
-			      dts, ndts, slp_node, cost_vec);
+      vect_model_simple_cost (vinfo, 1 + (bitop2 != NOP_EXPR),
+			      slp_node, cost_vec);
       return true;
     }
 
@@ -12815,9 +12126,7 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
   if (lhs)
     mask = vect_create_destination_var (lhs, mask_type);
 
-  vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies,
-		     rhs1, vectype, &vec_oprnds0,
-		     rhs2, vectype, &vec_oprnds1);
+  vect_get_vec_defs (vinfo, slp_node, rhs1, &vec_oprnds0, rhs2, &vec_oprnds1);
   if (swap_p)
     std::swap (vec_oprnds0, vec_oprnds1);
 
@@ -12856,15 +12165,9 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
 	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 	    }
 	}
-      if (slp_node)
-	slp_node->push_vec_def (new_stmt);
-      else
-	STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
+      slp_node->push_vec_def (new_stmt);
     }
 
-  if (!slp_node)
-    *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0];
-
   vec_oprnds0.release ();
   vec_oprnds1.release ();
 
@@ -12882,7 +12185,6 @@ vectorizable_comparison_1 (vec_info *vinfo, tree vectype,
 static bool
 vectorizable_comparison (vec_info *vinfo,
 			 stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
-			 gimple **vec_stmt,
 			 slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
@@ -12898,13 +12200,13 @@ vectorizable_comparison (vec_info *vinfo,
     return false;
 
   enum tree_code code = gimple_assign_rhs_code (stmt);
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi,
-				  vec_stmt, slp_node, cost_vec))
+				  slp_node, cost_vec))
     return false;
 
-  if (!vec_stmt)
-    STMT_VINFO_TYPE (stmt_info) = comparison_vec_info_type;
+  if (cost_vec)
+    SLP_TREE_TYPE (slp_node) = comparison_vec_info_type;
 
   return true;
 }
@@ -12914,7 +12216,7 @@ vectorizable_comparison (vec_info *vinfo,
 
 bool
 vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
-			 gimple_stmt_iterator *gsi, gimple **vec_stmt,
+			 gimple_stmt_iterator *gsi,
 			 slp_tree slp_node, stmt_vector_for_cost *cost_vec)
 {
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
@@ -12932,48 +12234,17 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
 
   auto code = gimple_cond_code (STMT_VINFO_STMT (stmt_info));
 
-  tree vectype = NULL_TREE;
-  slp_tree slp_op0;
-  tree op0;
-  enum vect_def_type dt0;
-
-  /* Early break gcond kind SLP trees can be root only and have no children,
-     for instance in the case where the argument is an external.  If that's
-     the case there is no operand to analyse use of.  */
-  if ((!slp_node || !SLP_TREE_CHILDREN (slp_node).is_empty ())
-      && !vect_is_simple_use (vinfo, stmt_info, slp_node, 0, &op0, &slp_op0, &dt0,
-			      &vectype))
-    {
-      if (dump_enabled_p ())
-	  dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			   "use not simple.\n");
-	return false;
-    }
-
   /* For SLP we don't want to use the type of the operands of the SLP node, when
      vectorizing using SLP slp_node will be the children of the gcond and we
      want to use the type of the direct children which since the gcond is root
      will be the current node, rather than a child node as vect_is_simple_use
      assumes.  */
-  if (slp_node)
-    vectype = SLP_TREE_VECTYPE (slp_node);
-
+  tree vectype = SLP_TREE_VECTYPE (slp_node);
   if (!vectype)
     return false;
 
   machine_mode mode = TYPE_MODE (vectype);
-  int ncopies, vec_num;
-
-  if (slp_node)
-    {
-      ncopies = 1;
-      vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
-    }
-  else
-    {
-      ncopies = vect_get_num_copies (loop_vinfo, vectype);
-      vec_num = 1;
-    }
+  int vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
 
   vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
   vec_loop_lens *lens = &LOOP_VINFO_LENS (loop_vinfo);
@@ -13003,7 +12274,7 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
 					exit_true_edge->dest);
 
   /* Analyze only.  */
-  if (!vec_stmt)
+  if (cost_vec)
     {
       if (direct_optab_handler (cbranch_optab, mode) == CODE_FOR_nothing)
 	{
@@ -13015,31 +12286,17 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
 	  return false;
 	}
 
-      if (ncopies > 1
-	  && direct_optab_handler (ior_optab, mode) == CODE_FOR_nothing)
-	{
-	  if (dump_enabled_p ())
-	      dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-			       "can't vectorize early exit because the "
-			       "target does not support boolean vector IOR "
-			       "for type %T.\n",
-			       vectype);
-	  return false;
-	}
-
       if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi,
-				      vec_stmt, slp_node, cost_vec))
+				      slp_node, cost_vec))
 	return false;
 
       if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
 	{
 	  if (direct_internal_fn_supported_p (IFN_VCOND_MASK_LEN, vectype,
 					      OPTIMIZE_FOR_SPEED))
-	    vect_record_loop_len (loop_vinfo, lens, ncopies * vec_num,
-				  vectype, 1);
+	    vect_record_loop_len (loop_vinfo, lens, vec_num, vectype, 1);
 	  else
-	    vect_record_loop_mask (loop_vinfo, masks, ncopies * vec_num,
-				   vectype, NULL);
+	    vect_record_loop_mask (loop_vinfo, masks, vec_num, vectype, NULL);
 	}
 
       return true;
@@ -13059,28 +12316,13 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
      vectorized.  It's not very clean to do this here, But the masking code below is
      complex and this keeps it all in one place to ease fixes and backports.  Once we
      drop the non-SLP loop vect or split vectorizable_* this can be simplified.  */
-  if (!slp_node)
-    {
-      if (!vectorizable_comparison_1 (vinfo, vectype, stmt_info, code, gsi,
-				      vec_stmt, slp_node, cost_vec))
-	gcc_unreachable ();
-    }
 
   gimple *stmt = STMT_VINFO_STMT (stmt_info);
   basic_block cond_bb = gimple_bb (stmt);
   gimple_stmt_iterator  cond_gsi = gsi_last_bb (cond_bb);
 
   auto_vec<tree> stmts;
-
-  if (slp_node)
-    stmts.safe_splice (SLP_TREE_VEC_DEFS (slp_node));
-  else
-    {
-      auto vec_stmts = STMT_VINFO_VEC_STMTS (stmt_info);
-      stmts.reserve_exact (vec_stmts.length ());
-      for (auto stmt : vec_stmts)
-	stmts.quick_push (gimple_assign_lhs (stmt));
-    }
+  stmts.safe_splice (SLP_TREE_VEC_DEFS (slp_node));
 
   /* If we're comparing against a previous forall we need to negate the resullts
      before we do the final comparison or reduction.  */
@@ -13119,7 +12361,7 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
 	for (unsigned i = 0; i < stmts.length (); i++)
 	  {
 	    tree stmt_mask
-	      = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies * vec_num,
+	      = vect_get_loop_mask (loop_vinfo, gsi, masks, vec_num,
 				    vectype, i);
 	    stmt_mask
 	      = prepare_vec_mask (loop_vinfo, TREE_TYPE (stmt_mask), stmt_mask,
@@ -13130,7 +12372,7 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
 	for (unsigned i = 0; i < stmts.length (); i++)
 	  {
 	    tree len_mask = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi,
-						    lens, ncopies * vec_num,
+						    lens, vec_num,
 						    vectype, stmts[i], i, 1);
 
 	    workset.quick_push (len_mask);
@@ -13155,13 +12397,13 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
       if (masked_loop_p)
 	{
 	  tree mask
-	    = vect_get_loop_mask (loop_vinfo, gsi, masks, ncopies, vectype, 0);
+	    = vect_get_loop_mask (loop_vinfo, gsi, masks, 1, vectype, 0);
 	  new_temp = prepare_vec_mask (loop_vinfo, TREE_TYPE (mask), mask,
 				       new_temp, &cond_gsi);
 	}
       else if (len_loop_p)
 	new_temp = vect_gen_loop_len_mask (loop_vinfo, gsi, &cond_gsi, lens,
-					   ncopies, vectype, new_temp, 0, 1);
+					   1, vectype, new_temp, 0, 1);
     }
 
   gcc_assert (new_temp);
@@ -13169,13 +12411,8 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
   gimple_cond_set_condition (cond_stmt, NE_EXPR, new_temp, cst);
   update_stmt (orig_stmt);
 
-  if (slp_node)
-    SLP_TREE_VEC_DEFS (slp_node).truncate (0);
-   else
-    STMT_VINFO_VEC_STMTS (stmt_info).truncate (0);
-
-  if (!slp_node)
-    *vec_stmt = orig_stmt;
+  /* ??? */
+  SLP_TREE_VEC_DEFS (slp_node).truncate (0);
 
   return true;
 }
@@ -13186,37 +12423,27 @@ vectorizable_early_exit (vec_info *vinfo, stmt_vec_info stmt_info,
    VEC_STMT_P is as for vectorizable_live_operation.  */
 
 static bool
-can_vectorize_live_stmts (vec_info *vinfo, stmt_vec_info stmt_info,
+can_vectorize_live_stmts (vec_info *vinfo,
 			  slp_tree slp_node, slp_instance slp_node_instance,
 			  bool vec_stmt_p,
 			  stmt_vector_for_cost *cost_vec)
 {
   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
-  if (slp_node)
-    {
-      stmt_vec_info slp_stmt_info;
-      unsigned int i;
-      FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info)
-	{
-	  if (slp_stmt_info
-	      && (STMT_VINFO_LIVE_P (slp_stmt_info)
-		  || (loop_vinfo
-		      && LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
-		      && STMT_VINFO_DEF_TYPE (slp_stmt_info)
-		      == vect_induction_def))
-	      && !vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
-					       slp_node_instance, i,
-					       vec_stmt_p, cost_vec))
-	    return false;
-	}
+  stmt_vec_info slp_stmt_info;
+  unsigned int i;
+  FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt_info)
+    {
+      if (slp_stmt_info
+	  && (STMT_VINFO_LIVE_P (slp_stmt_info)
+	      || (loop_vinfo
+		  && LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
+		  && STMT_VINFO_DEF_TYPE (slp_stmt_info)
+		  == vect_induction_def))
+	  && !vectorizable_live_operation (vinfo, slp_stmt_info, slp_node,
+					   slp_node_instance, i,
+					   vec_stmt_p, cost_vec))
+	return false;
     }
-  else if ((STMT_VINFO_LIVE_P (stmt_info)
-	    || (LOOP_VINFO_EARLY_BREAKS (loop_vinfo)
-		&& STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def))
-	   && !vectorizable_live_operation (vinfo, stmt_info,
-					    slp_node, slp_node_instance, -1,
-					    vec_stmt_p, cost_vec))
-    return false;
 
   return true;
 }
@@ -13225,115 +12452,42 @@ can_vectorize_live_stmts (vec_info *vinfo, stmt_vec_info stmt_info,
 
 opt_result
 vect_analyze_stmt (vec_info *vinfo,
-		   stmt_vec_info stmt_info, bool *need_to_vectorize,
 		   slp_tree node, slp_instance node_instance,
 		   stmt_vector_for_cost *cost_vec)
 {
+  stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
   enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
   bool ok;
-  gimple_seq pattern_def_seq;
 
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: %G",
 		     stmt_info->stmt);
 
   if (gimple_has_volatile_ops (stmt_info->stmt))
-    return opt_result::failure_at (stmt_info->stmt,
-				   "not vectorized:"
-				   " stmt has volatile operands: %G\n",
-				   stmt_info->stmt);
-
-  if (STMT_VINFO_IN_PATTERN_P (stmt_info)
-      && node == NULL
-      && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
     {
-      gimple_stmt_iterator si;
-
-      for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si))
-	{
-	  stmt_vec_info pattern_def_stmt_info
-	    = vinfo->lookup_stmt (gsi_stmt (si));
-	  if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info)
-	      || STMT_VINFO_LIVE_P (pattern_def_stmt_info))
-	    {
-	      /* Analyze def stmt of STMT if it's a pattern stmt.  */
-	      if (dump_enabled_p ())
-		dump_printf_loc (MSG_NOTE, vect_location,
-				 "==> examining pattern def statement: %G",
-				 pattern_def_stmt_info->stmt);
-
-	      opt_result res
-		= vect_analyze_stmt (vinfo, pattern_def_stmt_info,
-				     need_to_vectorize, node, node_instance,
-				     cost_vec);
-	      if (!res)
-		return res;
-	    }
-	}
+      /* ???  This shouldn't really happen, volatile stmts should
+	 not end up in the SLP graph.  */
+      return opt_result::failure_at (stmt_info->stmt,
+				     "not vectorized:"
+				     " stmt has volatile operands: %G\n",
+				     stmt_info->stmt);
     }
 
-  /* Skip stmts that do not need to be vectorized. In loops this is expected
-     to include:
-     - the COND_EXPR which is the loop exit condition
-     - any LABEL_EXPRs in the loop
-     - computations that are used only for array indexing or loop control.
-     In basic blocks we only analyze statements that are a part of some SLP
-     instance, therefore, all the statements are relevant.
-
-     Pattern statement needs to be analyzed instead of the original statement
-     if the original statement is not relevant.  Otherwise, we analyze both
-     statements.  In basic blocks we are called from some SLP instance
-     traversal, don't analyze pattern stmts instead, the pattern stmts
-     already will be part of SLP instance.  */
-
-  stmt_vec_info pattern_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info);
+  /* Skip stmts that do not need to be vectorized.  */
   if (!STMT_VINFO_RELEVANT_P (stmt_info)
       && !STMT_VINFO_LIVE_P (stmt_info))
     {
-      if (STMT_VINFO_IN_PATTERN_P (stmt_info)
-	  && pattern_stmt_info
-	  && (STMT_VINFO_RELEVANT_P (pattern_stmt_info)
-	      || STMT_VINFO_LIVE_P (pattern_stmt_info)))
-        {
-          /* Analyze PATTERN_STMT instead of the original stmt.  */
-	  stmt_info = pattern_stmt_info;
-          if (dump_enabled_p ())
-	    dump_printf_loc (MSG_NOTE, vect_location,
-			     "==> examining pattern statement: %G",
-			     stmt_info->stmt);
-        }
-      else
-        {
-          if (dump_enabled_p ())
-            dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
-
-	  if (node)
-	    return opt_result::failure_at (stmt_info->stmt,
-					   "not vectorized:"
-					   " irrelevant stmt as SLP node %p "
-					   "representative.\n",
-					   (void *)node);
-          return opt_result::success ();
-        }
-    }
-  else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
-	   && node == NULL
-	   && pattern_stmt_info
-	   && (STMT_VINFO_RELEVANT_P (pattern_stmt_info)
-	       || STMT_VINFO_LIVE_P (pattern_stmt_info)))
-    {
-      /* Analyze PATTERN_STMT too.  */
       if (dump_enabled_p ())
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "==> examining pattern statement: %G",
-			 pattern_stmt_info->stmt);
+	dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
 
-      opt_result res
-	= vect_analyze_stmt (vinfo, pattern_stmt_info, need_to_vectorize, node,
-			     node_instance, cost_vec);
-      if (!res)
-	return res;
+      /* ???  This shouldn't really happen, irrelevant stmts should
+	 not end up in the SLP graph.  */
+      return opt_result::failure_at (stmt_info->stmt,
+				     "not vectorized:"
+				     " irrelevant stmt as SLP node %p "
+				     "representative.\n",
+				     (void *)node);
     }
 
   switch (STMT_VINFO_DEF_TYPE (stmt_info))
@@ -13369,8 +12523,7 @@ vect_analyze_stmt (vec_info *vinfo,
     }
 
   tree saved_vectype = STMT_VINFO_VECTYPE (stmt_info);
-  if (node)
-    STMT_VINFO_VECTYPE (stmt_info) = SLP_TREE_VECTYPE (node);
+  STMT_VINFO_VECTYPE (stmt_info) = SLP_TREE_VECTYPE (node);
 
   if (STMT_VINFO_RELEVANT_P (stmt_info))
     {
@@ -13378,23 +12531,8 @@ vect_analyze_stmt (vec_info *vinfo,
       gcc_assert (STMT_VINFO_VECTYPE (stmt_info)
 		  || gimple_code (stmt_info->stmt) == GIMPLE_COND
 		  || (call && gimple_call_lhs (call) == NULL_TREE));
-      *need_to_vectorize = true;
-    }
-
-  if (PURE_SLP_STMT (stmt_info) && !node)
-    {
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "handled only by SLP analysis\n");
-      return opt_result::success ();
     }
 
-  /* When we arrive here with a non-SLP statement and we are supposed
-     to use SLP for everything fail vectorization.  */
-  if (!node)
-    return opt_result::failure_at (stmt_info->stmt,
-				   "needs non-SLP handling\n");
-
   ok = true;
   if (!bb_vinfo
       && (STMT_VINFO_RELEVANT_P (stmt_info)
@@ -13402,64 +12540,63 @@ vect_analyze_stmt (vec_info *vinfo,
     /* Prefer vectorizable_call over vectorizable_simd_clone_call so
        -mveclibabi= takes preference over library functions with
        the simd attribute.  */
-    ok = (vectorizable_call (vinfo, stmt_info, NULL, NULL, node, cost_vec)
-	  || vectorizable_simd_clone_call (vinfo, stmt_info, NULL, NULL, node,
+    ok = (vectorizable_call (vinfo, stmt_info, NULL, node, cost_vec)
+	  || vectorizable_simd_clone_call (vinfo, stmt_info, NULL, node,
 					   cost_vec)
 	  || vectorizable_conversion (vinfo, stmt_info,
-				      NULL, NULL, node, cost_vec)
+				      NULL, node, cost_vec)
 	  || vectorizable_operation (vinfo, stmt_info,
-				     NULL, NULL, node, cost_vec)
+				     NULL, node, cost_vec)
 	  || vectorizable_assignment (vinfo, stmt_info,
-				      NULL, NULL, node, cost_vec)
-	  || vectorizable_load (vinfo, stmt_info, NULL, NULL, node, cost_vec)
-	  || vectorizable_store (vinfo, stmt_info, NULL, NULL, node, cost_vec)
+				      NULL, node, cost_vec)
+	  || vectorizable_load (vinfo, stmt_info, NULL, node, cost_vec)
+	  || vectorizable_store (vinfo, stmt_info, NULL, node, cost_vec)
 	  || vectorizable_lane_reducing (as_a <loop_vec_info> (vinfo),
 					 stmt_info, node, cost_vec)
 	  || vectorizable_reduction (as_a <loop_vec_info> (vinfo), stmt_info,
 				     node, node_instance, cost_vec)
 	  || vectorizable_induction (as_a <loop_vec_info> (vinfo), stmt_info,
-				     NULL, node, cost_vec)
-	  || vectorizable_shift (vinfo, stmt_info, NULL, NULL, node, cost_vec)
+				     node, cost_vec)
+	  || vectorizable_shift (vinfo, stmt_info, NULL, node, cost_vec)
 	  || vectorizable_condition (vinfo, stmt_info,
-				     NULL, NULL, node, cost_vec)
-	  || vectorizable_comparison (vinfo, stmt_info, NULL, NULL, node,
+				     NULL, node, cost_vec)
+	  || vectorizable_comparison (vinfo, stmt_info, NULL, node,
 				      cost_vec)
 	  || vectorizable_lc_phi (as_a <loop_vec_info> (vinfo),
 				  stmt_info, node)
 	  || vectorizable_recurr (as_a <loop_vec_info> (vinfo),
-				   stmt_info, NULL, node, cost_vec)
-	  || vectorizable_early_exit (vinfo, stmt_info, NULL, NULL, node,
+				   stmt_info, node, cost_vec)
+	  || vectorizable_early_exit (vinfo, stmt_info, NULL, node,
 				      cost_vec));
   else
     {
       if (bb_vinfo)
-	ok = (vectorizable_call (vinfo, stmt_info, NULL, NULL, node, cost_vec)
+	ok = (vectorizable_call (vinfo, stmt_info, NULL, node, cost_vec)
 	      || vectorizable_simd_clone_call (vinfo, stmt_info,
-					       NULL, NULL, node, cost_vec)
-	      || vectorizable_conversion (vinfo, stmt_info, NULL, NULL, node,
+					       NULL, node, cost_vec)
+	      || vectorizable_conversion (vinfo, stmt_info, NULL, node,
 					  cost_vec)
 	      || vectorizable_shift (vinfo, stmt_info,
-				     NULL, NULL, node, cost_vec)
+				     NULL, node, cost_vec)
 	      || vectorizable_operation (vinfo, stmt_info,
-					 NULL, NULL, node, cost_vec)
-	      || vectorizable_assignment (vinfo, stmt_info, NULL, NULL, node,
+					 NULL, node, cost_vec)
+	      || vectorizable_assignment (vinfo, stmt_info, NULL, node,
 					  cost_vec)
 	      || vectorizable_load (vinfo, stmt_info,
-				    NULL, NULL, node, cost_vec)
+				    NULL, node, cost_vec)
 	      || vectorizable_store (vinfo, stmt_info,
-				     NULL, NULL, node, cost_vec)
+				     NULL, node, cost_vec)
 	      || vectorizable_condition (vinfo, stmt_info,
-					 NULL, NULL, node, cost_vec)
-	      || vectorizable_comparison (vinfo, stmt_info, NULL, NULL, node,
+					 NULL, node, cost_vec)
+	      || vectorizable_comparison (vinfo, stmt_info, NULL, node,
 					  cost_vec)
-	      || vectorizable_phi (vinfo, stmt_info, NULL, node, cost_vec)
-	      || vectorizable_early_exit (vinfo, stmt_info, NULL, NULL, node,
+	      || vectorizable_phi (vinfo, stmt_info, node, cost_vec)
+	      || vectorizable_early_exit (vinfo, stmt_info, NULL, node,
 					  cost_vec));
 
     }
 
-  if (node)
-    STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
+  STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
 
   if (!ok)
     return opt_result::failure_at (stmt_info->stmt,
@@ -13470,11 +12607,12 @@ vect_analyze_stmt (vec_info *vinfo,
   /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
       need extra handling, except for vectorizable reductions.  */
   if (!bb_vinfo
-      && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type
-      && STMT_VINFO_TYPE (stmt_info) != lc_phi_info_type
-      && (!node || !node->ldst_lanes || SLP_TREE_CODE (node) == VEC_PERM_EXPR)
+      && SLP_TREE_TYPE (node) != reduc_vec_info_type
+      && (SLP_TREE_TYPE (node) != lc_phi_info_type
+	  || STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def)
+      && (!node->ldst_lanes || SLP_TREE_CODE (node) == VEC_PERM_EXPR)
       && !can_vectorize_live_stmts (as_a <loop_vec_info> (vinfo),
-				    stmt_info, node, node_instance,
+				    node, node_instance,
 				    false, cost_vec))
     return opt_result::failure_at (stmt_info->stmt,
 				   "not vectorized:"
@@ -13495,57 +12633,49 @@ vect_transform_stmt (vec_info *vinfo,
 		     slp_tree slp_node, slp_instance slp_node_instance)
 {
   bool is_store = false;
-  gimple *vec_stmt = NULL;
   bool done;
 
-  gcc_assert (slp_node || !PURE_SLP_STMT (stmt_info));
+  gcc_assert (slp_node);
 
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location,
 		     "------>vectorizing statement: %G", stmt_info->stmt);
 
   tree saved_vectype = STMT_VINFO_VECTYPE (stmt_info);
-  if (slp_node)
-    STMT_VINFO_VECTYPE (stmt_info) = SLP_TREE_VECTYPE (slp_node);
+  STMT_VINFO_VECTYPE (stmt_info) = SLP_TREE_VECTYPE (slp_node);
 
-  switch (STMT_VINFO_TYPE (stmt_info))
+  switch (SLP_TREE_TYPE (slp_node))
     {
     case type_demotion_vec_info_type:
     case type_promotion_vec_info_type:
     case type_conversion_vec_info_type:
-      done = vectorizable_conversion (vinfo, stmt_info,
-				      gsi, &vec_stmt, slp_node, NULL);
+      done = vectorizable_conversion (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case induc_vec_info_type:
       done = vectorizable_induction (as_a <loop_vec_info> (vinfo),
-				     stmt_info, &vec_stmt, slp_node,
-				     NULL);
+				     stmt_info, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case shift_vec_info_type:
-      done = vectorizable_shift (vinfo, stmt_info,
-				 gsi, &vec_stmt, slp_node, NULL);
+      done = vectorizable_shift (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case op_vec_info_type:
-      done = vectorizable_operation (vinfo, stmt_info, gsi, &vec_stmt, slp_node,
-				     NULL);
+      done = vectorizable_operation (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case assignment_vec_info_type:
-      done = vectorizable_assignment (vinfo, stmt_info,
-				      gsi, &vec_stmt, slp_node, NULL);
+      done = vectorizable_assignment (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case load_vec_info_type:
-      done = vectorizable_load (vinfo, stmt_info, gsi, &vec_stmt, slp_node,
-				NULL);
+      done = vectorizable_load (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
@@ -13561,32 +12691,28 @@ vect_transform_stmt (vec_info *vinfo,
 	;
       else
 	{
-	  done = vectorizable_store (vinfo, stmt_info,
-				     gsi, &vec_stmt, slp_node, NULL);
+	  done = vectorizable_store (vinfo, stmt_info, gsi, slp_node, NULL);
 	  gcc_assert (done);
 	  is_store = true;
 	}
       break;
 
     case condition_vec_info_type:
-      done = vectorizable_condition (vinfo, stmt_info,
-				     gsi, &vec_stmt, slp_node, NULL);
+      done = vectorizable_condition (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case comparison_vec_info_type:
-      done = vectorizable_comparison (vinfo, stmt_info, gsi, &vec_stmt,
-				      slp_node, NULL);
+      done = vectorizable_comparison (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case call_vec_info_type:
-      done = vectorizable_call (vinfo, stmt_info,
-				gsi, &vec_stmt, slp_node, NULL);
+      done = vectorizable_call (vinfo, stmt_info, gsi, slp_node, NULL);
       break;
 
     case call_simd_clone_vec_info_type:
-      done = vectorizable_simd_clone_call (vinfo, stmt_info, gsi, &vec_stmt,
+      done = vectorizable_simd_clone_call (vinfo, stmt_info, gsi,
 					   slp_node, NULL);
       break;
 
@@ -13610,18 +12736,17 @@ vect_transform_stmt (vec_info *vinfo,
 
     case recurr_info_type:
       done = vectorizable_recurr (as_a <loop_vec_info> (vinfo),
-				  stmt_info, &vec_stmt, slp_node, NULL);
+				  stmt_info, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case phi_info_type:
-      done = vectorizable_phi (vinfo, stmt_info, &vec_stmt, slp_node, NULL);
+      done = vectorizable_phi (vinfo, stmt_info, slp_node, NULL);
       gcc_assert (done);
       break;
 
     case loop_exit_ctrl_vec_info_type:
-      done = vectorizable_early_exit (vinfo, stmt_info, gsi, &vec_stmt,
-				      slp_node, NULL);
+      done = vectorizable_early_exit (vinfo, stmt_info, gsi, slp_node, NULL);
       gcc_assert (done);
       break;
 
@@ -13636,23 +12761,18 @@ vect_transform_stmt (vec_info *vinfo,
       done = true;
     }
 
-  if (!slp_node && vec_stmt)
-    gcc_assert (STMT_VINFO_VEC_STMTS (stmt_info).exists ());
-
-  if (STMT_VINFO_TYPE (stmt_info) != store_vec_info_type
-      && (!slp_node
-	  || !slp_node->ldst_lanes
+  if (SLP_TREE_TYPE (slp_node) != store_vec_info_type
+      && (!slp_node->ldst_lanes
 	  || SLP_TREE_CODE (slp_node) == VEC_PERM_EXPR))
     {
       /* Handle stmts whose DEF is used outside the loop-nest that is
 	 being vectorized.  */
-      done = can_vectorize_live_stmts (vinfo, stmt_info, slp_node,
+      done = can_vectorize_live_stmts (vinfo, slp_node,
 				       slp_node_instance, true, NULL);
       gcc_assert (done);
     }
 
-  if (slp_node)
-    STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
+  STMT_VINFO_VECTYPE (stmt_info) = saved_vectype;
 
   return is_store;
 }
@@ -14087,123 +13207,49 @@ vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt,
 
 /* Function vect_is_simple_use.
 
-   Same as vect_is_simple_use but also determines the vector operand
-   type of OPERAND and stores it to *VECTYPE.  If the definition of
-   OPERAND is vect_uninitialized_def, vect_constant_def or
-   vect_external_def *VECTYPE will be set to NULL_TREE and the caller
-   is responsible to compute the best suited vector type for the
-   scalar operand.  */
-
-bool
-vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt,
-		    tree *vectype, stmt_vec_info *def_stmt_info_out,
-		    gimple **def_stmt_out)
-{
-  stmt_vec_info def_stmt_info;
-  gimple *def_stmt;
-  if (!vect_is_simple_use (operand, vinfo, dt, &def_stmt_info, &def_stmt))
-    return false;
-
-  if (def_stmt_out)
-    *def_stmt_out = def_stmt;
-  if (def_stmt_info_out)
-    *def_stmt_info_out = def_stmt_info;
-
-  /* Now get a vector type if the def is internal, otherwise supply
-     NULL_TREE and leave it up to the caller to figure out a proper
-     type for the use stmt.  */
-  if (*dt == vect_internal_def
-      || *dt == vect_induction_def
-      || *dt == vect_reduction_def
-      || *dt == vect_double_reduction_def
-      || *dt == vect_nested_cycle
-      || *dt == vect_first_order_recurrence)
-    {
-      *vectype = STMT_VINFO_VECTYPE (def_stmt_info);
-      gcc_assert (*vectype != NULL_TREE);
-      if (dump_enabled_p ())
-	dump_printf_loc (MSG_NOTE, vect_location,
-			 "vect_is_simple_use: vectype %T\n", *vectype);
-    }
-  else if (*dt == vect_uninitialized_def
-	   || *dt == vect_constant_def
-	   || *dt == vect_external_def)
-    *vectype = NULL_TREE;
-  else
-    gcc_unreachable ();
-
-  return true;
-}
-
-/* Function vect_is_simple_use.
-
    Same as vect_is_simple_use but determines the operand by operand
    position OPERAND from either STMT or SLP_NODE, filling in *OP
    and *SLP_DEF (when SLP_NODE is not NULL).  */
 
 bool
-vect_is_simple_use (vec_info *vinfo, stmt_vec_info stmt, slp_tree slp_node,
+vect_is_simple_use (vec_info *vinfo, slp_tree slp_node,
 		    unsigned operand, tree *op, slp_tree *slp_def,
 		    enum vect_def_type *dt,
 		    tree *vectype, stmt_vec_info *def_stmt_info_out)
 {
-  if (slp_node)
-    {
-      slp_tree child = SLP_TREE_CHILDREN (slp_node)[operand];
-      *slp_def = child;
-      *vectype = SLP_TREE_VECTYPE (child);
-      if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
-	{
-	  /* ???  VEC_PERM nodes might be intermediate and their lane value
-	     have no representative (nor do we build a VEC_PERM stmt for
-	     the actual operation).  Note for two-operator nodes we set
-	     a representative but leave scalar stmts empty as we'd only
-	     have one for a subset of lanes.  Ideally no caller would
-	     require *op for internal defs.  */
-	  if (SLP_TREE_REPRESENTATIVE (child))
-	    {
-	      *op = gimple_get_lhs (SLP_TREE_REPRESENTATIVE (child)->stmt);
-	      return vect_is_simple_use (*op, vinfo, dt, def_stmt_info_out);
-	    }
-	  else
-	    {
-	      gcc_assert (SLP_TREE_CODE (child) == VEC_PERM_EXPR);
-	      *op = error_mark_node;
-	      *dt = vect_internal_def;
-	      if (def_stmt_info_out)
-		*def_stmt_info_out = NULL;
-	      return true;
-	    }
+  slp_tree child = SLP_TREE_CHILDREN (slp_node)[operand];
+  *slp_def = child;
+  *vectype = SLP_TREE_VECTYPE (child);
+  if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
+    {
+      /* ???  VEC_PERM nodes might be intermediate and their lane value
+	 have no representative (nor do we build a VEC_PERM stmt for
+	 the actual operation).  Note for two-operator nodes we set
+	 a representative but leave scalar stmts empty as we'd only
+	 have one for a subset of lanes.  Ideally no caller would
+	 require *op for internal defs.  */
+      if (SLP_TREE_REPRESENTATIVE (child))
+	{
+	  *op = gimple_get_lhs (SLP_TREE_REPRESENTATIVE (child)->stmt);
+	  return vect_is_simple_use (*op, vinfo, dt, def_stmt_info_out);
 	}
       else
 	{
+	  gcc_assert (SLP_TREE_CODE (child) == VEC_PERM_EXPR);
+	  *op = error_mark_node;
+	  *dt = vect_internal_def;
 	  if (def_stmt_info_out)
 	    *def_stmt_info_out = NULL;
-	  *op = SLP_TREE_SCALAR_OPS (child)[0];
-	  *dt = SLP_TREE_DEF_TYPE (child);
 	  return true;
 	}
     }
   else
     {
-      *slp_def = NULL;
-      if (gassign *ass = dyn_cast <gassign *> (stmt->stmt))
-	{
-	  if (gimple_assign_rhs_code (ass) == COND_EXPR
-	      && COMPARISON_CLASS_P (gimple_assign_rhs1 (ass)))
-	    gcc_unreachable ();
-	  else if (gimple_assign_rhs_code (ass) == VIEW_CONVERT_EXPR)
-	    *op = TREE_OPERAND (gimple_assign_rhs1 (ass), 0);
-	  else
-	    *op = gimple_op (ass, operand + 1);
-	}
-      else if (gcond *cond = dyn_cast <gcond *> (stmt->stmt))
-	*op = gimple_op (cond, operand);
-      else if (gcall *call = dyn_cast <gcall *> (stmt->stmt))
-	*op = gimple_call_arg (call, operand);
-      else
-	gcc_unreachable ();
-      return vect_is_simple_use (*op, vinfo, dt, vectype, def_stmt_info_out);
+      if (def_stmt_info_out)
+	*def_stmt_info_out = NULL;
+      *op = SLP_TREE_SCALAR_OPS (child)[0];
+      *dt = SLP_TREE_DEF_TYPE (child);
+      return true;
     }
 }
 
@@ -14410,6 +13456,8 @@ supportable_widening_operation (vec_info *vinfo,
 
       internal_fn lo, hi, even, odd;
       lookup_hilo_internal_fn (ifn, &lo, &hi);
+      if (BYTES_BIG_ENDIAN)
+	std::swap (lo, hi);
       *code1 = as_combined_fn (lo);
       *code2 = as_combined_fn (hi);
       optab1 = direct_internal_fn_optab (lo, {vectype, vectype});
diff --git a/gcc/tree-vectorizer.cc b/gcc/tree-vectorizer.cc
index 2f77e46..f992856 100644
--- a/gcc/tree-vectorizer.cc
+++ b/gcc/tree-vectorizer.cc
@@ -715,7 +715,6 @@ vec_info::new_stmt_vec_info (gimple *stmt)
   stmt_vec_info res = XCNEW (class _stmt_vec_info);
   res->stmt = stmt;
 
-  STMT_VINFO_TYPE (res) = undef_vec_info_type;
   STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
   STMT_VINFO_VECTORIZABLE (res) = true;
   STMT_VINFO_REDUC_TYPE (res) = TREE_CODE_REDUCTION;
@@ -724,7 +723,6 @@ vec_info::new_stmt_vec_info (gimple *stmt)
   STMT_VINFO_REDUC_IDX (res) = -1;
   STMT_VINFO_SLP_VECT_ONLY (res) = false;
   STMT_VINFO_SLP_VECT_ONLY_PATTERN (res) = false;
-  STMT_VINFO_VEC_STMTS (res) = vNULL;
   res->reduc_initial_values = vNULL;
   res->reduc_scalar_results = vNULL;
 
@@ -790,8 +788,6 @@ vec_info::free_stmt_vec_info (stmt_vec_info stmt_info)
 
   stmt_info->reduc_initial_values.release ();
   stmt_info->reduc_scalar_results.release ();
-  STMT_VINFO_SIMD_CLONE_INFO (stmt_info).release ();
-  STMT_VINFO_VEC_STMTS (stmt_info).release ();
   free (stmt_info);
 }
 
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 66a2964..0a75ee1 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -173,6 +173,8 @@ struct vect_scalar_ops_slice_hash : typed_noop_remove<vect_scalar_ops_slice>
 /* Describes how we're going to vectorize an individual load or store,
    or a group of loads or stores.  */
 enum vect_memory_access_type {
+  VMAT_UNINITIALIZED,
+
   /* An access to an invariant address.  This is used only for loads.  */
   VMAT_INVARIANT,
 
@@ -184,11 +186,6 @@ enum vect_memory_access_type {
      of invariants.  */
   VMAT_CONTIGUOUS_DOWN,
 
-  /* A simple contiguous access in which the elements need to be permuted
-     after loading or before storing.  Only used for loop vectorization;
-     SLP uses separate permutes.  */
-  VMAT_CONTIGUOUS_PERMUTE,
-
   /* A simple contiguous access in which the elements need to be reversed
      after loading or before storing.  */
   VMAT_CONTIGUOUS_REVERSE,
@@ -210,6 +207,32 @@ enum vect_memory_access_type {
   VMAT_GATHER_SCATTER
 };
 
+/*-----------------------------------------------------------------*/
+/* Info on vectorized defs.                                        */
+/*-----------------------------------------------------------------*/
+enum stmt_vec_info_type {
+  undef_vec_info_type = 0,
+  load_vec_info_type,
+  store_vec_info_type,
+  shift_vec_info_type,
+  op_vec_info_type,
+  call_vec_info_type,
+  call_simd_clone_vec_info_type,
+  assignment_vec_info_type,
+  condition_vec_info_type,
+  comparison_vec_info_type,
+  reduc_vec_info_type,
+  induc_vec_info_type,
+  type_promotion_vec_info_type,
+  type_demotion_vec_info_type,
+  type_conversion_vec_info_type,
+  cycle_phi_info_type,
+  lc_phi_info_type,
+  phi_info_type,
+  recurr_info_type,
+  loop_exit_ctrl_vec_info_type
+};
+
 /************************************************************************
   SLP
  ************************************************************************/
@@ -218,6 +241,23 @@ typedef auto_vec<std::pair<unsigned, unsigned>, 16> auto_lane_permutation_t;
 typedef vec<unsigned> load_permutation_t;
 typedef auto_vec<unsigned, 16> auto_load_permutation_t;
 
+struct vect_data {
+  virtual ~vect_data () = default;
+};
+
+/* Analysis data from vectorizable_simd_clone_call for
+   call_simd_clone_vec_info_type.  */
+struct vect_simd_clone_data : vect_data {
+  virtual ~vect_simd_clone_data () = default;
+  vect_simd_clone_data () = default;
+  vect_simd_clone_data (vect_simd_clone_data &&other) = default;
+
+  /* Selected SIMD clone's function info.  First vector element
+     is SIMD clone's function decl, followed by a pair of trees (base + step)
+     for linear arguments (pair of NULLs for other arguments).  */
+  auto_vec<tree> simd_clone_info;
+};
+
 /* A computation tree of an SLP instance.  Each node corresponds to a group of
    stmts to be packed in a SIMD stmt.  */
 struct _slp_tree {
@@ -246,11 +286,6 @@ struct _slp_tree {
      denotes the number of output lanes.  */
   lane_permutation_t lane_permutation;
 
-  /* Selected SIMD clone's function info.  First vector element
-     is SIMD clone's function decl, followed by a pair of trees (base + step)
-     for linear arguments (pair of NULLs for other arguments).  */
-  vec<tree> simd_clone_info;
-
   tree vectype;
   /* Vectorized defs.  */
   vec<tree> vec_defs;
@@ -284,6 +319,14 @@ struct _slp_tree {
      for loop vectorization.  */
   vect_memory_access_type memory_access_type;
 
+  /* The kind of operation as determined by analysis and optional
+     kind specific data.  */
+  enum stmt_vec_info_type type;
+  vect_data *data;
+
+  template <class T>
+  T& get_data (T& else_) { return data ? *static_cast <T *> (data) : else_; }
+
   /* If not NULL this is a cached failed SLP discovery attempt with
      the lanes that failed during SLP discovery as 'false'.  This is
      a copy of the matches array.  */
@@ -362,13 +405,13 @@ public:
 #define SLP_TREE_NUMBER_OF_VEC_STMTS(S)          (S)->vec_stmts_size
 #define SLP_TREE_LOAD_PERMUTATION(S)             (S)->load_permutation
 #define SLP_TREE_LANE_PERMUTATION(S)             (S)->lane_permutation
-#define SLP_TREE_SIMD_CLONE_INFO(S)              (S)->simd_clone_info
 #define SLP_TREE_DEF_TYPE(S)			 (S)->def_type
 #define SLP_TREE_VECTYPE(S)			 (S)->vectype
 #define SLP_TREE_REPRESENTATIVE(S)		 (S)->representative
 #define SLP_TREE_LANES(S)			 (S)->lanes
 #define SLP_TREE_CODE(S)			 (S)->code
 #define SLP_TREE_MEMORY_ACCESS_TYPE(S)		 (S)->memory_access_type
+#define SLP_TREE_TYPE(S)			 (S)->type
 
 enum vect_partial_vector_style {
     vect_partial_vectors_none,
@@ -1168,6 +1211,10 @@ public:
    || LOOP_REQUIRES_VERSIONING_FOR_NITERS (L)		\
    || LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (L))
 
+#define LOOP_VINFO_USE_VERSIONING_WITHOUT_PEELING(L)	\
+  ((L)->may_misalign_stmts.length () > 0		\
+   && !LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (L))
+
 #define LOOP_VINFO_NITERS_KNOWN_P(L)          \
   (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0)
 
@@ -1216,32 +1263,6 @@ public:
 #define BB_VINFO_DATAREFS(B)         (B)->shared->datarefs
 #define BB_VINFO_DDRS(B)             (B)->shared->ddrs
 
-/*-----------------------------------------------------------------*/
-/* Info on vectorized defs.                                        */
-/*-----------------------------------------------------------------*/
-enum stmt_vec_info_type {
-  undef_vec_info_type = 0,
-  load_vec_info_type,
-  store_vec_info_type,
-  shift_vec_info_type,
-  op_vec_info_type,
-  call_vec_info_type,
-  call_simd_clone_vec_info_type,
-  assignment_vec_info_type,
-  condition_vec_info_type,
-  comparison_vec_info_type,
-  reduc_vec_info_type,
-  induc_vec_info_type,
-  type_promotion_vec_info_type,
-  type_demotion_vec_info_type,
-  type_conversion_vec_info_type,
-  cycle_phi_info_type,
-  lc_phi_info_type,
-  phi_info_type,
-  recurr_info_type,
-  loop_exit_ctrl_vec_info_type
-};
-
 /* Indicates whether/how a variable is used in the scope of loop/basic
    block.  */
 enum vect_relevant {
@@ -1334,8 +1355,6 @@ typedef struct data_reference *dr_p;
 class _stmt_vec_info {
 public:
 
-  enum stmt_vec_info_type type;
-
   /* Indicates whether this stmts is part of a computation whose result is
      used outside the loop.  */
   bool live;
@@ -1359,9 +1378,6 @@ public:
   /* The vector type to be used for the LHS of this statement.  */
   tree vectype;
 
-  /* The vectorized stmts.  */
-  vec<gimple *> vec_stmts;
-
   /* The following is relevant only for stmts that contain a non-scalar
      data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
      at most one such data-ref.  */
@@ -1398,11 +1414,6 @@ public:
      pattern statement.  */
   gimple_seq pattern_def_seq;
 
-  /* Selected SIMD clone's function info.  First vector element
-     is SIMD clone's function decl, followed by a pair of trees (base + step)
-     for linear arguments (pair of NULLs for other arguments).  */
-  vec<tree> simd_clone_info;
-
   /* Classify the def of this stmt.  */
   enum vect_def_type def_type;
 
@@ -1441,10 +1452,6 @@ public:
   /* For both loads and stores.  */
   unsigned simd_lane_access_p : 3;
 
-  /* Classifies how the load or store is going to be implemented
-     for loop vectorization.  */
-  vect_memory_access_type memory_access_type;
-
   /* For INTEGER_INDUC_COND_REDUCTION, the initial value to be used.  */
   tree induc_cond_initial_val;
 
@@ -1557,6 +1564,10 @@ struct gather_scatter_info {
   /* The loop-invariant base value.  */
   tree base;
 
+  /* The TBBA alias pointer the value of which determines the alignment
+     of the scalar accesses.  */
+  tree alias_ptr;
+
   /* The original scalar offset, which is a non-loop-invariant SSA_NAME.  */
   tree offset;
 
@@ -1564,9 +1575,6 @@ struct gather_scatter_info {
      being added to the base.  */
   int scale;
 
-  /* The definition type for the vectorized offset.  */
-  enum vect_def_type offset_dt;
-
   /* The type of the vectorized offset.  */
   tree offset_vectype;
 
@@ -1578,17 +1586,14 @@ struct gather_scatter_info {
 };
 
 /* Access Functions.  */
-#define STMT_VINFO_TYPE(S)                 (S)->type
 #define STMT_VINFO_STMT(S)                 (S)->stmt
 #define STMT_VINFO_RELEVANT(S)             (S)->relevant
 #define STMT_VINFO_LIVE_P(S)               (S)->live
 #define STMT_VINFO_VECTYPE(S)              (S)->vectype
-#define STMT_VINFO_VEC_STMTS(S)            (S)->vec_stmts
 #define STMT_VINFO_VECTORIZABLE(S)         (S)->vectorizable
 #define STMT_VINFO_DATA_REF(S)             ((S)->dr_aux.dr + 0)
 #define STMT_VINFO_GATHER_SCATTER_P(S)	   (S)->gather_scatter_p
 #define STMT_VINFO_STRIDED_P(S)	   	   (S)->strided_p
-#define STMT_VINFO_MEMORY_ACCESS_TYPE(S)   (S)->memory_access_type
 #define STMT_VINFO_SIMD_LANE_ACCESS_P(S)   (S)->simd_lane_access_p
 #define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val
 #define STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT(S) (S)->reduc_epilogue_adjustment
@@ -1614,7 +1619,6 @@ struct gather_scatter_info {
 #define STMT_VINFO_IN_PATTERN_P(S)         (S)->in_pattern_p
 #define STMT_VINFO_RELATED_STMT(S)         (S)->related_stmt
 #define STMT_VINFO_PATTERN_DEF_SEQ(S)      (S)->pattern_def_seq
-#define STMT_VINFO_SIMD_CLONE_INFO(S)	   (S)->simd_clone_info
 #define STMT_VINFO_DEF_TYPE(S)             (S)->def_type
 #define STMT_VINFO_GROUPED_ACCESS(S) \
   ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
@@ -1655,6 +1659,14 @@ struct gather_scatter_info {
 #define PURE_SLP_STMT(S)                  ((S)->slp_type == pure_slp)
 #define STMT_SLP_TYPE(S)                   (S)->slp_type
 
+#define GATHER_SCATTER_LEGACY_P(info) ((info).decl != NULL_TREE \
+				       && (info).ifn == IFN_LAST)
+#define GATHER_SCATTER_IFN_P(info) ((info).decl == NULL_TREE \
+				    && (info).ifn != IFN_LAST)
+#define GATHER_SCATTER_EMULATED_P(info) ((info).decl == NULL_TREE \
+					 && (info).ifn == IFN_LAST)
+
+
 /* Contains the scalar or vector costs for a vec_info.  */
 class vector_costs
 {
@@ -1714,7 +1726,8 @@ public:
   unsigned int outside_cost () const;
   unsigned int total_cost () const;
   unsigned int suggested_unroll_factor () const;
-  machine_mode suggested_epilogue_mode () const;
+  machine_mode suggested_epilogue_mode (int &masked) const;
+  bool costing_for_scalar () const { return m_costing_for_scalar; }
 
 protected:
   unsigned int record_stmt_cost (stmt_vec_info, vect_cost_model_location,
@@ -1738,8 +1751,13 @@ protected:
   unsigned int m_suggested_unroll_factor;
 
   /* The suggested mode to be used for a vectorized epilogue or VOIDmode,
-     determined at finish_cost.  */
+     determined at finish_cost.  m_masked_epilogue specifies whether the
+     epilogue should use masked vectorization, regardless of the
+     --param vect-partial-vector-usage default.  If -1 then the
+     --param setting takes precedence.  If the user explicitly specified
+     --param vect-partial-vector-usage then that takes precedence.  */
   machine_mode m_suggested_epilogue_mode;
+  int m_masked_epilogue;
 
   /* True if finish_cost has been called.  */
   bool m_finished;
@@ -1755,6 +1773,7 @@ vector_costs::vector_costs (vec_info *vinfo, bool costing_for_scalar)
     m_costs (),
     m_suggested_unroll_factor(1),
     m_suggested_epilogue_mode(VOIDmode),
+    m_masked_epilogue (-1),
     m_finished (false)
 {
 }
@@ -1815,9 +1834,10 @@ vector_costs::suggested_unroll_factor () const
 /* Return the suggested epilogue mode.  */
 
 inline machine_mode
-vector_costs::suggested_epilogue_mode () const
+vector_costs::suggested_epilogue_mode (int &masked_p) const
 {
   gcc_checking_assert (m_finished);
+  masked_p = m_masked_epilogue;
   return m_suggested_epilogue_mode;
 }
 
@@ -1990,6 +2010,13 @@ add_stmt_cost (vector_costs *costs, int count,
 	       tree vectype, int misalign,
 	       enum vect_cost_model_location where)
 {
+  /* Even though a vector type might be set on stmt do not pass that on when
+     costing the scalar IL.  A SLP node shouldn't have been recorded.  */
+  if (costs->costing_for_scalar ())
+    {
+      vectype = NULL_TREE;
+      gcc_checking_assert (node == NULL);
+    }
   unsigned cost = costs->add_stmt_cost (count, kind, stmt_info, node, vectype,
 					misalign, where);
   if (dump_file && (dump_flags & TDF_DETAILS))
@@ -2415,10 +2442,7 @@ extern bool vect_chooses_same_modes_p (machine_mode, machine_mode);
 extern bool vect_get_loop_mask_type (loop_vec_info);
 extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
 				stmt_vec_info * = NULL, gimple ** = NULL);
-extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
-				tree *, stmt_vec_info * = NULL,
-				gimple ** = NULL);
-extern bool vect_is_simple_use (vec_info *, stmt_vec_info, slp_tree,
+extern bool vect_is_simple_use (vec_info *, slp_tree,
 				unsigned, tree *, slp_tree *,
 				enum vect_def_type *,
 				tree *, stmt_vec_info * = NULL);
@@ -2479,18 +2503,11 @@ extern void vect_finish_stmt_generation (vec_info *, stmt_vec_info, gimple *,
 					 gimple_stmt_iterator *);
 extern opt_result vect_mark_stmts_to_be_vectorized (loop_vec_info, bool *);
 extern tree vect_get_store_rhs (stmt_vec_info);
-void vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info, unsigned,
-				    tree op, vec<tree> *, tree = NULL);
-void vect_get_vec_defs (vec_info *, stmt_vec_info, slp_tree, unsigned,
+void vect_get_vec_defs (vec_info *, slp_tree,
 			tree, vec<tree> *,
 			tree = NULL, vec<tree> * = NULL,
 			tree = NULL, vec<tree> * = NULL,
 			tree = NULL, vec<tree> * = NULL);
-void vect_get_vec_defs (vec_info *, stmt_vec_info, slp_tree, unsigned,
-			tree, tree, vec<tree> *,
-			tree = NULL, tree = NULL, vec<tree> * = NULL,
-			tree = NULL, tree = NULL, vec<tree> * = NULL,
-			tree = NULL, tree = NULL, vec<tree> * = NULL);
 extern tree vect_init_vector (vec_info *, stmt_vec_info, tree, tree,
                               gimple_stmt_iterator *);
 extern tree vect_get_slp_vect_def (slp_tree, unsigned);
@@ -2499,8 +2516,7 @@ extern bool vect_transform_stmt (vec_info *, stmt_vec_info,
 				 slp_tree, slp_instance);
 extern void vect_remove_stores (vec_info *, stmt_vec_info);
 extern bool vect_nop_conversion_p (stmt_vec_info);
-extern opt_result vect_analyze_stmt (vec_info *, stmt_vec_info, bool *,
-				     slp_tree,
+extern opt_result vect_analyze_stmt (vec_info *, slp_tree,
 				     slp_instance, stmt_vector_for_cost *);
 extern void vect_get_load_cost (vec_info *, stmt_vec_info, slp_tree, int,
 				dr_alignment_support, int, bool,
@@ -2528,7 +2544,8 @@ extern bool ref_within_array_bound (gimple *, tree);
 /* In tree-vect-data-refs.cc.  */
 extern bool vect_can_force_dr_alignment_p (const_tree, poly_uint64);
 extern enum dr_alignment_support vect_supportable_dr_alignment
-				   (vec_info *, dr_vec_info *, tree, int);
+				   (vec_info *, dr_vec_info *, tree, int,
+				     gather_scatter_info * = nullptr);
 extern tree vect_get_smallest_scalar_type (stmt_vec_info, tree);
 extern opt_result vect_analyze_data_ref_dependences (loop_vec_info, unsigned int *);
 extern bool vect_slp_analyze_instance_dependence (vec_info *, slp_instance);
@@ -2546,7 +2563,7 @@ extern bool vect_check_gather_scatter (stmt_vec_info, loop_vec_info,
 extern opt_result vect_find_stmt_data_reference (loop_p, gimple *,
 						 vec<data_reference_p> *,
 						 vec<int> *, int);
-extern opt_result vect_analyze_data_refs (vec_info *, poly_uint64 *, bool *);
+extern opt_result vect_analyze_data_refs (vec_info *, bool *);
 extern void vect_record_base_alignments (vec_info *);
 extern tree vect_create_data_ref_ptr (vec_info *,
 				      stmt_vec_info, tree, class loop *, tree,
@@ -2562,17 +2579,10 @@ extern internal_fn vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT, boo
 extern bool vect_grouped_load_supported (tree, bool, unsigned HOST_WIDE_INT);
 extern internal_fn vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT,
 					      bool, vec<int> * = nullptr);
-extern void vect_permute_store_chain (vec_info *, vec<tree> &,
-				      unsigned int, stmt_vec_info,
-				      gimple_stmt_iterator *, vec<tree> *);
 extern tree vect_setup_realignment (vec_info *,
 				    stmt_vec_info, gimple_stmt_iterator *,
 				    tree *, enum dr_alignment_support, tree,
 	                            class loop **);
-extern void vect_transform_grouped_load (vec_info *, stmt_vec_info, vec<tree>,
-					 int, gimple_stmt_iterator *);
-extern void vect_record_grouped_load_vectors (vec_info *,
-					      stmt_vec_info, vec<tree>);
 extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
 extern tree vect_get_new_ssa_name (tree, enum vect_var_kind,
 				   const char * = NULL);
@@ -2641,8 +2651,7 @@ extern bool vectorizable_reduction (loop_vec_info, stmt_vec_info,
 				    slp_tree, slp_instance,
 				    stmt_vector_for_cost *);
 extern bool vectorizable_induction (loop_vec_info, stmt_vec_info,
-				    gimple **, slp_tree,
-				    stmt_vector_for_cost *);
+				    slp_tree, stmt_vector_for_cost *);
 extern bool vect_transform_reduction (loop_vec_info, stmt_vec_info,
 				      gimple_stmt_iterator *,
 				      slp_tree);
@@ -2650,12 +2659,12 @@ extern bool vect_transform_cycle_phi (loop_vec_info, stmt_vec_info,
 				      slp_tree, slp_instance);
 extern bool vectorizable_lc_phi (loop_vec_info, stmt_vec_info, slp_tree);
 extern bool vect_transform_lc_phi (loop_vec_info, stmt_vec_info, slp_tree);
-extern bool vectorizable_phi (vec_info *, stmt_vec_info, gimple **, slp_tree,
+extern bool vectorizable_phi (vec_info *, stmt_vec_info, slp_tree,
 			      stmt_vector_for_cost *);
 extern bool vectorizable_recurr (loop_vec_info, stmt_vec_info,
-				  gimple **, slp_tree, stmt_vector_for_cost *);
+				  slp_tree, stmt_vector_for_cost *);
 extern bool vectorizable_early_exit (vec_info *, stmt_vec_info,
-				     gimple_stmt_iterator *, gimple **,
+				     gimple_stmt_iterator *,
 				     slp_tree, stmt_vector_for_cost *);
 extern bool vect_emulated_vector_p (tree);
 extern bool vect_can_vectorize_without_simd_p (tree_code);
@@ -2682,7 +2691,7 @@ extern bool vect_slp_analyze_operations (vec_info *);
 extern void vect_schedule_slp (vec_info *, const vec<slp_instance> &);
 extern opt_result vect_analyze_slp (vec_info *, unsigned, bool);
 extern bool vect_make_slp_decision (loop_vec_info);
-extern void vect_detect_hybrid_slp (loop_vec_info);
+extern bool vect_detect_hybrid_slp (loop_vec_info);
 extern void vect_optimize_slp (vec_info *);
 extern void vect_gather_slp_loads (vec_info *);
 extern tree vect_get_slp_scalar_def (slp_tree, unsigned);
@@ -2833,18 +2842,6 @@ vect_is_reduction (stmt_vec_info stmt_info)
   return STMT_VINFO_REDUC_IDX (stmt_info) >= 0;
 }
 
-/* Returns the memory acccess type being used to vectorize the statement.  If
-   SLP this is read from NODE, otherwise it's read from the STMT_VINFO.  */
-
-inline vect_memory_access_type
-vect_mem_access_type (stmt_vec_info stmt_info, slp_tree node)
-{
-  if (node)
-    return SLP_TREE_MEMORY_ACCESS_TYPE (node);
-  else
-    return STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info);
-}
-
 /* If STMT_INFO describes a reduction, return the vect_reduction_type
    of the reduction it describes, otherwise return -1.  */
 inline int
diff --git a/gcc/tree.cc b/gcc/tree.cc
index 6a055c8..0f02924 100644
--- a/gcc/tree.cc
+++ b/gcc/tree.cc
@@ -32,6 +32,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "coretypes.h"
 #include "backend.h"
 #include "target.h"
+#include "memmodel.h"
+#include "tm_p.h"
 #include "tree.h"
 #include "gimple.h"
 #include "tree-pass.h"
@@ -15418,6 +15420,65 @@ get_target_clone_attr_len (tree arglist)
   return str_len_sum;
 }
 
+/* Returns an auto_vec of string_slices containing the version strings from
+   ARGLIST.  DEFAULT_COUNT is incremented for each default version found.  */
+
+auto_vec<string_slice>
+get_clone_attr_versions (const tree arglist, int *default_count)
+{
+  gcc_assert (TREE_CODE (arglist) == TREE_LIST);
+  auto_vec<string_slice> versions;
+
+  static const char separator_str[] = {TARGET_CLONES_ATTR_SEPARATOR, 0};
+  string_slice separators = string_slice (separator_str);
+
+  for (tree arg = arglist; arg; arg = TREE_CHAIN (arg))
+    {
+      string_slice str = string_slice (TREE_STRING_POINTER (TREE_VALUE (arg)));
+      while (str.is_valid ())
+	{
+	  string_slice attr = string_slice::tokenize (&str, separators);
+	  attr = attr.strip ();
+
+	  if (attr == "default" && default_count)
+	    (*default_count)++;
+	  versions.safe_push (attr);
+	}
+    }
+  return versions;
+}
+
+/* Returns an auto_vec of string_slices containing the version strings from
+   the target_clone attribute from DECL.  DEFAULT_COUNT is incremented for each
+   default version found.  */
+auto_vec<string_slice>
+get_clone_versions (const tree decl, int *default_count)
+{
+  tree attr = lookup_attribute ("target_clones", DECL_ATTRIBUTES (decl));
+  if (!attr)
+    return auto_vec<string_slice> ();
+  tree arglist = TREE_VALUE (attr);
+  return get_clone_attr_versions (arglist, default_count);
+}
+
+/* If DECL has a target_version attribute, returns a string_slice containing the
+   attribute value.  Otherwise, returns string_slice::invalid.
+   Only works for target_version due to target attributes allowing multiple
+   string arguments to specify one target.  */
+string_slice
+get_target_version (const tree decl)
+{
+  gcc_assert (!TARGET_HAS_FMV_TARGET_ATTRIBUTE);
+
+  tree attr = lookup_attribute ("target_version", DECL_ATTRIBUTES (decl));
+
+  if (!attr)
+    return string_slice::invalid ();
+
+  return string_slice (TREE_STRING_POINTER (TREE_VALUE (TREE_VALUE (attr))))
+	   .strip ();
+}
+
 void
 tree_cc_finalize (void)
 {
diff --git a/gcc/tree.h b/gcc/tree.h
index e87fa0f..c0e434b 100644
--- a/gcc/tree.h
+++ b/gcc/tree.h
@@ -22,6 +22,7 @@ along with GCC; see the file COPYING3.  If not see
 
 #include "tree-core.h"
 #include "options.h"
+#include "vec.h"
 
 /* Convert a target-independent built-in function code to a combined_fn.  */
 
@@ -2232,6 +2233,12 @@ class auto_suppress_location_wrappers
 #define OMP_CLAUSE_OPERAND(NODE, I)				\
 	OMP_CLAUSE_ELT_CHECK (NODE, I)
 
+/* True if the clause decl NODE contains an OpenMP iterator.  */
+#define OMP_ITERATOR_DECL_P(NODE) \
+	(TREE_CODE (NODE) == TREE_LIST				\
+	 && TREE_PURPOSE (NODE)					\
+	 && TREE_CODE (TREE_PURPOSE (NODE)) == TREE_VEC)
+
 /* In a BLOCK (scope) node:
    Variables declared in the scope NODE.  */
 #define BLOCK_VARS(NODE) (BLOCK_CHECK (NODE)->block.vars)
@@ -7082,4 +7089,14 @@ extern tree get_attr_nonstring_decl (tree, tree * = NULL);
 
 extern int get_target_clone_attr_len (tree);
 
+/* Returns the version string for a decl with target_version attribute.
+   Returns an invalid string_slice if no attribute is present.  */
+extern string_slice get_target_version (const tree);
+/* Returns a vector of the version strings from a target_clones attribute on
+   a decl.  Can also record the number of default versions found.  */
+extern auto_vec<string_slice> get_clone_versions (const tree, int * = NULL);
+/* Returns a vector of the version strings from a target_clones attribute
+   directly.  */
+extern auto_vec<string_slice> get_clone_attr_versions (const tree, int *);
+
 #endif  /* GCC_TREE_H  */
diff --git a/gcc/value-range.h b/gcc/value-range.h
index 5c358f3..3bc02db 100644
--- a/gcc/value-range.h
+++ b/gcc/value-range.h
@@ -324,7 +324,7 @@ public:
   virtual void update_bitmask (const class irange_bitmask &) override;
   virtual irange_bitmask get_bitmask () const override;
 
-  virtual void verify_range () const;
+  virtual void verify_range () const override;
 protected:
   void maybe_resize (int needed);
   virtual void set (tree, tree, value_range_kind = VR_RANGE) override;
@@ -422,7 +422,7 @@ public:
   bool contains_p (const wide_int &) const;
   wide_int lower_bound () const;
   wide_int upper_bound () const;
-  virtual void verify_range () const;
+  virtual void verify_range () const final override;
   irange_bitmask get_bitmask () const final override;
   void update_bitmask (const irange_bitmask &) final override;
 protected:
@@ -594,7 +594,7 @@ public:
   bool nan_signbit_p (bool &signbit) const;
   bool known_isnormal () const;
   bool known_isdenormal_or_zero () const;
-  virtual void verify_range () const;
+  virtual void verify_range () const override;
 protected:
   virtual bool contains_p (tree cst) const override;
   virtual void set (tree, tree, value_range_kind = VR_RANGE) override;
diff --git a/gcc/varasm.cc b/gcc/varasm.cc
index 10c1d2e..000ad9e 100644
--- a/gcc/varasm.cc
+++ b/gcc/varasm.cc
@@ -871,7 +871,7 @@ mergeable_string_section (tree decl ATTRIBUTE_UNUSED,
   if (HAVE_GAS_SHF_MERGE && flag_merge_constants
       && TREE_CODE (decl) == STRING_CST
       && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
-      && align <= 256
+      && align <= MAX_ALIGN_MERGABLE
       && (len = int_size_in_bytes (TREE_TYPE (decl))) > 0
       && TREE_STRING_LENGTH (decl) == len)
     {
@@ -885,7 +885,7 @@ mergeable_string_section (tree decl ATTRIBUTE_UNUSED,
 
       mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (TREE_TYPE (decl)));
       modesize = GET_MODE_BITSIZE (mode);
-      if (modesize >= 8 && modesize <= 256
+      if (modesize >= 8 && modesize <= MAX_ALIGN_MERGABLE
 	  && (modesize & (modesize - 1)) == 0)
 	{
 	  if (align < modesize)
@@ -919,16 +919,14 @@ mergeable_string_section (tree decl ATTRIBUTE_UNUSED,
 /* Return the section to use for constant merging.  */
 
 section *
-mergeable_constant_section (machine_mode mode ATTRIBUTE_UNUSED,
-			    unsigned HOST_WIDE_INT align ATTRIBUTE_UNUSED,
-			    unsigned int flags ATTRIBUTE_UNUSED)
+mergeable_constant_section (unsigned HOST_WIDE_INT size_bits,
+			    unsigned HOST_WIDE_INT align,
+			    unsigned int flags)
 {
   if (HAVE_GAS_SHF_MERGE && flag_merge_constants
-      && mode != VOIDmode
-      && mode != BLKmode
-      && known_le (GET_MODE_BITSIZE (mode), align)
+      && size_bits <= align
       && align >= 8
-      && align <= 256
+      && align <= MAX_ALIGN_MERGABLE
       && (align & (align - 1)) == 0)
     {
       const char *prefix = function_mergeable_rodata_prefix ();
@@ -940,6 +938,38 @@ mergeable_constant_section (machine_mode mode ATTRIBUTE_UNUSED,
     }
   return readonly_data_section;
 }
+
+
+/* Return the section to use for constant merging. Like the above
+   but the size stored as a tree.  */
+static section *
+mergeable_constant_section (tree size_bits,
+			    unsigned HOST_WIDE_INT align,
+			    unsigned int flags)
+{
+  if (!size_bits || !tree_fits_uhwi_p (size_bits))
+    return readonly_data_section;
+  return mergeable_constant_section (tree_to_uhwi (size_bits), align, flags);
+}
+
+
+/* Return the section to use for constant merging. Like the above
+   but given a mode rather than the size.  */
+
+section *
+mergeable_constant_section (machine_mode mode,
+			    unsigned HOST_WIDE_INT align,
+			    unsigned int flags)
+{
+  /* If the mode is unknown (BLK or VOID), then return a non mergable section.  */
+  if (mode == BLKmode || mode == VOIDmode)
+    return readonly_data_section;
+  unsigned HOST_WIDE_INT size;
+  if (!GET_MODE_BITSIZE (mode).is_constant (&size))
+    return readonly_data_section;
+  return mergeable_constant_section (size, align, flags);
+}
+
 
 /* Given NAME, a putative register name, discard any customary prefixes.  */
 
@@ -1734,7 +1764,8 @@ assemble_asm (tree asm_str)
 	  constraints[i]
 	    = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (tail)));
 	  if (!parse_output_constraint (&constraints[i], i, ninputs, noutputs,
-					&allows_mem, &allows_reg, &is_inout))
+					&allows_mem, &allows_reg, &is_inout,
+					nullptr))
 	    goto done;
 	  if (is_inout)
 	    {
@@ -1776,7 +1807,7 @@ assemble_asm (tree asm_str)
 	    = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (tail)));
 	  if (!parse_input_constraint (&constraints[i + noutputs], i,
 				       ninputs, noutputs, 0, constraints,
-				       &allows_mem, &allows_reg))
+				       &allows_mem, &allows_reg, nullptr))
 	    goto done;
 	  if (strchr (constraints[i], '%'))
 	    {
@@ -2907,7 +2938,7 @@ assemble_name_resolve (const char *name)
       ultimate_transparent_alias_target (&id);
       if (id != id_orig)
 	name = IDENTIFIER_POINTER (id);
-      gcc_assert (! TREE_CHAIN (id));
+      gcc_assert (!IDENTIFIER_TRANSPARENT_ALIAS (id));
     }
 
   return name;
@@ -7452,7 +7483,7 @@ default_elf_select_section (tree decl, int reloc,
     case SECCAT_RODATA_MERGE_STR_INIT:
       return mergeable_string_section (DECL_INITIAL (decl), align, 0);
     case SECCAT_RODATA_MERGE_CONST:
-      return mergeable_constant_section (DECL_MODE (decl), align, 0);
+      return mergeable_constant_section (DECL_SIZE (decl), align, 0);
     case SECCAT_SRODATA:
       sname = ".sdata2";
       break;
diff --git a/gcc/vec.cc b/gcc/vec.cc
index 55f5f3d..38314d7 100644
--- a/gcc/vec.cc
+++ b/gcc/vec.cc
@@ -176,6 +176,74 @@ dump_vec_loc_statistics (void)
   vec_mem_desc.dump (VEC_ORIGIN);
 }
 
+/* Gets the next token from STR delimited by DELIMS (deliminator not included
+   in returned string).
+
+   Updates STR to be the remaining string after the given token.
+
+   STR and DELIMS must both be valid string_slices.
+
+   If there aren't any of the chars in DELIM in STR (ie no more tokens in STR)
+   then returns the string, and updates STR to be invalid.  */
+string_slice
+string_slice::tokenize (string_slice *str, string_slice delims)
+{
+  const char *ptr = str->begin ();
+
+  gcc_assert (str->is_valid () && delims.is_valid ());
+
+  for (; ptr < str->end (); ptr++)
+    for (char c : delims)
+      if (*ptr == c)
+	{
+	  /* Update the input string to be the remaining string.  */
+	  const char *str_begin = str->begin ();
+	  *str = string_slice (ptr  + 1, str->end ());
+	  return string_slice (str_begin, ptr);
+	}
+
+  /* If no deliminators between the start and end, return the whole string.  */
+  string_slice res = *str;
+  *str = string_slice::invalid ();
+  return res;
+}
+
+/* Compares the string_slices STR1 and STR2 giving a lexograpical ordering.
+   Returns -1 if STR1 comes before STR2, 1 if STR1 comes after, and 0 if the
+   string_slices have the same contents.  */
+
+int
+string_slice::strcmp (string_slice str1, string_slice str2)
+{
+  for (unsigned int i = 0; i < str1.size () && i < str2.size (); i++)
+    {
+      if (str1[i] < str2[i])
+	return -1;
+      if (str1[i] > str2[i])
+	return 1;
+    }
+
+  if (str1.size () < str2.size ())
+    return -1;
+  if (str1.size () > str2.size ())
+    return 1;
+  return 0;
+}
+
+string_slice
+string_slice::strip ()
+{
+  const char *start = this->begin ();
+  const char *end = this->end ();
+
+  while (start < end && ISSPACE (*start))
+    start++;
+  while (end > start && ISSPACE (*(end-1)))
+    end--;
+
+  return string_slice (start, end);
+}
+
 #if CHECKING_P
 /* Report qsort comparator CMP consistency check failure with P1, P2, P3 as
    witness elements.  */
@@ -584,6 +652,159 @@ test_auto_alias ()
   ASSERT_EQ (val, 0);
 }
 
+static void
+test_string_slice_initializers ()
+{
+  string_slice str1 = string_slice ();
+  ASSERT_TRUE (str1.is_valid ());
+  ASSERT_EQ (str1.size (), 0);
+
+  string_slice str2 = string_slice ("Test string");
+  ASSERT_TRUE (str2.is_valid ());
+  ASSERT_EQ (str2.size (), 11);
+
+  string_slice str3 = "Test string the second";
+  ASSERT_TRUE (str3.is_valid ());
+  ASSERT_EQ (str3.size (), 22);
+
+  string_slice str4 = string_slice ("Test string", 4);
+  ASSERT_TRUE (str4.is_valid ());
+  ASSERT_EQ (str4.size (), 4);
+}
+
+static void
+test_string_slice_tokenize ()
+{
+  string_slice test_string_slice = "";
+  string_slice test_delims = ",";
+
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), "");
+  ASSERT_FALSE (test_string_slice.is_valid ());
+
+  test_string_slice = ",";
+  test_delims = ",";
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     string_slice (""));
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     string_slice (""));
+  ASSERT_FALSE (test_string_slice.is_valid ());
+
+  test_string_slice = ",test.,.test, ,  test  ";
+  test_delims = ",.";
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), "");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), "test");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), "");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), "");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), "test");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims), " ");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     "  test  ");
+  ASSERT_FALSE (test_string_slice.is_valid ());
+
+  const char *test_string
+    = "This is the test string, it \0 is for testing, 123 ,,";
+  test_string_slice = string_slice (test_string, 52);
+  test_delims = string_slice (",\0", 2);
+
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     "This is the test string");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     " it ");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     " is for testing");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     " 123 ");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     "");
+  ASSERT_EQ (string_slice::tokenize (&test_string_slice, test_delims),
+	     "");
+  ASSERT_FALSE (test_string_slice.is_valid ());
+}
+
+static void
+test_string_slice_strcmp ()
+{
+  ASSERT_EQ (string_slice::strcmp (string_slice (),
+				   string_slice ()), 0);
+  ASSERT_EQ (string_slice::strcmp (string_slice ("test"),
+				   string_slice ()), 1);
+  ASSERT_EQ (string_slice::strcmp (string_slice (),
+				   string_slice ("test")), -1);
+  ASSERT_EQ (string_slice::strcmp (string_slice ("test"),
+				   string_slice ("test")), 0);
+  ASSERT_EQ (string_slice::strcmp (string_slice ("a"),
+				   string_slice ("b")), -1);
+  ASSERT_EQ (string_slice::strcmp (string_slice ("b"),
+				   string_slice ("a")), 1);
+  ASSERT_EQ (string_slice::strcmp (string_slice ("ab", 1),
+				   string_slice ("a")), 0);
+  ASSERT_EQ (string_slice::strcmp (string_slice ("ab", 2),
+				   string_slice ("a")), 1);
+}
+
+static void
+test_string_slice_equality ()
+{
+  ASSERT_TRUE (string_slice () == string_slice ());
+  ASSERT_FALSE (string_slice ("test") == string_slice ());
+  ASSERT_FALSE ("test" == string_slice ());
+  ASSERT_FALSE (string_slice () == string_slice ("test"));
+  ASSERT_FALSE (string_slice () == "test");
+  ASSERT_TRUE (string_slice ("test") == string_slice ("test"));
+  ASSERT_TRUE ("test" == string_slice ("test"));
+  ASSERT_TRUE (string_slice ("test") == "test");
+  ASSERT_FALSE (string_slice ("a") == string_slice ("b"));
+  ASSERT_FALSE ("a" == string_slice ("b"));
+  ASSERT_FALSE (string_slice ("a") == "b");
+  ASSERT_FALSE (string_slice ("b") == string_slice ("a"));
+  ASSERT_TRUE (string_slice ("ab", 1) == string_slice ("a"));
+  ASSERT_TRUE (string_slice ("ab", 1) == "a");
+  ASSERT_FALSE (string_slice ("ab", 2) == string_slice ("a"));
+  ASSERT_FALSE (string_slice ("ab", 2) == "a");
+}
+
+static void
+test_string_slice_inequality ()
+{
+  ASSERT_FALSE (string_slice () != string_slice ());
+  ASSERT_TRUE (string_slice ("test") != string_slice ());
+  ASSERT_TRUE ("test" != string_slice ());
+  ASSERT_TRUE (string_slice () != string_slice ("test"));
+  ASSERT_TRUE (string_slice () != "test");
+  ASSERT_FALSE (string_slice ("test") != string_slice ("test"));
+  ASSERT_FALSE ("test" != string_slice ("test"));
+  ASSERT_FALSE (string_slice ("test") != "test");
+  ASSERT_TRUE (string_slice ("a") != string_slice ("b"));
+  ASSERT_TRUE ("a" != string_slice ("b"));
+  ASSERT_TRUE (string_slice ("a") != "b");
+  ASSERT_TRUE (string_slice ("b") != string_slice ("a"));
+  ASSERT_FALSE (string_slice ("ab", 1) != string_slice ("a"));
+  ASSERT_FALSE (string_slice ("ab", 1) != "a");
+  ASSERT_TRUE (string_slice ("ab", 2) != string_slice ("a"));
+  ASSERT_TRUE (string_slice ("ab", 2) != "a");
+}
+
+static void
+test_string_slice_invalid ()
+{
+  ASSERT_FALSE (string_slice::invalid ().is_valid ());
+  ASSERT_FALSE (string_slice (NULL, 1).is_valid ());
+  ASSERT_TRUE (string_slice (NULL, (size_t) 0).is_valid ());
+  ASSERT_TRUE (string_slice ("Test", (size_t) 0).is_valid ());
+  ASSERT_TRUE (string_slice ().is_valid ());
+}
+
+static void
+test_string_slice_strip ()
+{
+  ASSERT_EQ (string_slice ("   test   ").strip (), string_slice ("test"));
+  ASSERT_EQ (string_slice ("\t   test string\t   \n ").strip (),
+	     string_slice ("test string"));
+  ASSERT_EQ (string_slice ("test").strip (), string_slice ("test"));
+  ASSERT_EQ (string_slice ().strip (), string_slice ());
+  ASSERT_EQ (string_slice ("\t  \n \t   ").strip (), string_slice ());
+}
+
 /* Run all of the selftests within this file.  */
 
 void
@@ -604,6 +825,13 @@ vec_cc_tests ()
   test_reverse ();
   test_auto_delete_vec ();
   test_auto_alias ();
+  test_string_slice_initializers ();
+  test_string_slice_tokenize ();
+  test_string_slice_strcmp ();
+  test_string_slice_equality ();
+  test_string_slice_inequality ();
+  test_string_slice_invalid ();
+  test_string_slice_strip ();
 }
 
 } // namespace selftest
diff --git a/gcc/vec.h b/gcc/vec.h
index 7e112d1..0ea7a49 100644
--- a/gcc/vec.h
+++ b/gcc/vec.h
@@ -2495,4 +2495,54 @@ make_array_slice (T *base, unsigned int size)
 # pragma GCC poison m_vec m_vecpfx m_vecdata
 #endif
 
+/* string_slice inherits from array_slice, specifically to refer to a substring
+   of a character array.
+   It includes some string like helpers.  */
+class string_slice : public array_slice<const char>
+{
+public:
+  string_slice () : array_slice<const char> () {}
+  string_slice (const char *str) : array_slice (str, strlen (str)) {}
+  explicit string_slice (const char *str, size_t len)
+    : array_slice (str, len) {}
+  explicit string_slice (const char *start, const char *end)
+    : array_slice (start, end - start) {}
+
+  friend bool operator== (const string_slice &lhs, const string_slice &rhs)
+  {
+    if (!lhs.is_valid () || !rhs.is_valid ())
+      return false;
+    if (lhs.size () != rhs.size ())
+      return false;
+    /* Case where either is a NULL pointer and therefore, as both are valid,
+       both are empty slices with length 0.  */
+    if (lhs.size () == 0)
+      return true;
+    return memcmp (lhs.begin (), rhs.begin (), lhs.size ()) == 0;
+  }
+
+  friend bool operator!= (const string_slice &lhs, const string_slice &rhs)
+  {
+    return !(lhs == rhs);
+  }
+
+  /* Returns an invalid string_slice.  */
+  static string_slice invalid ()
+  {
+    return string_slice (nullptr, ~0U);
+  }
+
+  /* tokenize is used to split a string by some deliminator into
+     string_slice's.  Similarly to the posix strtok_r.but without modifying the
+     input string, and returning all tokens which may be empty in the case
+     of an empty input string of consecutive deliminators.  */
+  static string_slice tokenize (string_slice *str, string_slice delims);
+
+  /* Removes white space from the front and back of the string_slice.  */
+  string_slice strip ();
+
+  /* Compares two string_slices in lexographical ordering.  */
+  static int strcmp (string_slice str1, string_slice str2);
+};
+
 #endif // GCC_VEC_H
diff --git a/gcc/warning-control.cc b/gcc/warning-control.cc
index 48446a7..e04ef69 100644
--- a/gcc/warning-control.cc
+++ b/gcc/warning-control.cc
@@ -28,7 +28,7 @@
 #include "gimple.h"
 #include "cgraph.h"
 #include "hash-map.h"
-#include "diagnostic-spec.h"
+#include "gcc-diagnostic-spec.h"
 
 /* Return the no-warning bit for EXPR.  */
 
diff --git a/include/ChangeLog b/include/ChangeLog
index b11102d..b3fdd3c 100644
--- a/include/ChangeLog
+++ b/include/ChangeLog
@@ -1,3 +1,7 @@
+2025-07-09  Matthieu Longo  <matthieu.longo@arm.com>
+
+	* doubly-linked-list.h: New file.
+
 2025-06-20  Sosutha Sethuramapandian  <ssethuramapa@marvell.com>
 
 	PR target/110181
diff --git a/include/doubly-linked-list.h b/include/doubly-linked-list.h
new file mode 100644
index 0000000..3f5ea28
--- /dev/null
+++ b/include/doubly-linked-list.h
@@ -0,0 +1,447 @@
+/* Manipulate doubly linked lists.
+   Copyright (C) 2025 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+
+#ifndef _DOUBLY_LINKED_LIST_H
+#define _DOUBLY_LINKED_LIST_H
+
+/* Doubly linked list implementation enforcing typing.
+
+   This implementation of doubly linked list tries to achieve the enforcement of
+   typing similarly to C++ templates, but without encapsulation.
+
+   All the functions are prefixed with the type of the value: "AType_xxx".
+   Some functions are prefixed with "_AType_xxx" and are not part of the public
+   API, so should not be used, except for _##LTYPE##_merge_sort with a caveat
+   (see note above its definition).
+
+   Each function (### is a placeholder for method name) has a macro for:
+   (1) its invocation LINKED_LIST_###(LTYPE).
+   (2) its prototype LINKED_LIST_DECL_###(A, A2, scope). To add in a header
+       file, or a source file for forward declaration. 'scope' should be set
+       respectively to 'extern', or 'static'.
+   (3) its definition LINKED_LIST_DEFN_###(A, A2, scope). To add in a source
+       file with the 'scope' set respectively to nothing, or 'static' depending
+       on (2).
+
+   Data structures requirements:
+   - LTYPE corresponds to the node of a doubly linked list. It needs to define
+     attributes 'prev' and 'next' which are pointers on the type of a node.
+     For instance:
+       struct my_list_node
+       {
+	 T value;
+	 struct my_list_node *prev;
+	 struct my_list_node *next;
+       };
+   - LWRAPPERTYPE is a structure wrapping the nodes and others metadata (first,
+     last, size).
+ */
+
+
+/* Mutative operations:
+    - append
+    - prepend
+    - insert_before
+    - pop_front
+    - pop_back
+    - remove
+    - swap
+   The header and body of each of those operation can be declared individually,
+   or as a whole via LINKED_LIST_MUTATIVE_OPS_PROTOTYPE for the prototypes, and
+   LINKED_LIST_MUTATIVE_OPS_DECL for the implementations.  */
+
+/* Append the given node new_ to the exising list.
+   Precondition: prev and next of new_ must be NULL.  */
+#define LINKED_LIST_APPEND(LTYPE)		LTYPE##_append
+
+#define LINKED_LIST_DECL_APPEND(LWRAPPERTYPE, LTYPE, EXPORT)		\
+  EXPORT void								\
+  LTYPE##_append (LWRAPPERTYPE *wrapper, LTYPE *new_)
+
+#define LINKED_LIST_DEFN_APPEND(LWRAPPERTYPE, LTYPE, EXPORT)		\
+EXPORT void								\
+LTYPE##_append (LWRAPPERTYPE *wrapper, LTYPE *new_)			\
+{									\
+  if (wrapper->last == NULL)						\
+    wrapper->first = new_;						\
+  else									\
+    {									\
+      new_->prev = wrapper->last;					\
+      wrapper->last->next = new_;					\
+    }									\
+  wrapper->last = new_;							\
+  ++wrapper->size;							\
+}
+
+/* Prepend the given node new_ to the existing list.
+   Precondition: prev and next of new_ must be NULL.  */
+#define LINKED_LIST_PREPEND(LTYPE)		LTYPE##_prepend
+
+#define LINKED_LIST_DECL_PREPEND(LWRAPPERTYPE, LTYPE, EXPORT)		\
+  EXPORT void								\
+  LTYPE##_prepend (LWRAPPERTYPE *wrapper, LTYPE *new_)
+
+#define LINKED_LIST_DEFN_PREPEND(LWRAPPERTYPE, LTYPE, EXPORT)		\
+EXPORT void								\
+LTYPE##_prepend (LWRAPPERTYPE *wrapper, LTYPE *new_)			\
+{									\
+  if (wrapper->first == NULL)						\
+    wrapper->last = new_;						\
+  else									\
+    {									\
+      new_->next = wrapper->first;					\
+      wrapper->first->prev = new_;					\
+    }									\
+  wrapper->first = new_;						\
+  ++wrapper->size;							\
+}
+
+/* Insert the given node new_ before 'where' in the existing list.
+   If where == NULL, the insertion is equivalent to an append.
+   If where == first, the insertion is equivalent to a prepend.  */
+#define LINKED_LIST_INSERT_BEFORE(LTYPE)	LTYPE##_insert_before
+
+#define LINKED_LIST_DECL_INSERT_BEFORE(LWRAPPERTYPE, LTYPE, EXPORT)	\
+  EXPORT void								\
+  LTYPE##_insert_before (LWRAPPERTYPE *wrapper,				\
+			 LTYPE *new_,					\
+			 LTYPE *where)
+
+#define LINKED_LIST_DEFN_INSERT_BEFORE(LWRAPPERTYPE, LTYPE, EXPORT)	\
+EXPORT void								\
+LTYPE##_insert_before (LWRAPPERTYPE *wrapper,				\
+		       LTYPE *new_,					\
+		       LTYPE *where)					\
+{									\
+  if (where == wrapper->first)						\
+    LTYPE##_prepend (wrapper, new_);					\
+  else if (where == NULL)						\
+    LTYPE##_append (wrapper, new_);					\
+  else									\
+    {									\
+      where->prev->next = new_;						\
+      new_->prev = where->prev;						\
+      where->prev = new_;						\
+      new_->next = where;						\
+      ++wrapper->size;							\
+    }									\
+}
+
+/* Pop the first node of the list.  */
+#define LINKED_LIST_POP_FRONT(LTYPE)		LTYPE##_pop_front
+
+#define LINKED_LIST_DECL_POP_FRONT(LWRAPPERTYPE, LTYPE, EXPORT)		\
+  EXPORT LTYPE *							\
+  LTYPE##_pop_front (LWRAPPERTYPE *wrapper)
+
+#define LINKED_LIST_DEFN_POP_FRONT(LWRAPPERTYPE, LTYPE, EXPORT)		\
+EXPORT LTYPE *								\
+LTYPE##_pop_front (LWRAPPERTYPE *wrapper)				\
+{									\
+  LTYPE *front_node = wrapper->first;					\
+  if (front_node != NULL)						\
+    {									\
+      wrapper->first = front_node->next;				\
+      if (wrapper->last == front_node)					\
+	wrapper->last = NULL;						\
+      else								\
+	{								\
+	  front_node->next->prev = NULL;				\
+	  front_node->next = NULL;					\
+	}								\
+      front_node->next = NULL;						\
+      --wrapper->size;							\
+    }									\
+  return front_node;							\
+}
+
+/* Pop the last node of the list.  */
+#define LINKED_LIST_POP_BACK(LTYPE)		LTYPE##_pop_back
+
+#define LINKED_LIST_DECL_POP_BACK(LWRAPPERTYPE, LTYPE, EXPORT)		\
+  EXPORT LTYPE *							\
+  LTYPE##_pop_back (LWRAPPERTYPE *wrapper)
+
+#define LINKED_LIST_DEFN_POP_BACK(LWRAPPERTYPE, LTYPE, EXPORT)		\
+EXPORT LTYPE *								\
+LTYPE##_pop_back (LWRAPPERTYPE *wrapper)				\
+{									\
+  LTYPE *back_node = wrapper->last;					\
+  if (back_node != NULL)						\
+    {									\
+      wrapper->last = back_node->prev;					\
+      if (wrapper->first == back_node)					\
+	wrapper->first = NULL;						\
+      else								\
+	{								\
+	  back_node->prev->next = NULL;					\
+	  back_node->prev = NULL;					\
+	}								\
+      back_node->prev = NULL;						\
+      --wrapper->size;							\
+    }									\
+  return back_node;							\
+}
+
+/* Remove the given node from the existing list, and return the previous
+   node.  */
+#define LINKED_LIST_REMOVE(LTYPE)		LTYPE##_remove
+
+#define LINKED_LIST_DECL_REMOVE(LWRAPPERTYPE, LTYPE, EXPORT)		\
+  EXPORT LTYPE *							\
+  LTYPE##_remove (LWRAPPERTYPE *wrapper, LTYPE *node)
+
+#define LINKED_LIST_DEFN_REMOVE(LWRAPPERTYPE, LTYPE, EXPORT)		\
+EXPORT LTYPE *								\
+LTYPE##_remove (LWRAPPERTYPE *wrapper, LTYPE *node)			\
+{									\
+  LTYPE *previous = NULL;						\
+									\
+  if (node->prev != NULL)						\
+    {									\
+      node->prev->next = node->next;					\
+      if (node->next == NULL)						\
+	wrapper->last = node->prev;					\
+      else								\
+	node->next->prev = node->prev;					\
+      previous = node->prev;						\
+      node->next = NULL;						\
+      node->prev = NULL;						\
+      --wrapper->size;							\
+    }									\
+  else									\
+    LTYPE##_pop_front (wrapper);					\
+									\
+  return previous;							\
+}
+
+/* Generic swap.  */
+#define LINKED_LIST_SWAP(LTYPE)			LTYPE##_swap
+
+#define LINKED_LIST_DECL_SWAP(LWRAPPERTYPE, LTYPE, EXPORT)		\
+  EXPORT void								\
+  LTYPE##_swap (LWRAPPERTYPE *wrapper, LTYPE *node1, LTYPE *node2)
+
+/* Swap two nodes in a list.  */
+#define LINKED_LIST_DEFN_SWAP(LWRAPPERTYPE, LTYPE, EXPORT)		\
+EXPORT void								\
+LTYPE##_swap (LWRAPPERTYPE *wrapper, LTYPE *node1, LTYPE *node2)	\
+{									\
+  LTYPE *prev1 = node1->prev;						\
+  LTYPE *next1 = node1->next;						\
+  LTYPE *prev2 = node2->prev;						\
+  LTYPE *next2 = node2->next;						\
+									\
+  if (prev1 != NULL)							\
+    prev1->next = node2;						\
+  else									\
+    wrapper->first = node2;						\
+  if (prev2 != NULL)							\
+    prev2->next = node1;						\
+  else									\
+    wrapper->first = node1;						\
+									\
+  if (next1 != NULL)							\
+    next1->prev = node2;						\
+  else									\
+    wrapper->last = node2;						\
+  if (next2 != NULL)							\
+    next2->prev = node1;						\
+  else									\
+    wrapper->last = node1;						\
+									\
+  {									\
+    LTYPE *temp = node1->next;						\
+    node1->next = node2->next;						\
+    node2->next = temp;							\
+  }									\
+  {									\
+    LTYPE *temp = node1->prev;						\
+    node1->prev = node2->prev;						\
+    node2->prev = temp;							\
+  }									\
+}
+
+/* Note: all the mutative operations below also update the data in the wrapper,
+   i.e. first, last and size.  */
+#define LINKED_LIST_MUTATIVE_OPS_PROTOTYPE(LWRAPPERTYPE, LTYPE, EXPORT)	\
+  LINKED_LIST_DECL_APPEND(LWRAPPERTYPE, LTYPE, EXPORT);			\
+  LINKED_LIST_DECL_PREPEND(LWRAPPERTYPE, LTYPE, EXPORT);		\
+  LINKED_LIST_DECL_INSERT_BEFORE(LWRAPPERTYPE, LTYPE, EXPORT);		\
+  LINKED_LIST_DECL_POP_FRONT(LWRAPPERTYPE, LTYPE, EXPORT);		\
+  LINKED_LIST_DECL_POP_BACK(LWRAPPERTYPE, LTYPE, EXPORT);		\
+  LINKED_LIST_DECL_REMOVE(LWRAPPERTYPE, LTYPE, EXPORT);			\
+  LINKED_LIST_DECL_SWAP(LWRAPPERTYPE, LTYPE, EXPORT)
+
+#define LINKED_LIST_MUTATIVE_OPS_DECL(LWRAPPERTYPE, LTYPE, EXPORT)	\
+  LINKED_LIST_DEFN_APPEND(LWRAPPERTYPE, LTYPE, EXPORT)			\
+  LINKED_LIST_DEFN_PREPEND(LWRAPPERTYPE, LTYPE, EXPORT)			\
+  LINKED_LIST_DEFN_INSERT_BEFORE(LWRAPPERTYPE, LTYPE, EXPORT)		\
+  LINKED_LIST_DEFN_POP_FRONT(LWRAPPERTYPE, LTYPE, EXPORT)		\
+  LINKED_LIST_DEFN_POP_BACK(LWRAPPERTYPE, LTYPE, EXPORT)		\
+  LINKED_LIST_DEFN_REMOVE(LWRAPPERTYPE, LTYPE, EXPORT)			\
+  LINKED_LIST_DEFN_SWAP(LWRAPPERTYPE, LTYPE, EXPORT)
+
+
+/* Sorting.  */
+
+#define LINKED_LIST_MERGE_SORT_(LTYPE)	LTYPE##_merge_sort_
+
+#define LINKED_LIST_MERGE_SORT(LTYPE)	LTYPE##_merge_sort
+
+#define LINKED_LIST_MERGE_SORT_PROTOTYPE_(LTYPE, EXPORT)		\
+  EXPORT LTYPE *							\
+  LTYPE##_merge_sort_ (LTYPE *node,					\
+		       int (*fn_cmp) (const LTYPE *, const LTYPE *))
+
+#define LINKED_LIST_MERGE_SORT_PROTOTYPE(LWRAPPERTYPE, LTYPE, EXPORT)	\
+  EXPORT void								\
+  LTYPE##_merge_sort (LWRAPPERTYPE *wrapper,				\
+		      int (*fn_cmp) (const LTYPE *, const LTYPE *))
+
+/* Note: all the functions and macros below starting with "_" should be
+   considered private.  */
+
+/* Compute the middle element of the list based on the turtle and hare
+   approach, i.e. the hare runs twice faster than the turtle.  */
+#define _MERGE_SORT_IMPL_COMPUTE_TURTLE(LTYPE)				\
+static inline LTYPE *							\
+LTYPE##_merge_sort_compute_turtle_ (LTYPE *node)			\
+{									\
+  if (node == NULL)							\
+    return node;							\
+									\
+  LTYPE *turtle = node, *hare = node->next;				\
+  while (hare != NULL && hare->next != NULL)				\
+    {									\
+      turtle = turtle->next;						\
+      hare = hare->next->next;						\
+    }									\
+  return turtle;							\
+}
+
+/* Append n at the end of l_out, and return the next node after n.
+   l_out and l_last should be ideally encapsulated into a list structure
+   but this is overkill for what we need here.  */
+#define _MERGE_SORT_IMPL_OUT_APPEND(LTYPE)				\
+static inline LTYPE *							\
+LTYPE##_merge_sort_out_append_ (LTYPE **l_out, LTYPE **l_last,		\
+				LTYPE *n)				\
+{									\
+  if (*l_last == NULL)							\
+    {									\
+      *l_last = n;							\
+      *l_out = n;							\
+      n->prev = NULL;							\
+    }									\
+  else									\
+    {									\
+      (*l_last)->next = n;						\
+      n->prev = *l_last;						\
+      *l_last = n;							\
+    }									\
+									\
+  return n->next;							\
+}
+
+/* Merge two sorted lists together.
+   The returned value corresponds to the first element of the list.
+   Note: both input lists are invalidated after the call.  */
+#define _MERGE_SORT_IMPL_MERGE(LTYPE)					\
+static inline LTYPE *							\
+LTYPE##_merge_sort_merge_ (LTYPE *l_left, LTYPE *l_right,		\
+			   int (*fn_cmp) (const LTYPE *, const LTYPE *))\
+{									\
+  if (l_left == NULL)							\
+    return l_right;							\
+  else if (l_right == NULL)						\
+    return l_left;							\
+									\
+  LTYPE *l_out = NULL, *l_last = NULL;					\
+									\
+  LTYPE *l_l = l_left, *l_r = l_right;					\
+  while (l_l != NULL && l_r != NULL)					\
+    {									\
+      int cmp = fn_cmp (l_l, l_r);					\
+      if (cmp <= 0)							\
+	l_l = LTYPE##_merge_sort_out_append_ (&l_out, &l_last, l_l);	\
+      else								\
+	l_r = LTYPE##_merge_sort_out_append_ (&l_out, &l_last, l_r);	\
+    }									\
+									\
+  LTYPE *l_remaining = (l_l != NULL) ? l_l : l_r;			\
+  while (l_remaining != NULL)						\
+    l_remaining =							\
+      LTYPE##_merge_sort_out_append_ (&l_out, &l_last, l_remaining);	\
+									\
+  return l_out;								\
+}
+
+/* Merge sort implementation taking the first node of the list to sort,
+   and the comparison function. Returns the first node of the sorted list.
+   Note: use this if you don't care about updating the information in the
+   wrapper.  */
+#define _MERGE_SORT_DEFN_SORT(LTYPE, EXPORT)				\
+EXPORT LTYPE *								\
+LTYPE##_merge_sort_ (LTYPE *node,					\
+		     int (*fn_cmp)(const LTYPE *, const LTYPE *))	\
+{									\
+  if (node == NULL)							\
+    return NULL;							\
+  else if (node->next == NULL)						\
+    return node;							\
+									\
+  LTYPE *left_end = LTYPE##_merge_sort_compute_turtle_ (node);		\
+  LTYPE *left_begin = node;						\
+  LTYPE *right_begin = left_end->next;					\
+  /* break the list. */							\
+  left_end->next = NULL;						\
+  right_begin->prev = NULL;						\
+									\
+  left_begin = LTYPE##_merge_sort_ (left_begin, fn_cmp);		\
+  right_begin = LTYPE##_merge_sort_ (right_begin, fn_cmp);		\
+  return LTYPE##_merge_sort_merge_ (left_begin, right_begin, fn_cmp);	\
+}
+
+/* Merge sort wrapper that the end-user should be using as it updates the
+   first and last metadata of the list in wrapper as well.
+   If the user does not want to pay the cost of the update of the data,
+   it can directly use _##LTYPE##_merge_sort_merge.  */
+#define _MERGE_SORT_DEFN_WRAPPER_SORT(LWRAPPERTYPE, LTYPE, EXPORT)	\
+EXPORT void								\
+LTYPE##_merge_sort (LWRAPPERTYPE *wrapper,				\
+		    int (*fn_cmp) (const LTYPE *, const LTYPE *))	\
+{									\
+  wrapper->first = LTYPE##_merge_sort_ (wrapper->first, fn_cmp);	\
+									\
+  if (wrapper->first == NULL || wrapper->first->next == NULL)		\
+    wrapper->last = wrapper->first;					\
+  else									\
+    for (LTYPE *node = wrapper->first;					\
+	 node != NULL;							\
+	 node = node->next)						\
+      wrapper->last = node;						\
+}
+
+#define LINKED_LIST_MERGE_SORT_DECL(LWRAPPERTYPE, LTYPE, EXPORT)	\
+  _MERGE_SORT_IMPL_COMPUTE_TURTLE(LTYPE)				\
+  _MERGE_SORT_IMPL_OUT_APPEND(LTYPE)					\
+  _MERGE_SORT_IMPL_MERGE(LTYPE)						\
+  _MERGE_SORT_DEFN_SORT(LTYPE, EXPORT)					\
+  _MERGE_SORT_DEFN_WRAPPER_SORT(LWRAPPERTYPE, LTYPE, EXPORT)
+
+#endif /* _DOUBLY_LINKED_LIST_H */
diff --git a/libatomic/ChangeLog b/libatomic/ChangeLog
index 6716f04..f9bbaae 100644
--- a/libatomic/ChangeLog
+++ b/libatomic/ChangeLog
@@ -1,3 +1,8 @@
+2025-07-31  Yury Khrustalev  <yury.khrustalev@arm.com>
+
+	* config/linux/aarch64/host-config.h (__ifunc_arg_t):
+	Remove sys/ifunc.h and add new fields _hwcap3 and _hwcap4.
+
 2025-05-22  Alexandre Oliva  <oliva@adacore.com>
 
 	* configure.tgt: Set partial_libatomic on *-*-vxworks*.
diff --git a/libatomic/config/linux/aarch64/host-config.h b/libatomic/config/linux/aarch64/host-config.h
index d0d44bf..c6f8d69 100644
--- a/libatomic/config/linux/aarch64/host-config.h
+++ b/libatomic/config/linux/aarch64/host-config.h
@@ -40,16 +40,20 @@
 # define HWCAP2_LSE128	(1UL << 47)
 #endif
 
-#if __has_include(<sys/ifunc.h>)
-# include <sys/ifunc.h>
-#else
+/* The following struct is ABI-correct description of the 2nd argument for an
+   ifunc resolver as per SYSVABI spec (see link below).  It is safe to extend
+   it with new fields.  The ifunc resolver implementations must always check
+   the runtime size of the buffer using the value in the _size field.
+   https://github.com/ARM-software/abi-aa/blob/main/sysvabi64/sysvabi64.rst.  */
 typedef struct __ifunc_arg_t {
   unsigned long _size;
   unsigned long _hwcap;
   unsigned long _hwcap2;
+  unsigned long _hwcap3;
+  unsigned long _hwcap4;
 } __ifunc_arg_t;
+
 # define _IFUNC_ARG_HWCAP (1ULL << 62)
-#endif
 
 /* From the file which imported `host-config.h' we can ascertain which
    architectural extension provides relevant atomic support.  From this,
diff --git a/libcc1/ChangeLog b/libcc1/ChangeLog
index 0eb8fc6..bd1394a 100644
--- a/libcc1/ChangeLog
+++ b/libcc1/ChangeLog
@@ -1,3 +1,13 @@
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* context.cc: Update usage of "diagnostic_info" to explicitly
+	refer to "diagnostics::diagnostic_info".
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* context.cc: Update for move of diagnostics output formats into
+	namespace "diagnostics" as "sinks".
+
 2024-12-23  Simon Martin  <simon@nasilyan.com>
 
 	* Makefile.am: Remove reference to deleted marshall-c.h.
diff --git a/libcc1/context.cc b/libcc1/context.cc
index af5884b..38343a7 100644
--- a/libcc1/context.cc
+++ b/libcc1/context.cc
@@ -39,7 +39,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "diagnostic.h"
 #include "langhooks.h"
 #include "langhooks-def.h"
-#include "diagnostic-format-text.h"
+#include "diagnostics/text-sink.h"
 
 #include "gcc-interface.h"
 
@@ -63,9 +63,9 @@ cc1_plugin::plugin_context *cc1_plugin::current_context;
 // This is put into the lang hooks when the plugin starts.
 
 static void
-plugin_print_error_function (diagnostic_text_output_format &text_output,
+plugin_print_error_function (diagnostics::text_sink &text_output,
 			     const char *file,
-			     const diagnostic_info *diagnostic)
+			     const diagnostics::diagnostic_info *diagnostic)
 {
   if (current_function_decl != NULL_TREE
       && DECL_NAME (current_function_decl) != NULL_TREE
diff --git a/libcpp/ChangeLog b/libcpp/ChangeLog
index 3c5bae1..ca67a64 100644
--- a/libcpp/ChangeLog
+++ b/libcpp/ChangeLog
@@ -1,3 +1,33 @@
+2025-07-30  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/120778
+	* internal.h (struct lexer_state): Add comma_ok member.
+	* expr.cc (_cpp_parse_expr): Initialize it to 0, increment on
+	CPP_OPEN_PAREN and CPP_QUERY, decrement on CPP_CLOSE_PAREN
+	and CPP_COLON.
+	(num_binary_op): For C++ pedwarn on comma operator if
+	pfile->state.comma_ok is 0 instead of !c99 or skip_eval.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* charset.cc: Update comment for file rename.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* internal.h: Update comment for diagnostic_t becoming
+	enum class diagnostics::kind.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* include/cpplib.h: Update for moves to "source-printing".
+	* include/rich-location.h (class label_effects): Move to...
+	(class diagnostics::label_effects): ...here.
+
+2025-07-25  David Malcolm  <dmalcolm@redhat.com>
+
+	* include/rich-location.h: Replace diagnostic_path with
+	diagnostics::paths::path.
+
 2025-06-17  Jason Merrill  <jason@redhat.com>
 
 	* line-map.cc (linemap_location_from_module_p): Add.
diff --git a/libcpp/charset.cc b/libcpp/charset.cc
index e3accf8..95ba376 100644
--- a/libcpp/charset.cc
+++ b/libcpp/charset.cc
@@ -2420,7 +2420,7 @@ convert_escape (cpp_reader *pfile, const uchar *from, const uchar *limit,
 	{
 	  encoding_rich_location rich_loc (pfile);
 
-	  /* diagnostic.cc does not support "%03o".  When it does, this
+	  /* pretty-print.cc does not support "%03o".  When it does, this
 	     code can use %03o directly in the diagnostic again.  */
 	  char buf[32];
 	  sprintf(buf, "%03o", (int) c);
diff --git a/libcpp/expr.cc b/libcpp/expr.cc
index 7bb57a3..910848d 100644
--- a/libcpp/expr.cc
+++ b/libcpp/expr.cc
@@ -1460,6 +1460,7 @@ _cpp_parse_expr (cpp_reader *pfile, const char *dir,
   location_t virtual_location = 0;
 
   pfile->state.skip_eval = 0;
+  pfile->state.comma_ok = 0;
 
   /* Set up detection of #if ! defined().  */
   pfile->mi_ind_cmacro = 0;
@@ -1521,6 +1522,10 @@ _cpp_parse_expr (cpp_reader *pfile, const char *dir,
 	  lex_count--;
 	  continue;
 
+	case CPP_OPEN_PAREN:
+	  pfile->state.comma_ok++;
+	  break;
+
 	default:
 	  if ((int) op.op <= (int) CPP_EQ || (int) op.op >= (int) CPP_PLUS_EQ)
 	    SYNTAX_ERROR2_AT (op.loc,
@@ -1574,13 +1579,16 @@ _cpp_parse_expr (cpp_reader *pfile, const char *dir,
 	case CPP_CLOSE_PAREN:
 	  if (pfile->state.in_directive == 3 && top == pfile->op_stack)
 	    goto embed_done;
+	  pfile->state.comma_ok--;
 	  continue;
 	case CPP_OR_OR:
 	  if (!num_zerop (top->value))
 	    pfile->state.skip_eval++;
 	  break;
-	case CPP_AND_AND:
 	case CPP_QUERY:
+	  pfile->state.comma_ok++;
+	  /* FALLTHRU */
+	case CPP_AND_AND:
 	  if (num_zerop (top->value))
 	    pfile->state.skip_eval++;
 	  break;
@@ -1592,6 +1600,8 @@ _cpp_parse_expr (cpp_reader *pfile, const char *dir,
 	    pfile->state.skip_eval++;
 	  else
 	    pfile->state.skip_eval--;
+	  pfile->state.comma_ok--;
+	  break;
 	default:
 	  break;
 	}
@@ -2209,8 +2219,10 @@ num_binary_op (cpp_reader *pfile, cpp_num lhs, cpp_num rhs, enum cpp_ttype op)
 
       /* Comma.  */
     default: /* case CPP_COMMA: */
-      if (CPP_PEDANTIC (pfile) && (!CPP_OPTION (pfile, c99)
-				   || !pfile->state.skip_eval))
+      if (CPP_PEDANTIC (pfile)
+	  && (CPP_OPTION (pfile, cplusplus)
+	      ? !pfile->state.comma_ok
+	      : (!CPP_OPTION (pfile, c99) || !pfile->state.skip_eval)))
 	cpp_pedwarning (pfile, CPP_W_PEDANTIC,
 			"comma operator in operand of #%s",
 			pfile->state.in_directive == 3
diff --git a/libcpp/include/cpplib.h b/libcpp/include/cpplib.h
index 7c147ae..75efdcd 100644
--- a/libcpp/include/cpplib.h
+++ b/libcpp/include/cpplib.h
@@ -1610,7 +1610,8 @@ struct cpp_decoded_char
    This is a tabstop value, along with a callback for getting the
    widths of characters.  Normally this callback is cpp_wcwidth, but we
    support other schemes for escaping non-ASCII unicode as a series of
-   ASCII chars when printing the user's source code in diagnostic-show-locus.cc
+   ASCII chars when printing the user's source code in
+   gcc/diagnostics/source-printing.cc
 
    For example, consider:
    - the Unicode character U+03C0 "GREEK SMALL LETTER PI" (UTF-8: 0xCF 0x80)
diff --git a/libcpp/include/rich-location.h b/libcpp/include/rich-location.h
index fe9868d..c74e80e 100644
--- a/libcpp/include/rich-location.h
+++ b/libcpp/include/rich-location.h
@@ -25,7 +25,7 @@ along with this program; see the file COPYING3.  If not see
 #include "label-text.h"
 
 class range_label;
-class label_effects;
+namespace diagnostics { class label_effects; }
 
 /* A hint to diagnostic_show_locus on how to print a source range within a
    rich_location.
@@ -213,7 +213,7 @@ semi_embedded_vec<T, NUM_EMBEDDED>::truncate (int len)
 }
 
 class fixit_hint;
-class diagnostic_path;
+namespace diagnostics { namespace paths { class path; }}
 
 /* A "rich" source code location, for use when printing diagnostics.
    A rich_location has one or more carets&ranges, where the carets
@@ -520,8 +520,8 @@ class rich_location
   }
 
   /* An optional path through the code.  */
-  const diagnostic_path *get_path () const { return m_path; }
-  void set_path (const diagnostic_path *path) { m_path = path; }
+  const diagnostics::paths::path *get_path () const { return m_path; }
+  void set_path (const diagnostics::paths::path *path) { m_path = path; }
 
   /* A flag for hinting that the diagnostic involves character encoding
      issues, and thus that it will be helpful to the user if we show some
@@ -567,7 +567,7 @@ protected:
   static const int MAX_STATIC_FIXIT_HINTS = 2;
   semi_embedded_vec <fixit_hint *, MAX_STATIC_FIXIT_HINTS> m_fixit_hints;
 
-  const diagnostic_path *m_path;
+  const diagnostics::paths::path *m_path;
 };
 
 /* Abstract base class for labelling a range within a rich_location
@@ -596,7 +596,8 @@ class range_label
   virtual label_text get_text (unsigned range_idx) const = 0;
 
   /* Get any special effects for the label (e.g. links to other labels).  */
-  virtual const label_effects *get_effects (unsigned /*range_idx*/) const
+  virtual const diagnostics::label_effects *
+  get_effects (unsigned /*range_idx*/) const
   {
     return nullptr;
   }
diff --git a/libcpp/internal.h b/libcpp/internal.h
index 9973836..bcf5559 100644
--- a/libcpp/internal.h
+++ b/libcpp/internal.h
@@ -282,6 +282,9 @@ struct lexer_state
   /* Nonzero to skip evaluating part of an expression.  */
   unsigned int skip_eval;
 
+  /* Nonzero if CPP_COMMA is valid in expression in C++.  */
+  unsigned int comma_ok;
+
   /* Nonzero when tokenizing a deferred pragma.  */
   unsigned char in_deferred_pragma;
 
@@ -622,8 +625,8 @@ struct cpp_reader
      zero of said file.  */
   location_t main_loc;
 
-  /* If non-zero, override diagnostic locations (other than DK_NOTE
-     diagnostics) to this one.  */
+  /* If non-zero, override diagnostic locations (other than
+     diagnostics::kind::note diagnostics) to this one.  */
   location_t diagnostic_override_loc;
 
   /* Returns true iff we should warn about UTF-8 bidirectional control
diff --git a/libcpp/po/ChangeLog b/libcpp/po/ChangeLog
index d9f18ea..99d0fb4 100644
--- a/libcpp/po/ChangeLog
+++ b/libcpp/po/ChangeLog
@@ -1,3 +1,7 @@
+2025-07-30  Joseph Myers  <josmyers@redhat.com>
+
+	* sr.po: Update.
+
 2025-05-16  Joseph Myers  <josmyers@redhat.com>
 
 	* es.po: Update.
diff --git a/libcpp/po/sr.po b/libcpp/po/sr.po
index 365f1ae..9e19ea1 100644
--- a/libcpp/po/sr.po
+++ b/libcpp/po/sr.po
@@ -1,14 +1,14 @@
 # Serbian translation of cpplib.
 # Copyright © 2020 Free Software Foundation, Inc.
 # This file is distributed under the same license as the gcc package.
-# Миро�лав �иколић <miroslavnikolic@rocketmail.com>, 2012-2024.
+# Миро�лав �иколић <miroslavnikolic@rocketmail.com>, 2012-2025.
 #
 msgid ""
 msgstr ""
-"Project-Id-Version: cpplib-14.1-b20240218\n"
+"Project-Id-Version: cpplib-15.1-b20250316\n"
 "Report-Msgid-Bugs-To: https://gcc.gnu.org/bugs/\n"
 "POT-Creation-Date: 2025-03-14 22:05+0000\n"
-"PO-Revision-Date: 2024-12-14 16:23+0100\n"
+"PO-Revision-Date: 2025-07-18 09:21+0200\n"
 "Last-Translator: Миро�лав �иколић <miroslavnikolic@rocketmail.com>\n"
 "Language-Team: Serbian <(nothing)>\n"
 "Language: sr\n"
@@ -17,7 +17,7 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);\n"
 "X-Bugs: Report translation errors to the Language-Team address.\n"
-"X-Generator: Gtranslator 42.0\n"
+"X-Generator: Poedit 3.5\n"
 
 #: charset.cc:759
 #, gcc-internal-format, gfc-internal-format
@@ -26,7 +26,7 @@ msgstr "„ikonv“ не подржава претварање из %s у %s"
 
 #: charset.cc:762
 msgid "iconv_open"
-msgstr "иконв_отвори"
+msgstr "iconv_open"
 
 #: charset.cc:772
 #, gcc-internal-format, gfc-internal-format
@@ -34,10 +34,9 @@ msgid "no iconv implementation, cannot convert from %s to %s"
 msgstr "нема иконв примене, не могу да претворим из %s у %s"
 
 #: charset.cc:870
-#, fuzzy, gcc-internal-format
-#| msgid "character 0x%lx is not in the basic source character set\n"
+#, gcc-internal-format
 msgid "character 0x%lx is not in the basic source character set"
-msgstr "знак 0x%lx није у знаковном �купу о�новног извора\n"
+msgstr "знак 0x%lx није у знаковном �купу о�новног извора"
 
 #: charset.cc:887 charset.cc:2639
 msgid "converting to execution character set"
@@ -50,23 +49,20 @@ msgstr "знак 0x%lx није једног бајта у знаковном �
 
 #: charset.cc:1549
 msgid "universal character names are only valid in C++ and C99"
-msgstr "називи �веопштег знака �у једино и�правни у Ц++ и Ц99"
+msgstr "називи �веопштег знака �у једино и�правни у C++ и C99"
 
 #: charset.cc:1553
-#, fuzzy, gcc-internal-format
-#| msgid "C99's universal character names are incompatible with C90"
+#, gcc-internal-format
 msgid "C99%'s universal character names are incompatible with C90"
-msgstr "називи �веопштег знака Ц99 ни�у �агла�ни �а Ц90"
+msgstr "називи �веопштег знака C99%'s ни�у �агла�ни �а C90"
 
 #: charset.cc:1556
-#, fuzzy, gcc-internal-format
-#| msgid "the meaning of '\\%c' is different in traditional C"
+#, gcc-internal-format
 msgid "the meaning of %<\\%c%> is different in traditional C"
-msgstr "значење „\\%c“ је другачије у уобичајеном Ц-у"
+msgstr "значење „\\%c“ је другачије у �тандардном C-у"
 
 #: charset.cc:1595
-#, fuzzy, gcc-internal-format
-#| msgid "'\\N' not followed by '{'"
+#, gcc-internal-format
 msgid "%<\\N%> not followed by %<{%>"
 msgstr "„\\N“ без пратеће „{“"
 
@@ -80,43 +76,37 @@ msgstr "празан именован Ñ�веопшти низ промене зÐ
 
 #: charset.cc:1639
 msgid "named universal character escapes are only valid in C++23"
-msgstr "именоване �веопште промене знака �у и�правне �амо у Ц++23"
+msgstr "именоване �веопште промене знака �у и�правне �амо у C++23"
 
 #: charset.cc:1659
-#, fuzzy, gcc-internal-format
-#| msgid "\\N{%.*s} is not a valid universal character; treating it as separate tokens"
+#, gcc-internal-format
 msgid "%<\\N{%.*s}%> is not a valid universal character; treating it as separate tokens"
 msgstr "„\\N{%.*s}“ није и�праван �веопшти знак; �матрам то одвојеним �купинама"
 
 #: charset.cc:1665
-#, fuzzy, gcc-internal-format
-#| msgid "\\N{%.*s} is not a valid universal character"
+#, gcc-internal-format
 msgid "%<\\N{%.*s}%> is not a valid universal character"
 msgstr "„\\N{%.*s}“ није и�праван �веопшти знак"
 
 #: charset.cc:1675
-#, fuzzy, gcc-internal-format
-#| msgid "did you mean \\N{%s}?"
+#, gcc-internal-format
 msgid "did you mean %<\\N{%s}%>?"
 msgstr "да ли �те ми�лили „\\N{%s}“?"
 
 #: charset.cc:1693
-#, fuzzy, gcc-internal-format
-#| msgid "'\\N{' not terminated with '}' after %.*s; treating it as separate tokens"
+#, gcc-internal-format
 msgid "%<\\N{%> not terminated with %<}%> after %.*s; treating it as separate tokens"
-msgstr "„\\N{' �е не завршава �а '}“ по�ле „%.*s“; �матрам то одвојеним �купинама"
+msgstr "„\\N{“ �е не завршава �а „}“ по�ле „%.*s“; �матрам то одвојеним �купинама"
 
 #: charset.cc:1702
-#, fuzzy, gcc-internal-format
-#| msgid "'\\N{' not terminated with '}' after %.*s"
+#, gcc-internal-format
 msgid "%<\\N{%> not terminated with %<}%> after %.*s"
-msgstr "„\\N{' �е не завршава �а '}“ по�ле „%.*s“"
+msgstr "„\\N{“ �е не завршава �а „}“ након „%.*s“"
 
 #: charset.cc:1710
-#, fuzzy, gcc-internal-format
-#| msgid "In _cpp_valid_ucn but not a UCN"
+#, gcc-internal-format
 msgid "in %<_cpp_valid_ucn%> but not a UCN"
-msgstr "У _цпп_и�праван_н�з али није �СЗ"
+msgstr "у „_cpp_valid_ucn“ али није UCN"
 
 #: charset.cc:1753
 msgid "empty delimited escape sequence; treating it as separate tokens"
@@ -128,20 +118,17 @@ msgstr "празан ограничен низ промене"
 
 #: charset.cc:1769 charset.cc:2172 charset.cc:2289
 msgid "delimited escape sequences are only valid in C++23"
-msgstr "ограничени низови промене �у једино и�правни у Ц++23"
+msgstr "ограничени низови промене �у једино и�правни у C++23"
 
 #: charset.cc:1774 charset.cc:1779 charset.cc:2177 charset.cc:2182
 #: charset.cc:2294 charset.cc:2299
-#, fuzzy
-#| msgid "delimited escape sequences are only valid in C++23"
 msgid "delimited escape sequences are only valid in C2Y"
-msgstr "ограничени низови промене �у једино и�правни у Ц++23"
+msgstr "ограничени низови промене �у једино и�правни у C2Y"
 
 #: charset.cc:1794
-#, fuzzy, gcc-internal-format
-#| msgid "'\\u{' not terminated with '}' after %.*s; treating it as separate tokens"
+#, gcc-internal-format
 msgid "%<\\u{%> not terminated with %<}%> after %.*s; treating it as separate tokens"
-msgstr "„\\u{' �е не завршава �а '}“ по�ле „%.*s“; �матрам то одвојеним токенима"
+msgstr "„\\u{“ �е не завршава �а „}“ по�ле „%.*s“; �матрам то одвојеним токенима"
 
 #: charset.cc:1806
 #, gcc-internal-format
@@ -149,27 +136,24 @@ msgid "incomplete universal character name %.*s"
 msgstr "непотпун назив �веопштег знака %.*s"
 
 #: charset.cc:1810
-#, fuzzy, gcc-internal-format
-#| msgid "'\\u{' not terminated with '}' after %.*s"
+#, gcc-internal-format
 msgid "%<\\u{%> not terminated with %<}%> after %.*s"
-msgstr "„\\u{' �е не завршава �а '}“ по�ле „%.*s“"
+msgstr "„\\u{“ �е не завршава �а „}“ након „%.*s“"
 
 #: charset.cc:1818
 #, gcc-internal-format
 msgid "%.*s is not a valid universal character"
-msgstr "%.*s није и�праван �веопшти знак"
+msgstr "„%.*s“ није и�праван �веопшти знак"
 
 #: charset.cc:1834 charset.cc:1838
-#, fuzzy, gcc-internal-format
-#| msgid "%.*s is not a valid universal character"
+#, gcc-internal-format
 msgid "%.*s is not a valid universal character name before C23"
-msgstr "%.*s није и�праван �веопшти знак"
+msgstr "„%.*s“ није и�праван назив �веопштег знака пре C23"
 
 #: charset.cc:1854 lex.cc:2096
-#, fuzzy, gcc-internal-format
-#| msgid "'$' in identifier or number"
+#, gcc-internal-format
 msgid "%<$%> in identifier or number"
-msgstr "„$“ у одреднику или број"
+msgstr "„$“ у одреднику или броју"
 
 #: charset.cc:1864
 #, gcc-internal-format
@@ -184,15 +168,15 @@ msgstr "�веопшти знак %.*s није и�праван на почет
 #: charset.cc:1875
 #, gcc-internal-format
 msgid "%.*s is outside the UCS codespace"
-msgstr "„%.*s“ је ван УЦС кодног про�тора"
+msgstr "„%.*s“ је ван UCS кодног про�тора"
 
 #: charset.cc:1919 charset.cc:3072
 msgid "converting UCN to source character set"
-msgstr "претварам �СЗ у изворни знаковни �куп"
+msgstr "претварам UCN у изворни знаковни �куп"
 
 #: charset.cc:1926
 msgid "converting UCN to execution character set"
-msgstr "претварам �СЗ у извршни знаковни �куп"
+msgstr "претварам UCN у извршни знаковни �куп"
 
 #: charset.cc:1990
 #, gcc-internal-format
@@ -205,70 +189,60 @@ msgid "extended character %.*s is not valid at the start of an identifier"
 msgstr "проширени знак „%.*s“ није и�праван на почетку одредника"
 
 #: charset.cc:2129
-#, fuzzy, gcc-internal-format
-#| msgid "the meaning of '\\x' is different in traditional C"
+#, gcc-internal-format
 msgid "the meaning of %<\\x%> is different in traditional C"
-msgstr "значење „\\x“ је другачије у уобичајеном Ц-у"
+msgstr "значење „\\x“ је другачије у �тандардном C-у"
 
 #: charset.cc:2190
-#, fuzzy, gcc-internal-format
-#| msgid "\\x used with no following hex digits"
+#, gcc-internal-format
 msgid "%<\\x%> used with no following hex digits"
-msgstr "„\\x“ је коришћено �а не пратећим хек�а цифрама"
+msgstr "„\\x“ је коришћено �а не пратећим хек� цифрама"
 
 #: charset.cc:2196
-#, fuzzy, gcc-internal-format
-#| msgid "'\\x{' not terminated with '}' after %.*s"
+#, gcc-internal-format
 msgid "%<\\x{%> not terminated with %<}%> after %.*s"
-msgstr "„\\x{' �е не завршава �а '}“ по�ле „%.*s“"
+msgstr "„\\x{“ �е не завршава �а „}“ након „%.*s“"
 
 #: charset.cc:2204
 msgid "hex escape sequence out of range"
 msgstr "хек� низ промене је изван оп�ега"
 
 #: charset.cc:2247
-#, fuzzy, gcc-internal-format
-#| msgid "'\\o' not followed by '{'"
+#, gcc-internal-format
 msgid "%<\\o%> not followed by %<{%>"
 msgstr "за „\\o“ не �леди „{“"
 
 #: charset.cc:2305
-#, fuzzy, gcc-internal-format
-#| msgid "'\\o{' not terminated with '}' after %.*s"
+#, gcc-internal-format
 msgid "%<\\o{%> not terminated with %<}%> after %.*s"
-msgstr "„\\o{' �е не завршава �а '}“ по�ле „%.*s“"
+msgstr "„\\o{“ �е не завршава �а „}“ по�ле „%.*s“"
 
 #: charset.cc:2314
 msgid "octal escape sequence out of range"
 msgstr "октални низ промене је изван оп�ега"
 
 #: charset.cc:2366 charset.cc:2376
-#, fuzzy, gcc-internal-format
-#| msgid "numeric escape sequence in unevaluated string: '\\%c'"
+#, gcc-internal-format
 msgid "numeric escape sequence in unevaluated string: %<\\%c%>"
 msgstr "бројевни низ промене низа у непроцењеној ни�ки: „\\%c“"
 
 #: charset.cc:2404
-#, fuzzy, gcc-internal-format
-#| msgid "the meaning of '\\a' is different in traditional C"
+#, gcc-internal-format
 msgid "the meaning of %<\\a%> is different in traditional C"
-msgstr "значење „\\a“ је другачије у уобичајеном Ц-у"
+msgstr "значење „\\a“ је другачије у �тандардном C-у"
 
 #: charset.cc:2410
-#, fuzzy, gcc-internal-format
-#| msgid "non-ISO-standard escape sequence, '\\%c'"
+#, gcc-internal-format
 msgid "non-ISO-standard escape sequence, %<\\%c%>"
 msgstr "низ промене не-ИСО-�тандарда, „\\%c“"
 
 #: charset.cc:2418
-#, fuzzy, gcc-internal-format
-#| msgid "unknown escape sequence: '\\%c'"
+#, gcc-internal-format
 msgid "unknown escape sequence: %<\\%c%>"
 msgstr "непознат низ промене: „\\%c“"
 
 #: charset.cc:2428
-#, fuzzy, gcc-internal-format
-#| msgid "unknown escape sequence: '\\%s'"
+#, gcc-internal-format
 msgid "unknown escape sequence: %<\\%s%>"
 msgstr "непознат низ промене: „\\%s“"
 
@@ -289,8 +263,7 @@ msgid "at least one character in a multi-character literal not encodable in a si
 msgstr "барем један знак у вишезнаковном литералу �е не може прекодирати у једној јединици знаковног �купа извршавања"
 
 #: charset.cc:2830
-#, fuzzy, gcc-internal-format
-#| msgid "multi-character literal with %ld characters exceeds 'int' size of %ld bytes"
+#, gcc-internal-format
 msgid "multi-character literal with %ld characters exceeds %<int%> size of %ld bytes"
 msgstr "више-знаковни литерал �а %ld знака премашује „int“ величину од %ld бајта"
 
@@ -316,58 +289,50 @@ msgid "failure to convert %s to %s"
 msgstr "неу�пех у претварању %s у %s"
 
 #: directives.cc:243
-#, fuzzy, gcc-internal-format
-#| msgid "extra tokens at end of #%s directive"
+#, gcc-internal-format
 msgid "extra tokens at end of %<#%s%> directive"
-msgstr "по�ебни �имболи на крају #%s директиве"
+msgstr "ек�тра �купине на крају „#%s“ директиве"
 
 #: directives.cc:286
 #, gcc-internal-format, gfc-internal-format
 msgid "extra tokens at end of #%s directive"
-msgstr "по�ебни �имболи на крају #%s директиве"
+msgstr "по�ебни �имболи на крају „#%s“ директиве"
 
 #: directives.cc:396
-#, fuzzy, gcc-internal-format
-#| msgid "#%s is a GCC extension"
+#, gcc-internal-format
 msgid "%<#%s%> is a GCC extension"
-msgstr "#%s је ГЦЦ проширење"
+msgstr "„#%s“ је GCC проширење"
 
 #: directives.cc:404 directives.cc:2686 directives.cc:2725
-#, fuzzy, gcc-internal-format
-#| msgid "#%s before C++23 is a GCC extension"
+#, gcc-internal-format
 msgid "%<#%s%> before C++23 is a GCC extension"
-msgstr "„#%s“ пре „C++23“ је�те GCC проширење"
+msgstr "„#%s“ пре C++23 је GCC проширење"
 
 #: directives.cc:409 directives.cc:415 directives.cc:1374 directives.cc:2690
 #: directives.cc:2730
-#, fuzzy, gcc-internal-format
-#| msgid "#%s before C23 is a GCC extension"
+#, gcc-internal-format
 msgid "%<#%s%> before C23 is a GCC extension"
-msgstr "„#%s“ пре „C23“ је�те GCC проширење"
+msgstr "„#%s“ пре C23 је GCC проширење"
 
 #: directives.cc:423
-#, fuzzy, gcc-internal-format
-#| msgid "#%s is a deprecated GCC extension"
+#, gcc-internal-format
 msgid "%<#%s%> is a deprecated GCC extension"
-msgstr "#%s је превазиђено ГЦЦ проширење"
+msgstr "„#%s“ је за�тарело GCC проширење"
 
 #: directives.cc:436
-#, fuzzy, gcc-internal-format
-#| msgid "suggest not using #elif in traditional C"
+#, gcc-internal-format
 msgid "suggest not using %<#elif%> in traditional C"
-msgstr "�аветује да �е не кори�ти „#elif“ у уобичајеном Ц-у"
+msgstr "�аветује да �е не кори�ти „#elif“ у �тандардном C-у"
 
 #: directives.cc:439
-#, fuzzy, gcc-internal-format
-#| msgid "traditional C ignores #%s with the # indented"
+#, gcc-internal-format
 msgid "traditional C ignores %<#%s%> with the %<#%> indented"
-msgstr "уобичајени Ц занемарује #%s �а # увученим"
+msgstr "�тандардни C занемарује „#%s“ �а „#“ увученим"
 
 #: directives.cc:443
-#, fuzzy, gcc-internal-format
-#| msgid "suggest hiding #%s from traditional C with an indented #"
+#, gcc-internal-format
 msgid "suggest hiding %<#%s%> from traditional C with an indented %<#%>"
-msgstr "�аветује �кривање #%s из уобичајеног Ц-а �а увученим #"
+msgstr "�аветује �кривање „#%s“ из �тандардног C-а �а увученим „#“"
 
 #: directives.cc:468
 msgid "embedding a directive within macro arguments is not portable"
@@ -375,191 +340,168 @@ msgstr "угнежђивање директиве унутар аргуменаÑ
 
 #: directives.cc:497
 msgid "style of line directive is a GCC extension"
-msgstr "�тил директиве реда је ГЦЦ проширење"
+msgstr "�тил директиве реда је GCC проширење"
 
 #: directives.cc:572
 #, gcc-internal-format, gfc-internal-format
 msgid "invalid preprocessing directive #%s; did you mean #%s?"
-msgstr "неи�правна директива предобраде #%s; да ли �те ми�лили „#%s“?"
+msgstr "неи�правна директива предобраде „#%s“; да ли �те ми�лили „#%s“?"
 
 #: directives.cc:578
 #, gcc-internal-format, gfc-internal-format
 msgid "invalid preprocessing directive #%s"
-msgstr "неи�правна директива предобраде #%s"
+msgstr "неи�правна директива предобраде „#%s“"
 
 #: directives.cc:656
-#, fuzzy, gcc-internal-format
-#| msgid "\"%s\" cannot be used as a macro name"
+#, gcc-internal-format
 msgid "%qs cannot be used as a macro name"
-msgstr "„%s“ не може бити коришћено као назив макроа"
+msgstr "„%qs“ не може бити коришћено као назив макроа"
 
 #: directives.cc:663
-#, fuzzy, gcc-internal-format
-#| msgid "\"%s\" cannot be used as a macro name as it is an operator in C++"
+#, gcc-internal-format
 msgid "%qs cannot be used as a macro name as it is an operator in C++"
-msgstr "„%s“ не може бити коришћено као назив макроа јер је то оператор у Ц++-у"
+msgstr "„%qs“ не може бити коришћено као назив макроа јер је то оператор у C++-у"
 
 #: directives.cc:666
-#, fuzzy, gcc-internal-format
-#| msgid "no macro name given in #%s directive"
+#, gcc-internal-format
 msgid "no macro name given in %<#%s%> directive"
-msgstr "није дат назив макроа у #%s директиви"
+msgstr "није дат назив макроа у „#%s“ директиви"
 
 #: directives.cc:669
 msgid "macro names must be identifiers"
 msgstr "називи макроа морају бити одредници"
 
 #: directives.cc:743 directives.cc:747
-#, fuzzy, gcc-internal-format
-#| msgid "undefining \"%s\""
+#, gcc-internal-format
 msgid "undefining %qs"
-msgstr "неодређујем „%s“"
+msgstr "неодређујем „%qs“"
 
 #: directives.cc:805
-#, fuzzy, gcc-internal-format
-#| msgid "missing terminating > character"
+#, gcc-internal-format
 msgid "missing terminating %<>%> character"
-msgstr "недо�таје завршни знак >"
+msgstr "недо�таје завршни знак „>“"
 
 #: directives.cc:865
-#, fuzzy, gcc-internal-format
-#| msgid "#%s expects \"FILENAME\" or <FILENAME>"
+#, gcc-internal-format
 msgid "%<#%s%> expects %<\"FILENAME\"%> or %<<FILENAME>%>"
-msgstr "#%s очекује „��ЗИВД�ТОТЕКЕ“ или <��ЗИВД�ТОТЕКЕ>"
+msgstr "„#%s“ очекује „��ЗИВД�ТОТЕКЕ“ или „��ЗИВД�ТОТЕКЕ“"
 
 #: directives.cc:911 directives.cc:1391
 #, gcc-internal-format, gfc-internal-format
 msgid "empty filename in #%s"
-msgstr "празан назив датотеке у #%s"
+msgstr "празан назив датотеке у „#%s“"
 
 #: directives.cc:920
-#, fuzzy, gcc-internal-format
-#| msgid "#include nested depth %u exceeds maximum of %u (use -fmax-include-depth=DEPTH to increase the maximum)"
+#, gcc-internal-format
 msgid "%<#include%> nested depth %u exceeds maximum of %u (use %<-fmax-include-depth=DEPTH%> to increase the maximum)"
 msgstr "„#include“ дубоко угнеждено %u превазилази мак�имум од %u (кори�тите „-fmax-include-depth=ДУБИ��“ да повећате мак�имум)"
 
 #: directives.cc:965
-#, fuzzy, gcc-internal-format
-#| msgid "#include_next in primary source file"
+#, gcc-internal-format
 msgid "%<#include_next%> in primary source file"
-msgstr "#include_next у примарној изворној датотеци"
+msgstr "„#include_next“ у примарној изворној датотеци"
 
 #: directives.cc:1037 directives.cc:1059 directives.cc:1062 directives.cc:1065
-#, fuzzy, gcc-internal-format, gfc-internal-format
-#| msgid "unbalanced stack in %s"
+#, gcc-internal-format, gfc-internal-format
 msgid "unbalanced '%c'"
-msgstr "неуравнотежен �тек у %s"
+msgstr "неуравнотежено „%c“"
 
 #: directives.cc:1120 directives.cc:1311
 #, gcc-internal-format
 msgid "expected %<)%>"
-msgstr ""
+msgstr "очекивах „)“"
 
 #: directives.cc:1126 directives.cc:1177
-#, fuzzy
-#| msgid "expected parameter name, found \"%s\""
 msgid "expected parameter name"
-msgstr "очекивах назив параметра, нађох „%s“"
+msgstr "очекивах назив параметра"
 
 #: directives.cc:1137
 #, gcc-internal-format
 msgid "%<gnu::base64%> parameter conflicts with %<limit%> or %<gnu::offset%> parameters"
-msgstr ""
+msgstr "„gnu::base64“ параметар �е �укобљава �а „limit“ или „gnu::offset“ параметром"
 
 #: directives.cc:1147
 #, gcc-internal-format
 msgid "%<gnu::base64%> parameter required in preprocessed source"
-msgstr ""
+msgstr "„gnu::base64“ параметар је захтеван у преобрађеном извору"
 
 #: directives.cc:1168
 #, gcc-internal-format
 msgid "expected %<:%>"
-msgstr ""
+msgstr "очекивах „:“"
 
 #: directives.cc:1235
-#, fuzzy, gcc-internal-format
-#| msgid "duplicate macro parameter \"%s\""
+#, gcc-internal-format
 msgid "duplicate embed parameter '%.*s%s%.*s'"
-msgstr "параметар дво�труког макроа „%s“"
+msgstr "дво�труки угнеждени параметар „%.*s%s%.*s“"
 
 #: directives.cc:1247
 #, gcc-internal-format
 msgid "unknown embed parameter '%.*s%s%.*s'"
-msgstr ""
+msgstr "непознат угнеждени параметар „%.*s%s%.*s“"
 
 #: directives.cc:1256
 #, gcc-internal-format
 msgid "expected %<(%>"
-msgstr ""
+msgstr "очекивах „(“"
 
 #: directives.cc:1269
 #, gcc-internal-format
 msgid "too large %<gnu::offset%> argument"
-msgstr ""
+msgstr "превелик „gnu::offset“ аргумент"
 
 #: directives.cc:1316
-#, fuzzy
-#| msgid "null character(s) preserved in literal"
 msgid "expected character string literal"
-msgstr "ништаван знак је причуван у �ловно�ти"
+msgstr "очекивах �ловно�т ни�ке знака"
 
 #: directives.cc:1361
-#, fuzzy, gcc-internal-format
-#| msgid "suggest not using #elif in traditional C"
+#, gcc-internal-format
 msgid "%<#embed%> not supported in traditional C"
-msgstr "�аветује да �е не кори�ти „#elif“ у уобичајеном Ц-у"
+msgstr "„#embed“ није подржано у �тандардном C-у"
 
 #: directives.cc:1370
-#, fuzzy, gcc-internal-format
-#| msgid "#%s before C++23 is a GCC extension"
+#, gcc-internal-format
 msgid "%<#%s%> before C++26 is a GCC extension"
-msgstr "„#%s“ пре „C++23“ је�те GCC проширење"
+msgstr "„#%s“ пре C++26 је GCC проширење"
 
 #: directives.cc:1379
-#, fuzzy, gcc-internal-format
-#| msgid "#%s is a GCC extension"
+#, gcc-internal-format
 msgid "%<#%s%> is a C23 feature"
-msgstr "#%s је ГЦЦ проширење"
+msgstr "„#%s“ је C23 функционално�т"
 
 #: directives.cc:1436
-#, fuzzy, gcc-internal-format
-#| msgid "invalid flag \"%s\" in line directive"
+#, gcc-internal-format
 msgid "invalid flag %qs in line directive"
-msgstr "неи�правна опција „%s“ у директиви реда"
+msgstr "неи�правна за�тавица „%qs“ у директиви реда"
 
 #: directives.cc:1504
-#, fuzzy, gcc-internal-format
-#| msgid "unexpected end of file after #line"
+#, gcc-internal-format
 msgid "unexpected end of file after %<#line%>"
 msgstr "неочекивани крај датотеке након „#line“"
 
 #: directives.cc:1507
-#, fuzzy, gcc-internal-format
-#| msgid "\"%s\" after #line is not a positive integer"
+#, gcc-internal-format
 msgid "%qs after %<#line%> is not a positive integer"
-msgstr "„%s“ након „#line“ није позитиван цео број"
+msgstr "„%qs“ након „#line“ није позитиван цео број"
 
 #: directives.cc:1513 directives.cc:1516
 msgid "line number out of range"
 msgstr "број реда је ван оп�ега"
 
 #: directives.cc:1529 directives.cc:1610
-#, fuzzy, gcc-internal-format
-#| msgid "\"%s\" is not a valid filename"
+#, gcc-internal-format
 msgid "%qs is not a valid filename"
-msgstr "„%s“ није и�праван назив датотеке"
+msgstr "„%qs“ није и�праван назив датотеке"
 
 #: directives.cc:1570
-#, fuzzy, gcc-internal-format
-#| msgid "\"%s\" after # is not a positive integer"
+#, gcc-internal-format
 msgid "%qs after %<#%> is not a positive integer"
-msgstr "„%s“ након „#“ није позитиван цео број"
+msgstr "„%qs“ након „#“ није позитиван цео број"
 
 #: directives.cc:1637
-#, fuzzy, gcc-internal-format
-#| msgid "file \"%s\" linemarker ignored due to incorrect nesting"
+#, gcc-internal-format
 msgid "file %qs linemarker ignored due to incorrect nesting"
-msgstr "датотека „%s“ означавач реда занемаренa због нетачног угњежђавања"
+msgstr "„%qs“ означавач реда датотеке је занемарен због нетачног угнежђавања"
 
 #: directives.cc:1715 directives.cc:1717 directives.cc:1719 directives.cc:2326
 #, gcc-internal-format, gfc-internal-format
@@ -569,69 +511,59 @@ msgstr "%s"
 #: directives.cc:1743
 #, gcc-internal-format, gfc-internal-format
 msgid "invalid #%s directive"
-msgstr "неи�правна #%s директива"
+msgstr "неи�правна „#%s“ директива"
 
 #: directives.cc:1806
-#, fuzzy, gcc-internal-format
-#| msgid "registering pragmas in namespace \"%s\" with mismatched name expansion"
+#, gcc-internal-format
 msgid "registering pragmas in namespace %qs with mismatched name expansion"
-msgstr "бележим прагме у про�тору назива „%s“ �а неодговарајућим ширењем назива"
+msgstr "бележим прагме у про�тору назива „%qs“ �а неодговарајућим ширењем назива"
 
 #: directives.cc:1815
-#, fuzzy, gcc-internal-format
-#| msgid "registering pragma \"%s\" with name expansion and no namespace"
+#, gcc-internal-format
 msgid "registering pragma %qs with name expansion and no namespace"
-msgstr "бележим прагму „%s“ �а ширењем назива и без про�тора назива"
+msgstr "бележим прагму „%qs“ �а ширењем назива и без про�тора назива"
 
 #: directives.cc:1833
-#, fuzzy, gcc-internal-format
-#| msgid "registering \"%s\" as both a pragma and a pragma namespace"
+#, gcc-internal-format
 msgid "registering %qs as both a pragma and a pragma namespace"
-msgstr "бележим „%s“ и као прагму и као про�тор назива прагме"
+msgstr "бележим „%qs“ и као прагму и као про�тор назива прагме"
 
 #: directives.cc:1836
-#, fuzzy, gcc-internal-format
-#| msgid "#pragma %s %s is already registered"
+#, gcc-internal-format
 msgid "%<#pragma %s %s%> is already registered"
-msgstr "#pragma %s %s је већ забележено"
+msgstr "„#pragma %s %s“ је већ забележено"
 
 #: directives.cc:1839
-#, fuzzy, gcc-internal-format
-#| msgid "#pragma %s is already registered"
+#, gcc-internal-format
 msgid "%<#pragma %s%> is already registered"
-msgstr "#pragma %s је већ забележено"
+msgstr "„#pragma %s“ је већ забележено"
 
 #: directives.cc:1870
 msgid "registering pragma with NULL handler"
 msgstr "бележим прагму �а �ИШТ�В�ИМ руковаоцем"
 
 #: directives.cc:2088
-#, fuzzy, gcc-internal-format
-#| msgid "#pragma once in main file"
+#, gcc-internal-format
 msgid "%<#pragma once%> in main file"
-msgstr "#pragma једном у главној датотеци"
+msgstr "„#pragma once“ у главној датотеци"
 
 #: directives.cc:2158
-#, fuzzy, gcc-internal-format
-#| msgid "invalid #pragma push_macro directive"
+#, gcc-internal-format
 msgid "invalid %<#pragma %s_macro%> directive"
-msgstr "неи�правна „#pragma push_macro“ директива"
+msgstr "неи�правна „#pragma %s_macro“ директива"
 
 #: directives.cc:2237
-#, fuzzy, gcc-internal-format
-#| msgid "invalid #pragma GCC poison directive"
+#, gcc-internal-format
 msgid "invalid %<#pragma GCC poison%> directive"
 msgstr "неи�правна „#pragma GCC poison“ директива"
 
 #: directives.cc:2246
-#, fuzzy, gcc-internal-format
-#| msgid "poisoning existing macro \"%s\""
+#, gcc-internal-format
 msgid "poisoning existing macro %qs"
-msgstr "трујем по�тојећи макро „%s“"
+msgstr "трујем по�тојећи макро „%qs“"
 
 #: directives.cc:2268
-#, fuzzy, gcc-internal-format
-#| msgid "#pragma system_header ignored outside include file"
+#, gcc-internal-format
 msgid "%<#pragma system_header%> ignored outside include file"
 msgstr "„#pragma system_header“ је занемарено изван датотеке укључивања"
 
@@ -646,26 +578,22 @@ msgid "current file is older than %s"
 msgstr "текућа датотека је �тарија од %s"
 
 #: directives.cc:2321
-#, fuzzy, gcc-internal-format
-#| msgid "invalid \"#pragma GCC %s\" directive"
+#, gcc-internal-format
 msgid "invalid %<#pragma GCC %s%> directive"
-msgstr "неи�правна директива „#pragma GCC %s“"
+msgstr "неи�правна „#pragma GCC %s“ директива"
 
 #: directives.cc:2541
-#, fuzzy, gcc-internal-format
-#| msgid "_Pragma takes a parenthesized string literal"
+#, gcc-internal-format
 msgid "%<_Pragma%> takes a parenthesized string literal"
-msgstr "_Прагма узима �ловно�т ни�ке затворене заградом"
+msgstr "„_Pragma“ узима �ловно�т ни�ке затворене заградом"
 
 #: directives.cc:2624
-#, fuzzy, gcc-internal-format
-#| msgid "#else without #if"
+#, gcc-internal-format
 msgid "%<#else%> without %<#if%>"
 msgstr "„#else“ без „#if“"
 
 #: directives.cc:2629
-#, fuzzy, gcc-internal-format
-#| msgid "#else after #else"
+#, gcc-internal-format
 msgid "%<#else%> after %<#else%>"
 msgstr "„#else“ након „#else“"
 
@@ -674,40 +602,34 @@ msgid "the conditional began here"
 msgstr "у�ловљено�т почиње овде"
 
 #: directives.cc:2657
-#, fuzzy, gcc-internal-format
-#| msgid "#%s without #if"
+#, gcc-internal-format
 msgid "%<#%s%> without %<#if%>"
 msgstr "„#%s“ без „#if“"
 
 #: directives.cc:2663
-#, fuzzy, gcc-internal-format
-#| msgid "#%s after #else"
+#, gcc-internal-format
 msgid "%<#%s%> after %<#else%>"
 msgstr "„#%s“ након „#else“"
 
 #: directives.cc:2767
-#, fuzzy, gcc-internal-format
-#| msgid "#endif without #if"
+#, gcc-internal-format
 msgid "%<#endif%> without %<#if%>"
 msgstr "„#endif“ без „#if“"
 
 #: directives.cc:2852
-#, fuzzy, gcc-internal-format
-#| msgid "missing '(' after predicate"
+#, gcc-internal-format
 msgid "missing %<(%> after predicate"
-msgstr "недо�таје ( након предиката"
+msgstr "недо�таје „(“ након предиката"
 
 #: directives.cc:2870
-#, fuzzy, gcc-internal-format
-#| msgid "missing ')' to complete answer"
+#, gcc-internal-format
 msgid "missing %<)%> to complete answer"
-msgstr "недо�таје ) да доврши одговор"
+msgstr "недо�таје „)“ да доврши одговор"
 
 #: directives.cc:2882
-#, fuzzy, gcc-internal-format
-#| msgid "predicate's answer is empty"
+#, gcc-internal-format
 msgid "predicate%'s answer is empty"
-msgstr "одговор предиката је празан"
+msgstr "„predicate%'s“ одговор је празан"
 
 #: directives.cc:2912
 msgid "assertion without predicate"
@@ -718,10 +640,9 @@ msgid "predicate must be an identifier"
 msgstr "предикат мора бити одредник"
 
 #: directives.cc:2997
-#, fuzzy, gcc-internal-format
-#| msgid "\"%s\" re-asserted"
+#, gcc-internal-format
 msgid "%qs re-asserted"
-msgstr "„%s“ је поново утврђен"
+msgstr "„%qs“ је поново утврђено"
 
 #: directives.cc:3305
 #, gcc-internal-format, gfc-internal-format
@@ -739,104 +660,92 @@ msgstr "�тдизлаз"
 
 #: expr.cc:694 expr.cc:822
 msgid "fixed-point constants are a GCC extension"
-msgstr "кон�танте �талне тачке �у ГЦЦ проширења"
+msgstr "кон�танте �талне тачке �у GCC проширења"
 
 #: expr.cc:721
-#, fuzzy, gcc-internal-format
-#| msgid "invalid prefix \"0b\" for floating constant"
+#, gcc-internal-format
 msgid "invalid prefix %<0b%> for floating constant"
 msgstr "неи�праван префик� „0b“ за покретну кон�танту"
 
 #: expr.cc:727
-#, fuzzy, gcc-internal-format
-#| msgid "invalid prefix \"0b\" for floating constant"
+#, gcc-internal-format
 msgid "invalid prefix %<0o%> for floating constant"
-msgstr "неи�праван префик� „0b“ за покретну кон�танту"
+msgstr "неи�праван префик� „0o“ за покретну кон�танту"
 
 #: expr.cc:740
 msgid "use of C++17 hexadecimal floating constant"
-msgstr "употреба Ц++17 хек�адецималне покретне кон�танте"
+msgstr "употреба C++17 хек�адецималне покретне кон�танте"
 
 #: expr.cc:744
 msgid "use of C99 hexadecimal floating constant"
-msgstr "употреба Ц99 хек�адецималне покретне кон�танте"
+msgstr "употреба C99 хек�адецималне покретне кон�танте"
 
 #: expr.cc:789
-#, fuzzy, gcc-internal-format
-#| msgid "invalid suffix \"%.*s\" on floating constant"
+#, gcc-internal-format
 msgid "invalid suffix %<%.*s%> on floating constant"
 msgstr "неи�праван �уфик� „%.*s“ на покретној кон�танти"
 
 #: expr.cc:800 expr.cc:870
-#, fuzzy, gcc-internal-format
-#| msgid "traditional C rejects the \"%.*s\" suffix"
+#, gcc-internal-format
 msgid "traditional C rejects the %<%.*s%> suffix"
-msgstr "уобичајени Ц одбацује „%.*s“ �уфик�"
+msgstr "�тандардни C одбацује „%.*s“ �уфик�"
 
 #: expr.cc:809
 msgid "suffix for double constant is a GCC extension"
-msgstr "�уфик� за дво�труку кон�танту је ГЦЦ проширење"
+msgstr "�уфик� за дво�труку кон�танту је GCC проширење"
 
 #: expr.cc:815
-#, fuzzy, gcc-internal-format
-#| msgid "invalid suffix \"%.*s\" with hexadecimal floating constant"
+#, gcc-internal-format
 msgid "invalid suffix %<%.*s%> with hexadecimal floating constant"
 msgstr "неи�праван �уфик� „%.*s“ �а хек�адецималном покретном кон�тантом"
 
 #: expr.cc:829 expr.cc:833
-#, fuzzy
-#| msgid "decimal float constants are a C23 feature"
 msgid "decimal floating constants are a C23 feature"
-msgstr "децималне покретне кон�танте �у „C23“ функције"
+msgstr "децималне покретне кон�танте �у C23 функције"
 
 #: expr.cc:853
-#, fuzzy, gcc-internal-format
-#| msgid "invalid suffix \"%.*s\" on integer constant"
+#, gcc-internal-format
 msgid "invalid suffix %<%.*s%> on integer constant"
 msgstr "неи�праван �уфик� „%.*s“ на кон�танти целог броја"
 
 #: expr.cc:878
 msgid "use of C++11 long long integer constant"
-msgstr "употреба Ц++11 дуге дуге кон�танте целог броја"
+msgstr "употреба C++11 дуге дуге кон�танте целог броја"
 
 #: expr.cc:879
 msgid "use of C99 long long integer constant"
-msgstr "употреба Ц99 дуге дуге кон�танте целог броја"
+msgstr "употреба C99 дуге дуге кон�танте целог броја"
 
 #: expr.cc:894
 #, gcc-internal-format
 msgid "use of C++23 %<size_t%> integer constant"
-msgstr "употреба Ц++23 %<size_t%> кон�танте целог броја"
+msgstr "употреба C++23 „size_t“ кон�танте целог броја"
 
 #: expr.cc:895
 #, gcc-internal-format
 msgid "use of C++23 %<make_signed_t<size_t>%> integer constant"
-msgstr "употреба Ц++23 %<make_signed_t<size_t>%> кон�танте целог броја"
+msgstr "употреба C++23 „make_signed_t<size_t>“ кон�танте целог броја"
 
 #: expr.cc:905 expr.cc:916
 #, gcc-internal-format
 msgid "ISO C does not support literal %<wb%> suffixes before C23"
-msgstr "ISO C не подржава префик�е %<wb%> литерала пре C23"
+msgstr "ISO C не подржава префик�е „wb“ литерала пре C23"
 
 #: expr.cc:931
 msgid "imaginary constants are a GCC extension"
-msgstr "имагинарне кон�танте �у ГЦЦ проширења"
+msgstr "имагинарне кон�танте �у GCC проширења"
 
 #: expr.cc:939
-#, fuzzy
-#| msgid "binary constants are a C23 feature or GCC extension"
 msgid "imaginary constants are a C2Y feature or GCC extension"
-msgstr "бинарне кон�танте �у „C23“ функције или GCC проширења"
+msgstr "имагинарне кон�танте �у C23 функције или GCC проширења"
 
 #: expr.cc:944
-#, fuzzy
-#| msgid "binary constants are a C23 feature"
 msgid "imaginary constants are a C2Y feature"
-msgstr "бинарне кон�танте �у „C23“ функције"
+msgstr "имагинарне кон�танте �у C23 функције"
 
 #: expr.cc:956
 msgid "binary constants are a C++14 feature or GCC extension"
-msgstr "бинарне кон�танте �у Ц++14 функције или ГЦЦ проширења"
+msgstr "бинарне кон�танте �у C++14 функције или GCC проширења"
 
 #: expr.cc:961
 msgid "binary constants are a C23 feature or GCC extension"
@@ -847,16 +756,14 @@ msgid "binary constants are a C23 feature"
 msgstr "бинарне кон�танте �у „C23“ функције"
 
 #: expr.cc:974
-#, fuzzy, gcc-internal-format
-#| msgid "binary constants are a C23 feature or GCC extension"
+#, gcc-internal-format
 msgid "%<0o%> prefixed constants are a C2Y feature or GCC extension"
-msgstr "бинарне кон�танте �у „C23“ функције или GCC проширења"
+msgstr "„0o“ префик�иране кон�танте �у C2Y функције или GCC проширење"
 
 #: expr.cc:979
-#, fuzzy, gcc-internal-format
-#| msgid "binary constants are a C23 feature"
+#, gcc-internal-format
 msgid "%<0o%> prefixed constants are a C2Y feature"
-msgstr "бинарне кон�танте �у „C23“ функције"
+msgstr "„0o“ префик�иране кон�танте �у C2Y функције"
 
 #: expr.cc:1077
 msgid "integer constant is too large for its type"
@@ -869,29 +776,25 @@ msgstr "кон�танта целог броја је толико велика
 #: expr.cc:1189
 #, gcc-internal-format
 msgid "%<defined%> in %<#embed%> parameter"
-msgstr ""
+msgstr "„defined“ у „#embed“ параметру"
 
 #: expr.cc:1206
-#, fuzzy, gcc-internal-format
-#| msgid "missing ')' after \"defined\""
+#, gcc-internal-format
 msgid "missing %<)%> after %<defined%>"
-msgstr "недо�таје ( након „defined“"
+msgstr "недо�таје „(“ након „defined“"
 
 #: expr.cc:1213
-#, fuzzy, gcc-internal-format
-#| msgid "operator \"defined\" requires an identifier"
+#, gcc-internal-format
 msgid "operator %<defined%> requires an identifier"
 msgstr "оператор „defined“ захтева одредника"
 
 #: expr.cc:1221
-#, fuzzy, gcc-internal-format
-#| msgid "(\"%s\" is an alternative token for \"%s\" in C++)"
+#, gcc-internal-format
 msgid "(%qs is an alternative token for %qs in C++)"
-msgstr "(„%s“ је резервни �имбол за „%s“ у Ц++-у)"
+msgstr "(„%qs“ је резервни �имбол за „%qs“ у C++-)"
 
 #: expr.cc:1234
-#, fuzzy, gcc-internal-format
-#| msgid "this use of \"defined\" may not be portable"
+#, gcc-internal-format
 msgid "this use of %<defined%> may not be portable"
 msgstr "ова употреба „defined“ не може бити прено�ива"
 
@@ -901,21 +804,20 @@ msgstr "�ловно�т коју је одредио кори�ник у изр
 
 #: expr.cc:1284
 msgid "floating constant in preprocessor expression"
-msgstr "покретна кон�танта у изразу претптоце�ора"
+msgstr "покретна кон�танта у изразу претпроце�ора"
 
 #: expr.cc:1290
 msgid "imaginary number in preprocessor expression"
-msgstr "имагинаран број у изразу претптоце�ора"
+msgstr "имагинаран број у изразу претпроце�ора"
 
 #: expr.cc:1339
-#, fuzzy, gcc-internal-format
-#| msgid "\"%s\" is not defined, evaluates to 0"
+#, gcc-internal-format
 msgid "%qs is not defined, evaluates to %<0%>"
-msgstr "„%s“ није одређено, процењује �е на 0"
+msgstr "„%qs“ није одређено, процењује �е на „0“"
 
 #: expr.cc:1351
 msgid "assertions are a GCC extension"
-msgstr "тврдње �у ГЦЦ проширења"
+msgstr "тврдње �у GCC проширења"
 
 #: expr.cc:1355
 msgid "assertions are a deprecated extension"
@@ -928,11 +830,11 @@ msgstr "неуравнотежен �тек у %s"
 
 #: expr.cc:1633
 msgid "negative embed parameter operand"
-msgstr ""
+msgstr "негативан операнд угнежденог параметра"
 
 #: expr.cc:1639
 msgid "too large embed parameter operand"
-msgstr ""
+msgstr "превелик операнд угнежденог параметра"
 
 #: expr.cc:1658
 #, gcc-internal-format, gfc-internal-format
@@ -940,68 +842,59 @@ msgid "impossible operator '%u'"
 msgstr "немогућ оператор у „%u“"
 
 #: expr.cc:1759
-#, fuzzy, gcc-internal-format
-#| msgid "missing ')' in expression"
+#, gcc-internal-format
 msgid "missing %<)%> in expression"
-msgstr "недо�таје ) у изразу"
+msgstr "недо�таје „)“ у изразу"
 
 #: expr.cc:1788
-#, fuzzy, gcc-internal-format
-#| msgid "'?' without following ':'"
+#, gcc-internal-format
 msgid "%<?%> without following %<:%>"
-msgstr "„?“ без пратећег „:“"
+msgstr "„?“ без пратеће „:“"
 
 #: expr.cc:1798
 msgid "integer overflow in preprocessor expression"
 msgstr "прекорачење целог броја у изразу претпроце�ора"
 
 #: expr.cc:1803
-#, fuzzy, gcc-internal-format
-#| msgid "missing '(' in expression"
+#, gcc-internal-format
 msgid "missing %<(%> in expression"
-msgstr "недо�таје ( у изразу"
+msgstr "недо�таје „(“ у изразу"
 
 #: expr.cc:1835
-#, fuzzy, gcc-internal-format
-#| msgid "the left operand of \"%s\" changes sign when promoted"
+#, gcc-internal-format
 msgid "the left operand of %qs changes sign when promoted"
-msgstr "леви операнд за „%s“ мења знак када је претворен"
+msgstr "леви операнд за „%qs“ мења знак када је претворен"
 
 #: expr.cc:1840
-#, fuzzy, gcc-internal-format
-#| msgid "the right operand of \"%s\" changes sign when promoted"
+#, gcc-internal-format
 msgid "the right operand of %qs changes sign when promoted"
-msgstr "де�ни операнд за „%s“ мења знак када је претворен"
+msgstr "де�ни операнд за „%qs“ мења знак када је претворен"
 
 #: expr.cc:2099
 msgid "traditional C rejects the unary plus operator"
-msgstr "уобичајени Ц одбацује унарни плу� оператор"
+msgstr "�тандардни C одбацује унарни плу� оператор"
 
 #: expr.cc:2197
-#, fuzzy, gcc-internal-format, gfc-internal-format
-#| msgid "comma operator in operand of #if"
+#, gcc-internal-format, gfc-internal-format
 msgid "comma operator in operand of #%s"
-msgstr "оператор зареза у операнду „#if“"
+msgstr "оператор зареза у операнду „#%s“"
 
 #: expr.cc:2335
-#, fuzzy, gcc-internal-format, gfc-internal-format
-#| msgid "division by zero in #if"
+#, gcc-internal-format, gfc-internal-format
 msgid "division by zero in #%s"
-msgstr "дељење нулом у „#if“"
+msgstr "дељење нулом у „#%s“"
 
 #: files.cc:530
-#, fuzzy, gcc-internal-format
-#| msgid "NULL directory in find_file"
+#, gcc-internal-format
 msgid "NULL directory in %<find_file%>"
-msgstr "�ИШТ�В�� директоријум у пронађи_датотеку"
+msgstr "�ИШТ� директоријум у „find_file“"
 
 #: files.cc:603
 msgid "one or more PCH files were found, but they were invalid"
-msgstr "једна или више ПЦХ датотека је пронађено, али �у неи�правне"
+msgstr "једна или више PCH датотека је пронађено, али �у неи�правне"
 
 #: files.cc:607
-#, fuzzy, gcc-internal-format
-#| msgid "use -Winvalid-pch for more information"
+#, gcc-internal-format
 msgid "use %<-Winvalid-pch%> for more information"
 msgstr "кори�тите „-Winvalid-pch“ за више података"
 
@@ -1028,12 +921,12 @@ msgstr "нема путање укључивања у којој потражиÑ
 #: files.cc:1455
 #, gcc-internal-format
 msgid "%<gnu::base64%> parameter can be only used with %<\".\"%>"
-msgstr ""
+msgstr "„gnu::base64“ параметар �е може кори�тити �амо �а „.“"
 
 #: files.cc:1472
 #, gcc-internal-format
 msgid "%<gnu::base64%> argument not valid base64 encoded string"
-msgstr ""
+msgstr "„gnu::base64“ аргумент није и�правна шифрована ни�ка о�нове64"
 
 #: files.cc:2228
 msgid "Multiple include guards may be useful for:\n"
@@ -1042,16 +935,15 @@ msgstr "Више чувара укључивања може бити кориÑ�Ð
 #: files.cc:2306
 #, gcc-internal-format
 msgid "header guard %qs followed by %<#define%> of a different macro"
-msgstr ""
+msgstr "за чуварем заглавља „%qs“ �леди „#define“ другог макроа"
 
 #: files.cc:2310
 #, gcc-internal-format
 msgid "%qs is defined here; did you mean %qs?"
-msgstr ""
+msgstr "„%qs“ је дефини�ано овде; да ли �те ми�лили „%qs“?"
 
 #: init.cc:676
-#, fuzzy, gcc-internal-format
-#| msgid "cppchar_t must be an unsigned type"
+#, gcc-internal-format
 msgid "%<cppchar_t%> must be an unsigned type"
 msgstr "„cppchar_t“ мора бити неозначена вр�та"
 
@@ -1061,37 +953,33 @@ msgid "preprocessor arithmetic has maximum precision of %lu bits; target require
 msgstr "аритметика претпроце�ора има највећу тачно�т од %lu бита; мета затева %lu бита"
 
 #: init.cc:687
-#, fuzzy, gcc-internal-format
-#| msgid "CPP arithmetic must be at least as precise as a target int"
+#, gcc-internal-format
 msgid "CPP arithmetic must be at least as precise as a target %<int%>"
-msgstr "ЦПП аритметика мора бити тачна барем колико и циљни цео број"
+msgstr "CPP аритметика мора бити тачна барем колико и циљни „int“"
 
 #: init.cc:691
-#, fuzzy, gcc-internal-format
-#| msgid "target char is less than 8 bits wide"
+#, gcc-internal-format
 msgid "target %<char%> is less than 8 bits wide"
-msgstr "циљни знак је дужи мање од 8 бита"
+msgstr "циљни „char“ је дужи мање од 8 бита"
 
 #: init.cc:695
-#, fuzzy, gcc-internal-format
-#| msgid "target wchar_t is narrower than target char"
+#, gcc-internal-format
 msgid "target %<wchar_t%> is narrower than target %<char%>"
-msgstr "циљни широки знак (wchar_t) је ужи од циљног знака"
+msgstr "циљни „wchar_t“ је ужи од циљног „char“"
 
 #: init.cc:699
-#, fuzzy, gcc-internal-format
-#| msgid "target int is narrower than target char"
+#, gcc-internal-format
 msgid "target %<int%> is narrower than target %<char%>"
-msgstr "циљни цео број је ужи од циљног знака"
+msgstr "циљни „int“ је ужи од циљног „char“"
 
 #: init.cc:704
 msgid "CPP half-integer narrower than CPP character"
-msgstr "ЦПП полуцео број је ужи од ЦПП знака"
+msgstr "CPP полуцео број је ужи од CPP знака"
 
 #: init.cc:708
 #, gcc-internal-format, gfc-internal-format
 msgid "CPP on this host cannot handle wide character constants over %lu bits, but the target requires %lu bits"
-msgstr "ЦПП на овом рачунару не може да рукује дуге кон�танте знака преко %lu бита, али мета захтева %lu бита"
+msgstr "CPP на овом рачунару не може да рукује дуге кон�танте знака преко %lu бита, али мета захтева %lu бита"
 
 #: lex.cc:1104
 msgid "backslash and newline separated by space"
@@ -1099,43 +987,41 @@ msgstr "контра коÑ�а и нови ред Ñ�у раздвојени раÐ
 
 #: lex.cc:1109 lex.cc:1145
 msgid "trailing whitespace"
-msgstr ""
+msgstr "пратећа празнина"
 
 #: lex.cc:1116
 msgid "backslash-newline at end of file"
 msgstr "контра ко�а-нови ред на крају датотеке"
 
 #: lex.cc:1132
-#, fuzzy, gcc-internal-format
-#| msgid "trigraph ??%c converted to %c"
+#, gcc-internal-format
 msgid "trigraph %<??%c%> converted to %<%c%>"
-msgstr "триграф ??%c је претворен у %c"
+msgstr "триграф „??%c“ је претворен у „%c“"
 
 #: lex.cc:1138
-#, fuzzy, gcc-internal-format
-#| msgid "trigraph ??%c ignored, use -trigraphs to enable"
+#, gcc-internal-format
 msgid "trigraph %<??%c%> ignored, use %<-trigraphs%> to enable"
-msgstr "триграф ??%c је занемарен, кори�тите „-trigraphs“ да укључите"
+msgstr "триграф „??%c“ је занемарен, кори�тите „-trigraphs“ да укључите"
 
 #: lex.cc:1149
 msgid "too many consecutive spaces in leading whitespace"
-msgstr ""
+msgstr "превише уза�топних размака у водећој празнини"
 
 #: lex.cc:1154
 msgid "tab after space in leading whitespace"
-msgstr ""
+msgstr "табулатор након размака у водећој празнини"
 
 #: lex.cc:1161
 msgid "whitespace other than spaces in leading whitespace"
-msgstr ""
+msgstr "празнина о�им размака у водећој празнини"
 
 #: lex.cc:1167
 msgid "whitespace other than tabs in leading whitespace"
-msgstr ""
+msgstr "празнина о�им табулатора у водећој празнини"
 
 #: lex.cc:1173
 msgid "whitespace other than spaces and tabs in leading whitespace"
-msgstr ""
+msgstr "празнина о�им размака и табулатора у водећој празнини"
 
 #: lex.cc:1623
 msgid "end of bidirectional context"
@@ -1150,50 +1036,42 @@ msgid "unpaired UTF-8 bidirectional control character detected"
 msgstr "неупарени УТФ-8 дво�мерни контролни знак је откривен"
 
 #: lex.cc:1706
-#, fuzzy, gcc-internal-format
-#| msgid "UTF-8 vs UCN mismatch when closing a context by \"%s\""
+#, gcc-internal-format
 msgid "UTF-8 vs UCN mismatch when closing a context by %qs"
-msgstr "„UTF-8 vs UCN“ не одговара када затварам контек�т �а „%s“"
+msgstr "„UTF-8 vs UCN“ не одговара када затварам контек�т �а „%qs“"
 
 #: lex.cc:1715
-#, fuzzy, gcc-internal-format
-#| msgid "\"%s\" is closing an unopened context"
+#, gcc-internal-format
 msgid "%qs is closing an unopened context"
-msgstr "„%s“ затвара неотворени контек�т"
+msgstr "„%qs“ затвара неотворени контек�т"
 
 #: lex.cc:1719
-#, fuzzy, gcc-internal-format
-#| msgid "found problematic Unicode character \"%s\""
+#, gcc-internal-format
 msgid "found problematic Unicode character %qs"
-msgstr "нађох проблематичан Јуникод знак „%s“"
+msgstr "нађох проблематичан Јуникод знак „%qs“"
 
 #: lex.cc:1749 lex.cc:1755
-#, fuzzy, gcc-internal-format
-#| msgid "invalid UTF-8 character <%x><%x>"
+#, gcc-internal-format
 msgid "invalid UTF-8 character %<<%x>%>"
-msgstr "неи�праван УТФ-8 знак „<%x><%x>“"
+msgstr "неи�праван УТФ-8 знак „<%x>“"
 
 #: lex.cc:1765 lex.cc:1771
-#, fuzzy, gcc-internal-format
-#| msgid "invalid UTF-8 character <%x><%x><%x>"
+#, gcc-internal-format
 msgid "invalid UTF-8 character %<<%x><%x>%>"
-msgstr "неи�праван УТФ-8 знак „<%x><%x><%x>“"
+msgstr "неи�праван УТФ-8 знак „<%x><%x>“"
 
 #: lex.cc:1781 lex.cc:1787
-#, fuzzy, gcc-internal-format
-#| msgid "invalid UTF-8 character <%x><%x><%x><%x>"
+#, gcc-internal-format
 msgid "invalid UTF-8 character %<<%x><%x><%x>%>"
-msgstr "неи�праван УТФ-8 знак „<%x><%x><%x><%x>“"
+msgstr "неи�праван УТФ-8 знак „<%x><%x><%x>“"
 
 #: lex.cc:1797 lex.cc:1803
-#, fuzzy, gcc-internal-format
-#| msgid "invalid UTF-8 character <%x><%x><%x><%x>"
+#, gcc-internal-format
 msgid "invalid UTF-8 character %<<%x><%x><%x><%x>%>"
 msgstr "неи�праван УТФ-8 знак „<%x><%x><%x><%x>“"
 
 #: lex.cc:1885
-#, fuzzy, gcc-internal-format
-#| msgid "\"/*\" within comment"
+#, gcc-internal-format
 msgid "%</*%> within comment"
 msgstr "„/*“ унутар напомене"
 
@@ -1207,62 +1085,53 @@ msgid "null character(s) ignored"
 msgstr "ништаван знак(ци) је занемарен(и)"
 
 #: lex.cc:2063
-#, fuzzy, gcc-internal-format
-#| msgid "`%.*s' is not in NFKC"
+#, gcc-internal-format
 msgid "%<%.*s%> is not in NFKC"
-msgstr "„%.*s“ није у �ФКЦ-у"
+msgstr "„%.*s“ није у NFKC-у"
 
 #: lex.cc:2066 lex.cc:2069
-#, fuzzy, gcc-internal-format
-#| msgid "`%.*s' is not in NFC"
+#, gcc-internal-format
 msgid "%<%.*s%> is not in NFC"
-msgstr "„%.*s“ није у �ФЦ-у"
+msgstr "„%.*s“ није у NFC-у"
 
 #: lex.cc:2158
-#, fuzzy, gcc-internal-format
-#| msgid "__VA_OPT__ is not available until C++20"
+#, gcc-internal-format
 msgid "%<__VA_OPT__%> is not available until C++20"
-msgstr "„__VA_OPT__“ није до�тупно �ве до „Ц++20“"
+msgstr "„__VA_OPT__“ није до�тупно �ве до C++20"
 
 #: lex.cc:2161
-#, fuzzy, gcc-internal-format
-#| msgid "__VA_OPT__ is not available until C23"
+#, gcc-internal-format
 msgid "%<__VA_OPT__%> is not available until C23"
-msgstr "„__VA_OPT__“ није до�тупно �ве до „C23“"
+msgstr "„__VA_OPT__“ није до�тупно �ве до C23"
 
 #: lex.cc:2169
-#, fuzzy, gcc-internal-format
-#| msgid "__VA_OPT__ can only appear in the expansion of a C++20 variadic macro"
+#, gcc-internal-format
 msgid "%<__VA_OPT__%> can only appear in the expansion of a C++20 variadic macro"
-msgstr "„__VA_OPT__“ може �амо да �е појави у ширењу вариадик макроа „Ц++20“"
+msgstr "„__VA_OPT__“ може �амо да �е појави у ширењу вариадик макроа C++20"
 
 #: lex.cc:2186
-#, fuzzy, gcc-internal-format
-#| msgid "attempt to use poisoned \"%s\""
+#, gcc-internal-format
 msgid "attempt to use poisoned %qs"
-msgstr "покушај коришћења отрованог „%s“"
+msgstr "покушај коришћења отрованог „%qs“"
 
 #: lex.cc:2191
 msgid "poisoned here"
 msgstr "отровано овде"
 
 #: lex.cc:2201
-#, fuzzy, gcc-internal-format
-#| msgid "__VA_ARGS__ can only appear in the expansion of a C++11 variadic macro"
+#, gcc-internal-format
 msgid "%<__VA_ARGS__%> can only appear in the expansion of a C++11 variadic macro"
-msgstr "„__VA_ARGS__“ може �амо да �е појави у ширењу вариадик макроа Ц++11"
+msgstr "„__VA_ARGS__“ може �амо да �е појави у ширењу вариадик макроа C++11"
 
 #: lex.cc:2205
-#, fuzzy, gcc-internal-format
-#| msgid "__VA_ARGS__ can only appear in the expansion of a C99 variadic macro"
+#, gcc-internal-format
 msgid "%<__VA_ARGS__%> can only appear in the expansion of a C99 variadic macro"
-msgstr "„__VA_ARGS__“ може �амо да �е појави у ширењу вариадик макроа Ц99"
+msgstr "„__VA_ARGS__“ може �амо да �е појави у ширењу вариадик макроа C99"
 
 #: lex.cc:2217
-#, fuzzy, gcc-internal-format
-#| msgid "identifier \"%s\" is a special operator name in C++"
+#, gcc-internal-format
 msgid "identifier %qs is a special operator name in C++"
-msgstr "одредник „%s“ је назив по�ебног оператора у Ц++-у"
+msgstr "одредник „%qs“ је назив по�ебног оператора у C++-у"
 
 #: lex.cc:2353
 msgid "adjacent digit separators"
@@ -1270,7 +1139,7 @@ msgstr "бли�ки раздвојници цифре"
 
 #: lex.cc:2502
 msgid "invalid suffix on literal; C++11 requires a space between literal and string macro"
-msgstr "неи�праван �уфик� у �ловно�ти; Ц++11 захтева размак између �ловно�ти и макроа ни�ке"
+msgstr "неи�праван �уфик� у �ловно�ти; C++11 захтева размак између �ловно�ти и макроа ни�ке"
 
 #: lex.cc:2723
 msgid "raw string delimiter longer than 16 characters"
@@ -1296,44 +1165,40 @@ msgstr "ништаван знак је причуван у �ловно�ти"
 #: lex.cc:2953
 #, gcc-internal-format, gfc-internal-format
 msgid "missing terminating %c character"
-msgstr "недо�таје завршни знак %c"
+msgstr "недо�таје завршни знак „%c“"
 
 #: lex.cc:2986
 msgid "C++11 requires a space between string literal and macro"
-msgstr "Ц++11 захтева размак између �ловно�ти и макроа ни�ке"
+msgstr "C++11 захтева размак између �ловно�ти и макроа ни�ке"
 
 #: lex.cc:3579
 msgid "module control-line cannot be in included file"
 msgstr "контролни ред модула не може бити у укљученој датотеци"
 
 #: lex.cc:3593
-#, fuzzy, gcc-internal-format
-#| msgid "module control-line \"%s\" cannot be an object-like macro"
+#, gcc-internal-format
 msgid "module control-line %qs cannot be an object-like macro"
-msgstr "контролни ред модула „%s“ не може бити макро налик објекту"
+msgstr "контролни ред модула „%qs“ не може бити макро налик објекту"
 
 #: lex.cc:3631
-#, fuzzy, gcc-internal-format
-#| msgid "module control-line \"%s\" cannot be an object-like macro"
+#, gcc-internal-format
 msgid "module name %qs cannot be an object-like macro"
-msgstr "контролни ред модула „%s“ не може бити макро налик објекту"
+msgstr "контролни ред модула „%qs“ не може бити макро налик објекту"
 
 #: lex.cc:3637
-#, fuzzy, gcc-internal-format
-#| msgid "module control-line \"%s\" cannot be an object-like macro"
+#, gcc-internal-format
 msgid "module partition %qs cannot be an object-like macro"
-msgstr "контролни ред модула „%s“ не може бити макро налик објекту"
+msgstr "партиција модула „%qs“ не може бити макро налик објекту"
 
 #: lex.cc:3658
-#, fuzzy, gcc-internal-format
-#| msgid "'\\o' not followed by '{'"
+#, gcc-internal-format
 msgid "module name followed by %<(%>"
-msgstr "за „\\o“ не �леди „{“"
+msgstr "за називом модула �леди „(“"
 
 #: lex.cc:3662
 #, gcc-internal-format
 msgid "module partition followed by %<(%>"
-msgstr ""
+msgstr "за партицијом модула �леди „(“"
 
 #: lex.cc:4071 lex.cc:5504 traditional.cc:174
 msgid "unterminated comment"
@@ -1341,7 +1206,7 @@ msgstr "неокончана напомена"
 
 #: lex.cc:4085 lex.cc:4120
 msgid "C++ style comments are not allowed in ISO C90"
-msgstr "�апомене Ц++ �тила ни�у допуштене у ИСО Ц90"
+msgstr "�апомене C++ �тила ни�у допуштене у ISO C90"
 
 #: lex.cc:4088 lex.cc:4099 lex.cc:4123
 msgid "(this will be reported only once per input file)"
@@ -1349,7 +1214,7 @@ msgstr "(ово ће бити извештено �амо једном по ул
 
 #: lex.cc:4097
 msgid "C++ style comments are incompatible with C90"
-msgstr "�апомене Ц++ �тила ни�у �агла�не �а Ц90"
+msgstr "�апомене C++ �тила ни�у �агла�не �а C90"
 
 #: lex.cc:4129
 msgid "multi-line comment"
@@ -1374,70 +1239,59 @@ msgid "'##' cannot appear at either end of __VA_OPT__"
 msgstr "„##“ не може да �е појави у или на крају „__VA_OPT__“"
 
 #: macro.cc:144
-#, fuzzy, gcc-internal-format
-#| msgid "__VA_OPT__ may not appear in a __VA_OPT__"
+#, gcc-internal-format
 msgid "%<__VA_OPT__%> may not appear in a %<__VA_OPT__%>"
 msgstr "„__VA_OPT__“ не може да �е појави у „__VA_OPT__“"
 
 #: macro.cc:157
-#, fuzzy, gcc-internal-format
-#| msgid "__VA_OPT__ must be followed by an open parenthesis"
+#, gcc-internal-format
 msgid "%<__VA_OPT__%> must be followed by an open parenthesis"
 msgstr "„__VA_OPT__“ мора да буде праћено отвореном заградом"
 
 #: macro.cc:235
-#, fuzzy, gcc-internal-format
-#| msgid "unterminated __VA_OPT__"
+#, gcc-internal-format
 msgid "unterminated %<__VA_OPT__%>"
 msgstr "неокончано „__VA_OPT__“"
 
 #: macro.cc:396
-#, fuzzy, gcc-internal-format
-#| msgid "\"%s\" used outside of preprocessing directive"
+#, gcc-internal-format
 msgid "%qs used outside of preprocessing directive"
-msgstr "„%s“ �е кори�ти ван директиве претпроце�ора"
+msgstr "„%qs“ �е кори�ти ван директиве претпроце�ора"
 
 #: macro.cc:407
-#, fuzzy, gcc-internal-format
-#| msgid "missing '(' before \"%s\" operand"
+#, gcc-internal-format
 msgid "missing %<(%> before %qs operand"
-msgstr "недо�таје ( пре операнда „%s“"
+msgstr "недо�таје „(“ пре операнда „%qs“"
 
 #: macro.cc:425
-#, fuzzy, gcc-internal-format
-#| msgid "operator \"%s\" requires a header-name"
+#, gcc-internal-format
 msgid "operator %qs requires a header-name"
-msgstr "оператор „%s“ захтева назив заглавља"
+msgstr "оператор „%qs“ захтева назив заглавља"
 
 #: macro.cc:454
-#, fuzzy, gcc-internal-format
-#| msgid "missing ')' after \"%s\" operand"
+#, gcc-internal-format
 msgid "missing %<)%> after %qs operand"
-msgstr "недо�таје ) након операнда „%s“"
+msgstr "недо�таје „)“ након операнда „%qs“"
 
 #: macro.cc:499
-#, fuzzy, gcc-internal-format
-#| msgid "empty filename in #%s"
+#, gcc-internal-format
 msgid "empty filename in %qs"
-msgstr "празан назив датотеке у #%s"
+msgstr "празан назив датотеке у „%qs“"
 
 #: macro.cc:533
-#, fuzzy, gcc-internal-format
-#| msgid "macro \"%s\" is not used"
+#, gcc-internal-format
 msgid "macro %qs is not used"
-msgstr "макро „%s“ није коришћен"
+msgstr "макро „%qs“ није коришћен"
 
 #: macro.cc:572 macro.cc:888
-#, fuzzy, gcc-internal-format
-#| msgid "invalid built-in macro \"%s\""
+#, gcc-internal-format
 msgid "invalid built-in macro %qs"
-msgstr "неи�праван макро уградње „%s“"
+msgstr "неи�праван макро уградње „%qs“"
 
 #: macro.cc:579 macro.cc:687
-#, fuzzy, gcc-internal-format
-#| msgid "macro \"%s\" might prevent reproducible builds"
+#, gcc-internal-format
 msgid "macro %qs might prevent reproducible builds"
-msgstr "макро „%s“ може да �пречи поновљиве изградње"
+msgstr "макро „%qs“ може да �пречи поновљиве изградње"
 
 #: macro.cc:610
 msgid "could not determine file timestamp"
@@ -1448,22 +1302,19 @@ msgid "could not determine date and time"
 msgstr "не могу да одредим датум и време"
 
 #: macro.cc:733
-#, fuzzy, gcc-internal-format
-#| msgid "__COUNTER__ expanded inside directive with -fdirectives-only"
+#, gcc-internal-format
 msgid "%<__COUNTER__%> expanded inside directive with %<-fdirectives-only%>"
 msgstr "„__COUNTER__“ је проширен у директиви �а „-fdirectives-only“"
 
 #: macro.cc:760
-#, fuzzy, gcc-internal-format
-#| msgid "suggest not using #elif in traditional C"
+#, gcc-internal-format
 msgid "%<__has_embed%> not supported in traditional C"
-msgstr "�аветује да �е не кори�ти „#elif“ у уобичајеном Ц-у"
+msgstr "„__has_embed“ није подржано у �тандардном C-у"
 
 #: macro.cc:1007
-#, fuzzy, gcc-internal-format
-#| msgid "invalid string literal, ignoring final '\\'"
+#, gcc-internal-format
 msgid "invalid string literal, ignoring final %<\\%>"
-msgstr "неи�правна �ловно�т ни�ке, занемарујем завршну \\"
+msgstr "неи�правна �ловно�т ни�ке, занемарујем завршну „\\“"
 
 #: macro.cc:1071
 #, gcc-internal-format
@@ -1471,183 +1322,162 @@ msgid "pasting \"%.*s\" and \"%.*s\" does not give a valid preprocessing token"
 msgstr "убацивање „%.*s“ и „%.*s“ не даје и�правну �купину предобраде"
 
 #: macro.cc:1203
-#, fuzzy, gcc-internal-format
-#| msgid "ISO C++11 requires at least one argument for the \"...\" in a variadic macro"
+#, gcc-internal-format
 msgid "ISO C++11 requires at least one argument for the %<...%> in a variadic macro"
-msgstr "ИСО Ц++11 захтева барем један аргумент за „...“ у вариадик макроу"
+msgstr "ISO C++11 захтева барем један аргумент за „...“ у вариадик макроу"
 
 #: macro.cc:1207
-#, fuzzy, gcc-internal-format
-#| msgid "ISO C99 requires at least one argument for the \"...\" in a variadic macro"
+#, gcc-internal-format
 msgid "ISO C99 requires at least one argument for the %<...%> in a variadic macro"
-msgstr "ИСО Ц99 захтева барем један аргумент за „...“ у вариадик макроу"
+msgstr "ISO C99 захтева барем један аргумент за „...“ у вариадик макроу"
 
 #: macro.cc:1214
-#, fuzzy, gcc-internal-format
-#| msgid "macro \"%s\" requires %u arguments, but only %u given"
+#, gcc-internal-format
 msgid "macro %qs requires %u arguments, but only %u given"
-msgstr "макро „%s“ захтева %u аргумента, али �амо %u �у дата"
+msgstr "макро „%qs“ захтева %u аргумента, али �амо %u �у дата"
 
 #: macro.cc:1219
-#, fuzzy, gcc-internal-format
-#| msgid "macro \"%s\" passed %u arguments, but takes just %u"
+#, gcc-internal-format
 msgid "macro %qs passed %u arguments, but takes just %u"
-msgstr "макро „%s“ је про�ледио %u аргумента, али узима �амо %u"
+msgstr "макро „%qs“ је про�ледио %u аргумента, али узима �амо %u"
 
 #: macro.cc:1223
-#, fuzzy, gcc-internal-format
-#| msgid "macro \"%s\" defined here"
+#, gcc-internal-format
 msgid "macro %qs defined here"
-msgstr "макро „%s“ је дефини�ан овде"
+msgstr "макро „%qs“ је дефини�ан овде"
 
 #: macro.cc:1417 traditional.cc:822
-#, fuzzy, gcc-internal-format
-#| msgid "unterminated argument list invoking macro \"%s\""
+#, gcc-internal-format
 msgid "unterminated argument list invoking macro %qs"
-msgstr "бе�коначан �пи�ак аргумената призивајући макро „%s“"
+msgstr "неокончан �пи�ак аргумената призивајући макро „%qs“"
 
 #: macro.cc:1563
-#, fuzzy, gcc-internal-format
-#| msgid "function-like macro \"%s\" must be used with arguments in traditional C"
+#, gcc-internal-format
 msgid "function-like macro %qs must be used with arguments in traditional C"
-msgstr "макро налик функцији „%s“ мора бити коришћен �а аргументима у уобичајеном Ц-у"
+msgstr "макро налик функцији „%qs“ мора бити коришћен �а аргументима у �тандардном C-у"
 
 #: macro.cc:2398
 #, gcc-internal-format, gfc-internal-format
 msgid "invoking macro %s argument %d: empty macro arguments are undefined in ISO C++98"
-msgstr "призивајући макро %s аргумент %d: празни аргументи макроа ни�у одређени у ИСО Ц++98"
+msgstr "призивајући макро %s аргумент %d: празни аргументи макроа ни�у одређени у ISO C++98"
 
 #: macro.cc:2406 macro.cc:2415
 #, gcc-internal-format, gfc-internal-format
 msgid "invoking macro %s argument %d: empty macro arguments are undefined in ISO C90"
-msgstr "призивајући макро %s аргумент %d: празни аргументи макроа ни�у одређени у ИСО Ц90"
+msgstr "призивајући макро %s аргумент %d: празни аргументи макроа ни�у одређени у ISO C90"
 
 #: macro.cc:3238
 #, gcc-internal-format
 msgid "%qc in module name or partition comes from or after macro expansion"
-msgstr ""
+msgstr "„%qc“ у називу модула или партицији долази из или након ширења макроа"
 
 #: macro.cc:3481
-#, fuzzy, gcc-internal-format
-#| msgid "duplicate macro parameter \"%s\""
+#, gcc-internal-format
 msgid "duplicate macro parameter %qs"
-msgstr "параметар дво�труког макроа „%s“"
+msgstr "дво�труки параметар макроа „%qs“"
 
 #: macro.cc:3563
-#, fuzzy, gcc-internal-format
-#| msgid "expected parameter name, found \"%s\""
+#, gcc-internal-format
 msgid "expected parameter name, found %qs"
-msgstr "очекивах назив параметра, нађох „%s“"
+msgstr "очекивах назив параметра, нађох „%qs“"
 
 #: macro.cc:3564
-#, fuzzy, gcc-internal-format
-#| msgid "expected ',' or ')', found \"%s\""
+#, gcc-internal-format
 msgid "expected %<,%> or %<)%>, found %qs"
-msgstr "очекивах , или ), нађох „%s“"
+msgstr "очекивах „,“ или „)“, нађох %qs"
 
 #: macro.cc:3565
 msgid "expected parameter name before end of line"
 msgstr "очекивах назив параметра пре краја реда"
 
 #: macro.cc:3566
-#, fuzzy, gcc-internal-format
-#| msgid "expected ')' before end of line"
+#, gcc-internal-format
 msgid "expected %<)%> before end of line"
-msgstr "очекивах ) пре краја реда"
+msgstr "очекивах „)“ пре краја реда"
 
 #: macro.cc:3567
-#, fuzzy, gcc-internal-format
-#| msgid "expected ')' after \"...\""
+#, gcc-internal-format
 msgid "expected %<)%> after %<...%>"
-msgstr "очекивах ) након ..."
+msgstr "очекивах „)“ након „...“"
 
 #: macro.cc:3624
 msgid "anonymous variadic macros were introduced in C++11"
-msgstr "безимени вариадик макрои �у уведени у Ц++11"
+msgstr "безимени вариадик макрои �у уведени у C++11"
 
 #: macro.cc:3625 macro.cc:3629
 msgid "anonymous variadic macros were introduced in C99"
-msgstr "безимени вариадик макрои �у уведени у Ц99"
+msgstr "безимени вариадик макрои �у уведени у C99"
 
 #: macro.cc:3635
 msgid "ISO C++ does not permit named variadic macros"
-msgstr "ИСО Ц++ не дозвољава именоване вариадик макрое"
+msgstr "ISO C++ не дозвољава именоване вариадик макрое"
 
 #: macro.cc:3636
 msgid "ISO C does not permit named variadic macros"
-msgstr "ИСО Ц не дозвољава именоване вариадик макрое"
+msgstr "ISO C не дозвољава именоване вариадик макрое"
 
 #: macro.cc:3682
-#, fuzzy, gcc-internal-format
-#| msgid "'##' cannot appear at either end of a macro expansion"
+#, gcc-internal-format
 msgid "%<##%> cannot appear at either end of a macro expansion"
 msgstr "„##“ не може да �е појави у или на крају макро ширења"
 
 #: macro.cc:3720
 msgid "ISO C++11 requires whitespace after the macro name"
-msgstr "ИСО Ц++11 захтева празнине након назива макроа"
+msgstr "ISO C++11 захтева празнине након назива макроа"
 
 #: macro.cc:3721
 msgid "ISO C99 requires whitespace after the macro name"
-msgstr "ИСО Ц99 захтева празнине након назива макроа"
+msgstr "ISO C99 захтева празнине након назива макроа"
 
 #: macro.cc:3745
 msgid "missing whitespace after the macro name"
 msgstr "недо�таје празнина након назива макроа"
 
 #: macro.cc:3798
-#, fuzzy, gcc-internal-format
-#| msgid "'#' is not followed by a macro parameter"
+#, gcc-internal-format
 msgid "%<#%> is not followed by a macro parameter"
 msgstr "за „#“ не �леди параметар макроа"
 
 #: macro.cc:3961
-#, fuzzy, gcc-internal-format
-#| msgid "\"%s\" redefined"
+#, gcc-internal-format
 msgid "%qs redefined"
-msgstr "„%s“ је поново одређено"
+msgstr "„%qs“ је поново одређено"
 
 #: macro.cc:3965
 msgid "this is the location of the previous definition"
 msgstr "ово је ме�то претходне одреднице"
 
 #: macro.cc:4103
-#, fuzzy, gcc-internal-format
-#| msgid "macro argument \"%s\" would be stringified in traditional C"
+#, gcc-internal-format
 msgid "macro argument %qs would be stringified in traditional C"
-msgstr "макро аргумент „%s“ ће бити ни�кован у уобичајеном Ц-у"
+msgstr "макро аргумент „%qs“ ће бити ни�кован у �тандардном C-у"
 
 #: pch.cc:90 pch.cc:342 pch.cc:356 pch.cc:374 pch.cc:380 pch.cc:389 pch.cc:396
 msgid "while writing precompiled header"
 msgstr "за време упи�ивања претходно преведеног заглавља"
 
 #: pch.cc:616
-#, fuzzy, gcc-internal-format
-#| msgid "%s: not used because `%.*s' is poisoned"
+#, gcc-internal-format
 msgid "%s: not used because %<%.*s%> is poisoned"
 msgstr "%s: није употребљено зато што је „%.*s“ отровано"
 
 #: pch.cc:638
-#, fuzzy, gcc-internal-format
-#| msgid "%s: not used because `%.*s' not defined"
+#, gcc-internal-format
 msgid "%s: not used because %<%.*s%> not defined"
 msgstr "%s: није употребљено зато што „%.*s“ није одређено"
 
 #: pch.cc:650
-#, fuzzy, gcc-internal-format
-#| msgid "%s: not used because `%.*s' defined as `%s' not `%.*s'"
+#, gcc-internal-format
 msgid "%s: not used because %<%.*s%> defined as %qs not %<%.*s%>"
-msgstr "%s: није употребљено зато што „%.*s“ одређено као „%s“ није „%.*s“"
+msgstr "%s: није употребљено зато што „%.*s“ одређено као „%qs“ није „%.*s“"
 
 #: pch.cc:693
-#, fuzzy, gcc-internal-format
-#| msgid "%s: not used because `%s' is defined"
+#, gcc-internal-format
 msgid "%s: not used because %qs is defined"
-msgstr "%s: није употребљено зато што је „%s“ одређено"
+msgstr "%s: није употребљено зато што је „%qs“ одређено"
 
 #: pch.cc:713
-#, fuzzy, gcc-internal-format
-#| msgid "%s: not used because `__COUNTER__' is invalid"
+#, gcc-internal-format
 msgid "%s: not used because %<__COUNTER__%> is invalid"
 msgstr "%s: није употребљено зато што „__COUNTER__“ није и�правно"
 
@@ -1656,10 +1486,9 @@ msgid "while reading precompiled header"
 msgstr "за време читања претходно преведеног заглавља"
 
 #: traditional.cc:891
-#, fuzzy, gcc-internal-format
-#| msgid "detected recursion whilst expanding macro \"%s\""
+#, gcc-internal-format
 msgid "detected recursion whilst expanding macro %qs"
-msgstr "откривено је дубачење за време ширења макроа „%s“"
+msgstr "откривено је дубачење за време ширења макроа „%qs“"
 
 #: traditional.cc:1114
 msgid "syntax error in macro parameter list"
diff --git a/libgcc/ChangeLog b/libgcc/ChangeLog
index 7e907b6..0962ddf 100644
--- a/libgcc/ChangeLog
+++ b/libgcc/ChangeLog
@@ -1,3 +1,37 @@
+2025-07-31  Wilco Dijkstra  <wilco.dijkstra@arm.com>
+
+	* config/aarch64/cpuinfo.c (__init_cpu_features_constructor):
+	Remove unused features, add support for CSSC and MOPS.
+
+2025-07-31  Wilco Dijkstra  <wilco.dijkstra@arm.com>
+
+	* config/aarch64/cpuinfo.c: Cleanup HWCAP defines.
+
+2025-07-31  Yury Khrustalev  <yury.khrustalev@arm.com>
+
+	* config/aarch64/cpuinfo.c (__ifunc_arg_t): Likewise.
+	(__init_cpu_features): obtain and assign values for the
+	fields _hwcap3 and _hwcap4.
+	(__init_cpu_features_constructor): check _size in the
+	arg argument.
+
+2025-07-17  Richard Sandiford  <richard.sandiford@arm.com>
+	    Yury Khrustalev  <yury.khrustalev@arm.com>
+
+	* config/aarch64/linux-unwind.h (aarch64_fallback_frame_state):
+	If a signal was raised while there was an uncommitted lazy save,
+	commit the save as part of the unwind process.
+
+2025-07-16  John Ericson  <git@JohnEricson.me>
+
+	* Makefile.in:  Delete dead `MACHMODE_H` variable
+
+2025-07-10  Jan Dubiec  <jdx@o2.pl>
+
+	PR target/116363
+	* libgcc2.c (__fixunssfDI): Fix SFtype to UDWtype conversion for targets
+	without LIBGCC2_HAS_DF_MODE defined
+
 2025-05-27  Jakub Jelinek  <jakub@redhat.com>
 
 	* config/t-softfp (softfp_bid_list): Don't guard with
diff --git a/libgcc/Makefile.in b/libgcc/Makefile.in
index 0719fd0..f7b48dc 100644
--- a/libgcc/Makefile.in
+++ b/libgcc/Makefile.in
@@ -193,7 +193,6 @@ AWK = @AWK@
 GCC_FOR_TARGET = $(CC)
 LIPO = @LIPO@
 LIPO_FOR_TARGET = $(LIPO)
-MACHMODE_H = machmode.h mode-classes.def insn-modes.h
 NM = @NM@
 NM_FOR_TARGET = $(NM)
 RANLIB_FOR_TARGET = $(RANLIB)
@@ -220,7 +219,6 @@ export INSTALL_DATA
 export LIB1ASMSRC
 export LIBGCC2_CFLAGS
 export LIPO_FOR_TARGET
-export MACHMODE_H
 export NM_FOR_TARGET
 export STRIP_FOR_TARGET
 export RANLIB_FOR_TARGET
diff --git a/libgcc/config/aarch64/cpuinfo.c b/libgcc/config/aarch64/cpuinfo.c
index dda9dc6..f8c10037 100644
--- a/libgcc/config/aarch64/cpuinfo.c
+++ b/libgcc/config/aarch64/cpuinfo.c
@@ -27,18 +27,18 @@
 #if __has_include(<sys/auxv.h>)
 #include <sys/auxv.h>
 
-#if __has_include(<sys/ifunc.h>)
-#include <sys/ifunc.h>
-#else
+/* The following struct is ABI-correct description of the 2nd argument for an
+   ifunc resolver as per SYSVABI spec (see link below).  It is safe to extend
+   it with new fields.  The ifunc resolver implementations must always check
+   the runtime size of the buffer using the value in the _size field.
+   https://github.com/ARM-software/abi-aa/blob/main/sysvabi64/sysvabi64.rst.  */
 typedef struct __ifunc_arg_t {
   unsigned long _size;
   unsigned long _hwcap;
   unsigned long _hwcap2;
+  unsigned long _hwcap3;
+  unsigned long _hwcap4;
 } __ifunc_arg_t;
-#endif
-
-#if __has_include(<asm/hwcap.h>)
-#include <asm/hwcap.h>
 
 /* Architecture features used in Function Multi Versioning.  */
 struct {
@@ -46,196 +46,64 @@ struct {
   /* As features grows new fields could be added.  */
 } __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon));
 
-#ifndef _IFUNC_ARG_HWCAP
 #define _IFUNC_ARG_HWCAP (1ULL << 62)
-#endif
-#ifndef AT_HWCAP
 #define AT_HWCAP 16
-#endif
-#ifndef HWCAP_FP
-#define HWCAP_FP (1 << 0)
-#endif
-#ifndef HWCAP_ASIMD
-#define HWCAP_ASIMD (1 << 1)
-#endif
-#ifndef HWCAP_EVTSTRM
-#define HWCAP_EVTSTRM (1 << 2)
-#endif
-#ifndef HWCAP_AES
-#define HWCAP_AES (1 << 3)
-#endif
-#ifndef HWCAP_PMULL
-#define HWCAP_PMULL (1 << 4)
-#endif
-#ifndef HWCAP_SHA1
-#define HWCAP_SHA1 (1 << 5)
-#endif
-#ifndef HWCAP_SHA2
-#define HWCAP_SHA2 (1 << 6)
-#endif
-#ifndef HWCAP_CRC32
-#define HWCAP_CRC32 (1 << 7)
-#endif
-#ifndef HWCAP_ATOMICS
-#define HWCAP_ATOMICS (1 << 8)
-#endif
-#ifndef HWCAP_FPHP
-#define HWCAP_FPHP (1 << 9)
-#endif
-#ifndef HWCAP_ASIMDHP
-#define HWCAP_ASIMDHP (1 << 10)
-#endif
-#ifndef HWCAP_CPUID
-#define HWCAP_CPUID (1 << 11)
-#endif
-#ifndef HWCAP_ASIMDRDM
-#define HWCAP_ASIMDRDM (1 << 12)
-#endif
-#ifndef HWCAP_JSCVT
-#define HWCAP_JSCVT (1 << 13)
-#endif
-#ifndef HWCAP_FCMA
-#define HWCAP_FCMA (1 << 14)
-#endif
-#ifndef HWCAP_LRCPC
-#define HWCAP_LRCPC (1 << 15)
-#endif
-#ifndef HWCAP_DCPOP
-#define HWCAP_DCPOP (1 << 16)
-#endif
-#ifndef HWCAP_SHA3
-#define HWCAP_SHA3 (1 << 17)
-#endif
-#ifndef HWCAP_SM3
-#define HWCAP_SM3 (1 << 18)
-#endif
-#ifndef HWCAP_SM4
-#define HWCAP_SM4 (1 << 19)
-#endif
-#ifndef HWCAP_ASIMDDP
-#define HWCAP_ASIMDDP (1 << 20)
-#endif
-#ifndef HWCAP_SHA512
-#define HWCAP_SHA512 (1 << 21)
-#endif
-#ifndef HWCAP_SVE
-#define HWCAP_SVE (1 << 22)
-#endif
-#ifndef HWCAP_ASIMDFHM
-#define HWCAP_ASIMDFHM (1 << 23)
-#endif
-#ifndef HWCAP_DIT
-#define HWCAP_DIT (1 << 24)
-#endif
-#ifndef HWCAP_ILRCPC
-#define HWCAP_ILRCPC (1 << 26)
-#endif
-#ifndef HWCAP_FLAGM
-#define HWCAP_FLAGM (1 << 27)
-#endif
-#ifndef HWCAP_SSBS
-#define HWCAP_SSBS (1 << 28)
-#endif
-#ifndef HWCAP_SB
-#define HWCAP_SB (1 << 29)
-#endif
-#ifndef HWCAP_PACA
-#define HWCAP_PACA (1 << 30)
-#endif
-#ifndef HWCAP_PACG
-#define HWCAP_PACG (1UL << 31)
-#endif
-
-#ifndef AT_HWCAP2
 #define AT_HWCAP2 26
-#endif
-#ifndef HWCAP2_DCPODP
-#define HWCAP2_DCPODP (1 << 0)
-#endif
-#ifndef HWCAP2_SVE2
-#define HWCAP2_SVE2 (1 << 1)
-#endif
-#ifndef HWCAP2_SVEAES
-#define HWCAP2_SVEAES (1 << 2)
-#endif
-#ifndef HWCAP2_SVEPMULL
-#define HWCAP2_SVEPMULL (1 << 3)
-#endif
-#ifndef HWCAP2_SVEBITPERM
-#define HWCAP2_SVEBITPERM (1 << 4)
-#endif
-#ifndef HWCAP2_SVESHA3
-#define HWCAP2_SVESHA3 (1 << 5)
-#endif
-#ifndef HWCAP2_SVESM4
-#define HWCAP2_SVESM4 (1 << 6)
-#endif
-#ifndef HWCAP2_FLAGM2
-#define HWCAP2_FLAGM2 (1 << 7)
-#endif
-#ifndef HWCAP2_FRINT
-#define HWCAP2_FRINT (1 << 8)
-#endif
-#ifndef HWCAP2_SVEI8MM
-#define HWCAP2_SVEI8MM (1 << 9)
-#endif
-#ifndef HWCAP2_SVEF32MM
-#define HWCAP2_SVEF32MM (1 << 10)
-#endif
-#ifndef HWCAP2_SVEF64MM
-#define HWCAP2_SVEF64MM (1 << 11)
-#endif
-#ifndef HWCAP2_SVEBF16
-#define HWCAP2_SVEBF16 (1 << 12)
-#endif
-#ifndef HWCAP2_I8MM
-#define HWCAP2_I8MM (1 << 13)
-#endif
-#ifndef HWCAP2_BF16
-#define HWCAP2_BF16 (1 << 14)
-#endif
-#ifndef HWCAP2_DGH
-#define HWCAP2_DGH (1 << 15)
-#endif
-#ifndef HWCAP2_RNG
-#define HWCAP2_RNG (1 << 16)
-#endif
-#ifndef HWCAP2_BTI
-#define HWCAP2_BTI (1 << 17)
-#endif
-#ifndef HWCAP2_MTE
-#define HWCAP2_MTE (1 << 18)
-#endif
-#ifndef HWCAP2_RPRES
-#define HWCAP2_RPRES (1 << 21)
-#endif
-#ifndef HWCAP2_MTE3
-#define HWCAP2_MTE3 (1 << 22)
-#endif
-#ifndef HWCAP2_SME
-#define HWCAP2_SME (1 << 23)
-#endif
-#ifndef HWCAP2_SME_I16I64
-#define HWCAP2_SME_I16I64 (1 << 24)
-#endif
-#ifndef HWCAP2_SME_F64F64
-#define HWCAP2_SME_F64F64 (1 << 25)
-#endif
-#ifndef HWCAP2_WFXT
-#define HWCAP2_WFXT (1UL << 31)
-#endif
-#ifndef HWCAP2_EBF16
-#define HWCAP2_EBF16 (1UL << 32)
-#endif
-#ifndef HWCAP2_SVE_EBF16
-#define HWCAP2_SVE_EBF16 (1UL << 33)
-#endif
-#ifndef HWCAP2_SME2
-#define HWCAP2_SME2 (1UL << 37)
-#endif
-#ifndef HWCAP2_LRCPC3
-#define HWCAP2_LRCPC3	(1UL << 46)
-#endif
+#define AT_HWCAP3 29
+#define AT_HWCAP4 30
+
+#define HWCAP_FP		(1 << 0)
+#define HWCAP_ASIMD		(1 << 1)
+#define HWCAP_PMULL		(1 << 4)
+#define HWCAP_SHA2		(1 << 6)
+#define HWCAP_CRC32		(1 << 7)
+#define HWCAP_ATOMICS		(1 << 8)
+#define HWCAP_FPHP		(1 << 9)
+#define HWCAP_ASIMDHP		(1 << 10)
+#define HWCAP_ASIMDRDM		(1 << 12)
+#define HWCAP_JSCVT		(1 << 13)
+#define HWCAP_FCMA		(1 << 14)
+#define HWCAP_LRCPC		(1 << 15)
+#define HWCAP_DCPOP		(1 << 16)
+#define HWCAP_SHA3		(1 << 17)
+#define HWCAP_SM3		(1 << 18)
+#define HWCAP_SM4		(1 << 19)
+#define HWCAP_ASIMDDP		(1 << 20)
+#define HWCAP_SVE		(1 << 22)
+#define HWCAP_ASIMDFHM		(1 << 23)
+#define HWCAP_DIT		(1 << 24)
+#define HWCAP_ILRCPC		(1 << 26)
+#define HWCAP_FLAGM		(1 << 27)
+#define HWCAP_SSBS		(1 << 28)
+#define HWCAP_SB		(1 << 29)
+
+#define HWCAP2_DCPODP		(1 << 0)
+#define HWCAP2_SVE2		(1 << 1)
+#define HWCAP2_SVEPMULL		(1 << 3)
+#define HWCAP2_SVEBITPERM	(1 << 4)
+#define HWCAP2_SVESHA3		(1 << 5)
+#define HWCAP2_SVESM4		(1 << 6)
+#define HWCAP2_FLAGM2		(1 << 7)
+#define HWCAP2_FRINT		(1 << 8)
+#define HWCAP2_SVEF32MM		(1 << 10)
+#define HWCAP2_SVEF64MM		(1 << 11)
+#define HWCAP2_I8MM		(1 << 13)
+#define HWCAP2_BF16		(1 << 14)
+#define HWCAP2_RNG		(1 << 16)
+#define HWCAP2_BTI		(1 << 17)
+#define HWCAP2_MTE		(1 << 18)
+#define HWCAP2_SME		(1 << 23)
+#define HWCAP2_SME_I16I64	(1 << 24)
+#define HWCAP2_SME_F64F64	(1 << 25)
+#define HWCAP2_WFXT		(1UL << 31)
+#define HWCAP2_CSSC		(1UL << 34)
+#define HWCAP2_SME2		(1UL << 37)
+#define HWCAP2_MOPS		(1UL << 43)
+#define HWCAP2_LRCPC3		(1UL << 46)
+
+#define __IFUNC_ARG_SIZE_HWCAP2 (sizeof (unsigned long) * 3)
+#define __IFUNC_ARG_SIZE_HWCAP3 (sizeof (unsigned long) * 4)
+#define __IFUNC_ARG_SIZE_HWCAP4 (sizeof (unsigned long) * 5)
 
 static void
 __init_cpu_features_constructor (unsigned long hwcap,
@@ -247,8 +115,14 @@ __init_cpu_features_constructor (unsigned long hwcap,
 #define extractBits(val, start, number) \
   (val & ((1UL << number) - 1UL) << start) >> start
   unsigned long hwcap2 = 0;
-  if (hwcap & _IFUNC_ARG_HWCAP)
+  if ((hwcap & _IFUNC_ARG_HWCAP) && arg->_size >= __IFUNC_ARG_SIZE_HWCAP2)
     hwcap2 = arg->_hwcap2;
+  unsigned long hwcap3 __attribute__ ((unused)) = 0;
+  if ((hwcap & _IFUNC_ARG_HWCAP) && arg->_size >= __IFUNC_ARG_SIZE_HWCAP3)
+    hwcap3 = arg->_hwcap3;
+  unsigned long hwcap4 __attribute__ ((unused)) = 0;
+  if ((hwcap & _IFUNC_ARG_HWCAP) && arg->_size >= __IFUNC_ARG_SIZE_HWCAP4)
+    hwcap4 = arg->_hwcap4;
   if (hwcap & HWCAP_CRC32)
     setCPUFeature(FEAT_CRC);
   if (hwcap & HWCAP_PMULL)
@@ -269,10 +143,6 @@ __init_cpu_features_constructor (unsigned long hwcap,
     setCPUFeature(FEAT_DIT);
   if (hwcap & HWCAP_ASIMDRDM)
     setCPUFeature(FEAT_RDM);
-  if (hwcap & HWCAP_AES)
-    setCPUFeature(FEAT_AES);
-  if (hwcap & HWCAP_SHA1)
-    setCPUFeature(FEAT_SHA1);
   if (hwcap & HWCAP_SHA2)
     setCPUFeature(FEAT_SHA2);
   if (hwcap & HWCAP_JSCVT)
@@ -282,19 +152,9 @@ __init_cpu_features_constructor (unsigned long hwcap,
   if (hwcap & HWCAP_SB)
     setCPUFeature(FEAT_SB);
   if (hwcap & HWCAP_SSBS)
-    {
-      setCPUFeature(FEAT_SSBS);
-      setCPUFeature(FEAT_SSBS2);
-    }
+    setCPUFeature(FEAT_SSBS2);
   if (hwcap2 & HWCAP2_MTE)
-    {
-      setCPUFeature(FEAT_MEMTAG);
-      setCPUFeature(FEAT_MEMTAG2);
-    }
-  if (hwcap2 & HWCAP2_MTE3)
-    setCPUFeature(FEAT_MEMTAG3);
-  if (hwcap2 & HWCAP2_SVEAES)
-    setCPUFeature(FEAT_SVE_AES);
+    setCPUFeature(FEAT_MEMTAG2);
   if (hwcap2 & HWCAP2_SVEPMULL)
     setCPUFeature(FEAT_SVE_PMULL128);
   if (hwcap2 & HWCAP2_SVEBITPERM)
@@ -311,24 +171,14 @@ __init_cpu_features_constructor (unsigned long hwcap,
     setCPUFeature(FEAT_RNG);
   if (hwcap2 & HWCAP2_I8MM)
     setCPUFeature(FEAT_I8MM);
-  if (hwcap2 & HWCAP2_EBF16)
-    setCPUFeature(FEAT_EBF16);
-  if (hwcap2 & HWCAP2_SVE_EBF16)
-    setCPUFeature(FEAT_SVE_EBF16);
-  if (hwcap2 & HWCAP2_DGH)
-    setCPUFeature(FEAT_DGH);
   if (hwcap2 & HWCAP2_FRINT)
     setCPUFeature(FEAT_FRINTTS);
-  if (hwcap2 & HWCAP2_SVEI8MM)
-    setCPUFeature(FEAT_SVE_I8MM);
   if (hwcap2 & HWCAP2_SVEF32MM)
     setCPUFeature(FEAT_SVE_F32MM);
   if (hwcap2 & HWCAP2_SVEF64MM)
     setCPUFeature(FEAT_SVE_F64MM);
   if (hwcap2 & HWCAP2_BTI)
     setCPUFeature(FEAT_BTI);
-  if (hwcap2 & HWCAP2_RPRES)
-    setCPUFeature(FEAT_RPRES);
   if (hwcap2 & HWCAP2_WFXT)
     setCPUFeature(FEAT_WFXT);
   if (hwcap2 & HWCAP2_SME)
@@ -339,6 +189,10 @@ __init_cpu_features_constructor (unsigned long hwcap,
     setCPUFeature(FEAT_SME_I64);
   if (hwcap2 & HWCAP2_SME_F64F64)
     setCPUFeature(FEAT_SME_F64);
+  if (hwcap2 & HWCAP2_MOPS)
+    setCPUFeature(FEAT_MOPS);
+  if (hwcap2 & HWCAP2_CSSC)
+    setCPUFeature(FEAT_CSSC);
   if (hwcap & HWCAP_FP)
     {
       setCPUFeature(FEAT_FP);
@@ -355,8 +209,6 @@ __init_cpu_features_constructor (unsigned long hwcap,
     setCPUFeature(FEAT_RCPC3);
   if (hwcap2 & HWCAP2_BF16)
     setCPUFeature(FEAT_BF16);
-  if (hwcap2 & HWCAP2_SVEBF16)
-    setCPUFeature(FEAT_SVE_BF16);
   if (hwcap & HWCAP_SVE)
     setCPUFeature(FEAT_SVE);
   if (hwcap2 & HWCAP2_SVE2)
@@ -383,20 +235,26 @@ __init_cpu_features(void)
 {
   unsigned long hwcap;
   unsigned long hwcap2;
+  unsigned long hwcap3;
+  unsigned long hwcap4;
 
   /* CPU features already initialized.  */
   if (__atomic_load_n (&__aarch64_cpu_features.features, __ATOMIC_RELAXED))
     return;
-  hwcap = getauxval(AT_HWCAP);
-  hwcap2 = getauxval(AT_HWCAP2);
+  hwcap = getauxval (AT_HWCAP);
+  hwcap2 = getauxval (AT_HWCAP2);
+  hwcap3 = getauxval (AT_HWCAP3);
+  hwcap4 = getauxval (AT_HWCAP4);
+
   __ifunc_arg_t arg;
-  arg._size = sizeof(__ifunc_arg_t);
+  arg._size = sizeof (__ifunc_arg_t);
   arg._hwcap = hwcap;
   arg._hwcap2 = hwcap2;
-  __init_cpu_features_constructor(hwcap | _IFUNC_ARG_HWCAP, &arg);
+  arg._hwcap3 = hwcap3;
+  arg._hwcap4 = hwcap4;
+  __init_cpu_features_constructor (hwcap | _IFUNC_ARG_HWCAP, &arg);
 #undef extractBits
 #undef getCPUFeature
 #undef setCPUFeature
 }
-#endif /* __has_include(<asm/hwcap.h>)  */
 #endif /* __has_include(<sys/auxv.h>)  */
diff --git a/libgcc/config/aarch64/linux-unwind.h b/libgcc/config/aarch64/linux-unwind.h
index e41ca6a..f5b73a0 100644
--- a/libgcc/config/aarch64/linux-unwind.h
+++ b/libgcc/config/aarch64/linux-unwind.h
@@ -27,7 +27,7 @@
 
 #include <signal.h>
 #include <sys/ucontext.h>
-
+#include <stdint.h>
 
 /* Since insns are always stored LE, on a BE system the opcodes will
    be loaded byte-reversed.  Therefore, define two sets of opcodes,
@@ -43,6 +43,22 @@
 
 #define MD_FALLBACK_FRAME_STATE_FOR aarch64_fallback_frame_state
 
+#ifndef FPSIMD_MAGIC
+#define FPSIMD_MAGIC 0x46508001
+#endif
+
+#ifndef TPIDR2_MAGIC
+#define TPIDR2_MAGIC 0x54504902
+#endif
+
+#ifndef ZA_MAGIC
+#define ZA_MAGIC 0x54366345
+#endif
+
+#ifndef EXTRA_MAGIC
+#define EXTRA_MAGIC 0x45585401
+#endif
+
 static _Unwind_Reason_Code
 aarch64_fallback_frame_state (struct _Unwind_Context *context,
 			      _Unwind_FrameState * fs)
@@ -58,6 +74,21 @@ aarch64_fallback_frame_state (struct _Unwind_Context *context,
     ucontext_t uc;
   };
 
+  struct tpidr2_block
+  {
+    uint64_t za_save_buffer;
+    uint16_t num_za_save_slices;
+    uint8_t reserved[6];
+  };
+
+  struct za_block
+  {
+    struct _aarch64_ctx head;
+    uint16_t vl;
+    uint16_t reserved[3];
+    uint64_t data;
+  };
+
   struct rt_sigframe *rt_;
   _Unwind_Ptr new_cfa;
   unsigned *pc = context->ra;
@@ -103,11 +134,15 @@ aarch64_fallback_frame_state (struct _Unwind_Context *context,
      field can be used to skip over unrecognized context extensions.
      The end of the context sequence is marked by a context with magic
      0 or size 0.  */
+  struct tpidr2_block *tpidr2 = 0;
+  struct za_block *za_ctx = 0;
+
   for (extension_marker = (struct _aarch64_ctx *) &sc->__reserved;
        extension_marker->magic;
        extension_marker = (struct _aarch64_ctx *)
        ((unsigned char *) extension_marker + extension_marker->size))
     {
+    restart:
       if (extension_marker->magic == FPSIMD_MAGIC)
 	{
 	  struct fpsimd_context *ctx =
@@ -139,12 +174,83 @@ aarch64_fallback_frame_state (struct _Unwind_Context *context,
 	      fs->regs.reg[AARCH64_DWARF_V0 + i].loc.offset = offset;
 	    }
 	}
+      else if (extension_marker->magic == TPIDR2_MAGIC)
+	{
+	  /* A TPIDR2 context.
+
+	     All the casting is to support big-endian ILP32.  We could read
+	     directly into TPIDR2 otherwise.  */
+	  struct { struct _aarch64_ctx h; uint64_t tpidr2; } *ctx
+		  = (void *)extension_marker;
+#if defined (__ILP32__)
+	  tpidr2 = (struct tpidr2_block *) (uintptr_t) ctx->tpidr2;
+#else
+	  tpidr2 = (struct tpidr2_block *) ctx->tpidr2;
+#endif
+	}
+      else if (extension_marker->magic == ZA_MAGIC)
+	/* A ZA context.  We interpret this later.  */
+	za_ctx = (void *)extension_marker;
+      else if (extension_marker->magic == EXTRA_MAGIC)
+	{
+	  /* Extra context.  The ABI guarantees that the next _aarch64_ctx
+	     in the current list will be the zero terminator, so we can simply
+	     switch to the new list and continue from there.  The new list is
+	     also zero-terminated.
+
+	     As above, the casting is to support big-endian ILP32.  */
+	  struct { struct _aarch64_ctx h; uint64_t next; } *ctx
+		  = (void *)extension_marker;
+#if defined (__ILP32__)
+	  extension_marker = (struct _aarch64_ctx *) (uintptr_t) ctx->next;
+#else
+	  extension_marker = (struct _aarch64_ctx *) ctx->next;
+#endif
+	  goto restart;
+	}
       else
 	{
 	  /* There is context provided that we do not recognize!  */
 	}
     }
 
+  /* Signal handlers are entered with ZA in the off state (TPIDR2_ELO==0 and
+     PSTATE.ZA==0).  The normal process when transitioning from ZA being
+     dormant to ZA being off is to commit the lazy save; see the AAPCS64
+     for details.  However, this is not done when entering a signal handler.
+     Instead, linux saves the old contents of ZA and TPIDR2_EL0 to the
+     sigcontext without interpreting them further.
+
+     Therefore, if a signal handler throws an exception to code outside the
+     signal handler, the unwinder must commit the lazy save after the fact.
+     Committing a lazy save means:
+
+     (1) Storing the contents of ZA into the buffer provided by TPIDR2_EL0.
+     (2) Setting TPIDR2_EL0 to zero.
+     (3) Turning ZA off.
+
+     (2) and (3) have already been done by the call to __libgcc_arm_za_disable.
+     (1) involves copying data from the ZA sigcontext entry to the
+     corresponding lazy save buffer.  */
+  if (tpidr2 && za_ctx && tpidr2->za_save_buffer)
+    {
+      /* There is a 16-bit vector length (measured in bytes) at ZA_CTX + 8.
+	 The data itself starts at ZA_CTX + 16.
+	 As above, the casting is to support big-endian ILP32.  */
+      uint16_t vl = za_ctx->vl;
+#if defined (__ILP32__)
+      void *save_buffer = (void *) (uintptr_t) tpidr2->za_save_buffer;
+      const void *za_buffer = (void *) (uintptr_t) &za_ctx->data;
+#else
+      void *save_buffer = (void *) tpidr2->za_save_buffer;
+      const void *za_buffer = (void *) &za_ctx->data;
+#endif
+      uint64_t num_slices = tpidr2->num_za_save_slices;
+      if (num_slices > vl)
+	num_slices = vl;
+      memcpy (save_buffer, za_buffer, num_slices * vl);
+    }
+
   fs->regs.how[31] = REG_SAVED_OFFSET;
   fs->regs.reg[31].loc.offset = (_Unwind_Ptr) & (sc->sp) - new_cfa;
 
diff --git a/libgcc/config/libbid/ChangeLog b/libgcc/config/libbid/ChangeLog
index 9bc4518..fd82d6e 100644
--- a/libgcc/config/libbid/ChangeLog
+++ b/libgcc/config/libbid/ChangeLog
@@ -1,3 +1,8 @@
+2025-07-15  Andrew Pinski  <quic_apinski@quicinc.com>
+
+	* bid_binarydecimal.c (__mul_10x256_to_256): Mark c3 as being
+	used.
+
 2025-05-15  liuhongt  <hongtao.liu@intel.com>
 
 	* bid128_string.c (MIN_DIGITS): New macro.
diff --git a/libgcc/config/libbid/bid_binarydecimal.c b/libgcc/config/libbid/bid_binarydecimal.c
index daca2ff..12e32b9 100644
--- a/libgcc/config/libbid/bid_binarydecimal.c
+++ b/libgcc/config/libbid/bid_binarydecimal.c
@@ -132,6 +132,7 @@ UINT64 CY;                                                      \
   __mul_10x64(p1,c1,a1,c0);                                     \
   __mul_10x64(p2,c2,a2,c1);                                     \
   __mul_10x64(p3,c3,a3,c2);                                     \
+  (void)c3;							\
 }
 
 // Set up indices for low and high parts, depending on the endian-ness.
diff --git a/libgcc/libgcc2.c b/libgcc/libgcc2.c
index faefff3..df99c78 100644
--- a/libgcc/libgcc2.c
+++ b/libgcc/libgcc2.c
@@ -2187,36 +2187,21 @@ __fixunssfDI (SFtype a)
   if (a < 1)
     return 0;
   if (a < Wtype_MAXp1_F)
-    return (UWtype)a;
+    return (UWtype) a;
   if (a < Wtype_MAXp1_F * Wtype_MAXp1_F)
     {
-      /* Since we know that there are fewer significant bits in the SFmode
-	 quantity than in a word, we know that we can convert out all the
-	 significant bits in one step, and thus avoid losing bits.  */
-
-      /* ??? This following loop essentially performs frexpf.  If we could
-	 use the real libm function, or poke at the actual bits of the fp
-	 format, it would be significantly faster.  */
-
-      UWtype shift = 0, counter;
-      SFtype msb;
-
-      a /= Wtype_MAXp1_F;
-      for (counter = W_TYPE_SIZE / 2; counter != 0; counter >>= 1)
-	{
-	  SFtype counterf = (UWtype)1 << counter;
-	  if (a >= counterf)
-	    {
-	      shift |= counter;
-	      a /= counterf;
-	    }
-	}
-
-      /* Rescale into the range of one word, extract the bits of that
-	 one word, and shift the result into position.  */
-      a *= Wtype_MAXp1_F;
-      counter = a;
-      return (DWtype)counter << shift;
+      /* We assume that SFtype -> UWtype and UWtype -> UDWtype casts work
+         properly. Obviously, we *cannot* assume that SFtype -> UDWtype
+         works as expected.  */
+      SFtype a_hi, a_lo;
+
+      a_hi = a / Wtype_MAXp1_F;
+      a_lo = a - a_hi * Wtype_MAXp1_F;
+
+      /* A lot of parentheses. This is to make it very clear what is
+         the sequence of operations.  */
+      return ((UDWtype) ((UWtype) a_hi)) << W_TYPE_SIZE
+            | (UDWtype) ((UWtype) a_lo);
     }
   return -1;
 #else
diff --git a/libgcobol/ChangeLog b/libgcobol/ChangeLog
index e851b39..91a3b86 100644
--- a/libgcobol/ChangeLog
+++ b/libgcobol/ChangeLog
@@ -1,3 +1,28 @@
+2025-07-13  Robert Dubner  <rdubner@symas.com>
+
+	* common-defs.h (PTRCAST): Moved here from libgcobol.h.
+	* libgcobol.h (PTRCAST): Deleted.
+
+2025-07-10  James K. Lowden  <jklowden@cobolworx.com>
+
+	* common-defs.h (cdf_enabled_exceptions): Use new CDF state.
+
+2025-07-09  Robert Dubner  <rdubner@symas.com>
+	    James K. Lowden  <jklowden@cobolworx.com>
+
+	* libgcobol.cc (__gg__accept_envar): ACCEPT/DISPLAY environment variables.
+	(accept_envar): Likewise.
+	(default_exception_handler): Refine system log entries.
+	(open_syslog): Likewise.
+	(__gg__set_env_name): ACCEPT/DISPLAY environment variables.
+	(__gg__get_env_name): ACCEPT/DISPLAY environment variables.
+	(__gg__get_env_value): ACCEPT/DISPLAY environment variables.
+	(__gg__set_env_value): ACCEPT/DISPLAY environment variables.
+	(__gg__fprintf_stderr): Adjust __attribute__ for printf.
+	(__gg__set_arg_num): ACCEPT/DISPLAY command-line arguments.
+	(__gg__accept_arg_value): ACCEPT/DISPLAY command-line arguments.
+	(__gg__get_file_descriptor): DISPLAY on os_filename[] /dev device.
+
 2025-06-20  James K. Lowden  <jklowden@cobolworx.com>
 
 	* LICENSE: New file.
diff --git a/libgcobol/common-defs.h b/libgcobol/common-defs.h
index a78022a..15d0683 100644
--- a/libgcobol/common-defs.h
+++ b/libgcobol/common-defs.h
@@ -84,6 +84,14 @@
 
 #define MINIMUM_ALLOCATION_SIZE 16
 
+// This was part of an exercise to make cppcheck shut up about invalid
+// pointer type conversions.
+// It was also to avoid having reinterpret_cast<> all over the place.
+// So, instead of                 reinterpret_cast<char *>(VALUE)
+// I sometimes use                PTRCAST(char, VALUE)
+// Note that "(char *)" is implied by "PTRCAST(char, VALUE)"
+#define PTRCAST(TYPE, VALUE) static_cast<TYPE *>(static_cast<void *>(VALUE))
+
 /*
  * User-defined names in IBM COBOL can have at most 30 characters.
  * For DBCS, the maximum is 14.
@@ -589,7 +597,7 @@ class cbl_enabled_exceptions_t : protected std::set<cbl_enabled_exception_t>
   cbl_enabled_exceptions_t& operator=( const cbl_enabled_exceptions_t& ) = default;
 };
 
-extern cbl_enabled_exceptions_t enabled_exceptions;
+cbl_enabled_exceptions_t& cdf_enabled_exceptions();
 
 template <typename T>
 T enabled_exception_match( T beg, T end, ec_type_t type, size_t file ) {
diff --git a/libgcobol/libgcobol.cc b/libgcobol/libgcobol.cc
index 81b5b7a..c3d78d4 100644
--- a/libgcobol/libgcobol.cc
+++ b/libgcobol/libgcobol.cc
@@ -10132,49 +10132,45 @@ __gg__classify( classify_t type,
   return retval;
   }
 
-extern "C"
+static
 int
-__gg__accept_envar( cblc_field_t *tgt,
-                    size_t        tgt_offset,
-                    size_t        tgt_length,
-                    cblc_field_t *name,
-                    size_t        name_offset,
-                    size_t        name_length)
+accept_envar( cblc_field_t *tgt,
+              size_t        tgt_offset,
+              size_t        tgt_length,
+              const char   *psz_name)
   {
-  int retval;
-  tgt_length  = tgt_length  ? tgt_length  : tgt->capacity;
-  name_length = name_length ? name_length : name->capacity;
-
-  // Pick up the environment variable name, which is in teh internal codeset
-  static char   *env        = NULL;
-  static size_t  env_length = 0;
-  if( env_length < name_length+1 )
+  int retval = 1; // 1 means we couldn't find it
+  if( psz_name )
     {
-    env_length = name_length+1;
-    env = static_cast<char *>(realloc(env, env_length));
-    }
-  memcpy(env, name->data + name_offset, name_length);
-  env[name_length] = '\0';
+    tgt_length = tgt_length  ? tgt_length  : tgt->capacity;
 
-  // Get rid of leading and trailing internal_space characters:
-  char *trimmed_env = brute_force_trim(env);
+    // Pick up the environment variable name, which is in the internal codeset
+    char *env = strdup(psz_name);
+    massert(env);
 
-  // Convert the name to the console codeset:
-  __gg__internal_to_console_in_place(trimmed_env, strlen(trimmed_env));
+    // Get rid of leading and trailing internal_space characters:
+    char *trimmed_env = brute_force_trim(env);
 
-  // Pick up the environment variable, and convert it to the internal codeset
-  const char *p = getenv(trimmed_env);
-  if(p)
-    {
-    char *pp = strdup(p);
-    console_to_internal(pp, strlen(pp));
-    retval = 0; // Okay
-    move_string(tgt, tgt_offset, tgt_length, pp);
-    free(pp);
+    // Convert the name to the console codeset:
+    __gg__internal_to_console_in_place(trimmed_env, strlen(trimmed_env));
+
+    // Pick up the environment variable, and convert it to the internal codeset
+    const char *p = getenv(trimmed_env);
+    if(p)
+      {
+      char *pp = strdup(p);
+      massert(pp);
+      console_to_internal(pp, strlen(pp));
+      retval = 0; // Okay
+      move_string(tgt, tgt_offset, tgt_length, pp);
+      free(pp);
+      }
+    free(env);
     }
-  else
+
+  if( retval == 1 )
     {
-    retval = 1; // Could't find it
+    // Could't find it
     exception_raise(ec_argument_imp_environment_e);
     }
 
@@ -10182,6 +10178,28 @@ __gg__accept_envar( cblc_field_t *tgt,
   }
 
 extern "C"
+int
+__gg__accept_envar( cblc_field_t *tgt,
+                    size_t        tgt_offset,
+                    size_t        tgt_length,
+              const cblc_field_t *name,
+                    size_t        name_offset,
+                    size_t        name_length)
+  {
+  //  We need the name to be nul-terminated:
+  char *p = static_cast<char *>(malloc(name_length + 1));
+  massert(p);
+  memcpy(p, name->data+name_offset, name_length);
+  p[name_length] = '\0';
+  int retval = accept_envar(tgt,
+                            tgt_offset,
+                            tgt_length,
+                            p);
+  free(p);
+  return retval;
+  }
+
+extern "C"
 bool
 __gg__set_envar(cblc_field_t    *name,
                 size_t           name_offset,
@@ -11247,35 +11265,42 @@ match_declarative( bool enabled,
   return matches;
 }
 
-/*
- * The default exception handler is called if:
- *   1.  The EC is enabled and was not handled by a Declarative, or
- *   2.  The EC is EC-I-O and was not handled by a Format-1 Declarative, or
- *   3.  The EC is EC-I-O, associated with a file, and is not OPEN or CLOSE.
- */
-static void
-default_exception_handler( ec_type_t ec )
+static
+void open_syslog(int option, int facility)
 {
+  static bool first_time = true;
+  if( first_time ) {
 #if HAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME
   /* Declared in errno.h, when available.  */
-  static const char * const ident = program_invocation_short_name;
+    static const char * const ident = program_invocation_short_name;
 #elif defined (HAVE_GETPROGNAME)
   /* Declared in stdlib.h.  */
-  static const char * const ident = getprogname();
+    static const char * const ident = getprogname();
 #else
   /* Avoid a NULL entry.  */
-  static const char * const ident = "unnamed_COBOL_program";
+    static const char * const ident = "unnamed_COBOL_program";
 #endif
-  static bool first_time = true;
-  static const int priority = LOG_INFO, option = LOG_PERROR, facility = LOG_USER;
-  ec_disposition_t disposition = ec_category_fatal_e;
-
-  if( first_time ) {
     // TODO: Program to set option in library via command-line and/or environment.
     //       Library listens to program, not to the environment.
     openlog(ident, option, facility);
     first_time = false;
   }
+}
+
+/*
+ * The default exception handler is called if:
+ *   1.  The EC is enabled and was not handled by a Declarative, or
+ *   2.  The EC is EC-I-O and was not handled by a Format-1 Declarative, or
+ *   3.  The EC is EC-I-O, associated with a file, and is not OPEN or CLOSE.
+ */
+static void
+default_exception_handler( ec_type_t ec )
+{
+  static const int priority = LOG_INFO, option = LOG_PERROR, facility = LOG_USER;
+  open_syslog(option, facility);
+
+  ec_disposition_t disposition = ec_category_fatal_e;
+
 
   if( ec != ec_none_e ) {
     auto pec = std::find_if( __gg__exception_table, __gg__exception_table_end,
@@ -13149,6 +13174,7 @@ operator<<( std::vector<cbl_declarative_t>& dcls,
 }
 
 // The first element of each array is the number of elements that follow
+// The first element of each array is the number of elements that follow
 extern "C"
 void
 __gg__set_exception_environment( uint64_t *ecs, uint64_t *dcls )
@@ -13207,6 +13233,7 @@ __gg__set_env_name( const cblc_field_t *var,
                     size_t              offset,
                     size_t              length )
   {
+  // implements DISPLAY UPON ENVIRONMENT-NAME
   free(sv_envname);
   sv_envname = static_cast<char *>(malloc(length+1));
   massert(sv_envname);
@@ -13214,12 +13241,41 @@ __gg__set_env_name( const cblc_field_t *var,
   sv_envname[length] = '\0';
   }
 
+
+extern "C"
+void
+__gg__get_env_name( cblc_field_t *dest,
+                    size_t dest_offset,
+                    size_t dest_length)
+  {
+  // Implements ACCEPT FROM ENVIRONMENT-NAME
+  // It returns the value previously established by __gg__set_env_name.
+  if( sv_envname )
+    {
+    sv_envname = strdup("");
+    }
+  move_string(dest, dest_offset, dest_length, sv_envname);
+  }
+
+extern "C"
+int
+__gg__get_env_value(cblc_field_t *dest,
+                    size_t dest_offset,
+                    size_t dest_length)
+  {
+  return accept_envar(dest,
+                      dest_offset,
+                      dest_length,
+                      sv_envname);
+  }
+
 extern "C"
 void
 __gg__set_env_value(const cblc_field_t *value,
                     size_t              offset,
                     size_t              length )
   {
+  // implements DISPLAY UPON ENVIRONMENT-VALUE
   size_t name_length  = strlen(sv_envname);
   size_t value_length = length;
 
@@ -13261,6 +13317,11 @@ __gg__set_env_value(const cblc_field_t *value,
 extern "C"
 void
 __gg__fprintf_stderr(const char *format_string, ...)
+  __attribute__ ((__format__ (__printf__, 1, 2)));
+
+extern "C"
+void
+__gg__fprintf_stderr(const char *format_string, ...)
   {
   /*  This routine allows the compiler to send stuff to stderr in a way
       that is straightforward to use..  */
@@ -13270,3 +13331,81 @@ __gg__fprintf_stderr(const char *format_string, ...)
   va_end(ap);
   }
 
+
+static int sv_argument_number = 0;
+
+extern "C"
+void
+__gg__set_arg_num( const cblc_field_t *index,
+                   size_t              index_offset,
+                   size_t              index_size )
+  {
+  // Implements DISPLAY UPON ARGUMENT-NUMBER.
+  int rdigits;
+  __int128 N = get_binary_value_local(&rdigits,
+                                      index,
+                                      index->data + index_offset,
+                                      index_size);
+  // If he gives us fractional digits, just truncate
+  N /= __gg__power_of_ten(rdigits);
+
+  // N is 1-based, per normal COBOL.  We have to decrement it here:
+  N -= 1;
+  sv_argument_number = static_cast<int>(N);
+  }
+
+extern "C"
+int
+__gg__accept_arg_value( cblc_field_t *dest,
+                        size_t dest_offset,
+                        size_t dest_length)
+  {
+  // Implements ACCEPT FROM ARGUMENT-VALUE
+  int retcode;
+  command_line_plan_b();
+  if( sv_argument_number >= stashed_argc || sv_argument_number < 0 )
+    {
+    exception_raise(ec_argument_imp_command_e);
+    retcode = 1;  // Error
+    }
+  else
+    {
+    char *retval = strdup(stashed_argv[sv_argument_number]);
+    console_to_internal(retval, strlen(retval));
+    move_string(dest, dest_offset, dest_length, retval);
+    free(retval);
+    retcode = 0;  // Okay
+
+    // The Fujitsu spec says bump this value by one.
+    sv_argument_number += 1;
+    }
+  return retcode;
+  }
+
+extern "C"
+int
+__gg__get_file_descriptor(const char *device)
+  {
+  int retval = open(device, O_WRONLY);
+  
+  if( retval == -1 )
+    {
+    char *msg;
+    int ec = asprintf(&msg,
+                      "Trying to open %s.  Got error %s",
+                      device,
+                      strerror(errno));
+    if( ec != -1 )
+      {
+      static const int priority = LOG_INFO,
+                         option = LOG_PERROR,
+                       facility = LOG_USER;
+      open_syslog(option, facility);
+      syslog(priority, "%s", msg);
+      }
+    
+    // Open a new handle to /dev/stdout, since our caller will be closing it
+    retval = open("/dev/stdout", O_WRONLY);
+    }
+  return retval;
+  }
diff --git a/libgcobol/libgcobol.h b/libgcobol/libgcobol.h
index 2e338c2..2f5c8b0 100644
--- a/libgcobol/libgcobol.h
+++ b/libgcobol/libgcobol.h
@@ -47,13 +47,6 @@ extern void __gg__mabort();
 // malloc().
 #define massert(p) if(!p){__gg__mabort();abort();}
 
-// This was part of an exercise to make cppcheck shut up about invalid
-// pointer type conversions.
-// It was also to avoid having reinterpret_cast<> all over the place.
-// So, instead of                 reinterpret_cast<char *>(VALUE)
-// I sometimes use                PTRCAST(char, VALUE)
-#define PTRCAST(TYPE, VALUE) static_cast<TYPE *>(static_cast<void *>(VALUE))
-
 extern "C" __int128 __gg__power_of_ten(int n);
 
 extern "C" __int128 __gg__dirty_to_binary_source( const char *dirty,
diff --git a/libgfortran/ChangeLog b/libgfortran/ChangeLog
index 057b850..ea41616 100644
--- a/libgfortran/ChangeLog
+++ b/libgfortran/ChangeLog
@@ -1,3 +1,9 @@
+2025-07-30  Yuao Ma  <c8ef@outlook.com>
+
+	* gfortran.map: Add split symbol.
+	* intrinsics/string_intrinsics_inc.c (string_split):
+	Runtime support for SPLIT.
+
 2025-06-18  Harald Anlauf  <anlauf@gmx.de>
 
 	PR fortran/82480
diff --git a/libgfortran/gfortran.map b/libgfortran/gfortran.map
index 742dddf..98808dc 100644
--- a/libgfortran/gfortran.map
+++ b/libgfortran/gfortran.map
@@ -2032,3 +2032,9 @@ GFORTRAN_15.2 {
     _gfortran_mmaxloc1_16_m16;
     _gfortran_smaxloc1_16_m16;
 } GFORTRAN_15;
+
+GFORTRAN_16 {
+  global:
+    _gfortran_string_split;
+    _gfortran_string_split_char4;
+} GFORTRAN_15.2;
diff --git a/libgfortran/intrinsics/string_intrinsics_inc.c b/libgfortran/intrinsics/string_intrinsics_inc.c
index d86bb6c8..aeecf68 100644
--- a/libgfortran/intrinsics/string_intrinsics_inc.c
+++ b/libgfortran/intrinsics/string_intrinsics_inc.c
@@ -33,6 +33,7 @@ see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
 #define string_verify SUFFIX(string_verify)
 #define string_trim SUFFIX(string_trim)
 #define string_minmax SUFFIX(string_minmax)
+#define string_split SUFFIX(string_split)
 #define zero_length_string SUFFIX(zero_length_string)
 #define compare_string SUFFIX(compare_string)
 
@@ -72,6 +73,10 @@ export_proto(string_trim);
 extern void string_minmax (gfc_charlen_type *, CHARTYPE **, int, int, ...);
 export_proto(string_minmax);
 
+extern gfc_charlen_type string_split (gfc_charlen_type, const CHARTYPE *,
+				      gfc_charlen_type, const CHARTYPE *,
+				      gfc_charlen_type, GFC_LOGICAL_4);
+export_proto (string_split);
 
 /* Use for functions which can return a zero-length string.  */
 static CHARTYPE zero_length_string = 0;
@@ -459,3 +464,50 @@ string_minmax (gfc_charlen_type *rlen, CHARTYPE **dest, int op, int nargs, ...)
       *dest = tmp;
     }
 }
+
+gfc_charlen_type
+string_split (gfc_charlen_type stringlen, const CHARTYPE *string,
+	      gfc_charlen_type setlen, const CHARTYPE *set,
+	      gfc_charlen_type pos, GFC_LOGICAL_4 back)
+{
+  gfc_charlen_type i, j;
+
+  if (!back)
+    {
+      if (pos > stringlen)
+	runtime_error ("If BACK is present with the value false, the value of "
+		       "POS shall be in the range [0, LEN (STRING)], "
+		       "where POS = %ld and LEN (STRING) = %ld",
+		       pos, stringlen);
+
+      for (i = pos + 1; i <= stringlen; i++)
+	{
+	  for (j = 0; j < setlen; j++)
+	    {
+	      if (string[i - 1] == set[j])
+		return i;
+	    }
+	}
+
+      return stringlen + 1;
+    }
+  else
+    {
+      if (pos < 1 || pos > (stringlen + 1))
+	runtime_error ("If BACK is present with the value true, the value of "
+		       "POS shall be in the range [1, LEN (STRING) + 1], "
+		       "where POS = %ld and LEN (STRING) = %ld",
+		       pos, stringlen);
+
+      for (i = pos - 1; i != 0; i--)
+	{
+	  for (j = 0; j < setlen; j++)
+	    {
+	      if (string[i - 1] == set[j])
+		return i;
+	    }
+	}
+
+      return 0;
+    }
+}
diff --git a/libgm2/ChangeLog b/libgm2/ChangeLog
index 7ca3c03..c068292 100644
--- a/libgm2/ChangeLog
+++ b/libgm2/ChangeLog
@@ -1,3 +1,15 @@
+2025-07-23  Gaius Mulley  <gaiusmod2@gmail.com>
+
+	PR modula2/121164
+	* libm2iso/Makefile.am (libm2iso_la_M2FLAGS): Add -Wall.
+	* libm2iso/Makefile.in: Regenerate.
+	* libm2log/Makefile.am (libm2log_la_M2FLAGS): Add -Wall.
+	* libm2log/Makefile.in: Regenerate.
+	* libm2min/Makefile.am (libm2min_la_M2FLAGS): Add -Wall.
+	* libm2min/Makefile.in: Regenerate.
+	* libm2pim/Makefile.am (libm2pim_la_M2FLAGS): Add -Wall.
+	* libm2pim/Makefile.in: Regenerate.
+
 2025-06-29  Gaius Mulley  <gaiusmod2@gmail.com>
 
 	PR modula2/117203
diff --git a/libgm2/libm2iso/Makefile.am b/libgm2/libm2iso/Makefile.am
index bd8af62..1af1d6a 100644
--- a/libgm2/libm2iso/Makefile.am
+++ b/libgm2/libm2iso/Makefile.am
@@ -206,6 +206,7 @@ libm2iso_la_M2FLAGS = \
   -fm2-pathname=m2iso -I. -Ilibm2iso -I$(GM2_SRC)/gm2-libs-iso \
   -fm2-pathname=m2pim -I$(GM2_SRC)/gm2-libs \
   -fiso -fextended-opaque -fm2-g -g -Wcase-enum \
+  -Wall \
   -Wreturn-type -fcase -fm2-prefix=m2iso
 
 if TARGET_DARWIN
diff --git a/libgm2/libm2iso/Makefile.in b/libgm2/libm2iso/Makefile.in
index 1e48ad0..308c023 100644
--- a/libgm2/libm2iso/Makefile.in
+++ b/libgm2/libm2iso/Makefile.in
@@ -594,6 +594,7 @@ FLAGS_TO_PASS = $(AM_MAKEFLAGS)
 @BUILD_ISOLIB_TRUE@  -fm2-pathname=m2iso -I. -Ilibm2iso -I$(GM2_SRC)/gm2-libs-iso \
 @BUILD_ISOLIB_TRUE@  -fm2-pathname=m2pim -I$(GM2_SRC)/gm2-libs \
 @BUILD_ISOLIB_TRUE@  -fiso -fextended-opaque -fm2-g -g -Wcase-enum \
+@BUILD_ISOLIB_TRUE@  -Wall \
 @BUILD_ISOLIB_TRUE@  -Wreturn-type -fcase -fm2-prefix=m2iso
 
 @BUILD_ISOLIB_TRUE@@TARGET_DARWIN_FALSE@libm2iso_la_link_flags =  \
diff --git a/libgm2/libm2log/Makefile.am b/libgm2/libm2log/Makefile.am
index 8da0d14..8078be8 100644
--- a/libgm2/libm2log/Makefile.am
+++ b/libgm2/libm2log/Makefile.am
@@ -136,6 +136,7 @@ libm2log_la_M2FLAGS = \
  -fm2-pathname=m2log -I$(GM2_SRC)/gm2-libs-log \
  -fm2-pathname=m2pim -I$(GM2_SRC)/gm2-libs \
  -fm2-pathname=m2iso -I$(GM2_SRC)/gm2-libs-iso \
+ -Wall \
  -Wcase-enum -Wreturn-type \
  -fcase -fm2-prefix=m2log
 
diff --git a/libgm2/libm2log/Makefile.in b/libgm2/libm2log/Makefile.in
index 6df3f16..dcd5941 100644
--- a/libgm2/libm2log/Makefile.in
+++ b/libgm2/libm2log/Makefile.in
@@ -485,6 +485,7 @@ FLAGS_TO_PASS = $(AM_MAKEFLAGS)
 @BUILD_LOGLIB_TRUE@ -fm2-pathname=m2log -I$(GM2_SRC)/gm2-libs-log \
 @BUILD_LOGLIB_TRUE@ -fm2-pathname=m2pim -I$(GM2_SRC)/gm2-libs \
 @BUILD_LOGLIB_TRUE@ -fm2-pathname=m2iso -I$(GM2_SRC)/gm2-libs-iso \
+@BUILD_LOGLIB_TRUE@ -Wall \
 @BUILD_LOGLIB_TRUE@ -Wcase-enum -Wreturn-type \
 @BUILD_LOGLIB_TRUE@ -fcase -fm2-prefix=m2log
 
diff --git a/libgm2/libm2min/Makefile.am b/libgm2/libm2min/Makefile.am
index b95b5dd..8cee85f 100644
--- a/libgm2/libm2min/Makefile.am
+++ b/libgm2/libm2min/Makefile.am
@@ -107,6 +107,7 @@ libm2min_la_CFLAGS = -I. -I$(GM2_SRC)/gm2-libs-min -I$(GM2_SRC)/gm2-libs
 libm2min_la_M2FLAGS = \
    -fm2-pathname=m2min -I. -I$(GM2_SRC)/gm2-libs-min \
    -fm2-pathname=m2pim -I$(GM2_SRC)/gm2-libs -fno-exceptions \
+   -Wall \
    -fno-m2-plugin -fno-scaffold-dynamic -fno-scaffold-main -fm2-prefix=m2min
 
 if TARGET_DARWIN
diff --git a/libgm2/libm2min/Makefile.in b/libgm2/libm2min/Makefile.in
index ce0efff..93bf8a4 100644
--- a/libgm2/libm2min/Makefile.in
+++ b/libgm2/libm2min/Makefile.in
@@ -449,6 +449,7 @@ libm2min_la_CFLAGS = -I. -I$(GM2_SRC)/gm2-libs-min -I$(GM2_SRC)/gm2-libs
 libm2min_la_M2FLAGS = \
    -fm2-pathname=m2min -I. -I$(GM2_SRC)/gm2-libs-min \
    -fm2-pathname=m2pim -I$(GM2_SRC)/gm2-libs -fno-exceptions \
+   -Wall \
    -fno-m2-plugin -fno-scaffold-dynamic -fno-scaffold-main -fm2-prefix=m2min
 
 @TARGET_DARWIN_FALSE@libm2min_la_link_flags = $(am__append_1)
diff --git a/libgm2/libm2pim/Makefile.am b/libgm2/libm2pim/Makefile.am
index f8e9aae..56bbb3e 100644
--- a/libgm2/libm2pim/Makefile.am
+++ b/libgm2/libm2pim/Makefile.am
@@ -185,6 +185,7 @@ libm2pim_la_M2FLAGS = \
   -fm2-pathname=m2pim -I. -I$(GM2_SRC)/gm2-libs \
   -fm2-pathname=m2iso -I$(GM2_SRC)/gm2-libs-iso \
   -fm2-g -g -Wcase-enum -Wreturn-type \
+  -Wall \
   -fcase -fm2-prefix=m2pim
 
 if TARGET_DARWIN
diff --git a/libgm2/libm2pim/Makefile.in b/libgm2/libm2pim/Makefile.in
index 8d101c4..63f8fc9 100644
--- a/libgm2/libm2pim/Makefile.in
+++ b/libgm2/libm2pim/Makefile.in
@@ -564,6 +564,7 @@ FLAGS_TO_PASS = $(AM_MAKEFLAGS)
 @BUILD_PIMLIB_TRUE@  -fm2-pathname=m2pim -I. -I$(GM2_SRC)/gm2-libs \
 @BUILD_PIMLIB_TRUE@  -fm2-pathname=m2iso -I$(GM2_SRC)/gm2-libs-iso \
 @BUILD_PIMLIB_TRUE@  -fm2-g -g -Wcase-enum -Wreturn-type \
+@BUILD_PIMLIB_TRUE@  -Wall \
 @BUILD_PIMLIB_TRUE@  -fcase -fm2-prefix=m2pim
 
 @BUILD_PIMLIB_TRUE@@TARGET_DARWIN_FALSE@libm2pim_la_link_flags =  \
diff --git a/libgo/go/syscall/socket.go b/libgo/go/syscall/socket.go
index 54a4a99..35665d5 100644
--- a/libgo/go/syscall/socket.go
+++ b/libgo/go/syscall/socket.go
@@ -467,7 +467,7 @@ func Sendto(fd int, p []byte, flags int, to Sockaddr) (err error) {
 
 func recvmsgRaw(fd int, p, oob []byte, flags int, rsa *RawSockaddrAny) (n, oobn int, recvflags int, err error) {
 	var msg Msghdr
-	msg.Name = (*byte)(unsafe.Pointer(&rsa))
+	msg.Name = (*byte)(unsafe.Pointer(rsa))
 	msg.Namelen = uint32(SizeofSockaddrAny)
 	var iov Iovec
 	if len(p) > 0 {
diff --git a/libgo/runtime/go-memclr.c b/libgo/runtime/go-memclr.c
index 53b8117..84df98d 100644
--- a/libgo/runtime/go-memclr.c
+++ b/libgo/runtime/go-memclr.c
@@ -11,50 +11,39 @@ void memclrNoHeapPointers(void *, uintptr)
   __attribute__ ((no_split_stack));
 
 void
-memclrNoHeapPointers (void *p1, uintptr len)
+memclrNoHeapPointers(void *p1, uintptr len)
 {
-
-#if !defined(__PPC64__)
-  __builtin_memset(p1, 0, len);
-#else
-  int64 rem,drem,i;
-  uint64 offset;
-  volatile uint64 *vp;
+  const uintptr ptr_size = sizeof(p1);
+  uintptr rem,drem,i;
+  uintptr offset;
+  volatile uintptr *vp;
 
   if (len == 0) {
     return;
   }
   rem = len;
 
-  offset = (uint64)p1 % 8;
-  // This memset is OK since it can't contain
-  // an 8 byte aligned pointer.
-  if ((rem < 8) || (offset > 0 && offset+rem <= 16)) {
+  offset = (uintptr)p1 % ptr_size;
+  if (rem < ptr_size || offset > 0) {
+    // This memset is OK since it can't contain
+    // an pointer aligned pointer.
     __builtin_memset(p1, 0, rem);
     return;
   }
-  // Move initial bytes to get to 8 byte boundary
-  if (offset > 0) {
-    __builtin_memset(p1, 0, 8-offset);
-    p1 = (void*)((char*)p1+8-offset);
-    rem -= 8-offset;
-  }
 
-  // If at least 8 bytes left, clear
-  drem = rem>>3;
+  drem = rem / ptr_size;
 
-  vp = (volatile uint64*)(p1);
+  vp = (volatile uintptr*)(p1);
   // Without the use of volatile here, the compiler
   // might convert the loop into a memset.
   for (i=0; i<drem; i++) {
     *vp = 0;
     vp++;
-    rem -= 8;
+    rem -= ptr_size;
   }
-  p1 = (void*)((char*)p1 + 8*drem);
-  // Clear any remaining
+  // Clear any remaining bytes.
   if (rem > 0) {
-    __builtin_memset (p1, 0, rem);
+    p1 = (void*)((char*)p1 + ptr_size*drem);
+    __builtin_memset(p1, 0, rem);
   }
-#endif
 }
diff --git a/libgo/runtime/go-memmove.c b/libgo/runtime/go-memmove.c
index 1ca3f48..1dbd2b3 100644
--- a/libgo/runtime/go-memmove.c
+++ b/libgo/runtime/go-memmove.c
@@ -12,78 +12,60 @@ void gomemmove(void *, void *, uintptr)
 
 // This implementation is necessary since
 // the __builtin_memmove might use __libc_memmove
-// which doesn't require atomicity of 8 byte
+// which doesn't require atomicity of pointer-sized
 // moves.
 
 void
-gomemmove (void *dst, void *src, uintptr len)
+gomemmove(void *dst, void *src, uintptr len)
 {
-#if !defined(__PPC64__)
-  __builtin_memmove(dst, src, len);
-#else
-  uint64 offset, tail;
-  int64 rem;
-  uint64 dwords;
-  uint64 i;
-  char *bdst,*bsrc;
-
-  rem = len;
+  const uintptr ptr_size = sizeof(dst);
+  uintptr tail;
+  uintptr rem;
+  uintptr dwords;
+  uintptr i;
+  char *bdst, *bsrc;
 
   if (len == 0) {
-	return;
+    return;
   }
 
-  // If src and dst don't have the same 8 byte alignment then
-  // there is no issue with copying pointer atomicity. Use the
-  // builtin.
-  if (((uint64)dst % 8) != ((uint64)src % 8) || len < 8) {
-	__builtin_memmove(dst, src, len);
-	return;
+  // We expect pointer-containing values to be pointer-aligned.
+  // If these pointers are not aligned, they don't contain pointers.
+  if ((uintptr)dst % ptr_size != 0 || (uintptr)src % ptr_size != 0 || len < ptr_size) {
+    __builtin_memmove(dst, src, len);
+    return;
   }
 
-  // Length >= 8 && same ptr alignment
-  offset = (uint64)dst % 8;
-
-  // If not 8 byte alignment, move the intial bytes.
-  if (offset > 0) {
-	__builtin_memmove(dst, src, 8-offset);
-	dst += (8-offset);
-	src += (8-offset);
-	rem -= (8-offset);
-  }
+  bdst = (char*)dst;
+  bsrc = (char*)src;
 
-  // Move the tail bytes to make the backward move
-  // easier.
-  tail = rem % 8;
+  // Move the tail bytes to make the backward move easier.
+  rem = len;
+  tail = rem % ptr_size;
   if (tail > 0) {
-	__builtin_memmove(dst+rem-tail, src+rem-tail, tail);
-	rem -= tail;
-  }
-
-  if (rem == 0) {
-	return;
+    __builtin_memmove(bdst+rem-tail, bsrc+rem-tail, tail);
+    rem -= tail;
   }
 
-  // Must now be 8 byte alignment and rem is multiple of 8.
-  dwords = len>>3;
+  // Must now be pointer alignment and rem is multiple of ptr_size.
+  dwords = rem / ptr_size;
 
-  // Determine if a backwards move is needed
-  // Forward or backward, move all doublewords
+  // Determine if a backwards move is needed.
+  // Forward or backward, move all words.
 
-  if ((uint64)(dst - src) < (uint64)rem) {
-	bdst = dst+rem-8;
-	bsrc = src+rem-8;
-	for (i = 0; i<dwords; i++) {
-		*(uint64*)bdst = *(uint64*)bsrc;
-		bdst -= 8;
-		bsrc -= 8;
-	}
+  if ((uintptr)(bdst - bsrc) < rem) {
+    bdst += rem - ptr_size;
+    bsrc += rem - ptr_size;
+    for (i = 0; i<dwords; i++) {
+      *(uintptr*)bdst = *(uintptr*)bsrc;
+      bdst -= ptr_size;
+      bsrc -= ptr_size;
+    }
   } else {
-	for (i = 0; i<dwords; i++) {
-		*(uint64*)dst = *(uint64*)src;
-		dst += 8;
-		src += 8;
-	}
+    for (i = 0; i<dwords; i++) {
+      *(uintptr*)bdst = *(uintptr*)bsrc;
+      bdst += ptr_size;
+      bsrc += ptr_size;
+    }
   }
-#endif
 }
diff --git a/libgomp/ChangeLog b/libgomp/ChangeLog
index d3384b0..2689f3e 100644
--- a/libgomp/ChangeLog
+++ b/libgomp/ChangeLog
@@ -1,3 +1,28 @@
+2025-07-21  Thomas Schwinge  <tschwinge@baylibre.com>
+
+	PR target/119853
+	PR target/119854
+	* testsuite/libgomp.c++/target-cdtor-1.C: Adjust for
+	'targetm.cxx.use_aeabi_atexit'.
+	* testsuite/libgomp.c++/target-cdtor-2.C: Likewise.
+
+2025-07-18  Andrew Stubbs  <ams@baylibre.com>
+
+	PR target/121156
+	* config/gcn/bar.c (gomp_team_barrier_wait_end): Remove unused
+	"generation" variable.
+	(gomp_team_barrier_wait_cancel_end): Likewise.
+
+2025-07-17  Thomas Schwinge  <tschwinge@baylibre.com>
+
+	PR target/119692
+	* testsuite/libgomp.c++/pr119692-1-4.C: '{ dg-timeout 10 { target offload_device } }'.
+	* testsuite/libgomp.c++/pr119692-1-5.C: Likewise.
+	* testsuite/libgomp.c++/target-exceptions-bad_cast-1.C: Likewise.
+	* testsuite/libgomp.c++/target-exceptions-bad_cast-2.C: Likewise.
+	* testsuite/libgomp.oacc-c++/exceptions-bad_cast-1.C: Likewise.
+	* testsuite/libgomp.oacc-c++/exceptions-bad_cast-2.C: Likewise.
+
 2025-06-24  Tobias Burnus  <tburnus@baylibre.com>
 
 	* libgomp.texi (acc_attach, acc_detach): Update for Fortran
diff --git a/libgomp/config/gcn/bar.c b/libgomp/config/gcn/bar.c
index 52b344c..57ac648 100644
--- a/libgomp/config/gcn/bar.c
+++ b/libgomp/config/gcn/bar.c
@@ -79,7 +79,7 @@ gomp_team_barrier_wake (gomp_barrier_t *bar, int count)
 void
 gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
 {
-  unsigned int generation, gen;
+  unsigned int gen;
 
   if (__builtin_expect (state & BAR_WAS_LAST, 0))
     {
@@ -105,7 +105,6 @@ gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
 	}
     }
 
-  generation = state;
   state &= ~BAR_CANCELLED;
   int retry = 100;
   do
@@ -128,7 +127,6 @@ gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
 	  gomp_barrier_handle_tasks (state);
 	  gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
 	}
-      generation |= gen & BAR_WAITING_FOR_TASK;
     }
   while (gen != state + BAR_INCR);
 }
@@ -152,7 +150,7 @@ bool
 gomp_team_barrier_wait_cancel_end (gomp_barrier_t *bar,
 				   gomp_barrier_state_t state)
 {
-  unsigned int generation, gen;
+  unsigned int gen;
 
   if (__builtin_expect (state & BAR_WAS_LAST, 0))
     {
@@ -184,7 +182,6 @@ gomp_team_barrier_wait_cancel_end (gomp_barrier_t *bar,
   if (__builtin_expect (state & BAR_CANCELLED, 0))
     return true;
 
-  generation = state;
   int retry = 100;
   do
     {
@@ -209,7 +206,6 @@ gomp_team_barrier_wait_cancel_end (gomp_barrier_t *bar,
 	  gomp_barrier_handle_tasks (state);
 	  gen = __atomic_load_n (&bar->generation, MEMMODEL_RELAXED);
 	}
-      generation |= gen & BAR_WAITING_FOR_TASK;
     }
   while (gen != state + BAR_INCR);
 
diff --git a/libgomp/testsuite/libgomp.c++/pr119692-1-4.C b/libgomp/testsuite/libgomp.c++/pr119692-1-4.C
index af9fe1c..d329ad3 100644
--- a/libgomp/testsuite/libgomp.c++/pr119692-1-4.C
+++ b/libgomp/testsuite/libgomp.c++/pr119692-1-4.C
@@ -5,7 +5,7 @@
    { dg-xfail-run-if PR119692 { offload_device } } */
 /* There are configurations where we 'WARNING: program timed out.' while in
    'dynamic_cast', see <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=119692#c6>.
-   { dg-timeout 10 } ... to make sure that happens quickly.  */
+   { dg-timeout 10 { target offload_device } } ... to make sure that happens quickly.  */
 /* { dg-additional-options -fdump-tree-gimple } */
 
 #include "pr119692-1-1.C"
diff --git a/libgomp/testsuite/libgomp.c++/pr119692-1-5.C b/libgomp/testsuite/libgomp.c++/pr119692-1-5.C
index e5c6e07..6bbe186 100644
--- a/libgomp/testsuite/libgomp.c++/pr119692-1-5.C
+++ b/libgomp/testsuite/libgomp.c++/pr119692-1-5.C
@@ -5,7 +5,7 @@
    { dg-xfail-run-if PR119692 { offload_device } } */
 /* There are configurations where we 'WARNING: program timed out.' while in
    'dynamic_cast', see <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=119692#c6>.
-   { dg-timeout 10 } ... to make sure that happens quickly.  */
+   { dg-timeout 10 { target offload_device } } ... to make sure that happens quickly.  */
 /* { dg-additional-options -fdump-tree-gimple } */
 
 #include "pr119692-1-1.C"
diff --git a/libgomp/testsuite/libgomp.c++/target-cdtor-1.C b/libgomp/testsuite/libgomp.c++/target-cdtor-1.C
index ecb029e..7e8cc58 100644
--- a/libgomp/testsuite/libgomp.c++/target-cdtor-1.C
+++ b/libgomp/testsuite/libgomp.c++/target-cdtor-1.C
@@ -63,14 +63,19 @@ int main()
   return 0;
 }
 
-/* Verify '__cxa_atexit' calls.
+/* Verify '__cxa_atexit' calls (or '__aeabi_atexit', per 'targetm.cxx.use_aeabi_atexit').
 
    For the host, there are four expected calls:
-   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, } 4 optimized { target cxa_atexit } } }
-   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZN1SD1Ev, \&sH1, \&__dso_handle>} 1 optimized { target cxa_atexit } } }
-   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZN1SD1Ev, \&sHD1, \&__dso_handle>} 1 optimized { target cxa_atexit } } }
-   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZNSt6vectorI1SSaIS0_EED1Ev, \&svHD1, \&__dso_handle>} 1 optimized { target cxa_atexit } } }
-   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZN1SD1Ev, \&sH2, \&__dso_handle>} 1 optimized { target cxa_atexit } } }
+   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, } 4 optimized { target { cxa_atexit && { ! arm_eabi } } } } }
+     { dg-final { scan-tree-dump-times {gimple_call <__aeabi_atexit, } 4 optimized { target { cxa_atexit && arm_eabi } } } }
+   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZN1SD1Ev, \&sH1, \&__dso_handle>} 1 optimized { target { cxa_atexit && { ! arm_eabi } } } } }
+     { dg-final { scan-tree-dump-times {gimple_call <__aeabi_atexit, NULL, \&sH1, _ZN1SD1Ev, \&__dso_handle>} 1 optimized { target { cxa_atexit && arm_eabi } } } }
+   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZN1SD1Ev, \&sHD1, \&__dso_handle>} 1 optimized { target { cxa_atexit && { ! arm_eabi } } } } }
+     { dg-final { scan-tree-dump-times {gimple_call <__aeabi_atexit, NULL, \&sHD1, _ZN1SD1Ev, \&__dso_handle>} 1 optimized { target { cxa_atexit && arm_eabi } } } }
+   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZNSt6vectorI1SSaIS0_EED1Ev, \&svHD1, \&__dso_handle>} 1 optimized { target { cxa_atexit && { ! arm_eabi } } } } }
+     { dg-final { scan-tree-dump-times {gimple_call <__aeabi_atexit, NULL, \&svHD1, _ZNSt6vectorI1SSaIS0_EED1Ev, \&__dso_handle>} 1 optimized { target { cxa_atexit && arm_eabi } } } }
+   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZN1SD1Ev, \&sH2, \&__dso_handle>} 1 optimized { target { cxa_atexit && { ! arm_eabi } } } } }
+     { dg-final { scan-tree-dump-times {gimple_call <__aeabi_atexit, NULL, \&sH2, _ZN1SD1Ev, \&__dso_handle>} 1 optimized { target { cxa_atexit && arm_eabi } } } }
 
    For the device, there are two expected calls:
    { dg-final { scan-offload-tree-dump-times {gimple_call <__cxa_atexit, } 2 optimized { target cxa_atexit } } }
diff --git a/libgomp/testsuite/libgomp.c++/target-cdtor-2.C b/libgomp/testsuite/libgomp.c++/target-cdtor-2.C
index 75e48ca..9c85122 100644
--- a/libgomp/testsuite/libgomp.c++/target-cdtor-2.C
+++ b/libgomp/testsuite/libgomp.c++/target-cdtor-2.C
@@ -93,14 +93,19 @@ int main()
   return 0;
 }
 
-/* Verify '__cxa_atexit' calls.
+/* Verify '__cxa_atexit' calls (or '__aeabi_atexit', per 'targetm.cxx.use_aeabi_atexit').
 
    For the host, there are four expected calls:
-   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, } 4 optimized { target cxa_atexit } } }
-   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZN1SD1Ev, \&sH1, \&__dso_handle>} 1 optimized { target cxa_atexit } } }
-   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZN1SD1Ev, \&sHD1, \&__dso_handle>} 1 optimized { target cxa_atexit } } }
-   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZNSt6vectorI1SSaIS0_EED1Ev, \&svHD1, \&__dso_handle>} 1 optimized { target cxa_atexit } } }
-   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZN1SD1Ev, \&sH2, \&__dso_handle>} 1 optimized { target cxa_atexit } } }
+   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, } 4 optimized { target { cxa_atexit && { ! arm_eabi } } } } }
+     { dg-final { scan-tree-dump-times {gimple_call <__aeabi_atexit, } 4 optimized { target { cxa_atexit && arm_eabi } } } }
+   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZN1SD1Ev, \&sH1, \&__dso_handle>} 1 optimized { target { cxa_atexit && { ! arm_eabi } } } } }
+     { dg-final { scan-tree-dump-times {gimple_call <__aeabi_atexit, NULL, \&sH1, _ZN1SD1Ev, \&__dso_handle>} 1 optimized { target { cxa_atexit && arm_eabi } } } }
+   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZN1SD1Ev, \&sHD1, \&__dso_handle>} 1 optimized { target { cxa_atexit && { ! arm_eabi } } } } }
+     { dg-final { scan-tree-dump-times {gimple_call <__aeabi_atexit, NULL, \&sHD1, _ZN1SD1Ev, \&__dso_handle>} 1 optimized { target { cxa_atexit && arm_eabi } } } }
+   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZNSt6vectorI1SSaIS0_EED1Ev, \&svHD1, \&__dso_handle>} 1 optimized { target { cxa_atexit && { ! arm_eabi } } } } }
+     { dg-final { scan-tree-dump-times {gimple_call <__aeabi_atexit, NULL, \&svHD1, _ZNSt6vectorI1SSaIS0_EED1Ev, \&__dso_handle>} 1 optimized { target { cxa_atexit && arm_eabi } } } }
+   { dg-final { scan-tree-dump-times {gimple_call <__cxa_atexit, NULL, _ZN1SD1Ev, \&sH2, \&__dso_handle>} 1 optimized { target { cxa_atexit && { ! arm_eabi } } } } }
+     { dg-final { scan-tree-dump-times {gimple_call <__aeabi_atexit, NULL, \&sH2, _ZN1SD1Ev, \&__dso_handle>} 1 optimized { target { cxa_atexit && arm_eabi } } } }
 
    For the device, there are two expected calls:
    { dg-final { scan-offload-tree-dump-times {gimple_call <__cxa_atexit, } 2 optimized { target cxa_atexit } } }
diff --git a/libgomp/testsuite/libgomp.c++/target-exceptions-bad_cast-1.C b/libgomp/testsuite/libgomp.c++/target-exceptions-bad_cast-1.C
index a862652..4158ece 100644
--- a/libgomp/testsuite/libgomp.c++/target-exceptions-bad_cast-1.C
+++ b/libgomp/testsuite/libgomp.c++/target-exceptions-bad_cast-1.C
@@ -25,4 +25,4 @@
    { dg-shouldfail {'std::bad_cast' exception} } */
 /* There are configurations where we 'WARNING: program timed out.' while in
    'dynamic_cast', see <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=119692#c6>.
-   { dg-timeout 10 } ... to make sure that happens quickly.  */
+   { dg-timeout 10 { target offload_device } } ... to make sure that happens quickly.  */
diff --git a/libgomp/testsuite/libgomp.c++/target-exceptions-bad_cast-2.C b/libgomp/testsuite/libgomp.c++/target-exceptions-bad_cast-2.C
index ff15c9f..ca72e57 100644
--- a/libgomp/testsuite/libgomp.c++/target-exceptions-bad_cast-2.C
+++ b/libgomp/testsuite/libgomp.c++/target-exceptions-bad_cast-2.C
@@ -24,4 +24,4 @@
    { dg-shouldfail {'MyException' exception} { offload_device } } */
 /* There are configurations where we 'WARNING: program timed out.' while in
    'dynamic_cast', see <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=119692#c6>.
-   { dg-timeout 10 } ... to make sure that happens quickly.  */
+   { dg-timeout 10 { target offload_device } } ... to make sure that happens quickly.  */
diff --git a/libgomp/testsuite/libgomp.oacc-c++/exceptions-bad_cast-1.C b/libgomp/testsuite/libgomp.oacc-c++/exceptions-bad_cast-1.C
index 6957a6c..e6cbe17 100644
--- a/libgomp/testsuite/libgomp.oacc-c++/exceptions-bad_cast-1.C
+++ b/libgomp/testsuite/libgomp.oacc-c++/exceptions-bad_cast-1.C
@@ -54,4 +54,4 @@ int main()
    { dg-shouldfail {'std::bad_cast' exception} } */
 /* There are configurations where we 'WARNING: program timed out.' while in
    'dynamic_cast', see <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=119692#c6>.
-   { dg-timeout 10 } ... to make sure that happens quickly.  */
+   { dg-timeout 10 { target offload_device } } ... to make sure that happens quickly.  */
diff --git a/libgomp/testsuite/libgomp.oacc-c++/exceptions-bad_cast-2.C b/libgomp/testsuite/libgomp.oacc-c++/exceptions-bad_cast-2.C
index 0f84cf2..599425f 100644
--- a/libgomp/testsuite/libgomp.oacc-c++/exceptions-bad_cast-2.C
+++ b/libgomp/testsuite/libgomp.oacc-c++/exceptions-bad_cast-2.C
@@ -60,4 +60,4 @@ int main()
    { dg-shouldfail {'std::bad_cast' exception} { ! openacc_host_selected } } */
 /* There are configurations where we 'WARNING: program timed out.' while in
    'dynamic_cast', see <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=119692#c6>.
-   { dg-timeout 10 } ... to make sure that happens quickly.  */
+   { dg-timeout 10 { target offload_device } } ... to make sure that happens quickly.  */
diff --git a/libiberty/ChangeLog b/libiberty/ChangeLog
index 2ae5626..cf90917 100644
--- a/libiberty/ChangeLog
+++ b/libiberty/ChangeLog
@@ -1,3 +1,9 @@
+2025-07-09  Matthieu Longo  <matthieu.longo@arm.com>
+
+	* Makefile.in: Add new header.
+	* testsuite/Makefile.in: Add new test.
+	* testsuite/test-doubly-linked-list.c: New test.
+
 2025-05-13  Andreas Schwab  <schwab@suse.de>
 
 	* regex.c (regex_compile): Don't write beyond array bounds when
diff --git a/libiberty/Makefile.in b/libiberty/Makefile.in
index ce54d88..e4d7d0a 100644
--- a/libiberty/Makefile.in
+++ b/libiberty/Makefile.in
@@ -237,6 +237,7 @@ CONFIGURED_OFILES = ./asprintf.$(objext) ./atexit.$(objext)		\
 INSTALLED_HEADERS =                                                     \
 	$(INCDIR)/ansidecl.h                                            \
 	$(INCDIR)/demangle.h                                            \
+	$(INCDIR)/doubly-linked-list.h                                  \
 	$(INCDIR)/dyn-string.h                                          \
 	$(INCDIR)/fibheap.h                                             \
 	$(INCDIR)/floatformat.h                                         \
diff --git a/libiberty/testsuite/Makefile.in b/libiberty/testsuite/Makefile.in
index 2b0883c..ef549ca 100644
--- a/libiberty/testsuite/Makefile.in
+++ b/libiberty/testsuite/Makefile.in
@@ -45,7 +45,8 @@ all:
 check: @CHECK@
 
 really-check: check-cplus-dem check-d-demangle check-rust-demangle \
-		check-pexecute check-expandargv check-strtol
+		check-pexecute check-expandargv check-strtol \
+		check-doubly-linked-list
 
 # Run some tests of the demangler.
 check-cplus-dem: test-demangle $(srcdir)/demangle-expected
@@ -69,6 +70,10 @@ check-expandargv: test-expandargv
 check-strtol: test-strtol
 	./test-strtol
 
+# Check the linked list functionality
+check-doubly-linked-list: test-doubly-linked-list
+	./test-doubly-linked-list
+
 # Run the demangler fuzzer
 fuzz-demangler: demangler-fuzzer
 	./demangler-fuzzer
@@ -90,6 +95,10 @@ test-strtol: $(srcdir)/test-strtol.c ../libiberty.a
 	$(TEST_COMPILE) -DHAVE_CONFIG_H -I.. -o test-strtol \
 		$(srcdir)/test-strtol.c ../libiberty.a
 
+test-doubly-linked-list: $(srcdir)/test-doubly-linked-list.c
+	$(TEST_COMPILE) -DHAVE_CONFIG_H -I.. -o test-doubly-linked-list \
+		$(srcdir)/test-doubly-linked-list.c
+
 demangler-fuzzer: $(srcdir)/demangler-fuzzer.c ../libiberty.a
 	$(TEST_COMPILE) -o demangler-fuzzer \
 		$(srcdir)/demangler-fuzzer.c ../libiberty.a
@@ -104,6 +113,7 @@ mostlyclean:
 	rm -f test-pexecute
 	rm -f test-expandargv
 	rm -f test-strtol
+	rm -f test-doubly-linked-list
 	rm -f demangler-fuzzer
 	rm -f core
 clean: mostlyclean
diff --git a/libiberty/testsuite/test-doubly-linked-list.c b/libiberty/testsuite/test-doubly-linked-list.c
new file mode 100644
index 0000000..1e1fc63
--- /dev/null
+++ b/libiberty/testsuite/test-doubly-linked-list.c
@@ -0,0 +1,269 @@
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "doubly-linked-list.h"
+
+#ifndef EXIT_SUCCESS
+#define EXIT_SUCCESS 0
+#endif
+
+#ifndef EXIT_FAILURE
+#define EXIT_FAILURE 1
+#endif
+
+/* Implementation */
+
+typedef int T;
+
+typedef struct ListNodeType
+{
+  T value;
+  struct ListNodeType *next;
+  struct ListNodeType *prev;
+} ListNodeType;
+
+ListNodeType * l_new_node (T value)
+{
+  ListNodeType *n = malloc (sizeof (ListNodeType));
+  n->next = NULL;
+  n->prev = NULL;
+  n->value = value;
+  return n;
+}
+
+typedef struct LinkedListWrapperType
+{
+  ListNodeType *first;
+  ListNodeType *last;
+  size_t size;
+} LinkedListWrapperType;
+
+int compare_nodes (const ListNodeType *n1, const ListNodeType *n2)
+{
+  if (n1->value == n2->value)
+    return 0;
+  else if (n1->value < n2->value)
+    return -1;
+  else
+    return 1;
+}
+
+LINKED_LIST_MUTATIVE_OPS_PROTOTYPE (LinkedListWrapperType, ListNodeType, static);
+LINKED_LIST_MERGE_SORT_PROTOTYPE (LinkedListWrapperType, ListNodeType, static);
+
+LINKED_LIST_MUTATIVE_OPS_DECL (LinkedListWrapperType, ListNodeType, static)
+LINKED_LIST_MERGE_SORT_DECL (LinkedListWrapperType, ListNodeType, static)
+
+ListNodeType * find_last_node (ListNodeType *head)
+{
+  if (head == NULL)
+    return NULL;
+
+  ListNodeType *n = head;
+  while (n->next != NULL)
+    n = n->next;
+
+  return n;
+}
+
+void l_print (ListNodeType *node)
+{
+  for (ListNodeType *l = node; l != NULL; l = l->next)
+    printf ("%d ", l->value);
+  printf ("\n");
+}
+
+void l_reverse_print (ListNodeType *last_node)
+{
+  for (ListNodeType *l = last_node; l != NULL; l = l->prev)
+    printf ("%d ", l->value);
+  printf ("\n");
+}
+
+struct test_data_t
+{
+  T const *content;
+  size_t size;
+};
+
+bool run_test (const struct test_data_t *expect,
+	       LinkedListWrapperType *current,
+	       bool reversed)
+{
+  ListNodeType *node = (reversed) ? current->last : current->first;
+  bool passed = true;
+  for (int i=0; i<expect->size && node != NULL; ++i)
+    {
+      if (reversed)
+	{
+	  if (expect->content[expect->size - 1 - i] != node->value)
+	    {
+	      printf ("FAIL: mismatching expected (%d) VS current (%d).\n",
+		      expect->content[expect->size - 1 - i], node->value);
+	      passed = false;
+	    }
+	  if (node->prev == NULL && current->first != node)
+	    {
+	      printf ("FAIL: first is not matching the first node.\n");
+	      passed = false;
+	    }
+	}
+      else
+	{
+	  if (expect->content[i] != node->value)
+	    {
+	      printf ("FAIL: mismatching expected (%d) VS current (%d).\n",
+		      expect->content[i], node->value);
+	      passed = false;
+	    }
+	  if (node->next == NULL && current->last != node)
+	    {
+	      printf ("FAIL: last_ is not matching the last node.\n");
+	      passed = false;
+	    }
+	}
+
+      if (!passed)
+	return false;
+
+      if (reversed)
+	node = node->prev;
+      else
+	node = node->next;
+    }
+
+  if (node != NULL)
+    {
+      printf ("FAIL: the list is longer than expected.\n");
+      passed = false;
+    }
+  if (expect->size != current->size)
+    {
+      printf ("FAIL: size (%ld) is not matching the real size of the list (%ld).\n",
+	      current->size, expect->size);
+      passed = false;
+    }
+
+  return passed;
+}
+
+bool check(const char *op,
+	  const struct test_data_t *expect,
+	  LinkedListWrapperType *wrapper)
+{
+  bool success = true;
+  bool res;
+
+  l_print (wrapper->first);
+  res = run_test (expect, wrapper, false);
+  printf ("%s: test-linked-list::%s: check forward conformity\n",
+	  res ? "PASS": "FAIL", op);
+  success &= res;
+
+  l_reverse_print (wrapper->last);
+  res = run_test (expect, wrapper, true);
+  printf ("%s: test-linked-list::%s: check backward conformity\n",
+	  res ? "PASS": "FAIL", op);
+  success &= res;
+
+  printf("\n");
+
+  return success;
+}
+
+const int EXPECT_0 [] = { 10, 4, 3, 1, 9, 2 };
+const int EXPECT_1 [] = { 1, 2, 3, 4, 9, 10 };
+const int EXPECT_2 [] = { 11, 1, 2, 3, 4, 9, 10 };
+const int EXPECT_3 [] = { 11, 1, 2, 3, 4, 9, 8, 10 };
+const int EXPECT_4 [] = { 11, 2, 3, 4, 9, 8, 10 };
+const int EXPECT_5 [] = { 10, 2, 3, 4, 9, 8, 11 };
+const int EXPECT_6 [] = { 10, 3, 2, 4, 9, 8, 11 };
+const int EXPECT_7 [] = { 10, 9, 2, 4, 3, 8, 11 };
+const int EXPECT_8 [] = { 2, 3, 4, 8, 9, 10, 11 };
+const int EXPECT_9 [] = { 3, 4, 8, 9, 10, 11 };
+const int EXPECT_10 [] = { 3, 4, 8, 9, 10 };
+const struct test_data_t test_data[] = {
+  { .content = EXPECT_0, .size = sizeof(EXPECT_0) / sizeof(EXPECT_0[0]) },
+  { .content = EXPECT_1, .size = sizeof(EXPECT_1) / sizeof(EXPECT_1[0]) },
+  { .content = EXPECT_2, .size = sizeof(EXPECT_2) / sizeof(EXPECT_2[0]) },
+  { .content = EXPECT_3, .size = sizeof(EXPECT_3) / sizeof(EXPECT_3[0]) },
+  { .content = EXPECT_4, .size = sizeof(EXPECT_4) / sizeof(EXPECT_4[0]) },
+  { .content = EXPECT_5, .size = sizeof(EXPECT_5) / sizeof(EXPECT_5[0]) },
+  { .content = EXPECT_6, .size = sizeof(EXPECT_6) / sizeof(EXPECT_6[0]) },
+  { .content = EXPECT_7, .size = sizeof(EXPECT_7) / sizeof(EXPECT_7[0]) },
+  { .content = EXPECT_8, .size = sizeof(EXPECT_8) / sizeof(EXPECT_8[0]) },
+  { .content = EXPECT_9, .size = sizeof(EXPECT_9) / sizeof(EXPECT_9[0]) },
+  { .content = EXPECT_10, .size = sizeof(EXPECT_10) / sizeof(EXPECT_10[0]) },
+};
+
+int main (void)
+{
+  int failures = 0;
+
+  LinkedListWrapperType wrapper = {
+    .first = NULL,
+    .last = NULL,
+    .size = 0,
+  };
+
+  /* Append nodes.  */
+  LINKED_LIST_APPEND(ListNodeType) (&wrapper, l_new_node (10));
+  LINKED_LIST_APPEND(ListNodeType) (&wrapper, l_new_node (4));
+  LINKED_LIST_APPEND(ListNodeType) (&wrapper, l_new_node (3));
+  LINKED_LIST_APPEND(ListNodeType) (&wrapper, l_new_node (1));
+  LINKED_LIST_APPEND(ListNodeType) (&wrapper, l_new_node (9));
+  LINKED_LIST_APPEND(ListNodeType) (&wrapper, l_new_node (2));
+
+  failures += ! check ("append", &test_data[0], &wrapper);
+
+  /* Sort nodes (without updating wrapper).  */
+  wrapper.first =
+    LINKED_LIST_MERGE_SORT_(ListNodeType) (wrapper.first, compare_nodes);
+  wrapper.last = find_last_node (wrapper.first);
+
+  failures += ! check ("sort", &test_data[1], &wrapper);
+
+  /* Save a reference to this node for later.  */
+  ListNodeType *n_to_remove = wrapper.first;
+
+  /* Prepend node.  */
+  LINKED_LIST_PREPEND(ListNodeType) (&wrapper, l_new_node (11));
+  failures += ! check ("prepend", &test_data[2], &wrapper);
+
+  /* Insert node.  */
+  LINKED_LIST_INSERT_BEFORE(ListNodeType) (&wrapper, l_new_node (8), wrapper.last);
+  failures += ! check ("insert_before", &test_data[3], &wrapper);
+
+  /* Remove a node.  */
+  LINKED_LIST_REMOVE(ListNodeType) (&wrapper, n_to_remove);
+  failures += ! check ("remove", &test_data[4], &wrapper);
+
+  /* Swap first and last.  */
+  LINKED_LIST_SWAP(ListNodeType) (&wrapper, wrapper.first, wrapper.last);
+  failures += ! check ("swap first and last", &test_data[5], &wrapper);
+
+  /* Swap adjacent nodes.  */
+  LINKED_LIST_SWAP(ListNodeType) (&wrapper, wrapper.first->next,
+				  wrapper.first->next->next);
+  failures += ! check ("swap adjacent nodes", &test_data[6], &wrapper);
+
+  /* Swap non-adjacent nodes, but neither first nor last.  */
+  LINKED_LIST_SWAP(ListNodeType) (&wrapper, wrapper.first->next,
+				  wrapper.first->next->next->next->next);
+  failures += ! check ("swap non-adjacent nodes", &test_data[7], &wrapper);
+
+  /* Sort nodes.  */
+  LINKED_LIST_MERGE_SORT(ListNodeType) (&wrapper, compare_nodes);
+  failures += ! check ("sort", &test_data[8], &wrapper);
+
+  /* Pop front.  */
+  LINKED_LIST_POP_FRONT(ListNodeType) (&wrapper);
+  failures += ! check ("pop_front", &test_data[9], &wrapper);
+
+  /* Pop back.  */
+  LINKED_LIST_POP_BACK(ListNodeType) (&wrapper);
+  failures += ! check ("pop_back", &test_data[10], &wrapper);
+
+  exit (failures ? EXIT_FAILURE : EXIT_SUCCESS);
+}
diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cpp b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cpp
index 490e75f..9856ac3 100644
--- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cpp
+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cpp
@@ -762,13 +762,9 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
   unsigned IOCTL_SOUND_PCM_WRITE_FILTER = SOUND_PCM_WRITE_FILTER;
 #endif // SOUND_VERSION
   unsigned IOCTL_TCFLSH = TCFLSH;
-  unsigned IOCTL_TCGETA = TCGETA;
   unsigned IOCTL_TCGETS = TCGETS;
   unsigned IOCTL_TCSBRK = TCSBRK;
   unsigned IOCTL_TCSBRKP = TCSBRKP;
-  unsigned IOCTL_TCSETA = TCSETA;
-  unsigned IOCTL_TCSETAF = TCSETAF;
-  unsigned IOCTL_TCSETAW = TCSETAW;
   unsigned IOCTL_TCSETS = TCSETS;
   unsigned IOCTL_TCSETSF = TCSETSF;
   unsigned IOCTL_TCSETSW = TCSETSW;
diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
index a80df65..cfac490 100644
--- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
+++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -1274,13 +1274,9 @@ extern unsigned IOCTL_SNDCTL_COPR_SENDMSG;
 extern unsigned IOCTL_SNDCTL_COPR_WCODE;
 extern unsigned IOCTL_SNDCTL_COPR_WDATA;
 extern unsigned IOCTL_TCFLSH;
-extern unsigned IOCTL_TCGETA;
 extern unsigned IOCTL_TCGETS;
 extern unsigned IOCTL_TCSBRK;
 extern unsigned IOCTL_TCSBRKP;
-extern unsigned IOCTL_TCSETA;
-extern unsigned IOCTL_TCSETAF;
-extern unsigned IOCTL_TCSETAW;
 extern unsigned IOCTL_TCSETS;
 extern unsigned IOCTL_TCSETSF;
 extern unsigned IOCTL_TCSETSW;
diff --git a/libstdc++-v3/ChangeLog b/libstdc++-v3/ChangeLog
index c87e61f..7f697f2 100644
--- a/libstdc++-v3/ChangeLog
+++ b/libstdc++-v3/ChangeLog
@@ -1,3 +1,969 @@
+2025-07-30  François Dumont  <frs.dumont@gmail.com>
+
+	* testsuite/std/time/format/data_not_present_neg.cc: Remove _GLIBCXX_USE_DUAL_ABI
+	check.
+
+2025-07-29  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	* testsuite/std/format/ranges/format_kind.cc: New test.
+
+2025-07-28  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/102181
+	* include/bits/stl_iterator_base_funcs.h (distance, advance):
+	Check C++20 iterator concepts and handle appropriately.
+	(__detail::__iter_category_converts_to_concept): New concept.
+	(__detail::__promotable_iterator): New concept.
+	* testsuite/24_iterators/operations/cxx20_iterators.cc: New
+	test.
+
+2025-07-28  Nathan Myers  <ncm@cantrip.org>
+
+	PR libstdc++/119742
+	* include/debug/bitset: Add new ctor.
+
+2025-07-28  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	* include/std/mdspan: Small stylistic adjustments.
+
+2025-07-28  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	* testsuite/23_containers/mdspan/accessors/generic.cc: Refactor
+	test_ctor.
+
+2025-07-28  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	PR libstdc++/121196
+	* include/std/inplace_vector (std::erase): Provide default argument
+	for _Up parameter.
+	* testsuite/23_containers/inplace_vector/erasure.cc: Add test for
+	using braces-init-list as arguments to erase_if and use function
+	to verify content of inplace_vector
+
+2025-07-25  Tuur Martens  <tuurmartens4@gmail.com>
+
+	* include/bits/unordered_map.h: Rectify referencing of
+	non-existent type.
+
+2025-07-24  Nathaniel Shead  <nathanieloshead@gmail.com>
+
+	PR c++/117294
+	PR c++/113854
+	* testsuite/20_util/any/misc/any_cast_neg.cc: Adjust
+	diagnostics.
+	* testsuite/20_util/expected/illformed_neg.cc: Likewise.
+	* testsuite/20_util/optional/monadic/or_else_neg.cc: Likewise.
+	* testsuite/23_containers/array/creation/3_neg.cc: Likewise.
+	* testsuite/24_iterators/range_generators/lwg3900.cc: Likewise.
+	* testsuite/29_atomics/atomic/requirements/types_neg.cc:
+	Likewise.
+	* testsuite/30_threads/stop_token/stop_callback/invocable_neg.cc:
+	Likewise.
+	* testsuite/30_threads/stop_token/stop_callback/destructible_neg.cc:
+	Likewise.
+	* testsuite/std/format/arguments/args_neg.cc: Likewise.
+	* testsuite/std/format/string_neg.cc: Likewise.
+
+2025-07-24  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	PR libstdc++/104874
+	* testsuite/24_iterators/random_access/string_vector_iterators.cc:
+	Reworked.
+
+2025-07-24  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	* testsuite/21_strings/basic_string/cons/from_range.cc: Replace
+	test_constexpr with test_ranges inside static_assert.
+	* testsuite/21_strings/basic_string/modifiers/append/append_range.cc:
+	Likewise.
+	* testsuite/21_strings/basic_string/modifiers/assign/assign_range.cc:
+	Likewise.
+	* testsuite/21_strings/basic_string/modifiers/insert/insert_range.cc:
+	Likewise.
+	* testsuite/21_strings/basic_string/modifiers/replace/replace_with_range.cc:
+	Likewise.
+	* testsuite/23_containers/vector/bool/cons/from_range.cc: Likewise.
+	* testsuite/23_containers/vector/bool/modifiers/assign/assign_range.cc:
+	Likewise.
+	* testsuite/23_containers/vector/bool/modifiers/insert/insert_range.cc:
+	Likewise.
+	* testsuite/23_containers/vector/cons/from_range.cc: Likewise.
+	* testsuite/23_containers/vector/modifiers/assign/assign_range.cc:
+	Likewise.
+	* testsuite/23_containers/vector/modifiers/insert/insert_range.cc:
+	Likewise.
+	* testsuite/23_containers/vector/bool/modifiers/insert/append_range.cc:
+	Run full test_ranges instead of span-only in test_constexpr.
+	* testsuite/23_containers/vector/modifiers/append_range.cc:
+	Replace test_constexpr with calls to test_ranges and test_overlapping.
+	* testsuite/util/testsuite_allocator.h (__gnu_test::SimpleAllocator):
+	Declared member functions as constexpr.
+
+2025-07-23  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	* testsuite/23_containers/mdspan/accessors/default.cc: Delete.
+	* testsuite/23_containers/mdspan/accessors/generic.cc: Slightly
+	generalize the test code previously in default.cc.
+
+2025-07-23  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	* testsuite/23_containers/mdspan/extents/ctor_ints.cc: Remove
+	superfluous parens.
+	* testsuite/23_containers/mdspan/extents/ctor_shape.cc: Ditto.
+	* testsuite/23_containers/mdspan/mdspan.cc: Ditto.
+
+2025-07-23  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	PR libstdc++/119137
+	* testsuite/23_containers/inplace_vector/access/elem.cc: Cover
+	front and back methods and const calls.
+	* testsuite/23_containers/inplace_vector/access/elem_neg.cc:
+	Likewise.
+	* testsuite/23_containers/inplace_vector/modifiers/erase_neg.cc:
+	New test.
+	* testsuite/23_containers/inplace_vector/modifiers/single_insert_neg.cc:
+	New test.
+
+2025-07-22  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	PR libstdc++/119137
+	* testsuite/23_containers/inplace_vector/cons/from_range.cc: Run
+	iterators and range test at compile-time.
+	* testsuite/23_containers/inplace_vector/modifiers/assign.cc:
+	Likewise.
+	* testsuite/23_containers/inplace_vector/modifiers/multi_insert.cc:
+	Likewise.
+	* testsuite/util/testsuite_iterators.h (__gnu_test::BoundsContainer)
+	(__gnu_test::OutputContainer, __gnu_test::WritableObject)
+	(__gnu_test::output_iterator_wrapper, __gnu_test::input_iterator_wrapper)
+	(__gnu_test::forward_iterator_wrapper)
+	(__gnu_test::bidirectional_iterator_wrapper)
+	(__gnu_test::random_access_iterator_wrapper)
+	(__gnu_test::test_container): Add appropriate _GLIBCXXNN_CONSTEXPR
+	macros to member functions.
+	(__gnu_test::contiguous_iterator_wrapper)
+	(__gnu_test::input_iterator_wrapper_rval)
+	(__gnu_test::test_range, __gnu_test::test_range_nocopy)
+	(__gnu_test::test_sized_range_sized_sent)
+	(__gnu_test::test_sized_range): Add constexpr specifier to member
+	functions.
+
+2025-07-22  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	PR libstdc++/119137
+	* include/std/inplace_vector (inplace_vector::assign_range):
+	Replace _Nm with _M_size in the assigment loop.
+
+2025-07-21  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	* include/std/mdspan (mdspan::mdspan): Make default ctor
+	conditionally noexcept.
+	* testsuite/23_containers/mdspan/mdspan.cc: Add tests.
+
+2025-07-21  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	* include/std/mdspan (mdspan::is_always_unique): Make
+	conditionally noexcept.
+	(mdspan::is_always_exhaustive): Ditto.
+	(mdspan::is_always_strided): Ditto.
+	(mdspan::is_unique): Ditto.
+	(mdspan::is_exhaustive): Ditto.
+	(mdspan::is_strided): Ditto.
+	* testsuite/23_containers/mdspan/layout_like.h: Make noexcept
+	configurable. Add ThrowingLayout.
+	* testsuite/23_containers/mdspan/mdspan.cc: Add tests for
+	noexcept.
+
+2025-07-20  Jakub Jelinek  <jakub@redhat.com>
+
+	PR libstdc++/121174
+	* src/c++23/std.cc.in (std::dextents): Export.  Add to FIXME comments
+	other not yet implemented nor exported <mdspan> entities.
+
+2025-07-18  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/stl_iterator_base_types.h (__any_input_iterator):
+	Only define when __cpp_lib_concepts is defined.
+
+2025-07-18  Jonathan Wakely  <jwakely@redhat.com>
+
+	* doc/xml/manual/appendix_contributing.xml: Remove Paolo from
+	list of maintainers to contact about contributing.
+	* doc/html/manual/appendix_contributing.html: Regenerate.
+
+2025-07-18  Jonathan Wakely  <jwakely@redhat.com>
+
+	* doc/xml/manual/build_hacking.xml: Document that
+	windows_zones-map.h is a generated file.
+	* doc/html/manual/appendix_porting.html: Regenerate.
+
+2025-07-18  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	PR libstdc++/119137
+	* include/std/inplace_vector (inplace_vector::operator=):
+	Qualify call to std::addressof.
+
+2025-07-18  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	PR libstdc++/121154
+	* include/bits/chrono_io.h (_ChronoSpec::_M_time_point): Remove.
+	(_ChronoSpec::_M_needs_ok_check): Define
+	(__formatter_chrono::_M_parse): Set _M_needs_ok_check.
+	(__formatter_chrono::_M_check_ok): Check values also for debug mode,
+	and return __string_view.
+	(__formatter_chrono::_M_format_to): Handle results of _M_check_ok.
+	(__formatter_chrono::_M_wi, __formatter_chrono::_M_a_A)
+	(__formatter_chrono::_M_b_B, __formatter_chrono::_M_C_y_Y)
+	(__formatter_chrono::_M_d_e, __formatter_chrono::_M_F):
+	Removed handling of _M_debug.
+	(__formatter_chrono::__M_m): Print zero unpadded in _M_debug mode.
+	(__formatter_duration::_S_spec_for): Remove _M_time_point refernce.
+	(__formatter_duration::_M_parse): Override _M_needs_ok_check.
+	* testsuite/std/time/month/io.cc: Test for localized !ok() values.
+	* testsuite/std/time/weekday/io.cc: Test for localized !ok() values.
+
+2025-07-18  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/121150
+	* testsuite/20_util/hash/int128.cc: Cast expected values to
+	size_t.
+
+2025-07-18  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/unicode.h (_Utf_iterator::operator--): Reorder
+	conditions and update position after reading a code unit.
+	(_Utf_iterator::_M_read_reverse): Define.
+	(_Utf_iterator::_M_read_utf8): Return extracted code point.
+	(_Utf_iterator::_M_read_reverse_utf8): Define.
+	(_Utf_iterator::_M_read_reverse_utf16): Define.
+	(_Utf_iterator::_M_read_reverse_utf32): Define.
+	* testsuite/ext/unicode/view.cc: Add checks for reversed views
+	and reverse iteration.
+
+2025-07-18  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/unicode.h (_Utf_iterator): Reorder data members
+	to be more compact.
+
+2025-07-18  Jonathan Wakely  <jwakely@redhat.com>
+	    Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	PR libstdc++/119137
+	* doc/doxygen/user.cfg.in (INPUT): Add new header.
+	* include/Makefile.am: Add new header.
+	* include/Makefile.in: Regenerate.
+	* include/bits/stl_iterator_base_types.h (__any_input_iterator):
+	Define.
+	* include/bits/version.def (inplace_vector): Define.
+	* include/bits/version.h: Regenerate.
+	* include/precompiled/stdc++.h: Include new header.
+	* src/c++23/std.cc.in: Export contents if new header.
+	* include/std/inplace_vector: New file.
+	* testsuite/23_containers/inplace_vector/access/capacity.cc: New file.
+	* testsuite/23_containers/inplace_vector/access/elem.cc: New file.
+	* testsuite/23_containers/inplace_vector/access/elem_neg.cc: New file.
+	* testsuite/23_containers/inplace_vector/cons/1.cc: New file.
+	* testsuite/23_containers/inplace_vector/cons/from_range.cc: New file.
+	* testsuite/23_containers/inplace_vector/cons/throws.cc: New file.
+	* testsuite/23_containers/inplace_vector/copy.cc: New file.
+	* testsuite/23_containers/inplace_vector/erasure.cc: New file.
+	* testsuite/23_containers/inplace_vector/modifiers/assign.cc: New file.
+	* testsuite/23_containers/inplace_vector/modifiers/erase.cc: New file.
+	* testsuite/23_containers/inplace_vector/modifiers/multi_insert.cc:
+	New file.
+	* testsuite/23_containers/inplace_vector/modifiers/single_insert.cc:
+	New file.
+	* testsuite/23_containers/inplace_vector/move.cc: New file.
+	* testsuite/23_containers/inplace_vector/relops.cc: New file.
+	* testsuite/23_containers/inplace_vector/version.cc: New file.
+	* testsuite/util/testsuite_iterators.h (input_iterator_wrapper::base):
+	Define.
+
+2025-07-17  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/96710
+	* doc/xml/manual/evolution.xml: Document change or __int128.
+	* doc/html/manual/api.html: Regenerate.
+
+2025-07-17  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	PR libstdc++/121061
+	* include/std/mdspan (extents::extents): Perform conversion to
+	index_type of an r-value reference.
+	(layout_left::mapping::operator()): Ditto.
+	(layout_right::mapping::operator()): Ditto.
+	(layout_stride::mapping::operator()): Ditto.
+	* testsuite/23_containers/mdspan/extents/custom_integer.cc: Add
+	tests for RValueInt and MutatingInt.
+	* testsuite/23_containers/mdspan/int_like.h (RValueInt): Add.
+	* testsuite/23_containers/mdspan/layouts/mapping.cc: Test with
+	RValueInt.
+	* testsuite/23_containers/mdspan/mdspan.cc: Ditto.
+
+2025-07-17  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	PR libstdc++/121061
+	* include/std/mdspan (extents::extents): Fix constraint to
+	prevent non-const conversion to index_type.
+	(layout_stride::mapping::mapping): Ditto.
+	(mdspan::mdspan): Ditto.
+	(mdspan::operator[]): Ditto.
+	* testsuite/23_containers/mdspan/extents/custom_integer.cc: Add
+	test for MutatingInt.
+	* testsuite/23_containers/mdspan/int_like.h (MutatingInt): Add.
+	* testsuite/23_containers/mdspan/layouts/mapping.cc: Add test for
+	MutatingInt.
+	* testsuite/23_containers/mdspan/layouts/stride.cc: Ditto.
+	* testsuite/23_containers/mdspan/mdspan.cc: Ditto.
+
+2025-07-17  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	PR libstdc++/121061
+	* testsuite/23_containers/mdspan/extents/custom_integer.cc:
+	Enable checking with different custom integers. Improve
+	checking non-existence of overloads for incompatible custom
+	integers.
+	* testsuite/23_containers/mdspan/layouts/mapping.cc: ditto. Also
+	improve reuse of int_like.h.
+	* testsuite/23_containers/mdspan/layouts/stride.cc: ditto.
+	* testsuite/23_containers/mdspan/mdspan.cc: ditto.
+	* testsuite/23_containers/mdspan/extents/int_like.h: Rename (old
+	name).
+	* testsuite/23_containers/mdspan/int_like.h: Rename (new name).
+	(ThrowingInt): Add.
+	(NotIntLike): Add.
+
+2025-07-17  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/unicode.h (__unicode::_Utf_iterator): Add
+	comments.
+	(__unicode:_Utf_iterator::operator++()): Check whether to
+	iterate over the buffer first.
+
+2025-07-17  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/121097
+	* include/c_global/cmath (hypot): Use __promote_3 instead of
+	__promoted.
+
+2025-07-16  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/std/bitset (__bitset::__string) [__cpp_lib_bitset]:
+	Change alias to refer to basic_string_view instead.
+
+2025-07-16  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	PR libstdc++/121114
+	* include/bits/version.def (constexpr_exceptions): Add no_stdname
+	and changed value.
+	* include/bits/version.h: Regenerated.
+	* testsuite/18_support/exception/version.cc: Test that macro is
+	not exported.
+
+2025-07-15  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/move.h (swap): Replace enable_if with concepts
+	when available, and with __enable_if_t alias otherwise.
+
+2025-07-15  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/ranges_base.h (ranges::advance(i, n, bound)):
+	Ensure that observable side effects on iterators match what is
+	specified in the standard.
+
+2025-07-15  Patrick Palka  <ppalka@redhat.com>
+
+	PR libstdc++/119962
+	* include/std/ranges (join_view::_Iterator::_M_outer): Initialize.
+	(lazy_split_view::_OuterIter::_M_current): Initialize.
+	(join_with_view::_Iterator::_M_outer_it): Initialize.
+	* testsuite/std/ranges/adaptors/join.cc (test15): New test.
+	* testsuite/std/ranges/adaptors/join_with/1.cc (test05): New test.
+	* testsuite/std/ranges/adaptors/lazy_split.cc (test13): New test.
+
+2025-07-15  Björn Schäpers  <bjoern@hazardy.de>
+
+	PR libstdc++/108409
+	* scripts/gen_windows_zones_map.py: New file, generates
+	windows_zones-map.h.
+	* src/c++20/windows_zones-map.h: New file, contains the look up
+	table.
+	* src/c++20/tzdb.cc (tzdb::current_zone): Add Windows code path.
+
+2025-07-15  Björn Schäpers  <bjoern@hazardy.de>
+
+	* src/c++20/tzdb.cc (zoneinfo_file): On Windows look relative
+	from the DLL path for the time zone information.
+
+2025-07-15  Jonathan Wakely  <jwakely@redhat.com>
+
+	* testsuite/25_algorithms/copy/debug/constexpr_neg.cc:
+	* testsuite/25_algorithms/copy_backward/debug/constexpr_neg.cc:
+	* testsuite/25_algorithms/equal/debug/constexpr_neg.cc:
+	* testsuite/25_algorithms/lower_bound/debug/constexpr_partitioned_neg.cc:
+	* testsuite/25_algorithms/lower_bound/debug/constexpr_partitioned_pred_neg.cc:
+	* testsuite/25_algorithms/lower_bound/debug/constexpr_valid_range_neg.cc:
+	* testsuite/25_algorithms/upper_bound/debug/constexpr_partitioned_neg.cc:
+	* testsuite/25_algorithms/upper_bound/debug/constexpr_partitioned_pred_neg.cc:
+	* testsuite/25_algorithms/upper_bound/debug/constexpr_valid_range_neg.cc:
+
+2025-07-15  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/121024
+	* include/bits/ranges_uninitialized.h (ranges::destroy): Do not
+	optimize away trivial destructors during constant evaluation.
+	(ranges::destroy_n): Likewise.
+	* testsuite/20_util/specialized_algorithms/destroy/121024.cc:
+	New test.
+
+2025-07-15  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/experimental/memory (swap, make_observer_ptr): Add
+	constexpr.
+	(operator==, operator!=, operator<, operator>, operator<=)
+	(operator>=): Likewise.
+	* testsuite/experimental/memory/observer_ptr/make_observer.cc:
+	Checks for constant evaluation.
+	* testsuite/experimental/memory/observer_ptr/relops/relops.cc:
+	Likewise.
+	* testsuite/experimental/memory/observer_ptr/swap/swap.cc:
+	Likewise.
+
+2025-07-15  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/std/type_traits (__make_unsigned_selector): Add
+	unsigned __int128 to type list.
+	* testsuite/20_util/make_unsigned/int128.cc: New test.
+
+2025-07-15  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/96710
+	* include/bits/functional_hash.h (hash<__int128>): Define for
+	strict modes.
+	(hash<unsigned __int128>): Likewise.
+	* testsuite/20_util/hash/int128.cc: New test.
+
+2025-07-15  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	PR libstdc++/110739
+	* include/bits/chrono_io.h (__formatter_chrono::_S_weekdays)
+	(__formatter_chrono::_S_months, __formatter_chrono::_S_fill_ampm):
+	Define.
+	(__formatter_chrono::_M_format_to): Do not pass context parameter
+	to functions listed below.
+	(__formatter_chrono::_M_a_A, __formatter_chrono::_M_b_B): Implement
+	using harcoded list of names, and remove format context parameter.
+	(__formatter_chrono::_M_p, __formatter_chrono::_M_r): Implement
+	using _S_fill_ampm.
+	(__formatter_chrono::_M_c): Removed format context parameter.
+	(__formatter_chrono::_M_subsecs): Call __ctx.locale() directly,
+	instead of _M_locale and do not compare with locale::classic().
+	Add [[unlikely]] attributes.
+	(__formatter_chrono::_M_locale): Move to __formatter_duration.
+	(__formatter_duration::_M_locale): Moved from __formatter_chrono.
+
+2025-07-14  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/stl_pair.h (swap): Add comment to deleted
+	overload.
+	* include/bits/unique_ptr.h (swap): Likewise.
+	* include/std/array (swap): Likewise.
+	* include/std/optional (swap): Likewise.
+	* include/std/tuple (swap): Likewise.
+	* include/std/variant (swap): Likewise.
+	* testsuite/23_containers/array/tuple_interface/get_neg.cc:
+	Adjust dg-error line numbers.
+
+2025-07-14  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/pstl/algorithm_impl.h (__for_each_n_it_serial):
+	Protect against overloaded comma operator.
+	(__brick_walk2): Likewise.
+	(__brick_walk2_n): Likewise.
+	(__brick_walk3): Likewise.
+	(__brick_move_destroy::operator()): Likewise.
+	(__brick_calc_mask_1): Likewise.
+	(__brick_copy_by_mask): Likewise.
+	(__brick_partition_by_mask): Likewise.
+	(__brick_calc_mask_2): Likewise.
+	(__brick_reverse): Likewise.
+	(__pattern_partial_sort_copy): Likewise.
+	* include/pstl/memory_impl.h (__brick_uninitialized_move):
+	Likewise.
+	(__brick_uninitialized_copy): Likewise.
+	* include/pstl/numeric_impl.h (__brick_transform_scan):
+	Likewise.
+
+2025-07-14  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/117785
+	* include/bits/version.def (constexpr_exceptions): Define
+	correct value.
+	* include/bits/version.h: Regenerate.
+	* libsupc++/exception: Check correct value.
+	* testsuite/18_support/exception/version.cc: New test.
+
+2025-07-14  Jonathan Wakely  <jwakely@redhat.com>
+
+	* libsupc++/exception_ptr.h (make_exception_ptr): Return null
+	for consteval when -fno-exceptions is used.
+	(exception_ptr_cast): Likewise. Allow consteval path to work
+	with -fno-rtti.
+
+2025-07-11  Jakub Jelinek  <jakub@redhat.com>
+
+	* libsupc++/exception_ptr.h: Implement C++26 P3748R0 - Inspecting
+	exception_ptr should be constexpr.
+	(std::exception_ptr_cast): Make constexpr, remove inline keyword.  Add
+	static_asserts for Mandates.  For if consteval use std::rethrow_exception,
+	catch it and return its address or nullptr.
+	* testsuite/18_support/exception_ptr/exception_ptr_cast.cc (E::~E): Add
+	constexpr.
+	(G::G): Likewise.
+	(test01): Likewise.  Return bool and take bool argument, throw if the
+	argument is true.  Add static_assert(test01(false)).
+	(main): Call test01(true) in try.
+
+2025-07-11  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/cpp_type_traits.h (__is_floating<__float128>):
+	Do not depend on __STRICT_ANSI__.
+	* include/bits/stl_algobase.h (__size_to_integer(__float128)):
+	Likewise.
+	* include/std/type_traits (__is_floating_point_helper<__float128>):
+	Likewise.
+
+2025-07-11  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/96710
+	* include/bits/cpp_type_traits.h (__is_integer): Define explicit
+	specializations for __int128.
+	(__memcpyable_integer): Remove explicit specializations for
+	__int128.
+	* include/bits/iterator_concepts.h (incrementable_traits):
+	Likewise.
+	(__is_signed_int128, __is_unsigned_int128, __is_int128): Remove.
+	(__is_integer_like, __is_signed_integer_like): Remove check for
+	__int128.
+	* include/bits/max_size_type.h: Remove all uses of __is_int128
+	in constraints.
+	* include/bits/ranges_base.h (__to_unsigned_like): Remove
+	overloads for __int128.
+	(ranges::ssize): Remove special case for __int128.
+	* include/bits/stl_algobase.h (__size_to_integer): Define
+	__int128 overloads for strict modes.
+	* include/ext/numeric_traits.h (__is_integer_nonstrict): Remove
+	explicit specializations for __int128.
+	* include/std/charconv (to_chars): Define overloads for
+	__int128.
+	* include/std/format (__format::make_unsigned_t): Remove.
+	(__format::to_chars): Remove.
+	* include/std/limits (numeric_limits): Define explicit
+	specializations for __int128.
+	* include/std/type_traits (__is_integral_helper): Likewise.
+	(__make_unsigned, __make_signed): Likewise.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	PR c++/117785
+	* include/bits/version.def (constexpr_exceptions): New.
+	* include/bits/version.h: Regenerate.
+	* libsupc++/exception (std::bad_exception::bad_exception): Add
+	_GLIBCXX26_CONSTEXPR.
+	(std::bad_exception::~bad_exception, std::bad_exception::what): For
+	C++26 add constexpr and define inline.
+	* libsupc++/exception.h (std::exception::exception,
+	std::exception::operator=): Add _GLIBCXX26_CONSTEXPR.
+	(std::exception::~exception, std::exception::what): For C++26 add
+	constexpr and define inline.
+	* libsupc++/exception_ptr.h (std::make_exception_ptr): Add
+	_GLIBCXX26_CONSTEXPR.  For if consteval use just throw with
+	current_exception() in catch.
+	(std::exception_ptr::exception_ptr(void*)): For C++26 add constexpr
+	and define inline.
+	(std::exception_ptr::exception_ptr()): Add _GLIBCXX26_CONSTEXPR.
+	(std::exception_ptr::exception_ptr(const exception_ptr&)): Likewise.
+	Use __builtin_eh_ptr_adjust_ref if consteval and compiler has it
+	instead of _M_addref.
+	(std::exception_ptr::exception_ptr(nullptr_t)): Add
+	_GLIBCXX26_CONSTEXPR.
+	(std::exception_ptr::exception_ptr(exception_ptr&&)): Likewise.
+	(std::exception_ptr::operator=): Likewise.
+	(std::exception_ptr::~exception_ptr): Likewise.  Use
+	__builtin_eh_ptr_adjust_ref if consteval and compiler has it
+	instead of _M_release.
+	(std::exception_ptr::swap): Add _GLIBCXX26_CONSTEXPR.
+	(std::exception_ptr::operator bool): Likewise.
+	(std::exception_ptr::operator==): Likewise.
+	* libsupc++/nested_exception.h
+	(std::nested_exception::nested_exception): Add _GLIBCXX26_CONSTEXPR.
+	(std::nested_exception::operator=): Likewise.
+	(std::nested_exception::~nested_exception): For C++26 add constexpr
+	and define inline.
+	(std::nested_exception::rethrow_if_nested): Add _GLIBCXX26_CONSTEXPR.
+	(std::nested_exception::nested_ptr): Likewise.
+	(std::_Nested_exception::_Nested_exception): Likewise.
+	(std::throw_with_nested, std::rethrow_if_nested): Likewise.
+	* libsupc++/new (std::bad_alloc::bad_alloc): Likewise.
+	(std::bad_alloc::operator=): Likewise.
+	(std::bad_alloc::~bad_alloc): For C++26 add constexpr and define
+	inline.
+	(std::bad_alloc::what): Likewise.
+	(std::bad_array_new_length::bad_array_new_length): Add
+	_GLIBCXX26_CONSTEXPR.
+	(std::bad_array_new_length::~bad_array_new_length): For C++26 add
+	constexpr and define inline.
+	(std::bad_array_new_length::what): Likewise.
+	* libsupc++/typeinfo (std::bad_cast::bad_cast): Add
+	_GLIBCXX26_CONSTEXPR.
+	(std::bad_cast::~bad_cast): For C++26 add constexpr and define inline.
+	(std::bad_cast::what): Likewise.
+	(std::bad_typeid::bad_typeid): Add _GLIBCXX26_CONSTEXPR.
+	(std::bad_typeid::~bad_typeid): For C++26 add constexpr and define
+	inline.
+	(std::bad_typeid::what): Likewise.
+
+2025-07-10  Jakub Jelinek  <jakub@redhat.com>
+
+	* testsuite/22_locale/codecvt/codecvt_unicode.h
+	(ucs2_to_utf8_out_error): Comment spelling fix: bellow -> below.
+	(utf16_to_ucs2_in_error): Likewise.
+
+2025-07-09  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/stl_uninitialized.h (__uninitialized_default):
+	Do not use optimized implementation for constexpr case. Use
+	_GLIBCXX20_CONSTEXPR instead of _GLIBCXX26_CONSTEXPR.
+
+2025-07-09  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/std/mdspan (mdspan): Add template keyword for
+	dependent name.
+
+2025-07-09  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/120997
+	* include/std/span (span::first, span::last, span::subspan): Do
+	not use braced-init-list for return statements.
+	* testsuite/23_containers/span/120997.cc: New test.
+
+2025-07-09  Paul Keir  <pkeir@outlook.com>
+
+	PR libstdc++/117403
+	* include/bits/shared_ptr.h (shared_ptr::owner_equal)
+	(shared_ptr::owner_hash, weak_ptr::owner_equal)
+	(weak_ptr::owner_hash): Define new member functions.
+	* include/bits/shared_ptr_base.h (owner_equal, owner_hash):
+	Define new structs.
+	* include/bits/version.def (smart_ptr_owner_equality): Define.
+	* include/bits/version.h: Regenerate.
+	* include/std/memory: Added define for
+	__glibcxx_want_smart_ptr_owner_equality.
+	* testsuite/20_util/owner_equal/version.cc: New test.
+	* testsuite/20_util/owner_equal/cmp.cc: New test.
+	* testsuite/20_util/owner_equal/noexcept.cc: New test.
+	* testsuite/20_util/owner_hash/cmp.cc: New test.
+	* testsuite/20_util/owner_hash/noexcept.cc: New test.
+	* testsuite/20_util/shared_ptr/observers/owner_equal.cc: New
+	test.
+	* testsuite/20_util/shared_ptr/observers/owner_hash.cc:
+	New test.
+	* testsuite/20_util/weak_ptr/observers/owner_equal.cc: New test.
+	* testsuite/20_util/weak_ptr/observers/owner_hash.cc: New test.
+
+2025-07-09  Mateusz Zych  <mte.zych@gmail.com>
+
+	* include/bits/max_size_type.h (numeric_limits<__max_size_type>):
+	New members.
+	(numeric_limits<__max_diff_type>): Likewise.
+	* testsuite/std/ranges/iota/max_size_type.cc: New test cases.
+
+2025-07-09  Jonathan Wakely  <jwakely@redhat.com>
+
+	* src/c++17/memory_resource.cc: Adjust indentation of unnamed
+	namespaces.
+	(pool_sizes): Add comment.
+	(choose_block_size): Move outside preprocessor group for
+	gthreads targets.
+	* testsuite/20_util/synchronized_pool_resource/118681.cc:
+	Require gthreads.
+
+2025-07-09  H.J. Lu  <hjl.tools@gmail.com>
+
+	* config/abi/post/x86_64-linux-gnu/x32/baseline_symbols.txt:
+	Updated.
+
+2025-07-08  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/118681
+	* testsuite/20_util/unsynchronized_pool_resource/118681.cc: Fix
+	deallocate argument.
+
+2025-07-08  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/118681
+	* src/c++17/memory_resource.cc (choose_block_size): New
+	function.
+	(synchronized_pool_resource::do_allocate): Use choose_block_size
+	to determine appropriate block size.
+	(synchronized_pool_resource::do_deallocate): Likewise
+	(unsynchronized_pool_resource::do_allocate): Likewise.
+	(unsynchronized_pool_resource::do_deallocate): Likewise
+	* testsuite/20_util/synchronized_pool_resource/118681.cc: New
+	test.
+	* testsuite/20_util/unsynchronized_pool_resource/118681.cc: New
+	test.
+
+2025-07-08  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/debug/forward_list (_Safe_forward_list<>::_M_swap):
+	Adapt to _M_this() signature change.
+
+2025-07-08  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	PR libstdc++/119861
+	* include/std/format (formatter<_Rg, _CharT>::set_separator)
+	(formatter<_Rg, _CharT>::set_brackets): Constrain with
+	(format_kind<_Rg> == range_format::sequence).
+	* testsuite/std/format/ranges/pr119861_neg.cc: New test.
+
+2025-07-08  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	PR libstdc++/120914
+	* include/std/span (span): Update CTAD to enable
+	integral constants [P3029R1].
+	* include/std/mdspan (extents): ditto.
+	(mdspan): ditto.
+	* testsuite/23_containers/span/deduction.cc: Test deduction
+	guide.
+	* testsuite/23_containers/mdspan/extents/misc.cc: ditto.
+	* testsuite/23_containers/mdspan/mdspan.cc: ditto.
+
+2025-07-08  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	* testsuite/23_containers/span/contiguous_range_neg.cc: Silence
+	warning about unused variable myspan.
+
+2025-07-08  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	PR libstdc++/107761
+	* include/bits/version.def (mdspan): Set to 202207 and remove
+	no_stdname.
+	* include/bits/version.h: Regenerate.
+	* testsuite/23_containers/mdspan/version.cc: Test presence
+	of feature test macro.
+
+2025-07-08  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	PR libstdc++/107761
+	* include/std/mdspan (mdspan): New class.
+	* src/c++23/std.cc.in (mdspan): Add.
+	* testsuite/23_containers/mdspan/class_mandate_neg.cc: New test.
+	* testsuite/23_containers/mdspan/mdspan.cc: New test.
+	* testsuite/23_containers/mdspan/layout_like.h: Add class
+	LayoutLike which models a user-defined layout.
+	* testsuite/23_containers/mdspan/out_of_bounds_neg.cc: New file.
+
+2025-07-08  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	* include/std/mdspan (__mdspan::__size): New function.
+
+2025-07-08  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	* testsuite/23_containers/mdspan/extents/custom_integer.cc:
+	Delete IntLike and include "int_like.h".
+	* testsuite/23_containers/mdspan/extents/int_like.h: Add
+	IntLike.
+
+2025-07-08  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	* include/std/mdspan (extents): Check prerequisite of the ctor that
+	static_extent(i) == dynamic_extent || extent(i) == other.extent(i).
+	* testsuite/23_containers/mdspan/extents/class_mandates_neg.cc:
+	Test the implemented prerequisite.
+	* testsuite/23_containers/mdspan/extents/extents_mismatch_neg.cc: New file.
+
+2025-07-08  Luc Grosheintz  <luc.grosheintz@gmail.com>
+
+	* include/std/mdspan: Check prerequisites of
+	layout_*::operator() with _GLIBCXX_DEBUG_ASSERTs.
+	* testsuite/23_containers/mdspan/layouts/debug/out_of_bounds_neg.cc:
+	Add tests for prerequisites.
+
+2025-07-08  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	* include/std/queue (formatter<queue<_Tp, _Container>, _CharT>)
+	(formatter<priority_queue<_Tp, _Container, _Compare>, _CharT>):
+	Add _GLIBCXX_RESOLVE_LIB_DEFECTS comments.
+
+2025-07-08  François Dumont  <frs.dumont@gmail.com>
+
+	PR c++/116369
+	* config/abi/pre/gnu-versioned-namespace.ver: Use new const qualified symbols.
+	* config/abi/pre/gnu.ver: Add new const qualified symbols.
+	* include/debug/safe_base.h
+	(_Safe_iterator_base::_M_sequence): Declare as pointer-to-const.
+	(_Safe_iterator_base::_M_attach, _M_attach_single): New, take pointer-to-const
+	_Safe_sequence_base.
+	(_Safe_sequence_base::_M_detach_all, _M_detach_singular, _M_revalidate_singular)
+	(_M_swap, _M_get_mutex): New, const qualified.
+	(_Safe_sequence_base::_M_attach, _M_attach_single, _M_detach, _M_detach_single):
+	const qualify.
+	* include/debug/safe_container.h (_Safe_container<>::_M_cont): Add const qualifier.
+	(_Safe_container<>::_M_swap_base): New.
+	(_Safe_container(_Safe_container&&, const _Alloc&, std::false_type)):
+	Adapt to use latter.
+	(_Safe_container<>::operator=(_Safe_container&&)): Likewise.
+	(_Safe_container<>::_M_swap): Likewise and take parameter as const reference.
+	* include/debug/safe_unordered_base.h
+	(_Safe_local_iterator_base::_M_safe_container): New.
+	(_Safe_local_iterator_base::_Safe_local_iterator_base): Take
+	_Safe_unordered_container_base as pointer-to-const.
+	(_Safe_unordered_container_base::_M_attach, _M_attach_single): New, take
+	container as _Safe_unordered_container_base pointer-to-const.
+	(_Safe_unordered_container_base::_M_local_iterators, _M_const_local_iterators):
+	Add mutable.
+	(_Safe_unordered_container_base::_M_detach_all, _M_swap): New, const qualify.
+	(_Safe_unordered_container_base::_M_attach_local, _M_attach_local_single)
+	(_M_detach_local, _M_detach_local_single): Add const qualifier.
+	* include/debug/safe_unordered_container.h (_Safe_unordered_container::_M_self()): New.
+	* include/debug/safe_unordered_container.tcc
+	(_Safe_unordered_container::_M_invalidate_if, _M_invalidated_local_if): Use latter.
+	* include/debug/safe_iterator.h (_Safe_iterator<>::_M_attach, _M_attach_single):
+	Take _Safe_sequence_base as pointer-to-const.
+	(_Safe_iterator<>::_M_get_sequence): Add const_cast and comment about it.
+	* include/debug/safe_local_iterator.h (_Safe_local_iterator<>): Replace usages
+	of _M_sequence member by _M_safe_container().
+	(_Safe_local_iterator<>::_M_attach, _M_attach_single): Take
+	_Safe_unordered_container_base as pointer-to-const.
+	(_Safe_local_iterator<>::_M_get_sequence): Rename into...
+	(_Safe_local_iterator<>::_M_get_ucontainer): ...this. Add necessary const_cast and
+	comment to explain it.
+	(_Safe_local_iterator<>::_M_is_begin, _M_is_end): Adapt.
+	* include/debug/safe_local_iterator.tcc: Adapt.
+	* include/debug/safe_sequence.h
+	(_Safe_sequence<>::_M_invalidate_if, _M_transfer_from_if): Add const qualifier.
+	* include/debug/safe_sequence.tcc: Adapt.
+	* include/debug/deque (std::__debug::deque::erase): Adapt to use new const
+	qualified methods.
+	* include/debug/formatter.h: Adapt.
+	* include/debug/forward_list (_Safe_forward_list::_M_this): Add const
+	qualification and return pointer for consistency with 'this' keyword.
+	(_Safe_forward_list::_M_swap_aux): Rename into...
+	(_Safe_forward_list::_S_swap_aux): ...this and take sequence as const reference.
+	(forward_list<>::resize): Adapt to use const methods.
+	* include/debug/list (list<>::resize): Likewise.
+	* src/c++11/debug.cc: Adapt to const qualification.
+	* testsuite/util/testsuite_containers.h
+	(forward_members_unordered::forward_members_unordered): Add check on local_iterator
+	conversion to const_local_iterator.
+	(forward_members::forward_members): Add check on iterator conversion to
+	const_iterator.
+	* testsuite/23_containers/unordered_map/const_container.cc: New test case.
+	* testsuite/23_containers/unordered_multimap/const_container.cc: New test case.
+	* testsuite/23_containers/unordered_multiset/const_container.cc: New test case.
+	* testsuite/23_containers/unordered_set/const_container.cc: New test case.
+	* testsuite/23_containers/vector/debug/mutex_association.cc: Adapt.
+
+2025-07-07  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/120949
+	* include/bits/stl_iterator.h (__normal_iterator): Fix order of
+	always_inline and nodiscard attributes for Clang compatibility.
+
+2025-07-07  Jonathan Wakely  <jwakely@redhat.com>
+
+	* testsuite/util/testsuite_hooks.h (VERIFY): Define as variadic
+	macro.
+	* testsuite/ext/verify_neg.cc: New test.
+
+2025-07-07  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/std/mdspan (__mapping_of): Add template keyword.
+
+2025-07-07  XU Kailiang  <xu2k3l4@outlook.com>
+	    Tomasz Kaminski  <tkaminsk@redhat.com>
+
+	PR libstdc++/117214
+	* include/bits/chrono_io.h (_ChronoSpec::_M_time_only): Remove.
+	(_ChronoSpec::_M_time_point): Define.
+	(__formatter_chrono::_M_parse): Use __parts to determine
+	interpretation of j.
+	(__formatter_chrono::_M_check_ok): Define.
+	(__formatter_chrono::_M_format_to): Invoke _M_check_ok.
+	(__formatter_chrono::_M_a_A, __formatter_chrono::_M_b_B): Move
+	exception throwing to _M_check_ok.
+	(__formatter_chrono::_M_j): Use _M_needs to define interpretation.
+	(__formatter_duration::_S_spec_for): Set _M_time_point.
+	* testsuite/std/time/format/format.cc: Test for exception for !ok()
+	months/weekday.
+	* testsuite/std/time/format/pr117214_custom_timeput.cc: New
+	test.
+
+2025-07-07  Tomasz Kamiński  <tkaminsk@redhat.com>
+
+	PR libstdc++/120976
+	* include/std/format (formatter<__float128, _Char_T): Define if
+	_GLIBCXX_FORMAT_F128 == 2.
+
+2025-07-06  Patrick Palka  <ppalka@redhat.com>
+
+	* include/bits/ranges_algo.h (shift_left, shift_right): Guard
+	with __glibcxx_shift >= 201806L.
+	(ranges::__shift_left_fn, ranges::shift_left): Define for C++23.
+	(ranges::__shift_right_fn, ranges::shift_right): Likewise.
+	* include/bits/version.def (shift): Update for C++23.
+	* include/bits/version.h: Regenerate.
+	* src/c++23/std.cc.in: Add ranges::shift_left/right.
+	* testsuite/25_algorithms/shift_left/constrained.cc: New test,
+	based off of 1.cc.
+	* testsuite/25_algorithms/shift_right/constrained.cc: New test,
+	based off of 1.cc.
+
+2025-07-04  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/chrono_io.h (__formatter_duration::_S_spec_for):
+	Add default case to switch and use __builtin_unreachable.
+
+2025-07-04  Jonathan Wakely  <jwakely@redhat.com>
+
+	* include/bits/stl_algobase.h (__size_to_integer): Move
+	misplaced unsigned keyword on __size_to_integer overloads for
+	__GLIBCXX_TYPE_INT_N_3 integer type.
+
+2025-07-03  Nathan Myers  <ncm@cantrip.org>
+
+	* include/bits/version.def: Fix typo.
+	* include/bits/version.h: Rebuilt.
+
+2025-07-03  Patrick Palka  <ppalka@redhat.com>
+
+	PR libstdc++/120934
+	* include/std/ranges (concat_view::end): Refine condition
+	for returning an iterator instead of default_sentinel as
+	per the updated P/R for LWG 4166.
+	* testsuite/std/ranges/concat/1.cc (test05): New test.
+
+2025-07-03  Nathan Myers  <ncm@cantrip.org>
+
+	PR libstdc++/119742
+	* include/bits/version.def: Add preprocessor symbol.
+	* include/bits/version.h: Add preprocessor symbol.
+	* include/std/bitset: Add constructor.
+	* testsuite/20_util/bitset/cons/1.cc: Fix.
+	* testsuite/20_util/bitset/cons/6282.cc: Fix.
+	* testsuite/20_util/bitset/cons/string_view.cc: Test new ctor.
+	* testsuite/20_util/bitset/cons/string_view_wide.cc: Test new ctor.
+
+2025-07-03  Jonathan Wakely  <jwakely@redhat.com>
+
+	PR libstdc++/120931
+	* include/bits/stl_uninitialized.h (__uninitialized_fill<true>):
+	Fix typo resulting in call to __do_uninit_copy instead of
+	__do_uninit_fill.
+	* testsuite/20_util/specialized_algorithms/uninitialized_fill/120931.cc:
+	New test.
+
 2025-07-02  Jonathan Wakely  <jwakely@redhat.com>
 
 	* include/bits/stl_iterator.h (__normal_iterator): Make all
diff --git a/libstdc++-v3/config/abi/post/x86_64-linux-gnu/x32/baseline_symbols.txt b/libstdc++-v3/config/abi/post/x86_64-linux-gnu/x32/baseline_symbols.txt
index ac11d5d..5d55287 100644
--- a/libstdc++-v3/config/abi/post/x86_64-linux-gnu/x32/baseline_symbols.txt
+++ b/libstdc++-v3/config/abi/post/x86_64-linux-gnu/x32/baseline_symbols.txt
@@ -2124,6 +2124,10 @@ FUNC:_ZNSt12__shared_ptrINSt10filesystem7__cxx114_DirELN9__gnu_cxx12_Lock_policy
 FUNC:_ZNSt12__shared_ptrINSt10filesystem7__cxx114_DirELN9__gnu_cxx12_Lock_policyE2EEC2EOS5_@@GLIBCXX_3.4.28
 FUNC:_ZNSt12__shared_ptrINSt10filesystem7__cxx114_DirELN9__gnu_cxx12_Lock_policyE2EEC2Ev@@GLIBCXX_3.4.27
 FUNC:_ZNSt12__shared_ptrINSt10filesystem7__cxx114_DirELN9__gnu_cxx12_Lock_policyE2EEaSEOS5_@@GLIBCXX_3.4.26
+FUNC:_ZNSt12__sso_stringC1Ev@@GLIBCXX_3.4.34
+FUNC:_ZNSt12__sso_stringC2Ev@@GLIBCXX_3.4.34
+FUNC:_ZNSt12__sso_stringD1Ev@@GLIBCXX_3.4.34
+FUNC:_ZNSt12__sso_stringD2Ev@@GLIBCXX_3.4.34
 FUNC:_ZNSt12bad_weak_ptrD0Ev@@GLIBCXX_3.4.15
 FUNC:_ZNSt12bad_weak_ptrD1Ev@@GLIBCXX_3.4.15
 FUNC:_ZNSt12bad_weak_ptrD2Ev@@GLIBCXX_3.4.15
@@ -3221,6 +3225,8 @@ FUNC:_ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE12_Alloc_hiderC1EPcRKS
 FUNC:_ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE12_Alloc_hiderC2EPcOS3_@@GLIBCXX_3.4.23
 FUNC:_ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE12_Alloc_hiderC2EPcRKS3_@@GLIBCXX_3.4.21
 FUNC:_ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE12_M_constructEjc@@GLIBCXX_3.4.21
+FUNC:_ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE12_M_constructILb0EEEvPKcj@@GLIBCXX_3.4.34
+FUNC:_ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE12_M_constructILb1EEEvPKcj@@GLIBCXX_3.4.34
 FUNC:_ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE12_M_constructIN9__gnu_cxx17__normal_iteratorIPKcS4_EEEEvT_SB_St20forward_iterator_tag@@GLIBCXX_3.4.21
 FUNC:_ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE12_M_constructIN9__gnu_cxx17__normal_iteratorIPcS4_EEEEvT_SA_St20forward_iterator_tag@@GLIBCXX_3.4.21
 FUNC:_ZNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE12_M_constructIPKcEEvT_S8_St20forward_iterator_tag@@GLIBCXX_3.4.21
@@ -3374,6 +3380,8 @@ FUNC:_ZNSt7__cxx1112basic_stringIwSt11char_traitsIwESaIwEE12_Alloc_hiderC1EPwRKS
 FUNC:_ZNSt7__cxx1112basic_stringIwSt11char_traitsIwESaIwEE12_Alloc_hiderC2EPwOS3_@@GLIBCXX_3.4.23
 FUNC:_ZNSt7__cxx1112basic_stringIwSt11char_traitsIwESaIwEE12_Alloc_hiderC2EPwRKS3_@@GLIBCXX_3.4.21
 FUNC:_ZNSt7__cxx1112basic_stringIwSt11char_traitsIwESaIwEE12_M_constructEjw@@GLIBCXX_3.4.21
+FUNC:_ZNSt7__cxx1112basic_stringIwSt11char_traitsIwESaIwEE12_M_constructILb0EEEvPKwj@@GLIBCXX_3.4.34
+FUNC:_ZNSt7__cxx1112basic_stringIwSt11char_traitsIwESaIwEE12_M_constructILb1EEEvPKwj@@GLIBCXX_3.4.34
 FUNC:_ZNSt7__cxx1112basic_stringIwSt11char_traitsIwESaIwEE12_M_constructIN9__gnu_cxx17__normal_iteratorIPKwS4_EEEEvT_SB_St20forward_iterator_tag@@GLIBCXX_3.4.21
 FUNC:_ZNSt7__cxx1112basic_stringIwSt11char_traitsIwESaIwEE12_M_constructIN9__gnu_cxx17__normal_iteratorIPwS4_EEEEvT_SA_St20forward_iterator_tag@@GLIBCXX_3.4.21
 FUNC:_ZNSt7__cxx1112basic_stringIwSt11char_traitsIwESaIwEE12_M_constructIPKwEEvT_S8_St20forward_iterator_tag@@GLIBCXX_3.4.21
@@ -3941,6 +3949,8 @@ FUNC:_ZNSt8__detail15_List_node_base11_M_transferEPS0_S1_@@GLIBCXX_3.4.15
 FUNC:_ZNSt8__detail15_List_node_base4swapERS0_S1_@@GLIBCXX_3.4.15
 FUNC:_ZNSt8__detail15_List_node_base7_M_hookEPS0_@@GLIBCXX_3.4.15
 FUNC:_ZNSt8__detail15_List_node_base9_M_unhookEv@@GLIBCXX_3.4.15
+FUNC:_ZNSt8__format25__locale_encoding_to_utf8ERKSt6localeSt17basic_string_viewIcSt11char_traitsIcEEPv@@GLIBCXX_3.4.34
+FUNC:_ZNSt8__format26__with_encoding_conversionERKSt6locale@@GLIBCXX_3.4.34
 FUNC:_ZNSt8bad_castD0Ev@@GLIBCXX_3.4
 FUNC:_ZNSt8bad_castD1Ev@@GLIBCXX_3.4
 FUNC:_ZNSt8bad_castD2Ev@@GLIBCXX_3.4
@@ -4617,6 +4627,7 @@ OBJECT:0:GLIBCXX_3.4.30
 OBJECT:0:GLIBCXX_3.4.31
 OBJECT:0:GLIBCXX_3.4.32
 OBJECT:0:GLIBCXX_3.4.33
+OBJECT:0:GLIBCXX_3.4.34
 OBJECT:0:GLIBCXX_3.4.4
 OBJECT:0:GLIBCXX_3.4.5
 OBJECT:0:GLIBCXX_3.4.6
diff --git a/libstdc++-v3/config/abi/pre/gnu-versioned-namespace.ver b/libstdc++-v3/config/abi/pre/gnu-versioned-namespace.ver
index 2818ab3..1c423ff 100644
--- a/libstdc++-v3/config/abi/pre/gnu-versioned-namespace.ver
+++ b/libstdc++-v3/config/abi/pre/gnu-versioned-namespace.ver
@@ -117,11 +117,11 @@ GLIBCXX_8.0 {
     _ZN9__gnu_cxx3__818stdio_sync_filebufI[cw]NSt3__811char_traitsI[cw]EEE[5-9]*;
 
     # debug mode
-    _ZN11__gnu_debug19_Safe_sequence_base12_M_get_mutexEv;
-    _ZN11__gnu_debug19_Safe_sequence_base13_M_detach_allEv;
-    _ZN11__gnu_debug19_Safe_sequence_base18_M_detach_singularEv;
-    _ZN11__gnu_debug19_Safe_sequence_base22_M_revalidate_singularEv;
-    _ZN11__gnu_debug19_Safe_sequence_base7_M_swapERS0_;
+    _ZNK11__gnu_debug19_Safe_sequence_base12_M_get_mutexEv;
+    _ZNK11__gnu_debug19_Safe_sequence_base13_M_detach_allEv;
+    _ZNK11__gnu_debug19_Safe_sequence_base18_M_detach_singularEv;
+    _ZNK11__gnu_debug19_Safe_sequence_base22_M_revalidate_singularEv;
+    _ZNK11__gnu_debug19_Safe_sequence_base7_M_swapERKS0_;
 
     _ZN11__gnu_debug19_Safe_iterator_base9_M_attach*;
     _ZN11__gnu_debug19_Safe_iterator_base16_M_attach_single*;
@@ -136,11 +136,11 @@ GLIBCXX_8.0 {
 
     # __gnu_debug::_Safe_unordered_container_base
     # __gnu_debug::_Safe_local_iterator_base
-    _ZN11__gnu_debug30_Safe_unordered_container_base7_M_swapERS0_;
-    _ZN11__gnu_debug30_Safe_unordered_container_base13_M_detach_allEv;
-    _ZN11__gnu_debug25_Safe_local_iterator_base9_M_attachEPNS_19_Safe_sequence_baseEb;
+    _ZNK11__gnu_debug30_Safe_unordered_container_base7_M_swapERKS0_;
+    _ZNK11__gnu_debug30_Safe_unordered_container_base13_M_detach_allEv;
+    _ZN11__gnu_debug25_Safe_local_iterator_base9_M_attachEPKNS_30_Safe_unordered_container_baseEb;
     _ZN11__gnu_debug25_Safe_local_iterator_base9_M_detachEv;
-    _ZN11__gnu_debug25_Safe_local_iterator_base16_M_attach_singleEPNS_19_Safe_sequence_baseEb;
+    _ZN11__gnu_debug25_Safe_local_iterator_base16_M_attach_singleEPKNS_30_Safe_unordered_container_baseEb;
 
     # parallel mode
     _ZN14__gnu_parallel9_Settings3getEv;
diff --git a/libstdc++-v3/config/abi/pre/gnu.ver b/libstdc++-v3/config/abi/pre/gnu.ver
index 73b6f33..b5a89c3 100644
--- a/libstdc++-v3/config/abi/pre/gnu.ver
+++ b/libstdc++-v3/config/abi/pre/gnu.ver
@@ -2555,6 +2555,20 @@ GLIBCXX_3.4.35 {
     _ZNSt8__detail17__wait_until_implEPKvRNS_16__wait_args_baseERKNSt6chrono8durationI[lx]St5ratioIL[lx]1EL[lx]1000000000EEEE;
     _ZNSt8__detail11__wait_args22_M_load_proxy_wait_valEPKv;
 
+    # __gnu_debug::_Safe_iterator_base and _Safe_sequence_base const
+    _ZN11__gnu_debug19_Safe_iterator_base9_M_attachEPKNS_19_Safe_sequence_baseEb;
+    _ZN11__gnu_debug19_Safe_iterator_base16_M_attach_singleEPKNS_19_Safe_sequence_baseEb;
+    _ZNK11__gnu_debug19_Safe_sequence_base13_M_detach_allEv;
+    _ZNK11__gnu_debug19_Safe_sequence_base18_M_detach_singularEv;
+    _ZNK11__gnu_debug19_Safe_sequence_base12_M_get_mutexEv;
+    _ZNK11__gnu_debug19_Safe_sequence_base22_M_revalidate_singularEv;
+    _ZNK11__gnu_debug19_Safe_sequence_base7_M_swapERKS0_;
+
+    # __gnu_debug::_Safe_local_iterator_base and _Safe_unordered_container_base const
+    _ZN11__gnu_debug25_Safe_local_iterator_base9_M_attachEPKNS_30_Safe_unordered_container_baseEb;
+    _ZN11__gnu_debug25_Safe_local_iterator_base16_M_attach_singleEPKNS_30_Safe_unordered_container_baseEb;
+    _ZNK11__gnu_debug30_Safe_unordered_container_base13_M_detach_allEv;
+    _ZNK11__gnu_debug30_Safe_unordered_container_base7_M_swapERKS0_;
 } GLIBCXX_3.4.34;
 
 # Symbols in the support library (libsupc++) have their own tag.
diff --git a/libstdc++-v3/doc/doxygen/user.cfg.in b/libstdc++-v3/doc/doxygen/user.cfg.in
index 536e035..8969bb8 100644
--- a/libstdc++-v3/doc/doxygen/user.cfg.in
+++ b/libstdc++-v3/doc/doxygen/user.cfg.in
@@ -869,6 +869,7 @@ INPUT                  = @srcdir@/doc/doxygen/doxygroups.cc \
                          include/functional \
                          include/future \
                          include/generator \
+                         include/inplace_vector \
                          include/iomanip \
                          include/ios \
                          include/iosfwd \
diff --git a/libstdc++-v3/doc/html/manual/api.html b/libstdc++-v3/doc/html/manual/api.html
index 4441d9c..f004ddc 100644
--- a/libstdc++-v3/doc/html/manual/api.html
+++ b/libstdc++-v3/doc/html/manual/api.html
@@ -512,4 +512,7 @@ Nested <code class="code">result_type</code> and <code class="code">argument_typ
 </p></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="api.rel_16"></a><code class="constant">16</code></h3></div></div></div><p>
 Deprecated the non-standard overload of <code class="code">std::fabs</code> for
 <code class="code">std::complex</code> arguments.
+</p><p>
+Made <code class="code">std::is_integral_v&lt;__int128&gt;</code> true for strict
+<code class="code">-std</code> dialects.
 </p></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="abi.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="appendix_porting.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="backwards.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">ABI Policy and Guidelines </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> Backwards Compatibility</td></tr></table></div></body></html>
\ No newline at end of file
diff --git a/libstdc++-v3/doc/html/manual/appendix_contributing.html b/libstdc++-v3/doc/html/manual/appendix_contributing.html
index 24d5a04..baac20a 100644
--- a/libstdc++-v3/doc/html/manual/appendix_contributing.html
+++ b/libstdc++-v3/doc/html/manual/appendix_contributing.html
@@ -60,13 +60,11 @@
       this question would be appreciated.
     </p><p>
       Please contact
-      Paolo Carlini at <code class="email">&lt;<a class="email" href="mailto:paolo.carlini@oracle.com">paolo.carlini@oracle.com</a>&gt;</code>
-      or
       Jonathan Wakely at <code class="email">&lt;<a class="email" href="mailto:jwakely+assign@redhat.com">jwakely+assign@redhat.com</a>&gt;</code>
       if you are confused about the assignment or have general licensing
       questions. When requesting an assignment form from
       <code class="email">&lt;<a class="email" href="mailto:assign@gnu.org">assign@gnu.org</a>&gt;</code>, please CC the libstdc++
-      maintainers above so that progress can be monitored.
+      maintainer above so that progress can be monitored.
     </p></div><div class="section"><div class="titlepage"><div><div><h3 class="title"><a id="list.getting"></a>Getting Sources</h3></div></div></div><p>
       <a class="link" href="https://gcc.gnu.org/gitwrite.html" target="_top">Getting write access
 	(look for "Write after approval")</a>
diff --git a/libstdc++-v3/doc/html/manual/appendix_porting.html b/libstdc++-v3/doc/html/manual/appendix_porting.html
index e0f52db..887fa50 100644
--- a/libstdc++-v3/doc/html/manual/appendix_porting.html
+++ b/libstdc++-v3/doc/html/manual/appendix_porting.html
@@ -497,4 +497,11 @@ baseline file.
    See <a class="xref" href="configure.html" title="Configure">Configuring</a> for the
    <code class="code">--with-libstdcxx-zoneinfo</code> option that determines whether
    this file is used.
+   </p></li><li class="listitem"><p>
+   The header file
+   <code class="filename">src/c++20/windows_zones-map.h</code>
+   is generated by the Python script
+   <code class="filename">scripts/gen_windows_zones_map.py</code>
+   using the XML file <a class="link" href="https://github.com/unicode-org/cldr/blob/main/common/supplemental/windowsZones.xml" target="_top">https://github.com/unicode-org/cldr/blob/main/common/supplemental/windowsZones.xml</a>
+   from the Unicode CLDR Project.
    </p></li></ul></div></div></div></div><div class="navfooter"><hr /><table width="100%" summary="Navigation footer"><tr><td width="40%" align="left"><a accesskey="p" href="source_design_notes.html">Prev</a> </td><td width="20%" align="center"><a accesskey="u" href="appendix.html">Up</a></td><td width="40%" align="right"> <a accesskey="n" href="documentation_hacking.html">Next</a></td></tr><tr><td width="40%" align="left" valign="top">Design Notes </td><td width="20%" align="center"><a accesskey="h" href="../index.html">Home</a></td><td width="40%" align="right" valign="top"> Writing and Generating Documentation</td></tr></table></div></body></html>
\ No newline at end of file
diff --git a/libstdc++-v3/doc/xml/manual/appendix_contributing.xml b/libstdc++-v3/doc/xml/manual/appendix_contributing.xml
index b924545..b2bc1d7 100644
--- a/libstdc++-v3/doc/xml/manual/appendix_contributing.xml
+++ b/libstdc++-v3/doc/xml/manual/appendix_contributing.xml
@@ -113,13 +113,11 @@
 
     <para>
       Please contact
-      Paolo Carlini at <email>paolo.carlini@oracle.com</email>
-      or
       Jonathan Wakely at <email>jwakely+assign@redhat.com</email>
       if you are confused about the assignment or have general licensing
       questions. When requesting an assignment form from
       <email>assign@gnu.org</email>, please CC the libstdc++
-      maintainers above so that progress can be monitored.
+      maintainer above so that progress can be monitored.
     </para>
   </section>
 
diff --git a/libstdc++-v3/doc/xml/manual/build_hacking.xml b/libstdc++-v3/doc/xml/manual/build_hacking.xml
index 077c063..20de49f 100644
--- a/libstdc++-v3/doc/xml/manual/build_hacking.xml
+++ b/libstdc++-v3/doc/xml/manual/build_hacking.xml
@@ -742,6 +742,17 @@ baseline file.
    this file is used.
    </para>
  </listitem>
+ <listitem>
+   <para>
+   The header file
+   <filename class="headerfile">src/c++20/windows_zones-map.h</filename>
+   is generated by the Python script
+   <filename>scripts/gen_windows_zones_map.py</filename>
+   using the XML file <link xmlns:xlink="http://www.w3.org/1999/xlink"
+     xlink:href="https://github.com/unicode-org/cldr/blob/main/common/supplemental/windowsZones.xml"/>
+   from the Unicode CLDR Project.
+   </para>
+ </listitem>
 </itemizedlist>
 
 </section> <!-- Generated files -->
diff --git a/libstdc++-v3/doc/xml/manual/evolution.xml b/libstdc++-v3/doc/xml/manual/evolution.xml
index 73b9f17..911db8a 100644
--- a/libstdc++-v3/doc/xml/manual/evolution.xml
+++ b/libstdc++-v3/doc/xml/manual/evolution.xml
@@ -1157,6 +1157,11 @@ Nested <code>result_type</code> and <code>argument_type</code> removed from
 Deprecated the non-standard overload of <code>std::fabs</code> for
 <code>std::complex</code> arguments.
 </para>
+
+<para>
+Made <code>std::is_integral_v&lt;__int128&gt;</code> true for strict
+<code>-std</code> dialects.
+</para>
 </section>
 
 </section>
diff --git a/libstdc++-v3/include/Makefile.am b/libstdc++-v3/include/Makefile.am
index cc402f06..6f248fe 100644
--- a/libstdc++-v3/include/Makefile.am
+++ b/libstdc++-v3/include/Makefile.am
@@ -77,6 +77,7 @@ std_headers = \
 	${std_srcdir}/forward_list \
 	${std_srcdir}/fstream \
 	${std_srcdir}/future \
+	${std_srcdir}/inplace_vector \
 	${std_srcdir}/iomanip \
 	${std_srcdir}/ios \
 	${std_srcdir}/iosfwd \
diff --git a/libstdc++-v3/include/Makefile.in b/libstdc++-v3/include/Makefile.in
index 0ef8564..014466f 100644
--- a/libstdc++-v3/include/Makefile.in
+++ b/libstdc++-v3/include/Makefile.in
@@ -433,6 +433,7 @@ std_freestanding = \
 @GLIBCXX_HOSTED_TRUE@	${std_srcdir}/forward_list \
 @GLIBCXX_HOSTED_TRUE@	${std_srcdir}/fstream \
 @GLIBCXX_HOSTED_TRUE@	${std_srcdir}/future \
+@GLIBCXX_HOSTED_TRUE@	${std_srcdir}/inplace_vector \
 @GLIBCXX_HOSTED_TRUE@	${std_srcdir}/iomanip \
 @GLIBCXX_HOSTED_TRUE@	${std_srcdir}/ios \
 @GLIBCXX_HOSTED_TRUE@	${std_srcdir}/iosfwd \
diff --git a/libstdc++-v3/include/bits/chrono_io.h b/libstdc++-v3/include/bits/chrono_io.h
index bcf9830..809d795 100644
--- a/libstdc++-v3/include/bits/chrono_io.h
+++ b/libstdc++-v3/include/bits/chrono_io.h
@@ -280,8 +280,9 @@ namespace __format
       // in the format-spec, e.g. "{:L%a}" is localized and locale-specific,
       // but "{:L}" is only localized and "{:%a}" is only locale-specific.
       unsigned _M_locale_specific : 1;
-      // Indicates that we are handling duration.
-      unsigned _M_time_only : 1;
+      // Indicates if parts that are checked for ok come directly from the
+      // input, instead of being computed.
+      unsigned _M_needs_ok_check : 1;
       // Indicates that duration should be treated as floating point.
       unsigned _M_floating_point_rep : 1;
       // Indicate that duration uses user-defined representation.
@@ -570,7 +571,15 @@ namespace __format
 
 	  _ChronoSpec<_CharT> __spec = __def;
 
-	  auto __finalize = [this, &__spec] {
+	  auto __finalize = [this, &__spec, &__def] {
+	    using enum _ChronoParts;
+            _ChronoParts __checked 
+	      = __spec._M_debug ? _YearMonthDay|_IndexedWeekday
+	                        : _Month|_Weekday;
+	    // n.b. for calendar types __def._M_needed contains only parts
+	    // copied from the input, remaining ones are computed, and thus ok
+	    __spec._M_needs_ok_check 
+	      = __spec._M_needs(__def._M_needed & __checked);
 	    _M_spec = __spec;
 	  };
 
@@ -693,8 +702,11 @@ namespace __format
 		  __allowed_mods = _Mod_O;
 		  break;
 		case 'j':
-		  __needed = __spec._M_time_only ? _HoursMinutesSeconds
-						 : _DayOfYear;
+		  __needed = __parts & _DayOfYear;
+		  // If we do not know day-of-year then we must have a duration,
+		  // which is to be formatted as decimal number of days.
+		  if (__needed == _None)
+		    __needed = _HoursMinutesSeconds;
 		  break;
 		case 'm':
 		  __needed = _Month;
@@ -818,10 +830,10 @@ namespace __format
 	    __throw_format_error("chrono format error: unescaped '%' in "
 				 "chrono-specs");
 
-	  _M_spec = __spec;
-	  _M_spec._M_chrono_specs
-		 = __string_view(__chrono_specs, __first - __chrono_specs);
+	  __spec._M_chrono_specs
+	    = __string_view(__chrono_specs, __first - __chrono_specs);
 
+	  __finalize();
 	  return __first;
 	}
 
@@ -878,16 +890,32 @@ namespace __format
       _S_empty_fs()
       { return _Runtime_format_string<_CharT>(_S_empty_spec); }
 
-      // Return the formatting locale.
-      template<typename _FormatContext>
-	std::locale
-	_M_locale(_FormatContext& __fc) const
-	{
-	  if (!_M_spec._M_localized)
-	    return std::locale::classic();
-	  else
-	    return __fc.locale();
-	}
+      static constexpr const _CharT* _S_weekdays[]
+      {
+	_GLIBCXX_WIDEN("Sunday"),
+	_GLIBCXX_WIDEN("Monday"),
+	_GLIBCXX_WIDEN("Tuesday"),
+	_GLIBCXX_WIDEN("Wednesday"),
+	_GLIBCXX_WIDEN("Thursday"),
+	_GLIBCXX_WIDEN("Friday"),
+	_GLIBCXX_WIDEN("Saturday"),
+      };
+
+      static constexpr const _CharT* _S_months[]
+      {
+	_GLIBCXX_WIDEN("January"),
+	_GLIBCXX_WIDEN("February"),
+	_GLIBCXX_WIDEN("March"),
+	_GLIBCXX_WIDEN("April"),
+	_GLIBCXX_WIDEN("May"),
+	_GLIBCXX_WIDEN("June"),
+	_GLIBCXX_WIDEN("July"),
+	_GLIBCXX_WIDEN("August"),
+	_GLIBCXX_WIDEN("September"),
+	_GLIBCXX_WIDEN("October"),
+	_GLIBCXX_WIDEN("November"),
+	_GLIBCXX_WIDEN("December"),
+      };
 
     private:
       template<typename _OutIter>
@@ -919,7 +947,13 @@ namespace __format
       {
 	switch (__conv)
 	  {
+	  case 'a':
+	  case 'A':
+	  case 'b':
+	  case 'B':
 	  case 'c':
+	  case 'h':
+	  case 'p':
 	  case 'r':
 	  case 'x':
 	  case 'X':
@@ -947,6 +981,73 @@ namespace __format
 	  return __out;
 	}
 
+      __string_view
+      _M_check_ok(const _ChronoData<_CharT>& __t, _CharT& __conv) const
+      {
+	if (!_M_spec._M_debug)
+	  {
+	    switch (__conv)
+	    {
+	    case 'a':
+	    case 'A':
+	      if (!__t._M_weekday.ok()) [[unlikely]]
+		__throw_format_error("format error: invalid weekday");
+	      break;
+	    case 'b':
+	    case 'h':
+	    case 'B':
+	      if (!__t._M_month.ok()) [[unlikely]]
+		__throw_format_error("format error: invalid month");
+	      break;
+	    default:
+	      break;
+	    }
+	    return __string_view();
+	  }
+
+	switch (__conv)
+	{
+	// %\0 is extension for handling weekday index
+	case '\0':
+	  if (__t._M_weekday_index < 1 || __t._M_weekday_index > 5) [[unlikely]]
+	    return _GLIBCXX_WIDEN("index");
+	  break;
+	case 'a':
+	case 'A':
+	  if (!__t._M_weekday.ok()) [[unlikely]]
+	    {
+	      __conv = 'w'; // print as decimal number
+	      return _GLIBCXX_WIDEN("weekday");
+	    }
+	  break;
+	case 'b':
+	case 'h':
+	case 'B':
+	  if (!__t._M_month.ok()) [[unlikely]]
+	    {
+	      __conv = 'm'; // print as decimal number
+	      return _GLIBCXX_WIDEN("month");
+	    }
+	  break;
+	case 'd':
+	case 'e':
+	  if (!__t._M_day.ok()) [[unlikely]]
+	    return _GLIBCXX_WIDEN("day");
+	  break;
+	case 'F':
+	  if (!(__t._M_year/__t._M_month/__t._M_day).ok()) [[unlikely]]
+	    return _GLIBCXX_WIDEN("date");
+	  break;
+	case 'Y':
+	  if (!__t._M_year.ok()) [[unlikely]]
+	    return _GLIBCXX_WIDEN("year");
+	  break;
+	default:
+	  break;
+	}
+	return __string_view();
+      }
+
       template<typename _OutIter, typename _FormatContext>
 	_OutIter
 	_M_format_to(const _ChronoData<_CharT>& __t, _OutIter __out,
@@ -1003,7 +1104,12 @@ namespace __format
 	  do
 	    {
 	      _CharT __c = *__first++;
-	      if (__use_locale_fmt && _S_localized_spec(__c, __mod)) [[unlikely]]
+	      __string_view __invalid;
+	      if (_M_spec._M_needs_ok_check)
+		__invalid = _M_check_ok(__t, __c);
+
+	      if (__invalid.empty() &&__use_locale_fmt
+		    && _S_localized_spec(__c, __mod)) [[unlikely]]
 		__out = _M_locale_fmt(std::move(__out), __fc.locale(),
 				      __tm, __c, __mod);
 	      else switch (__c)
@@ -1014,15 +1120,15 @@ namespace __format
 		  break;
 		case 'a':
 		case 'A':
-		  __out = _M_a_A(__t._M_weekday, std::move(__out), __fc, __c == 'A');
+		  __out = _M_a_A(__t._M_weekday, std::move(__out), __c == 'A');
 		  break;
 		case 'b':
 		case 'h':
 		case 'B':
-		  __out = _M_b_B(__t._M_month, std::move(__out), __fc, __c == 'B');
+		  __out = _M_b_B(__t._M_month, std::move(__out), __c == 'B');
 		  break;
 		case 'c':
-		  __out = _M_c(__t, std::move(__out), __fc);
+		  __out = _M_c(__t, std::move(__out));
 		  break;
 		case 'C':
 		case 'y':
@@ -1058,7 +1164,7 @@ namespace __format
 		  __out = _M_M(__t._M_minutes, __print_sign());
 		  break;
 		case 'p':
-		  __out = _M_p(__t._M_hours, std::move(__out), __fc);
+		  __out = _M_p(__t._M_hours, std::move(__out));
 		  break;
 		case 'q':
 		  __out = _M_q(__t._M_unit_suffix, std::move(__out));
@@ -1067,7 +1173,7 @@ namespace __format
 		  __out = _M_Q(__t, __print_sign(), __fc);
 		  break;
 		case 'r':
-		  __out = _M_r(__t, __print_sign(), __fc);
+		  __out = _M_r(__t, __print_sign());
 		  break;
 		case 'R':
 		case 'X':
@@ -1111,6 +1217,14 @@ namespace __format
 		  __first = __last;
 		  break;
 		}
+
+	      if (!__invalid.empty())
+		{
+		  constexpr __string_view __pref = _GLIBCXX_WIDEN(" is not a valid ");
+		  __out = __format::__write(std::move(__out), __pref);
+		  __out = __format::__write(std::move(__out), __invalid);
+		}
+
 	      __mod = _CharT();
 	      // Scan for next '%' and write out everything before it.
 	      __string_view __str(__first, __last - __first);
@@ -1140,83 +1254,43 @@ namespace __format
 	  // %\0 Extension to format weekday index, used only by empty format spec
 	  _CharT __buf[3];
 	  __out = __format::__write(std::move(__out), _S_str_d1(__buf, __wi));
-	  if (_M_spec._M_debug && (__wi < 1 || __wi > 5))
-	    __out = __format::__write(std::move(__out),
-		      __string_view(_GLIBCXX_WIDEN(" is not a valid index")));
 	  return std::move(__out);
 	}
 
-      template<typename _OutIter, typename _FormatContext>
+      template<typename _OutIter>
 	_OutIter
-	_M_a_A(chrono::weekday __wd, _OutIter __out,
-	       _FormatContext& __ctx, bool __full) const
+	_M_a_A(chrono::weekday __wd, _OutIter __out, bool __full) const
 	{
 	  // %a Locale's abbreviated weekday name.
 	  // %A Locale's full weekday name.
-	  if (!__wd.ok())
-	    {
-	      if (!_M_spec._M_debug)
-		__throw_format_error("format error: invalid weekday");
-
-	      _CharT __buf[3];
-	      __out = __format::__write(std::move(__out),
-					_S_str_d1(__buf, __wd.c_encoding()));
-	      return __format::__write(std::move(__out),
-			__string_view(_GLIBCXX_WIDEN(" is not a valid weekday")));
-	    }
-
-	  locale __loc = _M_locale(__ctx);
-	  const auto& __tp = use_facet<__timepunct<_CharT>>(__loc);
-	  const _CharT* __days[7];
-	  if (__full)
-	    __tp._M_days(__days);
-	  else
-	    __tp._M_days_abbreviated(__days);
-	  __string_view __str(__days[__wd.c_encoding()]);
-	  return _M_write(std::move(__out), __loc, __str);
+	  __string_view __str = _S_weekdays[__wd.c_encoding()];
+	  if (!__full)
+	    __str = __str.substr(0, 3);
+	  return __format::__write(std::move(__out), __str);
 	}
 
-      template<typename _OutIter, typename _FormatContext>
+      template<typename _OutIter>
 	_OutIter
-	_M_b_B(chrono::month __m, _OutIter __out,
-	       _FormatContext& __ctx, bool __full) const
+	_M_b_B(chrono::month __m, _OutIter __out, bool __full) const
 	{
 	  // %b Locale's abbreviated month name.
 	  // %B Locale's full month name.
-	  if (!__m.ok())
-	    {
-	      if (!_M_spec._M_debug)
-		__throw_format_error("format error: invalid month");
-
-	      _CharT __buf[3];
-	      __out = __format::__write(std::move(__out),
-					_S_str_d1(__buf, (unsigned)__m));
-	      return __format::__write(std::move(__out),
-			__string_view(_GLIBCXX_WIDEN(" is not a valid month")));
-	    }
-
-	  locale __loc = _M_locale(__ctx);
-	  const auto& __tp = use_facet<__timepunct<_CharT>>(__loc);
-	  const _CharT* __months[12];
-	  if (__full)
-	    __tp._M_months(__months);
-	  else
-	    __tp._M_months_abbreviated(__months);
-	  __string_view __str(__months[(unsigned)__m - 1]);
-	  return _M_write(std::move(__out), __loc, __str);
+	  __string_view __str = _S_months[(unsigned)__m - 1];
+	  if (!__full)
+	    __str = __str.substr(0, 3);
+	  return __format::__write(std::move(__out), __str);
 	}
 
-      template<typename _OutIter, typename _FormatContext>
+      template<typename _OutIter>
 	_OutIter
-	_M_c(const _ChronoData<_CharT>& __t, _OutIter __out,
-	     _FormatContext& __ctx) const
+	_M_c(const _ChronoData<_CharT>& __t, _OutIter __out) const
 	{
 	  // %c  Locale's date and time representation, for C-locale: %a %b %e %T %Y
 	  // %Ec Locale's alternate date and time representation, for C-locale same as above
 
-	  __out = _M_a_A(__t._M_weekday, std::move(__out), __ctx, false);
+	  __out = _M_a_A(__t._M_weekday, std::move(__out), false);
 	  *__out = _S_space;
-	  __out = _M_b_B(__t._M_month, std::move(++__out), __ctx, false);
+	  __out = _M_b_B(__t._M_month, std::move(++__out), false);
 	  *__out = _S_space;
 	  __out = _M_d_e(__t._M_day, std::move(++__out), 'e');
 	  *__out = _S_space;
@@ -1269,10 +1343,6 @@ namespace __format
 		}
 	      __out = __format::__write(std::move(__out), __sv);
 	    }
-
-	  if (_M_spec._M_debug && __conv == 'Y' && !__y.ok()) [[unlikely]]
-	    __out = __format::__write(std::move(__out),
-		      __string_view(_GLIBCXX_WIDEN(" is not a valid year")));
 	  return __out;
 	}
 
@@ -1331,9 +1401,6 @@ namespace __format
 	    }
 
 	  __out = __format::__write(std::move(__out), __sv);
-	  if (_M_spec._M_debug && !__d.ok()) [[unlikely]]
-	    __out = __format::__write(std::move(__out),
-		      __string_view(_GLIBCXX_WIDEN(" is not a valid day")));
 	  return std::move(__out);
 	}
 
@@ -1370,9 +1437,6 @@ namespace __format
 	      __out = __format::__write(std::move(__out), __sv);
 	    }
 
-	  if (_M_spec._M_debug && !(__t._M_year/__t._M_month/__t._M_day).ok())
-	    __out = __format::__write(std::move(__out),
-		      __string_view(_GLIBCXX_WIDEN(" is not a valid date")));
 	  return std::move(__out);
 	}
 
@@ -1419,7 +1483,7 @@ namespace __format
 	_OutIter
 	_M_j(const _ChronoData<_CharT>& __t, _OutIter __out) const
 	{
-	  if (_M_spec._M_time_only)
+	  if (!_M_spec._M_needs(_ChronoParts::_DayOfYear))
 	  {
 	    // Decimal number of days, without padding.
 	    auto __d = chrono::floor<chrono::days>(__t._M_hours).count();
@@ -1440,8 +1504,10 @@ namespace __format
 	{
 	  // %m  month as a decimal number.
 	  // %Om Locale's alternative representation.
-
 	  auto __i = (unsigned)__m;
+	  if (__i == 0 && _M_spec._M_debug) [[unlikely]]
+	    // 0 should not be padded to two digits
+	    return __format::__write(std::move(__out), _S_digit(0));
 
 	  _CharT __buf[3];
 	  return __format::__write(std::move(__out), _S_str_d2(__buf, __i));
@@ -1458,20 +1524,15 @@ namespace __format
 	  return __format::__write(std::move(__out), _S_two_digits(__i));
 	}
 
-      template<typename _OutIter, typename _FormatContext>
+      template<typename _OutIter>
 	_OutIter
-	_M_p(chrono::hours __h, _OutIter __out, _FormatContext& __ctx) const
+	_M_p(chrono::hours __h, _OutIter __out) const
 	{
 	  // %p The locale's equivalent of the AM/PM designations.
-	  auto __hi =  __h.count();
-	  if (__hi >= 24) [[unlikely]]
-	    __hi %= 24;
-
-	  locale __loc = _M_locale(__ctx);
-	  const auto& __tp = use_facet<__timepunct<_CharT>>(__loc);
-	  const _CharT* __ampm[2];
-	  __tp._M_am_pm(__ampm);
-	  return _M_write(std::move(__out), __loc, __ampm[__hi >= 12]);
+
+	  _CharT __buf[2];
+	  _S_fill_ampm(__buf, __h);
+	  return __format::__write(std::move(__out), __string_view(__buf, 2));
 	}
 
       template<typename _OutIter>
@@ -1491,27 +1552,25 @@ namespace __format
 	  return std::vformat_to(std::move(__out), _S_empty_spec, __t._M_ereps);
 	}
 
-      template<typename _OutIter, typename _FormatContext>
+      template<typename _OutIter>
 	_OutIter
-	_M_r(const _ChronoData<_CharT>& __t, _OutIter __out,
-	     _FormatContext& __ctx) const
+	_M_r(const _ChronoData<_CharT>& __t, _OutIter __out) const
 	{
 	  // %r Locale's 12-hour clock time, for C-locale: %I:%M:%S %p
 	  auto __hi = __t._M_hours.count() % 12;
 	  if (__hi == 0)
 	    __hi = 12;
 
-	  _CharT __buf[9];
+	  _CharT __buf[11];
 	  __buf[2] = _S_colon;
 	  __buf[5] = _S_colon;
 	  __buf[8] = _S_space;
 	  _S_fill_two_digits(__buf, __hi);
 	  _S_fill_two_digits(__buf + 3, __t._M_minutes.count());
 	  _S_fill_two_digits(__buf + 6, __t._M_seconds.count());
+	  _S_fill_ampm(__buf + 9, __t._M_hours);
 
-	  __string_view __sv(__buf, 9);
-	  __out = __format::__write(std::move(__out), __sv);
-	  return _M_p(__t._M_hours, std::move(__out), __ctx);
+	  return __format::__write(std::move(__out), __string_view(__buf, 11));
 	}
 
       template<typename _OutIter>
@@ -1575,9 +1634,9 @@ namespace __format
 	    return __out;
 
 	  _CharT __dot = _S_dot;
-	  if (_M_spec._M_localized)
-	    if (auto __loc = __ctx.locale(); __loc != locale::classic())
+	  if (_M_spec._M_localized) [[unlikely]]
 	    {
+	      auto __loc = __ctx.locale();
 	      const auto& __np = use_facet<numpunct<_CharT>>(__loc);
 	      __dot = __np.decimal_point();
 	    }
@@ -1587,8 +1646,8 @@ namespace __format
 	  if (_M_spec._M_floating_point_rep)
 	    {
 	      _Str_sink<_CharT> __sink;
-	      if (_M_spec._M_localized && _M_spec._M_custom_rep)
-		 std::vformat_to(__sink.out(), _M_locale(__ctx),
+	      if (_M_spec._M_localized && _M_spec._M_custom_rep) [[unlikely]]
+		 std::vformat_to(__sink.out(), __ctx.locale(),
 				 _GLIBCXX_WIDEN("{1:0.{2}Lf}"), __t._M_ereps);
 	      else
 		 std::vformat_to(__sink.out(),
@@ -1747,6 +1806,20 @@ namespace __format
 	__buf[1] = __sv[1];
       }
 
+      // Fills __buf[0] and __buf[1] with "AM", "PM" depending on __h.
+      [[__gnu__::__always_inline__]]
+      static void
+      _S_fill_ampm(_CharT* __buf, chrono::hours __h)
+      {
+	auto __hi =  __h.count();
+	if (__hi >= 24) [[unlikely]]
+	  __hi %= 24;
+
+	constexpr const _CharT* __apm = _GLIBCXX_WIDEN("APM");
+	__buf[0] = __apm[__hi >= 12];
+	__buf[1] = __apm[2];
+      }
+
       // Returns decimal representation of __n.
       // Returned string_view may point to __buf.
       [[__gnu__::__always_inline__]]
@@ -1766,7 +1839,7 @@ namespace __format
       {
 	if (__n < 100) [[likely]]
 	  return _S_two_digits(__n);
-        return _S_str_d3(__buf, __n);
+	return _S_str_d3(__buf, __n);
       }
 
       // Returns decimal representation of __n, padded to 3 digits.
@@ -1811,7 +1884,6 @@ namespace __format
 	  using enum _ChronoParts;
 
 	  _ChronoSpec<_CharT> __res{};
-	  __res._M_time_only = (__parts & _Date) == 0;
 	  __res._M_floating_point_rep = chrono::treat_as_floating_point_v<_Rep>;
 	  __res._M_custom_rep = !is_arithmetic_v<_Rep>;
 	  __res._M_prec = chrono::hh_mm_ss<_Duration>::fractional_width;
@@ -1841,12 +1913,13 @@ namespace __format
 	      break;
 	    case _None:
 	      break;
+	    default:
+	      __builtin_unreachable();
 	    }
 	  return __res;
 	};
 
       using __formatter_chrono<_CharT>::__formatter_chrono;
-      using __formatter_chrono<_CharT>::_M_locale;
       using __formatter_chrono<_CharT>::_M_spec;
 
       template<typename _Duration, typename _ParseContext>
@@ -1859,6 +1932,10 @@ namespace __format
 
 	  auto __res
 	    = __formatter_chrono<_CharT>::_M_parse(__pc, __parts, __def);
+	  // n.b. durations do not contain date parts, and for time point all
+	  // date parts are computed, and they are always ok.
+	  _M_spec._M_needs_ok_check = false;
+
 	  // check for custom floating point durations, if digits of output
 	  // will contain subseconds, then formatters must support specifying
 	  // precision.
@@ -1876,6 +1953,17 @@ namespace __format
 	  return __res;
 	}
 
+      // Return the formatting locale.
+      template<typename _FormatContext>
+	std::locale
+	_M_locale(_FormatContext& __fc) const
+	{
+	  if (!_M_spec._M_localized)
+	    return std::locale::classic();
+	  else
+	    return __fc.locale();
+	}
+
       // Format duration for empty chrono-specs, e.g. "{}" (C++20 [time.format] p6).
       template<typename _Rep, typename _Period, typename _FormatContext>
 	typename _FormatContext::iterator
diff --git a/libstdc++-v3/include/bits/cpp_type_traits.h b/libstdc++-v3/include/bits/cpp_type_traits.h
index b1a6206..38cea4c 100644
--- a/libstdc++-v3/include/bits/cpp_type_traits.h
+++ b/libstdc++-v3/include/bits/cpp_type_traits.h
@@ -273,6 +273,12 @@ __INT_N(__GLIBCXX_TYPE_INT_N_2)
 __INT_N(__GLIBCXX_TYPE_INT_N_3)
 #endif
 
+#if defined __STRICT_ANSI__ && defined __SIZEOF_INT128__
+// In strict modes __GLIBCXX_TYPE_INT_N_0 is not defined for __int128,
+// but we want to always treat signed/unsigned __int128 as integral types.
+__INT_N(__int128)
+#endif
+
 #undef __INT_N
 
   //
@@ -307,6 +313,15 @@ __INT_N(__GLIBCXX_TYPE_INT_N_3)
       typedef __true_type __type;
     };
 
+#ifdef _GLIBCXX_USE_FLOAT128
+  template<>
+    struct __is_floating<__float128>
+    {
+      enum { __value = 1 };
+      typedef __true_type __type;
+    };
+#endif
+
 #ifdef __STDCPP_FLOAT16_T__
   template<>
     struct __is_floating<_Float16>
@@ -545,17 +560,6 @@ __INT_N(__GLIBCXX_TYPE_INT_N_3)
     { enum { __width = __GLIBCXX_BITSIZE_INT_N_3 }; };
 #endif
 
-#if defined __STRICT_ANSI__ && defined __SIZEOF_INT128__
-  // In strict modes __is_integer<__int128> is false,
-  // but we want to allow memcpy between signed/unsigned __int128.
-  __extension__
-  template<>
-    struct __memcpyable_integer<__int128> { enum { __width = 128 }; };
-  __extension__
-  template<>
-    struct __memcpyable_integer<unsigned __int128> { enum { __width = 128 }; };
-#endif
-
 #if _GLIBCXX_DOUBLE_IS_IEEE_BINARY64 && _GLIBCXX_LDOUBLE_IS_IEEE_BINARY64
   template<>
     struct __memcpyable<double*, long double*> { enum { __value = true }; };
diff --git a/libstdc++-v3/include/bits/functional_hash.h b/libstdc++-v3/include/bits/functional_hash.h
index e84c9ee..8456089 100644
--- a/libstdc++-v3/include/bits/functional_hash.h
+++ b/libstdc++-v3/include/bits/functional_hash.h
@@ -199,6 +199,15 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   _Cxx_hashtable_define_trivial_hash(__GLIBCXX_TYPE_INT_N_3 unsigned)
 #endif
 
+#if defined __STRICT_ANSI__ && defined __SIZEOF_INT128__
+  // In strict modes __GLIBCXX_TYPE_INT_N_0 is not defined for __int128,
+  // but we want to always treat signed/unsigned __int128 as integral types.
+  __extension__
+  _Cxx_hashtable_define_trivial_hash(__int128)
+  __extension__
+  _Cxx_hashtable_define_trivial_hash(__int128 unsigned)
+#endif
+
 #undef _Cxx_hashtable_define_trivial_hash
 
   struct _Hash_impl
diff --git a/libstdc++-v3/include/bits/iterator_concepts.h b/libstdc++-v3/include/bits/iterator_concepts.h
index d31e4f1..979039e 100644
--- a/libstdc++-v3/include/bits/iterator_concepts.h
+++ b/libstdc++-v3/include/bits/iterator_concepts.h
@@ -214,17 +214,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	= make_signed_t<decltype(std::declval<_Tp>() - std::declval<_Tp>())>;
     };
 
-#if defined __STRICT_ANSI__ && defined __SIZEOF_INT128__
-  // __int128 is incrementable even if !integral<__int128>
-  template<>
-    struct incrementable_traits<__int128>
-    { using difference_type = __int128; };
-
-  template<>
-    struct incrementable_traits<unsigned __int128>
-    { using difference_type = __int128; };
-#endif
-
   namespace __detail
   {
     // An iterator such that iterator_traits<_Iter> names a specialization
@@ -611,24 +600,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     class __max_diff_type;
     class __max_size_type;
 
-    __extension__
-    template<typename _Tp>
-      concept __is_signed_int128
-#if __SIZEOF_INT128__
-	= same_as<_Tp, __int128>;
-#else
-	= false;
-#endif
-
-    __extension__
-    template<typename _Tp>
-      concept __is_unsigned_int128
-#if __SIZEOF_INT128__
-	= same_as<_Tp, unsigned __int128>;
-#else
-	= false;
-#endif
-
     template<typename _Tp>
       concept __cv_bool = same_as<const volatile _Tp, const volatile bool>;
 
@@ -636,16 +607,11 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       concept __integral_nonbool = integral<_Tp> && !__cv_bool<_Tp>;
 
     template<typename _Tp>
-      concept __is_int128 = __is_signed_int128<_Tp> || __is_unsigned_int128<_Tp>;
-
-    template<typename _Tp>
       concept __is_integer_like = __integral_nonbool<_Tp>
-	|| __is_int128<_Tp>
 	|| same_as<_Tp, __max_diff_type> || same_as<_Tp, __max_size_type>;
 
     template<typename _Tp>
       concept __is_signed_integer_like = signed_integral<_Tp>
-	|| __is_signed_int128<_Tp>
 	|| same_as<_Tp, __max_diff_type>;
 
   } // namespace ranges::__detail
diff --git a/libstdc++-v3/include/bits/max_size_type.h b/libstdc++-v3/include/bits/max_size_type.h
index 73a6d14..a34b91a 100644
--- a/libstdc++-v3/include/bits/max_size_type.h
+++ b/libstdc++-v3/include/bits/max_size_type.h
@@ -38,6 +38,7 @@
 #include <ext/numeric_traits.h>
 #include <bit> // __bit_width
 #include <numbers>
+#include <limits> // __glibcxx_integral_traps
 
 // This header implements unsigned and signed integer-class types (as per
 // [iterator.concept.winc]) that are one bit wider than the widest supported
@@ -64,7 +65,7 @@ namespace ranges
     public:
       __max_size_type() = default;
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	constexpr
 	__max_size_type(_Tp __i) noexcept
 	  : _M_val(__i), _M_msb(__i < 0)
@@ -73,7 +74,7 @@ namespace ranges
       constexpr explicit
       __max_size_type(const __max_diff_type& __d) noexcept;
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	constexpr explicit
 	operator _Tp() const noexcept
 	{ return _M_val; }
@@ -259,52 +260,52 @@ namespace ranges
 	return *this;
       }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator+=(_Tp& __a, const __max_size_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a + __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator-=(_Tp& __a, const __max_size_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a - __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator*=(_Tp& __a, const __max_size_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a * __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator/=(_Tp& __a, const __max_size_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a / __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator%=(_Tp& __a, const __max_size_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a % __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator&=(_Tp& __a, const __max_size_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a & __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator|=(_Tp& __a, const __max_size_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a | __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator^=(_Tp& __a, const __max_size_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a ^ __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator<<=(_Tp& __a, const __max_size_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a << __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator>>=(_Tp& __a, const __max_size_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a >> __b)); }
@@ -446,7 +447,7 @@ namespace ranges
     public:
       __max_diff_type() = default;
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	constexpr
 	__max_diff_type(_Tp __i) noexcept
 	  : _M_rep(__i)
@@ -457,7 +458,7 @@ namespace ranges
 	: _M_rep(__d)
       { }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	constexpr explicit
 	operator _Tp() const noexcept
 	{ return static_cast<_Tp>(_M_rep); }
@@ -590,52 +591,52 @@ namespace ranges
 	return *this;
       }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator+=(_Tp& __a, const __max_diff_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a + __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator-=(_Tp& __a, const __max_diff_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a - __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator*=(_Tp& __a, const __max_diff_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a * __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator/=(_Tp& __a, const __max_diff_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a / __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator%=(_Tp& __a, const __max_diff_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a % __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator&=(_Tp& __a, const __max_diff_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a & __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator|=(_Tp& __a, const __max_diff_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a | __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator^=(_Tp& __a, const __max_diff_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a ^ __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator<<=(_Tp& __a, const __max_diff_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a << __b)); }
 
-      template<typename _Tp> requires integral<_Tp> || __is_int128<_Tp>
+      template<typename _Tp> requires integral<_Tp>
 	friend constexpr _Tp&
 	operator>>=(_Tp& __a, const __max_diff_type& __b) noexcept
 	{ return (__a = static_cast<_Tp>(__a >> __b)); }
@@ -775,10 +776,27 @@ namespace ranges
       static constexpr bool is_signed = false;
       static constexpr bool is_integer = true;
       static constexpr bool is_exact = true;
+      static constexpr bool is_bounded = true;
+      static constexpr bool is_modulo = true;
+      static constexpr bool traps = __glibcxx_integral_traps;
+      static constexpr int radix = 2;
       static constexpr int digits
 	= __gnu_cxx::__int_traits<_Sp::__rep>::__digits + 1;
       static constexpr int digits10
 	= static_cast<int>(digits * numbers::ln2 / numbers::ln10);
+      static constexpr int max_digits10 = 0;
+      static constexpr int min_exponent = 0;
+      static constexpr int min_exponent10 = 0;
+      static constexpr int max_exponent = 0;
+      static constexpr int max_exponent10 = 0;
+      static constexpr bool is_iec559 = false;
+      static constexpr bool has_infinity = false;
+      static constexpr bool has_quiet_NaN = false;
+      static constexpr bool has_signaling_NaN = false;
+      static constexpr bool has_denorm_loss = false;
+      static constexpr bool tinyness_before = false;
+      static constexpr float_denorm_style has_denorm = denorm_absent;
+      static constexpr float_round_style round_style = round_toward_zero;
 
       static constexpr _Sp
       min() noexcept
@@ -791,6 +809,30 @@ namespace ranges
       static constexpr _Sp
       lowest() noexcept
       { return min(); }
+
+      static constexpr _Sp
+      denorm_min() noexcept
+      { return 0; }
+
+      static constexpr _Sp
+      epsilon() noexcept
+      { return 0; }
+
+      static constexpr _Sp
+      round_error() noexcept
+      { return 0; }
+
+      static constexpr _Sp
+      infinity() noexcept
+      { return 0; }
+
+      static constexpr _Sp
+      quiet_NaN() noexcept
+      { return 0; }
+
+      static constexpr _Sp
+      signaling_NaN() noexcept
+      { return 0; }
     };
 
   template<>
@@ -802,9 +844,26 @@ namespace ranges
       static constexpr bool is_signed = true;
       static constexpr bool is_integer = true;
       static constexpr bool is_exact = true;
+      static constexpr bool is_bounded = true;
+      static constexpr bool is_modulo = false;
+      static constexpr bool traps = __glibcxx_integral_traps;
+      static constexpr int radix = 2;
       static constexpr int digits = numeric_limits<_Sp>::digits - 1;
       static constexpr int digits10
 	= static_cast<int>(digits * numbers::ln2 / numbers::ln10);
+      static constexpr int max_digits10 = 0;
+      static constexpr int min_exponent = 0;
+      static constexpr int min_exponent10 = 0;
+      static constexpr int max_exponent = 0;
+      static constexpr int max_exponent10 = 0;
+      static constexpr bool is_iec559 = false;
+      static constexpr bool has_infinity = false;
+      static constexpr bool has_quiet_NaN = false;
+      static constexpr bool has_signaling_NaN = false;
+      static constexpr bool has_denorm_loss = false;
+      static constexpr bool tinyness_before = false;
+      static constexpr float_denorm_style has_denorm = denorm_absent;
+      static constexpr float_round_style round_style = round_toward_zero;
 
       static constexpr _Dp
       min() noexcept
@@ -817,6 +876,30 @@ namespace ranges
       static constexpr _Dp
       lowest() noexcept
       { return min(); }
+
+      static constexpr _Dp
+      denorm_min() noexcept
+      { return 0; }
+
+      static constexpr _Dp
+      epsilon() noexcept
+      { return 0; }
+
+      static constexpr _Dp
+      round_error() noexcept
+      { return 0; }
+
+      static constexpr _Dp
+      infinity() noexcept
+      { return 0; }
+
+      static constexpr _Dp
+      quiet_NaN() noexcept
+      { return 0; }
+
+      static constexpr _Dp
+      signaling_NaN() noexcept
+      { return 0; }
     };
 
   template<>
diff --git a/libstdc++-v3/include/bits/move.h b/libstdc++-v3/include/bits/move.h
index 085ca07..061e6b4 100644
--- a/libstdc++-v3/include/bits/move.h
+++ b/libstdc++-v3/include/bits/move.h
@@ -215,14 +215,18 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
    *  @return   Nothing.
   */
   template<typename _Tp>
-    _GLIBCXX20_CONSTEXPR
-    inline
-#if __cplusplus >= 201103L
-    typename enable_if<__and_<__not_<__is_tuple_like<_Tp>>,
-			      is_move_constructible<_Tp>,
-			      is_move_assignable<_Tp>>::value>::type
+#if __glibcxx_concepts // >= C++20
+    requires (! __is_tuple_like<_Tp>::value)
+      && is_move_constructible_v<_Tp>
+      && is_move_assignable_v<_Tp>
+    constexpr void
+#elif __cplusplus >= 201103L
+    _GLIBCXX20_CONSTEXPR inline
+    __enable_if_t<__and_<__not_<__is_tuple_like<_Tp>>,
+			 is_move_constructible<_Tp>,
+			 is_move_assignable<_Tp>>::value>
 #else
-    void
+    inline void
 #endif
     swap(_Tp& __a, _Tp& __b)
     _GLIBCXX_NOEXCEPT_IF(__and_<is_nothrow_move_constructible<_Tp>,
@@ -241,12 +245,14 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   // DR 809. std::swap should be overloaded for array types.
   /// Swap the contents of two arrays.
   template<typename _Tp, size_t _Nm>
-    _GLIBCXX20_CONSTEXPR
-    inline
-#if __cplusplus >= 201103L
-    typename enable_if<__is_swappable<_Tp>::value>::type
+#if __glibcxx_concepts // >= C++20
+    requires is_swappable_v<_Tp>
+    constexpr void
+#elif __cplusplus >= 201103L
+    _GLIBCXX20_CONSTEXPR inline
+    __enable_if_t<__is_swappable<_Tp>::value>
 #else
-    void
+    inline void
 #endif
     swap(_Tp (&__a)[_Nm], _Tp (&__b)[_Nm])
     _GLIBCXX_NOEXCEPT_IF(__is_nothrow_swappable<_Tp>::value)
diff --git a/libstdc++-v3/include/bits/ranges_algo.h b/libstdc++-v3/include/bits/ranges_algo.h
index cf369c5..9f8945a 100644
--- a/libstdc++-v3/include/bits/ranges_algo.h
+++ b/libstdc++-v3/include/bits/ranges_algo.h
@@ -5218,6 +5218,7 @@ namespace ranges
 #endif // __glibcxx_ranges_fold
 } // namespace ranges
 
+#if __glibcxx_shift >= 201806L // C++ >= 20
   template<typename _ForwardIterator>
     constexpr _ForwardIterator
     shift_left(_ForwardIterator __first, _ForwardIterator __last,
@@ -5308,6 +5309,120 @@ namespace ranges
 	    }
 	}
     }
+#endif
+
+namespace ranges
+{
+#if __glibcxx_shift >= 202202L // C++ >= 23
+  struct __shift_left_fn
+  {
+    template<permutable _Iter, sentinel_for<_Iter> _Sent>
+      constexpr subrange<_Iter>
+      operator()(_Iter __first, _Sent __last, iter_difference_t<_Iter> __n) const
+      {
+	__glibcxx_assert(__n >= 0);
+	if (__n == 0)
+	  return {__first, ranges::next(__first, __last)};
+
+	auto __mid = ranges::next(__first, __n, __last);
+	if (__mid == __last)
+	  return {__first, __first};
+	return {__first, ranges::move(__mid, __last, __first).out};
+      }
+
+    template<forward_range _Range>
+      requires permutable<iterator_t<_Range>>
+      constexpr borrowed_subrange_t<_Range>
+      operator()(_Range&& __r, range_difference_t<_Range> __n) const
+      { return (*this)(ranges::begin(__r), ranges::end(__r), __n); }
+  };
+
+  inline constexpr __shift_left_fn shift_left{};
+
+  struct __shift_right_fn
+  {
+    template<permutable _Iter, sentinel_for<_Iter> _Sent>
+      constexpr subrange<_Iter>
+      operator()(_Iter __first, _Sent __last, iter_difference_t<_Iter> __n) const
+      {
+	__glibcxx_assert(__n >= 0);
+	if (__n == 0)
+	  return {__first, ranges::next(__first, __last)};
+
+	if constexpr (bidirectional_iterator<_Iter> && same_as<_Iter, _Sent>)
+	  {
+	    auto __mid = ranges::next(__last, -__n, __first);
+	    if (__mid == __first)
+	      return {__last, __last};
+
+	    return {ranges::move_backward(__first, __mid, __last).out, __last};
+	  }
+	else
+	  {
+	    auto __result = ranges::next(__first, __n, __last);
+	    if (__result == __last)
+	      return {__result, __result};
+
+	    auto __dest_head = __first, __dest_tail = __result;
+	    while (__dest_head != __result)
+	      {
+		if (__dest_tail == __last)
+		  {
+		    // If we get here, then we must have
+		    //     2*n >= distance(__first, __last)
+		    // i.e. we are shifting out at least half of the range.  In
+		    // this case we can safely perform the shift with a single
+		    // move.
+		    auto __lasti = ranges::move(__first, __dest_head, __result).out;
+		    // __glibcxx_assert(__lasti == __last);
+		    return {__result, __lasti};
+		  }
+		++__dest_head;
+		++__dest_tail;
+	      }
+
+	    for (;;)
+	      {
+		// At the start of each iteration of this outer loop, the range
+		// [__first, __result) contains those elements that after shifting
+		// the whole range right by __n, should end up in
+		// [__dest_head, __dest_tail) in order.
+
+		// The below inner loop swaps the elements of [__first, __result)
+		// and [__dest_head, __dest_tail), while simultaneously shifting
+		// the latter range by __n.
+		auto __cursor = __first;
+		while (__cursor != __result)
+		  {
+		    if (__dest_tail == __last)
+		      {
+			// At this point the ranges [__first, result) and
+			// [__dest_head, dest_tail) are disjoint, so we can safely
+			// move the remaining elements.
+			__dest_head = ranges::move(__cursor, __result, __dest_head).out;
+			auto __lasti = ranges::move(__first, __cursor, __dest_head).out;
+			// __glibcxx_assert(__lasti == __last);
+			return {__result, __lasti};
+		      }
+		    ranges::iter_swap(__cursor, __dest_head);
+		    ++__dest_head;
+		    ++__dest_tail;
+		    ++__cursor;
+		  }
+	      }
+	  }
+      }
+
+    template<forward_range _Range>
+      requires permutable<iterator_t<_Range>>
+      constexpr borrowed_subrange_t<_Range>
+      operator()(_Range&& __r, range_difference_t<_Range> __n) const
+      { return (*this)(ranges::begin(__r), ranges::end(__r), __n); }
+  };
+
+  inline constexpr __shift_right_fn shift_right{};
+#endif // C++23
+} // namespace ranges
 
 _GLIBCXX_END_NAMESPACE_VERSION
 } // namespace std
diff --git a/libstdc++-v3/include/bits/ranges_base.h b/libstdc++-v3/include/bits/ranges_base.h
index c09f729..7df638d 100644
--- a/libstdc++-v3/include/bits/ranges_base.h
+++ b/libstdc++-v3/include/bits/ranges_base.h
@@ -81,16 +81,6 @@ namespace ranges
       __to_unsigned_like(_Tp __t) noexcept
       { return static_cast<make_unsigned_t<_Tp>>(__t); }
 
-#if defined __STRICT_ANSI__ && defined __SIZEOF_INT128__
-    constexpr unsigned __int128
-    __to_unsigned_like(__int128 __t) noexcept
-    { return __t; }
-
-    constexpr unsigned __int128
-    __to_unsigned_like(unsigned __int128 __t) noexcept
-    { return __t; }
-#endif
-
     template<typename _Tp>
       using __make_unsigned_like_t
 	= decltype(__detail::__to_unsigned_like(std::declval<_Tp>()));
@@ -398,11 +388,6 @@ namespace ranges
 	      else
 		return static_cast<make_signed_t<__size_type>>(__size);
 	    }
-#if defined __STRICT_ANSI__ && defined __SIZEOF_INT128__
-	  // For strict-ansi modes integral<__int128> is false
-	  else if constexpr (__detail::__is_int128<__size_type>)
-	    return static_cast<__int128>(__size);
-#endif
 	  else // Must be one of __max_diff_type or __max_size_type.
 	    return __detail::__max_diff_type(__size);
 	}
@@ -894,10 +879,17 @@ namespace ranges
       {
 	if constexpr (sized_sentinel_for<_Sent, _It>)
 	  {
-	    const auto __diff = __bound - __it;
+	    const iter_difference_t<_It> __diff = __bound - __it;
 
 	    if (__diff == 0)
-	      return __n;
+	      {
+		// inline any possible side effects of advance(it, bound)
+		if constexpr (assignable_from<_It&, _Sent>)
+		  __it = std::move(__bound);
+		else if constexpr (random_access_iterator<_It>)
+		  __it += 0;
+		return __n;
+	      }
 	    else if (__diff > 0 ? __n >= __diff : __n <= __diff)
 	      {
 		(*this)(__it, __bound);
@@ -912,9 +904,14 @@ namespace ranges
 		return 0;
 	      }
 	    else
-	      return 0;
+	      {
+		// inline any possible side effects of advance(it, n)
+		if constexpr (random_access_iterator<_It>)
+		  __it += 0;
+		return 0;
+	      }
 	  }
-	else if (__it == __bound || __n == 0)
+	else if (__n == 0 || __it == __bound)
 	  return __n;
 	else if (__n > 0)
 	  {
diff --git a/libstdc++-v3/include/bits/ranges_uninitialized.h b/libstdc++-v3/include/bits/ranges_uninitialized.h
index 12a714b..3f9a07f 100644
--- a/libstdc++-v3/include/bits/ranges_uninitialized.h
+++ b/libstdc++-v3/include/bits/ranges_uninitialized.h
@@ -556,13 +556,12 @@ namespace ranges
     __destroy_fn::operator()(_Iter __first, _Sent __last) const noexcept
     {
       if constexpr (is_trivially_destructible_v<iter_value_t<_Iter>>)
-	return ranges::next(std::move(__first), __last);
-      else
-	{
-	  for (; __first != __last; ++__first)
-	    ranges::destroy_at(std::__addressof(*__first));
-	  return __first;
-	}
+	if (!is_constant_evaluated())
+	  return ranges::next(std::move(__first), __last);
+
+      for (; __first != __last; ++__first)
+	ranges::destroy_at(std::__addressof(*__first));
+      return __first;
     }
 
   template<__detail::__nothrow_input_range _Range>
@@ -581,13 +580,12 @@ namespace ranges
       operator()(_Iter __first, iter_difference_t<_Iter> __n) const noexcept
       {
 	if constexpr (is_trivially_destructible_v<iter_value_t<_Iter>>)
-	  return ranges::next(std::move(__first), __n);
-	else
-	  {
-	    for (; __n > 0; ++__first, (void)--__n)
-	      ranges::destroy_at(std::__addressof(*__first));
-	    return __first;
-	  }
+	  if (!is_constant_evaluated())
+	    return ranges::next(std::move(__first), __n);
+
+	for (; __n > 0; ++__first, (void)--__n)
+	  ranges::destroy_at(std::__addressof(*__first));
+	return __first;
       }
   };
 
diff --git a/libstdc++-v3/include/bits/shared_ptr.h b/libstdc++-v3/include/bits/shared_ptr.h
index a196a0f..f2b4601 100644
--- a/libstdc++-v3/include/bits/shared_ptr.h
+++ b/libstdc++-v3/include/bits/shared_ptr.h
@@ -909,6 +909,64 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     : public _Sp_owner_less<weak_ptr<_Tp>, shared_ptr<_Tp>>
     { };
 
+#ifdef __glibcxx_smart_ptr_owner_equality // >= C++26
+
+  /**
+   * @brief Provides ownership-based hashing.
+   * @headerfile memory
+   * @since C++26
+   */
+  struct owner_hash
+  {
+    template<typename _Tp>
+      size_t
+      operator()(const shared_ptr<_Tp>& __s) const noexcept
+      { return __s.owner_hash(); }
+
+    template<typename _Tp>
+      size_t
+      operator()(const weak_ptr<_Tp>& __s) const noexcept
+      { return __s.owner_hash(); }
+
+    using is_transparent = void;
+  };
+
+  /**
+   * @brief Provides ownership-based mixed equality comparisons of
+   *        shared and weak pointers.
+   * @headerfile memory
+   * @since C++26
+   */
+  struct owner_equal
+  {
+    template<typename _Tp1, typename _Tp2>
+      bool
+      operator()(const shared_ptr<_Tp1>& __lhs,
+		 const shared_ptr<_Tp2>& __rhs) const noexcept
+      { return __lhs.owner_equal(__rhs); }
+
+    template<typename _Tp1, typename _Tp2>
+      bool
+      operator()(const shared_ptr<_Tp1>& __lhs,
+		 const   weak_ptr<_Tp2>& __rhs) const noexcept
+      { return __lhs.owner_equal(__rhs); }
+
+    template<typename _Tp1, typename _Tp2>
+      bool
+      operator()(const   weak_ptr<_Tp1>& __lhs,
+		 const shared_ptr<_Tp2>& __rhs) const noexcept
+      { return __lhs.owner_equal(__rhs); }
+
+    template<typename _Tp1, typename _Tp2>
+      bool
+      operator()(const weak_ptr<_Tp1>& __lhs,
+		 const weak_ptr<_Tp2>& __rhs)   const noexcept
+      { return __lhs.owner_equal(__rhs); }
+
+    using is_transparent = void;
+  };
+#endif
+
   /**
    * @brief Base class allowing use of the member function `shared_from_this`.
    * @headerfile memory
diff --git a/libstdc++-v3/include/bits/shared_ptr_base.h b/libstdc++-v3/include/bits/shared_ptr_base.h
index b4be1b4..fb868e7 100644
--- a/libstdc++-v3/include/bits/shared_ptr_base.h
+++ b/libstdc++-v3/include/bits/shared_ptr_base.h
@@ -1122,6 +1122,12 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       _M_less(const __weak_count<_Lp>& __rhs) const noexcept
       { return std::less<_Sp_counted_base<_Lp>*>()(this->_M_pi, __rhs._M_pi); }
 
+#ifdef __glibcxx_smart_ptr_owner_equality // >= C++26
+      size_t
+      _M_owner_hash() const noexcept
+      { return std::hash<_Sp_counted_base<_Lp>*>()(this->_M_pi); }
+#endif
+
       // Friend function injected into enclosing namespace and found by ADL
       friend inline bool
       operator==(const __shared_count& __a, const __shared_count& __b) noexcept
@@ -1225,6 +1231,12 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       _M_less(const __shared_count<_Lp>& __rhs) const noexcept
       { return std::less<_Sp_counted_base<_Lp>*>()(this->_M_pi, __rhs._M_pi); }
 
+#ifdef __glibcxx_smart_ptr_owner_equality // >= C++26
+      size_t
+      _M_owner_hash() const noexcept
+      { return std::hash<_Sp_counted_base<_Lp>*>()(this->_M_pi); }
+#endif
+
       // Friend function injected into enclosing namespace and found by ADL
       friend inline bool
       operator==(const __weak_count& __a, const __weak_count& __b) noexcept
@@ -1715,6 +1727,20 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	{ return _M_refcount._M_less(__rhs._M_refcount); }
       /// @}
 
+#ifdef __glibcxx_smart_ptr_owner_equality // >= C++26
+      size_t owner_hash() const noexcept { return _M_refcount._M_owner_hash(); }
+
+      template<typename _Tp1>
+	bool
+	owner_equal(__shared_ptr<_Tp1, _Lp> const& __rhs) const noexcept
+	{ return _M_refcount == __rhs._M_refcount; }
+
+      template<typename _Tp1>
+	bool
+	owner_equal(__weak_ptr<_Tp1, _Lp> const& __rhs) const noexcept
+	{ return _M_refcount == __rhs._M_refcount; }
+#endif
+
     protected:
       // This constructor is non-standard, it is used by allocate_shared.
       template<typename _Alloc, typename... _Args>
@@ -2098,6 +2124,20 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	owner_before(const __weak_ptr<_Tp1, _Lp>& __rhs) const noexcept
 	{ return _M_refcount._M_less(__rhs._M_refcount); }
 
+#ifdef __glibcxx_smart_ptr_owner_equality // >= C++26
+      size_t owner_hash() const noexcept { return _M_refcount._M_owner_hash(); }
+
+      template<typename _Tp1>
+      bool
+      owner_equal(const __shared_ptr<_Tp1, _Lp> & __rhs) const noexcept
+      { return _M_refcount == __rhs._M_refcount; }
+
+      template<typename _Tp1>
+      bool
+      owner_equal(const __weak_ptr<_Tp1, _Lp> & __rhs) const noexcept
+      { return _M_refcount == __rhs._M_refcount; }
+#endif
+
       void
       reset() noexcept
       { __weak_ptr().swap(*this); }
diff --git a/libstdc++-v3/include/bits/stl_algobase.h b/libstdc++-v3/include/bits/stl_algobase.h
index 119dbe9..b104ec2 100644
--- a/libstdc++-v3/include/bits/stl_algobase.h
+++ b/libstdc++-v3/include/bits/stl_algobase.h
@@ -1046,19 +1046,26 @@ _GLIBCXX_END_NAMESPACE_CONTAINER
   __size_to_integer(unsigned __GLIBCXX_TYPE_INT_N_2 __n) { return __n; }
 #endif
 #if defined(__GLIBCXX_TYPE_INT_N_3)
-  __extension__ inline _GLIBCXX_CONSTEXPR unsigned __GLIBCXX_TYPE_INT_N_3
-  __size_to_integer(__GLIBCXX_TYPE_INT_N_3 __n) { return __n; }
   __extension__ inline _GLIBCXX_CONSTEXPR __GLIBCXX_TYPE_INT_N_3
+  __size_to_integer(__GLIBCXX_TYPE_INT_N_3 __n) { return __n; }
+  __extension__ inline _GLIBCXX_CONSTEXPR unsigned __GLIBCXX_TYPE_INT_N_3
   __size_to_integer(unsigned __GLIBCXX_TYPE_INT_N_3 __n) { return __n; }
 #endif
 
+#if defined(__STRICT_ANSI__) && defined(__SIZEOF_INT128__)
+  __extension__ inline _GLIBCXX_CONSTEXPR __int128
+  __size_to_integer(__int128 __n) { return __n; }
+  __extension__ inline _GLIBCXX_CONSTEXPR unsigned __int128
+  __size_to_integer(unsigned __int128 __n) { return __n; }
+#endif
+
   inline _GLIBCXX_CONSTEXPR long long
   __size_to_integer(float __n) { return (long long)__n; }
   inline _GLIBCXX_CONSTEXPR long long
   __size_to_integer(double __n) { return (long long)__n; }
   inline _GLIBCXX_CONSTEXPR long long
   __size_to_integer(long double __n) { return (long long)__n; }
-#if !defined(__STRICT_ANSI__) && defined(_GLIBCXX_USE_FLOAT128)
+#ifdef _GLIBCXX_USE_FLOAT128
   __extension__ inline _GLIBCXX_CONSTEXPR long long
   __size_to_integer(__float128 __n) { return (long long)__n; }
 #endif
diff --git a/libstdc++-v3/include/bits/stl_iterator.h b/libstdc++-v3/include/bits/stl_iterator.h
index a7188f4..75e794f 100644
--- a/libstdc++-v3/include/bits/stl_iterator.h
+++ b/libstdc++-v3/include/bits/stl_iterator.h
@@ -1211,7 +1211,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #else
        // Forward iterator requirements
       template<typename _Iter>
-	__attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+	_GLIBCXX_NODISCARD __attribute__((__always_inline__))
 	friend
 	_GLIBCXX_CONSTEXPR
 	bool
@@ -1220,7 +1220,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	_GLIBCXX_NOEXCEPT
 	{ return __lhs.base() == __rhs.base(); }
 
-      __attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD __attribute__((__always_inline__))
       friend
       _GLIBCXX_CONSTEXPR
       bool
@@ -1229,7 +1229,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       { return __lhs.base() == __rhs.base(); }
 
       template<typename _Iter>
-	__attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+	_GLIBCXX_NODISCARD __attribute__((__always_inline__))
 	friend
 	_GLIBCXX_CONSTEXPR
 	bool
@@ -1238,7 +1238,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	_GLIBCXX_NOEXCEPT
 	{ return __lhs.base() != __rhs.base(); }
 
-      __attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD __attribute__((__always_inline__))
       friend
       _GLIBCXX_CONSTEXPR
       bool
@@ -1248,15 +1248,16 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
       // Random access iterator requirements
       template<typename _Iter>
+	_GLIBCXX_NODISCARD __attribute__((__always_inline__))
 	friend
-	__attribute__((__always_inline__)) _GLIBCXX_NODISCARD _GLIBCXX_CONSTEXPR
+	_GLIBCXX_CONSTEXPR
 	inline bool
 	operator<(const __normal_iterator& __lhs,
 		  const __normal_iterator<_Iter, _Container>& __rhs)
 	_GLIBCXX_NOEXCEPT
 	{ return __lhs.base() < __rhs.base(); }
 
-      __attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD __attribute__((__always_inline__))
       friend
       _GLIBCXX20_CONSTEXPR
       bool
@@ -1265,15 +1266,16 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       { return __lhs.base() < __rhs.base(); }
 
       template<typename _Iter>
+	_GLIBCXX_NODISCARD __attribute__((__always_inline__))
 	friend
-	__attribute__((__always_inline__)) _GLIBCXX_NODISCARD _GLIBCXX_CONSTEXPR
+	_GLIBCXX_CONSTEXPR
 	bool
 	operator>(const __normal_iterator& __lhs,
 		  const __normal_iterator<_Iter, _Container>& __rhs)
 	_GLIBCXX_NOEXCEPT
 	{ return __lhs.base() > __rhs.base(); }
 
-      __attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD __attribute__((__always_inline__))
       friend
       _GLIBCXX_CONSTEXPR
       bool
@@ -1282,7 +1284,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       { return __lhs.base() > __rhs.base(); }
 
       template<typename _Iter>
-	__attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+	_GLIBCXX_NODISCARD __attribute__((__always_inline__))
 	friend
 	_GLIBCXX_CONSTEXPR
 	bool
@@ -1291,7 +1293,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	_GLIBCXX_NOEXCEPT
 	{ return __lhs.base() <= __rhs.base(); }
 
-      __attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD __attribute__((__always_inline__))
       friend
       _GLIBCXX_CONSTEXPR
       bool
@@ -1300,7 +1302,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       { return __lhs.base() <= __rhs.base(); }
 
       template<typename _Iter>
-	__attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+	_GLIBCXX_NODISCARD __attribute__((__always_inline__))
 	friend
 	_GLIBCXX_CONSTEXPR
 	bool
@@ -1309,7 +1311,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	_GLIBCXX_NOEXCEPT
 	{ return __lhs.base() >= __rhs.base(); }
 
-      __attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD __attribute__((__always_inline__))
       friend
       _GLIBCXX_CONSTEXPR
       bool
@@ -1341,7 +1343,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #endif
 	{ return __lhs.base() - __rhs.base(); }
 
-      __attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD __attribute__((__always_inline__))
       friend
       _GLIBCXX_CONSTEXPR
       difference_type
@@ -1349,7 +1351,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	_GLIBCXX_NOEXCEPT
       { return __lhs.base() - __rhs.base(); }
 
-      __attribute__((__always_inline__)) _GLIBCXX_NODISCARD
+      _GLIBCXX_NODISCARD __attribute__((__always_inline__))
       friend
       _GLIBCXX_CONSTEXPR
       __normal_iterator
diff --git a/libstdc++-v3/include/bits/stl_iterator_base_funcs.h b/libstdc++-v3/include/bits/stl_iterator_base_funcs.h
index 637159f..f78e535 100644
--- a/libstdc++-v3/include/bits/stl_iterator_base_funcs.h
+++ b/libstdc++-v3/include/bits/stl_iterator_base_funcs.h
@@ -130,6 +130,28 @@ _GLIBCXX_END_NAMESPACE_CONTAINER
     __distance(_OutputIterator, _OutputIterator, output_iterator_tag) = delete;
 #endif
 
+#ifdef __glibcxx_concepts
+namespace __detail
+{
+  // Satisfied if ITER_TRAITS(Iter)::iterator_category is valid and is
+  // at least as strong as ITER_TRAITS(Iter)::iterator_concept.
+  template<typename _Iter>
+    concept __iter_category_converts_to_concept
+      = convertible_to<typename __iter_traits<_Iter>::iterator_category,
+		       typename __iter_traits<_Iter>::iterator_concept>;
+
+  // Satisfied if the type is a C++20 iterator that defines iterator_concept,
+  // and its iterator_concept is stronger than its iterator_category (if any).
+  // Used by std::distance and std::advance to detect iterators which should
+  // dispatch based on their C++20 concept not their C++17 category.
+  template<typename _Iter>
+    concept __promotable_iterator
+      = input_iterator<_Iter>
+	  && requires { typename __iter_traits<_Iter>::iterator_concept; }
+	  && ! __iter_category_converts_to_concept<_Iter>;
+} // namespace __detail
+#endif
+
   /**
    *  @brief A generalization of pointer arithmetic.
    *  @param  __first  An input iterator.
@@ -149,6 +171,24 @@ _GLIBCXX_END_NAMESPACE_CONTAINER
     typename iterator_traits<_InputIterator>::difference_type
     distance(_InputIterator __first, _InputIterator __last)
     {
+#ifdef __glibcxx_concepts
+      // A type which satisfies the C++20 random_access_iterator concept might
+      // have input_iterator_tag as its iterator_category type, which would
+      // mean we select the O(n) __distance. Or a C++20 std::input_iterator
+      // that is not a Cpp17InputIterator might have output_iterator_tag as
+      // its iterator_category type and then calling __distance with
+      // std::__iterator_category(__first) would be ill-formed.
+      // So for C++20 iterator types we can just choose to do the right thing.
+      if constexpr (__detail::__promotable_iterator<_InputIterator>)
+	{
+	  if constexpr (random_access_iterator<_InputIterator>)
+	    return __last - __first;
+	  else
+	    return std::__distance(std::move(__first), std::move(__last),
+				   input_iterator_tag());
+	}
+      else // assume it meets the Cpp17InputIterator requirements:
+#endif
       // concept requirements -- taken care of in __distance
       return std::__distance(__first, __last,
 			     std::__iterator_category(__first));
@@ -221,9 +261,31 @@ _GLIBCXX_END_NAMESPACE_CONTAINER
     inline _GLIBCXX17_CONSTEXPR void
     advance(_InputIterator& __i, _Distance __n)
     {
-      // concept requirements -- taken care of in __advance
-      typename iterator_traits<_InputIterator>::difference_type __d = __n;
-      std::__advance(__i, __d, std::__iterator_category(__i));
+#ifdef __glibcxx_concepts
+      // A type which satisfies the C++20 bidirectional_iterator concept might
+      // have input_iterator_tag as its iterator_category type, which would
+      // mean we select the __advance overload which cannot move backwards.
+      // A C++20 random_access_iterator we might select the O(n) __advance
+      // if it doesn't meet the Cpp17RandomAccessIterator requirements.
+      // So for C++20 iterator types we can just choose to do the right thing.
+      if constexpr (__detail::__promotable_iterator<_InputIterator>
+		      && ranges::__detail::__is_integer_like<_Distance>)
+	{
+	  auto __d = static_cast<iter_difference_t<_InputIterator>>(__n);
+	  if constexpr (random_access_iterator<_InputIterator>)
+	    std::__advance(__i, __d, random_access_iterator_tag());
+	  else if constexpr (bidirectional_iterator<_InputIterator>)
+	    std::__advance(__i, __d, bidirectional_iterator_tag());
+	  else
+	    std::__advance(__i, __d, input_iterator_tag());
+	}
+      else // assume it meets the Cpp17InputIterator requirements:
+#endif
+	{
+	  // concept requirements -- taken care of in __advance
+	  typename iterator_traits<_InputIterator>::difference_type __d = __n;
+	  std::__advance(__i, __d, std::__iterator_category(__i));
+	}
     }
 
 #if __cplusplus >= 201103L
diff --git a/libstdc++-v3/include/bits/stl_iterator_base_types.h b/libstdc++-v3/include/bits/stl_iterator_base_types.h
index a67d7bd..0c34ad7 100644
--- a/libstdc++-v3/include/bits/stl_iterator_base_types.h
+++ b/libstdc++-v3/include/bits/stl_iterator_base_types.h
@@ -257,6 +257,13 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   template<typename _InIter>
     concept __has_input_iter_cat
       = is_convertible_v<__iter_category_t<_InIter>, input_iterator_tag>;
+
+#ifdef __cpp_lib_concepts
+  // Is a Cpp17InputIterator or satisfies std::input_iterator.
+  template<typename _InIterator>
+    concept __any_input_iterator
+      = input_iterator<_InIterator> || __has_input_iter_cat<_InIterator>;
+#endif
 #endif
 
   template<typename _It,
diff --git a/libstdc++-v3/include/bits/stl_pair.h b/libstdc++-v3/include/bits/stl_pair.h
index 8c57712..393f6a0 100644
--- a/libstdc++-v3/include/bits/stl_pair.h
+++ b/libstdc++-v3/include/bits/stl_pair.h
@@ -1132,6 +1132,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #endif // C++23
 
 #if __cplusplus > 201402L || !defined(__STRICT_ANSI__) // c++1z or gnu++11
+  // _GLIBCXX_RESOLVE_LIB_DEFECTS
+  // 2766. Swapping non-swappable types
   template<typename _T1, typename _T2>
     typename enable_if<!__and_<__is_swappable<_T1>,
 			       __is_swappable<_T2>>::value>::type
diff --git a/libstdc++-v3/include/bits/stl_uninitialized.h b/libstdc++-v3/include/bits/stl_uninitialized.h
index 3a37ddc..351c3a1 100644
--- a/libstdc++-v3/include/bits/stl_uninitialized.h
+++ b/libstdc++-v3/include/bits/stl_uninitialized.h
@@ -922,11 +922,17 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   // __uninitialized_default
   // Fills [first, last) with value-initialized value_types.
   template<typename _ForwardIterator>
-    _GLIBCXX26_CONSTEXPR
+    _GLIBCXX20_CONSTEXPR
     inline void
     __uninitialized_default(_ForwardIterator __first,
 			    _ForwardIterator __last)
     {
+#ifdef __cpp_lib_is_constant_evaluated
+      if (std::is_constant_evaluated())
+	return __uninitialized_default_1<false>::
+		 __uninit_default(__first, __last);
+#endif
+
       typedef typename iterator_traits<_ForwardIterator>::value_type
 	_ValueType;
       // trivial types can have deleted assignment
diff --git a/libstdc++-v3/include/bits/unicode.h b/libstdc++-v3/include/bits/unicode.h
index f1b6bf4..88e97d4 100644
--- a/libstdc++-v3/include/bits/unicode.h
+++ b/libstdc++-v3/include/bits/unicode.h
@@ -86,6 +86,25 @@ namespace __unicode
       { return *__it == iter_value_t<_It>{}; }
   };
 
+  // An iterator over an input range of FromFmt code units that yields either
+  // UTF-8, UTF-16, or UTF-32, as a range of ToFmt code units.
+  // The code units from the input range are interpreted as Unicode code points
+  // and the iterator produces the individual code unit for each code point.
+  // Invalid sequences in the input are replaced with U+FFDD so that the result
+  // is always valid UTF-8, UTF-16, or UTF-32.
+  //
+  // The iterator knows the bounds of the underlying input range and will not
+  // read outside those bounds (incrementing or decrementing at the boundary
+  // is erroneously idempotent).
+  //
+  // On construction, the iterator attemps to decode a single code point from
+  // the input range and then encode it into an internal buffer in the output
+  // format, e.g. if the input is UTF-8 and the output is UTF-16, it might read
+  // three char8_t code units from the input and store two char16_t code units
+  // in its buffer. Incrementing the iterator will first iterate over buffer,
+  // yielding each code unit in turn, and then extract another code point from
+  // the input. Failure to extract a valid code point from the input will store
+  // U+FFFD in the buffer, encoded as the appropriate code units of type ToFmt.
   template<typename _FromFmt, typename _ToFmt,
 	   input_iterator _Iter, sentinel_for<_Iter> _Sent = _Iter,
 	   typename _ErrorHandler = _Repl>
@@ -162,17 +181,20 @@ namespace __unicode
       constexpr _Utf_iterator&
       operator++()
       {
-	if (_M_buf_index + 1 == _M_buf_last && _M_curr() != _M_last)
+	if (_M_buf_index + 1 < _M_buf_last)
+	  ++_M_buf_index; // Move to the next code unit in the buffer.
+	else if (_M_curr() != _M_last)
 	  {
+	    // Advance past the current code point (for non-forward iterators
+	    // we already moved there after decoding the last code point).
 	    if constexpr (forward_iterator<_Iter>)
 	      std::advance(_M_curr(), _M_to_increment);
 	    if (_M_curr() == _M_last)
 	      _M_buf_index = 0;
-	    else
+	    else // Decode next code point from the input and update buffer.
 	      _M_read();
 	  }
-	else if (_M_buf_index + 1 < _M_buf_last)
-	  ++_M_buf_index;
+	// else erroneous, but ignored for now.
 	return *this;
       }
 
@@ -187,10 +209,15 @@ namespace __unicode
       constexpr _Utf_iterator&
       operator--() requires bidirectional_iterator<_Iter>
       {
-	if (!_M_buf_index && _M_curr() != _M_first())
-	  _M_read_reverse();
-	else if (_M_buf_index)
+	if (_M_buf_index > 0)
 	  --_M_buf_index;
+	else if (_M_curr() != _M_first())
+	  {
+	    _M_read_reverse();
+	    _M_buf_index = _M_buf_last - 1;
+	    ranges::advance(_M_curr(), -_M_to_increment);
+	  }
+	// else erroneous, but ignored for now.
 	return *this;
       }
 
@@ -247,7 +274,18 @@ namespace __unicode
       }
 
       constexpr void
-      _M_read_reverse(); // TODO
+      _M_read_reverse() requires bidirectional_iterator<_Iter>
+      {
+	if constexpr (sizeof(_FromFmt) == sizeof(uint8_t))
+	  _M_read_reverse_utf8();
+	else if constexpr (sizeof(_FromFmt) == sizeof(uint16_t))
+	  _M_read_reverse_utf16();
+	else
+	  {
+	    static_assert(sizeof(_FromFmt) == sizeof(uint32_t));
+	    _M_read_reverse_utf32();
+	  }
+      }
 
       template<typename>
 	struct _Guard
@@ -263,7 +301,7 @@ namespace __unicode
 	  _It _M_orig;
 	};
 
-      constexpr void
+      constexpr char32_t
       _M_read_utf8()
       {
 	_Guard<_Iter> __g{this, _M_curr()};
@@ -361,6 +399,8 @@ namespace __unicode
 	  __c = _S_error();
 
 	_M_update(__c, __to_incr);
+
+	return __c;
       }
 
       constexpr void
@@ -403,6 +443,116 @@ namespace __unicode
 	_M_update(__c, 1);
       }
 
+      constexpr void
+      _M_read_reverse_utf8() requires bidirectional_iterator<_Iter>
+      {
+	const auto __first = _M_first();
+	auto __curr = _M_curr();
+	// The code point we decode:
+	char32_t __c{};
+	// The last code unit read:
+	uint8_t __u = *--__curr;
+	// Count of bytes read:
+	uint8_t __to_incr = 1;
+
+	if (__u <= 0x7F) [[likely]]
+	  {
+	    _M_update(__u, 1);
+	    return;
+	  }
+
+	// Continuation bytes match 10xxxxxx
+	auto __is_continuation = [](uint8_t __b) {
+	  return (__b & 0xC0) == 0x80;
+	};
+	// 0xC0 and 0xC1 would produce overlong encodings of ASCII characters.
+	// 0xF5-0xFF would produce code points above U+10FFFF
+	auto __is_invalid = [](uint8_t __b) {
+	  return (__b & 0xFE) == 0xC0 || __b >= 0xF5;
+	};
+
+	// No valid or invalid multibyte sequence is longer than 4 bytes,
+	// so skip back over at most four bytes.
+	while (__is_continuation(__u) && __to_incr < 4 && __curr != __first)
+	  {
+	    ++__to_incr;
+	    __u = *--__curr;
+	  }
+
+	// If the last byte read was a continuation byte then either we read
+	// four continuation bytes, or stopped at the start of the sequence.
+	// Either way, the maximal subparts are the individual continuation
+	// bytes so each one should be replaced with U+FFFD.
+	if (__is_continuation(__u) || __is_invalid(__u)) [[unlikely]]
+	  {
+	    // Either found four continuation bytes (maximum allowed is three)
+	    // or first non-continuation byte is an invalid UTF-8 code unit.
+	    _M_update(_S_error(), 1);
+	    return;
+	  }
+	// __u is a valid start byte so use countl_one to get the expected
+	// length of the multibyte sequence that starts with this byte.
+	int __seq_length = std::countl_one((unsigned char)__u);
+	if (__seq_length < __to_incr) [[unlikely]]
+	  {
+	    // If the expected number of continuation bytes is less than
+	    // the number we found, then the last continuation byte is a
+	    // maximal subpart and the decremented iterator points to it.
+	    _M_update(_S_error(), 1);
+	    return;
+	  }
+
+	auto __orig = std::__exchange(_M_curr(), std::move(__curr));
+	if (_M_read_utf8() == _S_error()) [[unlikely]]
+	  {
+	    if (_M_to_increment < __to_incr) // Read truncated sequence, set
+	      _M_to_increment = 1;           // curr to last continuation byte.
+	  }
+
+	_M_curr() = std::move(__orig);
+	// operator--() will move back by _M_to_increment
+      }
+
+      constexpr void
+      _M_read_reverse_utf16() requires bidirectional_iterator<_Iter>
+      {
+	_Guard<_Iter> __g{this, _M_curr()};
+	char32_t __c{};
+	uint16_t __u = *--_M_curr();
+	uint8_t __to_incr = 1;
+
+	if (__u < 0xD800 || __u > 0xDFFF) [[likely]]
+	  __c = __u;
+	else if (__u >= 0xDC00 && _M_curr() != _M_first()) [[likely]]
+	  {
+	    // read a low surrogate, expect a high surrogate before it.
+	    uint16_t __u2 = *--_M_curr();
+	    if (__u2 < 0xD800 || __u2 > 0xDC00) [[unlikely]]
+	      __c = _S_error(); // unpaired low surrogate
+	    else
+	      {
+		__to_incr = 2;
+		uint32_t __x = (__u2 & 0x3F) << 10 | (__u & 0x3FF);
+		uint32_t __w = (__u2 >> 6) & 0x1F;
+		__c = (__w + 1) << 16 | __x;
+	      }
+	  }
+	else
+	  __c = _S_error(); // unpaired surrogate
+
+	_M_update(__c, __to_incr);
+      }
+
+      constexpr void
+      _M_read_reverse_utf32() requires bidirectional_iterator<_Iter>
+      {
+	_Guard<_Iter> __g{this, _M_curr()};
+	char32_t __c = *--_M_curr();
+	if (!__is_scalar_value(__c)) [[unlikely]]
+	  __c = _S_error();
+	_M_update(__c, 1);
+      }
+
       // Encode the code point __c as one or more code units in _M_buf.
       constexpr void
       _M_update(char32_t __c, uint8_t __to_incr)
@@ -487,8 +637,7 @@ namespace __unicode
       constexpr _Iter
       _M_curr() const { return _M_first_and_curr._M_curr; }
 
-      array<value_type, 4 / sizeof(_ToFmt)> _M_buf;
-
+      // _M_first is not needed for non-bidirectional ranges.
       template<typename _It>
 	struct _First_and_curr
 	{
@@ -502,6 +651,8 @@ namespace __unicode
 	    _First_and_curr(const _First_and_curr<_It2>& __other)
 	    : _M_curr(__other._M_curr) { }
 
+	  // First code unit of the current code point for forward iterators,
+	  // past-the-end of the current code point for input iterators.
 	  _It _M_curr;
 	};
 
@@ -519,18 +670,24 @@ namespace __unicode
 	    _First_and_curr(const _First_and_curr<_It2>& __other)
 	    : _M_first(__other._M_first), _M_curr(__other._M_curr) { }
 
-	  _It _M_first;
-	  _It _M_curr;
+	  _It _M_first; // Start of the underlying range.
+	  _It _M_curr;  // First code unit of the current code point.
 	};
 
+      // Iterators pointing to the start of the underlying range and to the
+      // start (or end, for non-forward iterators) of the current code point.
       _First_and_curr<_Iter> _M_first_and_curr;
 
-      uint8_t _M_buf_index = 0;
-      uint8_t _M_buf_last = 0;
-      uint8_t _M_to_increment = 0;
-
+      // The end of the underlying input range.
       [[no_unique_address]] _Sent _M_last;
 
+      // Buffer holding the individual code units of the current code point.
+      array<value_type, 4 / sizeof(_ToFmt)> _M_buf;
+
+      uint8_t _M_buf_index = 0;    // Index of current code unit in the buffer.
+      uint8_t _M_buf_last = 0;     // Number of code units in the buffer.
+      uint8_t _M_to_increment = 0; // How far to advance _M_curr on increment.
+
       template<typename _FromFmt2, typename _ToFmt2,
 	       input_iterator _Iter2, sentinel_for<_Iter2> _Sent2,
 	       typename _ErrHandler>
diff --git a/libstdc++-v3/include/bits/unique_ptr.h b/libstdc++-v3/include/bits/unique_ptr.h
index 6ae46a9..d76ad63 100644
--- a/libstdc++-v3/include/bits/unique_ptr.h
+++ b/libstdc++-v3/include/bits/unique_ptr.h
@@ -832,6 +832,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     { __x.swap(__y); }
 
 #if __cplusplus > 201402L || !defined(__STRICT_ANSI__) // c++1z or gnu++11
+  // _GLIBCXX_RESOLVE_LIB_DEFECTS
+  // 2766. Swapping non-swappable types
   template<typename _Tp, typename _Dp>
     typename enable_if<!__is_swappable<_Dp>::value>::type
     swap(unique_ptr<_Tp, _Dp>&,
diff --git a/libstdc++-v3/include/bits/unordered_map.h b/libstdc++-v3/include/bits/unordered_map.h
index fc07ffc..cc9e2c4 100644
--- a/libstdc++-v3/include/bits/unordered_map.h
+++ b/libstdc++-v3/include/bits/unordered_map.h
@@ -91,9 +91,9 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
    *
    *  @tparam  _Key    Type of key objects.
    *  @tparam  _Tp     Type of mapped objects.
-   *  @tparam  _Hash   Hashing function object type, defaults to hash<_Value>.
+   *  @tparam  _Hash   Hashing function object type, defaults to hash<_Key>.
    *  @tparam  _Pred   Predicate function object type, defaults
-   *                   to equal_to<_Value>.
+   *                   to equal_to<_Key>.
    *  @tparam  _Alloc  Allocator type, defaults to
    *                   std::allocator<std::pair<const _Key, _Tp>>.
    *
@@ -1360,9 +1360,9 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
    *
    *  @tparam  _Key    Type of key objects.
    *  @tparam  _Tp     Type of mapped objects.
-   *  @tparam  _Hash   Hashing function object type, defaults to hash<_Value>.
+   *  @tparam  _Hash   Hashing function object type, defaults to hash<_Key>.
    *  @tparam  _Pred   Predicate function object type, defaults
-   *                   to equal_to<_Value>.
+   *                   to equal_to<_Key>.
    *  @tparam  _Alloc  Allocator type, defaults to
    *                   std::allocator<std::pair<const _Key, _Tp>>.
    *
diff --git a/libstdc++-v3/include/bits/version.def b/libstdc++-v3/include/bits/version.def
index b89b287..dbe2cb8 100644
--- a/libstdc++-v3/include/bits/version.def
+++ b/libstdc++-v3/include/bits/version.def
@@ -1007,9 +1007,8 @@ ftms = {
 
 ftms = {
   name = mdspan;
-  no_stdname = true; // FIXME: remove
   values = {
-    v = 1; // FIXME: 202207
+    v = 202207;
     cxxmin = 23;
   };
 };
@@ -1092,6 +1091,10 @@ ftms = {
 ftms = {
   name = shift;
   values = {
+    v = 202202;
+    cxxmin = 23;
+  };
+  values = {
     v = 201806;
     cxxmin = 20;
   };
@@ -1986,6 +1989,14 @@ ftms = {
 };
 
 ftms = {
+  name = inplace_vector;
+  values = {
+    v = 202406;
+    cxxmin = 26;
+  };
+};
+
+ftms = {
   name = indirect;
   values = {
     v = 202502;
@@ -2004,6 +2015,15 @@ ftms = {
 };
 
 ftms = {
+  name = smart_ptr_owner_equality;
+  values = {
+    v = 202306;
+    cxxmin = 26;
+    hosted = yes;
+  };
+};
+
+ftms = {
   name = sstream_from_string_view;
   values = {
     v = 202306;
@@ -2031,13 +2051,24 @@ ftms = {
 };
 
 ftms = {
-  name = bitset  // ...construct from string_view
+  name = bitset;  // ...construct from string_view
   values = {
     v = 202306;
     cxxmin = 26;
   };
 };
 
+ftms = {
+  name = constexpr_exceptions;
+  // TODO Remove when PR121114 is resolved
+  no_stdname = true;
+  values = {
+    v = 1; // TODO 202411;
+    cxxmin = 26;
+    extra_cond = "__cpp_constexpr_exceptions >= 202411L";
+  };
+};
+
 // Standard test specifications.
 stds[97] = ">= 199711L";
 stds[03] = ">= 199711L";
diff --git a/libstdc++-v3/include/bits/version.h b/libstdc++-v3/include/bits/version.h
index a70a7ed..7bb6016 100644
--- a/libstdc++-v3/include/bits/version.h
+++ b/libstdc++-v3/include/bits/version.h
@@ -1126,8 +1126,9 @@
 
 #if !defined(__cpp_lib_mdspan)
 # if (__cplusplus >= 202100L)
-#  define __glibcxx_mdspan 1L
+#  define __glibcxx_mdspan 202207L
 #  if defined(__glibcxx_want_all) || defined(__glibcxx_want_mdspan)
+#   define __cpp_lib_mdspan 202207L
 #  endif
 # endif
 #endif /* !defined(__cpp_lib_mdspan) && defined(__glibcxx_want_mdspan) */
@@ -1224,7 +1225,12 @@
 #undef __glibcxx_want_constexpr_utility
 
 #if !defined(__cpp_lib_shift)
-# if (__cplusplus >= 202002L)
+# if (__cplusplus >= 202100L)
+#  define __glibcxx_shift 202202L
+#  if defined(__glibcxx_want_all) || defined(__glibcxx_want_shift)
+#   define __cpp_lib_shift 202202L
+#  endif
+# elif (__cplusplus >= 202002L)
 #  define __glibcxx_shift 201806L
 #  if defined(__glibcxx_want_all) || defined(__glibcxx_want_shift)
 #   define __cpp_lib_shift 201806L
@@ -2223,6 +2229,16 @@
 #endif /* !defined(__cpp_lib_modules) && defined(__glibcxx_want_modules) */
 #undef __glibcxx_want_modules
 
+#if !defined(__cpp_lib_inplace_vector)
+# if (__cplusplus >  202302L)
+#  define __glibcxx_inplace_vector 202406L
+#  if defined(__glibcxx_want_all) || defined(__glibcxx_want_inplace_vector)
+#   define __cpp_lib_inplace_vector 202406L
+#  endif
+# endif
+#endif /* !defined(__cpp_lib_inplace_vector) && defined(__glibcxx_want_inplace_vector) */
+#undef __glibcxx_want_inplace_vector
+
 #if !defined(__cpp_lib_indirect)
 # if (__cplusplus >  202302L) && _GLIBCXX_HOSTED
 #  define __glibcxx_indirect 202502L
@@ -2243,6 +2259,16 @@
 #endif /* !defined(__cpp_lib_polymorphic) && defined(__glibcxx_want_polymorphic) */
 #undef __glibcxx_want_polymorphic
 
+#if !defined(__cpp_lib_smart_ptr_owner_equality)
+# if (__cplusplus >  202302L) && _GLIBCXX_HOSTED
+#  define __glibcxx_smart_ptr_owner_equality 202306L
+#  if defined(__glibcxx_want_all) || defined(__glibcxx_want_smart_ptr_owner_equality)
+#   define __cpp_lib_smart_ptr_owner_equality 202306L
+#  endif
+# endif
+#endif /* !defined(__cpp_lib_smart_ptr_owner_equality) && defined(__glibcxx_want_smart_ptr_owner_equality) */
+#undef __glibcxx_want_smart_ptr_owner_equality
+
 #if !defined(__cpp_lib_sstream_from_string_view)
 # if (__cplusplus >  202302L) && _GLIBCXX_HOSTED
 #  define __glibcxx_sstream_from_string_view 202306L
@@ -2281,6 +2307,15 @@
 #  endif
 # endif
 #endif /* !defined(__cpp_lib_bitset) && defined(__glibcxx_want_bitset) */
-#undef __glibcxx_bitset
+#undef __glibcxx_want_bitset
+
+#if !defined(__cpp_lib_constexpr_exceptions)
+# if (__cplusplus >  202302L) && (__cpp_constexpr_exceptions >= 202411L)
+#  define __glibcxx_constexpr_exceptions 1L
+#  if defined(__glibcxx_want_all) || defined(__glibcxx_want_constexpr_exceptions)
+#  endif
+# endif
+#endif /* !defined(__cpp_lib_constexpr_exceptions) && defined(__glibcxx_want_constexpr_exceptions) */
+#undef __glibcxx_want_constexpr_exceptions
 
 #undef __glibcxx_want_all
diff --git a/libstdc++-v3/include/c_global/cmath b/libstdc++-v3/include/c_global/cmath
index 27c21ca..65a3b81 100644
--- a/libstdc++-v3/include/c_global/cmath
+++ b/libstdc++-v3/include/c_global/cmath
@@ -3792,10 +3792,10 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   { return std::__hypot3<long double>(__x, __y, __z); }
 
   template<typename _Tp, typename _Up, typename _Vp>
-    __gnu_cxx::__promoted_t<_Tp, _Up, _Vp>
+    typename __gnu_cxx::__promote_3<_Tp, _Up, _Vp>::__type
     hypot(_Tp __x, _Up __y, _Vp __z)
     {
-      using __type = __gnu_cxx::__promoted_t<_Tp, _Up, _Vp>;
+      using __type = typename __gnu_cxx::__promote_3<_Tp, _Up, _Vp>::__type;
       return std::__hypot3<__type>(__x, __y, __z);
     }
 
diff --git a/libstdc++-v3/include/debug/bitset b/libstdc++-v3/include/debug/bitset
index ad9b7b5..e4d3e66 100644
--- a/libstdc++-v3/include/debug/bitset
+++ b/libstdc++-v3/include/debug/bitset
@@ -164,6 +164,17 @@ namespace __debug
 	       _CharT __zero, _CharT __one = _CharT('1'))
 	: _Base(__str, __pos, __n, __zero, __one) { }
 
+#ifdef __cpp_lib_bitset // ... from string_view
+      template<class _CharT, class _Traits>
+	constexpr explicit
+	bitset(std::basic_string_view<_CharT, _Traits> __s,
+	  std::basic_string_view<_CharT, _Traits>::size_type __position = 0,
+	  std::basic_string_view<_CharT, _Traits>::size_type __n =
+	    std::basic_string_view<_CharT, _Traits>::npos,
+	  _CharT __zero = _CharT('0'), _CharT __one = _CharT('1'))
+	: _Base(__s, __position, __n, __zero, __one) { }
+#endif
+
       _GLIBCXX23_CONSTEXPR
       bitset(const _Base& __x) : _Base(__x) { }
 
diff --git a/libstdc++-v3/include/debug/deque b/libstdc++-v3/include/debug/deque
index 59d60b2..ed69eb8 100644
--- a/libstdc++-v3/include/debug/deque
+++ b/libstdc++-v3/include/debug/deque
@@ -650,7 +650,8 @@ namespace __debug
 	else if (__first.base() == _Base::begin()
 		 || __last.base() == _Base::end())
 	  {
-	    this->_M_detach_singular();
+	    const deque* __this = this;
+	    __this->_M_detach_singular();
 	    for (_Base_const_iterator __position = __first.base();
 		 __position != __last.base(); ++__position)
 	      {
@@ -663,7 +664,7 @@ namespace __debug
 	      }
 	    __catch(...)
 	      {
-		this->_M_revalidate_singular();
+		__this->_M_revalidate_singular();
 		__throw_exception_again;
 	      }
 	  }
diff --git a/libstdc++-v3/include/debug/formatter.h b/libstdc++-v3/include/debug/formatter.h
index d80e8a7..8aa84ad 100644
--- a/libstdc++-v3/include/debug/formatter.h
+++ b/libstdc++-v3/include/debug/formatter.h
@@ -96,7 +96,7 @@ namespace __gnu_debug
   template<typename _Iterator, typename _Sequence, typename _Category>
     class _Safe_iterator;
 
-  template<typename _Iterator, typename _Sequence>
+  template<typename _Iterator, typename _UContainer>
     class _Safe_local_iterator;
 
   template<typename _Sequence>
@@ -316,8 +316,8 @@ namespace __gnu_debug
 	    }
 	}
 
-      template<typename _Iterator, typename _Sequence>
-	_Parameter(_Safe_local_iterator<_Iterator, _Sequence> const& __it,
+      template<typename _Iterator, typename _UContainer>
+	_Parameter(_Safe_local_iterator<_Iterator, _UContainer> const& __it,
 		   const char* __name, _Is_iterator)
 	: _M_kind(__iterator),  _M_variant()
 	{
@@ -326,8 +326,8 @@ namespace __gnu_debug
 	  _M_variant._M_iterator._M_type = _GLIBCXX_TYPEID(_Iterator);
 	  _M_variant._M_iterator._M_constness =
 	    __it._S_constant() ? __const_iterator : __mutable_iterator;
-	  _M_variant._M_iterator._M_sequence = __it._M_get_sequence();
-	  _M_variant._M_iterator._M_seq_type = _GLIBCXX_TYPEID(_Sequence);
+	  _M_variant._M_iterator._M_sequence = __it._M_get_ucontainer();
+	  _M_variant._M_iterator._M_seq_type = _GLIBCXX_TYPEID(_UContainer);
 
 	  if (__it._M_singular())
 	    {
diff --git a/libstdc++-v3/include/debug/forward_list b/libstdc++-v3/include/debug/forward_list
index 60a2542..9da7dda 100644
--- a/libstdc++-v3/include/debug/forward_list
+++ b/libstdc++-v3/include/debug/forward_list
@@ -58,44 +58,44 @@ namespace __gnu_debug
     class _Safe_forward_list
     : public _Safe_sequence<_SafeSequence>
     {
-      _SafeSequence&
-      _M_this() noexcept
-      { return *static_cast<_SafeSequence*>(this); }
+      const _SafeSequence*
+      _M_this() const noexcept
+      { return static_cast<const _SafeSequence*>(this); }
 
       static void
-      _M_swap_aux(_Safe_sequence_base& __lhs,
+      _S_swap_aux(const _Safe_forward_list& __lhs,
 		  _Safe_iterator_base*& __lhs_iterators,
-		  _Safe_sequence_base& __rhs,
+		  const _Safe_forward_list& __rhs,
 		  _Safe_iterator_base*& __rhs_iterators);
 
-      void _M_swap_single(_Safe_sequence_base&) noexcept;
+      void _M_swap_single(const _Safe_forward_list&) const noexcept;
 
     protected:
       void
-      _M_invalidate_all()
+      _M_invalidate_all() const
       {
-	using _Base_const_iterator = __decltype(_M_this()._M_base().cend());
+	using _Base_const_iterator = __decltype(_M_this()->_M_base().cend());
 	this->_M_invalidate_if([this](_Base_const_iterator __it)
 	{
-	  return __it != _M_this()._M_base().cbefore_begin()
-	    && __it != _M_this()._M_base().cend(); });
+	  return __it != _M_this()->_M_base().cbefore_begin()
+	    && __it != _M_this()->_M_base().cend(); });
       }
 
-      void _M_swap(_Safe_sequence_base&) noexcept;
+      void
+      _M_swap(const _Safe_forward_list&) const noexcept;
     };
 
    template<typename _SafeSequence>
     void
     _Safe_forward_list<_SafeSequence>::
-    _M_swap_aux(_Safe_sequence_base& __lhs,
+    _S_swap_aux(const _Safe_forward_list& __lhs,
 		_Safe_iterator_base*& __lhs_iterators,
-		_Safe_sequence_base& __rhs,
+		const _Safe_forward_list& __rhs,
 		_Safe_iterator_base*& __rhs_iterators)
     {
       using const_iterator = typename _SafeSequence::const_iterator;
       _Safe_iterator_base* __bbegin_its = 0;
       _Safe_iterator_base* __last_bbegin = 0;
-      _SafeSequence& __rseq = static_cast<_SafeSequence&>(__rhs);
 
       for (_Safe_iterator_base* __iter = __lhs_iterators; __iter;)
 	{
@@ -104,7 +104,7 @@ namespace __gnu_debug
 	  const_iterator* __victim =
 	    static_cast<const_iterator*>(__victim_base);
 	  __iter = __iter->_M_next;
-	  if (__victim->base() == __rseq._M_base().cbefore_begin())
+	  if (__victim->base() == __rhs._M_this()->_M_base().cbefore_begin())
 	    {
 	      __victim->_M_unlink();
 	      if (__lhs_iterators == __victim_base)
@@ -136,21 +136,21 @@ namespace __gnu_debug
    template<typename _SafeSequence>
     void
     _Safe_forward_list<_SafeSequence>::
-    _M_swap_single(_Safe_sequence_base& __other) noexcept
+    _M_swap_single(const _Safe_forward_list& __other) const noexcept
     {
-      std::swap(_M_this()._M_iterators, __other._M_iterators);
-      std::swap(_M_this()._M_const_iterators, __other._M_const_iterators);
+      std::swap(_M_this()->_M_iterators, __other._M_iterators);
+      std::swap(_M_this()->_M_const_iterators, __other._M_const_iterators);
       // Useless, always 1 on forward_list
-      //std::swap(_M_this()_M_version, __other._M_version);
-      _Safe_iterator_base* __this_its = _M_this()._M_iterators;
-      _M_swap_aux(__other, __other._M_iterators,
-		  _M_this(), _M_this()._M_iterators);
-      _Safe_iterator_base* __this_const_its = _M_this()._M_const_iterators;
-      _M_swap_aux(__other, __other._M_const_iterators,
-		  _M_this(), _M_this()._M_const_iterators);
-      _M_swap_aux(_M_this(), __this_its,
+      //std::swap(_M_this()->_M_version, __other._M_version);
+      _Safe_iterator_base* __this_its = _M_this()->_M_iterators;
+      _S_swap_aux(__other, __other._M_iterators,
+		  *_M_this(), _M_this()->_M_iterators);
+      _Safe_iterator_base* __this_const_its = _M_this()->_M_const_iterators;
+      _S_swap_aux(__other, __other._M_const_iterators,
+		  *_M_this(), _M_this()->_M_const_iterators);
+      _S_swap_aux(*_M_this(), __this_its,
 		  __other, __other._M_iterators);
-      _M_swap_aux(_M_this(), __this_const_its,
+      _S_swap_aux(*_M_this(), __this_const_its,
 		  __other, __other._M_const_iterators);
     }
 
@@ -159,13 +159,12 @@ namespace __gnu_debug
    template<typename _SafeSequence>
     void
     _Safe_forward_list<_SafeSequence>::
-    _M_swap(_Safe_sequence_base& __other) noexcept
+    _M_swap(const _Safe_forward_list& __other) const noexcept
     {
       // We need to lock both sequences to swap
       using namespace __gnu_cxx;
-      __mutex *__this_mutex = &_M_this()._M_get_mutex();
-      __mutex *__other_mutex =
-	&static_cast<_SafeSequence&>(__other)._M_get_mutex();
+      __mutex *__this_mutex = &_M_this()->_M_get_mutex();
+      __mutex *__other_mutex = &__other._M_get_mutex();
       if (__this_mutex == __other_mutex)
 	{
 	  __scoped_lock __lock(*__this_mutex);
@@ -565,7 +564,8 @@ namespace __debug
       void
       resize(size_type __sz)
       {
-	this->_M_detach_singular();
+	const forward_list* __this = this;
+	__this->_M_detach_singular();
 
 	// if __sz < size(), invalidate all iterators in [begin+__sz, end()
 	_Base_iterator __victim = _Base::begin();
@@ -585,7 +585,7 @@ namespace __debug
 	  }
 	__catch(...)
 	  {
-	    this->_M_revalidate_singular();
+	    __this->_M_revalidate_singular();
 	    __throw_exception_again;
 	  }
       }
@@ -593,7 +593,8 @@ namespace __debug
       void
       resize(size_type __sz, const value_type& __val)
       {
-	this->_M_detach_singular();
+	const forward_list* __this = this;
+	__this->_M_detach_singular();
 
 	// if __sz < size(), invalidate all iterators in [begin+__sz, end())
 	_Base_iterator __victim = _Base::begin();
@@ -613,7 +614,7 @@ namespace __debug
 	  }
 	__catch(...)
 	  {
-	    this->_M_revalidate_singular();
+	    __this->_M_revalidate_singular();
 	    __throw_exception_again;
 	  }
       }
diff --git a/libstdc++-v3/include/debug/list b/libstdc++-v3/include/debug/list
index a9d974c..c502c7c 100644
--- a/libstdc++-v3/include/debug/list
+++ b/libstdc++-v3/include/debug/list
@@ -321,7 +321,8 @@ namespace __debug
       void
       resize(size_type __sz)
       {
-	this->_M_detach_singular();
+	const list* __this = this;
+	__this->_M_detach_singular();
 
 	// if __sz < size(), invalidate all iterators in [begin + __sz, end())
 	_Base_iterator __victim = _Base::begin();
@@ -338,7 +339,7 @@ namespace __debug
 	  }
 	__catch(...)
 	  {
-	    this->_M_revalidate_singular();
+	    __this->_M_revalidate_singular();
 	    __throw_exception_again;
 	  }
       }
@@ -346,7 +347,8 @@ namespace __debug
       void
       resize(size_type __sz, const _Tp& __c)
       {
-	this->_M_detach_singular();
+	const list* __this = this;
+	__this->_M_detach_singular();
 
 	// if __sz < size(), invalidate all iterators in [begin + __sz, end())
 	_Base_iterator __victim = _Base::begin();
@@ -363,7 +365,7 @@ namespace __debug
 	  }
 	__catch(...)
 	  {
-	    this->_M_revalidate_singular();
+	    __this->_M_revalidate_singular();
 	    __throw_exception_again;
 	  }
       }
@@ -371,7 +373,8 @@ namespace __debug
       void
       resize(size_type __sz, _Tp __c = _Tp())
       {
-	this->_M_detach_singular();
+	const list* __this = this;
+	__this->_M_detach_singular();
 
 	// if __sz < size(), invalidate all iterators in [begin + __sz, end())
 	_Base_iterator __victim = _Base::begin();
@@ -388,7 +391,7 @@ namespace __debug
 	  }
 	__catch(...)
 	  {
-	    this->_M_revalidate_singular();
+	    __this->_M_revalidate_singular();
 	    __throw_exception_again;
 	  }
       }
diff --git a/libstdc++-v3/include/debug/safe_base.h b/libstdc++-v3/include/debug/safe_base.h
index cf3f170..4462297 100644
--- a/libstdc++-v3/include/debug/safe_base.h
+++ b/libstdc++-v3/include/debug/safe_base.h
@@ -53,8 +53,10 @@ namespace __gnu_debug
 
   public:
     /** The sequence this iterator references; may be NULL to indicate
-	a singular iterator. */
-    _Safe_sequence_base*	_M_sequence;
+     *  a singular iterator. Stored as pointer-to-const because sequence
+     *  could be declared as const.
+     */
+    const _Safe_sequence_base*	_M_sequence;
 
     /** The version number of this iterator. The sentinel value 0 is
      *  used to indicate an invalidated iterator (i.e., one that is
@@ -92,7 +94,7 @@ namespace __gnu_debug
     : _M_sequence(0), _M_version(0), _M_prior(0), _M_next(0)
     {
       if (!std::__is_constant_evaluated())
-	this->_M_attach(const_cast<_Safe_sequence_base*>(__seq), __constant);
+	this->_M_attach(__seq, __constant);
     }
 
     /** Initializes the iterator to reference the same sequence that
@@ -115,7 +117,7 @@ namespace __gnu_debug
 
     /** For use in _Safe_iterator. */
     __gnu_cxx::__mutex&
-    _M_get_mutex() throw ();
+    _M_get_mutex() _GLIBCXX_USE_NOEXCEPT;
 
     /** Attaches this iterator to the given sequence, detaching it
      *	from whatever sequence it was attached to originally. If the
@@ -123,11 +125,12 @@ namespace __gnu_debug
      *	unattached.
      */
     void
-    _M_attach(_Safe_sequence_base* __seq, bool __constant);
+    _M_attach(const _Safe_sequence_base* __seq, bool __constant);
 
     /** Likewise, but not thread-safe. */
     void
-    _M_attach_single(_Safe_sequence_base* __seq, bool __constant) throw ();
+    _M_attach_single(const _Safe_sequence_base* __seq,
+		     bool __constant) _GLIBCXX_USE_NOEXCEPT;
 
     /** Detach the iterator for whatever sequence it is attached to,
      *	if any.
@@ -135,10 +138,23 @@ namespace __gnu_debug
     void
     _M_detach();
 
+#if !_GLIBCXX_INLINE_VERSION
+  private:
+    /***************************************************************/
+    /** Not-const method preserved for abi backward compatibility. */
+    void
+    _M_attach(_Safe_sequence_base* __seq, bool __constant);
+
+    void
+    _M_attach_single(_Safe_sequence_base* __seq,
+		     bool __constant) _GLIBCXX_USE_NOEXCEPT;
+    /***************************************************************/
+#endif
+
   public:
     /** Likewise, but not thread-safe. */
     void
-    _M_detach_single() throw ();
+    _M_detach_single() _GLIBCXX_USE_NOEXCEPT;
 
     /** Determines if we are attached to the given sequence. */
     bool
@@ -147,13 +163,13 @@ namespace __gnu_debug
 
     /** Is this iterator singular? */
     _GLIBCXX_PURE bool
-    _M_singular() const throw ();
+    _M_singular() const _GLIBCXX_USE_NOEXCEPT;
 
     /** Can we compare this iterator to the given iterator @p __x?
 	Returns true if both iterators are nonsingular and reference
 	the same sequence. */
     _GLIBCXX_PURE bool
-    _M_can_compare(const _Safe_iterator_base& __x) const throw ();
+    _M_can_compare(const _Safe_iterator_base& __x) const _GLIBCXX_USE_NOEXCEPT;
 
     /** Invalidate the iterator, making it singular. */
     void
@@ -162,11 +178,11 @@ namespace __gnu_debug
 
     /** Reset all member variables */
     void
-    _M_reset() throw ();
+    _M_reset() _GLIBCXX_USE_NOEXCEPT;
 
     /** Unlink itself */
     void
-    _M_unlink() throw ()
+    _M_unlink() _GLIBCXX_USE_NOEXCEPT
     {
       if (_M_prior)
 	_M_prior->_M_next = _M_next;
@@ -246,14 +262,14 @@ namespace __gnu_debug
 
     /** Detach all iterators, leaving them singular. */
     void
-    _M_detach_all();
+    _M_detach_all() const;
 
     /** Detach all singular iterators.
      *  @post for all iterators i attached to this sequence,
      *   i->_M_version == _M_version.
      */
     void
-    _M_detach_singular();
+    _M_detach_singular() const;
 
     /** Revalidates all attached singular iterators.  This method may
      *  be used to validate iterators that were invalidated before
@@ -261,7 +277,7 @@ namespace __gnu_debug
      *  valid again).
      */
     void
-    _M_revalidate_singular();
+    _M_revalidate_singular() const;
 
     /** Swap this sequence with the given sequence. This operation
      *  also swaps ownership of the iterators, so that when the
@@ -269,11 +285,11 @@ namespace __gnu_debug
      *  one container now reference the other container.
      */
     void
-    _M_swap(_Safe_sequence_base& __x) _GLIBCXX_USE_NOEXCEPT;
+    _M_swap(const _Safe_sequence_base& __x) const _GLIBCXX_USE_NOEXCEPT;
 
     /** For use in _Safe_sequence. */
     __gnu_cxx::__mutex&
-    _M_get_mutex() throw ();
+    _M_get_mutex() const _GLIBCXX_USE_NOEXCEPT;
 
     /** Invalidates all iterators. */
     void
@@ -281,21 +297,42 @@ namespace __gnu_debug
     { if (++_M_version == 0) _M_version = 1; }
 
   private:
+#if !_GLIBCXX_INLINE_VERSION
+    /***************************************************************/
+    /** Not-const method preserved for abi backward compatibility. */
+    void
+    _M_detach_all();
+
+    void
+    _M_detach_singular();
+
+    void
+    _M_revalidate_singular();
+
+    void
+    _M_swap(_Safe_sequence_base& __x) _GLIBCXX_USE_NOEXCEPT;
+
+    __gnu_cxx::__mutex&
+    _M_get_mutex() _GLIBCXX_USE_NOEXCEPT;
+    /***************************************************************/
+#endif
+
     /** Attach an iterator to this sequence. */
     void
-    _M_attach(_Safe_iterator_base* __it, bool __constant);
+    _M_attach(_Safe_iterator_base* __it, bool __constant) const;
 
     /** Likewise but not thread safe. */
     void
-    _M_attach_single(_Safe_iterator_base* __it, bool __constant) throw ();
+    _M_attach_single(_Safe_iterator_base* __it,
+		     bool __constant) const _GLIBCXX_USE_NOEXCEPT;
 
     /** Detach an iterator from this sequence */
     void
-    _M_detach(_Safe_iterator_base* __it);
+    _M_detach(_Safe_iterator_base* __it) const;
 
     /** Likewise but not thread safe. */
     void
-    _M_detach_single(_Safe_iterator_base* __it) throw ();
+    _M_detach_single(_Safe_iterator_base* __it) const _GLIBCXX_USE_NOEXCEPT;
   };
 } // namespace __gnu_debug
 
diff --git a/libstdc++-v3/include/debug/safe_container.h b/libstdc++-v3/include/debug/safe_container.h
index cb1e69a..3341806 100644
--- a/libstdc++-v3/include/debug/safe_container.h
+++ b/libstdc++-v3/include/debug/safe_container.h
@@ -44,9 +44,9 @@ namespace __gnu_debug
       typedef _SafeBase<_SafeContainer> _Base;
 
       _GLIBCXX20_CONSTEXPR
-      _SafeContainer&
-      _M_cont() _GLIBCXX_NOEXCEPT
-      { return *static_cast<_SafeContainer*>(this); }
+      const _SafeContainer&
+      _M_cont() const _GLIBCXX_NOEXCEPT
+      { return *static_cast<const _SafeContainer*>(this); }
 
     protected:
 #if __cplusplus >= 201103L
@@ -56,6 +56,11 @@ namespace __gnu_debug
 
     private:
       _GLIBCXX20_CONSTEXPR
+      void
+      _M_swap_base(const _Safe_container& __x) const noexcept
+      { _Base::_M_swap(__x); }
+
+      _GLIBCXX20_CONSTEXPR
       _Safe_container(_Safe_container&& __x, const _Alloc&, std::true_type)
       : _Safe_container(std::move(__x))
       { }
@@ -67,7 +72,7 @@ namespace __gnu_debug
 	if (!std::__is_constant_evaluated())
 	  {
 	    if (__x._M_cont().get_allocator() == __a)
-	      _Base::_M_swap(__x);
+	      _M_swap_base(__x);
 	    else
 	      __x._M_invalidate_all();
 	  }
@@ -115,12 +120,12 @@ namespace __gnu_debug
 	    bool __xfer_memory = _Alloc_traits::_S_propagate_on_move_assign()
 	      || _M_cont().get_allocator() == __x._M_cont().get_allocator();
 	    if (__xfer_memory)
-	      _Base::_M_swap(__x);
+	      _M_swap_base(__x);
 	    else
 	      this->_M_invalidate_all();
 	  }
 	else
-	  _Base::_M_swap(__x);
+	  _M_swap_base(__x);
 
 	__x._M_invalidate_all();
 	return *this;
@@ -128,7 +133,7 @@ namespace __gnu_debug
 
       _GLIBCXX20_CONSTEXPR
       void
-      _M_swap(_Safe_container& __x) noexcept
+      _M_swap(const _Safe_container& __x) const noexcept
       {
 	if (_IsCxx11AllocatorAware)
 	  {
@@ -139,8 +144,12 @@ namespace __gnu_debug
 					   __x._M_cont()._M_base());
 	  }
 
-	_Base::_M_swap(__x);
+	_M_swap_base(__x);
       }
+#else
+      void
+      _M_swap(const _Safe_container& __x) const throw()
+      { _Base::_M_swap(__x); }
 #endif
     };
 
diff --git a/libstdc++-v3/include/debug/safe_iterator.h b/libstdc++-v3/include/debug/safe_iterator.h
index 7c56338..e0b1b46 100644
--- a/libstdc++-v3/include/debug/safe_iterator.h
+++ b/libstdc++-v3/include/debug/safe_iterator.h
@@ -224,7 +224,7 @@ namespace __gnu_debug
 			      _M_message(__msg_init_copy_singular)
 			      ._M_iterator(*this, "this")
 			      ._M_iterator(__x, "other"));
-	_Safe_sequence_base* __seq = __x._M_sequence;
+	const _Safe_sequence_base* __seq = __x._M_sequence;
 	__x._M_detach();
 	std::swap(base(), __x.base());
 	_M_attach(__seq);
@@ -445,12 +445,12 @@ namespace __gnu_debug
 
       /** Attach iterator to the given sequence. */
       void
-      _M_attach(_Safe_sequence_base* __seq)
+      _M_attach(const _Safe_sequence_base* __seq)
       { _Safe_base::_M_attach(__seq, _S_constant()); }
 
       /** Likewise, but not thread-safe. */
       void
-      _M_attach_single(_Safe_sequence_base* __seq)
+      _M_attach_single(const _Safe_sequence_base* __seq)
       { _Safe_base::_M_attach_single(__seq, _S_constant()); }
 
       /// Is the iterator dereferenceable?
@@ -500,7 +500,13 @@ namespace __gnu_debug
       typename __gnu_cxx::__conditional_type<
 	_IsConstant::__value, const _Sequence*, _Sequence*>::__type
       _M_get_sequence() const
-      { return static_cast<_Sequence*>(_M_sequence); }
+      {
+	// Looks like not const-correct, but if _IsConstant the constness
+	// is restored when returning the sequence pointer and if not
+	// _IsConstant we are allowed to remove constness.
+	return static_cast<_Sequence*>
+	  (const_cast<_Safe_sequence_base*>(_M_sequence));
+      }
 
       // Get distance to __rhs.
       typename _Distance_traits<_Iterator>::__type
diff --git a/libstdc++-v3/include/debug/safe_local_iterator.h b/libstdc++-v3/include/debug/safe_local_iterator.h
index c84f4f1..47b3a80 100644
--- a/libstdc++-v3/include/debug/safe_local_iterator.h
+++ b/libstdc++-v3/include/debug/safe_local_iterator.h
@@ -52,15 +52,15 @@ namespace __gnu_debug
   /** \brief Safe iterator wrapper.
    *
    *  The class template %_Safe_local_iterator is a wrapper around an
-   *  iterator that tracks the iterator's movement among sequences and
-   *  checks that operations performed on the "safe" iterator are
+   *  iterator that tracks the iterator's movement among unordered containers
+   *  and checks that operations performed on the "safe" iterator are
    *  legal. In additional to the basic iterator operations (which are
    *  validated, and then passed to the underlying iterator),
    *  %_Safe_local_iterator has member functions for iterator invalidation,
-   *  attaching/detaching the iterator from sequences, and querying
+   *  attaching/detaching the iterator from unordered containers, and querying
    *  the iterator's state.
    */
-  template<typename _Iterator, typename _Sequence>
+  template<typename _Iterator, typename _UContainer>
     class _Safe_local_iterator
     : private _Iterator
     , public _Safe_local_iterator_base
@@ -68,28 +68,27 @@ namespace __gnu_debug
       typedef _Iterator _Iter_base;
       typedef _Safe_local_iterator_base _Safe_base;
 
-      typedef typename _Sequence::size_type size_type;
+      typedef typename _UContainer::size_type size_type;
 
       typedef std::iterator_traits<_Iterator> _Traits;
 
-      typedef std::__are_same<
-	typename _Sequence::_Base::const_local_iterator,
-	_Iterator> _IsConstant;
+      using _IsConstant = std::__are_same<
+	typename _UContainer::_Base::const_local_iterator, _Iterator>;
 
-      typedef typename __gnu_cxx::__conditional_type<_IsConstant::__value,
-	typename _Sequence::_Base::local_iterator,
-	typename _Sequence::_Base::const_local_iterator>::__type
-      _OtherIterator;
+      using _OtherIterator = std::__conditional_t<
+	_IsConstant::__value,
+	typename _UContainer::_Base::local_iterator,
+	typename _UContainer::_Base::const_local_iterator>;
 
       typedef _Safe_local_iterator _Self;
-      typedef _Safe_local_iterator<_OtherIterator, _Sequence> _OtherSelf;
+      typedef _Safe_local_iterator<_OtherIterator, _UContainer> _OtherSelf;
 
       struct _Unchecked { };
 
       _Safe_local_iterator(const _Safe_local_iterator& __x,
 			   _Unchecked) noexcept
       : _Iter_base(__x.base())
-      { _M_attach(__x._M_sequence); }
+      { _M_attach(__x._M_safe_container()); }
 
     public:
       typedef _Iterator					iterator_type;
@@ -104,12 +103,13 @@ namespace __gnu_debug
 
       /**
        * @brief Safe iterator construction from an unsafe iterator and
-       * its sequence.
+       * its unordered container.
        *
-       * @pre @p seq is not NULL
+       * @pre @p cont is not NULL
        * @post this is not singular
        */
-      _Safe_local_iterator(_Iterator __i, const _Safe_sequence_base* __cont)
+      _Safe_local_iterator(_Iterator __i,
+			   const _Safe_unordered_container_base* __cont)
       : _Iter_base(__i), _Safe_base(__cont, _S_constant())
       { }
 
@@ -126,7 +126,7 @@ namespace __gnu_debug
 			      _M_message(__msg_init_copy_singular)
 			      ._M_iterator(*this, "this")
 			      ._M_iterator(__x, "other"));
-	_M_attach(__x._M_sequence);
+	_M_attach(__x._M_safe_container());
       }
 
       /**
@@ -141,7 +141,7 @@ namespace __gnu_debug
 			      _M_message(__msg_init_copy_singular)
 			      ._M_iterator(*this, "this")
 			      ._M_iterator(__x, "other"));
-	auto __cont = __x._M_sequence;
+	auto __cont = __x._M_safe_container();
 	__x._M_detach();
 	std::swap(base(), __x.base());
 	_M_attach(__cont);
@@ -156,7 +156,7 @@ namespace __gnu_debug
 	  const _Safe_local_iterator<_MutableIterator,
 	  typename __gnu_cxx::__enable_if<_IsConstant::__value &&
 	    std::__are_same<_MutableIterator, _OtherIterator>::__value,
-					  _Sequence>::__type>& __x) noexcept
+					  _UContainer>::__type>& __x) noexcept
 	: _Iter_base(__x.base())
 	{
 	  // _GLIBCXX_RESOLVE_LIB_DEFECTS
@@ -166,7 +166,7 @@ namespace __gnu_debug
 				_M_message(__msg_init_const_singular)
 				._M_iterator(*this, "this")
 				._M_iterator(__x, "other"));
-	  _M_attach(__x._M_sequence);
+	  _M_attach(__x._M_safe_container());
 	}
 
       /**
@@ -193,7 +193,7 @@ namespace __gnu_debug
 	  {
 	    _M_detach();
 	    base() = __x.base();
-	    _M_attach(__x._M_sequence);
+	    _M_attach(__x._M_safe_container());
 	  }
 
 	return *this;
@@ -225,7 +225,7 @@ namespace __gnu_debug
 	  {
 	    _M_detach();
 	    base() = __x.base();
-	    _M_attach(__x._M_sequence);
+	    _M_attach(__x._M_safe_container());
 	  }
 
 	__x._M_detach();
@@ -318,15 +318,15 @@ namespace __gnu_debug
        */
       operator _Iterator() const { return *this; }
 
-      /** Attach iterator to the given sequence. */
+      /** Attach iterator to the given unordered container. */
       void
-      _M_attach(_Safe_sequence_base* __seq)
-      { _Safe_base::_M_attach(__seq, _S_constant()); }
+      _M_attach(const _Safe_unordered_container_base* __cont)
+      { _Safe_base::_M_attach(__cont, _S_constant()); }
 
       /** Likewise, but not thread-safe. */
       void
-      _M_attach_single(_Safe_sequence_base* __seq)
-      { _Safe_base::_M_attach_single(__seq, _S_constant()); }
+      _M_attach_single(const _Safe_unordered_container_base* __cont)
+      { _Safe_base::_M_attach_single(__cont, _S_constant()); }
 
       /// Is the iterator dereferenceable?
       bool
@@ -353,25 +353,31 @@ namespace __gnu_debug
       typename _Distance_traits<_Iterator>::__type
       _M_get_distance_to(const _Safe_local_iterator& __rhs) const;
 
-      // The sequence this iterator references.
-      typename __gnu_cxx::__conditional_type<
-	_IsConstant::__value, const _Sequence*, _Sequence*>::__type
-      _M_get_sequence() const
-      { return static_cast<_Sequence*>(_M_sequence); }
+      // The unordered container this iterator references.
+      std::__conditional_t<
+	_IsConstant::__value, const _UContainer*, _UContainer*>
+      _M_get_ucontainer() const
+      {
+	// Looks like not const-correct, but if _IsConstant the constness
+	// is restored when returning the container pointer and if not
+	// _IsConstant we are allowed to remove constness.
+	return static_cast<_UContainer*>
+	  (const_cast<_Safe_unordered_container_base*>(_M_safe_container()));
+      }
 
-      /// Is this iterator equal to the sequence's begin(bucket) iterator?
+      /// Is this iterator equal to the container's begin(bucket) iterator?
       bool _M_is_begin() const
-      { return base() == _M_get_sequence()->_M_base().begin(bucket()); }
+      { return base() == _M_get_ucontainer()->_M_base().begin(bucket()); }
 
-      /// Is this iterator equal to the sequence's end(bucket) iterator?
+      /// Is this iterator equal to the container's end(bucket) iterator?
       bool _M_is_end() const
-      { return base() == _M_get_sequence()->_M_base().end(bucket()); }
+      { return base() == _M_get_ucontainer()->_M_base().end(bucket()); }
 
       /// Is this iterator part of the same bucket as the other one?
       template<typename _Other>
 	bool
 	_M_in_same_bucket(const _Safe_local_iterator<_Other,
-						     _Sequence>& __other) const
+						     _UContainer>& __other) const
 	{ return bucket() == __other.bucket(); }
 
       friend inline bool
@@ -404,31 +410,31 @@ namespace __gnu_debug
     };
 
   /** Safe local iterators know how to check if they form a valid range. */
-  template<typename _Iterator, typename _Sequence>
+  template<typename _Iterator, typename _UContainer>
     inline bool
-    __valid_range(const _Safe_local_iterator<_Iterator, _Sequence>& __first,
-		  const _Safe_local_iterator<_Iterator, _Sequence>& __last,
+    __valid_range(const _Safe_local_iterator<_Iterator, _UContainer>& __first,
+		  const _Safe_local_iterator<_Iterator, _UContainer>& __last,
 		  typename _Distance_traits<_Iterator>::__type& __dist_info)
     { return __first._M_valid_range(__last, __dist_info); }
 
-  template<typename _Iterator, typename _Sequence>
+  template<typename _Iterator, typename _UContainer>
     inline bool
-    __valid_range(const _Safe_local_iterator<_Iterator, _Sequence>& __first,
-		  const _Safe_local_iterator<_Iterator, _Sequence>& __last)
+    __valid_range(const _Safe_local_iterator<_Iterator, _UContainer>& __first,
+		  const _Safe_local_iterator<_Iterator, _UContainer>& __last)
     {
       typename _Distance_traits<_Iterator>::__type __dist_info;
       return __first._M_valid_range(__last, __dist_info);
     }
 
 #if __cplusplus < 201103L
-  template<typename _Iterator, typename _Sequence>
-    struct _Unsafe_type<_Safe_local_iterator<_Iterator, _Sequence> >
+  template<typename _Iterator, typename _UContainer>
+    struct _Unsafe_type<_Safe_local_iterator<_Iterator, _UContainer> >
     { typedef _Iterator _Type; };
 #endif
 
-  template<typename _Iterator, typename _Sequence>
+  template<typename _Iterator, typename _UContainer>
     inline _Iterator
-    __unsafe(const _Safe_local_iterator<_Iterator, _Sequence>& __it)
+    __unsafe(const _Safe_local_iterator<_Iterator, _UContainer>& __it)
     { return __it.base(); }
 
 } // namespace __gnu_debug
diff --git a/libstdc++-v3/include/debug/safe_local_iterator.tcc b/libstdc++-v3/include/debug/safe_local_iterator.tcc
index 71e5320..10fec3f 100644
--- a/libstdc++-v3/include/debug/safe_local_iterator.tcc
+++ b/libstdc++-v3/include/debug/safe_local_iterator.tcc
@@ -44,7 +44,7 @@ namespace __gnu_debug
 	  if (__rhs._M_is_end())
 	    return
 	      {
-		_M_get_sequence()->bucket_size(bucket()),
+		_M_get_ucontainer()->bucket_size(bucket()),
 		__dp_exact
 	      };
 
@@ -56,7 +56,7 @@ namespace __gnu_debug
 	  if (__rhs._M_is_begin())
 	    return
 	      {
-		-_M_get_sequence()->bucket_size(bucket()),
+		-_M_get_ucontainer()->bucket_size(bucket()),
 		__dp_exact
 	      };
 
diff --git a/libstdc++-v3/include/debug/safe_sequence.h b/libstdc++-v3/include/debug/safe_sequence.h
index 6b35afa..e10474a 100644
--- a/libstdc++-v3/include/debug/safe_sequence.h
+++ b/libstdc++-v3/include/debug/safe_sequence.h
@@ -114,7 +114,7 @@ namespace __gnu_debug
 	  in the safe ones. */
       template<typename _Predicate>
 	void
-	_M_invalidate_if(_Predicate __pred);
+	_M_invalidate_if(_Predicate __pred) const;
 
       /** Transfers all iterators @c x that reference @c from sequence,
 	  are not singular, and for which @c __pred(x) returns @c
@@ -122,7 +122,8 @@ namespace __gnu_debug
 	  in the safe ones. */
       template<typename _Predicate>
 	void
-	_M_transfer_from_if(_Safe_sequence& __from, _Predicate __pred);
+	_M_transfer_from_if(const _Safe_sequence& __from,
+			    _Predicate __pred) const;
     };
 
   /// Like _Safe_sequence but with a special _M_invalidate_all implementation
@@ -133,12 +134,12 @@ namespace __gnu_debug
     {
     protected:
       void
-      _M_invalidate_all()
+      _M_invalidate_all() const
       {
 	typedef typename _Sequence::const_iterator _Const_iterator;
 	typedef typename _Const_iterator::iterator_type _Base_const_iterator;
 	typedef __gnu_debug::_Not_equal_to<_Base_const_iterator> _Not_equal;
-	const _Sequence& __seq = *static_cast<_Sequence*>(this);
+	const _Sequence& __seq = *static_cast<const _Sequence*>(this);
 	this->_M_invalidate_if(_Not_equal(__seq._M_base().end()));
       }
     };
diff --git a/libstdc++-v3/include/debug/safe_sequence.tcc b/libstdc++-v3/include/debug/safe_sequence.tcc
index 336bf2a..053361d 100644
--- a/libstdc++-v3/include/debug/safe_sequence.tcc
+++ b/libstdc++-v3/include/debug/safe_sequence.tcc
@@ -35,7 +35,7 @@ namespace __gnu_debug
     template<typename _Predicate>
       void
       _Safe_sequence<_Sequence>::
-      _M_invalidate_if(_Predicate __pred)
+      _M_invalidate_if(_Predicate __pred) const
       {
 	typedef typename _Sequence::iterator iterator;
 	typedef typename _Sequence::const_iterator const_iterator;
@@ -66,7 +66,7 @@ namespace __gnu_debug
     template<typename _Predicate>
       void
       _Safe_sequence<_Sequence>::
-      _M_transfer_from_if(_Safe_sequence& __from, _Predicate __pred)
+      _M_transfer_from_if(const _Safe_sequence& __from, _Predicate __pred) const
       {
 	if (this == std::__addressof(__from))
 	  return;
@@ -104,7 +104,7 @@ namespace __gnu_debug
 	    }
 
 	  for (_Safe_iterator_base* __iter2 = __from._M_const_iterators;
-		 __iter2;)
+	       __iter2;)
 	    {
 	      _Safe_iterator_base* __victim_base = __iter2;
 	      const_iterator* __victim =
diff --git a/libstdc++-v3/include/debug/safe_unordered_base.h b/libstdc++-v3/include/debug/safe_unordered_base.h
index 1547f5b..55cf581 100644
--- a/libstdc++-v3/include/debug/safe_unordered_base.h
+++ b/libstdc++-v3/include/debug/safe_unordered_base.h
@@ -49,6 +49,10 @@ namespace __gnu_debug
    */
   class _Safe_local_iterator_base : public _Safe_iterator_base
   {
+  public:
+    const _Safe_unordered_container_base*
+    _M_safe_container() const noexcept;
+
   protected:
     /** Initializes the iterator and makes it singular. */
     _Safe_local_iterator_base()
@@ -61,32 +65,32 @@ namespace __gnu_debug
      *  singular. Otherwise, the iterator will reference @p __seq and
      *  be nonsingular.
      */
-    _Safe_local_iterator_base(const _Safe_sequence_base* __seq, bool __constant)
-    { this->_M_attach(const_cast<_Safe_sequence_base*>(__seq), __constant); }
+    _Safe_local_iterator_base(const _Safe_unordered_container_base* __seq,
+			      bool __constant)
+    { _M_attach(__seq, __constant); }
 
     /** Initializes the iterator to reference the same container that
 	@p __x does. @p __constant is true if this is a constant
 	iterator, and false if it is mutable. */
     _Safe_local_iterator_base(const _Safe_local_iterator_base& __x,
 			      bool __constant)
-    { this->_M_attach(__x._M_sequence, __constant); }
+    { this->_M_attach(__x._M_safe_container(), __constant); }
 
     ~_Safe_local_iterator_base() { this->_M_detach(); }
 
-    _Safe_unordered_container_base*
-    _M_get_container() const noexcept;
-
     /** Attaches this iterator to the given container, detaching it
      *	from whatever container it was attached to originally. If the
      *	new container is the NULL pointer, the iterator is left
      *	unattached.
      */
     void
-    _M_attach(_Safe_sequence_base* __seq, bool __constant);
+    _M_attach(const _Safe_unordered_container_base* __cont,
+	      bool __constant);
 
     /** Likewise, but not thread-safe. */
     void
-    _M_attach_single(_Safe_sequence_base* __seq, bool __constant) throw ();
+    _M_attach_single(const _Safe_unordered_container_base* __cont,
+		     bool __constant) noexcept;
 
     /** Detach the iterator for whatever container it is attached to,
      *	if any.
@@ -96,7 +100,19 @@ namespace __gnu_debug
 
     /** Likewise, but not thread-safe. */
     void
-    _M_detach_single() throw ();
+    _M_detach_single() noexcept;
+
+#if !_GLIBCXX_INLINE_VERSION
+  private:
+    /***************************************************************/
+    /** Not-const method preserved for abi backward compatibility. */
+    void
+    _M_attach(_Safe_sequence_base* __seq, bool __constant);
+
+    void
+    _M_attach_single(_Safe_sequence_base* __seq, bool __constant) noexcept;
+    /***************************************************************/
+#endif
   };
 
   /**
@@ -124,10 +140,10 @@ namespace __gnu_debug
 
   public:
     /// The list of mutable local iterators that reference this container
-    _Safe_iterator_base* _M_local_iterators;
+    mutable _Safe_iterator_base* _M_local_iterators;
 
     /// The list of constant local iterators that reference this container
-    _Safe_iterator_base* _M_const_local_iterators;
+    mutable _Safe_iterator_base* _M_const_local_iterators;
 
   protected:
     // Initialize with a version number of 1 and no iterators
@@ -153,7 +169,7 @@ namespace __gnu_debug
 
     /** Detach all iterators, leaving them singular. */
     void
-    _M_detach_all();
+    _M_detach_all() const;
 
     /** Swap this container with the given container. This operation
      *  also swaps ownership of the iterators, so that when the
@@ -161,25 +177,42 @@ namespace __gnu_debug
      *  one container now reference the other container.
      */
     void
-    _M_swap(_Safe_unordered_container_base& __x) noexcept;
+    _M_swap(const _Safe_unordered_container_base& __x) const noexcept;
 
   private:
+#if !_GLIBCXX_INLINE_VERSION
+    /***************************************************************/
+    /** Not-const method preserved for abi backward compatibility. */
+    void
+    _M_detach_all();
+
+    void
+    _M_swap(_Safe_unordered_container_base& __x) noexcept;
+    /***************************************************************/
+#endif
+
     /** Attach an iterator to this container. */
     void
-    _M_attach_local(_Safe_iterator_base* __it, bool __constant);
+    _M_attach_local(_Safe_iterator_base* __it, bool __constant) const;
 
     /** Likewise but not thread safe. */
     void
-    _M_attach_local_single(_Safe_iterator_base* __it, bool __constant) throw ();
+    _M_attach_local_single(_Safe_iterator_base* __it,
+			   bool __constant) const noexcept;
 
     /** Detach an iterator from this container */
     void
-    _M_detach_local(_Safe_iterator_base* __it);
+    _M_detach_local(_Safe_iterator_base* __it) const;
 
     /** Likewise but not thread safe. */
     void
-    _M_detach_local_single(_Safe_iterator_base* __it) throw ();
+    _M_detach_local_single(_Safe_iterator_base* __it) const noexcept;
   };
+
+  inline const _Safe_unordered_container_base*
+  _Safe_local_iterator_base::
+  _M_safe_container() const noexcept
+  { return static_cast<const _Safe_unordered_container_base*>(_M_sequence); }
 } // namespace __gnu_debug
 
 #endif
diff --git a/libstdc++-v3/include/debug/safe_unordered_container.h b/libstdc++-v3/include/debug/safe_unordered_container.h
index 2ba27db..b67b7e0 100644
--- a/libstdc++-v3/include/debug/safe_unordered_container.h
+++ b/libstdc++-v3/include/debug/safe_unordered_container.h
@@ -62,6 +62,10 @@ namespace __gnu_debug
       _M_cont() noexcept
       { return *static_cast<_Container*>(this); }
 
+      const _Safe_unordered_container*
+      _M_self() const
+      { return this; }
+
     protected:
       void
       _M_invalidate_locals()
diff --git a/libstdc++-v3/include/debug/safe_unordered_container.tcc b/libstdc++-v3/include/debug/safe_unordered_container.tcc
index 6819389..0732e63 100644
--- a/libstdc++-v3/include/debug/safe_unordered_container.tcc
+++ b/libstdc++-v3/include/debug/safe_unordered_container.tcc
@@ -40,7 +40,7 @@ namespace __gnu_debug
 	typedef typename _Container::iterator iterator;
 	typedef typename _Container::const_iterator const_iterator;
 
-	__gnu_cxx::__scoped_lock sentry(this->_M_get_mutex());
+	__gnu_cxx::__scoped_lock sentry(_M_self()->_M_get_mutex());
 	for (_Safe_iterator_base* __iter = _M_iterators; __iter;)
 	  {
 	    iterator* __victim = static_cast<iterator*>(__iter);
@@ -72,7 +72,7 @@ namespace __gnu_debug
 	typedef typename _Container::local_iterator local_iterator;
 	typedef typename _Container::const_local_iterator const_local_iterator;
 
-	__gnu_cxx::__scoped_lock sentry(this->_M_get_mutex());
+	__gnu_cxx::__scoped_lock sentry(_M_self()->_M_get_mutex());
 	for (_Safe_iterator_base* __iter = _M_local_iterators; __iter;)
 	  {
 	    local_iterator* __victim = static_cast<local_iterator*>(__iter);
diff --git a/libstdc++-v3/include/experimental/memory b/libstdc++-v3/include/experimental/memory
index 131e5ac..1b01462 100644
--- a/libstdc++-v3/include/experimental/memory
+++ b/libstdc++-v3/include/experimental/memory
@@ -148,42 +148,42 @@ inline namespace fundamentals_v2
     }; // observer_ptr<>
 
   template<typename _Tp>
-    void
+    constexpr void
     swap(observer_ptr<_Tp>& __p1, observer_ptr<_Tp>& __p2) noexcept
     {
       __p1.swap(__p2);
     }
 
   template<typename _Tp>
-    observer_ptr<_Tp>
+    constexpr observer_ptr<_Tp>
     make_observer(_Tp* __p) noexcept
     {
       return observer_ptr<_Tp>(__p);
     }
 
   template<typename _Tp, typename _Up>
-    bool
+    constexpr bool
     operator==(observer_ptr<_Tp> __p1, observer_ptr<_Up> __p2)
     {
       return __p1.get() == __p2.get();
     }
 
   template<typename _Tp, typename _Up>
-    bool
+    constexpr bool
     operator!=(observer_ptr<_Tp> __p1, observer_ptr<_Up> __p2)
     {
     return !(__p1 == __p2);
     }
 
   template<typename _Tp>
-    bool
+    constexpr bool
     operator==(observer_ptr<_Tp> __p, nullptr_t) noexcept
     {
       return !__p;
     }
 
   template<typename _Tp>
-    bool
+    constexpr bool
     operator==(nullptr_t, observer_ptr<_Tp> __p) noexcept
     {
       return !__p;
@@ -197,14 +197,14 @@ inline namespace fundamentals_v2
     }
 
   template<typename _Tp>
-    bool
+    constexpr bool
     operator!=(nullptr_t, observer_ptr<_Tp> __p) noexcept
     {
       return bool(__p);
     }
 
   template<typename _Tp, typename _Up>
-    bool
+    constexpr bool
     operator<(observer_ptr<_Tp> __p1, observer_ptr<_Up> __p2)
     {
       return std::less<typename common_type<typename add_pointer<_Tp>::type,
@@ -214,21 +214,21 @@ inline namespace fundamentals_v2
     }
 
   template<typename _Tp, typename _Up>
-    bool
+    constexpr bool
     operator>(observer_ptr<_Tp> __p1, observer_ptr<_Up> __p2)
     {
       return __p2 < __p1;
     }
 
   template<typename _Tp, typename _Up>
-    bool
+    constexpr bool
     operator<=(observer_ptr<_Tp> __p1, observer_ptr<_Up> __p2)
     {
       return !(__p2 < __p1);
     }
 
   template<typename _Tp, typename _Up>
-    bool
+    constexpr bool
     operator>=(observer_ptr<_Tp> __p1, observer_ptr<_Up> __p2)
     {
       return !(__p1 < __p2);
diff --git a/libstdc++-v3/include/ext/numeric_traits.h b/libstdc++-v3/include/ext/numeric_traits.h
index 2cd8943..6786dc6 100644
--- a/libstdc++-v3/include/ext/numeric_traits.h
+++ b/libstdc++-v3/include/ext/numeric_traits.h
@@ -126,12 +126,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   _GLIBCXX_INT_N_TRAITS(__GLIBCXX_TYPE_INT_N_3, __GLIBCXX_BITSIZE_INT_N_3)
 #endif
 
-#if defined __STRICT_ANSI__ && defined __SIZEOF_INT128__
-  // In strict modes __is_integer<__int128> is false,
-  // but we still want to define __numeric_traits_integer<__int128>.
-  _GLIBCXX_INT_N_TRAITS(__int128, 128)
-#endif
-
 #undef _GLIBCXX_INT_N_TRAITS
 
 #if __cplusplus >= 201103L
diff --git a/libstdc++-v3/include/precompiled/stdc++.h b/libstdc++-v3/include/precompiled/stdc++.h
index e7d89c9..733a5e5 100644
--- a/libstdc++-v3/include/precompiled/stdc++.h
+++ b/libstdc++-v3/include/precompiled/stdc++.h
@@ -237,6 +237,7 @@
 #endif
 
 #if __cplusplus > 202302L
+#include <inplace_vector>
 #include <text_encoding>
 #include <stdbit.h>
 #include <stdckdint.h>
diff --git a/libstdc++-v3/include/pstl/algorithm_impl.h b/libstdc++-v3/include/pstl/algorithm_impl.h
index 5b1cd20..2080e82 100644
--- a/libstdc++-v3/include/pstl/algorithm_impl.h
+++ b/libstdc++-v3/include/pstl/algorithm_impl.h
@@ -79,7 +79,7 @@ template <class _ForwardIterator, class _Size, class _Function>
 _ForwardIterator
 __for_each_n_it_serial(_ForwardIterator __first, _Size __n, _Function __f)
 {
-    for (; __n > 0; ++__first, --__n)
+    for (; __n > 0; ++__first, (void) --__n)
         __f(__first);
     return __first;
 }
@@ -221,7 +221,7 @@ _ForwardIterator2
 __brick_walk2(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Function __f,
               /*vector=*/std::false_type) noexcept
 {
-    for (; __first1 != __last1; ++__first1, ++__first2)
+    for (; __first1 != __last1; ++__first1, (void) ++__first2)
         __f(*__first1, *__first2);
     return __first2;
 }
@@ -240,7 +240,7 @@ _ForwardIterator2
 __brick_walk2_n(_ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2, _Function __f,
                 /*vector=*/std::false_type) noexcept
 {
-    for (; __n > 0; --__n, ++__first1, ++__first2)
+    for (; __n > 0; --__n, (void) ++__first1, ++__first2)
         __f(*__first1, *__first2);
     return __first2;
 }
@@ -364,7 +364,7 @@ _ForwardIterator3
 __brick_walk3(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
               _ForwardIterator3 __first3, _Function __f, /*vector=*/std::false_type) noexcept
 {
-    for (; __first1 != __last1; ++__first1, ++__first2, ++__first3)
+    for (; __first1 != __last1; ++__first1, (void) ++__first2, ++__first3)
         __f(*__first1, *__first2, *__first3);
     return __first3;
 }
@@ -961,7 +961,7 @@ struct __brick_move_destroy
     {
         using _IteratorValueType = typename std::iterator_traits<_RandomAccessIterator1>::value_type;
 
-        for (; __first != __last; ++__first, ++__result)
+        for (; __first != __last; ++__first, (void) ++__result)
         {
             *__result = std::move(*__first);
             (*__first).~_IteratorValueType();
@@ -1027,7 +1027,7 @@ __brick_calc_mask_1(_ForwardIterator __first, _ForwardIterator __last, bool* __r
     static_assert(__are_random_access_iterators<_ForwardIterator>::value,
                   "Pattern-brick error. Should be a random access iterator.");
 
-    for (; __first != __last; ++__first, ++__mask)
+    for (; __first != __last; ++__first, (void) ++__mask)
     {
         *__mask = __pred(*__first);
         if (*__mask)
@@ -1052,7 +1052,7 @@ void
 __brick_copy_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, bool* __mask,
                      _Assigner __assigner, /*vector=*/std::false_type) noexcept
 {
-    for (; __first != __last; ++__first, ++__mask)
+    for (; __first != __last; ++__first, (void) ++__mask)
     {
         if (*__mask)
         {
@@ -1079,7 +1079,7 @@ void
 __brick_partition_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
                           _OutputIterator2 __out_false, bool* __mask, /*vector=*/std::false_type) noexcept
 {
-    for (; __first != __last; ++__first, ++__mask)
+    for (; __first != __last; ++__first, (void) ++__mask)
     {
         if (*__mask)
         {
@@ -1383,7 +1383,7 @@ __brick_calc_mask_2(_RandomAccessIterator __first, _RandomAccessIterator __last,
                     _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
 {
     _DifferenceType __count = 0;
-    for (; __first != __last; ++__first, ++__mask)
+    for (; __first != __last; ++__first, (void) ++__mask)
     {
         *__mask = !__pred(*__first, *(__first - 1));
         __count += *__mask;
@@ -1483,7 +1483,7 @@ void
 __brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, _BidirectionalIterator __d_last,
                 /*is_vector=*/std::false_type) noexcept
 {
-    for (--__d_last; __first != __last; ++__first, --__d_last)
+    for (--__d_last; __first != __last; ++__first, (void) --__d_last)
     {
         using std::iter_swap;
         iter_swap(__first, __d_last);
@@ -2333,7 +2333,7 @@ __pattern_partial_sort_copy(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&&
                     _RandomAccessIterator1 __it = __first + (__i - __r);
 
                     // 1. Copy elements from input to raw memory
-                    for (_T1* __k = __i; __k != __j; ++__k, ++__it)
+                    for (_T1* __k = __i; __k != __j; ++__k, (void) ++__it)
                     {
                         ::new (__k) _T2(*__it);
                     }
@@ -3648,7 +3648,7 @@ __mismatch_serial(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _Forwar
 #if defined(_PSTL_CPP14_2RANGE_MISMATCH_EQUAL_PRESENT)
     return std::mismatch(__first1, __last1, __first2, __last2, __pred);
 #else
-    for (; __first1 != __last1 && __first2 != __last2 && __pred(*__first1, *__first2); ++__first1, ++__first2)
+    for (; __first1 != __last1 && __first2 != __last2 && __pred(*__first1, *__first2); ++__first1, (void) ++__first2)
     {
     }
     return std::make_pair(__first1, __first2);
diff --git a/libstdc++-v3/include/pstl/memory_impl.h b/libstdc++-v3/include/pstl/memory_impl.h
index 8cb32d0..080b6ca 100644
--- a/libstdc++-v3/include/pstl/memory_impl.h
+++ b/libstdc++-v3/include/pstl/memory_impl.h
@@ -29,7 +29,7 @@ __brick_uninitialized_move(_ForwardIterator __first, _ForwardIterator __last, _O
                            /*vector=*/std::false_type) noexcept
 {
     using _ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
-    for (; __first != __last; ++__first, ++__result)
+    for (; __first != __last; ++__first, (void) ++__result)
     {
         ::new (std::addressof(*__result)) _ValueType(std::move(*__first));
     }
@@ -80,7 +80,7 @@ __brick_uninitialized_copy(_ForwardIterator __first, _ForwardIterator __last, _O
                            /*vector=*/std::false_type) noexcept
 {
     using _ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
-    for (; __first != __last; ++__first, ++__result)
+    for (; __first != __last; ++__first, (void) ++__result)
     {
         ::new (std::addressof(*__result)) _ValueType(*__first);
     }
diff --git a/libstdc++-v3/include/pstl/numeric_impl.h b/libstdc++-v3/include/pstl/numeric_impl.h
index b285a66..af6f6a2 100644
--- a/libstdc++-v3/include/pstl/numeric_impl.h
+++ b/libstdc++-v3/include/pstl/numeric_impl.h
@@ -158,7 +158,7 @@ __brick_transform_scan(_ForwardIterator __first, _ForwardIterator __last, _Outpu
                        _UnaryOperation __unary_op, _Tp __init, _BinaryOperation __binary_op,
                        /*Inclusive*/ std::false_type, /*is_vector=*/std::false_type) noexcept
 {
-    for (; __first != __last; ++__first, ++__result)
+    for (; __first != __last; ++__first, (void) ++__result)
     {
 	_Tp __v = std::move(__init);
         _PSTL_PRAGMA_FORCEINLINE
@@ -175,7 +175,7 @@ __brick_transform_scan(_RandomAccessIterator __first, _RandomAccessIterator __la
                        _UnaryOperation __unary_op, _Tp __init, _BinaryOperation __binary_op,
                        /*Inclusive*/ std::true_type, /*is_vector=*/std::false_type) noexcept
 {
-    for (; __first != __last; ++__first, ++__result)
+    for (; __first != __last; ++__first, (void) ++__result)
     {
         _PSTL_PRAGMA_FORCEINLINE
         __init = __binary_op(__init, __unary_op(*__first));
diff --git a/libstdc++-v3/include/std/array b/libstdc++-v3/include/std/array
index fdcf0b0..12f0109 100644
--- a/libstdc++-v3/include/std/array
+++ b/libstdc++-v3/include/std/array
@@ -381,6 +381,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     { __one.swap(__two); }
 
 #if __cplusplus > 201402L || !defined(__STRICT_ANSI__) // c++1z or gnu++11
+  // _GLIBCXX_RESOLVE_LIB_DEFECTS
+  // 2766. Swapping non-swappable types
   template<typename _Tp, std::size_t _Nm>
     __enable_if_t<!__array_traits<_Tp, _Nm>::_Is_swappable::value>
     swap(array<_Tp, _Nm>&, array<_Tp, _Nm>&) = delete;
diff --git a/libstdc++-v3/include/std/bitset b/libstdc++-v3/include/std/bitset
index 1c1e167..93a03f6 100644
--- a/libstdc++-v3/include/std/bitset
+++ b/libstdc++-v3/include/std/bitset
@@ -720,7 +720,10 @@ _GLIBCXX_BEGIN_NAMESPACE_CONTAINER
 
   namespace __bitset
   {
-#if _GLIBCXX_HOSTED
+#ifdef __cpp_lib_bitset // ...construct from string_view
+    template<typename _CharT>
+      using __string = std::basic_string_view<_CharT>;
+#elif _GLIBCXX_HOSTED
     template<typename _CharT>
       using __string = std::basic_string<_CharT>;
 #else
diff --git a/libstdc++-v3/include/std/charconv b/libstdc++-v3/include/std/charconv
index dda49ce..8cf2c0b 100644
--- a/libstdc++-v3/include/std/charconv
+++ b/libstdc++-v3/include/std/charconv
@@ -390,6 +390,10 @@ _GLIBCXX_TO_CHARS(unsigned __GLIBCXX_TYPE_INT_N_2)
 _GLIBCXX_TO_CHARS(signed __GLIBCXX_TYPE_INT_N_3)
 _GLIBCXX_TO_CHARS(unsigned __GLIBCXX_TYPE_INT_N_3)
 #endif
+#if defined __SIZEOF_INT128__ && defined __STRICT_ANSI__
+_GLIBCXX_TO_CHARS(signed __int128)
+_GLIBCXX_TO_CHARS(unsigned __int128)
+#endif
 #undef _GLIBCXX_TO_CHARS
 
   // _GLIBCXX_RESOLVE_LIB_DEFECTS
diff --git a/libstdc++-v3/include/std/format b/libstdc++-v3/include/std/format
index 46bd5d5..d63c6fc 100644
--- a/libstdc++-v3/include/std/format
+++ b/libstdc++-v3/include/std/format
@@ -1854,20 +1854,6 @@ namespace __format
 					  __align, __nfill, __fill_char);
 	}
 
-#if defined __SIZEOF_INT128__ && defined __STRICT_ANSI__
-      template<typename _Tp>
-	using make_unsigned_t
-	  = typename __conditional_t<(sizeof(_Tp) <= sizeof(long long)),
-				     std::make_unsigned<_Tp>,
-				     type_identity<unsigned __int128>>::type;
-
-      // std::to_chars is not overloaded for int128 in strict mode.
-      template<typename _Int>
-	static to_chars_result
-	to_chars(char* __first, char* __last, _Int __value, int __base)
-	{ return std::__to_chars_i<_Int>(__first, __last, __value, __base); }
-#endif
-
       _Spec<_CharT> _M_spec{};
     };
 
@@ -2998,11 +2984,9 @@ namespace __format
     };
 #endif
 
-#if defined(__SIZEOF_FLOAT128__) && _GLIBCXX_FORMAT_F128 > 1
-  // Reuse __formatter_fp<C>::format<__format::__flt128_t, Out> for __float128.
-  // This formatter is not declared if _GLIBCXX_LONG_DOUBLE_ALT128_COMPAT is true,
-  // as __float128 when present is same type as __ieee128, which may be same as
-  // long double.
+#if defined(__SIZEOF_FLOAT128__) && _GLIBCXX_FORMAT_F128 == 2
+  // Use __formatter_fp<C>::format<__format::__flt128_t, Out> for __float128,
+  // when long double is not 128bit IEEE type.
   template<__format::__char _CharT>
     struct formatter<__float128, _CharT>
     {
@@ -3020,9 +3004,6 @@ namespace __format
 
     private:
       __format::__formatter_fp<_CharT> _M_f;
-
-      static_assert( !is_same_v<__float128, long double>,
-		     "This specialization should not be used for long double" );
     };
 #endif
 
@@ -6035,13 +6016,13 @@ namespace __format
 
       constexpr void
       set_separator(basic_string_view<_CharT> __sep) noexcept
-	requires (!_S_range_format_is_string)
+	requires (format_kind<_Rg> == range_format::sequence)
       { _M_under.set_separator(__sep); }
 
       constexpr void
       set_brackets(basic_string_view<_CharT> __open,
 		   basic_string_view<_CharT> __close) noexcept
-	requires (!_S_range_format_is_string)
+	requires (format_kind<_Rg> == range_format::sequence)
       { _M_under.set_brackets(__open, __close); }
 
       // We deviate from standard, that declares this as template accepting
diff --git a/libstdc++-v3/include/std/inplace_vector b/libstdc++-v3/include/std/inplace_vector
new file mode 100644
index 0000000..b5a81be
--- /dev/null
+++ b/libstdc++-v3/include/std/inplace_vector
@@ -0,0 +1,1379 @@
+// Sequence container with fixed capacity -*- C++ -*-
+
+// Copyright The GNU Toolchain Authors.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+/** @file include/inplace_vector
+ *  This is a Standard C++ Library header.
+ *  @ingroup sequences
+ */
+
+#ifndef _GLIBCXX_INPLACE_VECTOR
+#define _GLIBCXX_INPLACE_VECTOR 1
+
+#pragma GCC system_header
+
+#define __glibcxx_want_inplace_vector
+#include <bits/version.h>
+
+#ifdef __glibcxx_inplace_vector // C++ >= 26 
+#include <compare>
+#include <initializer_list>
+#include <bits/range_access.h>
+#include <bits/ranges_base.h> // borrowed_iterator_t, __detail::__container_compatible_range
+#include <bits/ranges_util.h> // subrange
+#include <bits/ranges_uninitialized.h>
+#include <bits/refwrap.h>
+#include <bits/stl_construct.h>
+#include <bits/stl_uninitialized.h>
+#include <bits/stl_algo.h> // rotate
+
+namespace std _GLIBCXX_VISIBILITY(default)
+{
+_GLIBCXX_BEGIN_NAMESPACE_VERSION
+
+  // [indirect], class template indirect
+  template<typename _Tp, size_t _Nm>
+    class inplace_vector
+    {
+    public:
+
+      // types:
+      using value_type             = _Tp;
+      using pointer                = _Tp*;
+      using const_pointer          = const _Tp*;
+      using reference              = value_type&;
+      using const_reference        = const value_type&;
+      using size_type              = size_t;
+      using difference_type        = ptrdiff_t;
+      using iterator
+	= __gnu_cxx::__normal_iterator<_Tp*, inplace_vector>;
+      using const_iterator
+	= __gnu_cxx::__normal_iterator<const _Tp*, inplace_vector>;
+      using reverse_iterator       = std::reverse_iterator<iterator>;
+      using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+
+      // [containers.sequences.inplace.vector.cons], construct/copy/destroy
+      constexpr
+      inplace_vector() noexcept
+      { _M_init(); }
+
+      constexpr explicit
+      inplace_vector(size_type __n)
+      {
+	_M_init();
+	_S_reserve(__n);
+	std::uninitialized_value_construct_n(data(), __n);
+	_M_size = __n;
+      }
+
+      constexpr
+      inplace_vector(size_type __n, const _Tp& __value)
+      {
+	_M_init();
+	_S_reserve(__n);
+	std::uninitialized_fill_n(data(), __n, __value);
+	_M_size = __n;
+      }
+
+      template<__any_input_iterator _InputIterator>
+	constexpr
+	inplace_vector(_InputIterator __first, _InputIterator __last)
+	: inplace_vector()
+	{
+	  if (const auto __n = _S_distance(__first, __last))
+	    {
+	      _S_reserve(__n);
+	      std::uninitialized_copy(__first, __last, data());
+	      _M_size = __n;
+	    }
+	  else
+	    {
+	      while (__first != __last)
+		emplace_back(*__first++);
+	    }
+	}
+
+      template <__detail::__container_compatible_range<_Tp> _Rg>
+	constexpr
+	inplace_vector(from_range_t, _Rg&& __rg)
+	: inplace_vector()
+	{ append_range(__rg); }
+
+      constexpr
+      inplace_vector(initializer_list<_Tp> __il)
+      {
+	_M_init();
+	_S_reserve(__il.size());
+	std::uninitialized_copy(__il.begin(), __il.end(), data());
+	_M_size = __il.size();
+      }
+
+      inplace_vector(const inplace_vector&)
+	requires is_trivially_copy_constructible_v<_Tp>
+	= default;
+
+      constexpr
+      inplace_vector(const inplace_vector& __other)
+      noexcept(is_nothrow_copy_constructible_v<_Tp>)
+      {
+	_M_init();
+	std::uninitialized_copy(__other.begin(), __other.end(), data());
+	_M_size = __other.size();
+      }
+
+      inplace_vector(inplace_vector&&)
+	requires is_trivially_move_constructible_v<_Tp>
+	= default;
+
+      constexpr
+      inplace_vector(inplace_vector&& __other)
+      noexcept(is_nothrow_move_constructible_v<_Tp>)
+      {
+	_M_init();
+	std::uninitialized_move(__other.begin(), __other.end(), data());
+	_M_size = __other.size();
+      }
+
+      ~inplace_vector()
+	requires is_trivially_destructible_v<_Tp>
+	= default;
+
+      constexpr
+      ~inplace_vector()
+      { clear(); }
+
+      inplace_vector&
+      operator=(const inplace_vector&)
+	requires is_trivially_copy_assignable_v<_Tp>
+		   && is_trivially_copy_constructible_v<_Tp>
+		   && is_trivially_destructible_v<_Tp>
+	= default;
+
+      constexpr inplace_vector&
+      operator=(const inplace_vector& __other)
+      noexcept(is_nothrow_copy_assignable_v<_Tp>
+		 && is_nothrow_copy_constructible_v<_Tp>)
+      {
+	if (std::addressof(__other) != this) [[likely]]
+	  assign(__other.begin(), __other.end());
+	return *this;
+      }
+
+      inplace_vector&
+      operator=(inplace_vector&&)
+	requires is_trivially_move_assignable_v<_Tp>
+		   && is_trivially_move_constructible_v<_Tp>
+		   && is_trivially_destructible_v<_Tp>
+	= default;
+
+      constexpr inplace_vector&
+      operator=(inplace_vector&& __other)
+      noexcept(is_nothrow_move_assignable_v<_Tp>
+		 && is_nothrow_move_constructible_v<_Tp>)
+      {
+	if (std::addressof(__other) != this) [[likely]]
+	  assign(std::make_move_iterator(__other.begin()),
+		 std::make_move_iterator(__other.end()));
+	return *this;
+      }
+
+      constexpr inplace_vector&
+      operator=(initializer_list<_Tp> __il)
+      {
+	assign(__il.begin(), __il.end());
+	return *this;
+      }
+
+      template<__any_input_iterator _InputIterator>
+	constexpr void
+	assign(_InputIterator __first, _InputIterator __last)
+	{
+	  if (const auto __n = _S_distance(__first, __last))
+	    {
+	      _S_reserve(__n);
+	      if (_M_size <= __n)
+		{
+		  for (size_t __i = 0; __i < _M_size; ++__i, (void)++__first)
+		    _M_elems[__i] = *__first;
+		  std::uninitialized_copy(__first, __last, end());
+		}
+	      else
+		std::destroy(std::copy(__first, __last, begin()), end());
+	      _M_size = __n;
+	    }
+	  else
+	    {
+	      size_t __i = 0;
+	      for (;__first != __last && __i < _M_size; ++__first)
+		_M_elems[__i++] = *__first;
+	      if (__first == __last)
+		{
+		  std::_Destroy_n(data() + __i, _M_size - __i);
+		  _M_size = __i;
+		}
+	      else
+		{
+		  while (__first != __last)
+		    emplace_back(*__first++);
+		}
+	    }
+	}
+
+      template<__detail::__container_compatible_range<_Tp> _Rg>
+	constexpr void
+	assign_range(_Rg&& __rg)
+	{
+	  if constexpr (ranges::forward_range<_Rg> || ranges::sized_range<_Rg>)
+	    {
+	      const auto __sz = ranges::distance(__rg);
+	      if (__sz > _Nm)
+		__throw_bad_alloc();
+	      if (__sz <= size())
+		{
+		  ranges::copy_n(ranges::begin(__rg), __sz, data());
+		  std::destroy(data() + __sz, data() + _M_size);
+		}
+	      else
+		{
+		  auto [__in, __out] = ranges::copy_n(
+					 ranges::begin(__rg), _M_size,
+					 data());
+		  ranges::uninitialized_copy(
+		    std::move(__in), ranges::end(__rg),
+		    __out, unreachable_sentinel);
+		}
+	      _M_size = __sz;
+	    }
+	  else
+	    {
+	      auto __in = ranges::begin(__rg);
+	      auto __end = ranges::end(__rg);
+	      size_type __n = 0;
+	      for (; __n < _M_size && __in != __end; ++__in)
+		_M_elems[__n++] = *__in;
+
+	      if (__in == __end)
+		{
+		  std::destroy(data() + __n, data() + _M_size);
+		  _M_size = __n;
+		  return;
+		}
+	      else if (__n < _Nm)
+		{
+		  auto __res = ranges::uninitialized_copy(
+				std::move(__in), __end,
+				data() + __n, data() + _Nm);
+		  _M_size = __res.out - data();
+		  if (__res.in == ranges::end(__rg))
+		    return;
+		}
+	      __throw_bad_alloc();
+	    }
+	}
+
+      constexpr void
+      assign(size_type __n, const _Tp& __u)
+      {
+	_S_reserve(__n);
+	if (_M_size <= __n)
+	  std::uninitialized_fill_n(std::fill_n(data(), _M_size, __u),
+				    __n - _M_size, __u);
+	else
+	  std::destroy_n(std::fill_n(data(), __n, __u), _M_size - __n);
+	_M_size = __n;
+      }
+
+      constexpr void
+      assign(initializer_list<_Tp> __il)
+      { assign(__il.begin(), __il.end()); }
+
+      // iterators
+      [[nodiscard]]
+      constexpr iterator
+      begin() noexcept { return iterator(data()); }
+
+      [[nodiscard]]
+      constexpr const_iterator
+      begin() const noexcept { return const_iterator(data()); }
+
+      [[nodiscard]]
+      constexpr iterator
+      end() noexcept
+      { return iterator(data() + _M_size); }
+
+      [[nodiscard]]
+      constexpr const_iterator
+      end() const noexcept
+      { return const_iterator(data() + _M_size); }
+
+      [[nodiscard]]
+      constexpr reverse_iterator
+      rbegin() noexcept
+      { return reverse_iterator(end()); }
+
+      [[nodiscard]]
+      constexpr const_reverse_iterator
+      rbegin() const noexcept
+      { return const_reverse_iterator(end()); }
+
+      [[nodiscard]]
+      constexpr reverse_iterator
+      rend() noexcept { return reverse_iterator(begin()); }
+
+      [[nodiscard]]
+      constexpr const_reverse_iterator
+      rend() const noexcept { return const_reverse_iterator(begin()); }
+
+      [[nodiscard]]
+      constexpr const_iterator
+      cbegin() const noexcept { return begin(); }
+
+      [[nodiscard]]
+      constexpr const_iterator
+      cend() const noexcept { return end(); }
+
+      [[nodiscard]]
+      constexpr const_reverse_iterator
+      crbegin() const noexcept { return rbegin(); }
+
+      [[nodiscard]]
+      constexpr const_reverse_iterator
+      crend() const noexcept { return rend(); }
+
+      // [containers.sequences.inplace.vector.members] size/capacity
+      [[nodiscard]]
+      constexpr bool
+      empty() const noexcept { return _M_size == 0; }
+
+      [[nodiscard]]
+      constexpr size_type
+      size() const noexcept
+      {
+	if (_M_size > _Nm)
+	  __builtin_unreachable();
+	return _M_size;
+      }
+
+      [[nodiscard]]
+      static constexpr size_type
+      max_size() noexcept { return _Nm; }
+
+      [[nodiscard]]
+      static constexpr size_type
+      capacity() noexcept { return _Nm; }
+
+      constexpr void
+      resize(size_type __n)
+      {
+	_S_reserve(__n);
+	if (__n > _M_size)
+	  std::uninitialized_value_construct_n(data() + _M_size, __n - _M_size);
+	else if (__n < _M_size)
+	  std::destroy_n(data() + __n, _M_size - __n);
+	_M_size = __n;
+      }
+
+      constexpr void
+      resize(size_type __n, const _Tp& __c)
+      {
+	_S_reserve(__n);
+	if (__n > _M_size)
+	  std::uninitialized_fill_n(data() + _M_size, __n - _M_size, __c);
+	else if (__n < _M_size)
+	  std::destroy_n(data() + __n, _M_size - __n);
+	_M_size = __n;
+      }
+
+      static constexpr void
+      reserve(size_type __n)
+      { _S_reserve(__n); }
+
+      static constexpr void
+      shrink_to_fit() { }
+
+      // element access
+      [[nodiscard]]
+      constexpr reference
+      operator[](size_type __n)
+      {
+	__glibcxx_requires_subscript(__n);
+	return _M_elems[__n];
+      }
+
+      [[nodiscard]]
+      constexpr const_reference
+      operator[](size_type __n) const
+      {
+	__glibcxx_requires_subscript(__n);
+	return _M_elems[__n];
+      }
+
+      [[nodiscard]]
+      constexpr const_reference
+      at(size_type __n) const
+      {
+	if (__n >= _M_size)
+	  std::__throw_out_of_range_fmt(__N("inplace_vector::at: __n "
+					    "(which is %zu) "
+					    ">= size() (which is %zu)"),
+					__n, _M_size);
+	return _M_elems[__n];
+      }
+
+      [[nodiscard]]
+      constexpr reference
+      at(size_type __n)
+      {
+	if (__n >= _M_size)
+	  std::__throw_out_of_range_fmt(__N("inplace_vector::at: __n "
+					    "(which is %zu) "
+					    ">= size() (which is %zu)"),
+					__n, _M_size);
+	return _M_elems[__n];
+      }
+
+      [[nodiscard]]
+      constexpr reference
+      front()
+      {
+	__glibcxx_requires_nonempty();
+	return _M_elems[0];
+      }
+
+      [[nodiscard]]
+      constexpr const_reference
+      front() const
+      {
+	__glibcxx_requires_nonempty();
+	return _M_elems[0];
+      }
+
+      [[nodiscard]]
+      constexpr reference
+      back()
+      {
+	__glibcxx_requires_nonempty();
+	return _M_elems[_M_size - 1];
+      }
+
+      [[nodiscard]]
+      constexpr const_reference
+      back() const
+      {
+	__glibcxx_requires_nonempty();
+	return _M_elems[_M_size - 1];
+      }
+
+      // [containers.sequences.inplace.vector.data], data access
+
+      [[nodiscard]]
+      constexpr _Tp*
+      data() noexcept
+      { return static_cast<pointer>(_M_elems); }
+
+      [[nodiscard]]
+      constexpr const _Tp*
+      data() const noexcept
+      { return static_cast<const_pointer>(_M_elems); }
+
+      // [containers.sequences.inplace.vector.modifiers], modifiers
+      template<typename... _Args>
+	constexpr _Tp&
+	emplace_back(_Args&&... __args)
+	{
+	  if (_M_size >= _Nm)
+	    __throw_bad_alloc();
+	  return unchecked_emplace_back(std::forward<_Args>(__args)...);
+	}
+
+      constexpr _Tp&
+      push_back(const _Tp& __x)
+      { return emplace_back(__x); }
+
+      constexpr _Tp&
+      push_back(_Tp&& __x)
+      { return emplace_back(std::move(__x)); }
+
+      template<__detail::__container_compatible_range<_Tp> _Rg>
+	constexpr void
+	append_range(_Rg&& __rg)
+	{
+	  if constexpr (ranges::forward_range<_Rg> || ranges::sized_range<_Rg>)
+	    {
+	      const auto __sz = ranges::distance(__rg);
+	      if (__sz > (_Nm - size()))
+		__throw_bad_alloc();
+	      // Bounded on output range due PR121143
+	      ranges::uninitialized_copy(
+		ranges::begin(__rg), unreachable_sentinel,
+		data() + _M_size, data() + _M_size + __sz);
+	      _M_size += size_type(__sz);
+	    }
+	  else
+	    {
+	      ranges::subrange<pointer> __tail(data() + _M_size, data() + _Nm);
+	      auto [__in, __out] = ranges::uninitialized_copy(__rg, __tail);
+	      _M_size = __out - data();
+	      if (__in != ranges::end(__rg))
+		__throw_bad_alloc();
+	    }
+	}
+
+      constexpr void
+      pop_back()
+      {
+	__glibcxx_requires_nonempty();
+	--_M_size;
+	_M_elems[_M_size].~_Tp();
+      }
+
+      template<typename... _Args>
+	constexpr _Tp*
+	try_emplace_back(_Args&&... __args)
+	{
+	  if (_M_size >= _Nm) [[unlikely]]
+	    return nullptr;
+	  auto& __r = unchecked_emplace_back(std::forward<_Args>(__args)...);
+	  return __builtin_addressof(__r);
+	}
+
+      constexpr _Tp*
+      try_push_back(const _Tp& __x)
+      {
+	if (_M_size >= _Nm) [[unlikely]]
+	  return nullptr;
+	return __builtin_addressof(unchecked_emplace_back(__x));
+      }
+
+      constexpr _Tp*
+      try_push_back(_Tp&& __x)
+      {
+	if (_M_size >= _Nm) [[unlikely]]
+	  return nullptr;
+	return __builtin_addressof(unchecked_emplace_back(std::move(__x)));
+      }
+
+      template<__detail::__container_compatible_range<_Tp> _Rg>
+	constexpr ranges::borrowed_iterator_t<_Rg>
+	try_append_range(_Rg&& __rg)
+	{
+	  if constexpr (ranges::sized_range<_Rg>)
+	    {
+	      auto __n = ranges::distance(__rg);
+	      if (__n == 0) [[unlikely]]
+		return ranges::begin(__rg);
+
+	      const auto __end = data() + _M_size;
+	      const size_t __avail = _Nm - size();
+	      if (__n <= __avail)
+		_M_size += size_type(__n);
+	      else
+		{
+		  __n = __avail;
+		  _M_size = _Nm;
+		}
+	      return ranges::uninitialized_copy_n(
+		       ranges::begin(__rg), __n,
+		       __end, unreachable_sentinel).in;
+	    }
+	  else
+	    {
+	      ranges::subrange<pointer> __tail(data() + _M_size, data() + _Nm);
+	      auto [__in, __out] = ranges::uninitialized_copy(__rg, __tail);
+	      _M_size = __out - data();
+	      return std::move(__in);
+	    }
+	}
+
+      template<typename... _Args>
+	constexpr _Tp&
+	unchecked_emplace_back(_Args&&... __args)
+	{
+	  __glibcxx_assert(_M_size < _Nm);
+	  auto __p = std::construct_at(data() + _M_size,
+				       std::forward<_Args>(__args)...);
+	  ++_M_size;
+	  return *__p;
+	}
+
+      constexpr _Tp&
+      unchecked_push_back(const _Tp& __x)
+      { return unchecked_emplace_back(__x); }
+
+      constexpr _Tp&
+      unchecked_push_back(_Tp&& __x)
+      { return unchecked_emplace_back(std::move(__x)); }
+
+      template<typename... _Args>
+	constexpr iterator
+	emplace(const_iterator __position, _Args&&... __args)
+	{
+	  size_t __b = __position - cbegin(); // elements before position
+	  __glibcxx_assert(__b <= _M_size);
+	  if (_M_size >= _Nm)
+	    __throw_bad_alloc();
+	  iterator __pos = begin() + __b;
+	  std::construct_at(data() + _M_size, std::forward<_Args>(__args)...);
+	  if (_M_size++)
+	    std::rotate(__pos, end() - 1, end());
+	  return __pos;
+	}
+
+      constexpr iterator
+      insert(const_iterator __position, const _Tp& __x)
+      { return emplace(__position, __x); }
+
+      constexpr iterator
+      insert(const_iterator __position, _Tp&& __x)
+      { return emplace(__position, std::move(__x)); }
+
+      constexpr iterator
+      insert(const_iterator __position, size_type __n, const _Tp& __x)
+      {
+	size_t __b = __position - cbegin(); // elements before position
+	__glibcxx_assert(__b <= _M_size);
+	if ((_Nm - _M_size) < __n)
+	  __throw_bad_alloc();
+	iterator __pos = begin() + __b;
+	std::uninitialized_fill_n(data() + _M_size, __n, __x);
+	if (std::__exchange(_M_size, _M_size + __n))
+	  std::rotate(__pos, end() - __n, end());
+	return __pos;
+      }
+
+      template<__any_input_iterator _InputIterator>
+	constexpr iterator
+	insert(const_iterator __position, _InputIterator __first,
+	       _InputIterator __last)
+	{
+	  size_t __b = __position - cbegin(); // elements before position
+	  __glibcxx_assert(__b <= _M_size);
+	  iterator __pos = begin() + __b;
+	  const size_t __s = _M_size;
+	  if (const auto __n = _S_distance(__first, __last))
+	    {
+	      if ((_Nm - _M_size) < __n)
+		__throw_bad_alloc();
+	      std::uninitialized_copy(__first, __last, data() + _M_size);
+	      _M_size += __n;
+	    }
+	  else
+	    {
+	      while (__first != __last)
+		emplace_back(*__first++);
+	    }
+	  if (__s)
+	    std::rotate(__pos, begin() + __s, end());
+	  return __pos;
+	}
+
+      template<__detail::__container_compatible_range<_Tp> _Rg>
+	constexpr iterator
+	insert_range(const_iterator __position, _Rg&& __rg)
+	{
+	  iterator __pos = begin() + (__position - cbegin());
+	  const auto __end = end();
+	  if constexpr (ranges::forward_range<_Rg> || ranges::sized_range<_Rg>)
+	    {
+	      const auto __len = ranges::distance(__rg);
+	      if (__len > (_Nm - size()))
+		__throw_bad_alloc();
+	      if (!__len) [[unlikely]]
+		return __pos;
+
+	      const size_type __n = size_type(__len);
+	      const size_type __num_after = __end - __pos;
+	      if (__num_after >= __n)
+		{
+		  ranges::uninitialized_move(__end - __n, __end,
+					     __end, unreachable_sentinel);
+		  _M_size += __n;
+		  ranges::move_backward(__pos, __end - __n, __end);
+		  ranges::copy(__rg, __pos);
+		}
+	      else if constexpr (ranges::forward_range<_Rg>)
+		{
+		  auto __mid = ranges::next(ranges::begin(__rg), __num_after);
+		  ranges::uninitialized_copy(__mid, ranges::end(__rg),
+					     __end, unreachable_sentinel);
+		  _M_size += __n - __num_after;
+		  ranges::uninitialized_move(__pos, __end,
+					     __pos + __n, unreachable_sentinel);
+		  _M_size += __num_after;
+		  ranges::copy(ranges::begin(__rg), __mid, __pos);
+		}
+	      else
+		{
+		  ranges::uninitialized_copy(
+		    ranges::begin(__rg), ranges::end(__rg),
+		    __end, unreachable_sentinel);
+		  _M_size += __n;
+		  std::rotate(__pos, __end, end());
+		}
+	    }
+	  else
+	    {
+	      append_range(__rg);
+	      std::rotate(__pos, __end, end());
+	    }
+	  return __pos;
+	}
+
+      constexpr iterator
+      insert(const_iterator __position, initializer_list<_Tp> __il)
+      { return insert(__position, __il.begin(), __il.end()); }
+
+      constexpr iterator
+      erase(const_iterator __position)
+      {
+	size_t __n = __position - cbegin();
+	__glibcxx_assert(__n < _M_size);
+	iterator __pos = begin() + __n;
+	std::move(__pos + 1, end(), __pos);
+	pop_back();
+	return __pos;
+      }
+
+      constexpr iterator
+      erase(const_iterator __first, const_iterator __last)
+      {
+	size_t __n = __first - cbegin();
+	size_t __x = __last - __first;
+	__glibcxx_assert(__n <= _M_size);
+	__glibcxx_assert(__x <= _M_size);
+	iterator __pos = begin() + __n;
+	iterator __end = std::move(__pos + __x, end(), __pos);
+	std::destroy_n(__end, __x);
+	_M_size -= __x;
+	return __pos;
+      }
+
+      constexpr void
+      swap(inplace_vector& __x)
+      noexcept(is_nothrow_swappable_v<_Tp> && is_nothrow_move_constructible_v<_Tp>)
+      {
+	inplace_vector* __vs[2]{ this, std::addressof(__x) };
+	const auto __smaller = __vs[__x.size() < size()];
+	const auto __bigger = __vs[__x.size() >= size()];
+	size_type __n = __smaller->size();
+	size_type __n2 = __bigger->size();
+
+	if constexpr (is_nothrow_move_constructible_v<_Tp>)
+	  {
+	    for (size_type __i = __n; __i < __n2; ++__i)
+	      {
+		std::construct_at(__smaller->data() + __i,
+				  std::move(*(__bigger->data() + __i)));
+		std::destroy_at(__bigger->data() + __i);
+	      }
+	  }
+	else
+	  {
+	    std::uninitialized_copy(__bigger->data() + __n,
+				    __bigger->data() + __n2,
+				    __smaller->data() + __n);
+	    std::destroy(__bigger->data() + __n, __bigger->data() + __n2);
+	  }
+	__smaller->_M_size = __n2;
+	__bigger->_M_size = __n;
+
+	using std::swap;
+	for (size_type __i = 0; __i < __n; __i++)
+	  swap(_M_elems[__i], __x._M_elems[__i]);
+      }
+
+      constexpr void
+      clear() noexcept
+      {
+	std::destroy_n(data(), size_t(_M_size));
+	_M_size = 0;
+      }
+
+      constexpr friend bool
+      operator==(const inplace_vector& __x, const inplace_vector& __y)
+      { return std::equal(__x.begin(), __x.end(), __y.begin(), __y.end()); }
+
+      constexpr friend auto
+      operator<=>(const inplace_vector& __x, const inplace_vector& __y)
+      requires requires (const _Tp __t) {
+	{ __t < __t } -> __detail::__boolean_testable;
+      }
+      {
+	return std::lexicographical_compare_three_way(__x.begin(), __x.end(),
+						      __y.begin(), __y.end(),
+						      __detail::__synth3way);
+      }
+
+      constexpr friend void
+      swap(inplace_vector& __x, inplace_vector& __y)
+      noexcept(is_nothrow_swappable_v<_Tp> && is_nothrow_move_constructible_v<_Tp>)
+      { __x.swap(__y); }
+
+    private:
+      union {
+	_Tp _M_elems[_Nm];
+      };
+
+      // Check whether integer type _UInt is wide enough to store _Nm,
+      // so that we use a smaller type for _M_size when that saves space.
+      template<typename _UInt, bool = (alignof(_Tp) <= sizeof(_UInt))>
+	static constexpr bool __fits
+	  = _Nm <= __gnu_cxx::__int_traits<_UInt>::__max;
+
+      // Don't bother using a smaller type if alignment of the array elements
+      // means that it doesn't actually save space.
+      template<typename _UInt>
+	static constexpr bool __fits<_UInt, false> = false;
+
+      static consteval auto __select_size_type()
+      {
+	if constexpr (__fits<unsigned char>)
+	  return (unsigned char)0;
+#if __SHRT_WIDTH__ < __SIZE_WIDTH__
+	else if constexpr (__fits<unsigned short>)
+	  return (unsigned short)0;
+#endif
+#if __INT_WIDTH__ < __SIZE_WIDTH__ && __INT_WIDTH__ > __SHRT_WIDTH__
+	else if constexpr (__fits<unsigned int>)
+	  return 0u;
+#endif
+#if __LONG_WIDTH__ < __SIZE_WIDTH__ && __LONG_WIDTH__ > __INT_WIDTH__
+	else if constexpr (__fits<unsigned long>)
+	  return 0ul;
+#endif
+	else // Just use size_t.
+	  return 0uz;
+      }
+      decltype(__select_size_type()) _M_size = 0;
+
+      constexpr void
+      _M_init()
+      {
+	if !consteval
+	  {
+#if __glibcxx_start_lifetime_as
+	    std::start_lifetime_as_array<_Tp>(data(), _Nm);
+#endif
+	  }
+	else
+	  {
+	    // TODO: use new(_M_elems) _Tp[_Nm]() once PR121068 is fixed
+	    if constexpr (is_trivial_v<_Tp>)
+	      for (size_t __i = 0; __i < _Nm; ++__i)
+		_M_elems[__i] = _Tp();
+	    else
+	      __builtin_unreachable(); // only trivial types are supported at compile time
+	  }
+      }
+
+      static constexpr void
+      _S_reserve(size_t __n)
+      {
+	if (__n > _Nm)
+	  __throw_bad_alloc();
+      }
+
+      template<typename _InputIterator>
+	constexpr static auto
+	_S_distance(_InputIterator __first, _InputIterator __last)
+	{
+	  if constexpr (sized_sentinel_for<_InputIterator, _InputIterator>
+			  || forward_iterator<_InputIterator>)
+	    return (size_type)ranges::distance(__first, __last);
+	  else if constexpr (derived_from<__iter_category_t<_InputIterator>,
+				     forward_iterator_tag>)
+	    return (size_type)std::distance(__first, __last);
+	  else
+	    return false_type{};
+	}
+    };
+
+  // [inplace.vector.special], specialized algorithms
+  template<typename _Tp, size_t _Nm>
+    constexpr void
+    swap(inplace_vector<_Tp, _Nm>& __x, inplace_vector<_Tp, _Nm>& __y)
+    noexcept(noexcept(__x.swap(__y)))
+    { __x.swap(__y); }
+
+  // specialization for zero capacity, that is required to be trivally copyable
+  // and empty regardless of _Tp.
+  template<typename _Tp>
+    class inplace_vector<_Tp, 0>
+    {
+    public:
+      // types:
+      using value_type             = _Tp;
+      using pointer                = _Tp*;
+      using const_pointer          = const _Tp*;
+      using reference              = value_type&;
+      using const_reference        = const value_type&;
+      using size_type              = size_t;
+      using difference_type        = ptrdiff_t;
+      using iterator
+	= __gnu_cxx::__normal_iterator<_Tp*, inplace_vector>;
+      using const_iterator
+	= __gnu_cxx::__normal_iterator<const _Tp*, inplace_vector>;
+      using reverse_iterator       = std::reverse_iterator<iterator>;
+      using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+
+      // [containers.sequences.inplace.vector.cons], construct/copy/destroy
+      inplace_vector() = default;
+
+      constexpr explicit
+      inplace_vector(size_type __n)
+      {
+	if (__n != 0)
+	  __throw_bad_alloc();
+      }
+
+      constexpr
+      inplace_vector(size_type __n, const _Tp& __value)
+      {
+	if (__n != 0)
+	  __throw_bad_alloc();
+      }
+
+      template<__any_input_iterator _InputIterator>
+	constexpr
+	inplace_vector(_InputIterator __first, _InputIterator __last)
+	{
+	  if (__first != __last)
+	    __throw_bad_alloc();
+	}
+
+      template <__detail::__container_compatible_range<_Tp> _Rg>
+	constexpr
+	inplace_vector(from_range_t, _Rg&& __rg)
+	{
+	  if (ranges::begin(__rg) != ranges::end(__rg))
+	    __throw_bad_alloc();
+	}
+
+      constexpr
+      inplace_vector(initializer_list<_Tp> __il)
+      {
+	if (__il.size() != 0)
+	  __throw_bad_alloc();
+      }
+
+      inplace_vector(const inplace_vector&) = default;
+      inplace_vector(inplace_vector&&) = default;
+
+      constexpr
+      ~inplace_vector() = default;
+
+      inplace_vector&
+      operator=(const inplace_vector&) = default;
+
+      inplace_vector&
+      operator=(inplace_vector&&) = default;
+
+      constexpr inplace_vector&
+      operator=(initializer_list<_Tp> __il)
+      {
+	if (__il.size() != 0)
+	  __throw_bad_alloc();
+      }
+
+      template<__any_input_iterator _InputIterator>
+	constexpr void
+	assign(_InputIterator __first, _InputIterator __last)
+	{
+	  if (__first != __last)
+	    __throw_bad_alloc();
+	}
+
+      template<__detail::__container_compatible_range<_Tp> _Rg>
+	constexpr void
+	assign_range(_Rg&& __rg)
+	{
+	  if (ranges::begin(__rg) != ranges::end(__rg))
+	    __throw_bad_alloc();
+	}
+
+      constexpr void
+      assign(size_type __n, const _Tp& __u)
+      {
+	if (__n != 0)
+	  __throw_bad_alloc();
+      }
+
+      constexpr void
+      assign(initializer_list<_Tp> __il)
+      {
+	if (__il.size() != 0)
+	  __throw_bad_alloc();
+      }
+
+      // iterators
+      [[nodiscard]]
+      constexpr iterator
+      begin() noexcept { return iterator(nullptr); }
+
+      [[nodiscard]]
+      constexpr const_iterator
+      begin() const noexcept { return const_iterator(nullptr); }
+
+      [[nodiscard]]
+      constexpr iterator
+      end() noexcept { return iterator(nullptr); }
+
+      [[nodiscard]]
+      constexpr const_iterator
+      end() const noexcept { return const_iterator(nullptr); }
+
+      [[nodiscard]]
+      constexpr reverse_iterator
+      rbegin() noexcept
+      { return reverse_iterator(end()); }
+
+      [[nodiscard]]
+      constexpr const_reverse_iterator
+      rbegin() const noexcept
+      { return const_reverse_iterator(end()); }
+
+      [[nodiscard]]
+      constexpr reverse_iterator
+      rend() noexcept { return reverse_iterator(begin()); }
+
+      [[nodiscard]]
+      constexpr const_reverse_iterator
+      rend() const noexcept { return const_reverse_iterator(begin()); }
+
+      [[nodiscard]]
+      constexpr const_iterator
+      cbegin() const noexcept { return begin(); }
+
+      [[nodiscard]]
+      constexpr const_iterator
+      cend() const noexcept { return end(); }
+
+      [[nodiscard]]
+      constexpr const_reverse_iterator
+      crbegin() const noexcept { return rbegin(); }
+
+      [[nodiscard]]
+      constexpr const_reverse_iterator
+      crend() const noexcept { return rend(); }
+
+      // [containers.sequences.inplace.vector.members] size/capacity
+      [[nodiscard]]
+      constexpr bool
+      empty() const noexcept { return true; }
+
+      [[nodiscard]]
+      constexpr size_type
+      size() const noexcept { return 0; }
+
+      [[nodiscard]]
+      static constexpr size_type
+      max_size() noexcept { return 0; }
+
+      [[nodiscard]]
+      static constexpr size_type
+      capacity() noexcept { return 0; }
+
+      constexpr void
+      resize(size_type __n)
+      {
+	if (__n != 0)
+	  __throw_bad_alloc();
+      }
+
+      constexpr void
+      resize(size_type __n, const _Tp&)
+      {
+	if (__n != 0)
+	  __throw_bad_alloc();
+      }
+
+      static constexpr void
+      reserve(size_type __n)
+      {
+	if (__n != 0)
+	  __throw_bad_alloc();
+      }
+
+      static constexpr void
+      shrink_to_fit() { }
+
+      // element access
+      [[nodiscard,noreturn]]
+      constexpr reference
+      operator[](size_type)
+      { __builtin_trap(); }
+
+      [[nodiscard,noreturn]]
+      constexpr const_reference
+      operator[](size_type) const
+      { __builtin_trap(); }
+
+      [[nodiscard,noreturn]]
+      constexpr const_reference
+      at(size_type __n) const
+      {
+	std::__throw_out_of_range_fmt(__N("inplace_vector::at: __n "
+					  "(which is %zu) "
+					  ">= size() (which is 0)"),
+				      __n);
+      }
+
+      [[nodiscard,noreturn]]
+      constexpr reference
+      at(size_type __n)
+      {
+	std::__throw_out_of_range_fmt(__N("inplace_vector::at: __n "
+					  "(which is %zu) "
+					  ">= size() (which is 0)"),
+				      __n);
+      }
+
+      [[nodiscard,noreturn]]
+      constexpr reference
+      front()
+      { __builtin_trap(); }
+
+      [[nodiscard,noreturn]]
+      constexpr const_reference
+      front() const
+      { __builtin_trap(); }
+
+      [[nodiscard,noreturn]]
+      constexpr reference
+      back()
+      { __builtin_trap(); }
+
+      [[nodiscard,noreturn]]
+      constexpr const_reference
+      back() const
+      { __builtin_trap(); }
+
+      // [containers.sequences.inplace.vector.data], data access
+
+      [[nodiscard]]
+      constexpr _Tp*
+      data() noexcept
+      { return nullptr; }
+
+      [[nodiscard]]
+      constexpr const _Tp*
+      data() const noexcept
+      { return nullptr; }
+
+      // [containers.sequences.inplace.vector.modifiers], modifiers
+      template<typename... _Args>
+	[[noreturn]]
+	constexpr _Tp&
+	emplace_back(_Args&&...)
+	{ __throw_bad_alloc(); }
+
+      [[noreturn]]
+      constexpr _Tp&
+      push_back(const _Tp&)
+      { __throw_bad_alloc(); }
+
+      [[noreturn]]
+      constexpr _Tp&
+      push_back(_Tp&&)
+      { __throw_bad_alloc(); }
+
+      template<__detail::__container_compatible_range<_Tp> _Rg>
+	constexpr void
+	append_range(_Rg&& __rg)
+	{
+	  if (ranges::begin(__rg) != ranges::end(__rg))
+	    __throw_bad_alloc();
+	}
+
+      [[noreturn]]
+      constexpr void
+      pop_back()
+      { __builtin_trap(); }
+
+      template<typename... _Args>
+	constexpr _Tp*
+	try_emplace_back(_Args&&...)
+	{ return nullptr; }
+
+      constexpr _Tp*
+      try_push_back(const _Tp&)
+      { return nullptr; }
+
+      constexpr _Tp*
+      try_push_back(_Tp&&)
+      { return nullptr; }
+
+      template<__detail::__container_compatible_range<_Tp> _Rg>
+	constexpr ranges::borrowed_iterator_t<_Rg>
+	try_append_range(_Rg&& __rg)
+	{ return ranges::begin(__rg); }
+
+      template<typename... _Args>
+	[[noreturn]]
+	constexpr _Tp&
+	unchecked_emplace_back(_Args&&...)
+	{ __builtin_trap(); }
+
+      [[noreturn]]
+      constexpr _Tp&
+      unchecked_push_back(const _Tp&)
+      { __builtin_trap(); }
+
+      [[noreturn]]
+      constexpr _Tp&
+      unchecked_push_back(_Tp&&)
+      { __builtin_trap(); }
+
+      template<typename... _Args>
+	[[noreturn]]
+	constexpr iterator
+	emplace(const_iterator, _Args&&...)
+	{ __throw_bad_alloc(); }
+
+      [[noreturn]]
+      constexpr iterator
+      insert(const_iterator, const _Tp&)
+      { __throw_bad_alloc(); }
+
+      [[noreturn]]
+      constexpr iterator
+      insert(const_iterator, _Tp&&)
+      { __throw_bad_alloc(); }
+
+      constexpr iterator
+      insert(const_iterator, size_type __n, const _Tp&)
+      {
+	if (__n != 0)
+	  __throw_bad_alloc();
+	return begin();
+      }
+
+      template<typename _InputIterator>
+	constexpr iterator
+	insert(const_iterator, _InputIterator __first, _InputIterator __last)
+	{
+	  if (__first != __last)
+	    __throw_bad_alloc();
+	  return begin();
+	}
+
+      template<__detail::__container_compatible_range<_Tp> _Rg>
+	constexpr iterator
+	insert_range(const_iterator, _Rg&& __rg)
+	{
+	  if (ranges::begin(__rg) != ranges::end(__rg))
+	    __throw_bad_alloc();
+	  return begin();
+	}
+
+      constexpr iterator
+      insert(const_iterator, initializer_list<_Tp> __il)
+      {
+	if (__il.size() != 0)
+	  __throw_bad_alloc();
+	return begin();
+      }
+
+      [[noreturn]]
+      constexpr iterator
+      erase(const_iterator)
+      { __builtin_trap(); }
+
+      constexpr iterator
+      erase(const_iterator __first, const_iterator __last)
+      {
+	__glibcxx_assert(__first == __last);
+	return begin();
+      }
+
+      constexpr void
+      swap(inplace_vector& __x)
+      noexcept
+      { }
+
+      constexpr void
+      clear() noexcept
+      { }
+
+      constexpr friend bool
+      operator==(const inplace_vector&, const inplace_vector&)
+      { return true; }
+
+      constexpr friend auto
+      operator<=>(const inplace_vector&, const inplace_vector&)
+      requires requires (const _Tp __t) {
+	{ __t < __t } -> __detail::__boolean_testable;
+      }
+      {	return std::strong_ordering::equal; }
+
+      // n.b. there is not explicit wording requiring that swap for inplace_vector,
+      // with zero size, works even if element type is not swappable. However given
+      // that move operations are required to be present and trivial, it makes sense
+      // to support them.
+      constexpr friend void
+      swap(inplace_vector&, inplace_vector&) noexcept
+      { }
+    };
+
+  template<typename _Tp, size_t _Nm, typename _Predicate>
+    constexpr size_t
+    erase_if(inplace_vector<_Tp, _Nm>& __cont, _Predicate __pred)
+    {
+      using namespace __gnu_cxx;
+      const auto __osz = __cont.size();
+      const auto __end = __cont.end();
+      auto __removed = std::__remove_if(__cont.begin(), __end,
+					__ops::__pred_iter(std::ref(__pred)));
+      if (__removed != __end)
+	{
+	  __cont.erase(__niter_wrap(__cont.begin(), __removed),
+		       __cont.end());
+	  return __osz - __cont.size();
+	}
+      return 0;
+    }
+
+
+  template<typename _Tp, size_t _Nm, typename _Up = _Tp>
+    constexpr  size_t
+    erase(inplace_vector<_Tp, _Nm>& __cont, const _Up& __value)
+    {
+      using namespace __gnu_cxx;
+      const auto __osz = __cont.size();
+      const auto __end = __cont.end();
+      auto __removed = std::__remove_if(__cont.begin(), __end,
+					__ops::__iter_equals_val(__value));
+      if (__removed != __end)
+	{
+	  __cont.erase(__niter_wrap(__cont.begin(), __removed),
+		       __cont.end());
+	  return __osz - __cont.size();
+	}
+      return 0;
+    }
+
+_GLIBCXX_END_NAMESPACE_VERSION
+} // namespace
+
+#endif // __glibcxx_inplace_vector
+#endif // _GLIBCXX_INPLACE_VECTOR
diff --git a/libstdc++-v3/include/std/limits b/libstdc++-v3/include/std/limits
index 2331c25..3567a32 100644
--- a/libstdc++-v3/include/std/limits
+++ b/libstdc++-v3/include/std/limits
@@ -1639,7 +1639,6 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #define __INT_N_U201103(TYPE)
 #endif
 
-#if !defined(__STRICT_ANSI__)
 #ifdef __GLIBCXX_TYPE_INT_N_0
   __INT_N(__GLIBCXX_TYPE_INT_N_0, __GLIBCXX_BITSIZE_INT_N_0,
 	  __INT_N_201103 (__GLIBCXX_TYPE_INT_N_0),
@@ -1661,7 +1660,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  __INT_N_U201103 (__GLIBCXX_TYPE_INT_N_3))
 #endif
 
-#elif defined __STRICT_ANSI__ && defined __SIZEOF_INT128__
+#if defined __STRICT_ANSI__ && defined __SIZEOF_INT128__
   __INT_N(__int128, 128,
 	  __INT_N_201103 (__int128),
 	  __INT_N_U201103 (__int128))
diff --git a/libstdc++-v3/include/std/mdspan b/libstdc++-v3/include/std/mdspan
index c72a640..5e79d4b 100644
--- a/libstdc++-v3/include/std/mdspan
+++ b/libstdc++-v3/include/std/mdspan
@@ -111,9 +111,22 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	}
 
 	template<size_t _OtherRank, typename _GetOtherExtent>
+	  static constexpr bool
+	  _S_is_compatible_extents(_GetOtherExtent __get_extent) noexcept
+	  {
+	    if constexpr (_OtherRank == _S_rank)
+	      for (size_t __i = 0; __i < _S_rank; ++__i)
+		if (_Extents[__i] != dynamic_extent
+		    && !cmp_equal(_Extents[__i], _S_int_cast(__get_extent(__i))))
+		  return false;
+	    return true;
+	  }
+
+	template<size_t _OtherRank, typename _GetOtherExtent>
 	  constexpr void
 	  _M_init_dynamic_extents(_GetOtherExtent __get_extent) noexcept
 	  {
+	    __glibcxx_assert(_S_is_compatible_extents<_OtherRank>(__get_extent));
 	    for (size_t __i = 0; __i < _S_rank_dynamic; ++__i)
 	      {
 		size_t __di = __i;
@@ -146,9 +159,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
 	static constexpr span<const size_t>
 	_S_static_extents(size_t __begin, size_t __end) noexcept
-	{
-	  return {_Extents.data() + __begin, _Extents.data() + __end};
-	}
+	{ return {_Extents.data() + __begin, _Extents.data() + __end}; }
 
 	constexpr span<const _IndexType>
 	_M_dynamic_extents(size_t __begin, size_t __end) const noexcept
@@ -172,10 +183,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       concept
       __valid_static_extent = _Extent == dynamic_extent
 	|| _Extent <= numeric_limits<_IndexType>::max();
-  }
 
-  namespace __mdspan
-  {
     template<typename _Extents>
       constexpr span<const size_t>
       __static_extents(size_t __begin = 0, size_t __end = _Extents::rank())
@@ -186,9 +194,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       constexpr span<const typename _Extents::index_type>
       __dynamic_extents(const _Extents& __exts, size_t __begin = 0,
 			size_t __end = _Extents::rank()) noexcept
-      {
-	return __exts._M_exts._M_dynamic_extents(__begin, __end);
-      }
+      { return __exts._M_exts._M_dynamic_extents(__begin, __end); }
   }
 
   template<typename _IndexType, size_t... _Extents>
@@ -272,18 +278,20 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 		  || sizeof...(_OIndexTypes) == rank_dynamic())
 	constexpr explicit extents(_OIndexTypes... __exts) noexcept
 	: _M_exts(span<const _IndexType, sizeof...(_OIndexTypes)>(
-	    initializer_list{_S_storage::_S_int_cast(__exts)...}))
+	    initializer_list{static_cast<_IndexType>(std::move(__exts))...}))
 	{ }
 
-      template<__mdspan::__valid_index_type<index_type> _OIndexType, size_t _Nm>
-	requires (_Nm == rank() || _Nm == rank_dynamic())
+      template<typename _OIndexType, size_t _Nm>
+	requires __mdspan::__valid_index_type<const _OIndexType&, index_type>
+	  && (_Nm == rank() || _Nm == rank_dynamic())
 	constexpr explicit(_Nm != rank_dynamic())
 	extents(span<_OIndexType, _Nm> __exts) noexcept
 	: _M_exts(span<const _OIndexType, _Nm>(__exts))
 	{ }
 
-      template<__mdspan::__valid_index_type<index_type> _OIndexType, size_t _Nm>
-	requires (_Nm == rank() || _Nm == rank_dynamic())
+      template<typename _OIndexType, size_t _Nm>
+	requires __mdspan::__valid_index_type<const _OIndexType&, index_type>
+	  && (_Nm == rank() || _Nm == rank_dynamic())
 	constexpr explicit(_Nm != rank_dynamic())
 	extents(const array<_OIndexType, _Nm>& __exts) noexcept
 	: _M_exts(span<const _OIndexType, _Nm>(__exts))
@@ -385,13 +393,14 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       __rev_prod(const _Extents& __exts, size_t __r) noexcept
       { return __exts_prod(__exts, __r + 1, __exts.rank()); }
 
+    template<typename _Extents>
+      constexpr typename _Extents::index_type
+      __size(const _Extents& __exts) noexcept
+      { return __fwd_prod(__exts, __exts.rank()); }
+
     template<typename _IndexType, size_t... _Counts>
       auto __build_dextents_type(integer_sequence<size_t, _Counts...>)
 	-> extents<_IndexType, ((void) _Counts, dynamic_extent)...>;
-
-    template<typename _Tp>
-      consteval size_t
-      __dynamic_extent() { return dynamic_extent; }
   }
 
   template<typename _IndexType, size_t _Rank>
@@ -401,7 +410,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
   template<typename... _Integrals>
     requires (is_convertible_v<_Integrals, size_t> && ...)
     explicit extents(_Integrals...) ->
-      extents<size_t, __mdspan::__dynamic_extent<_Integrals>()...>;
+      extents<size_t, __detail::__maybe_static_ext<_Integrals>...>;
 
   struct layout_left
   {
@@ -441,6 +450,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	    _IndexType __mult = 1;
 	    auto __update = [&, __pos = 0u](_IndexType __idx) mutable
 	      {
+		_GLIBCXX_DEBUG_ASSERT(cmp_less(__idx, __exts.extent(__pos)));
 		__res += __idx * __mult;
 		__mult *= __exts.extent(__pos);
 		++__pos;
@@ -509,7 +519,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
     template<typename _Layout, typename _Mapping>
       concept __mapping_of =
-	is_same_v<typename _Layout::mapping<typename _Mapping::extents_type>,
+	is_same_v<typename _Layout::template mapping<typename _Mapping::extents_type>,
 		  _Mapping>;
 
     template<typename _Mapping>
@@ -577,7 +587,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
       constexpr index_type
       required_span_size() const noexcept
-      { return __mdspan::__fwd_prod(_M_extents, extents_type::rank()); }
+      { return __mdspan::__size(_M_extents); }
 
       template<__mdspan::__valid_index_type<index_type>... _Indices>
 	requires (sizeof...(_Indices) == extents_type::rank())
@@ -585,7 +595,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	operator()(_Indices... __indices) const noexcept
 	{
 	  return __mdspan::__linear_index_left(_M_extents,
-	    static_cast<index_type>(__indices)...);
+	    static_cast<index_type>(std::move(__indices))...);
 	}
 
       static constexpr bool
@@ -651,6 +661,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	    auto __update = [&, __pos = __exts.rank()](_IndexType) mutable
 	      {
 		--__pos;
+		_GLIBCXX_DEBUG_ASSERT(cmp_less(__ind_arr[__pos],
+					       __exts.extent(__pos)));
 		__res += __ind_arr[__pos] * __mult;
 		__mult *= __exts.extent(__pos);
 	      };
@@ -714,7 +726,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
       constexpr index_type
       required_span_size() const noexcept
-      { return __mdspan::__fwd_prod(_M_extents, extents_type::rank()); }
+      { return __mdspan::__size(_M_extents); }
 
       template<__mdspan::__valid_index_type<index_type>... _Indices>
 	requires (sizeof...(_Indices) == extents_type::rank())
@@ -722,7 +734,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	operator()(_Indices... __indices) const noexcept
 	{
 	  return __mdspan::__linear_index_right(
-	    _M_extents, static_cast<index_type>(__indices)...);
+	    _M_extents, static_cast<index_type>(std::move(__indices))...);
 	}
 
       static constexpr bool
@@ -822,6 +834,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  {
 	    auto __update = [&, __pos = 0u](_IndexType __idx) mutable
 	      {
+		_GLIBCXX_DEBUG_ASSERT(cmp_less(__idx,
+					       __m.extents().extent(__pos)));
 		__res += __idx * __m.stride(__pos++);
 	      };
 	    (__update(__indices), ...);
@@ -859,7 +873,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       constexpr
       mapping(const mapping&) noexcept = default;
 
-      template<__mdspan::__valid_index_type<index_type> _OIndexType>
+      template<typename _OIndexType>
+	requires __mdspan::__valid_index_type<const _OIndexType&, index_type>
 	constexpr
 	mapping(const extents_type& __exts,
 		span<_OIndexType, extents_type::rank()> __strides) noexcept
@@ -869,7 +884,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	    _M_strides[__i] = index_type(as_const(__strides[__i]));
 	}
 
-      template<__mdspan::__valid_index_type<index_type> _OIndexType>
+      template<typename _OIndexType>
+	requires __mdspan::__valid_index_type<const _OIndexType&, index_type>
 	constexpr
 	mapping(const extents_type& __exts,
 		const array<_OIndexType, extents_type::rank()>& __strides)
@@ -940,7 +956,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	operator()(_Indices... __indices) const noexcept
 	{
 	  return __mdspan::__linear_index_strides(*this,
-	    static_cast<index_type>(__indices)...);
+	    static_cast<index_type>(std::move(__indices))...);
 	}
 
       static constexpr bool
@@ -968,8 +984,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       {
 	if constexpr (!is_always_exhaustive())
 	  {
-	    constexpr auto __rank = extents_type::rank();
-	    auto __size = __mdspan::__fwd_prod(_M_extents, __rank);
+	    auto __size = __mdspan::__size(_M_extents);
 	    if(__size > 0)
 	      return __size == required_span_size();
 	  }
@@ -1035,6 +1050,299 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       { return __p + __i; }
     };
 
+  namespace __mdspan
+  {
+    template<typename _Extents, typename _IndexType, size_t _Nm>
+      constexpr bool
+      __is_multi_index(const _Extents& __exts, span<_IndexType, _Nm> __indices)
+      {
+	static_assert(__exts.rank() == _Nm);
+	for (size_t __i = 0; __i < __exts.rank(); ++__i)
+	  if (__indices[__i] >= __exts.extent(__i))
+	    return false;
+	return true;
+      }
+  }
+
+  template<typename _ElementType, typename _Extents,
+	   typename _LayoutPolicy = layout_right,
+	   typename _AccessorPolicy = default_accessor<_ElementType>>
+    class mdspan
+    {
+      static_assert(!is_array_v<_ElementType>,
+	"ElementType must not be an array type");
+      static_assert(!is_abstract_v<_ElementType>,
+	"ElementType must not be an abstract class type");
+      static_assert(__mdspan::__is_extents<_Extents>,
+	"Extents must be a specialization of std::extents");
+      static_assert(is_same_v<_ElementType,
+			      typename _AccessorPolicy::element_type>);
+
+    public:
+      using extents_type = _Extents;
+      using layout_type = _LayoutPolicy;
+      using accessor_type = _AccessorPolicy;
+      using mapping_type = typename layout_type::template mapping<extents_type>;
+      using element_type = _ElementType;
+      using value_type = remove_cv_t<element_type>;
+      using index_type = typename extents_type::index_type;
+      using size_type = typename extents_type::size_type;
+      using rank_type = typename extents_type::rank_type;
+      using data_handle_type = typename accessor_type::data_handle_type;
+      using reference = typename accessor_type::reference;
+
+      static constexpr rank_type
+      rank() noexcept { return extents_type::rank(); }
+
+      static constexpr rank_type
+      rank_dynamic() noexcept { return extents_type::rank_dynamic(); }
+
+      static constexpr size_t
+      static_extent(rank_type __r) noexcept
+      { return extents_type::static_extent(__r); }
+
+      constexpr index_type
+      extent(rank_type __r) const noexcept { return extents().extent(__r); }
+
+      constexpr
+      mdspan()
+      requires (rank_dynamic() > 0)
+	  && is_default_constructible_v<data_handle_type>
+	  && is_default_constructible_v<mapping_type>
+	  && is_default_constructible_v<accessor_type> = default;
+
+      constexpr
+      mdspan(const mdspan& __other) = default;
+
+      constexpr
+      mdspan(mdspan&& __other) = default;
+
+      template<__mdspan::__valid_index_type<index_type>... _OIndexTypes>
+	requires (sizeof...(_OIndexTypes) == rank()
+		   || sizeof...(_OIndexTypes) == rank_dynamic())
+		 && is_constructible_v<mapping_type, extents_type>
+		 && is_default_constructible_v<accessor_type>
+	constexpr explicit
+	mdspan(data_handle_type __handle, _OIndexTypes... __exts)
+	: _M_accessor(),
+	  _M_mapping(_Extents(static_cast<index_type>(std::move(__exts))...)),
+	  _M_handle(std::move(__handle))
+	{ }
+
+      template<typename _OIndexType, size_t _Nm>
+	requires __mdspan::__valid_index_type<const _OIndexType&, index_type>
+		 && (_Nm == rank() || _Nm == rank_dynamic())
+		 && is_constructible_v<mapping_type, extents_type>
+		 && is_default_constructible_v<accessor_type>
+	constexpr explicit(_Nm != rank_dynamic())
+	mdspan(data_handle_type __handle, span<_OIndexType, _Nm> __exts)
+	: _M_accessor(), _M_mapping(extents_type(__exts)),
+	  _M_handle(std::move(__handle))
+	{ }
+
+      template<typename _OIndexType, size_t _Nm>
+	requires __mdspan::__valid_index_type<const _OIndexType&, index_type>
+		 && (_Nm == rank() || _Nm == rank_dynamic())
+		 && is_constructible_v<mapping_type, extents_type>
+		 && is_default_constructible_v<accessor_type>
+	constexpr explicit(_Nm != rank_dynamic())
+	mdspan(data_handle_type __handle, const array<_OIndexType, _Nm>& __exts)
+	: _M_accessor(), _M_mapping(extents_type(__exts)),
+	  _M_handle(std::move(__handle))
+	{ }
+
+      constexpr
+      mdspan(data_handle_type __handle, const extents_type& __exts)
+      requires is_constructible_v<mapping_type, const extents_type&>
+	       && is_default_constructible_v<accessor_type>
+      : _M_accessor(), _M_mapping(__exts), _M_handle(std::move(__handle))
+      { }
+
+      constexpr
+      mdspan(data_handle_type __handle, const mapping_type& __mapping)
+      requires is_default_constructible_v<accessor_type>
+      : _M_accessor(), _M_mapping(__mapping), _M_handle(std::move(__handle))
+      { }
+
+      constexpr
+      mdspan(data_handle_type __handle, const mapping_type& __mapping,
+	     const accessor_type& __accessor)
+      : _M_accessor(__accessor), _M_mapping(__mapping),
+        _M_handle(std::move(__handle))
+      { }
+
+      template<typename _OElementType, typename _OExtents, typename _OLayout,
+	       typename _OAccessor>
+	requires is_constructible_v<mapping_type,
+	      const typename _OLayout::template mapping<_OExtents>&>
+	  && is_constructible_v<accessor_type, const _OAccessor&>
+	constexpr explicit(!is_convertible_v<
+	    const typename _OLayout::template mapping<_OExtents>&, mapping_type>
+	  || !is_convertible_v<const _OAccessor&, accessor_type>)
+	mdspan(const mdspan<_OElementType, _OExtents, _OLayout, _OAccessor>&
+		 __other)
+	: _M_accessor(__other.accessor()), _M_mapping(__other.mapping()),
+	  _M_handle(__other.data_handle())
+	{
+	  static_assert(is_constructible_v<data_handle_type,
+	      const typename _OAccessor::data_handle_type&>);
+	  static_assert(is_constructible_v<extents_type, _OExtents>);
+	}
+
+      constexpr mdspan&
+      operator=(const mdspan& __other) = default;
+
+      constexpr mdspan&
+      operator=(mdspan&& __other) = default;
+
+      template<__mdspan::__valid_index_type<index_type>... _OIndexTypes>
+	requires (sizeof...(_OIndexTypes) == rank())
+	constexpr reference
+	operator[](_OIndexTypes... __indices) const
+	{
+	  auto __checked_call = [this](auto... __idxs) -> index_type
+	    {
+	      if constexpr (sizeof...(__idxs) > 0)
+		__glibcxx_assert(__mdspan::__is_multi_index(extents(),
+		  span<const index_type, sizeof...(__idxs)>({__idxs...})));
+	      return _M_mapping(__idxs...);
+	    };
+
+	  auto __index = __checked_call(
+	    static_cast<index_type>(std::move(__indices))...);
+	  return _M_accessor.access(_M_handle, __index);
+	}
+
+      template<typename _OIndexType>
+	requires __mdspan::__valid_index_type<const _OIndexType&, index_type>
+	constexpr reference
+	operator[](span<_OIndexType, rank()> __indices) const
+	{
+	  auto __call = [&]<size_t... _Counts>(index_sequence<_Counts...>)
+	    -> reference
+	    { return (*this)[index_type(as_const(__indices[_Counts]))...]; };
+	  return __call(make_index_sequence<rank()>());
+	}
+
+      template<typename _OIndexType>
+	requires __mdspan::__valid_index_type<const _OIndexType&, index_type>
+	constexpr reference
+	operator[](const array<_OIndexType, rank()>& __indices) const
+	{ return (*this)[span<const _OIndexType, rank()>(__indices)]; }
+
+      constexpr size_type
+      size() const noexcept
+      {
+	__glibcxx_assert(cmp_less_equal(_M_mapping.required_span_size(),
+					numeric_limits<size_t>::max()));
+	return size_type(__mdspan::__size(extents()));
+      }
+
+      [[nodiscard]]
+      constexpr bool
+      empty() const noexcept
+      { return __mdspan::__empty(extents()); }
+
+      friend constexpr void
+      swap(mdspan& __x, mdspan& __y) noexcept
+      {
+	using std::swap;
+	swap(__x._M_mapping, __y._M_mapping);
+	swap(__x._M_accessor, __y._M_accessor);
+	swap(__x._M_handle, __y._M_handle);
+      }
+
+      constexpr const extents_type&
+      extents() const noexcept { return _M_mapping.extents(); }
+
+      constexpr const data_handle_type&
+      data_handle() const noexcept { return _M_handle; }
+
+      constexpr const mapping_type&
+      mapping() const noexcept { return _M_mapping; }
+
+      constexpr const accessor_type&
+      accessor() const noexcept { return _M_accessor; }
+
+      // Strengthened noexcept for all `is_*` methods.
+
+      static constexpr bool
+      is_always_unique() noexcept(noexcept(mapping_type::is_always_unique()))
+      { return mapping_type::is_always_unique(); }
+
+      static constexpr bool
+      is_always_exhaustive()
+      noexcept(noexcept(mapping_type::is_always_exhaustive()))
+      { return mapping_type::is_always_exhaustive(); }
+
+      static constexpr bool
+      is_always_strided()
+      noexcept(noexcept(mapping_type::is_always_strided()))
+      { return mapping_type::is_always_strided(); }
+
+      constexpr bool
+      is_unique() const noexcept(noexcept(_M_mapping.is_unique()))
+      { return _M_mapping.is_unique(); }
+
+      constexpr bool
+      is_exhaustive() const noexcept(noexcept(_M_mapping.is_exhaustive()))
+      { return _M_mapping.is_exhaustive(); }
+
+      constexpr bool
+      is_strided() const noexcept(noexcept(_M_mapping.is_strided()))
+      { return _M_mapping.is_strided(); }
+
+      constexpr index_type
+      stride(rank_type __r) const { return _M_mapping.stride(__r); }
+
+    private:
+      [[no_unique_address]] accessor_type _M_accessor = accessor_type();
+      [[no_unique_address]] mapping_type _M_mapping = mapping_type();
+      [[no_unique_address]] data_handle_type _M_handle = data_handle_type();
+    };
+
+  template<typename _CArray>
+    requires is_array_v<_CArray> && (rank_v<_CArray> == 1)
+    mdspan(_CArray&)
+    -> mdspan<remove_all_extents_t<_CArray>,
+	      extents<size_t, extent_v<_CArray, 0>>>;
+
+  template<typename _Pointer>
+    requires is_pointer_v<remove_reference_t<_Pointer>>
+    mdspan(_Pointer&&)
+    -> mdspan<remove_pointer_t<remove_reference_t<_Pointer>>, extents<size_t>>;
+
+  template<typename _ElementType, typename... _Integrals>
+    requires (is_convertible_v<_Integrals, size_t> && ...)
+	      && (sizeof...(_Integrals) > 0)
+    explicit mdspan(_ElementType*, _Integrals...)
+    -> mdspan<_ElementType,
+	      extents<size_t, __detail::__maybe_static_ext<_Integrals>...>>;
+
+  template<typename _ElementType, typename _OIndexType, size_t _Nm>
+    mdspan(_ElementType*, span<_OIndexType, _Nm>)
+    -> mdspan<_ElementType, dextents<size_t, _Nm>>;
+
+  template<typename _ElementType, typename _OIndexType, size_t _Nm>
+    mdspan(_ElementType*, const array<_OIndexType, _Nm>&)
+    -> mdspan<_ElementType, dextents<size_t, _Nm>>;
+
+  template<typename _ElementType, typename _IndexType, size_t... _ExtentsPack>
+    mdspan(_ElementType*, const extents<_IndexType, _ExtentsPack...>&)
+    -> mdspan<_ElementType, extents<_IndexType, _ExtentsPack...>>;
+
+  template<typename _ElementType, typename _MappingType>
+    mdspan(_ElementType*, const _MappingType&)
+    -> mdspan<_ElementType, typename _MappingType::extents_type,
+	      typename _MappingType::layout_type>;
+
+  template<typename _MappingType, typename _AccessorType>
+    mdspan(const typename _AccessorType::data_handle_type&, const _MappingType&,
+	   const _AccessorType&)
+    -> mdspan<typename _AccessorType::element_type,
+	      typename _MappingType::extents_type,
+	      typename _MappingType::layout_type, _AccessorType>;
+
 _GLIBCXX_END_NAMESPACE_VERSION
 }
 #endif
diff --git a/libstdc++-v3/include/std/memory b/libstdc++-v3/include/std/memory
index 1da03b3..763a57e 100644
--- a/libstdc++-v3/include/std/memory
+++ b/libstdc++-v3/include/std/memory
@@ -121,6 +121,7 @@
 #define __glibcxx_want_smart_ptr_for_overwrite
 #define __glibcxx_want_to_address
 #define __glibcxx_want_transparent_operators
+#define __glibcxx_want_smart_ptr_owner_equality
 #include <bits/version.h>
 
 #if __cplusplus >= 201103L && __cplusplus <= 202002L && _GLIBCXX_HOSTED
diff --git a/libstdc++-v3/include/std/optional b/libstdc++-v3/include/std/optional
index cc7af5b..e5051d7 100644
--- a/libstdc++-v3/include/std/optional
+++ b/libstdc++-v3/include/std/optional
@@ -1740,6 +1740,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     noexcept(noexcept(__lhs.swap(__rhs)))
     { __lhs.swap(__rhs); }
 
+  // _GLIBCXX_RESOLVE_LIB_DEFECTS
+  // 2766. Swapping non-swappable types
   template<typename _Tp>
     enable_if_t<!(is_move_constructible_v<_Tp> && is_swappable_v<_Tp>)>
     swap(optional<_Tp>&, optional<_Tp>&) = delete;
diff --git a/libstdc++-v3/include/std/queue b/libstdc++-v3/include/std/queue
index 9052589..1b76088 100644
--- a/libstdc++-v3/include/std/queue
+++ b/libstdc++-v3/include/std/queue
@@ -105,7 +105,10 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	{ return _M_f.format(__a.c, __fc); }
 
     private:
-      // Standard uses formatter<ref_view<_Container>, _CharT>.
+      // Standard uses formatter<ref_view<_Container>, _CharT>, but range_formatter
+      // provides same behavior.
+      // _GLIBCXX_RESOLVE_LIB_DEFECTS
+      // 3881. Incorrect formatting of container adapters backed by std::string
       range_formatter<_Tp, _CharT> _M_f;
     };
 
@@ -136,7 +139,10 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	{ return _M_f.format(__a.c, __fc); }
 
     private:
-      // Standard uses formatter<ref_view<_Container>, _CharT>.
+      // Standard uses formatter<ref_view<_Container>, _CharT>, but range_formatter
+      // provides same behavior.
+      // _GLIBCXX_RESOLVE_LIB_DEFECTS
+      // 3881. Incorrect formatting of container adapters backed by std::string
       range_formatter<_Tp, _CharT> _M_f;
     };
 
diff --git a/libstdc++-v3/include/std/ranges b/libstdc++-v3/include/std/ranges
index f764aa7..efe6296 100644
--- a/libstdc++-v3/include/std/ranges
+++ b/libstdc++-v3/include/std/ranges
@@ -3022,7 +3022,8 @@ namespace views::__adaptor
 	  { _M_satisfy(); }
 
 	  [[no_unique_address]]
-	    __detail::__maybe_present_t<forward_range<_Base>, _Outer_iter> _M_outer;
+	    __detail::__maybe_present_t<forward_range<_Base>, _Outer_iter> _M_outer
+	      = decltype(_M_outer)();
 	  optional<_Inner_iter> _M_inner;
 	  _Parent* _M_parent = nullptr;
 
@@ -3376,7 +3377,8 @@ namespace views::__adaptor
 
 	  [[no_unique_address]]
 	    __detail::__maybe_present_t<forward_range<_Vp>,
-					iterator_t<_Base>> _M_current;
+					iterator_t<_Base>> _M_current
+	      = decltype(_M_current)();
 	  bool _M_trailing_empty = false;
 
 	public:
@@ -7400,7 +7402,8 @@ namespace views::__adaptor
 
     _Parent* _M_parent = nullptr;
     [[no_unique_address]]
-      __detail::__maybe_present_t<forward_range<_Base>, _OuterIter> _M_outer_it;
+      __detail::__maybe_present_t<forward_range<_Base>, _OuterIter> _M_outer_it
+	= decltype(_M_outer_it)();
     variant<_PatternIter, _InnerIter> _M_inner_it;
 
     constexpr _OuterIter&
@@ -9735,7 +9738,7 @@ namespace ranges
     end() requires (!(__detail::__simple_view<_Vs> && ...))
     {
       constexpr auto __n = sizeof...(_Vs);
-      if constexpr ((semiregular<iterator_t<_Vs>> && ...)
+      if constexpr (__detail::__all_forward<false, _Vs...>
 		    && common_range<_Vs...[__n - 1]>)
 	return _Iterator<false>(this, in_place_index<__n - 1>,
 				ranges::end(std::get<__n - 1>(_M_views)));
@@ -9747,7 +9750,7 @@ namespace ranges
     end() const requires (range<const _Vs> && ...) && __detail::__concatable<const _Vs...>
     {
       constexpr auto __n = sizeof...(_Vs);
-      if constexpr ((semiregular<iterator_t<const _Vs>> && ...)
+      if constexpr (__detail::__all_forward<true, _Vs...>
 		    && common_range<const _Vs...[__n - 1]>)
 	return _Iterator<true>(this, in_place_index<__n - 1>,
 			       ranges::end(std::get<__n - 1>(_M_views)));
diff --git a/libstdc++-v3/include/std/span b/libstdc++-v3/include/std/span
index 49ab910..44f9b36 100644
--- a/libstdc++-v3/include/std/span
+++ b/libstdc++-v3/include/std/span
@@ -376,7 +376,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  else
 	    static_assert(_Count <= extent);
 	  using _Sp = span<element_type, _Count>;
-	  return _Sp{ _SizedPtr{this->data()} };
+	  return _Sp(_SizedPtr{this->data()});
 	}
 
       [[nodiscard]]
@@ -384,7 +384,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       first(size_type __count) const noexcept
       {
 	__glibcxx_assert(__count <= size());
-	return { this->data(), __count };
+	return span<element_type>(this->data(), __count);
       }
 
       template<size_t _Count>
@@ -397,7 +397,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  else
 	    static_assert(_Count <= extent);
 	  using _Sp = span<element_type, _Count>;
-	  return _Sp{ _SizedPtr{this->data() + (this->size() - _Count)} };
+	  return _Sp(_SizedPtr{this->data() + (this->size() - _Count)});
 	}
 
       [[nodiscard]]
@@ -405,7 +405,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
       last(size_type __count) const noexcept
       {
 	__glibcxx_assert(__count <= size());
-	return { this->data() + (this->size() - __count), __count };
+	return span<element_type>(this->data() + (this->size() - __count),
+				  __count);
       }
 
       template<size_t _Offset, size_t _Count = dynamic_extent>
@@ -424,7 +425,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	  using _Sp = span<element_type, _S_subspan_extent<_Offset, _Count>()>;
 
 	  if constexpr (_Count == dynamic_extent)
-	    return _Sp{ this->data() + _Offset, this->size() - _Offset };
+	    return _Sp(this->data() + _Offset, this->size() - _Offset);
 	  else
 	    {
 	      if constexpr (_Extent == dynamic_extent)
@@ -437,7 +438,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 		  static_assert(_Count <= extent);
 		  static_assert(_Count <= (extent - _Offset));
 		}
-	      return _Sp{ _SizedPtr{this->data() + _Offset} };
+	      return _Sp(_SizedPtr{this->data() + _Offset});
 	    }
 	}
 
@@ -454,7 +455,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 	    __glibcxx_assert(__count <= size());
 	    __glibcxx_assert(__offset + __count <= size());
 	  }
-	return {this->data() + __offset, __count};
+	return span<element_type>(this->data() + __offset, __count);
       }
 
     private:
@@ -476,6 +477,23 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     };
 
   // deduction guides
+  namespace __detail
+  {
+    template<typename _Tp>
+      concept __integral_constant_like = is_integral_v<decltype(_Tp::value)>
+	&& !is_same_v<bool, remove_const_t<decltype(_Tp::value)>>
+	&& convertible_to<_Tp, decltype(_Tp::value)>
+	&& equality_comparable_with<_Tp, decltype(_Tp::value)>
+	&& bool_constant<_Tp() == _Tp::value>::value
+	&& bool_constant<static_cast<decltype(_Tp::value)>(_Tp()) == _Tp::value>
+	     ::value;
+
+    template<typename _Tp>
+      constexpr size_t __maybe_static_ext = dynamic_extent;
+
+    template<__integral_constant_like _Tp>
+      constexpr size_t __maybe_static_ext<_Tp> = {_Tp::value};
+  }
 
   template<typename _Type, size_t _ArrayExtent>
     span(_Type(&)[_ArrayExtent]) -> span<_Type, _ArrayExtent>;
@@ -489,7 +507,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 
   template<contiguous_iterator _Iter, typename _End>
     span(_Iter, _End)
-      -> span<remove_reference_t<iter_reference_t<_Iter>>>;
+      -> span<remove_reference_t<iter_reference_t<_Iter>>,
+				 __detail::__maybe_static_ext<_End>>;
 
   template<ranges::contiguous_range _Range>
     span(_Range &&)
diff --git a/libstdc++-v3/include/std/tuple b/libstdc++-v3/include/std/tuple
index b39ce71..2e6499e 100644
--- a/libstdc++-v3/include/std/tuple
+++ b/libstdc++-v3/include/std/tuple
@@ -2835,6 +2835,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     { __x.swap(__y); }
 
 #if __cpp_lib_ranges_zip // >= C++23
+  /// Exchange the values of two const tuples (if const elements can be swapped)
   template<typename... _Elements>
     requires (is_swappable_v<const _Elements> && ...)
     constexpr void
@@ -2844,7 +2845,8 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
 #endif // C++23
 
 #if __cplusplus > 201402L || !defined(__STRICT_ANSI__) // c++1z or gnu++11
-  /// Exchange the values of two const tuples (if const elements can be swapped)
+  // _GLIBCXX_RESOLVE_LIB_DEFECTS
+  // 2766. Swapping non-swappable types
   template<typename... _Elements>
     _GLIBCXX20_CONSTEXPR
     typename enable_if<!__and_<__is_swappable<_Elements>...>::value>::type
diff --git a/libstdc++-v3/include/std/type_traits b/libstdc++-v3/include/std/type_traits
index 0554111..ff23544 100644
--- a/libstdc++-v3/include/std/type_traits
+++ b/libstdc++-v3/include/std/type_traits
@@ -464,6 +464,19 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     struct __is_integral_helper<unsigned __GLIBCXX_TYPE_INT_N_3>
     : public true_type { };
 #endif
+
+#if defined __SIZEOF_INT128__ && defined __STRICT_ANSI__
+  __extension__
+  template<>
+    struct __is_integral_helper<__int128>
+    : public true_type { };
+
+  __extension__
+  template<>
+    struct __is_integral_helper<unsigned __int128>
+    : public true_type { };
+#endif
+
   /// @endcond
 
   /// is_integral
@@ -519,7 +532,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     : public true_type { };
 #endif
 
-#if !defined(__STRICT_ANSI__) && defined(_GLIBCXX_USE_FLOAT128)
+#ifdef _GLIBCXX_USE_FLOAT128
   template<>
     struct __is_floating_point_helper<__float128>
     : public true_type { };
@@ -1927,6 +1940,12 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     struct __make_unsigned<__GLIBCXX_TYPE_INT_N_3>
     { using __type = unsigned __GLIBCXX_TYPE_INT_N_3; };
 #endif
+#if defined __SIZEOF_INT128__ && defined __STRICT_ANSI__
+  __extension__
+  template<>
+    struct __make_unsigned<__int128>
+    { using __type = unsigned __int128; };
+#endif
 
   // Select between integral and enum: not possible to be both.
   template<typename _Tp,
@@ -1973,8 +1992,13 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     : __make_unsigned_selector_base
     {
       // With -fshort-enums, an enum may be as small as a char.
+      __extension__
       using _UInts = _List<unsigned char, unsigned short, unsigned int,
-			   unsigned long, unsigned long long>;
+			   unsigned long, unsigned long long
+#ifdef __SIZEOF_INT128__
+			   , unsigned __int128
+#endif
+			  >;
 
       using __unsigned_type = typename __select<sizeof(_Tp), _UInts>::__type;
 
@@ -2087,6 +2111,12 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
     struct __make_signed<unsigned __GLIBCXX_TYPE_INT_N_3>
     { using __type = __GLIBCXX_TYPE_INT_N_3; };
 #endif
+#if defined __SIZEOF_INT128__ && defined __STRICT_ANSI__
+  __extension__
+  template<>
+    struct __make_signed<unsigned __int128>
+    { using __type = __int128; };
+#endif
 
   // Select between integral and enum: not possible to be both.
   template<typename _Tp,
diff --git a/libstdc++-v3/include/std/variant b/libstdc++-v3/include/std/variant
index ec46ff1..2f44f97 100644
--- a/libstdc++-v3/include/std/variant
+++ b/libstdc++-v3/include/std/variant
@@ -1387,6 +1387,8 @@ namespace __detail::__variant
     noexcept(noexcept(__lhs.swap(__rhs)))
     { __lhs.swap(__rhs); }
 
+  // _GLIBCXX_RESOLVE_LIB_DEFECTS
+  // 2766. Swapping non-swappable types
   template<typename... _Types>
     enable_if_t<!((is_move_constructible_v<_Types> && ...)
 		   && (is_swappable_v<_Types> && ...))>
diff --git a/libstdc++-v3/libsupc++/exception b/libstdc++-v3/libsupc++/exception
index 61d4131..fc6f8d9 100644
--- a/libstdc++-v3/libsupc++/exception
+++ b/libstdc++-v3/libsupc++/exception
@@ -38,6 +38,7 @@
 #include <bits/exception.h>
 
 #define __glibcxx_want_uncaught_exceptions
+#define __glibcxx_want_constexpr_exceptions
 #define __glibcxx_want_exception_ptr_cast
 #include <bits/version.h>
 
@@ -57,8 +58,16 @@ namespace std _GLIBCXX_VISIBILITY(default)
   class bad_exception : public exception
   {
   public:
-    bad_exception() _GLIBCXX_USE_NOEXCEPT { }
+    _GLIBCXX26_CONSTEXPR bad_exception() _GLIBCXX_USE_NOEXCEPT { }
 
+#if __cplusplus >= 202400L
+    constexpr virtual ~bad_exception() _GLIBCXX_TXN_SAFE_DYN noexcept {}
+
+    constexpr virtual const char* what() const _GLIBCXX_TXN_SAFE_DYN noexcept
+    {
+      return "std::bad_exception";
+    }
+#else
     // This declaration is not useless:
     // http://gcc.gnu.org/onlinedocs/gcc-3.0.2/gcc_6.html#SEC118
     virtual ~bad_exception() _GLIBCXX_TXN_SAFE_DYN _GLIBCXX_USE_NOEXCEPT;
@@ -66,6 +75,7 @@ namespace std _GLIBCXX_VISIBILITY(default)
     // See comment in eh_exception.cc.
     virtual const char*
     what() const _GLIBCXX_TXN_SAFE_DYN _GLIBCXX_USE_NOEXCEPT;
+#endif
   };
 
   /// If you write a replacement %terminate handler, it must be of this type.
diff --git a/libstdc++-v3/libsupc++/exception.h b/libstdc++-v3/libsupc++/exception.h
index f8bca91..efb5fea 100644
--- a/libstdc++-v3/libsupc++/exception.h
+++ b/libstdc++-v3/libsupc++/exception.h
@@ -61,19 +61,28 @@ namespace std _GLIBCXX_VISIBILITY(default)
   class exception
   {
   public:
-    exception() _GLIBCXX_NOTHROW { }
+    _GLIBCXX26_CONSTEXPR exception() _GLIBCXX_NOTHROW { }
+#if __cplusplus >= 202400L
+    constexpr virtual ~exception() _GLIBCXX_TXN_SAFE_DYN noexcept {}
+#else
     virtual ~exception() _GLIBCXX_TXN_SAFE_DYN _GLIBCXX_NOTHROW;
+#endif
 #if __cplusplus >= 201103L
-    exception(const exception&) = default;
-    exception& operator=(const exception&) = default;
-    exception(exception&&) = default;
-    exception& operator=(exception&&) = default;
+    _GLIBCXX26_CONSTEXPR exception(const exception&) = default;
+    _GLIBCXX26_CONSTEXPR exception& operator=(const exception&) = default;
+    _GLIBCXX26_CONSTEXPR exception(exception&&) = default;
+    _GLIBCXX26_CONSTEXPR exception& operator=(exception&&) = default;
 #endif
 
     /** Returns a C-style character string describing the general cause
      *  of the current error.  */
+#if __cplusplus >= 202400L
+    constexpr virtual const char*
+    what() const _GLIBCXX_TXN_SAFE_DYN noexcept { return "std::exception"; }
+#else
     virtual const char*
     what() const _GLIBCXX_TXN_SAFE_DYN _GLIBCXX_NOTHROW;
+#endif
   };
 
   /// @}
diff --git a/libstdc++-v3/libsupc++/exception_ptr.h b/libstdc++-v3/libsupc++/exception_ptr.h
index a6ff8c0..f673a33 100644
--- a/libstdc++-v3/libsupc++/exception_ptr.h
+++ b/libstdc++-v3/libsupc++/exception_ptr.h
@@ -75,16 +75,17 @@ namespace std _GLIBCXX_VISIBILITY(default)
   exception_ptr current_exception() _GLIBCXX_USE_NOEXCEPT;
 
   template<typename _Ex>
-  exception_ptr make_exception_ptr(_Ex) _GLIBCXX_USE_NOEXCEPT;
+  _GLIBCXX26_CONSTEXPR exception_ptr make_exception_ptr(_Ex)
+  _GLIBCXX_USE_NOEXCEPT;
 
   /// Throw the object pointed to by the exception_ptr.
   void rethrow_exception(exception_ptr) __attribute__ ((__noreturn__));
 
 #if __cpp_lib_exception_ptr_cast >= 202506L
   template<typename _Ex>
-  const _Ex* exception_ptr_cast(const exception_ptr&) noexcept;
+    constexpr const _Ex* exception_ptr_cast(const exception_ptr&) noexcept;
   template<typename _Ex>
-  void exception_ptr_cast(const exception_ptr&&) = delete;
+    void exception_ptr_cast(const exception_ptr&&) = delete;
 #endif
 
   namespace __exception_ptr
@@ -105,7 +106,25 @@ namespace std _GLIBCXX_VISIBILITY(default)
     {
       void* _M_exception_object;
 
+#if __cplusplus >= 202400L
+      constexpr explicit exception_ptr(void* __e) noexcept
+      : _M_exception_object(__e)
+      {
+	if (_M_exception_object)
+	  {
+#if __cpp_if_consteval >= 202106L \
+  && _GLIBCXX_HAS_BUILTIN(__builtin_eh_ptr_adjust_ref)
+	    if consteval {
+	      __builtin_eh_ptr_adjust_ref(_M_exception_object, 1);
+	      return;
+	    }
+#endif
+	    _M_addref();
+	  }
+      }
+#else
       explicit exception_ptr(void* __e) _GLIBCXX_USE_NOEXCEPT;
+#endif
 
       void _M_addref() _GLIBCXX_USE_NOEXCEPT;
       void _M_release() _GLIBCXX_USE_NOEXCEPT;
@@ -115,26 +134,29 @@ namespace std _GLIBCXX_VISIBILITY(default)
       friend exception_ptr std::current_exception() _GLIBCXX_USE_NOEXCEPT;
       friend void std::rethrow_exception(exception_ptr);
       template<typename _Ex>
-      friend exception_ptr std::make_exception_ptr(_Ex) _GLIBCXX_USE_NOEXCEPT;
+      friend _GLIBCXX26_CONSTEXPR exception_ptr std::make_exception_ptr(_Ex)
+	_GLIBCXX_USE_NOEXCEPT;
 #if __cpp_lib_exception_ptr_cast >= 202506L
       template<typename _Ex>
-      friend const _Ex* std::exception_ptr_cast(const exception_ptr&) noexcept;
+	friend constexpr const _Ex*
+	std::exception_ptr_cast(const exception_ptr&) noexcept;
 #endif
 
       const void* _M_exception_ptr_cast(const type_info&) const
 	_GLIBCXX_USE_NOEXCEPT;
 
     public:
-      exception_ptr() _GLIBCXX_USE_NOEXCEPT;
+      _GLIBCXX26_CONSTEXPR exception_ptr() _GLIBCXX_USE_NOEXCEPT;
 
-      exception_ptr(const exception_ptr&) _GLIBCXX_USE_NOEXCEPT;
+      _GLIBCXX26_CONSTEXPR exception_ptr(const exception_ptr&)
+	_GLIBCXX_USE_NOEXCEPT;
 
 #if __cplusplus >= 201103L
-      exception_ptr(nullptr_t) noexcept
+      _GLIBCXX26_CONSTEXPR exception_ptr(nullptr_t) noexcept
       : _M_exception_object(nullptr)
       { }
 
-      exception_ptr(exception_ptr&& __o) noexcept
+      _GLIBCXX26_CONSTEXPR exception_ptr(exception_ptr&& __o) noexcept
       : _M_exception_object(__o._M_exception_object)
       { __o._M_exception_object = nullptr; }
 #endif
@@ -146,11 +168,11 @@ namespace std _GLIBCXX_VISIBILITY(default)
       exception_ptr(__safe_bool) _GLIBCXX_USE_NOEXCEPT;
 #endif
 
-      exception_ptr&
+      _GLIBCXX26_CONSTEXPR exception_ptr&
       operator=(const exception_ptr&) _GLIBCXX_USE_NOEXCEPT;
 
 #if __cplusplus >= 201103L
-      exception_ptr&
+      _GLIBCXX26_CONSTEXPR exception_ptr&
       operator=(exception_ptr&& __o) noexcept
       {
         exception_ptr(static_cast<exception_ptr&&>(__o)).swap(*this);
@@ -158,9 +180,9 @@ namespace std _GLIBCXX_VISIBILITY(default)
       }
 #endif
 
-      ~exception_ptr() _GLIBCXX_USE_NOEXCEPT;
+      _GLIBCXX26_CONSTEXPR ~exception_ptr() _GLIBCXX_USE_NOEXCEPT;
 
-      void
+      _GLIBCXX26_CONSTEXPR void
       swap(exception_ptr&) _GLIBCXX_USE_NOEXCEPT;
 
 #ifdef _GLIBCXX_EH_PTR_COMPAT
@@ -172,13 +194,13 @@ namespace std _GLIBCXX_VISIBILITY(default)
 #endif
 
 #if __cplusplus >= 201103L
-      explicit operator bool() const noexcept
+      _GLIBCXX26_CONSTEXPR explicit operator bool() const noexcept
       { return _M_exception_object; }
 #endif
 
 #if __cpp_impl_three_way_comparison >= 201907L \
       && ! defined _GLIBCXX_EH_PTR_RELOPS_COMPAT
-      friend bool
+      _GLIBCXX26_CONSTEXPR friend bool
       operator==(const exception_ptr&, const exception_ptr&) noexcept = default;
 #else
       friend _GLIBCXX_EH_PTR_USED bool
@@ -198,31 +220,49 @@ namespace std _GLIBCXX_VISIBILITY(default)
     };
 
     _GLIBCXX_EH_PTR_USED
-    inline
+    _GLIBCXX26_CONSTEXPR inline
     exception_ptr::exception_ptr() _GLIBCXX_USE_NOEXCEPT
     : _M_exception_object(0)
     { }
 
     _GLIBCXX_EH_PTR_USED
-    inline
+    _GLIBCXX26_CONSTEXPR inline
     exception_ptr::exception_ptr(const exception_ptr& __other)
     _GLIBCXX_USE_NOEXCEPT
     : _M_exception_object(__other._M_exception_object)
     {
       if (_M_exception_object)
-	_M_addref();
+	{
+#if __cpp_if_consteval >= 202106L \
+  && _GLIBCXX_HAS_BUILTIN(__builtin_eh_ptr_adjust_ref)
+	  if consteval {
+	    __builtin_eh_ptr_adjust_ref(_M_exception_object, 1);
+	    return;
+	  }
+#endif
+	  _M_addref();
+	}
     }
 
     _GLIBCXX_EH_PTR_USED
-    inline
+    _GLIBCXX26_CONSTEXPR inline
     exception_ptr::~exception_ptr() _GLIBCXX_USE_NOEXCEPT
     {
       if (_M_exception_object)
-	_M_release();
+	{
+#if __cpp_if_consteval >= 202106L \
+  && _GLIBCXX_HAS_BUILTIN(__builtin_eh_ptr_adjust_ref)
+	  if consteval {
+	    __builtin_eh_ptr_adjust_ref(_M_exception_object, -1);
+	    return;
+	  }
+#endif
+	  _M_release();
+	}
     }
 
     _GLIBCXX_EH_PTR_USED
-    inline exception_ptr&
+    _GLIBCXX26_CONSTEXPR inline exception_ptr&
     exception_ptr::operator=(const exception_ptr& __other) _GLIBCXX_USE_NOEXCEPT
     {
       exception_ptr(__other).swap(*this);
@@ -230,7 +270,7 @@ namespace std _GLIBCXX_VISIBILITY(default)
     }
 
     _GLIBCXX_EH_PTR_USED
-    inline void
+    _GLIBCXX26_CONSTEXPR inline void
     exception_ptr::swap(exception_ptr &__other) _GLIBCXX_USE_NOEXCEPT
     {
       void *__tmp = _M_exception_object;
@@ -239,7 +279,7 @@ namespace std _GLIBCXX_VISIBILITY(default)
     }
 
     /// @relates exception_ptr
-    inline void
+    _GLIBCXX26_CONSTEXPR inline void
     swap(exception_ptr& __lhs, exception_ptr& __rhs)
     { __lhs.swap(__rhs); }
 
@@ -256,28 +296,41 @@ namespace std _GLIBCXX_VISIBILITY(default)
   using __exception_ptr::swap; // So that std::swap(exp1, exp2) finds it.
 
   /// Obtain an exception_ptr pointing to a copy of the supplied object.
-#if (__cplusplus >= 201103L && __cpp_rtti) || __cpp_exceptions
   template<typename _Ex>
-    exception_ptr
+#if !(__cplusplus >= 201103L && __cpp_rtti) && !__cpp_exceptions
+    // This is always_inline so the linker will never use a useless definition
+    // instead of a working one compiled with RTTI and/or exceptions enabled.
+    __attribute__ ((__always_inline__)) inline
+#endif
+    _GLIBCXX26_CONSTEXPR exception_ptr
     make_exception_ptr(_Ex __ex) _GLIBCXX_USE_NOEXCEPT
     {
 #if __cplusplus >= 201103L && __cpp_rtti
-      using _Ex2 = typename decay<_Ex>::type;
-      void* __e = __cxxabiv1::__cxa_allocate_exception(sizeof(_Ex));
-      (void) __cxxabiv1::__cxa_init_primary_exception(
-	  __e, const_cast<std::type_info*>(&typeid(_Ex)),
-	  __exception_ptr::__dest_thunk<_Ex2>);
-      __try
-	{
-	  ::new (__e) _Ex2(__ex);
-	  return exception_ptr(__e);
-	}
-      __catch(...)
+      // For runtime calls with -frtti enabled we can avoid try-catch overhead.
+      // We can't use this for C++98 because it relies on std::decay.
+#ifdef __glibcxx_constexpr_exceptions
+      if ! consteval
+#endif
 	{
-	  __cxxabiv1::__cxa_free_exception(__e);
-	  return current_exception();
+	  using _Ex2 = typename decay<_Ex>::type;
+	  void* __e = __cxxabiv1::__cxa_allocate_exception(sizeof(_Ex));
+	  (void) __cxxabiv1::__cxa_init_primary_exception(
+	      __e, const_cast<std::type_info*>(&typeid(_Ex)),
+	      __exception_ptr::__dest_thunk<_Ex2>);
+	  __try
+	    {
+	      ::new (__e) _Ex2(__ex);
+	      return exception_ptr(__e);
+	    }
+	  __catch(...)
+	    {
+	      __cxxabiv1::__cxa_free_exception(__e);
+	      return current_exception();
+	    }
 	}
-#else
+#endif
+
+#ifdef __cpp_exceptions
       try
 	{
           throw __ex;
@@ -287,28 +340,46 @@ namespace std _GLIBCXX_VISIBILITY(default)
 	  return current_exception();
 	}
 #endif
+      return exception_ptr();
     }
-#else // no RTTI and no exceptions
-  // This is always_inline so the linker will never use this useless definition
-  // instead of a working one compiled with RTTI and/or exceptions enabled.
-  template<typename _Ex>
-    __attribute__ ((__always_inline__))
-    inline exception_ptr
-    make_exception_ptr(_Ex) _GLIBCXX_USE_NOEXCEPT
-    { return exception_ptr(); }
-#endif
 
 #if __cpp_lib_exception_ptr_cast >= 202506L
   template<typename _Ex>
     [[__gnu__::__always_inline__]]
-    inline const _Ex* exception_ptr_cast(const exception_ptr& __p) noexcept
+    constexpr const _Ex*
+    exception_ptr_cast(const exception_ptr& __p) noexcept
     {
+      static_assert(!std::is_const_v<_Ex>);
+      static_assert(!std::is_reference_v<_Ex>);
+      static_assert(std::is_object_v<_Ex>);
+      static_assert(!std::is_array_v<_Ex>);
+      static_assert(!std::is_pointer_v<_Ex>);
+      static_assert(!std::is_member_pointer_v<_Ex>);
+
 #ifdef __cpp_rtti
-      const type_info &__id = typeid(const _Ex&);
-      return static_cast<const _Ex*>(__p._M_exception_ptr_cast(__id));
-#else
-      return nullptr;
+      // For runtime calls with -frtti enabled we can avoid try-catch overhead.
+      if ! consteval {
+	const type_info &__id = typeid(const _Ex&);
+	return static_cast<const _Ex*>(__p._M_exception_ptr_cast(__id));
+      }
 #endif
+
+#ifdef __cpp_exceptions
+      if (__p._M_exception_object)
+	try
+	  {
+	    std::rethrow_exception(__p);
+	  }
+	catch (const _Ex& __exc)
+	  {
+	    return &__exc;
+	  }
+	catch (...)
+	  {
+	  }
+#endif
+
+      return nullptr;
     }
 #endif
 
diff --git a/libstdc++-v3/libsupc++/nested_exception.h b/libstdc++-v3/libsupc++/nested_exception.h
index c9f63e5..aff47bc 100644
--- a/libstdc++-v3/libsupc++/nested_exception.h
+++ b/libstdc++-v3/libsupc++/nested_exception.h
@@ -62,17 +62,24 @@ namespace std _GLIBCXX_VISIBILITY(default)
 
   public:
     /// The default constructor stores the current exception (if any).
+    _GLIBCXX26_CONSTEXPR
     nested_exception() noexcept : _M_ptr(current_exception()) { }
 
+    _GLIBCXX26_CONSTEXPR
     nested_exception(const nested_exception&) noexcept = default;
 
+    _GLIBCXX26_CONSTEXPR
     nested_exception& operator=(const nested_exception&) noexcept = default;
 
+#if __cplusplus >= 202400L
+    constexpr virtual ~nested_exception() noexcept {}
+#else
     virtual ~nested_exception() noexcept;
+#endif
 
     /// Rethrow the stored exception, or terminate if none was stored.
     [[noreturn]]
-    void
+    _GLIBCXX26_CONSTEXPR void
     rethrow_nested() const
     {
       if (_M_ptr)
@@ -81,7 +88,7 @@ namespace std _GLIBCXX_VISIBILITY(default)
     }
 
     /// Access the stored exception.
-    exception_ptr
+    _GLIBCXX26_CONSTEXPR exception_ptr
     nested_ptr() const noexcept
     { return _M_ptr; }
   };
@@ -91,11 +98,11 @@ namespace std _GLIBCXX_VISIBILITY(default)
   template<typename _Except>
     struct _Nested_exception : public _Except, public nested_exception
     {
-      explicit _Nested_exception(const _Except& __ex)
+      _GLIBCXX26_CONSTEXPR explicit _Nested_exception(const _Except& __ex)
       : _Except(__ex)
       { }
 
-      explicit _Nested_exception(_Except&& __ex)
+      _GLIBCXX26_CONSTEXPR explicit _Nested_exception(_Except&& __ex)
       : _Except(static_cast<_Except&&>(__ex))
       { }
     };
@@ -144,7 +151,7 @@ namespace std _GLIBCXX_VISIBILITY(default)
    */
   template<typename _Tp>
     [[noreturn]]
-    inline void
+    _GLIBCXX26_CONSTEXPR inline void
     throw_with_nested(_Tp&& __t)
     {
       using _Up = typename decay<_Tp>::type;
@@ -204,7 +211,7 @@ namespace std _GLIBCXX_VISIBILITY(default)
 # if ! __cpp_rtti
     [[__gnu__::__always_inline__]]
 #endif
-    inline void
+    _GLIBCXX26_CONSTEXPR inline void
     rethrow_if_nested(const _Ex& __ex)
     {
       const _Ex* __ptr = __builtin_addressof(__ex);
diff --git a/libstdc++-v3/libsupc++/new b/libstdc++-v3/libsupc++/new
index a550373..fb36dae 100644
--- a/libstdc++-v3/libsupc++/new
+++ b/libstdc++-v3/libsupc++/new
@@ -66,33 +66,51 @@ namespace std
   class bad_alloc : public exception
   {
   public:
-    bad_alloc() throw() { }
+    _GLIBCXX26_CONSTEXPR bad_alloc() throw() { }
 
 #if __cplusplus >= 201103L
-    bad_alloc(const bad_alloc&) = default;
-    bad_alloc& operator=(const bad_alloc&) = default;
+    _GLIBCXX26_CONSTEXPR bad_alloc(const bad_alloc&) = default;
+    _GLIBCXX26_CONSTEXPR bad_alloc& operator=(const bad_alloc&) = default;
 #endif
 
+#if __cplusplus >= 202400L
+    constexpr virtual ~bad_alloc() noexcept {}
+
+    constexpr virtual const char* what() const noexcept
+    {
+      return "std::bad_alloc";
+    }
+#else
     // This declaration is not useless:
     // http://gcc.gnu.org/onlinedocs/gcc-3.0.2/gcc_6.html#SEC118
     virtual ~bad_alloc() throw();
 
     // See comment in eh_exception.cc.
     virtual const char* what() const throw();
+#endif
   };
 
 #if __cplusplus >= 201103L
   class bad_array_new_length : public bad_alloc
   {
   public:
-    bad_array_new_length() throw() { }
+    _GLIBCXX26_CONSTEXPR bad_array_new_length() throw() { }
 
+#if __cplusplus >= 202400L
+    constexpr virtual ~bad_array_new_length() noexcept {}
+
+    constexpr virtual const char* what() const noexcept
+    {
+      return "std::bad_array_new_length";
+    }
+#else
     // This declaration is not useless:
     // http://gcc.gnu.org/onlinedocs/gcc-3.0.2/gcc_6.html#SEC118
     virtual ~bad_array_new_length() throw();
 
     // See comment in eh_exception.cc.
     virtual const char* what() const throw();
+#endif
   };
 #endif
 
diff --git a/libstdc++-v3/libsupc++/typeinfo b/libstdc++-v3/libsupc++/typeinfo
index 0a94b86..a31be7c 100644
--- a/libstdc++-v3/libsupc++/typeinfo
+++ b/libstdc++-v3/libsupc++/typeinfo
@@ -224,14 +224,23 @@ namespace std
   class bad_cast : public exception
   {
   public:
-    bad_cast() _GLIBCXX_USE_NOEXCEPT { }
+    _GLIBCXX26_CONSTEXPR bad_cast() _GLIBCXX_USE_NOEXCEPT { }
 
+#if __cplusplus >= 202400L
+    constexpr virtual ~bad_cast() noexcept {}
+
+    constexpr virtual const char* what() const noexcept
+    {
+      return "std::bad_cast";
+    }
+#else
     // This declaration is not useless:
     // http://gcc.gnu.org/onlinedocs/gcc-3.0.2/gcc_6.html#SEC118
     virtual ~bad_cast() _GLIBCXX_USE_NOEXCEPT;
 
     // See comment in eh_exception.cc.
     virtual const char* what() const _GLIBCXX_USE_NOEXCEPT;
+#endif
   };
 
   /**
@@ -241,14 +250,23 @@ namespace std
   class bad_typeid : public exception
   {
   public:
-    bad_typeid () _GLIBCXX_USE_NOEXCEPT { }
+    _GLIBCXX26_CONSTEXPR bad_typeid () _GLIBCXX_USE_NOEXCEPT { }
 
+#if __cplusplus >= 202400L
+    constexpr virtual ~bad_typeid() noexcept {}
+
+    constexpr virtual const char* what() const noexcept
+    {
+      return "std::bad_typeid";
+    }
+#else
     // This declaration is not useless:
     // http://gcc.gnu.org/onlinedocs/gcc-3.0.2/gcc_6.html#SEC118
     virtual ~bad_typeid() _GLIBCXX_USE_NOEXCEPT;
 
     // See comment in eh_exception.cc.
     virtual const char* what() const _GLIBCXX_USE_NOEXCEPT;
+#endif
   };
 } // namespace std
 
diff --git a/libstdc++-v3/scripts/gen_windows_zones_map.py b/libstdc++-v3/scripts/gen_windows_zones_map.py
new file mode 100755
index 0000000..7f547a7
--- /dev/null
+++ b/libstdc++-v3/scripts/gen_windows_zones_map.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+#
+# Script to generate the map for libstdc++ std::chrono::current_zone under Windows.
+#
+# This file is part of GCC.
+#
+# GCC is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation; either version 3, or (at your option) any later
+# version.
+#
+# GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GCC; see the file COPYING3.  If not see
+# <http://www.gnu.org/licenses/>.
+
+# To update the Libstdc++ static data in src/c++20/windows_zones-map.h download the latest:
+# https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/windowsZones.xml
+# Then run this script and save the output to
+# src/c++20/windows_zones-map.h
+
+import os
+import sys
+import xml.etree.ElementTree as et
+
+if len(sys.argv) != 2:
+    print("Usage: %s <windows zones xml>" % sys.argv[0], file=sys.stderr)
+    sys.exit(1)
+
+self = os.path.basename(__file__)
+print("// Generated by scripts/{}, do not edit.".format(self))
+print("""
+// Copyright The GNU Toolchain Authors.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+""")
+
+class WindowsZoneMapEntry:
+    def __init__(self, windows, territory, iana):
+        self.windows = windows
+        self.territory = territory
+        self.iana = iana
+
+    def __lt__(self, other):
+        return (self.windows, self.territory) < (other.windows, other.territory)
+
+windows_zone_map = []
+
+tree = et.parse(sys.argv[1])
+xml_zone_map = tree.getroot().find("windowsZones").find("mapTimezones")
+
+for entry in xml_zone_map.iter("mapZone"):
+    iana = entry.attrib["type"]
+    space = iana.find(" ")
+    if space != -1:
+        iana = iana[0:space]
+    windows_zone_map.append(WindowsZoneMapEntry(entry.attrib["other"], entry.attrib["territory"], iana))
+
+# Sort so we can use binary search on the array.
+windows_zone_map.sort()
+
+# Skip territories which have the same IANA zone as 001, so we can reduce the data.
+last_windows_zone = ""
+for entry in windows_zone_map[:]:
+    if entry.windows != last_windows_zone:
+        if entry.territory != "001":
+            raise ValueError('Territory "001" not the first one, this breaks assumptions in tzdb.cc!')
+        last_windows_zone = entry.windows
+        fallback_iana = entry.iana
+    else:
+        if entry.iana == fallback_iana:
+            windows_zone_map.remove(entry)
+
+print("""struct windows_zone_map_entry
+{{
+  wstring_view windows_name;
+  wstring_view territory;
+  string_view iana_name;
+}};
+
+static constexpr array<windows_zone_map_entry, {}> windows_zone_map{{
+  {{""".format(len(windows_zone_map)))
+
+for entry in windows_zone_map:
+    print('    {{L"{}", L"{}", "{}"}},'.format(entry.windows, entry.territory, entry.iana))
+
+print("""  }
+};""")
+
+# src/c++20/tzdb.cc gives an error if this macro is not defined.
+# Do this last, so that the generated output is not usable unless we reach here.
+print("\n#define _GLIBCXX_WINDOWS_ZONES_MAP_COMPLETE")
diff --git a/libstdc++-v3/src/c++11/debug.cc b/libstdc++-v3/src/c++11/debug.cc
index 3533b5e..c6f6ef7 100644
--- a/libstdc++-v3/src/c++11/debug.cc
+++ b/libstdc++-v3/src/c++11/debug.cc
@@ -54,7 +54,7 @@ namespace
    *  in order to limit contention without breaking current library binary
    *  compatibility. */
   __gnu_cxx::__mutex&
-  get_safe_base_mutex(void* address)
+  get_safe_base_mutex(const void* address)
   {
     // Use arbitrarily __gnu_debug::vector<int> as the container giving
     // alignment of debug containers.
@@ -70,9 +70,9 @@ namespace
 #pragma GCC diagnostic warning "-Wabi=6"
 
   void
-  swap_its(__gnu_debug::_Safe_sequence_base& __lhs,
+  swap_its(const __gnu_debug::_Safe_sequence_base& __lhs,
 	   __gnu_debug::_Safe_iterator_base*& __lhs_its,
-	   __gnu_debug::_Safe_sequence_base& __rhs,
+	   const __gnu_debug::_Safe_sequence_base& __rhs,
 	   __gnu_debug::_Safe_iterator_base*& __rhs_its)
   {
     swap(__lhs_its, __rhs_its);
@@ -84,8 +84,8 @@ namespace
   }
 
   void
-  swap_seq_single(__gnu_debug::_Safe_sequence_base& __lhs,
-		  __gnu_debug::_Safe_sequence_base& __rhs)
+  swap_seq_single(const __gnu_debug::_Safe_sequence_base& __lhs,
+		  const __gnu_debug::_Safe_sequence_base& __rhs)
   {
     swap(__lhs._M_version, __rhs._M_version);
     swap_its(__lhs, __lhs._M_iterators,
@@ -118,17 +118,17 @@ namespace
 
   void
   swap_seq(__gnu_cxx::__mutex& lhs_mutex,
-	   __gnu_debug::_Safe_sequence_base& lhs,
+	   const __gnu_debug::_Safe_sequence_base& lhs,
 	   __gnu_cxx::__mutex& rhs_mutex,
-	   __gnu_debug::_Safe_sequence_base& rhs)
+	   const __gnu_debug::_Safe_sequence_base& rhs)
   {
     lock_and_run(lhs_mutex, rhs_mutex,
 		 [&lhs, &rhs]() { swap_seq_single(lhs, rhs); });
   }
 
   void
-  swap_ucont_single(__gnu_debug::_Safe_unordered_container_base& __lhs,
-		    __gnu_debug::_Safe_unordered_container_base& __rhs)
+  swap_ucont_single(const __gnu_debug::_Safe_unordered_container_base& __lhs,
+		    const __gnu_debug::_Safe_unordered_container_base& __rhs)
   {
     swap_seq_single(__lhs, __rhs);
     swap_its(__lhs, __lhs._M_local_iterators,
@@ -139,9 +139,9 @@ namespace
 
   void
   swap_ucont(__gnu_cxx::__mutex& lhs_mutex,
-	     __gnu_debug::_Safe_unordered_container_base& lhs,
+	     const __gnu_debug::_Safe_unordered_container_base& lhs,
 	     __gnu_cxx::__mutex& rhs_mutex,
-	     __gnu_debug::_Safe_unordered_container_base& rhs)
+	     const __gnu_debug::_Safe_unordered_container_base& rhs)
   {
     lock_and_run(lhs_mutex, rhs_mutex,
 		 [&lhs, &rhs]() { swap_ucont_single(lhs, rhs); });
@@ -158,8 +158,8 @@ namespace
       }
   }
 
-  void*
-  acquire_sequence_ptr_for_lock(__gnu_debug::_Safe_sequence_base*& seq)
+  const void*
+  acquire_sequence_ptr_for_lock(__gnu_debug::_Safe_sequence_base const*& seq)
   {
 #ifdef __GTHREADS
     if (!__gnu_cxx::__is_single_threaded())
@@ -169,7 +169,7 @@ namespace
   }
 
   void
-  reset_sequence_ptr(__gnu_debug::_Safe_sequence_base*& seq)
+  reset_sequence_ptr(__gnu_debug::_Safe_sequence_base const*& seq)
   {
 #ifdef __GTHREADS
     if (!__gnu_cxx::__is_single_threaded())
@@ -327,7 +327,7 @@ namespace __gnu_debug
 
   void
   _Safe_sequence_base::
-  _M_detach_all()
+  _M_detach_all() const
   {
     __gnu_cxx::__scoped_lock sentry(_M_get_mutex());
     detach_all(_M_iterators);
@@ -339,7 +339,7 @@ namespace __gnu_debug
 
   void
   _Safe_sequence_base::
-  _M_detach_singular()
+  _M_detach_singular() const
   {
     __gnu_cxx::__scoped_lock sentry(_M_get_mutex());
     for (_Safe_iterator_base* __iter = _M_iterators; __iter;)
@@ -361,7 +361,7 @@ namespace __gnu_debug
 
   void
   _Safe_sequence_base::
-  _M_revalidate_singular()
+  _M_revalidate_singular() const
   {
     __gnu_cxx::__scoped_lock sentry(_M_get_mutex());
     for (_Safe_iterator_base* __iter = _M_iterators; __iter;
@@ -375,17 +375,59 @@ namespace __gnu_debug
 
   void
   _Safe_sequence_base::
-  _M_swap(_Safe_sequence_base& __x) noexcept
+  _M_swap(const _Safe_sequence_base& __x) const noexcept
   { swap_seq(_M_get_mutex(), *this, __x._M_get_mutex(), __x); }
 
   __gnu_cxx::__mutex&
   _Safe_sequence_base::
-  _M_get_mutex() noexcept
+  _M_get_mutex() const noexcept
   { return get_safe_base_mutex(this); }
 
+#if !_GLIBCXX_INLINE_VERSION
+  void
+  _Safe_sequence_base::
+  _M_detach_all()
+  {
+    const _Safe_sequence_base* __this = this;
+    __this->_M_detach_all();
+  }
+
+  void
+  _Safe_sequence_base::
+  _M_detach_singular()
+  {
+    const _Safe_sequence_base* __this = this;
+    __this->_M_detach_singular();
+  }
+
+  void
+  _Safe_sequence_base::
+  _M_revalidate_singular()
+  {
+    const _Safe_sequence_base* __this = this;
+    __this->_M_revalidate_singular();
+  }
+
   void
   _Safe_sequence_base::
-  _M_attach(_Safe_iterator_base* __it, bool __constant)
+  _M_swap(_Safe_sequence_base& __x) noexcept
+  {
+    const _Safe_sequence_base* __this = this;
+    __this->_M_swap(__x);
+  }
+
+  __gnu_cxx::__mutex&
+  _Safe_sequence_base::
+  _M_get_mutex() noexcept
+  {
+    const _Safe_sequence_base* __this = this;
+    return __this->_M_get_mutex();
+  }
+#endif
+
+  void
+  _Safe_sequence_base::
+  _M_attach(_Safe_iterator_base* __it, bool __constant) const
   {
     __gnu_cxx::__scoped_lock sentry(_M_get_mutex());
     _M_attach_single(__it, __constant);
@@ -393,7 +435,7 @@ namespace __gnu_debug
 
   void
   _Safe_sequence_base::
-  _M_attach_single(_Safe_iterator_base* __it, bool __constant) noexcept
+  _M_attach_single(_Safe_iterator_base* __it, bool __constant) const noexcept
   {
     _Safe_iterator_base*& __its =
       __constant ? _M_const_iterators : _M_iterators;
@@ -405,7 +447,7 @@ namespace __gnu_debug
 
   void
   _Safe_sequence_base::
-  _M_detach(_Safe_iterator_base* __it)
+  _M_detach(_Safe_iterator_base* __it) const
   {
     // Remove __it from this sequence's list
     __gnu_cxx::__scoped_lock sentry(_M_get_mutex());
@@ -414,7 +456,7 @@ namespace __gnu_debug
 
   void
   _Safe_sequence_base::
-  _M_detach_single(_Safe_iterator_base* __it) noexcept
+  _M_detach_single(_Safe_iterator_base* __it) const noexcept
   {
     // Remove __it from this sequence's list
     __it->_M_unlink();
@@ -426,7 +468,7 @@ namespace __gnu_debug
 
   void
   _Safe_iterator_base::
-  _M_attach(_Safe_sequence_base* __seq, bool __constant)
+  _M_attach(const _Safe_sequence_base* __seq, bool __constant)
   {
     _M_detach();
 
@@ -443,7 +485,7 @@ namespace __gnu_debug
 
   void
   _Safe_iterator_base::
-  _M_attach_single(_Safe_sequence_base* __seq, bool __constant) noexcept
+  _M_attach_single(const _Safe_sequence_base* __seq, bool __constant) noexcept
   {
     _M_detach_single();
 
@@ -514,14 +556,27 @@ namespace __gnu_debug
   _M_get_mutex() noexcept
   { return _M_sequence->_M_get_mutex(); }
 
-  _Safe_unordered_container_base*
-  _Safe_local_iterator_base::
-  _M_get_container() const noexcept
-  { return static_cast<_Safe_unordered_container_base*>(_M_sequence); }
+#if !_GLIBCXX_INLINE_VERSION
+  void
+  _Safe_iterator_base::
+  _M_attach(_Safe_sequence_base* __seq, bool __constant)
+  {
+    const _Safe_sequence_base* __cseq = __seq;
+    _M_attach(__cseq, __constant);
+  }
+
+  void
+  _Safe_iterator_base::
+  _M_attach_single(_Safe_sequence_base* __seq, bool __constant) noexcept
+  {
+    const _Safe_sequence_base* __cseq = __seq;
+    _M_attach_single(__cseq, __constant);
+  }
+#endif
 
   void
   _Safe_local_iterator_base::
-  _M_attach(_Safe_sequence_base* __cont, bool __constant)
+  _M_attach(const _Safe_unordered_container_base* __cont, bool __constant)
   {
     _M_detach();
 
@@ -530,7 +585,7 @@ namespace __gnu_debug
       {
 	_M_sequence = __cont;
 	_M_version = _M_sequence->_M_version;
-	_M_get_container()->_M_attach_local(this, __constant);
+	_M_safe_container()->_M_attach_local(this, __constant);
       }
     else
       _M_version = 0;
@@ -538,7 +593,8 @@ namespace __gnu_debug
 
   void
   _Safe_local_iterator_base::
-  _M_attach_single(_Safe_sequence_base* __cont, bool __constant) noexcept
+  _M_attach_single(const _Safe_unordered_container_base* __cont,
+		   bool __constant) noexcept
   {
     _M_detach_single();
 
@@ -547,7 +603,7 @@ namespace __gnu_debug
       {
 	_M_sequence = __cont;
 	_M_version = _M_sequence->_M_version;
-	_M_get_container()->_M_attach_local_single(this, __constant);
+	_M_safe_container()->_M_attach_local_single(this, __constant);
       }
     else
       _M_version = 0;
@@ -570,14 +626,34 @@ namespace __gnu_debug
   {
     if (_M_sequence)
       {
-	_M_get_container()->_M_detach_local_single(this);
+	_M_safe_container()->_M_detach_local_single(this);
 	_M_reset();
       }
   }
 
+#if !_GLIBCXX_INLINE_VERSION
+  void
+  _Safe_local_iterator_base::
+  _M_attach(_Safe_sequence_base* __seq, bool __constant)
+  {
+    const _Safe_unordered_container_base* __cont
+      = static_cast<_Safe_unordered_container_base*>(__seq);
+    _M_attach(__cont, __constant);
+  }
+
+  void
+  _Safe_local_iterator_base::
+  _M_attach_single(_Safe_sequence_base* __seq, bool __constant) noexcept
+  {
+    const _Safe_unordered_container_base* __cont
+      = static_cast<_Safe_unordered_container_base*>(__seq);
+    _M_attach_single(__cont, __constant);
+  }
+#endif
+
   void
   _Safe_unordered_container_base::
-  _M_detach_all()
+  _M_detach_all() const
   {
     __gnu_cxx::__scoped_lock sentry(_M_get_mutex());
     detach_all(_M_iterators);
@@ -595,12 +671,12 @@ namespace __gnu_debug
 
   void
   _Safe_unordered_container_base::
-  _M_swap(_Safe_unordered_container_base& __x) noexcept
+  _M_swap(const _Safe_unordered_container_base& __x) const noexcept
   { swap_ucont(_M_get_mutex(), *this, __x._M_get_mutex(), __x); }
 
   void
   _Safe_unordered_container_base::
-  _M_attach_local(_Safe_iterator_base* __it, bool __constant)
+  _M_attach_local(_Safe_iterator_base* __it, bool __constant) const
   {
     __gnu_cxx::__scoped_lock sentry(_M_get_mutex());
     _M_attach_local_single(__it, __constant);
@@ -608,7 +684,7 @@ namespace __gnu_debug
 
   void
   _Safe_unordered_container_base::
-  _M_attach_local_single(_Safe_iterator_base* __it, bool __constant) noexcept
+  _M_attach_local_single(_Safe_iterator_base* __it, bool __constant) const noexcept
   {
     _Safe_iterator_base*& __its =
       __constant ? _M_const_local_iterators : _M_local_iterators;
@@ -620,7 +696,7 @@ namespace __gnu_debug
 
   void
   _Safe_unordered_container_base::
-  _M_detach_local(_Safe_iterator_base* __it)
+  _M_detach_local(_Safe_iterator_base* __it) const
   {
     // Remove __it from this container's list
     __gnu_cxx::__scoped_lock sentry(_M_get_mutex());
@@ -629,7 +705,7 @@ namespace __gnu_debug
 
   void
   _Safe_unordered_container_base::
-  _M_detach_local_single(_Safe_iterator_base* __it) noexcept
+  _M_detach_local_single(_Safe_iterator_base* __it) const noexcept
   {
     // Remove __it from this container's list
     __it->_M_unlink();
@@ -638,6 +714,24 @@ namespace __gnu_debug
     if (_M_local_iterators == __it)
       _M_local_iterators = __it->_M_next;
   }
+
+#if !_GLIBCXX_INLINE_VERSION
+  void
+  _Safe_unordered_container_base::
+  _M_detach_all()
+  {
+    const _Safe_unordered_container_base* __this = this;
+    __this->_M_detach_all();
+  }
+
+  void
+  _Safe_unordered_container_base::
+  _M_swap(_Safe_unordered_container_base& __x) noexcept
+  {
+    const _Safe_unordered_container_base* __this = this;
+    __this->_M_swap(__x);
+  }
+#endif
 }
 
 namespace
diff --git a/libstdc++-v3/src/c++17/memory_resource.cc b/libstdc++-v3/src/c++17/memory_resource.cc
index fac4c78..c61569f 100644
--- a/libstdc++-v3/src/c++17/memory_resource.cc
+++ b/libstdc++-v3/src/c++17/memory_resource.cc
@@ -182,8 +182,8 @@ namespace pmr
   // versions will not use this symbol.
   monotonic_buffer_resource::~monotonic_buffer_resource() { release(); }
 
-  namespace {
-
+namespace
+{
   // aligned_size<N> stores the size and alignment of a memory allocation.
   // The size must be a multiple of N, leaving the low log2(N) bits free
   // to store the base-2 logarithm of the alignment.
@@ -221,7 +221,7 @@ namespace pmr
     return (n + alignment - 1) & ~(alignment - 1);
   }
 
-  } // namespace
+} // namespace
 
   // Memory allocated by the upstream resource is managed in a linked list
   // of _Chunk objects. A _Chunk object recording the size and alignment of
@@ -307,8 +307,8 @@ namespace pmr
 
   // Helper types for synchronized_pool_resource & unsynchronized_pool_resource
 
-  namespace {
-
+namespace
+{
   // Simple bitset with runtime size.
   // Tracks which blocks in a pool chunk are used/unused.
   struct bitset
@@ -636,7 +636,7 @@ namespace pmr
 
   static_assert(sizeof(big_block) == (2 * sizeof(void*)));
 
-  } // namespace
+} // namespace
 
   // A pool that serves blocks of a particular size.
   // Each pool manages a number of chunks.
@@ -868,7 +868,16 @@ namespace pmr
     using big_block::big_block;
   };
 
-  namespace {
+namespace
+{
+  // N.B. it is important that we don't skip any power of two sizes if there
+  // is a non-power of two size between them, e.g. must not have pool sizes
+  // of 24 and 40 without having a pool size of 32. Otherwise an allocation
+  // of 32 bytes with alignment 16 would choose the 40-byte pool which is not
+  // correctly aligned for 16-byte alignment. It would be OK (but suboptimal)
+  // to have no pool of size 32 if we have pool sizes of 16 and 64 and no
+  // non-power of two sizes between those, because the example of (32, 16)
+  // would choose the 64-byte pool, which would be correctly aligned.
 
   constexpr size_t pool_sizes[] = {
       8, 16, 24,
@@ -983,7 +992,7 @@ namespace pmr
   using exclusive_lock = lock_guard<shared_mutex>;
 #endif
 
-  } // namespace
+} // namespace
 
   __pool_resource::
   __pool_resource(const pool_options& opts, memory_resource* upstream)
@@ -1075,12 +1084,33 @@ namespace pmr
     return p;
   }
 
+  // Determine the appropriate allocation size, rounding up to a multiple
+  // of the alignment if needed.
+  static inline size_t
+  choose_block_size(size_t bytes, size_t alignment)
+  {
+    if (bytes == 0) [[unlikely]]
+      return alignment;
+
+    // Use bit_ceil in case alignment is invalid (i.e. not a power of two).
+    size_t mask = std::__bit_ceil(alignment) - 1;
+    // Round up to a multiple of alignment.
+    size_t block_size = (bytes + mask) & ~mask;
+
+    if (block_size >= bytes) [[likely]]
+      return block_size;
+
+    // Wrapped around to zero, bytes must have been impossibly large.
+    return numeric_limits<size_t>::max();
+  }
+
+
 #ifdef _GLIBCXX_HAS_GTHREADS
   // synchronized_pool_resource members.
 
   /* Notes on implementation and thread safety:
    *
-   * Each synchronized_pool_resource manages an linked list of N+1 _TPools
+   * Each synchronized_pool_resource manages a linked list of N+1 _TPools
    * objects, where N is the number of threads using the pool resource.
    * Each _TPools object has its own set of pools, with their own chunks.
    * The first element of the list, _M_tpools[0], can be used by any thread.
@@ -1247,7 +1277,7 @@ namespace pmr
   synchronized_pool_resource::
   do_allocate(size_t bytes, size_t alignment)
   {
-    const auto block_size = std::max(bytes, alignment);
+    const auto block_size = choose_block_size(bytes, alignment);
     const pool_options opts = _M_impl._M_opts;
     if (block_size <= opts.largest_required_pool_block)
       {
@@ -1294,7 +1324,7 @@ namespace pmr
   synchronized_pool_resource::
   do_deallocate(void* p, size_t bytes, size_t alignment)
   {
-    size_t block_size = std::max(bytes, alignment);
+    size_t block_size = choose_block_size(bytes, alignment);
     if (block_size <= _M_impl._M_opts.largest_required_pool_block)
       {
 	const ptrdiff_t index = pool_index(block_size, _M_impl._M_npools);
@@ -1453,7 +1483,7 @@ namespace pmr
   void*
   unsynchronized_pool_resource::do_allocate(size_t bytes, size_t alignment)
   {
-    const auto block_size = std::max(bytes, alignment);
+    const auto block_size = choose_block_size(bytes, alignment);
     if (block_size <= _M_impl._M_opts.largest_required_pool_block)
       {
 	// Recreate pools if release() has been called:
@@ -1470,7 +1500,7 @@ namespace pmr
   unsynchronized_pool_resource::
   do_deallocate(void* p, size_t bytes, size_t alignment)
   {
-    size_t block_size = std::max(bytes, alignment);
+    size_t block_size = choose_block_size(bytes, alignment);
     if (block_size <= _M_impl._M_opts.largest_required_pool_block)
       {
 	if (auto pool = _M_find_pool(block_size))
diff --git a/libstdc++-v3/src/c++20/tzdb.cc b/libstdc++-v3/src/c++20/tzdb.cc
index 6e244dc..d79b61a 100644
--- a/libstdc++-v3/src/c++20/tzdb.cc
+++ b/libstdc++-v3/src/c++20/tzdb.cc
@@ -44,6 +44,14 @@
 # include <cstdlib>   // getenv
 #endif
 
+#if _GLIBCXX_HAVE_WINDOWS_H
+# define WIN32_LEAN_AND_MEAN
+# include <windows.h>
+# include <psapi.h>
+
+# include <array>
+#endif
+
 #if defined __GTHREADS && ATOMIC_POINTER_LOCK_FREE == 2
 # define USE_ATOMIC_LIST_HEAD 1
 // TODO benchmark atomic<shared_ptr<>> vs mutex.
@@ -1145,6 +1153,34 @@ namespace std::chrono
       else
 	path = _GLIBCXX_ZONEINFO_DIR;
 #endif
+#ifdef _GLIBCXX_HAVE_WINDOWS_H
+      if (path.empty())
+	{
+	  HMODULE dll_module;
+	  if (GetModuleHandleExA(
+		  GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS
+		      | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
+		  reinterpret_cast<const char *>(&zoneinfo_file), &dll_module))
+	    {
+	      char dll_path[MAX_PATH];
+	      if (GetModuleFileNameA(dll_module, dll_path, MAX_PATH) != 0)
+		{
+		  string_view dll_path_view = dll_path;
+		  auto pos = dll_path_view.find_last_of('\\');
+		  dll_path_view = dll_path_view.substr(0, pos);
+		  if (dll_path_view.ends_with("\\bin"))
+		    {
+		      constexpr string_view remaining_path = "share\\zoneinfo";
+		      dll_path_view.remove_suffix(3); // Remove bin
+		      path.reserve(dll_path_view.size()
+				   + remaining_path.size());
+		      path = dll_path_view;
+		      path += remaining_path;
+		    }
+		}
+	    }
+	}
+#endif
       if (!path.empty())
 	path.append(filename);
       return path;
@@ -1734,6 +1770,98 @@ namespace std::chrono
 
       return nullptr; // not found
     }
+
+#ifdef _GLIBCXX_HAVE_WINDOWS_H
+    string_view
+    detect_windows_zone() noexcept
+    {
+      DYNAMIC_TIME_ZONE_INFORMATION information{};
+      if (GetDynamicTimeZoneInformation(&information) == TIME_ZONE_ID_INVALID)
+	return {};
+
+      constexpr SYSTEMTIME all_zero_time{};
+      const wstring_view zone_name{ information.TimeZoneKeyName };
+      auto equal = [](const SYSTEMTIME &lhs, const SYSTEMTIME &rhs) noexcept
+	{ return memcmp(&lhs, &rhs, sizeof(SYSTEMTIME)) == 0; };
+      // The logic is copied from icu, couldn't find the source.
+      // Detect if DST is disabled.
+      if (information.DynamicDaylightTimeDisabled
+	  && equal(information.StandardDate, information.DaylightDate)
+	  && ((!zone_name.empty()
+	       && equal(information.StandardDate, all_zero_time))
+	      || (zone_name.empty()
+		  && !equal(information.StandardDate, all_zero_time))))
+	{
+	  if (information.Bias == 0)
+	    return "Etc/UTC";
+
+	  if (information.Bias % 60 != 0)
+	    // If the offset is not in full hours, we can't do anything really.
+	    return {};
+
+	  const auto raw_index = information.Bias / 60;
+
+	  // The bias added to the local time equals UTC. And GMT+X corresponds
+	  // to UTC-X, the sign is negated. Thus we can use the hourly bias as
+	  // an index into an array.
+	  if (raw_index < 0 && raw_index >= -14)
+	    {
+	      static constexpr array<string_view, 14> table{
+		"Etc/GMT-1",  "Etc/GMT-2",  "Etc/GMT-3",  "Etc/GMT-4",
+		"Etc/GMT-5",  "Etc/GMT-6",  "Etc/GMT-7",  "Etc/GMT-8",
+		"Etc/GMT-9",  "Etc/GMT-10", "Etc/GMT-11", "Etc/GMT-12",
+		"Etc/GMT-13", "Etc/GMT-14"
+	      };
+	      return table[-raw_index - 1];
+	    }
+	  else if (raw_index > 0 && raw_index <= 12)
+	    {
+	      static constexpr array<string_view, 12> table{
+		"Etc/GMT+1", "Etc/GMT+2",  "Etc/GMT+3",	 "Etc/GMT+4",
+		"Etc/GMT+5", "Etc/GMT+6",  "Etc/GMT+7",	 "Etc/GMT+8",
+		"Etc/GMT+9", "Etc/GMT+10", "Etc/GMT+11", "Etc/GMT+12"
+	      };
+	      return table[raw_index - 1];
+	    }
+	  return {};
+	}
+
+#include "windows_zones-map.h"
+#ifndef _GLIBCXX_WINDOWS_ZONES_MAP_COMPLETE
+# error "Invalid windows_zones map"
+#endif
+
+      const auto zone_range
+	  = ranges::equal_range(windows_zone_map, zone_name, {},
+				&windows_zone_map_entry::windows_name);
+
+      const auto size = ranges::size(zone_range);
+      if (size == 0)
+	// Unknown zone, we can't detect anything.
+	return {};
+
+      if (size == 1)
+	// Some zones have only one territory, use the quick path.
+	return zone_range.front().iana_name;
+
+      const auto geo_id = GetUserGeoID(GEOCLASS_NATION);
+      // We ask for a 2-letter country code plus the zero terminator. "001" is
+      // only contained in the zone map, not returned by GetGeoInfoW.
+      wchar_t territory[3] = {};
+      if (GetGeoInfoW(geo_id, GEO_ISO2, territory, 3, 0) == 0)
+	// Couldn't detect the territory, fallback to "001", which is the first
+	// entry.
+	return zone_range.front().iana_name;
+
+      const auto iter = ranges::lower_bound(
+	  zone_range, territory, {}, &windows_zone_map_entry::territory);
+      if (iter == zone_range.end() || iter->territory != territory)
+	// Territory not within the the map, use "001".
+	return zone_range.front().iana_name;
+
+      return iter->iana_name;
+    }
+#endif
   } // namespace
 
   // Implementation of std::chrono::tzdb::locate_zone(string_view).
@@ -1756,7 +1884,7 @@ namespace std::chrono
   {
     // TODO cache this function's result?
 
-#ifndef _AIX
+#if !defined(_AIX) && !defined(_GLIBCXX_HAVE_WINDOWS_H)
     // Repeat the preprocessor condition used by filesystem::read_symlink,
     // to avoid a dependency on src/c++17/fs_ops.o if it won't work anyway.
 #if defined(_GLIBCXX_HAVE_READLINK) && defined(_GLIBCXX_HAVE_SYS_STAT_H)
@@ -1813,7 +1941,11 @@ namespace std::chrono
 		  return tz;
 	      }
       }
-#else
+#elif defined(_GLIBCXX_HAVE_WINDOWS_H)
+    if (auto tz
+	= do_locate_zone(this->zones, this->links, detect_windows_zone()))
+      return tz;
+#else // defined(_AIX)
     // AIX stores current zone in $TZ in /etc/environment but the value
     // is typically a POSIX time zone name, not IANA zone.
     // https://developer.ibm.com/articles/au-aix-posix/
diff --git a/libstdc++-v3/src/c++20/windows_zones-map.h b/libstdc++-v3/src/c++20/windows_zones-map.h
new file mode 100644
index 0000000..0f63a17
--- /dev/null
+++ b/libstdc++-v3/src/c++20/windows_zones-map.h
@@ -0,0 +1,399 @@
+// Generated by scripts/gen_windows_zones_map.py, do not edit.
+
+// Copyright The GNU Toolchain Authors.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+
+struct windows_zone_map_entry
+{
+  wstring_view windows_name;
+  wstring_view territory;
+  string_view iana_name;
+};
+
+static constexpr array<windows_zone_map_entry, 361> windows_zone_map{
+  {
+    {L"AUS Central Standard Time", L"001", "Australia/Darwin"},
+    {L"AUS Eastern Standard Time", L"001", "Australia/Sydney"},
+    {L"Afghanistan Standard Time", L"001", "Asia/Kabul"},
+    {L"Alaskan Standard Time", L"001", "America/Anchorage"},
+    {L"Aleutian Standard Time", L"001", "America/Adak"},
+    {L"Altai Standard Time", L"001", "Asia/Barnaul"},
+    {L"Arab Standard Time", L"001", "Asia/Riyadh"},
+    {L"Arab Standard Time", L"BH", "Asia/Bahrain"},
+    {L"Arab Standard Time", L"KW", "Asia/Kuwait"},
+    {L"Arab Standard Time", L"QA", "Asia/Qatar"},
+    {L"Arab Standard Time", L"YE", "Asia/Aden"},
+    {L"Arabian Standard Time", L"001", "Asia/Dubai"},
+    {L"Arabian Standard Time", L"OM", "Asia/Muscat"},
+    {L"Arabian Standard Time", L"ZZ", "Etc/GMT-4"},
+    {L"Arabic Standard Time", L"001", "Asia/Baghdad"},
+    {L"Argentina Standard Time", L"001", "America/Buenos_Aires"},
+    {L"Astrakhan Standard Time", L"001", "Europe/Astrakhan"},
+    {L"Atlantic Standard Time", L"001", "America/Halifax"},
+    {L"Atlantic Standard Time", L"BM", "Atlantic/Bermuda"},
+    {L"Atlantic Standard Time", L"GL", "America/Thule"},
+    {L"Aus Central W. Standard Time", L"001", "Australia/Eucla"},
+    {L"Azerbaijan Standard Time", L"001", "Asia/Baku"},
+    {L"Azores Standard Time", L"001", "Atlantic/Azores"},
+    {L"Azores Standard Time", L"GL", "America/Scoresbysund"},
+    {L"Bahia Standard Time", L"001", "America/Bahia"},
+    {L"Bangladesh Standard Time", L"001", "Asia/Dhaka"},
+    {L"Bangladesh Standard Time", L"BT", "Asia/Thimphu"},
+    {L"Belarus Standard Time", L"001", "Europe/Minsk"},
+    {L"Bougainville Standard Time", L"001", "Pacific/Bougainville"},
+    {L"Canada Central Standard Time", L"001", "America/Regina"},
+    {L"Cape Verde Standard Time", L"001", "Atlantic/Cape_Verde"},
+    {L"Cape Verde Standard Time", L"ZZ", "Etc/GMT+1"},
+    {L"Caucasus Standard Time", L"001", "Asia/Yerevan"},
+    {L"Cen. Australia Standard Time", L"001", "Australia/Adelaide"},
+    {L"Central America Standard Time", L"001", "America/Guatemala"},
+    {L"Central America Standard Time", L"BZ", "America/Belize"},
+    {L"Central America Standard Time", L"CR", "America/Costa_Rica"},
+    {L"Central America Standard Time", L"EC", "Pacific/Galapagos"},
+    {L"Central America Standard Time", L"HN", "America/Tegucigalpa"},
+    {L"Central America Standard Time", L"NI", "America/Managua"},
+    {L"Central America Standard Time", L"SV", "America/El_Salvador"},
+    {L"Central America Standard Time", L"ZZ", "Etc/GMT+6"},
+    {L"Central Asia Standard Time", L"001", "Asia/Bishkek"},
+    {L"Central Asia Standard Time", L"AQ", "Antarctica/Vostok"},
+    {L"Central Asia Standard Time", L"CN", "Asia/Urumqi"},
+    {L"Central Asia Standard Time", L"IO", "Indian/Chagos"},
+    {L"Central Asia Standard Time", L"ZZ", "Etc/GMT-6"},
+    {L"Central Brazilian Standard Time", L"001", "America/Cuiaba"},
+    {L"Central Europe Standard Time", L"001", "Europe/Budapest"},
+    {L"Central Europe Standard Time", L"AL", "Europe/Tirane"},
+    {L"Central Europe Standard Time", L"CZ", "Europe/Prague"},
+    {L"Central Europe Standard Time", L"ME", "Europe/Podgorica"},
+    {L"Central Europe Standard Time", L"RS", "Europe/Belgrade"},
+    {L"Central Europe Standard Time", L"SI", "Europe/Ljubljana"},
+    {L"Central Europe Standard Time", L"SK", "Europe/Bratislava"},
+    {L"Central European Standard Time", L"001", "Europe/Warsaw"},
+    {L"Central European Standard Time", L"BA", "Europe/Sarajevo"},
+    {L"Central European Standard Time", L"HR", "Europe/Zagreb"},
+    {L"Central European Standard Time", L"MK", "Europe/Skopje"},
+    {L"Central Pacific Standard Time", L"001", "Pacific/Guadalcanal"},
+    {L"Central Pacific Standard Time", L"AQ", "Antarctica/Casey"},
+    {L"Central Pacific Standard Time", L"FM", "Pacific/Ponape"},
+    {L"Central Pacific Standard Time", L"NC", "Pacific/Noumea"},
+    {L"Central Pacific Standard Time", L"VU", "Pacific/Efate"},
+    {L"Central Pacific Standard Time", L"ZZ", "Etc/GMT-11"},
+    {L"Central Standard Time", L"001", "America/Chicago"},
+    {L"Central Standard Time", L"CA", "America/Winnipeg"},
+    {L"Central Standard Time", L"MX", "America/Matamoros"},
+    {L"Central Standard Time (Mexico)", L"001", "America/Mexico_City"},
+    {L"Chatham Islands Standard Time", L"001", "Pacific/Chatham"},
+    {L"China Standard Time", L"001", "Asia/Shanghai"},
+    {L"China Standard Time", L"HK", "Asia/Hong_Kong"},
+    {L"China Standard Time", L"MO", "Asia/Macau"},
+    {L"Cuba Standard Time", L"001", "America/Havana"},
+    {L"Dateline Standard Time", L"001", "Etc/GMT+12"},
+    {L"E. Africa Standard Time", L"001", "Africa/Nairobi"},
+    {L"E. Africa Standard Time", L"AQ", "Antarctica/Syowa"},
+    {L"E. Africa Standard Time", L"DJ", "Africa/Djibouti"},
+    {L"E. Africa Standard Time", L"ER", "Africa/Asmera"},
+    {L"E. Africa Standard Time", L"ET", "Africa/Addis_Ababa"},
+    {L"E. Africa Standard Time", L"KM", "Indian/Comoro"},
+    {L"E. Africa Standard Time", L"MG", "Indian/Antananarivo"},
+    {L"E. Africa Standard Time", L"SO", "Africa/Mogadishu"},
+    {L"E. Africa Standard Time", L"TZ", "Africa/Dar_es_Salaam"},
+    {L"E. Africa Standard Time", L"UG", "Africa/Kampala"},
+    {L"E. Africa Standard Time", L"YT", "Indian/Mayotte"},
+    {L"E. Africa Standard Time", L"ZZ", "Etc/GMT-3"},
+    {L"E. Australia Standard Time", L"001", "Australia/Brisbane"},
+    {L"E. Europe Standard Time", L"001", "Europe/Chisinau"},
+    {L"E. South America Standard Time", L"001", "America/Sao_Paulo"},
+    {L"Easter Island Standard Time", L"001", "Pacific/Easter"},
+    {L"Eastern Standard Time", L"001", "America/New_York"},
+    {L"Eastern Standard Time", L"BS", "America/Nassau"},
+    {L"Eastern Standard Time", L"CA", "America/Toronto"},
+    {L"Eastern Standard Time (Mexico)", L"001", "America/Cancun"},
+    {L"Egypt Standard Time", L"001", "Africa/Cairo"},
+    {L"Ekaterinburg Standard Time", L"001", "Asia/Yekaterinburg"},
+    {L"FLE Standard Time", L"001", "Europe/Kiev"},
+    {L"FLE Standard Time", L"AX", "Europe/Mariehamn"},
+    {L"FLE Standard Time", L"BG", "Europe/Sofia"},
+    {L"FLE Standard Time", L"EE", "Europe/Tallinn"},
+    {L"FLE Standard Time", L"FI", "Europe/Helsinki"},
+    {L"FLE Standard Time", L"LT", "Europe/Vilnius"},
+    {L"FLE Standard Time", L"LV", "Europe/Riga"},
+    {L"Fiji Standard Time", L"001", "Pacific/Fiji"},
+    {L"GMT Standard Time", L"001", "Europe/London"},
+    {L"GMT Standard Time", L"ES", "Atlantic/Canary"},
+    {L"GMT Standard Time", L"FO", "Atlantic/Faeroe"},
+    {L"GMT Standard Time", L"GG", "Europe/Guernsey"},
+    {L"GMT Standard Time", L"IE", "Europe/Dublin"},
+    {L"GMT Standard Time", L"IM", "Europe/Isle_of_Man"},
+    {L"GMT Standard Time", L"JE", "Europe/Jersey"},
+    {L"GMT Standard Time", L"PT", "Europe/Lisbon"},
+    {L"GTB Standard Time", L"001", "Europe/Bucharest"},
+    {L"GTB Standard Time", L"CY", "Asia/Nicosia"},
+    {L"GTB Standard Time", L"GR", "Europe/Athens"},
+    {L"Georgian Standard Time", L"001", "Asia/Tbilisi"},
+    {L"Greenland Standard Time", L"001", "America/Godthab"},
+    {L"Greenwich Standard Time", L"001", "Atlantic/Reykjavik"},
+    {L"Greenwich Standard Time", L"BF", "Africa/Ouagadougou"},
+    {L"Greenwich Standard Time", L"CI", "Africa/Abidjan"},
+    {L"Greenwich Standard Time", L"GH", "Africa/Accra"},
+    {L"Greenwich Standard Time", L"GL", "America/Danmarkshavn"},
+    {L"Greenwich Standard Time", L"GM", "Africa/Banjul"},
+    {L"Greenwich Standard Time", L"GN", "Africa/Conakry"},
+    {L"Greenwich Standard Time", L"GW", "Africa/Bissau"},
+    {L"Greenwich Standard Time", L"LR", "Africa/Monrovia"},
+    {L"Greenwich Standard Time", L"ML", "Africa/Bamako"},
+    {L"Greenwich Standard Time", L"MR", "Africa/Nouakchott"},
+    {L"Greenwich Standard Time", L"SH", "Atlantic/St_Helena"},
+    {L"Greenwich Standard Time", L"SL", "Africa/Freetown"},
+    {L"Greenwich Standard Time", L"SN", "Africa/Dakar"},
+    {L"Greenwich Standard Time", L"TG", "Africa/Lome"},
+    {L"Haiti Standard Time", L"001", "America/Port-au-Prince"},
+    {L"Hawaiian Standard Time", L"001", "Pacific/Honolulu"},
+    {L"Hawaiian Standard Time", L"CK", "Pacific/Rarotonga"},
+    {L"Hawaiian Standard Time", L"PF", "Pacific/Tahiti"},
+    {L"Hawaiian Standard Time", L"ZZ", "Etc/GMT+10"},
+    {L"India Standard Time", L"001", "Asia/Calcutta"},
+    {L"Iran Standard Time", L"001", "Asia/Tehran"},
+    {L"Israel Standard Time", L"001", "Asia/Jerusalem"},
+    {L"Jordan Standard Time", L"001", "Asia/Amman"},
+    {L"Kaliningrad Standard Time", L"001", "Europe/Kaliningrad"},
+    {L"Korea Standard Time", L"001", "Asia/Seoul"},
+    {L"Libya Standard Time", L"001", "Africa/Tripoli"},
+    {L"Line Islands Standard Time", L"001", "Pacific/Kiritimati"},
+    {L"Line Islands Standard Time", L"ZZ", "Etc/GMT-14"},
+    {L"Lord Howe Standard Time", L"001", "Australia/Lord_Howe"},
+    {L"Magadan Standard Time", L"001", "Asia/Magadan"},
+    {L"Magallanes Standard Time", L"001", "America/Punta_Arenas"},
+    {L"Marquesas Standard Time", L"001", "Pacific/Marquesas"},
+    {L"Mauritius Standard Time", L"001", "Indian/Mauritius"},
+    {L"Mauritius Standard Time", L"RE", "Indian/Reunion"},
+    {L"Mauritius Standard Time", L"SC", "Indian/Mahe"},
+    {L"Middle East Standard Time", L"001", "Asia/Beirut"},
+    {L"Montevideo Standard Time", L"001", "America/Montevideo"},
+    {L"Morocco Standard Time", L"001", "Africa/Casablanca"},
+    {L"Morocco Standard Time", L"EH", "Africa/El_Aaiun"},
+    {L"Mountain Standard Time", L"001", "America/Denver"},
+    {L"Mountain Standard Time", L"CA", "America/Edmonton"},
+    {L"Mountain Standard Time", L"MX", "America/Ciudad_Juarez"},
+    {L"Mountain Standard Time (Mexico)", L"001", "America/Mazatlan"},
+    {L"Myanmar Standard Time", L"001", "Asia/Rangoon"},
+    {L"Myanmar Standard Time", L"CC", "Indian/Cocos"},
+    {L"N. Central Asia Standard Time", L"001", "Asia/Novosibirsk"},
+    {L"Namibia Standard Time", L"001", "Africa/Windhoek"},
+    {L"Nepal Standard Time", L"001", "Asia/Katmandu"},
+    {L"New Zealand Standard Time", L"001", "Pacific/Auckland"},
+    {L"New Zealand Standard Time", L"AQ", "Antarctica/McMurdo"},
+    {L"Newfoundland Standard Time", L"001", "America/St_Johns"},
+    {L"Norfolk Standard Time", L"001", "Pacific/Norfolk"},
+    {L"North Asia East Standard Time", L"001", "Asia/Irkutsk"},
+    {L"North Asia Standard Time", L"001", "Asia/Krasnoyarsk"},
+    {L"North Korea Standard Time", L"001", "Asia/Pyongyang"},
+    {L"Omsk Standard Time", L"001", "Asia/Omsk"},
+    {L"Pacific SA Standard Time", L"001", "America/Santiago"},
+    {L"Pacific Standard Time", L"001", "America/Los_Angeles"},
+    {L"Pacific Standard Time", L"CA", "America/Vancouver"},
+    {L"Pacific Standard Time (Mexico)", L"001", "America/Tijuana"},
+    {L"Pakistan Standard Time", L"001", "Asia/Karachi"},
+    {L"Paraguay Standard Time", L"001", "America/Asuncion"},
+    {L"Qyzylorda Standard Time", L"001", "Asia/Qyzylorda"},
+    {L"Romance Standard Time", L"001", "Europe/Paris"},
+    {L"Romance Standard Time", L"BE", "Europe/Brussels"},
+    {L"Romance Standard Time", L"DK", "Europe/Copenhagen"},
+    {L"Romance Standard Time", L"ES", "Europe/Madrid"},
+    {L"Russia Time Zone 10", L"001", "Asia/Srednekolymsk"},
+    {L"Russia Time Zone 11", L"001", "Asia/Kamchatka"},
+    {L"Russia Time Zone 3", L"001", "Europe/Samara"},
+    {L"Russian Standard Time", L"001", "Europe/Moscow"},
+    {L"Russian Standard Time", L"UA", "Europe/Simferopol"},
+    {L"SA Eastern Standard Time", L"001", "America/Cayenne"},
+    {L"SA Eastern Standard Time", L"AQ", "Antarctica/Rothera"},
+    {L"SA Eastern Standard Time", L"BR", "America/Fortaleza"},
+    {L"SA Eastern Standard Time", L"FK", "Atlantic/Stanley"},
+    {L"SA Eastern Standard Time", L"SR", "America/Paramaribo"},
+    {L"SA Eastern Standard Time", L"ZZ", "Etc/GMT+3"},
+    {L"SA Pacific Standard Time", L"001", "America/Bogota"},
+    {L"SA Pacific Standard Time", L"BR", "America/Rio_Branco"},
+    {L"SA Pacific Standard Time", L"CA", "America/Coral_Harbour"},
+    {L"SA Pacific Standard Time", L"EC", "America/Guayaquil"},
+    {L"SA Pacific Standard Time", L"JM", "America/Jamaica"},
+    {L"SA Pacific Standard Time", L"KY", "America/Cayman"},
+    {L"SA Pacific Standard Time", L"PA", "America/Panama"},
+    {L"SA Pacific Standard Time", L"PE", "America/Lima"},
+    {L"SA Pacific Standard Time", L"ZZ", "Etc/GMT+5"},
+    {L"SA Western Standard Time", L"001", "America/La_Paz"},
+    {L"SA Western Standard Time", L"AG", "America/Antigua"},
+    {L"SA Western Standard Time", L"AI", "America/Anguilla"},
+    {L"SA Western Standard Time", L"AW", "America/Aruba"},
+    {L"SA Western Standard Time", L"BB", "America/Barbados"},
+    {L"SA Western Standard Time", L"BL", "America/St_Barthelemy"},
+    {L"SA Western Standard Time", L"BQ", "America/Kralendijk"},
+    {L"SA Western Standard Time", L"BR", "America/Manaus"},
+    {L"SA Western Standard Time", L"CA", "America/Blanc-Sablon"},
+    {L"SA Western Standard Time", L"CW", "America/Curacao"},
+    {L"SA Western Standard Time", L"DM", "America/Dominica"},
+    {L"SA Western Standard Time", L"DO", "America/Santo_Domingo"},
+    {L"SA Western Standard Time", L"GD", "America/Grenada"},
+    {L"SA Western Standard Time", L"GP", "America/Guadeloupe"},
+    {L"SA Western Standard Time", L"GY", "America/Guyana"},
+    {L"SA Western Standard Time", L"KN", "America/St_Kitts"},
+    {L"SA Western Standard Time", L"LC", "America/St_Lucia"},
+    {L"SA Western Standard Time", L"MF", "America/Marigot"},
+    {L"SA Western Standard Time", L"MQ", "America/Martinique"},
+    {L"SA Western Standard Time", L"MS", "America/Montserrat"},
+    {L"SA Western Standard Time", L"PR", "America/Puerto_Rico"},
+    {L"SA Western Standard Time", L"SX", "America/Lower_Princes"},
+    {L"SA Western Standard Time", L"TT", "America/Port_of_Spain"},
+    {L"SA Western Standard Time", L"VC", "America/St_Vincent"},
+    {L"SA Western Standard Time", L"VG", "America/Tortola"},
+    {L"SA Western Standard Time", L"VI", "America/St_Thomas"},
+    {L"SA Western Standard Time", L"ZZ", "Etc/GMT+4"},
+    {L"SE Asia Standard Time", L"001", "Asia/Bangkok"},
+    {L"SE Asia Standard Time", L"AQ", "Antarctica/Davis"},
+    {L"SE Asia Standard Time", L"CX", "Indian/Christmas"},
+    {L"SE Asia Standard Time", L"ID", "Asia/Jakarta"},
+    {L"SE Asia Standard Time", L"KH", "Asia/Phnom_Penh"},
+    {L"SE Asia Standard Time", L"LA", "Asia/Vientiane"},
+    {L"SE Asia Standard Time", L"VN", "Asia/Saigon"},
+    {L"SE Asia Standard Time", L"ZZ", "Etc/GMT-7"},
+    {L"Saint Pierre Standard Time", L"001", "America/Miquelon"},
+    {L"Sakhalin Standard Time", L"001", "Asia/Sakhalin"},
+    {L"Samoa Standard Time", L"001", "Pacific/Apia"},
+    {L"Sao Tome Standard Time", L"001", "Africa/Sao_Tome"},
+    {L"Saratov Standard Time", L"001", "Europe/Saratov"},
+    {L"Singapore Standard Time", L"001", "Asia/Singapore"},
+    {L"Singapore Standard Time", L"BN", "Asia/Brunei"},
+    {L"Singapore Standard Time", L"ID", "Asia/Makassar"},
+    {L"Singapore Standard Time", L"MY", "Asia/Kuala_Lumpur"},
+    {L"Singapore Standard Time", L"PH", "Asia/Manila"},
+    {L"Singapore Standard Time", L"ZZ", "Etc/GMT-8"},
+    {L"South Africa Standard Time", L"001", "Africa/Johannesburg"},
+    {L"South Africa Standard Time", L"BI", "Africa/Bujumbura"},
+    {L"South Africa Standard Time", L"BW", "Africa/Gaborone"},
+    {L"South Africa Standard Time", L"CD", "Africa/Lubumbashi"},
+    {L"South Africa Standard Time", L"LS", "Africa/Maseru"},
+    {L"South Africa Standard Time", L"MW", "Africa/Blantyre"},
+    {L"South Africa Standard Time", L"MZ", "Africa/Maputo"},
+    {L"South Africa Standard Time", L"RW", "Africa/Kigali"},
+    {L"South Africa Standard Time", L"SZ", "Africa/Mbabane"},
+    {L"South Africa Standard Time", L"ZM", "Africa/Lusaka"},
+    {L"South Africa Standard Time", L"ZW", "Africa/Harare"},
+    {L"South Africa Standard Time", L"ZZ", "Etc/GMT-2"},
+    {L"South Sudan Standard Time", L"001", "Africa/Juba"},
+    {L"Sri Lanka Standard Time", L"001", "Asia/Colombo"},
+    {L"Sudan Standard Time", L"001", "Africa/Khartoum"},
+    {L"Syria Standard Time", L"001", "Asia/Damascus"},
+    {L"Taipei Standard Time", L"001", "Asia/Taipei"},
+    {L"Tasmania Standard Time", L"001", "Australia/Hobart"},
+    {L"Tocantins Standard Time", L"001", "America/Araguaina"},
+    {L"Tokyo Standard Time", L"001", "Asia/Tokyo"},
+    {L"Tokyo Standard Time", L"ID", "Asia/Jayapura"},
+    {L"Tokyo Standard Time", L"PW", "Pacific/Palau"},
+    {L"Tokyo Standard Time", L"TL", "Asia/Dili"},
+    {L"Tokyo Standard Time", L"ZZ", "Etc/GMT-9"},
+    {L"Tomsk Standard Time", L"001", "Asia/Tomsk"},
+    {L"Tonga Standard Time", L"001", "Pacific/Tongatapu"},
+    {L"Transbaikal Standard Time", L"001", "Asia/Chita"},
+    {L"Turkey Standard Time", L"001", "Europe/Istanbul"},
+    {L"Turks And Caicos Standard Time", L"001", "America/Grand_Turk"},
+    {L"US Eastern Standard Time", L"001", "America/Indianapolis"},
+    {L"US Mountain Standard Time", L"001", "America/Phoenix"},
+    {L"US Mountain Standard Time", L"CA", "America/Creston"},
+    {L"US Mountain Standard Time", L"MX", "America/Hermosillo"},
+    {L"US Mountain Standard Time", L"ZZ", "Etc/GMT+7"},
+    {L"UTC", L"001", "Etc/UTC"},
+    {L"UTC+12", L"001", "Etc/GMT-12"},
+    {L"UTC+12", L"KI", "Pacific/Tarawa"},
+    {L"UTC+12", L"MH", "Pacific/Majuro"},
+    {L"UTC+12", L"NR", "Pacific/Nauru"},
+    {L"UTC+12", L"TV", "Pacific/Funafuti"},
+    {L"UTC+12", L"UM", "Pacific/Wake"},
+    {L"UTC+12", L"WF", "Pacific/Wallis"},
+    {L"UTC+13", L"001", "Etc/GMT-13"},
+    {L"UTC+13", L"KI", "Pacific/Enderbury"},
+    {L"UTC+13", L"TK", "Pacific/Fakaofo"},
+    {L"UTC-02", L"001", "Etc/GMT+2"},
+    {L"UTC-02", L"BR", "America/Noronha"},
+    {L"UTC-02", L"GS", "Atlantic/South_Georgia"},
+    {L"UTC-08", L"001", "Etc/GMT+8"},
+    {L"UTC-08", L"PN", "Pacific/Pitcairn"},
+    {L"UTC-09", L"001", "Etc/GMT+9"},
+    {L"UTC-09", L"PF", "Pacific/Gambier"},
+    {L"UTC-11", L"001", "Etc/GMT+11"},
+    {L"UTC-11", L"AS", "Pacific/Pago_Pago"},
+    {L"UTC-11", L"NU", "Pacific/Niue"},
+    {L"UTC-11", L"UM", "Pacific/Midway"},
+    {L"Ulaanbaatar Standard Time", L"001", "Asia/Ulaanbaatar"},
+    {L"Venezuela Standard Time", L"001", "America/Caracas"},
+    {L"Vladivostok Standard Time", L"001", "Asia/Vladivostok"},
+    {L"Volgograd Standard Time", L"001", "Europe/Volgograd"},
+    {L"W. Australia Standard Time", L"001", "Australia/Perth"},
+    {L"W. Central Africa Standard Time", L"001", "Africa/Lagos"},
+    {L"W. Central Africa Standard Time", L"AO", "Africa/Luanda"},
+    {L"W. Central Africa Standard Time", L"BJ", "Africa/Porto-Novo"},
+    {L"W. Central Africa Standard Time", L"CD", "Africa/Kinshasa"},
+    {L"W. Central Africa Standard Time", L"CF", "Africa/Bangui"},
+    {L"W. Central Africa Standard Time", L"CG", "Africa/Brazzaville"},
+    {L"W. Central Africa Standard Time", L"CM", "Africa/Douala"},
+    {L"W. Central Africa Standard Time", L"DZ", "Africa/Algiers"},
+    {L"W. Central Africa Standard Time", L"GA", "Africa/Libreville"},
+    {L"W. Central Africa Standard Time", L"GQ", "Africa/Malabo"},
+    {L"W. Central Africa Standard Time", L"NE", "Africa/Niamey"},
+    {L"W. Central Africa Standard Time", L"TD", "Africa/Ndjamena"},
+    {L"W. Central Africa Standard Time", L"TN", "Africa/Tunis"},
+    {L"W. Central Africa Standard Time", L"ZZ", "Etc/GMT-1"},
+    {L"W. Europe Standard Time", L"001", "Europe/Berlin"},
+    {L"W. Europe Standard Time", L"AD", "Europe/Andorra"},
+    {L"W. Europe Standard Time", L"AT", "Europe/Vienna"},
+    {L"W. Europe Standard Time", L"CH", "Europe/Zurich"},
+    {L"W. Europe Standard Time", L"GI", "Europe/Gibraltar"},
+    {L"W. Europe Standard Time", L"IT", "Europe/Rome"},
+    {L"W. Europe Standard Time", L"LI", "Europe/Vaduz"},
+    {L"W. Europe Standard Time", L"LU", "Europe/Luxembourg"},
+    {L"W. Europe Standard Time", L"MC", "Europe/Monaco"},
+    {L"W. Europe Standard Time", L"MT", "Europe/Malta"},
+    {L"W. Europe Standard Time", L"NL", "Europe/Amsterdam"},
+    {L"W. Europe Standard Time", L"NO", "Europe/Oslo"},
+    {L"W. Europe Standard Time", L"SE", "Europe/Stockholm"},
+    {L"W. Europe Standard Time", L"SJ", "Arctic/Longyearbyen"},
+    {L"W. Europe Standard Time", L"SM", "Europe/San_Marino"},
+    {L"W. Europe Standard Time", L"VA", "Europe/Vatican"},
+    {L"W. Mongolia Standard Time", L"001", "Asia/Hovd"},
+    {L"West Asia Standard Time", L"001", "Asia/Tashkent"},
+    {L"West Asia Standard Time", L"AQ", "Antarctica/Mawson"},
+    {L"West Asia Standard Time", L"KZ", "Asia/Oral"},
+    {L"West Asia Standard Time", L"MV", "Indian/Maldives"},
+    {L"West Asia Standard Time", L"TF", "Indian/Kerguelen"},
+    {L"West Asia Standard Time", L"TJ", "Asia/Dushanbe"},
+    {L"West Asia Standard Time", L"TM", "Asia/Ashgabat"},
+    {L"West Asia Standard Time", L"ZZ", "Etc/GMT-5"},
+    {L"West Bank Standard Time", L"001", "Asia/Hebron"},
+    {L"West Pacific Standard Time", L"001", "Pacific/Port_Moresby"},
+    {L"West Pacific Standard Time", L"AQ", "Antarctica/DumontDUrville"},
+    {L"West Pacific Standard Time", L"FM", "Pacific/Truk"},
+    {L"West Pacific Standard Time", L"GU", "Pacific/Guam"},
+    {L"West Pacific Standard Time", L"MP", "Pacific/Saipan"},
+    {L"West Pacific Standard Time", L"ZZ", "Etc/GMT-10"},
+    {L"Yakutsk Standard Time", L"001", "Asia/Yakutsk"},
+    {L"Yukon Standard Time", L"001", "America/Whitehorse"},
+  }
+};
+
+#define _GLIBCXX_WINDOWS_ZONES_MAP_COMPLETE
diff --git a/libstdc++-v3/src/c++23/std.cc.in b/libstdc++-v3/src/c++23/std.cc.in
index 483b6c6..9301ed9 100644
--- a/libstdc++-v3/src/c++23/std.cc.in
+++ b/libstdc++-v3/src/c++23/std.cc.in
@@ -278,10 +278,14 @@ export namespace std
   }
   using std::shift_left;
   namespace ranges
-  {}
+  {
+    using std::ranges::shift_left;
+  }
   using std::shift_right;
   namespace ranges
-  {}
+  {
+    using std::ranges::shift_right;
+  }
   using std::sort;
   namespace ranges
   {
@@ -1509,7 +1513,15 @@ export namespace std
   using std::initializer_list;
 }
 
-// <inplace_vector> FIXME
+// <inplace_vector>
+#if __cpp_lib_inplace_vector
+export namespace std
+{
+  using std::inplace_vector;
+  using std::erase;
+  using std::erase_if;
+}
+#endif
 
 // <iomanip>
 export namespace std
@@ -1838,11 +1850,15 @@ export namespace std
 export namespace std
 {
   using std::extents;
+  using std::dextents;
   using std::layout_left;
   using std::layout_right;
   using std::layout_stride;
   using std::default_accessor;
-  // FIXME layout_left_padded, layout_right_padded, aligned_accessor and mdspan
+  using std::mdspan;
+  // FIXME layout_left_padded, layout_right_padded, aligned_accessor,
+  // strided_slice, submdspan_mapping_result, full_extent_t, full_extent,
+  // submdspan_extents, mdsubspan
 }
 #endif
 
diff --git a/libstdc++-v3/testsuite/18_support/exception/version.cc b/libstdc++-v3/testsuite/18_support/exception/version.cc
new file mode 100644
index 0000000..5707abd
--- /dev/null
+++ b/libstdc++-v3/testsuite/18_support/exception/version.cc
@@ -0,0 +1,9 @@
+// { dg-do preprocess { target c++26 } }
+// { dg-add-options no_pch }
+
+#include <exception>
+
+#ifdef __cpp_lib_constexpr_exceptions
+# error "Feature test macro for constexpr_exceptions should not be provided by <exception>"
+#endif
+
diff --git a/libstdc++-v3/testsuite/18_support/exception_ptr/exception_ptr_cast.cc b/libstdc++-v3/testsuite/18_support/exception_ptr/exception_ptr_cast.cc
index 6a6fbfe..de371d1 100644
--- a/libstdc++-v3/testsuite/18_support/exception_ptr/exception_ptr_cast.cc
+++ b/libstdc++-v3/testsuite/18_support/exception_ptr/exception_ptr_cast.cc
@@ -30,13 +30,13 @@ struct A { int a; };
 struct B : A {};
 struct C : B {};
 struct D {};
-struct E : virtual C { int e; virtual ~E () {} };
+struct E : virtual C { int e; constexpr virtual ~E () {} };
 struct F : virtual E, virtual C { int f; };
 struct G : virtual F, virtual C, virtual E {
-  G () : g (4) { a = 1; e = 2; f = 3; } int g;
+  constexpr G () : g (4) { a = 1; e = 2; f = 3; } int g;
 };
 
-void test01()
+constexpr bool test01(bool x)
 {
   auto a = std::make_exception_ptr(C{ 42 });
   auto b = std::exception_ptr_cast<C>(a);
@@ -73,9 +73,20 @@ void test01()
       auto n = std::exception_ptr_cast<G>(a);
       VERIFY( n == nullptr );
     }
+  if (x)
+    throw 1;
+  return true;
 }
 
+static_assert(test01(false));
+
 int main()
 {
-  test01();
+  try
+    {
+      test01(true);
+    }
+  catch (...)
+    {
+    }
 }
diff --git a/libstdc++-v3/testsuite/20_util/any/misc/any_cast_neg.cc b/libstdc++-v3/testsuite/20_util/any/misc/any_cast_neg.cc
index 9740e09..9e5c64c 100644
--- a/libstdc++-v3/testsuite/20_util/any/misc/any_cast_neg.cc
+++ b/libstdc++-v3/testsuite/20_util/any/misc/any_cast_neg.cc
@@ -27,6 +27,7 @@ void test01()
   const any y(1);
   any_cast<int&>(y); // { dg-error "here" }
   // { dg-error "Template argument must be constructible from a const value" "" { target { *-*-* } } 0 }
+  // { dg-error "binding reference of type 'int&' to 'const int' discards qualifiers" "" { target { *-*-* } } 0 }
 }
 
 void test02()
@@ -34,6 +35,7 @@ void test02()
   any y(1);
   any_cast<int&&>(y); // { dg-error "here" }
   // { dg-error "Template argument must be constructible from an lvalue" "" { target { *-*-* } } 0 }
+  // { dg-error "cannot bind rvalue reference of type 'int&&' to lvalue of type 'int'" "" { target { *-*-* } } 0 }
 }
 
 void test03()
@@ -41,6 +43,7 @@ void test03()
   any y(1);
   any_cast<int&>(std::move(y)); // { dg-error "here" }
   // { dg-error "Template argument must be constructible from an rvalue" "" { target { *-*-* } } 0 }
+  // { dg-error "cannot bind non-const lvalue reference of type 'int&' to an rvalue of type 'int'" "" { target { *-*-* } } 0 }
 }
 
 // { dg-prune-output "invalid 'static_cast'" }
diff --git a/libstdc++-v3/testsuite/20_util/expected/illformed_neg.cc b/libstdc++-v3/testsuite/20_util/expected/illformed_neg.cc
index 69c13b4..69aa4a1 100644
--- a/libstdc++-v3/testsuite/20_util/expected/illformed_neg.cc
+++ b/libstdc++-v3/testsuite/20_util/expected/illformed_neg.cc
@@ -13,6 +13,7 @@ test_unexpected()
   std::unexpected<void()> func(test_unexpected); // { dg-error "here" }
   // { dg-error "no matching function for call to" "" { target *-*-* } 0 }
   // { dg-error "invalidly declared function type" "" { target *-*-* } 0 }
+  // { dg-error "could not convert" "" { target *-*-* } 0 }
 
   // an array type,
   std::unexpected<int[2]> array(i); // { dg-error "here" }
diff --git a/libstdc++-v3/testsuite/20_util/hash/int128.cc b/libstdc++-v3/testsuite/20_util/hash/int128.cc
new file mode 100644
index 0000000..a26d2e2
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/hash/int128.cc
@@ -0,0 +1,20 @@
+// { dg-do run { target c++11 } }
+// { dg-add-options strict_std }
+
+#include <functional>
+#include <testsuite_hooks.h>
+
+int main()
+{
+#ifdef __SIZEOF_INT128__
+  std::hash<__int128> h;
+  __int128 i = (__int128)0x123456789;
+  VERIFY( h(i) == (std::size_t)i );
+  VERIFY( h(-i) == (std::size_t)-i );
+  VERIFY( h(~i) == (std::size_t)~i );
+  std::hash<unsigned __int128> hu;
+  unsigned __int128 u = i;
+  VERIFY( hu(u) == (std::size_t)u );
+  VERIFY( hu(~u) == (std::size_t)~u );
+#endif
+}
diff --git a/libstdc++-v3/testsuite/20_util/make_unsigned/int128.cc b/libstdc++-v3/testsuite/20_util/make_unsigned/int128.cc
new file mode 100644
index 0000000..46c07b7
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/make_unsigned/int128.cc
@@ -0,0 +1,14 @@
+// { dg-do compile { target c++11 } }
+// { dg-add-options strict_std }
+
+#include <type_traits>
+
+#ifdef __SIZEOF_INT128__
+enum E : __int128 { };
+using U = std::make_unsigned<E>::type;
+static_assert( std::is_integral<U>::value, "type is an integer" );
+static_assert( sizeof(U) == sizeof(E), "width of type is 128 bits" );
+using I = std::make_signed<E>::type;
+static_assert( std::is_integral<I>::value, "type is an integer" );
+static_assert( sizeof(I) == sizeof(E), "width of type is 128 bits" );
+#endif
diff --git a/libstdc++-v3/testsuite/20_util/optional/monadic/or_else_neg.cc b/libstdc++-v3/testsuite/20_util/optional/monadic/or_else_neg.cc
index f5028c1..12a67bb 100644
--- a/libstdc++-v3/testsuite/20_util/optional/monadic/or_else_neg.cc
+++ b/libstdc++-v3/testsuite/20_util/optional/monadic/or_else_neg.cc
@@ -26,4 +26,5 @@ test02()
 
   std::optional<move_only> mo;
   mo.or_else([]{ return std::optional<move_only>{}; }); // { dg-error "no matching function" }
+  // { dg-error "use of deleted function" "" { target *-*-* } 0 }
 }
diff --git a/libstdc++-v3/testsuite/20_util/owner_equal/cmp.cc b/libstdc++-v3/testsuite/20_util/owner_equal/cmp.cc
new file mode 100644
index 0000000..311ddf2
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/owner_equal/cmp.cc
@@ -0,0 +1,105 @@
+// { dg-do run { target c++26 } }
+// { dg-require-effective-target hosted }
+
+// N5008 20.3.2.6 Struct owner_equal [util.smartptr.owner.equal]
+
+#include <memory>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+struct A { };
+
+struct B { A a[2]; };
+
+int
+test01()
+{
+  // test empty shared_ptr owners compare equivalent
+  std::owner_equal eq;
+  std::shared_ptr<A> p1;
+  std::shared_ptr<A> p2;
+  VERIFY( eq(p1, p2) && eq(p2, p1) );
+  std::weak_ptr<A> p3;
+  VERIFY( eq(p1, p3) && eq(p3, p1) );
+  VERIFY( eq(p1, p3) && eq(p3, p1) );
+  return 0;
+}
+
+
+// Construction from pointer
+int
+test02()
+{
+  std::owner_equal eq;
+
+  std::shared_ptr<A> empty;
+
+  std::shared_ptr<A> a1(new A);
+  VERIFY( !eq(empty, a1) && !eq(a1, empty) );
+
+  std::shared_ptr<A> a2(new A);
+  VERIFY( !eq(a1, a2) && !eq(a2, a1) );
+
+  std::weak_ptr<A> w1(a1);
+  VERIFY( eq(a1, w1) && eq(w1, a1) );
+
+  std::weak_ptr<A> w2(a2);
+  VERIFY( !eq(w1, w2) && !eq(w2, w1) );
+
+  a1.reset();
+  VERIFY( eq(empty, a1) && eq(a1, empty) );
+  VERIFY( !eq(a1, w1) && !eq(w1, a1) );
+
+  a2.reset();
+  VERIFY( eq(a2, a1) && eq(a1, a2) );
+
+  return 0;
+}
+
+// aliasing
+int
+test03()
+{
+  std::owner_equal eq;
+
+  std::shared_ptr<B> b(new B);
+  std::shared_ptr<A> a0(b, &b->a[0]);
+  std::shared_ptr<A> a1(b, &b->a[1]);
+  // values are different but owners are equivalent:
+  VERIFY( a0 < a1 && eq(a0, a1) && eq(b, a0) && eq(b, a1) );
+
+  std::weak_ptr<A> w0(a0);
+  std::weak_ptr<A> w1(a1);
+  VERIFY( eq(w0, w1) && eq(w1, w0) );
+  VERIFY( eq(a0, w1) && eq(w1, a0) );
+  VERIFY( eq(w0, a1) && eq(a1, w0) );
+
+  return 0;
+}
+
+// as binary predicate
+int
+test04()
+{
+  std::owner_equal eq;
+
+  std::shared_ptr<B> b(new B);
+  std::shared_ptr<A> a0(b, &b->a[0]);
+  std::shared_ptr<A> a1(b, &b->a[1]);
+  std::shared_ptr<A> c(new A);
+  std::weak_ptr<A> a[3]{a0, a1, c};
+  std::weak_ptr<A>* p = std::unique(a, a+3, eq);
+  VERIFY( p == &a[2] );
+
+  return 0;
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  test04();
+  return 0;
+}
diff --git a/libstdc++-v3/testsuite/20_util/owner_equal/noexcept.cc b/libstdc++-v3/testsuite/20_util/owner_equal/noexcept.cc
new file mode 100644
index 0000000..fb479f6
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/owner_equal/noexcept.cc
@@ -0,0 +1,30 @@
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target hosted }
+
+// N5008 20.3.2.6 Struct owner_equal [util.smartptr.owner.equal]
+
+#include <memory>
+
+#ifndef __cpp_lib_smart_ptr_owner_equality
+# error "Feature-test macro for smart ptr owner equality missing in <memory>"
+#elif __cpp_lib_smart_ptr_owner_equality != 202306L
+# error "Feature-test macro for smart ptr owner equality has wrong value in <memory>"
+#endif
+
+const std::owner_equal eq;
+const std::shared_ptr<int> si;
+const std::weak_ptr<int> wi;
+static_assert( noexcept(!eq(si, si)) );
+static_assert( noexcept(!eq(si, wi)) );
+static_assert( noexcept(!eq(wi, si)) );
+static_assert( noexcept(!eq(wi, wi)) );
+static_assert( noexcept(!eq(si, wi)) );
+static_assert( noexcept(!eq(wi, si)) );
+const std::shared_ptr<long> sl;
+const std::weak_ptr<char> wc;
+static_assert( noexcept(!eq(si, si)) );
+static_assert( noexcept(!eq(si, sl)) );
+static_assert( noexcept(!eq(sl, si)) );
+static_assert( noexcept(!eq(si, wc)) );
+static_assert( noexcept(!eq(wc, si)) );
+static_assert( noexcept(!eq(wc, wi)) );
diff --git a/libstdc++-v3/testsuite/20_util/owner_equal/version.cc b/libstdc++-v3/testsuite/20_util/owner_equal/version.cc
new file mode 100644
index 0000000..db29154
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/owner_equal/version.cc
@@ -0,0 +1,13 @@
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target hosted }
+
+// N5008 17.3.2 Header <version> synopsis [version.syn]
+
+#include <version>
+
+#ifndef __cpp_lib_smart_ptr_owner_equality
+# error "Feature-test macro for smart ptr owner equality missing in <version>"
+#elif __cpp_lib_smart_ptr_owner_equality != 202306L
+# error "Feature-test macro for smart ptr owner equality has wrong value in <version>"
+#endif
+
diff --git a/libstdc++-v3/testsuite/20_util/owner_hash/cmp.cc b/libstdc++-v3/testsuite/20_util/owner_hash/cmp.cc
new file mode 100644
index 0000000..c03a926
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/owner_hash/cmp.cc
@@ -0,0 +1,87 @@
+// { dg-do run { target c++26 } }
+// { dg-require-effective-target hosted }
+
+// N5008 20.3.2.5 Struct owner_hash [util.smartptr.owner.hash]
+
+#include <memory>
+#include <algorithm>
+#include <testsuite_hooks.h>
+
+struct A { };
+
+struct B { A a[2]; };
+
+int
+test01()
+{
+  // test empty shared_ptr hashes compare equivalent
+  std::owner_hash oh;
+  std::shared_ptr<A> p1;
+  std::shared_ptr<A> p2;
+  VERIFY( oh(p1) == oh(p2) );
+  std::weak_ptr<A> p3;
+  VERIFY( oh(p1) == oh(p3) );
+  VERIFY( oh(p1) == oh(p3) );
+  return 0;
+}
+
+
+// Construction from pointer
+int
+test02()
+{
+  std::owner_hash oh;
+
+  std::shared_ptr<A> empty;
+
+  std::shared_ptr<A> a1(new A);
+  VERIFY( oh(empty) != oh(a1) );
+
+  std::shared_ptr<A> a2(new A);
+  VERIFY( oh(a1) != oh(a2) );
+
+  std::weak_ptr<A> w1(a1);
+  VERIFY( oh(a1) == oh(w1) );
+
+  std::weak_ptr<A> w2(a2);
+  VERIFY( oh(w1) != oh(w2) );
+
+  a1.reset();
+  VERIFY( oh(empty) == oh(a1) );
+  VERIFY( oh(a1) != oh(w1) );
+
+  a2.reset();
+  VERIFY( oh(a2) == oh(a1) );
+
+  return 0;
+}
+
+// aliasing
+int
+test03()
+{
+  std::owner_hash oh;
+
+  std::shared_ptr<B> b(new B);
+  std::shared_ptr<A> a0(b, &b->a[0]);
+  std::shared_ptr<A> a1(b, &b->a[1]);
+  // values are different but owners are ohuivalent:
+  VERIFY( a0 < a1 && oh(a0) == oh(a1) && oh(b) == oh(a0) && oh(b) == oh(a1) );
+
+  std::weak_ptr<A> w0(a0);
+  std::weak_ptr<A> w1(a1);
+  VERIFY( oh(w0) == oh(w1) );
+  VERIFY( oh(a0) == oh(w1) );
+  VERIFY( oh(w0) == oh(a1) );
+
+  return 0;
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  return 0;
+}
diff --git a/libstdc++-v3/testsuite/20_util/owner_hash/noexcept.cc b/libstdc++-v3/testsuite/20_util/owner_hash/noexcept.cc
new file mode 100644
index 0000000..12b2f2f
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/owner_hash/noexcept.cc
@@ -0,0 +1,16 @@
+// { dg-do compile { target c++26 } }
+// { dg-require-effective-target hosted }
+
+// N5008 20.3.2.5 Struct owner_hash [util.smartptr.owner.hash]
+
+#include <memory>
+
+const std::owner_hash oh;
+const std::shared_ptr<int> si;
+const std::weak_ptr<int> wi;
+static_assert( noexcept(!oh(si)) );
+static_assert( noexcept(!oh(wi)) );
+const std::shared_ptr<long> sl;
+const std::weak_ptr<char> wc;
+static_assert( noexcept(!oh(sl)) );
+static_assert( noexcept(!oh(wc)) );
diff --git a/libstdc++-v3/testsuite/20_util/shared_ptr/observers/owner_equal.cc b/libstdc++-v3/testsuite/20_util/shared_ptr/observers/owner_equal.cc
new file mode 100644
index 0000000..7ec8691
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/shared_ptr/observers/owner_equal.cc
@@ -0,0 +1,74 @@
+// { dg-do run { target c++26 } }
+// { dg-require-effective-target hosted }
+
+// N5008 20.3.2.2.6 shared_ptr observers [util.smartptr.shared.obs]
+
+#include <memory>
+#include <testsuite_hooks.h>
+
+struct A
+{
+  int i;
+  virtual ~A() { }
+};
+
+struct B : A
+{
+};
+
+void
+test01()
+{
+  // test empty shared_ptr owners compare equivalent
+  std::shared_ptr<A> p1;
+  std::shared_ptr<B> p2;
+  VERIFY( p1.owner_equal(p2) && p2.owner_equal(p1) );
+}
+
+
+// Construction from pointer
+void
+test02()
+{
+  std::shared_ptr<A> a0;
+
+  std::shared_ptr<A> a1(new A);
+  VERIFY( !a1.owner_equal(a0) && !a0.owner_equal(a1) );
+
+  std::shared_ptr<B> b1(new B);
+  VERIFY( !a1.owner_equal(b1) && !b1.owner_equal(a1) );
+
+  std::shared_ptr<A> a2(a1);
+  VERIFY( a1.owner_equal(a2) && a2.owner_equal(a1) );
+  a2 = b1;
+  VERIFY( b1.owner_equal(a2) && a2.owner_equal(b1) );
+
+  std::weak_ptr<A> w1(a1);
+  VERIFY( a1.owner_equal(w1) && w1.owner_equal(a1) );
+  std::weak_ptr<A> w2(a2);
+  VERIFY( b1.owner_equal(w2) && w2.owner_equal(b1) );
+
+  static_assert( noexcept(a1.owner_equal(a0)) );
+  static_assert( noexcept(a1.owner_equal(b1)) );
+  static_assert( noexcept(b1.owner_equal(a1)) );
+  static_assert( noexcept(a1.owner_equal(w1)) );
+  static_assert( noexcept(b1.owner_equal(w1)) );
+}
+
+// Aliasing
+void
+test03()
+{
+  std::shared_ptr<A> p1(new A());
+  std::shared_ptr<int> p2(p1, &p1->i);
+  VERIFY( p1.owner_equal(p2) && p2.owner_equal(p1) );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  return 0;
+}
diff --git a/libstdc++-v3/testsuite/20_util/shared_ptr/observers/owner_hash.cc b/libstdc++-v3/testsuite/20_util/shared_ptr/observers/owner_hash.cc
new file mode 100644
index 0000000..8e6c02c
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/shared_ptr/observers/owner_hash.cc
@@ -0,0 +1,71 @@
+// { dg-do run { target c++26 } }
+// { dg-require-effective-target hosted }
+
+// N5008 20.3.2.2.6 shared_ptr observers [util.smartptr.shared.obs]
+
+#include <memory>
+#include <testsuite_hooks.h>
+
+struct A
+{
+  int i;
+  virtual ~A() { }
+};
+
+struct B : A
+{
+};
+
+void
+test01()
+{
+  // test empty shared_ptr hashes compare equivalent
+  std::shared_ptr<A> p1;
+  std::shared_ptr<B> p2;
+  VERIFY( p1.owner_hash() == p2.owner_hash() );
+}
+
+
+// Construction from pointer
+void
+test02()
+{
+  std::shared_ptr<A> a0;
+
+  std::shared_ptr<A> a1(new A);
+  VERIFY( a1.owner_hash() != a0.owner_hash() );
+
+  std::shared_ptr<B> b1(new B);
+  VERIFY( a1.owner_hash() != b1.owner_hash() );
+
+  std::shared_ptr<A> a2(a1);
+  VERIFY( a1.owner_hash() == a2.owner_hash() );
+  a2 = b1;
+  VERIFY( b1.owner_hash() == a2.owner_hash() );
+
+  std::weak_ptr<A> w1(a1);
+  VERIFY( a1.owner_hash() == w1.owner_hash() );
+  std::weak_ptr<A> w2(a2);
+  VERIFY( b1.owner_hash() == w2.owner_hash() );
+
+  static_assert( noexcept(a1.owner_hash()) );
+  static_assert( noexcept(b1.owner_hash()) );
+}
+
+// Aliasing
+void
+test03()
+{
+  std::shared_ptr<A> p1(new A());
+  std::shared_ptr<int> p2(p1, &p1->i);
+  VERIFY( p1.owner_hash() == p2.owner_hash() );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  return 0;
+}
diff --git a/libstdc++-v3/testsuite/20_util/specialized_algorithms/destroy/121024.cc b/libstdc++-v3/testsuite/20_util/specialized_algorithms/destroy/121024.cc
new file mode 100644
index 0000000..781dd40
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/specialized_algorithms/destroy/121024.cc
@@ -0,0 +1,77 @@
+// { dg-do compile { target c++26 } }
+
+// Bug 121024
+// ranges::destroy and ranges::destroy_n do not end lifetime of trivial types
+
+#include <memory>
+
+consteval bool is_within_lifetime(const auto* p) noexcept
+{
+  return __builtin_constant_p(*p);
+}
+
+template<typename T>
+struct Buf
+{
+  constexpr Buf() : p(std::allocator<T>().allocate(2)) { }
+  constexpr ~Buf() { std::allocator<T>().deallocate(p, 2); }
+  T* p;
+};
+
+template<typename T>
+consteval bool
+test_destroy()
+{
+  Buf<T> buf;
+  std::uninitialized_value_construct(buf.p, buf.p + 2);
+  std::destroy(buf.p, buf.p + 2);
+  return not is_within_lifetime(buf.p) && not is_within_lifetime(buf.p + 1);
+}
+
+template<typename T>
+consteval bool
+test_destroy_n()
+{
+  Buf<T> buf;
+  std::uninitialized_value_construct_n(buf.p, 2);
+  std::destroy_n(buf.p, 2);
+  return not is_within_lifetime(buf.p) && not is_within_lifetime(buf.p + 1);
+}
+
+template<typename T>
+consteval bool
+test_ranges_destroy()
+{
+  Buf<T> buf;
+  std::uninitialized_value_construct(buf.p, buf.p + 2);
+  std::ranges::destroy(buf.p, buf.p + 2);
+  return not is_within_lifetime(buf.p) && not is_within_lifetime(buf.p + 1);
+}
+
+template<typename T>
+consteval bool
+test_ranges_destroy_n()
+{
+  Buf<T> buf;
+  std::uninitialized_value_construct_n(buf.p, 2);
+  std::ranges::destroy_n(buf.p, 2);
+  return not is_within_lifetime(buf.p) && not is_within_lifetime(buf.p + 1);
+}
+
+struct O
+{
+  constexpr O() { }
+  constexpr ~O() { }
+};
+
+// These all fail for GCC because is_within_lifetime still returns true
+// after the lifetime has been ended.
+// { dg-xfail-if "PR c++/102284" { *-*-* } }
+static_assert( test_destroy<int>() );
+static_assert( test_destroy<O>() );
+static_assert( test_destroy_n<int>() );
+static_assert( test_destroy_n<O>() );
+static_assert( test_ranges_destroy<int>() );
+static_assert( test_ranges_destroy<O>() );
+static_assert( test_ranges_destroy_n<int>() );
+static_assert( test_ranges_destroy_n<O>() );
diff --git a/libstdc++-v3/testsuite/20_util/synchronized_pool_resource/118681.cc b/libstdc++-v3/testsuite/20_util/synchronized_pool_resource/118681.cc
new file mode 100644
index 0000000..facbf00
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/synchronized_pool_resource/118681.cc
@@ -0,0 +1,6 @@
+// { dg-do run { target c++17 } }
+// { dg-require-gthreads "" }
+// Bug 118681 - unsynchronized_pool_resource may fail to respect alignment
+
+#define RESOURCE std::pmr::synchronized_pool_resource
+#include "../unsynchronized_pool_resource/118681.cc"
diff --git a/libstdc++-v3/testsuite/20_util/unsynchronized_pool_resource/118681.cc b/libstdc++-v3/testsuite/20_util/unsynchronized_pool_resource/118681.cc
new file mode 100644
index 0000000..9935f79
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/unsynchronized_pool_resource/118681.cc
@@ -0,0 +1,58 @@
+// { dg-do run { target c++17 } }
+// Bug 118681 - unsynchronized_pool_resource may fail to respect alignment
+
+#include <memory_resource>
+#include <cstdio>
+#include <testsuite_hooks.h>
+
+#ifndef RESOURCE
+# define RESOURCE std::pmr::unsynchronized_pool_resource
+#endif
+
+bool any_misaligned = false;
+
+bool
+is_aligned(void* p, [[maybe_unused]] std::size_t size, std::size_t alignment)
+{
+  const bool misaligned = reinterpret_cast<std::uintptr_t>(p) % alignment;
+#ifdef DEBUG
+  std::printf("allocate(%2zu, %2zu): %p is aligned %scorrectly\n",
+	      size, alignment, p, misaligned ? "in" : "");
+  any_misaligned |= misaligned;
+  return true;
+#endif
+  return ! misaligned;
+}
+
+void
+test_alignment(std::pmr::memory_resource& res, bool dealloc)
+{
+  for (std::size_t alignment : { 8, 16, 32, 64 })
+  {
+    for (std::size_t size : { 9, 12, 24, 40, 48, 56, 72 })
+    {
+      void* p1 = res.allocate(size, alignment);
+      void* p2 = res.allocate(size, alignment);
+
+      VERIFY( is_aligned(p1, size, alignment) );
+      VERIFY( is_aligned(p2, size, alignment) );
+
+      if (dealloc)
+      {
+	res.deallocate(p1, size, alignment);
+	res.deallocate(p2, size, alignment);
+      }
+    }
+  }
+}
+
+int main()
+{
+  RESOURCE res;
+  test_alignment(res, true);
+  res.release();
+  test_alignment(res, false);
+  res.release();
+
+  VERIFY( ! any_misaligned );
+}
diff --git a/libstdc++-v3/testsuite/20_util/weak_ptr/observers/owner_equal.cc b/libstdc++-v3/testsuite/20_util/weak_ptr/observers/owner_equal.cc
new file mode 100644
index 0000000..0217a6e
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/weak_ptr/observers/owner_equal.cc
@@ -0,0 +1,52 @@
+// { dg-do run { target c++26 } }
+// { dg-require-effective-target hosted }
+
+// N5008 20.3.2.3.6 weak_ptr observers [util.smartptr.weak.obs]
+
+#include <memory>
+#include <testsuite_hooks.h>
+
+struct A { };
+struct B { };
+
+void
+test01()
+{
+  // test empty weak_ptr owners compare equivalent
+  std::weak_ptr<A> p1;
+  std::weak_ptr<B> p2;
+  VERIFY( p1.owner_equal(p2) && p2.owner_equal(p1) );
+
+  std::shared_ptr<B> p3;
+  VERIFY( p1.owner_equal(p3) && p3.owner_equal(p1) );
+
+  static_assert( noexcept(p1.owner_equal(p1)) );
+  static_assert( noexcept(p1.owner_equal(p2)) );
+  static_assert( noexcept(p1.owner_equal(p3)) );
+  static_assert( noexcept(p2.owner_equal(p1)) );
+}
+
+
+void
+test02()
+{
+  std::shared_ptr<A> a0;
+  std::weak_ptr<A> w0(a0);
+
+  std::shared_ptr<A> a1(new A);
+  std::weak_ptr<A> w1(a1);
+  VERIFY(  a1.owner_equal(w1) &&  w1.owner_equal(a1) );
+  VERIFY( !w1.owner_equal(w0) && !w0.owner_equal(w1) );
+  VERIFY( !w1.owner_equal(a0) && !a0.owner_equal(w1) );
+
+  std::shared_ptr<B> b1(new B);
+  VERIFY( !w1.owner_equal(b1) && !b1.owner_equal(w1) );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  return 0;
+}
diff --git a/libstdc++-v3/testsuite/20_util/weak_ptr/observers/owner_hash.cc b/libstdc++-v3/testsuite/20_util/weak_ptr/observers/owner_hash.cc
new file mode 100644
index 0000000..148a93b
--- /dev/null
+++ b/libstdc++-v3/testsuite/20_util/weak_ptr/observers/owner_hash.cc
@@ -0,0 +1,50 @@
+// { dg-do run { target c++26 } }
+// { dg-require-effective-target hosted }
+
+// N5008 20.3.2.3.6 weak_ptr observers [util.smartptr.weak.obs]
+
+#include <memory>
+#include <testsuite_hooks.h>
+
+struct A { };
+struct B { };
+
+void
+test01()
+{
+  // test empty weak_ptr hashes compare equivalent
+  std::weak_ptr<A> p1;
+  std::weak_ptr<B> p2;
+  VERIFY( p1.owner_hash() == p2.owner_hash() );
+
+  std::shared_ptr<B> p3;
+  VERIFY( p1.owner_hash() == p3.owner_hash() );
+
+  static_assert( noexcept(p1.owner_hash()) );
+  static_assert( noexcept(p2.owner_hash()) );
+}
+
+
+void
+test02()
+{
+  std::shared_ptr<A> a0;
+  std::weak_ptr<A> w0(a0);
+
+  std::shared_ptr<A> a1(new A);
+  std::weak_ptr<A> w1(a1);
+  VERIFY( a1.owner_hash() == w1.owner_hash() );
+  VERIFY( w1.owner_hash() != w0.owner_hash() );
+  VERIFY( w1.owner_hash() != a0.owner_hash() );
+
+  std::shared_ptr<B> b1(new B);
+  VERIFY( w1.owner_hash() != b1.owner_hash() );
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  return 0;
+}
diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/cons/from_range.cc b/libstdc++-v3/testsuite/21_strings/basic_string/cons/from_range.cc
index 6331050..df9e4c3 100644
--- a/libstdc++-v3/testsuite/21_strings/basic_string/cons/from_range.cc
+++ b/libstdc++-v3/testsuite/21_strings/basic_string/cons/from_range.cc
@@ -73,16 +73,19 @@ do_test(Alloc alloc)
 }
 
 template<typename Range>
-void
+constexpr void
 do_test_a()
 {
   do_test<Range>(std::allocator<char>());
-  do_test<Range>(__gnu_test::uneq_allocator<char>(42));
   do_test<Range>(std::allocator<wchar_t>());
-  do_test<Range>(__gnu_test::uneq_allocator<wchar_t>(42));
+
+  if not consteval {
+    do_test<Range>(__gnu_test::uneq_allocator<char>(42));
+    do_test<Range>(__gnu_test::uneq_allocator<wchar_t>(42));
+  }
 }
 
-bool
+constexpr bool
 test_ranges()
 {
   using namespace __gnu_test;
@@ -101,9 +104,9 @@ test_ranges()
 
   // Not lvalue-convertible to char
   struct C {
-    C(char v) : val(v) { }
-    operator char() && { return val; }
-    bool operator==(char b) const { return b == val; }
+    constexpr C(char v) : val(v) { }
+    constexpr operator char() && { return val; }
+    constexpr bool operator==(char b) const { return b == val; }
     char val;
   };
   using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
@@ -112,18 +115,10 @@ test_ranges()
   return true;
 }
 
-constexpr bool
-test_constexpr()
-{
-#if _GLIBCXX_USE_CXX11_ABI
-  // XXX: this doesn't test the non-forward_range code paths are constexpr.
-  do_test<std::string_view>(std::allocator<char>());
-#endif // _GLIBCXX_USE_CXX11_ABI
-  return true;
-}
-
 int main()
 {
   test_ranges();
-  static_assert( test_constexpr() );
+#if _GLIBCXX_USE_CXX11_ABI
+  static_assert( test_ranges() );
+#endif // _GLIBCXX_USE_CXX11_ABI
 }
diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/append/append_range.cc b/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/append/append_range.cc
index 6c0bc0c..984db36 100644
--- a/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/append/append_range.cc
+++ b/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/append/append_range.cc
@@ -49,7 +49,7 @@ do_test()
 }
 
 template<typename Range>
-void
+constexpr void
 do_test_a()
 {
   do_test<Range, std::allocator<char>>();
@@ -58,7 +58,7 @@ do_test_a()
   do_test<Range, __gnu_test::SimpleAllocator<wchar_t>>();
 }
 
-bool
+constexpr bool
 test_ranges()
 {
   using namespace __gnu_test;
@@ -77,9 +77,9 @@ test_ranges()
 
   // Not lvalue-convertible to char
   struct C {
-    C(char v) : val(v) { }
-    operator char() && { return val; }
-    bool operator==(char b) const { return b == val; }
+    constexpr C(char v) : val(v) { }
+    constexpr operator char() && { return val; }
+    constexpr bool operator==(char b) const { return b == val; }
     char val;
   };
   using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
@@ -107,19 +107,11 @@ test_overlapping()
   VERIFY( c == "1234abcd1234" );
 }
 
-constexpr bool
-test_constexpr()
-{
-#if _GLIBCXX_USE_CXX11_ABI
-  // XXX: this doesn't test the non-forward_range code paths are constexpr.
-  do_test<std::string_view, std::allocator<char>>();
-#endif // _GLIBCXX_USE_CXX11_ABI
-  return true;
-}
-
 int main()
 {
   test_ranges();
   test_overlapping();
-  static_assert( test_constexpr() );
+#if _GLIBCXX_USE_CXX11_ABI
+  static_assert( test_ranges() );
+#endif // _GLIBCXX_USE_CXX11_ABI
 }
diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/assign/assign_range.cc b/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/assign/assign_range.cc
index 310c8bc..aa1b329 100644
--- a/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/assign/assign_range.cc
+++ b/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/assign/assign_range.cc
@@ -41,7 +41,7 @@ do_test()
 }
 
 template<typename Range>
-void
+constexpr void
 do_test_a()
 {
   do_test<Range, std::allocator<char>>();
@@ -50,7 +50,7 @@ do_test_a()
   do_test<Range, __gnu_test::SimpleAllocator<wchar_t>>();
 }
 
-bool
+constexpr bool
 test_ranges()
 {
   using namespace __gnu_test;
@@ -69,9 +69,9 @@ test_ranges()
 
   // Not lvalue-convertible to char
   struct C {
-    C(char v) : val(v) { }
-    operator char() && { return val; }
-    bool operator==(char b) const { return b == val; }
+    constexpr C(char v) : val(v) { }
+    constexpr operator char() && { return val; }
+    constexpr bool operator==(char b) const { return b == val; }
     char val;
   };
   using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
@@ -98,19 +98,11 @@ test_overlapping()
   VERIFY( c == "1234" );
 }
 
-constexpr bool
-test_constexpr()
-{
-#if _GLIBCXX_USE_CXX11_ABI
-  // XXX: this doesn't test the non-forward_range code paths are constexpr.
-  do_test<std::string_view, std::allocator<char>>();
-#endif // _GLIBCXX_USE_CXX11_ABI
-  return true;
-}
-
 int main()
 {
   test_ranges();
   test_overlapping();
-  static_assert( test_constexpr() );
+#if _GLIBCXX_USE_CXX11_ABI
+  static_assert( test_ranges() );
+#endif // _GLIBCXX_USE_CXX11_ABI
 }
diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/insert/insert_range.cc b/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/insert/insert_range.cc
index 4fead32..c026fd4 100644
--- a/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/insert/insert_range.cc
+++ b/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/insert/insert_range.cc
@@ -54,7 +54,7 @@ do_test()
 }
 
 template<typename Range>
-void
+constexpr void
 do_test_a()
 {
   do_test<Range, std::allocator<char>>();
@@ -63,7 +63,7 @@ do_test_a()
   do_test<Range, __gnu_test::SimpleAllocator<wchar_t>>();
 }
 
-bool
+constexpr bool
 test_ranges()
 {
   using namespace __gnu_test;
@@ -82,9 +82,9 @@ test_ranges()
 
   // Not lvalue-convertible to char
   struct C {
-    C(char v) : val(v) { }
-    operator char() && { return val; }
-    bool operator==(char b) const { return b == val; }
+    constexpr C(char v) : val(v) { }
+    constexpr operator char() && { return val; }
+    constexpr bool operator==(char b) const { return b == val; }
     char val;
   };
   using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
@@ -112,19 +112,11 @@ test_overlapping()
   VERIFY( c == "12123434abcd" );
 }
 
-constexpr bool
-test_constexpr()
-{
-#if _GLIBCXX_USE_CXX11_ABI
-  // XXX: this doesn't test the non-forward_range code paths are constexpr.
-  do_test<std::string_view, std::allocator<char>>();
-#endif // _GLIBCXX_USE_CXX11_ABI
-  return true;
-}
-
 int main()
 {
   test_ranges();
   test_overlapping();
-  static_assert( test_constexpr() );
+#if _GLIBCXX_USE_CXX11_ABI
+  static_assert( test_ranges() );
+#endif // _GLIBCXX_USE_CXX11_ABI
 }
diff --git a/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/replace/replace_with_range.cc b/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/replace/replace_with_range.cc
index 9acf11a..4c6bba5 100644
--- a/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/replace/replace_with_range.cc
+++ b/libstdc++-v3/testsuite/21_strings/basic_string/modifiers/replace/replace_with_range.cc
@@ -54,7 +54,7 @@ do_test()
 }
 
 template<typename Range>
-void
+constexpr void
 do_test_a()
 {
   do_test<Range, std::allocator<char>>();
@@ -63,7 +63,7 @@ do_test_a()
   do_test<Range, __gnu_test::SimpleAllocator<wchar_t>>();
 }
 
-bool
+constexpr bool
 test_ranges()
 {
   using namespace __gnu_test;
@@ -82,9 +82,9 @@ test_ranges()
 
   // Not lvalue-convertible to char
   struct C {
-    C(char v) : val(v) { }
-    operator char() && { return val; }
-    bool operator==(char b) const { return b == val; }
+    constexpr C(char v) : val(v) { }
+    constexpr operator char() && { return val; }
+    constexpr bool operator==(char b) const { return b == val; }
     char val;
   };
   using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
@@ -115,19 +115,11 @@ test_overlapping()
   VERIFY( c == "12123434abcd" );
 }
 
-constexpr bool
-test_constexpr()
-{
-#if _GLIBCXX_USE_CXX11_ABI
-  // XXX: this doesn't test the non-forward_range code paths are constexpr.
-  do_test<std::string_view, std::allocator<char>>();
-#endif // _GLIBCXX_USE_CXX11_ABI
-  return true;
-}
-
 int main()
 {
   test_ranges();
   test_overlapping();
-  static_assert( test_constexpr() );
+#if _GLIBCXX_USE_CXX11_ABI
+  static_assert( test_ranges() );
+#endif // _GLIBCXX_USE_CXX11_ABI
 }
diff --git a/libstdc++-v3/testsuite/22_locale/codecvt/codecvt_unicode.h b/libstdc++-v3/testsuite/22_locale/codecvt/codecvt_unicode.h
index 9fe9ac8..2ba23bf 100644
--- a/libstdc++-v3/testsuite/22_locale/codecvt/codecvt_unicode.h
+++ b/libstdc++-v3/testsuite/22_locale/codecvt/codecvt_unicode.h
@@ -1439,7 +1439,7 @@ ucs2_to_utf8_out_error (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
     // make the trailing surrogate a BMP char
     {5, 10, 3, 6, u'z', 4},
 
-    // don't replace anything in the test cases bellow, just show the surrogate
+    // don't replace anything in the test cases below, just show the surrogate
     // pair (fourth CP) fully or partially
     {5, 10, 3, 6, u'b', 0},
     {5, 7, 3, 6, u'b', 0}, // no space for fourth CP
@@ -2072,7 +2072,7 @@ utf16_to_ucs2_in_error (const std::codecvt<InternT, char, mbstate_t> &cvt,
     // make the trailing surrogate a BMP char
     {10, 5, 6, 3, u'z', 4},
 
-    // don't replace anything in the test cases bellow, just show the surrogate
+    // don't replace anything in the test cases below, just show the surrogate
     // pair (fourth CP) fully or partially (just the first surrogate)
     {10, 5, 6, 3, u'b', 0},
     {8, 5, 6, 3, u'b', 0},
diff --git a/libstdc++-v3/testsuite/23_containers/array/creation/3_neg.cc b/libstdc++-v3/testsuite/23_containers/array/creation/3_neg.cc
index ae06302..1335228 100644
--- a/libstdc++-v3/testsuite/23_containers/array/creation/3_neg.cc
+++ b/libstdc++-v3/testsuite/23_containers/array/creation/3_neg.cc
@@ -54,3 +54,5 @@ test03()
 }
 
 // { dg-prune-output "static assertion failed" }
+// { dg-prune-output "use of deleted function" }
+// { dg-prune-output "could not convert" }
diff --git a/libstdc++-v3/testsuite/23_containers/array/tuple_interface/get_neg.cc b/libstdc++-v3/testsuite/23_containers/array/tuple_interface/get_neg.cc
index 25511e7..e1e9ce9b 100644
--- a/libstdc++-v3/testsuite/23_containers/array/tuple_interface/get_neg.cc
+++ b/libstdc++-v3/testsuite/23_containers/array/tuple_interface/get_neg.cc
@@ -26,6 +26,6 @@ int n1 = std::get<1>(a);
 int n2 = std::get<1>(std::move(a));
 int n3 = std::get<1>(ca);
 
-// { dg-error "static assertion failed" "" { target *-*-* } 394 }
-// { dg-error "static assertion failed" "" { target *-*-* } 403 }
-// { dg-error "static assertion failed" "" { target *-*-* } 412 }
+// { dg-error "static assertion failed" "" { target *-*-* } 396 }
+// { dg-error "static assertion failed" "" { target *-*-* } 405 }
+// { dg-error "static assertion failed" "" { target *-*-* } 414 }
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/access/capacity.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/access/capacity.cc
new file mode 100644
index 0000000..2797e20
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/access/capacity.cc
@@ -0,0 +1,51 @@
+// { dg-do run { target c++26 } }
+
+#include <inplace_vector>
+
+#include <span>
+#include <testsuite_hooks.h>
+
+template<size_t N, typename T>
+constexpr void
+test_reserve()
+{
+  std::inplace_vector<T, N> v;
+
+  static_assert(v.max_size() == N);
+  static_assert(v.capacity() == N);
+
+  // static methods
+  v.shrink_to_fit();
+  v.reserve(0);
+  v.reserve(N);
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  try
+  {
+    v.reserve(N + 2);
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+#endif
+}
+
+int main()
+{
+  auto test_all = [] {
+    test_reserve<0, int>();
+    test_reserve<4, int>();
+    return true;
+  };
+
+  test_all();
+  static_assert(test_all());;
+}
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/access/elem.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/access/elem.cc
new file mode 100644
index 0000000..bc06aa0
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/access/elem.cc
@@ -0,0 +1,115 @@
+// { dg-do run { target c++26 } }
+
+#include <inplace_vector>
+
+#include <span>
+#include <stdexcept>
+#include <testsuite_hooks.h>
+#include <utility>
+
+template<size_t N, typename T>
+constexpr void
+test_out_of_capacity()
+{
+  std::inplace_vector<T, N> v;
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  try
+  {
+    (void)v.at(N + 2);
+    VERIFY(false);
+  }
+  catch (std::out_of_range const&)
+  {
+  }
+
+  try
+  {
+    (void)as_const(v).at(N + 2);
+    VERIFY(false);
+  }
+  catch (std::out_of_range const&)
+  {
+  }
+#endif
+}
+
+
+template<bool Const, typename T, size_t N>
+using InplaceVector = std::conditional_t<Const, 
+					 const std::inplace_vector<T, N>,
+					 std::inplace_vector<T, N>>;
+
+template<bool Const, typename T>
+constexpr void
+test_access()
+{
+  InplaceVector<Const, T, 10> v{1, 2, 3, 4, 5};
+
+  auto& e0a = v[0];
+  auto& e0b = v.at(0);
+  auto& e0c = v.front();
+  VERIFY( &e0a == &e0b );
+  VERIFY( &e0a == &e0c );
+  VERIFY( &e0a == &v.begin()[0] );
+  VERIFY( &e0a == &v.cbegin()[0] );
+  VERIFY( &e0a == v.data() );
+  VERIFY( e0a == T(1) );
+
+  auto& e3a = v[2];
+  auto& e3b = v.at(2);
+  VERIFY( &e3a == &e3b );
+  VERIFY( &e3a == &v.begin()[2] );
+  VERIFY( &e3a == &v.cbegin()[2] );
+  VERIFY( &e3a == v.data() + 2 );
+  VERIFY( e3a == T(3) );
+
+  auto& e4a = v[4];
+  auto& e4b = v.at(4);
+  auto& e4c = v.back();
+  VERIFY( &e4a == &e4b );
+  VERIFY( &e4a == &e4c );
+  VERIFY( &e4a == &v.begin()[4] );
+  VERIFY( &e4a == &v.cbegin()[4] );
+  VERIFY( &e4a == v.data() + 4 );
+  VERIFY( e4a == T(5) );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  try
+  {
+    (void)v.at(7);
+    VERIFY(false);
+  }
+  catch (std::out_of_range const&)
+  {
+  }
+#endif
+}
+
+int main()
+{
+  auto test_all = [] {
+    test_out_of_capacity<0, int>();
+    test_out_of_capacity<4, int>();
+    test_access<true, int>();
+    test_access<false, int>();
+    return true;
+  };
+
+  test_all();
+  static_assert(test_all());;
+}
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/access/elem_neg.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/access/elem_neg.cc
new file mode 100644
index 0000000..b2bff0d
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/access/elem_neg.cc
@@ -0,0 +1,65 @@
+// { dg-do compile { target c++26 } }
+
+#include <inplace_vector>
+
+template<bool Const, typename T, size_t N>
+using InplaceVector 
+  = std::conditional_t<Const, 
+		       const std::inplace_vector<T, N>,
+		       std::inplace_vector<T, N>>;
+
+template<bool Const, size_t N, typename T>
+constexpr bool
+test_front_on_empty()
+{
+  InplaceVector<Const, T, N> v;
+  (void)v.front(); // { dg-error "in 'constexpr' expansion of" }
+  return true;
+}
+
+template<bool Const, size_t N, typename T>
+constexpr bool
+test_back_on_empty()
+{
+  InplaceVector<Const, T, N> v;
+  (void)v.back(); // { dg-error "in 'constexpr' expansion of" }
+  return true;
+}
+
+template<bool Const, size_t N, typename T>
+constexpr bool
+test_out_of_capacity()
+{
+  InplaceVector<Const, T, N> v;
+  (void)v[N+2]; // { dg-error "in 'constexpr' expansion of" }
+  return true;
+}
+
+template<bool Const, typename T>
+constexpr bool
+test_out_of_size()
+{
+  InplaceVector<Const, T, 10> v{1, 2, 3, 4, 5};
+  (void)v[7]; // { dg-error "in 'constexpr' expansion of" }
+  return true;
+}
+
+static_assert(test_front_on_empty<false, 0, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_front_on_empty<false, 4, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_back_on_empty<false, 0, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_back_on_empty<false, 4, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_out_of_capacity<false, 0, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_out_of_capacity<false, 4, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_out_of_size<false, int>());        // { dg-error "in 'constexpr' expansion of" }
+
+static_assert(test_front_on_empty<true, 0, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_front_on_empty<true, 4, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_back_on_empty<true, 0, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_back_on_empty<true, 4, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_out_of_capacity<true, 0, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_out_of_capacity<true, 4, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_out_of_size<true, int>());        // { dg-error "in 'constexpr' expansion of" }
+
+// { dg-prune-output "non-constant condition for static assertion" }
+// { dg-prune-output "is not a constant expression" }
+// { dg-prune-output "call to non-'constexpr' function" }
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/cons/1.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/cons/1.cc
new file mode 100644
index 0000000..e9c2cdc
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/cons/1.cc
@@ -0,0 +1,385 @@
+// { dg-do run { target c++26 } }
+
+#include <inplace_vector>
+#include <testsuite_hooks.h>
+
+struct X
+{
+  constexpr X() { } // not trivially default constructible
+};
+
+struct N
+{
+  constexpr N() noexcept { } // not trivially default constructible
+};
+
+struct D
+{
+  ~D() {} // not trivially destructible
+};
+
+struct U
+{
+  U() noexcept(false) = default; // lies about noexcept
+};
+
+// n5008 inplace.vector.overview says for inplace_vector<T, 0>
+// provides trivial copy/move/default cosntructpr regardless of T
+struct Z
+{
+  constexpr Z(int) {}
+  Z() = delete;
+};
+
+static_assert(std::is_default_constructible_v<std::inplace_vector<int, 2>>);
+static_assert(std::is_default_constructible_v<std::inplace_vector<X, 2>>);
+static_assert(std::is_default_constructible_v<std::inplace_vector<N, 2>>);
+static_assert(std::is_default_constructible_v<std::inplace_vector<D, 2>>);
+static_assert(std::is_default_constructible_v<std::inplace_vector<U, 2>>);
+// The operators are not constrained, as for any other container
+static_assert(std::is_default_constructible_v<std::inplace_vector<Z, 2>>);
+
+static_assert(std::is_nothrow_default_constructible_v<std::inplace_vector<int, 2>>);
+static_assert(std::is_nothrow_default_constructible_v<std::inplace_vector<X, 2>>);
+static_assert(std::is_nothrow_default_constructible_v<std::inplace_vector<N, 2>>);
+static_assert(std::is_nothrow_default_constructible_v<std::inplace_vector<D, 2>>);
+static_assert(std::is_nothrow_default_constructible_v<std::inplace_vector<U, 2>>);
+
+// Needs to set size to zero, not trivial
+static_assert(!std::is_trivially_default_constructible_v<std::inplace_vector<int, 2>>);
+static_assert(!std::is_trivially_default_constructible_v<std::inplace_vector<X, 2>>);
+static_assert(!std::is_trivially_default_constructible_v<std::inplace_vector<N, 2>>);
+static_assert(!std::is_trivially_default_constructible_v<std::inplace_vector<D, 2>>);
+static_assert(!std::is_trivially_default_constructible_v<std::inplace_vector<U, 2>>);
+
+static_assert(std::is_trivially_destructible_v<std::inplace_vector<int, 2>>);
+static_assert(std::is_trivially_destructible_v<std::inplace_vector<X, 2>>);
+static_assert(std::is_trivially_destructible_v<std::inplace_vector<N, 2>>);
+static_assert(!std::is_trivially_destructible_v<std::inplace_vector<D, 2>>);
+static_assert(std::is_trivially_destructible_v<std::inplace_vector<U, 2>>);
+
+static_assert(std::is_nothrow_default_constructible_v<std::inplace_vector<int, 0>>);
+static_assert(std::is_nothrow_default_constructible_v<std::inplace_vector<X, 0>>);
+static_assert(std::is_nothrow_default_constructible_v<std::inplace_vector<N, 0>>);
+static_assert(std::is_nothrow_default_constructible_v<std::inplace_vector<D, 0>>);
+static_assert(std::is_nothrow_default_constructible_v<std::inplace_vector<U, 0>>);
+
+// Size is always zero, so trivial
+static_assert(std::is_trivially_default_constructible_v<std::inplace_vector<int, 0>>);
+static_assert(std::is_trivially_default_constructible_v<std::inplace_vector<X, 0>>);
+static_assert(std::is_trivially_default_constructible_v<std::inplace_vector<N, 0>>);
+static_assert(std::is_trivially_default_constructible_v<std::inplace_vector<D, 0>>);
+static_assert(std::is_trivially_default_constructible_v<std::inplace_vector<U, 0>>);
+static_assert(std::is_trivially_default_constructible_v<std::inplace_vector<Z, 0>>);
+
+static_assert(std::is_trivially_destructible_v<std::inplace_vector<int, 0>>);
+static_assert(std::is_trivially_destructible_v<std::inplace_vector<X, 0>>);
+static_assert(std::is_trivially_destructible_v<std::inplace_vector<N, 0>>);
+static_assert(std::is_trivially_destructible_v<std::inplace_vector<D, 0>>);
+static_assert(std::is_trivially_destructible_v<std::inplace_vector<U, 0>>);
+
+static_assert(std::is_empty_v<std::inplace_vector<int, 0>>);
+static_assert(std::is_empty_v<std::inplace_vector<X, 0>>);
+static_assert(std::is_empty_v<std::inplace_vector<N, 0>>);
+static_assert(std::is_empty_v<std::inplace_vector<D, 0>>);
+static_assert(std::is_empty_v<std::inplace_vector<U, 0>>);
+static_assert(std::is_empty_v<std::inplace_vector<Z, 0>>);
+
+constexpr void
+test_default()
+{
+  std::inplace_vector<int, 5> c;
+  VERIFY( c.size() == 0 );
+  VERIFY( c.capacity() == 5 );
+  VERIFY( c.empty() );
+  VERIFY( c.begin() == c.end() );
+
+  std::inplace_vector<int, 0> c0;
+  VERIFY( c0.size() == 0 );
+  VERIFY( c0.capacity() == 0 );
+  VERIFY( c0.empty() );
+  VERIFY( c0.begin() == c0.end() );
+
+  std::inplace_vector<Z, 0> z0;
+  VERIFY( z0.size() == 0 );
+  VERIFY( z0.capacity() == 0 );
+  VERIFY( z0.empty() );
+  VERIFY( z0.begin() == z0.end() );
+
+#ifdef __cpp_lib_constexpr_inplace_vector
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  std::inplace_vector<X, 5> cx;
+  VERIFY( cx.size() == 0 );
+  VERIFY( cx.capacity() == 5 );
+  VERIFY( cx.empty() );
+  VERIFY( cx.begin() == cx.end() );
+
+  std::inplace_vector<X, 0> cx0;
+  VERIFY( cx0.size() == 0 );
+  VERIFY( cx0.capacity() == 0 );
+  VERIFY( cx0.empty() );
+  VERIFY( cx0.begin() == cx0.end() );
+}
+
+constexpr void
+test_n()
+{
+  std::inplace_vector<int, 5> c(2);
+  VERIFY( c.size() == 2 );
+  VERIFY( c.capacity() == 5 );
+  VERIFY( not c.empty() );
+  VERIFY( c.begin() + 2 == c.end() );
+  VERIFY( c[0] == 0 );
+  VERIFY( c[1] == 0 );
+
+  std::inplace_vector<int, 2> c2(2);
+  VERIFY( c2.size() == 2 );
+  VERIFY( c2.capacity() == 2 );
+  VERIFY( not c2.empty() );
+  VERIFY( c2.begin() + 2 == c2.end() );
+  VERIFY( c2[0] == 0 );
+  VERIFY( c2[1] == 0 );
+
+  std::inplace_vector<int, 0> c0(0);
+  VERIFY( c0.size() == 0 );
+  VERIFY( c0.capacity() == 0 );
+  VERIFY( c0.empty() );
+  VERIFY( c0.begin() == c0.end() );
+
+  std::inplace_vector<int, 2> c20(0);
+  VERIFY( c20.size() == 0 );
+  VERIFY( c20.capacity() == 2 );
+  VERIFY( c20.empty() );
+  VERIFY( c20.begin() == c20.end() );
+
+  std::inplace_vector<Z, 0> z0(0);
+  VERIFY( z0.size() == 0 );
+  VERIFY( z0.capacity() == 0 );
+  VERIFY( z0.empty() );
+  VERIFY( z0.begin() == z0.end() );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if not consteval {
+    try
+    {
+      std::inplace_vector<int, 2> ct(3);
+      VERIFY(false);
+    }
+    catch (std::bad_alloc const&)
+    {
+    }
+
+    try
+    {
+      std::inplace_vector<int, 0> ct(1);
+      VERIFY(false);
+    }
+    catch (std::bad_alloc const&)
+    {
+    }
+
+  }
+#endif
+
+#ifdef __cpp_lib_constexpr_inplace_vector
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  std::inplace_vector<X, 5> cx(3);
+  VERIFY( cx.size() == 3 );
+  VERIFY( cx.capacity() == 5 );
+  VERIFY( not cx.empty() );
+  VERIFY( cx.begin() + 3 == cx.end() );
+  (void) cx[2];
+}
+
+constexpr void
+test_n_val()
+{
+  std::inplace_vector<int, 5> c(2, 99);
+  VERIFY( c.size() == 2 );
+  VERIFY( c.capacity() == 5 );
+  VERIFY( not c.empty() );
+  VERIFY( c.begin() + 2 == c.end() );
+  VERIFY( c[0] == 99 );
+  VERIFY( c[1] == 99 );
+
+  std::inplace_vector<int, 1> c1(1, 44);
+  VERIFY( c1.size() == 1 );
+  VERIFY( c1.capacity() == 1 );
+  VERIFY( not c1.empty() );
+  VERIFY( c1.begin() + 1 == c1.end() );
+  VERIFY( c1[0] == 44 );
+
+  std::inplace_vector<int, 0> c0(0, 33);
+  VERIFY( c0.size() == 0 );
+  VERIFY( c0.capacity() == 0 );
+  VERIFY( c0.empty() );
+  VERIFY( c0.begin() == c0.end() );
+
+  std::inplace_vector<int, 2> c20(0, 22);
+  VERIFY( c20.size() == 0 );
+  VERIFY( c20.capacity() == 2 );
+  VERIFY( c20.empty() );
+  VERIFY( c20.begin() == c20.end() );
+
+  std::inplace_vector<Z, 0> z0(0, 33);
+  VERIFY( z0.size() == 0 );
+  VERIFY( z0.capacity() == 0 );
+  VERIFY( z0.empty() );
+  VERIFY( z0.begin() == z0.end() );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if not consteval {
+    try
+    {
+      std::inplace_vector<int, 2> ct(3, 11);
+      VERIFY(false);
+    }
+    catch (std::bad_alloc const&)
+    {
+    }
+
+    try
+    {
+      std::inplace_vector<int, 0> ct(2, 11);
+      VERIFY(false);
+    }
+    catch (std::bad_alloc const&)
+    {
+    }
+  }
+#endif
+
+#ifdef __cpp_lib_constexpr_inplace_vector
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  std::inplace_vector<X, 5> cx(4);
+  VERIFY( cx.size() == 4 );
+  VERIFY( cx.capacity() == 5 );
+  VERIFY( not cx.empty() );
+  VERIFY( cx.begin() + 4 == cx.end() );
+  (void) cx[3];
+}
+
+constexpr void
+test_initializer_list()
+{
+  std::inplace_vector<int, 5> c{22, 33};
+  VERIFY( c.size() == 2 );
+  VERIFY( c.capacity() == 5 );
+  VERIFY( not c.empty() );
+  VERIFY( c.begin() + 2 == c.end() );
+  VERIFY( c[0] == 22 );
+  VERIFY( c[1] == 33 );
+
+  std::inplace_vector<int, 1> c1{44};
+  VERIFY( c1.size() == 1 );
+  VERIFY( c1.capacity() == 1 );
+  VERIFY( not c1.empty() );
+  VERIFY( c1.begin() + 1 == c1.end() );
+  VERIFY( c1[0] == 44 );
+
+  std::inplace_vector<int, 0> c0({});
+  VERIFY( c0.size() == 0 );
+  VERIFY( c0.capacity() == 0 );
+  VERIFY( c0.empty() );
+  VERIFY( c0.begin() == c0.end() );
+
+  std::inplace_vector<int, 2> c20({});
+  VERIFY( c20.size() == 0 );
+  VERIFY( c20.capacity() == 2 );
+  VERIFY( c20.empty() );
+  VERIFY( c20.begin() == c20.end() );
+
+  std::inplace_vector<Z, 0> z0({});
+  VERIFY( z0.size() == 0 );
+  VERIFY( z0.capacity() == 0 );
+  VERIFY( z0.empty() );
+  VERIFY( z0.begin() == z0.end() );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if not consteval {
+    try
+    {
+      std::inplace_vector<int, 2> ct{11, 22, 33};
+      VERIFY(false);
+    }
+    catch (std::bad_alloc const&)
+    {
+    }
+
+    try
+    {
+      std::inplace_vector<int, 0> ct{11, 22};
+      VERIFY(false);
+    }
+    catch (std::bad_alloc const&)
+    {
+    }
+  }
+#endif
+
+#ifdef __cpp_lib_constexpr_inplace_vector
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  std::inplace_vector<X, 5> cx{X(), X(), X(), X()};
+  VERIFY( cx.size() == 4 );
+  VERIFY( cx.capacity() == 5 );
+  VERIFY( not cx.empty() );
+  VERIFY( cx.begin() + 4 == cx.end() );
+  (void) cx[3];
+}
+
+constexpr std::inplace_vector<int, 0> e0;
+constexpr std::inplace_vector<X, 0> e1;
+constexpr std::inplace_vector<Z, 0> e2;
+
+constexpr std::inplace_vector<int, 5> g1;
+constexpr std::inplace_vector<int, 5> g2(2, 100);
+constexpr std::inplace_vector<int, 5> g3 = g2;
+constexpr std::inplace_vector<int, 5> g4{1, 2, 3};
+constexpr std::inplace_vector<int, 5> g5 = [] {
+  std::inplace_vector<int, 5> res;
+  res = g3;
+  return res;
+}();
+
+int main()
+{
+  auto tests = [] {
+    test_default();
+    test_n();
+    test_n_val();
+    test_initializer_list();
+    return true;
+  };
+
+  tests();
+  constexpr bool _ = tests();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/cons/from_range.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/cons/from_range.cc
new file mode 100644
index 0000000..4a2f193
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/cons/from_range.cc
@@ -0,0 +1,177 @@
+// { dg-do run { target c++26 } }
+
+#include <inplace_vector>
+
+#include <span>
+#include <testsuite_hooks.h>
+#include <testsuite_iterators.h>
+#include <testsuite_allocator.h>
+
+template<typename T, typename V, size_t N>
+constexpr bool
+eq(const std::inplace_vector<V, N>& l, std::span<const T> r) {
+  if (l.size() != r.size())
+    return false;
+  for (auto i = 0u; i < l.size(); ++i)
+    if (l[i] != r[i])
+      return false;
+  return true;
+};
+
+template<typename T, template<class TT> class ItType>
+constexpr void
+do_test_it()
+{
+  // The vector's value_type.
+  using V = int;
+
+  T a[]{1,2,3,4,5,6,7,8,9};
+  using It = ItType<T>;
+
+  auto bounds = typename It::ContainerType(a, a+9);
+  std::inplace_vector<V, 0> e0(It(a, &bounds), It(a, &bounds));
+  VERIFY( e0.empty() );
+
+  bounds = typename It::ContainerType(a, a+9);
+  std::inplace_vector<V, 10> v0(It(a, &bounds), It(a, &bounds));
+  VERIFY( v0.empty() );
+
+  bounds = typename It::ContainerType(a, a+9);
+  std::inplace_vector<V, 10> v4(It(a, &bounds), It(a+4, &bounds));
+  VERIFY( eq<T>(v4, {a, 4}) );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  bounds = typename It::ContainerType(a, a+9);
+  try
+  {
+    std::inplace_vector<int, 5> v9(It(a, &bounds), It(a+9, &bounds));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+
+  bounds = typename It::ContainerType(a, a+9);
+  try
+  {
+    std::inplace_vector<int, 0> v2(It(a, &bounds), It(a+2, &bounds));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+#endif
+}
+
+constexpr bool
+test_iterators()
+{
+  using namespace __gnu_test;
+
+  do_test_it<int, input_iterator_wrapper>();
+  do_test_it<int, forward_iterator_wrapper>();
+  do_test_it<int, random_access_iterator_wrapper>();
+  
+  do_test_it<short, forward_iterator_wrapper>();
+  return true;
+}
+
+template<typename Range>
+constexpr void
+do_test_r()
+{
+  // The vector's value_type.
+  using V = int;
+
+  // The range's value_type.
+  using T = std::ranges::range_value_t<Range>;
+  T a[]{1,2,3,4,5,6,7,8,9};
+
+  std::inplace_vector<V, 0> e0(std::from_range, Range(a, a+0));
+  VERIFY( e0.empty() );
+
+  std::inplace_vector<V, 10> v0(std::from_range, Range(a, a+0));
+  VERIFY( v0.empty() );
+
+  std::inplace_vector<V, 10> v4(std::from_range, Range(a, a+4));
+  VERIFY( eq<T>(v4, {a, 4}) );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+  
+  try
+  {
+    std::inplace_vector<V, 5> v9(std::from_range, Range(a, a+9));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+
+  try
+  {
+    std::inplace_vector<V, 0> v3(std::from_range, Range(a, a+3));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+#endif
+}
+
+constexpr bool
+test_ranges()
+{
+  using namespace __gnu_test;
+
+  do_test_r<test_forward_range<int>>();
+  do_test_r<test_sized_range_sized_sent<int, forward_iterator_wrapper>>();
+
+  do_test_r<test_input_range<int>>();
+  do_test_r<test_input_sized_range<int>>();
+  do_test_r<test_sized_range_sized_sent<int, input_iterator_wrapper>>();
+
+  do_test_r<test_range<int, input_iterator_wrapper_nocopy>>();
+  do_test_r<test_sized_range<int, input_iterator_wrapper_nocopy>>();
+  do_test_r<test_sized_range_sized_sent<int, input_iterator_wrapper_nocopy>>();
+
+  do_test_r<test_forward_range<short>>();
+  do_test_r<test_input_range<short>>();
+
+  // Not lvalue-convertible to int
+  struct C {
+    constexpr C(int v) : val(v) { }
+    constexpr operator int() && { return val; }
+    constexpr bool operator==(int b) const { return b == val; }
+    int val;
+  };
+  using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
+  do_test_r<rvalue_input_range>();
+
+  return true;
+}
+
+int main()
+{
+  auto test_all = [] {
+    test_iterators();
+    test_ranges();
+    return true;
+  };
+
+  test_all();
+  static_assert( test_all() );
+}
+
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/cons/throws.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/cons/throws.cc
new file mode 100644
index 0000000..4ce39d1
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/cons/throws.cc
@@ -0,0 +1,129 @@
+// { dg-do run { target c++26 } }
+
+#include <inplace_vector>
+
+#include <span>
+#include <testsuite_hooks.h>
+#include <testsuite_iterators.h>
+#include <testsuite_allocator.h>
+
+struct CopyFailed {};
+
+struct Thrower
+{
+  static inline size_t throw_after = 0;
+  static inline size_t incontainer = 0;
+
+  Thrower() {}
+  Thrower(int x) {}
+  Thrower(const Thrower&)
+  {
+    if (incontainer >= throw_after)
+      throw CopyFailed();
+    ++incontainer;
+  }
+
+  ~Thrower()
+  { --incontainer; }
+};
+
+template<template<class TT> class ItType>
+void
+do_test_it()
+{
+  // The vector's value_type.
+  using V = Thrower;
+
+  V a[]{1,2,3,4,5,6,7,8,9};
+  using It = ItType<V>;
+
+  auto bounds = typename It::ContainerType(a, a+9);
+  Thrower::throw_after = 100;
+  Thrower::incontainer = 0;
+  try
+  {
+    std::inplace_vector<V, 5> v9(It(a, &bounds), It(a+9, &bounds));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( Thrower::incontainer == 0 );
+
+  bounds = typename It::ContainerType(a, a+9);
+  Thrower::throw_after = 2;
+  Thrower::incontainer = 0;
+  try
+  {
+    std::inplace_vector<V, 5> v2(It(a, &bounds), It(a+3, &bounds));
+    VERIFY(false);
+  }
+  catch (CopyFailed const&)
+  {
+  }
+  VERIFY( Thrower::incontainer == 0 );
+}
+
+bool
+test_iterators()
+{
+  using namespace __gnu_test;
+  do_test_it<input_iterator_wrapper>();
+  do_test_it<forward_iterator_wrapper>();
+  do_test_it<random_access_iterator_wrapper>();
+  return true;
+}
+
+template<typename Range>
+void
+do_test_r()
+{
+  // The vector's value_type.
+  using V = Thrower;
+
+  V a[]{1,2,3,4,5,6,7,8,9};
+
+  Thrower::throw_after = 100;
+  Thrower::incontainer = 0;
+  try
+  {
+    std::inplace_vector<V, 5> v9(std::from_range, Range(a, a+9));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( Thrower::incontainer == 0 );
+
+  Thrower::throw_after = 2;
+  Thrower::incontainer = 0;
+  try
+  {
+    std::inplace_vector<V, 5> v9(std::from_range, Range(a, a+3));
+    VERIFY(false);
+  }
+  catch (CopyFailed const&)
+  {
+  }
+  VERIFY( Thrower::incontainer == 0 );
+}
+
+bool
+test_ranges()
+{
+  using namespace __gnu_test;
+  do_test_r<test_forward_range<Thrower>>();
+  do_test_r<test_sized_range_sized_sent<Thrower, forward_iterator_wrapper>>();
+
+  do_test_r<test_input_range<Thrower>>();
+  do_test_r<test_input_sized_range<Thrower>>();
+  do_test_r<test_sized_range_sized_sent<Thrower, input_iterator_wrapper>>();
+  return true;
+}
+
+int main()
+{
+  test_iterators();
+  test_ranges();
+}
+
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/copy.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/copy.cc
new file mode 100644
index 0000000..d149e63
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/copy.cc
@@ -0,0 +1,247 @@
+// { dg-do run { target c++26 } }
+
+#include <inplace_vector>
+#include <ranges>
+#include <testsuite_hooks.h>
+
+struct X
+{
+  X() = default;
+  constexpr X(int p) : v(p) {}
+  constexpr X(const X& o) : v(o.v) { } // not trivial
+  constexpr X& operator=(const X& o) // not trivial
+  { v = o.v; return *this; }
+
+  int v;
+
+  friend auto operator<=>(const X&, const X&) = default;
+};
+
+template<bool CNoex, bool ANoex>
+struct N
+{
+  N() = default;
+  constexpr N(const N&) noexcept(CNoex) { } // not trivial
+  constexpr N& operator=(const N& o) noexcept(ANoex) // not trivial
+  { return *this; }
+};
+
+struct D
+{
+  D() = default;
+  D(const D&) = default;
+  D& operator=(const D&) = default;
+  ~D() {} // not trivially destructible
+};
+
+struct U
+{
+  U() = default;
+  U(const U&) noexcept(false) = default; // lies about noexcept, is trivial but throwing
+  U& operator=(const U&) noexcept(false) = default; // lies about noexcept, is trivial but throwing
+};
+
+// n5008 inplace.vector.overview p5 says for inplace_vector<T, 0>
+// provides trivial copy/move/default cosntructpr regardless of T
+struct Z
+{
+  Z(Z&&) = delete;
+  Z& operator=(Z&&) = delete;
+};
+
+template<size_t N, typename T>
+  constexpr std::inplace_vector<T, N> const&
+  materialize(std::inplace_vector<T, N> const& r)
+  { return r; }
+
+static_assert(std::is_copy_constructible_v<std::inplace_vector<int, 2>>);
+static_assert(std::is_copy_constructible_v<std::inplace_vector<X, 2>>);
+static_assert(std::is_copy_constructible_v<std::inplace_vector<N<false, false>, 2>>);
+static_assert(std::is_copy_constructible_v<std::inplace_vector<D, 2>>);
+static_assert(std::is_copy_constructible_v<std::inplace_vector<U, 2>>);
+// The operators are not constrained, as for any other container
+static_assert(std::is_copy_constructible_v<std::inplace_vector<Z, 2>>);
+
+// conditional noexcept here is libstdc++ extension,
+static_assert(std::is_nothrow_copy_constructible_v<std::inplace_vector<int, 2>>);
+static_assert(!std::is_nothrow_copy_constructible_v<std::inplace_vector<X, 2>>);
+static_assert(std::is_nothrow_copy_constructible_v<std::inplace_vector<N<true, true>, 2>>);
+static_assert(std::is_nothrow_copy_constructible_v<std::inplace_vector<N<true, false>, 2>>);
+static_assert(!std::is_nothrow_copy_constructible_v<std::inplace_vector<N<false, true>, 2>>);
+static_assert(!std::is_nothrow_copy_constructible_v<std::inplace_vector<N<false, false>, 2>>);
+static_assert(std::is_nothrow_copy_constructible_v<std::inplace_vector<D, 2>>);
+static_assert(!std::is_nothrow_copy_constructible_v<std::inplace_vector<U, 2>>);
+
+static_assert(std::is_trivially_copy_constructible_v<std::inplace_vector<int, 2>>);
+static_assert(!std::is_trivially_copy_constructible_v<std::inplace_vector<X, 2>>);
+static_assert(!std::is_trivially_copy_constructible_v<std::inplace_vector<N<true, true>, 2>>);
+// is_trivially_copy_constructible_v checks destructor
+static_assert(!std::is_trivially_copy_constructible_v<std::inplace_vector<D, 2>>);
+static_assert(std::is_trivially_copy_constructible_v<std::inplace_vector<U, 2>>);
+
+static_assert(std::is_copy_assignable_v<std::inplace_vector<int, 2>>);
+static_assert(std::is_copy_assignable_v<std::inplace_vector<X, 2>>);
+static_assert(std::is_copy_assignable_v<std::inplace_vector<N<false, false>, 2>>);
+static_assert(std::is_copy_assignable_v<std::inplace_vector<D, 2>>);
+static_assert(std::is_copy_assignable_v<std::inplace_vector<U, 2>>);
+// The operators are not constrained, as for any other container
+static_assert(std::is_copy_assignable_v<std::inplace_vector<Z, 2>>);
+
+static_assert(std::is_nothrow_copy_assignable_v<std::inplace_vector<int, 2>>);
+static_assert(!std::is_nothrow_copy_assignable_v<std::inplace_vector<X, 2>>);
+static_assert(std::is_nothrow_copy_assignable_v<std::inplace_vector<N<true, true>, 2>>);
+static_assert(!std::is_nothrow_copy_assignable_v<std::inplace_vector<N<true, false>, 2>>);
+static_assert(!std::is_nothrow_copy_assignable_v<std::inplace_vector<N<false, true>, 2>>);
+static_assert(!std::is_nothrow_copy_assignable_v<std::inplace_vector<N<false, false>, 2>>);
+static_assert(std::is_nothrow_copy_assignable_v<std::inplace_vector<D, 2>>);
+static_assert(!std::is_nothrow_copy_assignable_v<std::inplace_vector<U, 2>>);
+
+// conditional noexcept here is libstdc++ extension,
+static_assert(std::is_trivially_copy_assignable_v<std::inplace_vector<int, 2>>);
+static_assert(!std::is_trivially_copy_assignable_v<std::inplace_vector<X, 2>>);
+static_assert(!std::is_trivially_copy_assignable_v<std::inplace_vector<N<true, true>, 2>>);
+// destructor is not trivial
+static_assert(!std::is_trivially_copy_assignable_v<std::inplace_vector<D, 2>>);
+static_assert(std::is_trivially_copy_assignable_v<std::inplace_vector<U, 2>>);
+
+static_assert(std::is_nothrow_copy_constructible_v<std::inplace_vector<int, 0>>);
+static_assert(std::is_nothrow_copy_constructible_v<std::inplace_vector<X, 0>>);
+static_assert(std::is_nothrow_copy_constructible_v<std::inplace_vector<N<false, false>, 0>>);
+static_assert(std::is_nothrow_copy_constructible_v<std::inplace_vector<D, 0>>);
+static_assert(std::is_nothrow_copy_constructible_v<std::inplace_vector<U, 0>>);
+static_assert(std::is_nothrow_copy_constructible_v<std::inplace_vector<Z, 0>>);
+
+static_assert(std::is_trivially_copy_constructible_v<std::inplace_vector<int, 0>>);
+static_assert(std::is_trivially_copy_constructible_v<std::inplace_vector<X, 0>>);
+static_assert(std::is_trivially_copy_constructible_v<std::inplace_vector<D, 0>>);
+static_assert(std::is_trivially_copy_constructible_v<std::inplace_vector<U, 0>>);
+static_assert(std::is_trivially_copy_constructible_v<std::inplace_vector<Z, 0>>);
+
+static_assert(std::is_nothrow_copy_assignable_v<std::inplace_vector<int, 0>>);
+static_assert(std::is_nothrow_copy_assignable_v<std::inplace_vector<X, 0>>);
+static_assert(std::is_nothrow_copy_assignable_v<std::inplace_vector<N<false, false>, 0>>);
+static_assert(std::is_nothrow_copy_assignable_v<std::inplace_vector<D, 0>>);
+static_assert(std::is_nothrow_copy_assignable_v<std::inplace_vector<U, 0>>);
+static_assert(std::is_nothrow_copy_assignable_v<std::inplace_vector<Z, 0>>);
+
+static_assert(std::is_trivially_copy_assignable_v<std::inplace_vector<int, 0>>);
+static_assert(std::is_trivially_copy_assignable_v<std::inplace_vector<X, 0>>);
+static_assert(std::is_trivially_copy_assignable_v<std::inplace_vector<N<false, false>, 0>>);
+static_assert(std::is_trivially_copy_assignable_v<std::inplace_vector<D, 0>>);
+static_assert(std::is_trivially_copy_assignable_v<std::inplace_vector<U, 0>>);
+static_assert(std::is_trivially_copy_assignable_v<std::inplace_vector<Z, 0>>);
+
+
+template<typename T, size_t N>
+constexpr bool
+eq(const std::inplace_vector<T, N>& s, std::span<const T> o)
+{ return std::ranges::equal(s, o); }
+
+constexpr void
+test_ctor()
+{
+  auto e0 = materialize<0, int>({});
+  VERIFY( e0.empty() );
+  auto e1 = materialize<0, X>({});
+  VERIFY( e1.empty() );
+  auto e2 = materialize<0, Z>({});
+  VERIFY( e2.empty() );
+
+  auto c0 = materialize<5, int>({});
+  VERIFY( c0.empty() );
+
+  auto c3 = materialize<5, int>({1, 2, 3});
+  VERIFY( eq(c3, {1, 2, 3}) );
+
+  auto c5 = materialize<5, int>({1, 2, 3, 4, 5});
+  VERIFY( eq(c5, {1, 2, 3, 4, 5}) );
+
+#ifdef __cpp_lib_constexpr_inplace_vector
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  auto x0 = materialize<3, X>({});
+  VERIFY( x0.empty() );
+
+  auto x2 = materialize<3, X>({1, 2});
+  VERIFY( eq(x2, {1, 2}) );
+
+  auto x3 = materialize<3, X>({1, 2, 3});
+  VERIFY( eq(x3, {1, 2, 3}) );
+}
+
+constexpr void
+test_assign()
+{
+  std::inplace_vector<int, 0> e0;
+  e0 = materialize<0, int>({});
+  VERIFY( e0.empty() );
+  std::inplace_vector<X, 0> e1;
+  e1 = materialize<0, X>({});
+  VERIFY( e1.empty() );
+  std::inplace_vector<Z, 0> e2;
+  e2 = materialize<0, Z>({});
+  VERIFY( e2.empty() );
+
+  std::inplace_vector<int, 5> c;
+  c = materialize<5, int>({});
+  VERIFY( c.empty() );
+
+  c = materialize<5, int>({1, 2, 3});
+  VERIFY( eq(c, {1, 2, 3}) );
+
+  c = materialize<5, int>({1, 2, 3, 4, 5});
+  VERIFY( eq(c, {1, 2, 3, 4, 5}) );
+
+  c = materialize<5, int>({4, 5});
+  VERIFY( eq(c, {4, 5}) );
+
+  c = materialize<5, int>({});
+  VERIFY( c.empty() );
+
+#ifdef __cpp_lib_constexpr_inplace_vector
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  std::inplace_vector<X, 5> x;
+  x = materialize<5, X>({});
+  VERIFY( x.empty() );
+
+  x = materialize<5, X>({1, 2, 3});
+  VERIFY( eq(x, {1, 2, 3}) );
+
+  x = materialize<5, X>({1, 2, 3, 4, 5});
+  VERIFY( eq(x, {1, 2, 3, 4, 5}) );
+
+  x = materialize<5, X>({4, 5});
+  VERIFY( eq(x, {4, 5}) );
+
+  x = materialize<5, X>({});
+  VERIFY( x.empty() );
+}
+
+constexpr auto e0 = materialize<0, int>({});
+constexpr auto e1 = materialize<0, X>({});
+constexpr auto e2 = materialize<0, Z>({});
+
+constexpr auto t1 = materialize<3, int>({});
+constexpr auto t2 = materialize<3, int>({1, 2});
+constexpr auto t3 = materialize<3, int>({11, 22, 33});
+
+int main()
+{
+  auto tests = [] {
+    test_ctor();
+    test_assign();
+    return true;
+  };
+
+  tests();
+  constexpr bool _ = tests();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/erasure.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/erasure.cc
new file mode 100644
index 0000000..8fb56e9
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/erasure.cc
@@ -0,0 +1,69 @@
+// { dg-do run { target c++26 } }
+
+#include <inplace_vector>
+#include <testsuite_hooks.h>
+#include <span>
+
+template<typename T, size_t N>
+constexpr bool
+eq(const std::inplace_vector<T, N>& l, std::span<const T> r) {
+  if (l.size() != r.size())
+    return false;
+  for (auto i = 0u; i < l.size(); ++i)
+    if (l[i] != r[i])
+      return false;
+  return true;
+};
+
+constexpr void
+test_erase()
+{
+  std::inplace_vector<int, 15> c{1, 0, 3, 4, 5, 6, 5, 4, 3, 0, 1, 4, 4, 9};
+  std::erase(c, 4);
+  VERIFY( c.size() == 10 );
+  std::erase(c, 1);
+  VERIFY( c.size() == 8 );
+  std::erase(c, 9);
+  VERIFY( c.size() == 7 );
+  VERIFY( eq(c, {0, 3, 5, 6, 5, 3, 0}) );
+  
+  std::erase(c, {});
+  VERIFY( c.size() == 5 );
+  VERIFY( eq(c, {3, 5, 6, 5, 3}) );
+
+  std::erase(c, {5});
+  VERIFY( c.size() == 3 );
+  VERIFY( eq(c, {3, 6, 3}) );
+
+  std::inplace_vector<int, 0> e;
+  std::erase(e, 10);
+  VERIFY( e.empty() );
+}
+
+constexpr void
+test_erase_if()
+{
+  std::inplace_vector<int, 15> c{1, 2, 3, 4, 5, 6, 5, 4, 3, 2, 1, 4, 4, 9};
+  std::erase_if(c, [](int i) { return i > 5; });
+  VERIFY( c.size() == 12 );
+  std::erase_if(c, [](int i) { return i == 4; });
+  VERIFY( c.size() == 8 );
+  std::erase_if(c, [](int i) { return i & 1; });
+  VERIFY( eq(c, {2, 2}) );
+
+  std::inplace_vector<int, 0> e;
+  std::erase_if(e, [](int i) { return i > 5; });
+  VERIFY( e.empty() );
+}
+
+int main()
+{
+  test_erase();
+  test_erase_if();
+
+  constexpr bool _ = [] {
+    test_erase();
+    test_erase_if();
+    return true;
+  }();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/assign.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/assign.cc
new file mode 100644
index 0000000..65b505e
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/assign.cc
@@ -0,0 +1,379 @@
+// { dg-do run { target c++26 } }
+
+#include <inplace_vector>
+
+#include <span>
+#include <testsuite_hooks.h>
+#include <testsuite_iterators.h>
+
+struct X
+{
+  X() = default;
+  constexpr X(int p) : v(p) {}
+  constexpr X(const X& o) : v(o.v) { } // not trivial
+  constexpr X& operator=(const X& o) // not trivial
+  { v = o.v; return *this; }
+
+  int v;
+
+  friend auto operator<=>(const X&, const X&) = default;
+};
+
+template<typename T, typename V, size_t N>
+constexpr bool
+eq(const std::inplace_vector<V, N>& l, std::span<const T> r) {
+  if (l.size() != r.size())
+    return false;
+  for (auto i = 0u; i < l.size(); ++i)
+    if (l[i] != r[i])
+      return false;
+  return true;
+};
+
+template<size_t N, typename T, template<class TT> class ItType>
+constexpr void
+test_assign_empty_it()
+{
+  using namespace __gnu_test;
+
+  T a[]{1,2,3,4,5,6,7,8,9,10};
+  using It = ItType<T>;
+  using Range = test_range<T, ItType>;
+  using SizedRange = test_sized_range<T, ItType>;
+
+  const std::inplace_vector<T, N> src(std::from_range, std::span(a, N));
+  std::inplace_vector<T, N> v;
+
+  v = src;
+  v.assign_range(Range(a, a));
+  VERIFY( v.empty() );
+  v.assign_range(Range(a, a));
+  VERIFY( v.empty() );
+
+  v = src;
+  v.assign_range(SizedRange(a, a));
+  VERIFY( v.empty() );
+
+  v = src;
+  auto bounds = typename It::ContainerType(a, a+9);
+  v.assign(It(a, &bounds), It(a, &bounds));
+  VERIFY( v.empty() );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  static_assert(N < 9);
+
+  v = src;
+  try
+  {
+    v.assign_range(Range(a, a+9));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  if constexpr (std::ranges::sized_range<Range> || std::ranges::forward_range<Range>)
+    VERIFY( eq<T>(v, {a, N}) );
+
+  v = src;
+  try
+  {
+    v.assign_range(SizedRange(a, a+9));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  v = src;
+  bounds = typename It::ContainerType(a, a+9);
+  try
+  {
+    v.assign(It(a, &bounds), It(a+9, &bounds));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  if constexpr(std::forward_iterator<It>)
+    VERIFY( eq<T>(v, {a, N}) );
+#endif
+}
+
+template<size_t N, typename T>
+constexpr void
+test_assign_empty_other()
+{
+  T a[]{1,2,3,4,5,6,7,8,9,10};
+  const std::inplace_vector<T, N> src(std::from_range, std::span(a, N));
+  std::inplace_vector<T, N> v;
+
+  v = src;
+  v.assign(0, T(4));
+  VERIFY( v.empty() );
+
+  v = src;
+  v.assign({});
+  VERIFY( v.empty() );
+
+  v = src;
+  v = {};
+  VERIFY( v.empty() );
+
+  v = src;
+  v.resize(0, T(3));
+  VERIFY( v.empty() );
+
+  v = src;
+  v.resize(0);
+  VERIFY( v.empty() );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  static_assert(N < 9);
+
+  v = src;
+  try
+  {
+    v.assign(9, T(4));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  std::initializer_list<T> il =
+  {T(0), T(1), T(2), T(3), T(4), T(5), T(6), T(7), T(8), T(9), T(10)};
+  try
+  {
+    v.assign(il);
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  try
+  {
+    v = il;
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  try
+  {
+    v.resize(9, T(3));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  try
+  {
+    v.resize(9);
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+#endif
+}
+
+template<size_t N, typename T>
+constexpr void
+test_assign_empty()
+{
+  using namespace __gnu_test;
+  test_assign_empty_it<N, T, input_iterator_wrapper>();
+  test_assign_empty_it<N, T, forward_iterator_wrapper>();
+  test_assign_empty_it<N, T, random_access_iterator_wrapper>();
+
+  test_assign_empty_other<N, T>;
+}
+
+template<typename Range>
+constexpr void
+test_assign_range()
+{
+  using T = std::ranges::range_value_t<Range>;
+  T a[]{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
+
+  std::inplace_vector<T, 10> v;
+  v.assign_range(Range(a,a+5));
+  VERIFY( eq<T>(v, {a, 5}) );
+
+  v.assign_range(Range(a,a+7));
+  VERIFY( eq<T>(v, {a, 7}) );
+
+  v.assign_range(Range(a,a+3));
+  VERIFY( eq<T>(v, {a, 3}) );
+
+  v.assign_range(Range(a,a+10));
+  VERIFY( eq<T>(v, {a, 10}) );
+}
+
+template<typename T, template<class TT> class ItType>
+constexpr void
+test_assign_iterators()
+{
+  T a[]{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
+  using It = ItType<T>;
+
+  std::inplace_vector<T, 10> v;
+
+  auto bounds = typename It::ContainerType(a, a+15);
+  v.assign(It(a, &bounds), It(a+5, &bounds));
+  VERIFY( eq<T>(v, {a, 5}) );
+
+  bounds = typename It::ContainerType(a, a+15);
+  v.assign(It(a, &bounds), It(a+7, &bounds));
+  VERIFY( eq<T>(v, {a, 7}) );
+
+  bounds = typename It::ContainerType(a, a+15);
+  v.assign(It(a, &bounds), It(a+3, &bounds));
+  VERIFY( eq<T>(v, {a, 3}) );
+
+  bounds = typename It::ContainerType(a, a+15);
+  v.assign(It(a, &bounds), It(a+10, &bounds));
+  VERIFY( eq<T>(v, {a, 10}) );
+}
+
+template<typename T>
+constexpr void
+test_assign_initializer_list()
+{
+  T a[]{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
+
+  std::inplace_vector<T, 10> v;
+
+  v.assign({T(1), T(2), T(3), T(4), T(5)});
+  VERIFY( eq<T>(v, {a, 5}) );
+
+  v = {T(1), T(2), T(3), T(4), T(5), T(6), T(7)};
+  VERIFY( eq<T>(v, {a, 7}) );
+
+  v.assign({T(1), T(2), T(3)});
+  VERIFY( eq<T>(v, {a, 3}) );
+
+  v = {T(1), T(2), T(3), T(4), T(5), T(6), T(7), T(8), T(9), T(10)};
+  VERIFY( eq<T>(v, {a, 10}) );
+}
+
+template<typename T>
+constexpr void
+test_assign_repeated()
+{
+  auto rep = [](const std::inplace_vector<T, 10>& v, size_t c, const T& t)
+  {
+    if (v.size() != c)
+      return false;
+    for (const T& o : v)
+      if (o != t)
+	return false;
+    return true;
+  };
+
+  std::inplace_vector<T, 10> v;
+
+  v.assign(5, T(1));
+  VERIFY( rep(v, 5, T(1)) );
+
+  v.assign(7, T(2));
+  VERIFY( rep(v, 7, T(2)) );
+
+  v.assign(3, T(4));
+  VERIFY( rep(v, 3, T(4)) );
+
+  v.assign(10, T(8));
+  VERIFY( rep(v, 10, T(8)) );
+}
+
+template<typename T>
+constexpr void
+test_resize()
+{
+  T a[]{1,1,1,1,2,2,2,0,0,0};
+
+  std::inplace_vector<T, 10> v;
+
+  v.resize(4, T(1));
+  VERIFY( eq<T>(v, {a, 4}) );
+
+  v.resize(7, T(2));
+  VERIFY( eq<T>(v, {a, 7}) );
+
+  v.resize(10);
+  VERIFY( eq<T>(v, {a, 10}) );
+
+  v.resize(6, T(1));
+  VERIFY( eq<T>(v, {a, 6}) );
+}
+
+template<typename T>
+constexpr void
+test_assigns()
+{
+  using namespace __gnu_test;
+  test_assign_range<test_forward_range<int>>();
+  test_assign_range<test_sized_range_sized_sent<int, forward_iterator_wrapper>>();
+
+  test_assign_range<test_input_range<int>>();
+  test_assign_range<test_input_sized_range<int>>();
+  test_assign_range<test_sized_range_sized_sent<int, input_iterator_wrapper>>();
+
+  test_assign_range<test_range<int, input_iterator_wrapper_nocopy>>();
+  test_assign_range<test_sized_range<int, input_iterator_wrapper_nocopy>>();
+  test_assign_range<test_sized_range_sized_sent<int, input_iterator_wrapper_nocopy>>();
+
+  test_assign_iterators<T, input_iterator_wrapper>();
+  test_assign_iterators<T, forward_iterator_wrapper>();
+  test_assign_iterators<T, random_access_iterator_wrapper>();
+
+  test_assign_initializer_list<T>();
+  test_assign_repeated<T>();
+  test_resize<T>();
+}
+
+int main()
+{
+  auto test_all = [] {
+    test_assign_empty<0, int>();
+    test_assign_empty<0, X>();
+    test_assign_empty<2, int>();
+
+    test_assigns<int>();
+#ifdef __cpp_lib_constexpr_inplace_vector
+#error uncomemnt test_inserts<X>()
+#endif
+    if !consteval {
+      test_assign_empty<2, X>();
+      test_assigns<X>();
+    }
+    return true;
+  };
+
+
+  test_all();
+  static_assert(test_all());
+}
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/erase.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/erase.cc
new file mode 100644
index 0000000..8b82ab4
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/erase.cc
@@ -0,0 +1,117 @@
+// { dg-do run { target c++26 } }
+
+#include <inplace_vector>
+
+#include <span>
+#include <testsuite_hooks.h>
+
+struct X
+{
+  X() = default;
+  constexpr X(int p) : v(p) {}
+  constexpr X(const X& o) : v(o.v) { } // not trivial
+  constexpr X& operator=(const X& o) // not trivial
+  { v = o.v; return *this; }
+
+  int v;
+
+  friend auto operator<=>(const X&, const X&) = default;
+};
+
+template<typename T, typename V, size_t N>
+constexpr bool
+eq(const std::inplace_vector<V, N>& l, std::span<const T> r) {
+  if (l.size() != r.size())
+    return false;
+  for (auto i = 0u; i < l.size(); ++i)
+    if (l[i] != r[i])
+      return false;
+  return true;
+};
+
+template<typename T, typename V, size_t N>
+constexpr bool
+eq(const std::inplace_vector<V, N>& l, std::initializer_list<T> r)
+{ return eq<T>(l, std::span<const T>(r)); }
+
+template<size_t N, typename T>
+constexpr void
+test_erase_all_or_none()
+{
+  using namespace __gnu_test;
+
+  T a[]{1,2,3,4,5,6,7,8,9};
+  const T c(10);
+
+  std::inplace_vector<T, N> src(std::from_range, std::span(a, a+N));
+  std::inplace_vector<T, N> v;
+
+  v = src;
+  auto it = v.erase(v.begin(), v.begin());
+  VERIFY( it == v.begin() );
+  VERIFY( eq<T>(v, {a, N}) );
+
+  it = v.erase(v.end(), v.end());
+  VERIFY( it == v.end() );
+  VERIFY( eq<T>(v, {a, N}) );
+
+  it = v.erase(v.begin(), v.end());
+  VERIFY( it == v.begin() );
+  VERIFY( v.empty() );
+
+  v = src;
+  v.clear();
+  VERIFY( v.empty() );
+}
+
+template<typename T>
+constexpr void
+test_erase()
+{
+  std::inplace_vector<T, 10> v{T(1), T(2), T(3), T(4), T(5), T(6), T(7)};
+
+  auto it = v.erase(v.begin());
+  VERIFY( eq<T>(v, {T(2), T(3), T(4), T(5), T(6), T(7)}) );
+  VERIFY( it == v.begin() );
+
+  it = v.erase(v.end()-1);
+  VERIFY( eq<T>(v, {T(2), T(3), T(4), T(5), T(6)}) );
+  VERIFY( it == v.end() );
+
+  it = v.erase(v.begin()+2, v.begin()+4);
+  VERIFY( eq<T>(v, {T(2), T(3), T(6)}) );
+  VERIFY( it == v.begin()+2 );
+
+  it = v.erase(v.end()-1, v.end());
+  VERIFY( eq<T>(v, {T(2), T(3)}) );
+  VERIFY( it == v.end() );
+
+  it = v.erase(v.begin(), v.begin()+1);
+  VERIFY( eq<T>(v, {T(3)}) );
+  VERIFY( it == v.begin() );
+
+  v.pop_back();
+  VERIFY( v.empty() );
+}
+
+int main()
+{
+  auto test_all = [] {
+    test_erase_all_or_none<0, int>();
+    test_erase_all_or_none<0, X>();
+
+    test_erase_all_or_none<4, int>();
+
+    test_erase<int>();
+#ifdef __cpp_lib_constexpr_inplace_vector
+#error uncomemnt test_inserts<X>()
+#endif
+    if ! consteval {
+      test_erase_all_or_none<4, X>();
+    }
+    return true;
+  };
+
+  test_all();
+  static_assert(test_all());;
+}
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/erase_neg.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/erase_neg.cc
new file mode 100644
index 0000000..a1f43b3
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/erase_neg.cc
@@ -0,0 +1,43 @@
+// { dg-do compile { target c++26 } }
+
+#include <inplace_vector>
+
+template<size_t N, typename T>
+constexpr bool
+test_pop_back_on_empty()
+{
+  std::inplace_vector<T, N> v;
+  v.pop_back(); // { dg-error "in 'constexpr' expansion of" }
+  return true;
+}
+
+template<size_t N, typename T>
+constexpr bool
+test_erase_begin_on_empty()
+{
+  std::inplace_vector<T, N> v;
+  v.erase(v.begin()); // { dg-error "in 'constexpr' expansion of" }
+  return true;
+}
+
+template<size_t N, typename T>
+constexpr bool
+test_erase_end(size_t size = 0)
+{
+  std::inplace_vector<T, N> v(size, T());
+  v.erase(v.end()); // { dg-error "in 'constexpr' expansion of" }
+  return true;
+}
+
+static_assert(test_pop_back_on_empty<0, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_pop_back_on_empty<4, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_erase_begin_on_empty<0, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_erase_begin_on_empty<4, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_erase_end<0, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_erase_end<4, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_erase_end<4, int>(2)); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_erase_end<4, int>(4)); // { dg-error "in 'constexpr' expansion of" }
+
+// { dg-prune-output "non-constant condition for static assertion" }
+// { dg-prune-output "is not a constant expression" }
+// { dg-prune-output "call to non-'constexpr' function" }
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/multi_insert.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/multi_insert.cc
new file mode 100644
index 0000000..6a5b62f
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/multi_insert.cc
@@ -0,0 +1,606 @@
+// { dg-do run { target c++26 } }
+
+#include <inplace_vector>
+
+#include <span>
+#include <testsuite_hooks.h>
+#include <testsuite_iterators.h>
+
+struct X
+{
+  X() = default;
+  constexpr X(int p) : v(p) {}
+  constexpr X(const X& o) : v(o.v) { } // not trivial
+  constexpr X& operator=(const X& o) // not trivial
+  { v = o.v; return *this; }
+
+  int v;
+
+  friend auto operator<=>(const X&, const X&) = default;
+};
+
+template<typename T, typename V, size_t N>
+constexpr bool
+eq(const std::inplace_vector<V, N>& l, std::span<const T> r) {
+  if (l.size() != r.size())
+    return false;
+  for (auto i = 0u; i < l.size(); ++i)
+    if (l[i] != r[i])
+      return false;
+  return true;
+};
+
+template<typename T, typename V, size_t N>
+constexpr bool
+prefix(const std::inplace_vector<V, N>& l, std::span<const T> r) {
+  if (l.size() < r.size())
+    return false;
+  for (auto i = 0u; i < r.size(); ++i)
+    if (l[i] != r[i])
+      return false;
+  return true;
+};
+
+template<size_t N, typename T, template<class TT> class ItType>
+constexpr void
+test_add_to_full_it()
+{
+  using namespace __gnu_test;
+
+  T a[]{1,2,3,4,5,6,7,8,9};
+  using It = ItType<T>;
+  using Range = test_range<T, ItType>;
+  using SizedRange = test_sized_range<T, ItType>;
+
+  std::inplace_vector<T, N> v(std::from_range, std::span(a, a+N));
+
+  Range r1(a, a);
+  auto rit1 = v.try_append_range(r1);
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( rit1.base() == a );
+
+  SizedRange r2(a, a);
+  auto rit2 = v.try_append_range(r2);
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( rit2.base() == a );
+
+  v.append_range(Range(a, a));
+  VERIFY( eq<T>(v, {a, N}) );
+  v.append_range(SizedRange(a, a));
+  VERIFY( eq<T>(v, {a, N}) );
+
+  auto it = v.insert_range(v.end(), Range(a, a));
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( it == v.end() );
+  it = v.insert_range(v.end(), SizedRange(a, a));
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( it == v.end() );
+
+  auto bounds = typename It::ContainerType(a, a+9);
+  it = v.insert(v.end(), It(a, &bounds), It(a, &bounds));
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( it == v.end() );
+
+  it = v.insert_range(v.begin(), SizedRange(a, a));
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( it == v.begin() );
+  it = v.insert_range(v.begin(), Range(a, a));
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( it == v.begin() );
+
+  bounds = typename It::ContainerType(a, a+9);
+  it = v.insert(v.begin(), It(a, &bounds), It(a, &bounds));
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( it == v.begin() );
+
+  // Inserting non-empty range
+  Range r3(a+3, a+5);
+  auto rit3 = v.try_append_range(r3);
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( rit3.base() == a+3 );
+
+  SizedRange r4(a+2, a+5);
+  auto rit4 = v.try_append_range(r4);
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( rit4.base() == a+2 );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  try
+  {
+    v.append_range(Range(a, a + 5));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  try
+  {
+    v.append_range(SizedRange(a, a + 5));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  try
+  {
+    v.insert_range(v.begin(), SizedRange(a, a+5));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  try
+  {
+    v.insert_range(v.begin(), Range(a, a+5));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  auto gn = std::ranges::sized_range<Range> || std::ranges::forward_range<Range> ? N : 0;
+  VERIFY( prefix<T>(v, {a, gn}) );
+
+  v = std::inplace_vector<T, N>(std::from_range, std::span(a, a+N));
+  try
+  {
+    v.insert_range(v.begin(), Range(a, a+5));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  gn = std::forward_iterator<It> ? N : 0;
+  VERIFY( prefix<T>(v, {a, gn}) );
+#endif
+}
+
+template<size_t N, typename T>
+constexpr void
+test_add_to_full_other()
+{
+  using namespace __gnu_test;
+
+  T a[]{1,2,3,4,5,6,7,8,9};
+  std::inplace_vector<T, N> v(std::from_range, std::span(a, a+N));
+
+  auto it = v.insert(v.end(), {});
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( it == v.end() );
+  it = v.insert(v.end(), 0u, T(2));
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( it == v.end() );
+
+  it = v.insert(v.begin(), {});
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( it == v.begin() );
+  it = v.insert(v.begin(), 0u, T(2));
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( it == v.begin() );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  v = std::inplace_vector<T, N>(std::from_range, std::span(a, a+N));
+  try
+  {
+    v.insert(v.begin(), {T(1), T(2), T(3)});
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  try
+  {
+    v.insert(v.begin(), 4u, T(3));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+#endif
+}
+
+
+template<size_t N, typename T>
+constexpr void
+test_add_to_full()
+{
+  using namespace __gnu_test;
+  test_add_to_full_it<N, T, input_iterator_wrapper>();
+  test_add_to_full_it<N, T, forward_iterator_wrapper>();
+  test_add_to_full_it<N, T, random_access_iterator_wrapper>();
+
+  test_add_to_full_other<N, T>();
+}
+
+template<typename Range>
+constexpr void
+test_append_range()
+{
+  using T = std::ranges::range_value_t<Range>;
+  T a[]{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
+
+  std::inplace_vector<T, 20> v;
+  v.append_range(Range(a,a+10));
+  VERIFY( eq<T>(v, {a, 10}) );
+
+  v.append_range(Range(a+10, a+15));
+  VERIFY( eq<T>(v, {a, 15}) );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  try
+  {
+    v.append_range(Range(a, a+10));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( prefix<T>(v, {a, 15}) );
+#endif
+}
+
+template<typename Range>
+constexpr void
+test_try_append_range()
+{
+  using T = std::ranges::range_value_t<Range>;
+  T a[]{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25};
+
+  std::inplace_vector<T, 20> v;
+  Range r1 = Range(a, a+10);
+  auto it1 = v.try_append_range(r1);
+  VERIFY( eq<T>(v, {a, 10}) );
+  VERIFY( it1.base() == a+10 );
+
+  Range r2 = Range(a+10, a+15);
+  auto it2 = v.try_append_range(r2);
+  VERIFY( eq<T>(v, {a, 15}) );
+  VERIFY( it2.base() == a+15 );
+
+  Range r3 = Range(a+15, a+25);
+  auto it3 = v.try_append_range(r3);
+  VERIFY( eq<T>(v, {a, 20}) );
+  VERIFY( it3.base() == a+20 );
+}
+
+template<typename Range>
+constexpr void
+test_insert_range()
+{
+  using T = std::ranges::range_value_t<Range>;
+  T a[]{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
+
+  std::inplace_vector<T, 20> v;
+  auto it = v.insert_range(v.begin(), Range(a+10,a+15));
+  VERIFY( eq<T>(v, {a+10, 5}) );
+  VERIFY( it == v.begin() );
+
+  it = v.insert_range(v.begin(), Range(a, a+5));
+  VERIFY( prefix<T>(v, {a, 5}) );
+  VERIFY( it == v.begin() );
+
+  it = v.insert_range(v.begin() + 5, Range(a+5, a+10));
+  VERIFY( eq<T>(v, {a, 15}) );
+  VERIFY( it == v.begin() + 5 );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  const bool seg = std::ranges::sized_range<Range> || std::ranges::forward_range<Range>;
+  auto vc = v;
+  try
+  {
+    vc.insert_range(vc.begin(), Range(a, a+10));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( prefix<T>(vc, {a, seg ? 15 : 0}) );
+
+  vc = v;
+  try
+  {
+    vc.insert_range(vc.begin()+5, Range(a, a+10));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( prefix<T>(vc, {a, seg ? 15 : 5}) );
+
+  vc = v;
+  try
+  {
+    vc.insert_range(vc.end(), Range(a, a+10));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( prefix<T>(vc, {a, 15}) );
+#endif
+}
+
+template<typename Range>
+constexpr void
+do_test_ranges()
+{
+  test_append_range<Range>();
+  test_try_append_range<Range>();
+  test_insert_range<Range>();
+}
+
+template<typename T, template<class TT> class ItType>
+constexpr void
+test_insert_iterators()
+{
+  T a[]{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
+  using It = ItType<T>;
+
+  std::inplace_vector<T, 20> v;
+
+  auto bounds = typename It::ContainerType(a, a+15);
+  auto it = v.insert(v.begin(), It(a+10, &bounds), It(a+15, &bounds));
+  VERIFY( eq<T>(v, {a+10, 5}) );
+  VERIFY( it == v.begin() );
+
+  bounds = typename It::ContainerType(a, a+15);
+  it = v.insert(v.begin(), It(a, &bounds), It(a+5, &bounds));
+  VERIFY( prefix<T>(v, {a, 5}) );
+  VERIFY( it == v.begin() );
+
+  bounds = typename It::ContainerType(a, a+15);
+  it = v.insert(v.begin() + 5, It(a+5, &bounds), It(a+10, &bounds));
+  VERIFY( eq<T>(v, {a, 15}) );
+  VERIFY( it == v.begin() + 5 );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  const bool seg = std::forward_iterator<It>;
+  auto vc = v;
+  bounds = typename It::ContainerType(a, a+15);
+  try
+  {
+    vc.insert(vc.begin(), It(a, &bounds), It(a+10, &bounds));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( prefix<T>(vc, {a, seg ? 15 : 0}) );
+
+  vc = v;
+  bounds = typename It::ContainerType(a, a+15);
+  try
+  {
+    vc.insert(vc.begin()+5, It(a, &bounds), It(a+10, &bounds));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( prefix<T>(vc, {a, seg ? 15 : 5}) );
+
+  vc = v;
+  bounds = typename It::ContainerType(a, a+15);
+  try
+  {
+    vc.insert(vc.end(), It(a, &bounds), It(a+10, &bounds));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( prefix<T>(vc, {a, 15}) );
+#endif
+}
+
+template<typename T>
+constexpr void
+test_insert_initializer_list()
+{
+  T a[]{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
+
+  std::inplace_vector<T, 20> v;
+
+  auto it = v.insert(v.begin(), {T(11), T(12), T(13), T(14), T(15)});
+  VERIFY( eq<T>(v, {a+10, 5}) );
+  VERIFY( it == v.begin() );
+
+  it = v.insert(v.begin(), {T(1), T(2), T(3), T(4), T(5)});
+  VERIFY( prefix<T>(v, {a, 5}) );
+  VERIFY( it == v.begin() );
+
+  it = v.insert(v.begin() + 5, {T(6), T(7), T(8), T(9), T(10)});
+  VERIFY( eq<T>(v, {a, 15}) );
+  VERIFY( it == v.begin() + 5 );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  std::initializer_list<T> il
+  = {T(0), T(1), T(2), T(3), T(4), T(5), T(6), T(7), T(8), T(9)};
+
+  try
+  {
+    v.insert(v.begin(), il);
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, 15}) );
+
+  try
+  {
+    v.insert(v.begin()+5, il);
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, 15}) );
+
+  try
+  {
+    v.insert(v.end(), il);
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, 15}) );
+#endif
+}
+
+template<typename T>
+constexpr void
+test_insert_repeated()
+{
+  T a[]{5,5,5,5,5,6,6,6,6,6,7,7,7,7,7};
+
+  std::inplace_vector<T, 20> v;
+
+  auto it = v.insert(v.begin(), 5, T(7));
+  VERIFY( eq<T>(v, {a+10, 5}) );
+  VERIFY( it == v.begin() );
+
+  it = v.insert(v.begin(), 5, T(5));
+  VERIFY( prefix<T>(v, {a, 5}) );
+  VERIFY( it == v.begin() );
+
+  it = v.insert(v.begin() + 5, 5, T(6));
+  VERIFY( eq<T>(v, {a, 15}) );
+  VERIFY( it == v.begin() + 5 );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  try
+  {
+    v.insert(v.begin(), 10u, T(6));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, 15}) );
+
+  try
+  {
+    v.insert(v.begin()+5, 10u, T(6));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, 15}) );
+
+  try
+  {
+    v.insert(v.end(), 10u, T(6));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, 15}) );
+#endif
+}
+
+template<typename T>
+constexpr void
+test_inserts()
+{
+  using namespace __gnu_test;
+  do_test_ranges<test_forward_range<int>>();
+  do_test_ranges<test_sized_range_sized_sent<int, forward_iterator_wrapper>>();
+
+  do_test_ranges<test_input_range<int>>();
+  do_test_ranges<test_input_sized_range<int>>();
+  do_test_ranges<test_sized_range_sized_sent<int, input_iterator_wrapper>>();
+
+  do_test_ranges<test_range<int, input_iterator_wrapper_nocopy>>();
+  do_test_ranges<test_sized_range<int, input_iterator_wrapper_nocopy>>();
+  do_test_ranges<test_sized_range_sized_sent<int, input_iterator_wrapper_nocopy>>();
+
+  test_insert_iterators<T, input_iterator_wrapper>();
+  test_insert_iterators<T, forward_iterator_wrapper>();
+  test_insert_iterators<T, random_access_iterator_wrapper>();
+
+test_insert_initializer_list<T>();
+test_insert_repeated<T>();
+}
+
+int main()
+{
+auto test_all = []{
+  test_add_to_full<0, int>();
+  test_add_to_full<0, X>();
+  test_add_to_full<4, int>();
+
+    test_inserts<int>();
+#ifdef __cpp_lib_constexpr_inplace_vector
+#error uncomemnt test_inserts<X>()
+#endif
+    if !consteval {
+      test_add_to_full<4, X>();
+      test_inserts<X>();
+    }
+    return true;
+  };
+
+  test_all();
+  static_assert(test_all());
+}
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/single_insert.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/single_insert.cc
new file mode 100644
index 0000000..d5e893c
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/single_insert.cc
@@ -0,0 +1,215 @@
+// { dg-do run { target c++26 } }
+
+#include <inplace_vector>
+
+#include <span>
+#include <testsuite_hooks.h>
+
+struct X
+{
+  X() = default;
+  constexpr X(int p) : v(p) {}
+  constexpr X(const X& o) : v(o.v) { } // not trivial
+  constexpr X& operator=(const X& o) // not trivial
+  { v = o.v; return *this; }
+
+  int v;
+
+  friend auto operator<=>(const X&, const X&) = default;
+};
+
+template<typename T, typename V, size_t N>
+constexpr bool
+eq(const std::inplace_vector<V, N>& l, std::span<const T> r) {
+  if (l.size() != r.size())
+    return false;
+  for (auto i = 0u; i < l.size(); ++i)
+    if (l[i] != r[i])
+      return false;
+  return true;
+};
+
+template<size_t N, typename T>
+constexpr void
+test_add_to_full()
+{
+  using namespace __gnu_test;
+
+  T a[]{1,2,3,4,5,6,7,8,9};
+  const T c(10);
+
+  std::inplace_vector<T, N> v(std::from_range, std::span(a, a+N));
+
+  VERIFY( v.try_emplace_back(1) == nullptr );
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( v.try_push_back(T(1)) == nullptr );
+  VERIFY( eq<T>(v, {a, N}) );
+  VERIFY( v.try_push_back(c) == nullptr );
+  VERIFY( eq<T>(v, {a, N}) );
+
+#ifdef __cpp_exceptions
+#ifdef __cpp_lib_constexpr_exceptions
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  try
+  {
+    v.emplace_back(1);
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  try
+  {
+    v.push_back(T(1));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  try
+  {
+    v.push_back(c);
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  try
+  {
+    v.insert(v.end(), T(1));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  try
+  {
+    v.insert(v.begin(), c);
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  try
+  {
+    v.emplace(v.end(), c);
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+
+  try
+  {
+    v.emplace(v.begin(), T(2));
+    VERIFY(false);
+  }
+  catch (std::bad_alloc const&)
+  {
+  }
+  VERIFY( eq<T>(v, {a, N}) );
+#endif
+}
+
+template<typename T>
+constexpr void
+test_inserts()
+{
+  T a[]{3,14,13,1,2,3,4,5,3,7,8,3,10,11,3};
+  const T c(3);
+
+  std::inplace_vector<T, 20> v;
+
+  v.emplace_back(1);
+  VERIFY( eq<T>(v, {a+3, 1}) );
+  v.push_back(T(2));
+  VERIFY( eq<T>(v, {a+3, 2}) );
+  v.push_back(c);
+  VERIFY( eq<T>(v, {a+3, 3}) );
+
+  v.unchecked_emplace_back(4);
+  VERIFY( eq<T>(v, {a+3, 4}) );
+  v.unchecked_push_back(T(5));
+  VERIFY( eq<T>(v, {a+3, 5}) );
+  v.unchecked_push_back(c);
+  VERIFY( eq<T>(v, {a+3, 6}) );
+
+  T* ptr = v.try_emplace_back(7);
+  VERIFY( eq<T>(v, {a+3, 7}) );
+  VERIFY( ptr = &v.back() );
+  ptr = v.try_push_back(T(8));
+  VERIFY( eq<T>(v, {a+3, 8}) );
+  VERIFY( ptr = &v.back() );
+  ptr = v.try_push_back(c);
+  VERIFY( eq<T>(v, {a+3, 9}) );
+  VERIFY( ptr = &v.back() );
+
+  auto it = v.emplace(v.end(), 10);
+  VERIFY( eq<T>(v, {a+3, 10}) );
+  VERIFY( it == v.end()-1 );
+  it = v.insert(v.end(), T(11));
+  VERIFY( eq<T>(v, {a+3, 11}) );
+  VERIFY( it == v.end()-1 );
+  it = v.insert(v.end(), c);
+  VERIFY( eq<T>(v, {a+3, 12}) );
+  VERIFY( it == v.end()-1 );
+
+  it = v.emplace(v.begin(), 13);
+  VERIFY( eq<T>(v, {a+2, 13}) );
+  VERIFY( it == v.begin() );
+  it = v.insert(v.begin(), T(14));
+  VERIFY( eq<T>(v, {a+1, 14}) );
+  VERIFY( it == v.begin() );
+  it = v.insert(v.begin(), c);
+  VERIFY( eq<T>(v, {a+0, 15}) );
+  VERIFY( it == v.begin() );
+
+  it = v.emplace(v.begin()+2, 22);
+  VERIFY( *it == 22 );
+  VERIFY( it == v.begin()+2 );
+  it = v.insert(v.begin()+6, T(24));
+  VERIFY( *it == 24 );
+  VERIFY( it == v.begin()+6 );
+  it = v.insert(v.begin()+13, c);
+  VERIFY( *it == 3 );
+  VERIFY( it == v.begin()+13 );
+}
+
+int main()
+{
+  auto test_all = [] {
+    test_add_to_full<0, int>();
+    test_add_to_full<0, X>();
+
+    test_add_to_full<4, int>();
+
+    test_inserts<int>();
+#ifdef __cpp_lib_constexpr_inplace_vector
+#error uncomemnt test_inserts<X>()
+#endif
+    if ! consteval {
+      test_add_to_full<4, X>();
+      test_inserts<X>();
+    }
+    return true;
+  };
+
+  test_all();
+  static_assert(test_all());;
+}
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/single_insert_neg.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/single_insert_neg.cc
new file mode 100644
index 0000000..0552b8c
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/modifiers/single_insert_neg.cc
@@ -0,0 +1,42 @@
+// { dg-do compile { target c++26 } }
+
+#include <inplace_vector>
+
+template<size_t N, typename T>
+constexpr bool
+test_unchecked_emplace_back()
+{
+  std::inplace_vector<T, N> v(N, T(1));
+  v.unchecked_emplace_back(); // { dg-error "in 'constexpr' expansion of" }
+  return true;
+}
+
+template<size_t N, typename T>
+constexpr bool
+test_push_back_lvalue()
+{
+  auto val = T();;
+  std::inplace_vector<T, N> v(N, T(1));
+  v.unchecked_push_back(val); // { dg-error "in 'constexpr' expansion of" }
+  return true;
+}
+
+template<size_t N, typename T>
+constexpr bool
+test_push_back_rvalue()
+{
+  std::inplace_vector<T, N> v(N, T(1));
+  v.unchecked_push_back(T()); // { dg-error "in 'constexpr' expansion of" }
+  return true;
+}
+
+static_assert(test_unchecked_emplace_back<0, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_unchecked_emplace_back<4, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_push_back_lvalue<0, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_push_back_lvalue<4, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_push_back_rvalue<0, int>()); // { dg-error "in 'constexpr' expansion of" }
+static_assert(test_push_back_rvalue<4, int>()); // { dg-error "in 'constexpr' expansion of" }
+
+// { dg-prune-output "non-constant condition for static assertion" }
+// { dg-prune-output "is not a constant expression" }
+// { dg-prune-output "call to non-'constexpr' function" }
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/move.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/move.cc
new file mode 100644
index 0000000..5abcc87
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/move.cc
@@ -0,0 +1,358 @@
+// { dg-do run { target c++26 } }
+
+#include <inplace_vector>
+#include <ranges>
+#include <testsuite_hooks.h>
+
+struct X
+{
+  X() = default;
+  constexpr X(int p) : v(p) {}
+  constexpr X(const X& o) : v(o.v) { } // not trivial
+  constexpr X(X&& o) : v(o.v) { } // not trivial
+
+  constexpr X& operator=(const X& o) // not trivial
+  { v = o.v; return *this; }
+  constexpr X& operator=(X&& o) // not trivial
+  { v = o.v; return *this; }
+
+  int v;
+
+  friend auto operator<=>(const X&, const X&) = default;
+};
+
+template<bool CNoex, bool ANoex>
+struct N
+{
+  N() = default;
+  constexpr N(N&&) noexcept(CNoex) { } // not trivial
+  constexpr N& operator=(N&& o) noexcept(ANoex) // not trivial
+  { return *this; }
+};
+
+struct D
+{
+  D() = default;
+  D(D&&) = default;
+  D& operator=(D&&) = default;
+  ~D() {} // not trivially destructible
+};
+
+struct U
+{
+  U() = default;
+  U(U&&) noexcept(false) = default; // lies about noexcept, is trivial but throwing
+  U& operator=(U&&) noexcept(false) = default; // lies about noexcept, is trivial but throwing
+};
+
+template<bool SNoex, bool CNoex, bool ANoex>
+struct S {
+  S() = default;
+  constexpr S(S&&) noexcept(CNoex) { } // not trivial
+  constexpr S& operator=(S&& o) noexcept(ANoex) // not trivial
+  { return *this; }
+
+  friend constexpr
+  void swap(S&, S&) noexcept(SNoex) {}
+};
+
+// n5008 inplace.vector.overview says for inplace_vector<T, 0>
+// provides trivial copy/move/default cosntructpr regardless of T
+struct Z
+{
+  Z(const Z&) = delete;
+  Z& operator=(const Z&) = delete;
+};
+
+template<size_t N, typename T>
+  constexpr std::inplace_vector<T, N>
+  materialize(std::initializer_list<int> il)
+  { 
+    std::inplace_vector<T, N> res;
+    for (int x : il)
+      res.emplace_back(x);
+    return res;
+  }
+
+static_assert(std::is_move_constructible_v<std::inplace_vector<int, 2>>);
+static_assert(std::is_move_constructible_v<std::inplace_vector<X, 2>>);
+static_assert(std::is_move_constructible_v<std::inplace_vector<N<false, false>, 2>>);
+static_assert(std::is_move_constructible_v<std::inplace_vector<D, 2>>);
+static_assert(std::is_move_constructible_v<std::inplace_vector<U, 2>>);
+// The operators are not constrained, as for any other container
+static_assert(std::is_move_constructible_v<std::inplace_vector<Z, 2>>);
+
+static_assert(std::is_nothrow_move_constructible_v<std::inplace_vector<int, 2>>);
+static_assert(!std::is_nothrow_move_constructible_v<std::inplace_vector<X, 2>>);
+static_assert(std::is_nothrow_move_constructible_v<std::inplace_vector<N<true, true>, 2>>);
+static_assert(std::is_nothrow_move_constructible_v<std::inplace_vector<N<true, false>, 2>>);
+static_assert(!std::is_nothrow_move_constructible_v<std::inplace_vector<N<false, true>, 2>>);
+static_assert(!std::is_nothrow_move_constructible_v<std::inplace_vector<N<false, false>, 2>>);
+static_assert(std::is_nothrow_move_constructible_v<std::inplace_vector<D, 2>>);
+static_assert(!std::is_nothrow_move_constructible_v<std::inplace_vector<U, 2>>);
+
+static_assert(std::is_trivially_move_constructible_v<std::inplace_vector<int, 2>>);
+static_assert(!std::is_trivially_move_constructible_v<std::inplace_vector<X, 2>>);
+static_assert(!std::is_trivially_move_constructible_v<std::inplace_vector<N<true, true>, 2>>);
+// is_trivially_move_constructible_v checks destructor
+static_assert(!std::is_trivially_move_constructible_v<std::inplace_vector<D, 2>>);
+static_assert(std::is_trivially_move_constructible_v<std::inplace_vector<U, 2>>);
+
+static_assert(std::is_move_assignable_v<std::inplace_vector<int, 2>>);
+static_assert(std::is_move_assignable_v<std::inplace_vector<X, 2>>);
+static_assert(std::is_move_assignable_v<std::inplace_vector<N<false, false>, 2>>);
+static_assert(std::is_move_assignable_v<std::inplace_vector<D, 2>>);
+static_assert(std::is_move_assignable_v<std::inplace_vector<U, 2>>);
+// The operators are not constrained, as for any other container
+static_assert(std::is_move_assignable_v<std::inplace_vector<Z, 2>>);
+
+static_assert(std::is_nothrow_move_assignable_v<std::inplace_vector<int, 2>>);
+static_assert(!std::is_nothrow_move_assignable_v<std::inplace_vector<X, 2>>);
+static_assert(std::is_nothrow_move_assignable_v<std::inplace_vector<N<true, true>, 2>>);
+static_assert(!std::is_nothrow_move_assignable_v<std::inplace_vector<N<true, false>, 2>>);
+static_assert(!std::is_nothrow_move_assignable_v<std::inplace_vector<N<false, true>, 2>>);
+static_assert(!std::is_nothrow_move_assignable_v<std::inplace_vector<N<false, false>, 2>>);
+static_assert(std::is_nothrow_move_assignable_v<std::inplace_vector<D, 2>>);
+static_assert(!std::is_nothrow_move_assignable_v<std::inplace_vector<U, 2>>);
+
+static_assert(std::is_trivially_move_assignable_v<std::inplace_vector<int, 2>>);
+static_assert(!std::is_trivially_move_assignable_v<std::inplace_vector<X, 2>>);
+static_assert(!std::is_trivially_move_assignable_v<std::inplace_vector<N<true, true>, 2>>);
+// destructor is not trivial
+static_assert(!std::is_trivially_move_assignable_v<std::inplace_vector<D, 2>>);
+static_assert(std::is_trivially_move_assignable_v<std::inplace_vector<U, 2>>);
+
+static_assert(std::is_nothrow_swappable_v<std::inplace_vector<int, 2>>);
+static_assert(!std::is_nothrow_swappable_v<std::inplace_vector<X, 2>>);
+static_assert(std::is_nothrow_swappable_v<std::inplace_vector<N<true, true>, 2>>);
+static_assert(!std::is_nothrow_swappable_v<std::inplace_vector<N<true, false>, 2>>);
+static_assert(!std::is_nothrow_swappable_v<std::inplace_vector<N<false, true>, 2>>);
+static_assert(!std::is_nothrow_swappable_v<std::inplace_vector<N<false, false>, 2>>);
+static_assert(std::is_nothrow_swappable_v<std::inplace_vector<S<true, true, true>, 2>>);
+static_assert(std::is_nothrow_swappable_v<std::inplace_vector<S<true, true, false>, 2>>);
+static_assert(!std::is_nothrow_swappable_v<std::inplace_vector<S<true, false, true>, 2>>);
+static_assert(!std::is_nothrow_swappable_v<std::inplace_vector<S<true, false, false>, 2>>);
+static_assert(!std::is_nothrow_swappable_v<std::inplace_vector<S<false, true, false>, 2>>);
+static_assert(!std::is_nothrow_swappable_v<std::inplace_vector<S<false, false, false>, 2>>);
+static_assert(std::is_nothrow_swappable_v<std::inplace_vector<D, 2>>);
+static_assert(!std::is_nothrow_swappable_v<std::inplace_vector<U, 2>>);
+
+static_assert(std::is_nothrow_move_constructible_v<std::inplace_vector<int, 0>>);
+static_assert(std::is_nothrow_move_constructible_v<std::inplace_vector<X, 0>>);
+static_assert(std::is_nothrow_move_constructible_v<std::inplace_vector<N<false, false>, 0>>);
+static_assert(std::is_nothrow_move_constructible_v<std::inplace_vector<D, 0>>);
+static_assert(std::is_nothrow_move_constructible_v<std::inplace_vector<U, 0>>);
+static_assert(std::is_nothrow_move_constructible_v<std::inplace_vector<Z, 0>>);
+
+static_assert(std::is_trivially_move_constructible_v<std::inplace_vector<int, 0>>);
+static_assert(std::is_trivially_move_constructible_v<std::inplace_vector<X, 0>>);
+static_assert(std::is_trivially_move_constructible_v<std::inplace_vector<D, 0>>);
+static_assert(std::is_trivially_move_constructible_v<std::inplace_vector<U, 0>>);
+static_assert(std::is_trivially_move_constructible_v<std::inplace_vector<Z, 0>>);
+
+static_assert(std::is_nothrow_move_assignable_v<std::inplace_vector<int, 0>>);
+static_assert(std::is_nothrow_move_assignable_v<std::inplace_vector<X, 0>>);
+static_assert(std::is_nothrow_move_assignable_v<std::inplace_vector<N<false, false>, 0>>);
+static_assert(std::is_nothrow_move_assignable_v<std::inplace_vector<D, 0>>);
+static_assert(std::is_nothrow_move_assignable_v<std::inplace_vector<U, 0>>);
+static_assert(std::is_nothrow_move_assignable_v<std::inplace_vector<Z, 0>>);
+
+static_assert(std::is_trivially_move_assignable_v<std::inplace_vector<int, 0>>);
+static_assert(std::is_trivially_move_assignable_v<std::inplace_vector<X, 0>>);
+static_assert(std::is_trivially_move_assignable_v<std::inplace_vector<N<false, false>, 0>>);
+static_assert(std::is_trivially_move_assignable_v<std::inplace_vector<D, 0>>);
+static_assert(std::is_trivially_move_assignable_v<std::inplace_vector<U, 0>>);
+static_assert(std::is_trivially_move_assignable_v<std::inplace_vector<Z, 0>>);
+
+static_assert(std::is_nothrow_swappable_v<std::inplace_vector<int, 0>>);
+static_assert(std::is_nothrow_swappable_v<std::inplace_vector<X, 0>>);
+static_assert(std::is_nothrow_swappable_v<std::inplace_vector<N<false, false>, 0>>);
+static_assert(std::is_nothrow_swappable_v<std::inplace_vector<D, 0>>);
+static_assert(std::is_nothrow_swappable_v<std::inplace_vector<U, 0>>);
+static_assert(std::is_nothrow_swappable_v<std::inplace_vector<Z, 0>>);
+
+template<typename T, size_t N>
+constexpr bool
+eq(const std::inplace_vector<T, N>& s, std::span<const T> o)
+{ return std::ranges::equal(s, o); }
+
+constexpr void
+test_ctor()
+{
+  auto e0 = materialize<0, int>({});
+  VERIFY( e0.empty() );
+  auto e1 = materialize<0, X>({});
+  VERIFY( e1.empty() );
+  auto e2 = materialize<0, Z>({});
+  VERIFY( e2.empty() );
+
+  auto c0 = materialize<5, int>({});
+  VERIFY( c0.empty() );
+
+  auto c3 = materialize<5, int>({1, 2, 3});
+  VERIFY( eq(c3, {1, 2, 3}) );
+
+  auto c5 = materialize<5, int>({1, 2, 3, 4, 5});
+  VERIFY( eq(c5, {1, 2, 3, 4, 5}) );
+
+#ifdef __cpp_lib_constexpr_inplace_vector
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  auto x0 = materialize<3, X>({});
+  VERIFY( x0.empty() );
+
+  auto x2 = materialize<3, X>({1, 2});
+  VERIFY( eq(x2, {1, 2}) );
+
+  auto x3 = materialize<3, X>({1, 2, 3});
+  VERIFY( eq(x3, {1, 2, 3}) );
+}
+
+constexpr void
+test_assign()
+{
+  std::inplace_vector<int, 0> e0;
+  e0 = materialize<0, int>({});
+  VERIFY( e0.empty() );
+  std::inplace_vector<X, 0> e1;
+  e1 = materialize<0, X>({});
+  VERIFY( e1.empty() );
+  std::inplace_vector<Z, 0> e2;
+  e2 = materialize<0, Z>({});
+  VERIFY( e2.empty() );
+
+  std::inplace_vector<int, 5> c;
+  c = materialize<5, int>({});
+  VERIFY( c.empty() );
+
+  c = materialize<5, int>({1, 2, 3});
+  VERIFY( eq(c, {1, 2, 3}) );
+ 
+  c = materialize<5, int>({1, 2, 3, 4, 5});
+  VERIFY( eq(c, {1, 2, 3, 4, 5}) );
+
+  c = materialize<5, int>({4, 5});
+  VERIFY( eq(c, {4, 5}) );
+
+  c = materialize<5, int>({});
+  VERIFY( c.empty() );
+
+#ifdef __cpp_lib_constexpr_inplace_vector
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  std::inplace_vector<X, 5> x;
+  x = materialize<5, X>({});
+  VERIFY( x.empty() );
+
+  x = materialize<5, X>({1, 2, 3});
+  VERIFY( eq(x, {1, 2, 3}) );
+ 
+  x = materialize<5, X>({1, 2, 3, 4, 5});
+  VERIFY( eq(x, {1, 2, 3, 4, 5}) );
+
+  x = materialize<5, X>({4, 5});
+  VERIFY( eq(x, {4, 5}) );
+
+  x = materialize<5, X>({});
+  VERIFY( x.empty() );
+}
+
+constexpr void
+test_swap()
+{
+  std::inplace_vector<int, 0> e0a, e0b;
+  swap(e0a, e0b);
+  VERIFY( e0a.empty() );
+  VERIFY( e0b.empty() );
+  e0a.swap(e0b);
+  VERIFY( e0a.empty() );
+  VERIFY( e0b.empty() );
+ 
+  std::inplace_vector<X, 0> e1a, e1b;
+  swap(e1a, e1b);
+  VERIFY( e1a.empty() );
+  VERIFY( e1b.empty() );
+  e1a.swap(e1b);
+  VERIFY( e1a.empty() );
+  VERIFY( e1b.empty() );
+
+  std::inplace_vector<Z, 0> e2a, e2b;
+  swap(e2a, e2b);
+  VERIFY( e2a.empty() );
+  VERIFY( e2b.empty() );
+  e2a.swap(e2b);
+  VERIFY( e2a.empty() );
+  VERIFY( e2b.empty() );
+
+  std::inplace_vector<int, 5> c0;
+  std::inplace_vector<int, 5> c3{1, 2, 3};
+  std::inplace_vector<int, 5> c5{1, 2, 3, 4, 5};
+
+  swap(c0, c3);
+  VERIFY( c3.empty() );
+  VERIFY( eq(c0, {1, 2, 3}) );
+  c0.swap(c3);
+  VERIFY( c0.empty() );
+  VERIFY( eq(c3, {1, 2, 3}) );
+
+  swap(c3, c5);
+  VERIFY( eq(c5, {1, 2, 3}) );
+  VERIFY( eq(c3, {1, 2, 3, 4, 5}) );
+  c5.swap(c3);
+  VERIFY( eq(c3, {1, 2, 3}) );
+  VERIFY( eq(c5, {1, 2, 3, 4, 5}) );
+
+#ifdef __cpp_lib_constexpr_inplace_vector
+#error remove the consteval check
+#endif
+  if consteval {
+    return;
+  }
+
+  std::inplace_vector<X, 5> x0;
+  std::inplace_vector<X, 5> x3 = {1, 2, 3};
+  std::inplace_vector<X, 5> x5 = {1, 2, 3, 4, 5};
+
+  swap(x0, x3);
+  VERIFY( x3.empty() );
+  VERIFY( eq(x0, {1, 2, 3}) );
+  x0.swap(x3);
+  VERIFY( x0.empty() );
+  VERIFY( eq(x3, {1, 2, 3}) );
+
+  swap(x3, x5);
+  VERIFY( eq(x5, {1, 2, 3}) );
+  VERIFY( eq(x3, {1, 2, 3, 4, 5}) );
+  x5.swap(x3);
+  VERIFY( eq(x3, {1, 2, 3}) );
+  VERIFY( eq(x5, {1, 2, 3, 4, 5}) );
+}
+
+constexpr auto e0 = materialize<0, int>({});
+constexpr auto e1 = materialize<0, X>({});
+constexpr auto e2 = materialize<0, Z>({});
+
+constexpr auto t1 = materialize<3, int>({});
+constexpr auto t2 = materialize<3, int>({1, 2});
+constexpr auto t3 = materialize<3, int>({11, 22, 33});
+
+int main()
+{
+  auto tests = [] {
+    test_ctor();
+    test_assign();
+    test_swap();
+    return true;
+  };
+
+  tests();
+  constexpr bool _ = tests();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/relops.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/relops.cc
new file mode 100644
index 0000000..f9c5ce9
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/relops.cc
@@ -0,0 +1,60 @@
+// { dg-do run { target c++26 } }
+
+#include <inplace_vector>
+
+#include <span>
+#include <testsuite_hooks.h>
+
+template<size_t N, typename T>
+constexpr void
+test_equal(size_t c)
+{
+  T a[]{1, 2, 3, 4, 5, 6, 7, 8, 9};
+  std::inplace_vector<T, N> v(a, a+c);
+  VERIFY(   v == v  );
+  VERIFY( !(v != v) );
+  VERIFY( !(v <  v) );
+  VERIFY( !(v >  v) );
+  VERIFY(   v <= v  );
+  VERIFY(   v >= v  );
+  VERIFY( (v <=> v) == 0 );
+}
+
+template<typename T>
+constexpr void
+test_not_equal()
+{
+  std::inplace_vector<T, 10> v3l{T{1}, T{2}, T{3}};
+  std::inplace_vector<T, 10> v3g{T{1}, T{3}, T{3}};
+  VERIFY( !(v3l == v3g) );
+  VERIFY(   v3l != v3g  );
+  VERIFY(   v3l <  v3g  );
+  VERIFY( !(v3l >  v3g) );
+  VERIFY(   v3l <= v3g  );
+  VERIFY( !(v3l >= v3g) );
+  VERIFY( (v3l <=> v3g) < 0 );
+
+  std::inplace_vector<T, 10> v2{T{1}, T{2}};
+  VERIFY( !(v2 == v3l) );
+  VERIFY(   v2 != v3l  );
+  VERIFY(   v2 <  v3l  );
+  VERIFY( !(v2 >  v3l) );
+  VERIFY(   v2 <= v3l  );
+  VERIFY( !(v2 >= v3l) );
+  VERIFY( (v2 <=> v3l) < 0 );
+}
+
+int main()
+{
+  auto test_all = [] {
+    test_equal<0, int>(0);
+    test_equal<4, int>(0);
+    test_equal<4, int>(2);
+    test_equal<4, int>(4);
+    test_not_equal<int>();
+    return true;
+  };
+
+  test_all();
+  static_assert(test_all());;
+}
diff --git a/libstdc++-v3/testsuite/23_containers/inplace_vector/version.cc b/libstdc++-v3/testsuite/23_containers/inplace_vector/version.cc
new file mode 100644
index 0000000..a5bbdb8
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/inplace_vector/version.cc
@@ -0,0 +1,20 @@
+// { dg-do preprocess { target c++26 } }
+// { dg-add-options no_pch }
+
+#include <inplace_vector>
+
+#ifndef __cpp_lib_inplace_vector
+# error "Feature-test macro for inplace_vector missing in <inplace_vector>"
+#elif __cpp_lib_inplace_vector != 202406L
+# error "Feature-test macro for inplace_vector has wrong value in <inplace_vector>"
+#endif
+
+#undef __cpp_lib_inplace_vector
+
+#include <version>
+
+#ifndef __cpp_lib_inplace_vector
+# error "Feature-test macro for inplace_vector missing in <version>"
+#elif __cpp_lib_inplace_vector != 202406L
+# error "Feature-test macro for inplace_vector has wrong value in <version>"
+#endif
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/accessors/default.cc b/libstdc++-v3/testsuite/23_containers/mdspan/accessors/default.cc
deleted file mode 100644
index c036f8a..0000000
--- a/libstdc++-v3/testsuite/23_containers/mdspan/accessors/default.cc
+++ /dev/null
@@ -1,99 +0,0 @@
-// { dg-do run { target c++23 } }
-#include <mdspan>
-
-#include <testsuite_hooks.h>
-
-constexpr size_t dyn = std::dynamic_extent;
-
-template<typename Accessor>
-  constexpr void
-  test_accessor_policy()
-  {
-    static_assert(std::copyable<Accessor>);
-    static_assert(std::is_nothrow_move_constructible_v<Accessor>);
-    static_assert(std::is_nothrow_move_assignable_v<Accessor>);
-    static_assert(std::is_nothrow_swappable_v<Accessor>);
-  }
-
-constexpr bool
-test_access()
-{
-  std::default_accessor<double> accessor;
-  std::array<double, 5> a{10, 11, 12, 13, 14};
-  VERIFY(accessor.access(a.data(), 0) == 10);
-  VERIFY(accessor.access(a.data(), 4) == 14);
-  return true;
-}
-
-constexpr bool
-test_offset()
-{
-  std::default_accessor<double> accessor;
-  std::array<double, 5> a{10, 11, 12, 13, 14};
-  VERIFY(accessor.offset(a.data(), 0) == a.data());
-  VERIFY(accessor.offset(a.data(), 4) == a.data() + 4);
-  return true;
-}
-
-class Base
-{ };
-
-class Derived : public Base
-{ };
-
-constexpr void
-test_ctor()
-{
-  // T -> T
-  static_assert(std::is_nothrow_constructible_v<std::default_accessor<double>,
-						std::default_accessor<double>>);
-  static_assert(std::is_convertible_v<std::default_accessor<double>,
-				      std::default_accessor<double>>);
-
-  // T -> const T
-  static_assert(std::is_convertible_v<std::default_accessor<double>,
-				      std::default_accessor<const double>>);
-  static_assert(std::is_convertible_v<std::default_accessor<Derived>,
-				      std::default_accessor<const Derived>>);
-
-  // const T -> T
-  static_assert(!std::is_constructible_v<std::default_accessor<double>,
-					 std::default_accessor<const double>>);
-  static_assert(!std::is_constructible_v<std::default_accessor<Derived>,
-					 std::default_accessor<const Derived>>);
-
-  // T <-> volatile T
-  static_assert(std::is_convertible_v<std::default_accessor<int>,
-				      std::default_accessor<volatile int>>);
-  static_assert(!std::is_constructible_v<std::default_accessor<int>,
-					 std::default_accessor<volatile int>>);
-
-  // size difference
-  static_assert(!std::is_constructible_v<std::default_accessor<char>,
-					 std::default_accessor<int>>);
-
-  // signedness
-  static_assert(!std::is_constructible_v<std::default_accessor<int>,
-					 std::default_accessor<unsigned int>>);
-  static_assert(!std::is_constructible_v<std::default_accessor<unsigned int>,
-					 std::default_accessor<int>>);
-
-  // Derived <-> Base
-  static_assert(!std::is_constructible_v<std::default_accessor<Base>,
-					 std::default_accessor<Derived>>);
-  static_assert(!std::is_constructible_v<std::default_accessor<Derived>,
-					 std::default_accessor<Base>>);
-
-}
-
-int
-main()
-{
-  test_accessor_policy<std::default_accessor<double>>();
-  test_access();
-  static_assert(test_access());
-  test_offset();
-  static_assert(test_offset());
-  test_ctor();
-  return 0;
-}
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/accessors/generic.cc b/libstdc++-v3/testsuite/23_containers/mdspan/accessors/generic.cc
new file mode 100644
index 0000000..66009ad
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/accessors/generic.cc
@@ -0,0 +1,140 @@
+// { dg-do run { target c++23 } }
+#include <mdspan>
+
+#include <testsuite_hooks.h>
+
+template<typename Accessor>
+  constexpr bool
+  test_class_properties()
+  {
+    static_assert(std::is_trivially_copyable_v<Accessor>);
+    static_assert(std::semiregular<Accessor>);
+    return true;
+  }
+
+template<typename Accessor>
+  constexpr bool
+  test_accessor_policy()
+  {
+    static_assert(std::copyable<Accessor>);
+    static_assert(std::is_nothrow_move_constructible_v<Accessor>);
+    static_assert(std::is_nothrow_move_assignable_v<Accessor>);
+    static_assert(std::is_nothrow_swappable_v<Accessor>);
+    return true;
+  }
+
+class Base
+{ };
+
+class Derived : public Base
+{ };
+
+template<typename RhsAccessor, typename LhsAccessor, bool ExpectConvertible>
+  constexpr void
+  check_convertible()
+  {
+    RhsAccessor rhs;
+    [[maybe_unused]] LhsAccessor lhs(rhs);
+    static_assert(std::is_nothrow_constructible_v<LhsAccessor, RhsAccessor>);
+    static_assert(std::is_convertible_v<RhsAccessor, LhsAccessor> == ExpectConvertible);
+  }
+
+template<template<typename T> typename LhsAccessor,
+	 template<typename T> typename RhsAccessor = LhsAccessor,
+	 bool ExpectConvertible = true>
+  constexpr bool
+  test_ctor()
+  {
+    // T -> T
+    check_convertible<RhsAccessor<double>, LhsAccessor<double>,
+		      ExpectConvertible>();
+
+    // T -> const T
+    check_convertible<RhsAccessor<double>, LhsAccessor<const double>,
+		      ExpectConvertible>();
+    check_convertible<RhsAccessor<Derived>, LhsAccessor<const Derived>,
+		      ExpectConvertible>();
+
+    // const T -> T
+    static_assert(!std::is_constructible_v<LhsAccessor<double>,
+					   RhsAccessor<const double>>);
+    static_assert(!std::is_constructible_v<LhsAccessor<Derived>,
+					   RhsAccessor<const Derived>>);
+
+    // T <-> volatile T
+    check_convertible<RhsAccessor<int>, LhsAccessor<volatile int>,
+		      ExpectConvertible>();
+    static_assert(!std::is_constructible_v<LhsAccessor<int>,
+					   RhsAccessor<volatile int>>);
+
+    // size difference
+    static_assert(!std::is_constructible_v<LhsAccessor<char>,
+					   RhsAccessor<int>>);
+
+    // signedness
+    static_assert(!std::is_constructible_v<LhsAccessor<int>,
+					   RhsAccessor<unsigned int>>);
+    static_assert(!std::is_constructible_v<LhsAccessor<unsigned int>,
+					   RhsAccessor<int>>);
+
+    // Derived <-> Base
+    static_assert(!std::is_constructible_v<LhsAccessor<Base>,
+					   RhsAccessor<Derived>>);
+    static_assert(!std::is_constructible_v<LhsAccessor<Derived>,
+					   RhsAccessor<Base>>);
+    return true;
+  }
+
+template<template<typename T> typename Accessor>
+  constexpr bool
+  test_properties()
+  {
+    test_class_properties<Accessor<double>>();
+    test_accessor_policy<Accessor<double>>();
+    test_ctor<Accessor>();
+    return true;
+  }
+
+static_assert(test_properties<std::default_accessor>());
+
+template<typename A>
+  constexpr size_t
+  accessor_alignment = alignof(typename A::element_type);
+
+template<typename Accessor>
+  constexpr void
+  test_access(Accessor accessor)
+  {
+    constexpr size_t N = accessor_alignment<Accessor>;
+    alignas(N) std::array<double, 5> a{10, 11, 12, 13, 14};
+    for (size_t i = 0; i < a.size(); ++i)
+      VERIFY(accessor.access(a.data(), i) == 10 + i);
+  }
+
+template<typename Accessor>
+  constexpr void
+  test_offset(Accessor accessor)
+  {
+    constexpr size_t N = accessor_alignment<Accessor>;
+    alignas(N) std::array<double, 5> a{10, 11, 12, 13, 14};
+    for (size_t i = 0; i < a.size(); ++i)
+      VERIFY(accessor.offset(a.data(), i) == a.data() + i);
+  }
+
+template<typename Accessor>
+  constexpr bool
+  test_all()
+  {
+    auto accessor = Accessor{};
+    test_offset(accessor);
+    test_access(accessor);
+    return true;
+  }
+
+int
+main()
+{
+  test_all<std::default_accessor<double>>();
+  static_assert(test_all<std::default_accessor<double>>());
+  return 0;
+}
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/class_mandate_neg.cc b/libstdc++-v3/testsuite/23_containers/mdspan/class_mandate_neg.cc
new file mode 100644
index 0000000..de19b6d
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/class_mandate_neg.cc
@@ -0,0 +1,41 @@
+// { dg-do compile { target c++23 } }
+#include<mdspan>
+
+#include <cstdint>
+#include "layout_like.h"
+
+struct ExtentsLike
+{
+  using index_type = int;
+  using size_type = unsigned int;
+  using rank_type = size_t;
+
+  static constexpr size_t rank() { return 1; }
+  static constexpr size_t rank_dynamic() { return 0; }
+};
+
+constexpr bool
+test_custom_extents_type()
+{
+  std::mdspan<double, ExtentsLike> md1; // { dg-error "required from here" }
+  return true;
+}
+static_assert(test_custom_extents_type());
+
+constexpr bool
+test_element_type_mismatch()
+{
+  using E = std::extents<int, 1>;
+  using L = std::layout_right;
+  using A = std::default_accessor<double>;
+
+  [[maybe_unused]] std::mdspan<float, E, L, A> md2; // { dg-error "required from here" }
+  return true;
+};
+static_assert(test_element_type_mismatch());
+
+// { dg-prune-output "Extents must be a specialization of std::extents" }
+// { dg-prune-output "no type named '_S_storage'" }
+// { dg-prune-output "non-constant condition" }
+// { dg-prune-output "static assertion failed" }
+// { dg-prune-output "__glibcxx_assert" }
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/extents/class_mandates_neg.cc b/libstdc++-v3/testsuite/23_containers/mdspan/extents/class_mandates_neg.cc
index f9c1c01..67d18fe 100644
--- a/libstdc++-v3/testsuite/23_containers/mdspan/extents/class_mandates_neg.cc
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/extents/class_mandates_neg.cc
@@ -7,6 +7,8 @@ std::extents<uint8_t, size_t(1) << 9> e1; // { dg-error "from here" }
 std::extents<char, 1> e2;                 // { dg-error "from here" }
 std::extents<bool, 1> e3;                 // { dg-error "from here" }
 std::extents<double, 1> e4;               // { dg-error "from here" }
+
 // { dg-prune-output "dynamic or representable as IndexType" }
 // { dg-prune-output "signed or unsigned integer" }
 // { dg-prune-output "invalid use of incomplete type" }
+// { dg-prune-output "non-constant condition for static assertion" }
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/extents/ctor_ints.cc b/libstdc++-v3/testsuite/23_containers/mdspan/extents/ctor_ints.cc
index 3a70efd..d5f07c1 100644
--- a/libstdc++-v3/testsuite/23_containers/mdspan/extents/ctor_ints.cc
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/extents/ctor_ints.cc
@@ -35,20 +35,20 @@ test_all()
   auto expected = std::extents<int, 1, 2, 3>(1, 2, 3);
 
   // From all extents.
-  VERIFY((std::extents<int, 1, 2, 3>(1, 2, 3)) == expected);
-  VERIFY((std::extents<int, dyn, 2, 3>(1, 2, 3)) == expected);
-  VERIFY((std::extents<int, dyn, 2, dyn>(1, 2, 3)) == expected);
+  VERIFY(std::extents<int, 1, 2, 3>(1, 2, 3) == expected);
+  VERIFY(std::extents<int, dyn, 2, 3>(1, 2, 3) == expected);
+  VERIFY(std::extents<int, dyn, 2, dyn>(1, 2, 3) == expected);
 
-  VERIFY((std::extents<int, 1, 2, 3>{1, 2, 3}) == expected);
-  VERIFY((std::extents<int, dyn, 2, 3>{1, 2, 3}) == expected);
-  VERIFY((std::extents<int, dyn, 2, dyn>{1, 2, 3}) == expected);
+  VERIFY(std::extents<int, 1, 2, 3>{1, 2, 3} == expected);
+  VERIFY(std::extents<int, dyn, 2, 3>{1, 2, 3} == expected);
+  VERIFY(std::extents<int, dyn, 2, dyn>{1, 2, 3} == expected);
 
   // From only dynamic extents.
-  VERIFY((std::extents<int, dyn, 2, 3>(1)) == expected);
-  VERIFY((std::extents<int, dyn, 2, dyn>(1, 3)) == expected);
+  VERIFY(std::extents<int, dyn, 2, 3>(1) == expected);
+  VERIFY(std::extents<int, dyn, 2, dyn>(1, 3) == expected);
 
-  VERIFY((std::extents<int, dyn, 2, 3>{1}) == expected);
-  VERIFY((std::extents<int, dyn, 2, dyn>{1, 3}) == expected);
+  VERIFY(std::extents<int, dyn, 2, 3>{1} == expected);
+  VERIFY(std::extents<int, dyn, 2, dyn>{1, 3} == expected);
 
   return true;
 }
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/extents/ctor_shape.cc b/libstdc++-v3/testsuite/23_containers/mdspan/extents/ctor_shape.cc
index 01624f2..1682cc5 100644
--- a/libstdc++-v3/testsuite/23_containers/mdspan/extents/ctor_shape.cc
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/extents/ctor_shape.cc
@@ -66,9 +66,9 @@ namespace all_extents
     test_ctor(Shape shape)
     {
       auto expected = std::extents<int, 1, 2, 3>();
-      VERIFY((std::extents<int, 1, dyn, 3>(shape)) == expected);
-      VERIFY((std::extents<int, dyn, dyn, dyn>(shape)) == expected);
-      VERIFY((std::extents<int, 1, 2, 3>(shape)) == expected);
+      VERIFY(std::extents<int, 1, dyn, 3>(shape) == expected);
+      VERIFY(std::extents<int, dyn, dyn, dyn>(shape) == expected);
+      VERIFY(std::extents<int, 1, 2, 3>(shape) == expected);
     }
 
   constexpr void
@@ -90,8 +90,8 @@ namespace all_extents
     auto span = std::span<int, 0>(shape);
 
     auto expected = std::extents<int>();
-    VERIFY((std::extents<int>(shape)) == expected);
-    VERIFY((std::extents<int>(span)) == expected);
+    VERIFY(std::extents<int>(shape) == expected);
+    VERIFY(std::extents<int>(span) == expected);
   }
 
   constexpr bool
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/extents/custom_integer.cc b/libstdc++-v3/testsuite/23_containers/mdspan/extents/custom_integer.cc
index 2907ad1..99de401 100644
--- a/libstdc++-v3/testsuite/23_containers/mdspan/extents/custom_integer.cc
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/extents/custom_integer.cc
@@ -2,86 +2,95 @@
 #include <mdspan>
 
 #include <testsuite_hooks.h>
+#include "../int_like.h"
 
 // Test construction from a custom integer-like object, that has
 // no copy/move ctor or copy/move assignment operator.
 
 constexpr size_t dyn = std::dynamic_extent;
 
-class IntLike
-{
-public:
-  explicit
-  IntLike(int i)
-  : _M_i(i)
-  { }
-
-  IntLike() = delete;
-  IntLike(const IntLike&) = delete;
-  IntLike(IntLike&&) = delete;
-
-  const IntLike&
-  operator=(const IntLike&) = delete;
-
-  const IntLike&
-  operator=(IntLike&&) = delete;
-
-  constexpr
-  operator int() const noexcept
-  { return _M_i; }
-
-private:
-  int _M_i;
-};
-
 static_assert(std::is_convertible_v<IntLike, int>);
 static_assert(std::is_nothrow_constructible_v<int, IntLike>);
 
-void
-test_shape(const auto& s2, const auto& s23)
-{
-  std::extents<int, 2, 3> expected;
-
-  std::extents<int, 2, 3> e1(s23);
-  VERIFY(e1 == expected);
-
-  std::extents<int, dyn, 3> e2(s2);
-  VERIFY(e2 == expected);
-
-  std::extents<int, dyn, 3> e3(s23);
-  VERIFY(e3 == expected);
-
-  std::extents<int, dyn, dyn> e4(s23);
-  VERIFY(e4 == expected);
-}
-
-void
-test_pack()
-{
-  std::extents<int, 2, 3> expected;
-
-  std::extents<int, dyn, 3> e1(IntLike(2));
-  VERIFY(e1 == expected);
-
-  std::extents<int, dyn, 3> e2(IntLike(2), IntLike(3));
-  VERIFY(e2 == expected);
-
-  std::extents<int, dyn, dyn> e3(IntLike(2), IntLike(3));
-  VERIFY(e3 == expected);
-}
+template<typename Extents, bool Valid>
+  void
+  test_shape(const auto& shape)
+  {
+    static_assert(std::is_constructible_v<Extents, decltype(shape)> == Valid);
+
+    if constexpr (Valid)
+      {
+	std::extents<int, 2, 3> expected;
+	Extents actual(shape);
+	VERIFY(actual == expected);
+      }
+  }
+
+template<typename Int, bool Valid>
+  void
+  test_shape_all()
+  {
+    auto a2 = std::array<Int, 1>{Int(2)};
+    auto s2 = std::span<Int, 1>(a2);
+
+    auto a23 = std::array<Int, 2>{Int(2), Int(3)};
+    auto s23 = std::span<Int, 2>(a23);
+
+    auto check = [](const auto& dyn_exts, const auto& full_exts)
+      {
+	test_shape<std::extents<int, 2, 3>, Valid>(full_exts);
+	test_shape<std::extents<int, dyn, 3>, Valid>(dyn_exts);
+	test_shape<std::extents<int, dyn, 3>, Valid>(full_exts);
+	test_shape<std::extents<int, dyn, dyn>, Valid>(full_exts);
+      };
+
+    check(a2, a23);
+    check(s2, s23);
+  }
+
+// Needed because is_constructible requires that Ints are move constructible.
+template<typename Extents, typename... Ints>
+  concept has_ctor = requires
+  {
+    { Extents(Ints(0)...) } -> std::same_as<Extents>;
+  };
+
+template<typename Int, bool Valid>
+  void
+  test_pack_all()
+  {
+    static_assert(has_ctor<std::extents<int, dyn, 3>, Int> == Valid);
+    static_assert(has_ctor<std::extents<int, dyn, 3>, Int, Int> == Valid);
+    static_assert(has_ctor<std::extents<int, dyn, dyn>, Int, Int> == Valid);
+
+    if constexpr (Valid)
+      {
+	std::extents<int, 2, 3> expected;
+
+	std::extents<int, dyn, 3> e1(Int(2));
+	VERIFY(e1 == expected);
+
+	std::extents<int, dyn, 3> e2(Int(2), Int(3));
+	VERIFY(e2 == expected);
+
+	std::extents<int, dyn, dyn> e3(Int(2), Int(3));
+	VERIFY(e3 == expected);
+      }
+  }
 
 int
 main()
 {
-  auto a2 = std::array<IntLike, 1>{IntLike(2)};
-  auto s2 = std::span<IntLike, 1>(a2);
-
-  auto a23 = std::array<IntLike, 2>{IntLike(2), IntLike(3)};
-  auto s23 = std::span<IntLike, 2>(a23);
-
-  test_shape(a2, a23);
-  test_shape(s2, s23);
-  test_pack();
-
+  test_shape_all<int, true>();
+  test_shape_all<IntLike, true>();
+  test_shape_all<ThrowingInt, false>();
+  test_shape_all<MutatingInt, false>();
+  test_shape_all<RValueInt, false>();
+
+  test_pack_all<int, true>();
+  test_pack_all<IntLike, true>();
+  test_pack_all<ThrowingInt, false>();
+  test_pack_all<MutatingInt, true>();
+  test_pack_all<RValueInt, true>();
   return 0;
 }
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/extents/extents_mismatch_neg.cc b/libstdc++-v3/testsuite/23_containers/mdspan/extents/extents_mismatch_neg.cc
new file mode 100644
index 0000000..b35e531
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/extents/extents_mismatch_neg.cc
@@ -0,0 +1,35 @@
+// { dg-do compile { target c++23 } }
+#include<mdspan>
+
+#include <cstdint>
+
+constexpr size_t dyn = std::dynamic_extent;
+
+constexpr bool
+test_dyn2sta_extents_mismatch_00()
+{
+  auto e0 = std::extents<int, dyn>{1};
+  [[maybe_unused]] auto e1 = std::extents<int, 2>{e0};        // { dg-error "expansion of" }
+  return true;
+}
+static_assert(test_dyn2sta_extents_mismatch_00());            // { dg-error "expansion of" }
+
+constexpr bool
+test_dyn2sta_extents_mismatch_01()
+{
+  [[maybe_unused]] auto e = std::extents<int, 1, dyn>{2, 2}; // { dg-error "expansion of" }
+  return true;
+}
+static_assert(test_dyn2sta_extents_mismatch_01());           // { dg-error "expansion of" }
+
+constexpr bool
+test_dyn2sta_extents_mismatch_02()
+{
+  std::array<int, 2> exts{2, 2};
+  [[maybe_unused]] auto e = std::extents<int, 1, dyn>{exts}; // { dg-error "expansion of" }
+  return true;
+}
+static_assert(test_dyn2sta_extents_mismatch_02());           // { dg-error "expansion of" }
+
+// { dg-prune-output "non-constant condition for static assertion" }
+// { dg-prune-output "__glibcxx_assert" }
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/extents/misc.cc b/libstdc++-v3/testsuite/23_containers/mdspan/extents/misc.cc
index e71fdc5..bca8901 100644
--- a/libstdc++-v3/testsuite/23_containers/mdspan/extents/misc.cc
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/extents/misc.cc
@@ -98,12 +98,32 @@ test_deduction(Extents... exts)
 }
 
 constexpr bool
+test_integral_constant_deduction()
+{
+  auto verify = [](auto actual, auto expected)
+    {
+      static_assert(std::same_as<decltype(actual), decltype(expected)>);
+      VERIFY(actual == expected);
+    };
+
+  constexpr auto c1 = std::integral_constant<size_t, 1>{};
+  constexpr auto c2 = std::integral_constant<int, 2>{};
+
+  verify(std::extents(1), std::extents<size_t, dyn>{1});
+  verify(std::extents(c1), std::extents<size_t, 1>{});
+  verify(std::extents(c2), std::extents<size_t, 2>{});
+  verify(std::extents(c1, 2), std::extents<size_t, 1, dyn>{2});
+  return true;
+}
+
+constexpr bool
 test_deduction_all()
 {
   test_deduction<0>();
   test_deduction<1>(1);
   test_deduction<2>(1.0, 2.0f);
   test_deduction<3>(int(1), short(2), size_t(3));
+  test_integral_constant_deduction();
   return true;
 }
 
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/int_like.h b/libstdc++-v3/testsuite/23_containers/mdspan/int_like.h
new file mode 100644
index 0000000..310dd8d
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/int_like.h
@@ -0,0 +1,63 @@
+#ifndef TEST_MDSPAN_INT_LIKE_H
+#define TEST_MDSPAN_INT_LIKE_H
+
+enum class CustomIndexKind
+{
+  Const,
+  Throwing,
+  Mutating,
+  RValue,
+};
+
+template<CustomIndexKind Kind>
+  class CustomIndexType
+  {
+  public:
+    explicit
+    CustomIndexType(int i)
+    : _M_i(i)
+    { }
+
+    CustomIndexType() = delete;
+    CustomIndexType(const CustomIndexType&) = delete;
+    CustomIndexType(CustomIndexType&&) = delete;
+
+    const CustomIndexType&
+    operator=(const CustomIndexType&) = delete;
+
+    const CustomIndexType&
+    operator=(CustomIndexType&&) = delete;
+
+    constexpr
+    operator int() const noexcept
+    requires (Kind == CustomIndexKind::Const)
+    { return _M_i; }
+
+    constexpr
+    operator int() const
+    requires (Kind == CustomIndexKind::Throwing)
+    { return _M_i; }
+
+    constexpr
+    operator int() noexcept
+    requires (Kind == CustomIndexKind::Mutating)
+    { return _M_i; }
+
+    constexpr
+    operator int() && noexcept
+    requires (Kind == CustomIndexKind::RValue)
+    { return _M_i; }
+
+  private:
+    int _M_i;
+  };
+
+using IntLike = CustomIndexType<CustomIndexKind::Const>;
+using ThrowingInt = CustomIndexType<CustomIndexKind::Throwing>;
+using MutatingInt = CustomIndexType<CustomIndexKind::Mutating>;
+using RValueInt = CustomIndexType<CustomIndexKind::RValue>;
+
+struct NotIntLike
+{ };
+
+#endif // TEST_MDSPAN_INT_LIKE_H
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/layout_like.h b/libstdc++-v3/testsuite/23_containers/mdspan/layout_like.h
new file mode 100644
index 0000000..ded6e9d
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/layout_like.h
@@ -0,0 +1,87 @@
+#ifndef TEST_MDSPAN_LAYOUT_LIKE_H
+#define TEST_MDSPAN_LAYOUT_LIKE_H 1
+
+template<bool Noexcept>
+  struct CustomLayout
+  {
+    template<typename Extents>
+      class mapping
+      {
+      public:
+	using extents_type = Extents;
+	using index_type = typename extents_type::index_type;
+	using size_type = typename extents_type::size_type;
+	using rank_type = typename extents_type::rank_type;
+	using layout_type = CustomLayout;
+
+	constexpr
+	mapping() noexcept = default;
+
+	constexpr
+	mapping(Extents exts)
+	: m_exts(exts)
+	{ }
+
+	constexpr const extents_type&
+	extents() const noexcept(Noexcept) { return m_exts; }
+
+	constexpr index_type
+	required_span_size() const noexcept(Noexcept)
+	{
+	  for (size_t i = 0; i < extents_type::rank(); ++i)
+	    if (m_exts.extent(i) == 0)
+	      return 0;
+	  return 1;
+	}
+
+	template<typename... Indices>
+	  requires (sizeof...(Indices) == extents_type::rank())
+	  constexpr index_type
+	  operator()(Indices...) const noexcept(Noexcept)
+	  { return 0; }
+
+	static constexpr index_type
+	stride(rank_type) noexcept(Noexcept)
+	{ return 0; }
+
+	static constexpr bool
+	is_always_unique() noexcept(Noexcept)
+	{ return false; }
+
+	static constexpr bool
+	is_always_exhaustive() noexcept(Noexcept)
+	{ return true; }
+
+	static constexpr bool
+	is_always_strided() noexcept(Noexcept)
+	{ return true; }
+
+	constexpr bool
+	is_unique() const noexcept(Noexcept)
+	{
+	  if (required_span_size() == 0)
+	    return true;
+
+	  for (size_t i = 0; i < extents_type::rank(); ++i)
+	    if (m_exts.extent(i) > 1)
+	      return false;
+	  return true;
+	}
+
+	static constexpr bool
+	is_exhaustive() noexcept(Noexcept)
+	{ return true; }
+
+	static constexpr bool
+	is_strided() noexcept(Noexcept)
+	{ return true; }
+
+      private:
+	Extents m_exts;
+      };
+  };
+
+using LayoutLike = CustomLayout<true>;
+using ThrowingLayout = CustomLayout<false>;
+
+#endif
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/layouts/debug/out_of_bounds_neg.cc b/libstdc++-v3/testsuite/23_containers/mdspan/layouts/debug/out_of_bounds_neg.cc
new file mode 100644
index 0000000..fb8ff01
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/layouts/debug/out_of_bounds_neg.cc
@@ -0,0 +1,30 @@
+// { dg-do compile { target c++23 } }
+// { dg-require-debug-mode "" }
+#include<mdspan>
+
+template<typename Layout>
+  constexpr bool
+  test_out_of_bounds_1d()
+  {
+    auto m = typename Layout::mapping<std::extents<int, 0>>{};
+    (void) m(0); // { dg-error "expansion of" }
+    return true;
+  }
+static_assert(test_out_of_bounds_1d<std::layout_left>()); // { dg-error "expansion of" }
+static_assert(test_out_of_bounds_1d<std::layout_right>()); // { dg-error "expansion of" }
+static_assert(test_out_of_bounds_1d<std::layout_stride>()); // { dg-error "expansion of" }
+
+template<typename Layout>
+  constexpr bool
+  test_out_of_bounds_3d()
+  {
+    auto m = typename Layout::mapping<std::extents<int, 3, 5, 7>>{};
+    (void) m(2, 5, 5); // { dg-error "expansion of" }
+    return true;
+  }
+static_assert(test_out_of_bounds_3d<std::layout_left>()); // { dg-error "expansion of" }
+static_assert(test_out_of_bounds_3d<std::layout_right>()); // { dg-error "expansion of" }
+static_assert(test_out_of_bounds_3d<std::layout_stride>()); // { dg-error "expansion of" }
+
+// { dg-prune-output "non-constant condition for static assertion" }
+// { dg-prune-output "__glibcxx_assert" }
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/layouts/mapping.cc b/libstdc++-v3/testsuite/23_containers/mdspan/layouts/mapping.cc
index 963c804..58bce51 100644
--- a/libstdc++-v3/testsuite/23_containers/mdspan/layouts/mapping.cc
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/layouts/mapping.cc
@@ -1,6 +1,7 @@
 // { dg-do run { target c++23 } }
 #include <mdspan>
 
+#include "../int_like.h"
 #include <cstdint>
 #include <testsuite_hooks.h>
 
@@ -59,14 +60,13 @@ template<typename Mapping, size_t N>
     return ret;
   }
 
-template<typename Mapping, typename... Indices>
+template<typename Int, typename Mapping, typename... Indices>
   constexpr void
   test_linear_index(const Mapping& m, Indices... i)
   {
     using index_type = typename Mapping::index_type;
     index_type expected = linear_index(m, std::array{index_type(i)...});
-    VERIFY(m(i...) == expected);
-    VERIFY(m(uint8_t(i)...) == expected);
+    VERIFY(m(Int(i)...) == expected);
   }
 
 template<typename Layout>
@@ -77,26 +77,26 @@ template<typename Layout>
     VERIFY(m() == 0);
   }
 
-template<typename Layout>
+template<typename Layout, typename Int>
   constexpr void
   test_linear_index_1d()
   {
     typename Layout::mapping<std::extents<int, 5>> m;
-    test_linear_index(m, 0);
-    test_linear_index(m, 1);
-    test_linear_index(m, 4);
+    test_linear_index<Int>(m, 0);
+    test_linear_index<Int>(m, 1);
+    test_linear_index<Int>(m, 4);
   }
 
-template<typename Layout>
+template<typename Layout, typename Int>
   constexpr void
   test_linear_index_2d()
   {
     typename Layout::mapping<std::extents<int, 3, 256>> m;
-    test_linear_index(m, 0, 0);
-    test_linear_index(m, 1, 0);
-    test_linear_index(m, 0, 1);
-    test_linear_index(m, 1, 1);
-    test_linear_index(m, 2, 4);
+    test_linear_index<Int>(m, 0, 0);
+    test_linear_index<Int>(m, 1, 0);
+    test_linear_index<Int>(m, 0, 1);
+    test_linear_index<Int>(m, 1, 1);
+    test_linear_index<Int>(m, 2, 4);
   }
 
 template<typename Layout>
@@ -141,44 +141,34 @@ template<>
       }
   };
 
-template<typename Layout>
+template<typename Layout, typename Int>
   constexpr void
   test_linear_index_3d()
   {
     auto m = MappingFactory<Layout>::create(std::extents(3, 5, 7));
-    test_linear_index(m, 0, 0, 0);
-    test_linear_index(m, 1, 0, 0);
-    test_linear_index(m, 0, 1, 0);
-    test_linear_index(m, 0, 0, 1);
-    test_linear_index(m, 1, 1, 0);
-    test_linear_index(m, 2, 4, 6);
+    test_linear_index<Int>(m, 0, 0, 0);
+    test_linear_index<Int>(m, 1, 0, 0);
+    test_linear_index<Int>(m, 0, 1, 0);
+    test_linear_index<Int>(m, 0, 0, 1);
+    test_linear_index<Int>(m, 1, 1, 0);
+    test_linear_index<Int>(m, 2, 4, 6);
   }
 
-struct IntLikeA
-{
-  operator int()
-  { return 0; }
-};
-
-struct IntLikeB
-{
-  operator int() noexcept
-  { return 0; }
-};
-
-struct NotIntLike
-{ };
+template<typename Mapping, typename... Ints>
+  concept has_linear_index = requires (Mapping m)
+  {
+    { m(Ints(0)...) } -> std::same_as<typename Mapping::index_type>;
+  };
 
 template<typename Layout>
   constexpr void
   test_has_linear_index_0d()
   {
     using Mapping = typename Layout::mapping<std::extents<int>>;
-    static_assert(std::invocable<Mapping>);
-    static_assert(!std::invocable<Mapping, int>);
-    static_assert(!std::invocable<Mapping, IntLikeA>);
-    static_assert(!std::invocable<Mapping, IntLikeB>);
-    static_assert(!std::invocable<Mapping, NotIntLike>);
+    static_assert(has_linear_index<Mapping>);
+    static_assert(!has_linear_index<Mapping, int>);
+    static_assert(!has_linear_index<Mapping, IntLike>);
+    static_assert(!has_linear_index<Mapping, NotIntLike>);
   }
 
 template<typename Layout>
@@ -186,12 +176,14 @@ template<typename Layout>
   test_has_linear_index_1d()
   {
     using Mapping = typename Layout::mapping<std::extents<int, 3>>;
-    static_assert(std::invocable<Mapping, int>);
-    static_assert(!std::invocable<Mapping>);
-    static_assert(!std::invocable<Mapping, IntLikeA>);
-    static_assert(std::invocable<Mapping, IntLikeB>);
-    static_assert(!std::invocable<Mapping, NotIntLike>);
-    static_assert(std::invocable<Mapping, double>);
+    static_assert(!has_linear_index<Mapping>);
+    static_assert(has_linear_index<Mapping, int>);
+    static_assert(has_linear_index<Mapping, double>);
+    static_assert(has_linear_index<Mapping, IntLike>);
+    static_assert(has_linear_index<Mapping, MutatingInt>);
+    static_assert(!has_linear_index<Mapping, ThrowingInt>);
+    static_assert(!has_linear_index<Mapping, NotIntLike>);
+    static_assert(!has_linear_index<Mapping, int, int>);
   }
 
 template<typename Layout>
@@ -199,22 +191,24 @@ template<typename Layout>
   test_has_linear_index_2d()
   {
     using Mapping = typename Layout::mapping<std::extents<int, 3, 5>>;
-    static_assert(std::invocable<Mapping, int, int>);
-    static_assert(!std::invocable<Mapping, int>);
-    static_assert(!std::invocable<Mapping, IntLikeA, int>);
-    static_assert(std::invocable<Mapping, IntLikeB, int>);
-    static_assert(!std::invocable<Mapping, NotIntLike, int>);
-    static_assert(std::invocable<Mapping, double, double>);
+    static_assert(!has_linear_index<Mapping, int>);
+    static_assert(has_linear_index<Mapping, int, int>);
+    static_assert(has_linear_index<Mapping, double, double>);
+    static_assert(has_linear_index<Mapping, IntLike, int>);
+    static_assert(has_linear_index<Mapping, MutatingInt, int>);
+    static_assert(!has_linear_index<Mapping, ThrowingInt, int>);
+    static_assert(!has_linear_index<Mapping, NotIntLike, int>);
+    static_assert(!has_linear_index<Mapping, int, int, int>);
   }
 
-template<typename Layout>
+template<typename Layout, typename Int>
   constexpr bool
   test_linear_index_all()
   {
     test_linear_index_0d<Layout>();
-    test_linear_index_1d<Layout>();
-    test_linear_index_2d<Layout>();
-    test_linear_index_3d<Layout>();
+    test_linear_index_1d<Layout, Int>();
+    test_linear_index_2d<Layout, Int>();
+    test_linear_index_3d<Layout, Int>();
     test_has_linear_index_0d<Layout>();
     test_has_linear_index_1d<Layout>();
     test_has_linear_index_2d<Layout>();
@@ -527,7 +521,15 @@ template<typename Layout>
   constexpr bool
   test_mapping_all()
   {
-    test_linear_index_all<Layout>();
+    test_linear_index_all<Layout, uint8_t>();
+    test_linear_index_all<Layout, int>();
+    if !consteval
+      {
+	test_linear_index_all<Layout, IntLike>();
+	test_linear_index_all<Layout, MutatingInt>();
+	test_linear_index_all<Layout, RValueInt>();
+      }
+
     test_required_span_size_all<Layout>();
     test_stride_all<Layout>();
 
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/layouts/stride.cc b/libstdc++-v3/testsuite/23_containers/mdspan/layouts/stride.cc
index c8af5c6..8d2fad2 100644
--- a/libstdc++-v3/testsuite/23_containers/mdspan/layouts/stride.cc
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/layouts/stride.cc
@@ -1,6 +1,7 @@
 // { dg-do run { target c++23 } }
 #include <mdspan>
 
+#include "../int_like.h"
 #include <testsuite_hooks.h>
 
 constexpr size_t dyn = std::dynamic_extent;
@@ -42,21 +43,6 @@ test_ctor_default_stride_all()
   return true;
 }
 
-struct IntLikeA
-{
-  operator int()
-  { return 0; }
-};
-
-struct IntLikeB
-{
-  operator int() noexcept
-  { return 0; }
-};
-
-struct NotIntLike
-{ };
-
 template<typename E, typename E_arg, typename T, size_t N, bool Expected>
 constexpr void
 test_stride_constructible()
@@ -77,8 +63,9 @@ test_stride_constructible_all()
   using E2 = std::extents<int, dyn>;
 
   test_stride_constructible<E0, E0, int, 0, true>();
-  test_stride_constructible<E0, E0, IntLikeA, 0, false>();
-  test_stride_constructible<E0, E0, IntLikeB, 0, true>();
+  test_stride_constructible<E0, E0, IntLike, 0, true>();
+  test_stride_constructible<E0, E0, ThrowingInt, 0, false>();
+  test_stride_constructible<E0, E0, MutatingInt, 0, false>();
   test_stride_constructible<E0, E0, NotIntLike, 0, false>();
   test_stride_constructible<E1, E1, int, 1, true>();
   test_stride_constructible<E2, E1, int, 1, true>();
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/mdspan.cc b/libstdc++-v3/testsuite/23_containers/mdspan/mdspan.cc
new file mode 100644
index 0000000..bdfc6eb
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/mdspan.cc
@@ -0,0 +1,771 @@
+// { dg-do run { target c++23 } }
+#include <mdspan>
+
+#include <testsuite_hooks.h>
+#include "int_like.h"
+#include "layout_like.h"
+#include <stdexcept>
+
+constexpr auto dyn = std::dynamic_extent;
+
+template<typename MDSpan, typename T, typename E, typename L = std::layout_right,
+	 typename A = std::default_accessor<T>>
+  constexpr void
+  assert_typedefs()
+  {
+    static_assert(std::same_as<typename MDSpan::extents_type, E>);
+    static_assert(std::same_as<typename MDSpan::layout_type, L>);
+    static_assert(std::same_as<typename MDSpan::accessor_type, A>);
+    static_assert(std::same_as<typename MDSpan::mapping_type,
+			       typename L::mapping<E>>);
+    static_assert(std::same_as<typename MDSpan::element_type, T>);
+    static_assert(std::same_as<typename MDSpan::value_type,
+			       std::remove_const_t<T>>);
+    static_assert(std::same_as<typename MDSpan::index_type,
+			       typename E::index_type>);
+    static_assert(std::same_as<typename MDSpan::size_type,
+			       typename E::size_type>);
+    static_assert(std::same_as<typename MDSpan::rank_type,
+			       typename E::rank_type>);
+    static_assert(std::same_as<typename MDSpan::data_handle_type,
+			       typename A::data_handle_type>);
+    static_assert(std::same_as<typename MDSpan::reference,
+			       typename A::reference>);
+  }
+
+template<typename T, typename E, typename L, template<typename U> typename A>
+  constexpr void
+  test_typedefs()
+  { assert_typedefs<std::mdspan<T, E, L, A<T>>, T, E, L, A<T>>(); }
+
+constexpr void
+test_typedefs_all()
+{
+  using E = std::extents<int, 1, 2>;
+  using L = std::layout_left;
+
+  test_typedefs<double, E, L, std::default_accessor>();
+  test_typedefs<const double, E, L, std::default_accessor>();
+}
+
+template<typename MDSpan>
+  constexpr void
+  test_rank()
+  {
+    using Extents = typename MDSpan::extents_type;
+    static_assert(MDSpan::rank() == Extents::rank());
+    static_assert(MDSpan::rank_dynamic() == Extents::rank_dynamic());
+  }
+
+constexpr bool
+test_rank_all()
+{
+  test_rank<std::mdspan<double, std::extents<int>>>();
+  test_rank<std::mdspan<double, std::extents<int, 1>>>();
+  test_rank<std::mdspan<double, std::extents<int, dyn>>>();
+  return true;
+}
+
+template<typename Extents>
+  constexpr void
+  test_extent(Extents exts)
+  {
+    double data =  1.0;
+    auto md = std::mdspan(&data, exts);
+    using MDSpan = decltype(md);
+
+    for(size_t i = 0; i < MDSpan::rank(); ++i)
+      {
+	VERIFY(MDSpan::static_extent(i) == Extents::static_extent(i));
+	VERIFY(md.extent(i) == exts.extent(i));
+      }
+  }
+
+constexpr bool
+test_extent_all()
+{
+  // For rank == 0, check existence of the methods without calling them.
+  test_extent(std::extents<int>{});
+  test_extent(std::extents<int, 0>{});
+  test_extent(std::extents<int, dyn>{});
+  return true;
+}
+
+template<typename MDSpan>
+  constexpr void
+  test_class_properties()
+  {
+    static_assert(std::copyable<MDSpan>);
+    static_assert(std::is_nothrow_move_constructible_v<MDSpan>);
+    static_assert(std::is_nothrow_move_assignable_v<MDSpan>);
+    static_assert(std::is_nothrow_swappable_v<MDSpan>);
+    constexpr bool trivially_copyable =
+      std::is_trivially_copyable_v<typename MDSpan::accessor_type>
+      && std::is_trivially_copyable_v<typename MDSpan::mapping_type>
+      && std::is_trivially_copyable_v<typename MDSpan::data_handle_type>;
+    static_assert(std::is_trivially_copyable_v<MDSpan> == trivially_copyable);
+  }
+
+constexpr bool
+test_class_properties_all()
+{
+  test_class_properties<std::mdspan<double, std::extents<int>>>();
+  test_class_properties<std::mdspan<double, std::extents<int, 1>>>();
+  test_class_properties<std::mdspan<double, std::extents<int, dyn>>>();
+  return true;
+}
+
+template<typename T>
+  class ThrowingDefaultAccessor
+  {
+  public:
+    using element_type = T;
+    using reference = element_type&;
+    using data_handle_type = element_type*;
+    using offset_policy = ThrowingDefaultAccessor;
+
+    ThrowingDefaultAccessor() noexcept(false)
+    { }
+
+    reference
+    access(data_handle_type p, size_t i) const
+    { return p[i]; }
+
+    typename offset_policy::data_handle_type
+    offset(data_handle_type p, size_t i) const
+    { return p + i; }
+  };
+
+constexpr bool
+test_default_ctor()
+{
+  static_assert(!std::is_default_constructible_v<std::mdspan<double,
+						 std::extents<int>>>);
+  static_assert(!std::is_default_constructible_v<std::mdspan<double,
+						 std::extents<int, 1>>>);
+  static_assert(std::is_default_constructible_v<std::mdspan<double,
+						std::extents<int, dyn>>>);
+
+  std::mdspan<double, std::extents<int, dyn>> md;
+  VERIFY(md.data_handle() == nullptr);
+  VERIFY(md.empty());
+  return true;
+}
+
+template<template<typename T> typename Accessor, bool Expected>
+  constexpr void
+  test_nothrow_default_ctor()
+  {
+    using Extents = std::extents<int, dyn>;
+    using Layout = std::layout_left;
+    using MDSpan = std::mdspan<double, Extents, Layout, Accessor<double>>;
+
+    static_assert(std::is_default_constructible_v<MDSpan>);
+    static_assert(std::is_nothrow_default_constructible_v<MDSpan> == Expected);
+  }
+
+constexpr bool
+test_from_other()
+{
+  using Extents = std::extents<int, 3, 5, 7>;
+  auto exts = Extents{};
+
+  auto mapping = std::layout_right::mapping(exts);
+  constexpr size_t n = mapping.required_span_size();
+  std::array<double, n> storage{};
+
+  auto md1 = std::mdspan(storage.data(), exts);
+  auto md2 = std::mdspan<double, std::dextents<int, 3>>(md1);
+
+  VERIFY(md1.data_handle() == md2.data_handle());
+  VERIFY(md1.size() == md2.size());
+
+  static_assert(!std::is_convertible_v<
+      std::mdspan<double, std::extents<unsigned int, 2>>,
+      std::mdspan<double, std::extents<int, 2>>>);
+
+  static_assert(std::is_convertible_v<
+      std::mdspan<double, std::extents<int, 2>>,
+      std::mdspan<const double, std::extents<int, 2>>>);
+
+  static_assert(!std::is_constructible_v<
+      std::mdspan<double, std::extents<int, 2>>,
+      std::mdspan<const double, std::extents<int, 2>>>);
+
+  return true;
+}
+
+template<typename T, typename E, typename L = std::layout_right,
+	 typename A = std::default_accessor<T>>
+  constexpr void
+  assert_deduced_typedefs(auto md)
+  { assert_typedefs<decltype(md), T, E, L, A>(); }
+
+constexpr bool
+test_from_carray()
+{
+  constexpr size_t n = 5;
+  double data[n] = {1.1, 2.2, 3.3, 4.4, 5.5};
+
+  auto md = std::mdspan(data);
+  assert_deduced_typedefs<double, std::extents<size_t, n>>(md);
+  VERIFY(md.rank() == 1);
+  VERIFY(md.rank_dynamic() == 0);
+  VERIFY(md[2] == data[2]);
+  return true;
+}
+
+constexpr bool
+test_from_pointer()
+{
+  double value = 12.3;
+  auto md = std::mdspan(&value);
+  assert_deduced_typedefs<double, std::extents<size_t>>(md);
+  VERIFY(md.rank() == 0);
+  VERIFY(md.rank_dynamic() == 0);
+  VERIFY(md[] == value);
+  return true;
+}
+
+constexpr bool
+test_from_pointer_and_shape()
+{
+  constexpr size_t n = 6;
+  std::array<double, n> data{1.1, 2.2, 3.3, 4.4, 5.5, 6.6};
+  std::array<int, 2> shape{2, 3};
+  std::span<const int, 2> shape_view(shape);
+
+  auto verify = [&data](auto md)
+  {
+    assert_deduced_typedefs<double, std::dextents<size_t, 2>>(md);
+    VERIFY(md.rank() == 2);
+    VERIFY(md.rank_dynamic() == 2);
+    VERIFY(md[0, 0] == data[0]);
+    VERIFY(md[0, 1] == data[1]);
+    VERIFY(md[1, 0] == data[3]);
+  };
+
+  verify(std::mdspan(data.data(), shape[0], shape[1]));
+  verify(std::mdspan(data.data(), shape));
+  verify(std::mdspan(data.data(), shape_view));
+
+  std::mdspan<double, std::dextents<size_t, 2>> md1 = {data.data(), shape};
+  verify(md1);
+
+  std::mdspan<double, std::dextents<size_t, 2>> md2 = {data.data(), shape_view};
+  verify(md2);
+
+  static_assert(std::is_constructible_v<
+      std::mdspan<float, std::extents<int, 3, 5>>, float*>);
+  static_assert(!std::is_constructible_v<
+      std::mdspan<float, std::extents<int, 3, 5>>, float*, int>);
+  static_assert(std::is_constructible_v<
+      std::mdspan<float, std::extents<int, 3, 5>>, float*, int, int>);
+  static_assert(std::is_constructible_v<
+      std::mdspan<float, std::extents<int, 3, 5>>, float*, std::span<int, 0>>);
+  static_assert(std::is_constructible_v<
+      std::mdspan<float, std::extents<int, 3, 5>>, float*, std::span<int, 2>>);
+  static_assert(!std::is_convertible_v<
+      float*, std::mdspan<float, std::extents<int, 3, 5>>>);
+
+  static_assert(std::is_constructible_v<
+      std::mdspan<float, std::dextents<int, 2>>, float*, std::span<int, 2>>);
+  static_assert(!std::is_constructible_v<
+      std::mdspan<float, std::dextents<int, 2>>, float*, std::span<int, 1>>);
+  static_assert(!std::is_constructible_v<
+      std::mdspan<float, std::dextents<int, 2>>, float*, std::span<int, 3>>);
+  static_assert(!std::is_constructible_v<
+      std::mdspan<float, std::dextents<int, 2>>, float*, std::span<int, dyn>>);
+  return true;
+}
+
+constexpr bool
+test_from_pointer_and_integral_constant()
+{
+  std::array<double, 6> buffer{};
+  double * ptr = buffer.data();
+
+  auto verify = [ptr](auto actual, auto exts)
+    {
+      auto expected = std::mdspan<double, decltype(exts)>(ptr, exts);
+      static_assert(std::same_as<decltype(actual), decltype(expected)>);
+      VERIFY(actual.extents() == expected.extents());
+    };
+
+  auto c3 = std::integral_constant<int, 3>{};
+  auto c6 = std::integral_constant<int, 6>{};
+
+  verify(std::mdspan(ptr, 6), std::extents(6));
+  verify(std::mdspan(ptr, c6), std::extents(c6));
+  verify(std::mdspan(ptr, 2, c3), std::extents(2, c3));
+  return true;
+}
+
+constexpr bool
+test_from_extents()
+{
+  constexpr size_t n = 3*5*7;
+  std::array<double, n> storage{};
+  using Extents = std::extents<int, 3, 5, 7>;
+  auto exts = Extents{};
+  auto md = std::mdspan(storage.data(), exts);
+
+  assert_deduced_typedefs<double, Extents>(md);
+  VERIFY(md.data_handle() == storage.data());
+  VERIFY(md.extents() == exts);
+  return true;
+}
+
+constexpr bool
+test_from_mapping()
+{
+  constexpr size_t n = 3*5*7;
+  std::array<double, n> storage{};
+  using Extents = std::extents<int, 3, 5, 7>;
+
+  auto exts = Extents{};
+  auto m = std::layout_left::mapping(exts);
+  auto md = std::mdspan(storage.data(), m);
+
+  assert_deduced_typedefs<double, Extents, std::layout_left>(md);
+  VERIFY(md.data_handle() == storage.data());
+  VERIFY(md.mapping() == m);
+  return true;
+}
+
+constexpr bool
+test_from_accessor()
+{
+  constexpr size_t n = 3*5*7;
+  std::array<double, n> storage{};
+  using Extents = std::extents<int, 3, 5, 7>;
+
+  auto exts = Extents{};
+  auto m = std::layout_left::mapping(exts);
+  auto a = std::default_accessor<double>{};
+  auto md = std::mdspan(storage.data(), m, a);
+
+  assert_deduced_typedefs<double, Extents, std::layout_left>(md);
+  VERIFY(md.data_handle() == storage.data());
+  VERIFY(md.mapping() == m);
+  return true;
+}
+
+template<typename MDSpan, typename Pointer, typename... Ints>
+  concept has_pack_ctor = requires
+  {
+    { MDSpan(Pointer{}, Ints(0)...) } -> std::same_as<MDSpan>;
+  };
+
+template<typename CustomInt, bool ValidForPacks, bool ValidForArrays>
+  constexpr bool
+  test_from_int_like()
+  {
+    constexpr size_t n = 3*5*7;
+    std::array<double, n> storage{};
+
+    auto verify = [&](auto md)
+      {
+	VERIFY(md.data_handle() == storage.data());
+	VERIFY(md.extent(0) == 3);
+	VERIFY(md.extent(1) == 5);
+	VERIFY(md.extent(2) == 7);
+      };
+
+    static_assert(has_pack_ctor<std::mdspan<float, std::dextents<int, 3>>,
+	float*, CustomInt, int, CustomInt> == ValidForPacks);
+
+    static_assert(std::is_constructible_v<
+	std::mdspan<float, std::dextents<int, 3>>, float*,
+	std::span<CustomInt, 3>> == ValidForArrays);
+
+    static_assert(std::is_constructible_v<
+	std::mdspan<float, std::dextents<int, 3>>, float*,
+	std::array<CustomInt, 3>> == ValidForArrays);
+
+    if constexpr (ValidForPacks)
+      verify(std::mdspan(storage.data(), CustomInt(3), 5, CustomInt(7)));
+
+    if constexpr (ValidForArrays)
+      {
+	auto shape = std::array{CustomInt(3), CustomInt(5), CustomInt(7)};
+	auto shape_view = std::span<CustomInt, 3>{shape};
+	verify(std::mdspan(storage.data(), shape));
+	verify(std::mdspan(storage.data(), shape_view));
+      }
+    return true;
+  }
+
+template<typename T, bool NothrowConstructible = true,
+	 bool NothrowAssignable = true>
+  class OpaqueAccessor
+  {
+    struct Handle
+    {
+      constexpr
+      Handle(T * other)
+      : ptr(other)
+      { }
+
+      constexpr
+      Handle(const Handle&) noexcept(NothrowConstructible) = default;
+
+      constexpr
+      Handle(Handle&&) noexcept(NothrowConstructible) = default;
+
+      constexpr Handle&
+      operator=(const Handle&) noexcept(NothrowAssignable) = default;
+
+      constexpr Handle&
+      operator=(Handle&&) noexcept(NothrowAssignable) = default;
+
+      T * ptr;
+    };
+
+  public:
+    using element_type = T;
+    using reference = T&;
+    using data_handle_type = Handle;
+    using offset_policy = OpaqueAccessor;
+
+    reference
+    access(data_handle_type p, size_t i) const
+    {
+      ++access_count;
+      return p.ptr[i];
+    }
+
+    typename offset_policy::data_handle_type
+    offset(data_handle_type p, size_t i) const
+    {
+      ++offset_count;
+      return typename offset_policy::data_handle_type{(void*)(p.ptr + i)};
+    }
+
+    mutable size_t access_count = 0;
+    mutable size_t offset_count = 0;
+  };
+
+void
+test_from_opaque_accessor()
+{
+  constexpr size_t n = 3*5*7;
+  std::array<double, n> storage{};
+  using Extents = std::extents<int, 3, 5, 7>;
+
+  auto exts = Extents{};
+  auto m = std::layout_left::mapping(exts);
+  auto a = OpaqueAccessor<double>{};
+  auto handle = OpaqueAccessor<double>::data_handle_type{storage.data()};
+  auto md = std::mdspan(handle, m, a);
+
+  using MDSpan = decltype(md);
+  static_assert(std::same_as<MDSpan::accessor_type, decltype(a)>);
+
+  VERIFY(md[0, 0, 0] == 0.0);
+  VERIFY(md.accessor().access_count == 1);
+
+  VERIFY(md[2, 4, 6] == 0.0);
+  VERIFY(md.accessor().access_count == 2);
+}
+
+template<typename T, typename Base>
+  class BaseClassAccessor
+  {
+  public:
+    using element_type = T;
+    using reference = Base&;
+    using data_handle_type = T*;
+    using offset_policy = BaseClassAccessor;
+
+    static_assert(std::common_reference_with<reference&&, element_type&>);
+
+    reference
+    access(data_handle_type p, size_t i) const
+    { return p[i]; }
+
+    typename offset_policy::data_handle_type
+    offset(data_handle_type p, size_t i) const
+    { return typename offset_policy::data_handle_type{p + i}; }
+  };
+
+struct Base
+{
+  double value = 1.0;
+};
+
+struct Derived : Base
+{
+  double value = 2.0;
+};
+
+void
+test_from_base_class_accessor()
+{
+  constexpr size_t n = 3*5*7;
+  std::array<Derived, n> storage{};
+  using Extents = std::extents<int, 3, 5, 7>;
+
+  auto exts = Extents{};
+  auto m = std::layout_left::mapping(exts);
+  auto a = BaseClassAccessor<Derived, Base>{};
+  auto md = std::mdspan(storage.data(), m, a);
+
+  using MDSpan = decltype(md);
+  static_assert(std::same_as<MDSpan::accessor_type, decltype(a)>);
+  static_assert(std::same_as<decltype(md[0, 0, 0]), Base&>);
+  VERIFY(md[0, 0, 0].value == 1.0);
+  VERIFY(md[2, 4, 6].value == 1.0);
+}
+
+constexpr bool
+test_from_mapping_like()
+{
+  double data = 1.1;
+  auto m = LayoutLike::mapping<std::extents<int, 1, 2, 3>>{};
+  auto md = std::mdspan(&data, m);
+  VERIFY(md[0, 0, 0] == data);
+  VERIFY(md[0, 1, 2] == data);
+  return true;
+}
+
+template<typename MDSpan>
+  constexpr void
+  test_empty(MDSpan md)
+  {
+    VERIFY(md.empty() == (md.size() == 0));
+  }
+
+constexpr bool
+test_empty_all()
+{
+  test_empty(std::mdspan<double, std::extents<int, dyn>>{});
+  return true;
+}
+
+template<typename MDSpan, typename... Args>
+concept indexable = requires (MDSpan md, Args... args)
+{
+  { md[args...] } -> std::same_as<typename MDSpan::reference>;
+};
+
+template<typename Int, bool ValidForPacks, bool ValidForArrays>
+  constexpr bool
+  test_access()
+  {
+    using Extents = std::extents<int, 3, 5, 7>;
+    auto exts = Extents{};
+
+    auto mapping = std::layout_left::mapping(exts);
+    constexpr size_t n = mapping.required_span_size();
+    std::array<double, n> storage{};
+
+    auto md = std::mdspan(storage.data(), mapping);
+    using MDSpan = decltype(md);
+
+    for(int i = 0; i < exts.extent(0); ++i)
+      for(int j = 0; j < exts.extent(1); ++j)
+	for(int k = 0; k < exts.extent(2); ++k)
+	  {
+	    storage[mapping(i, j, k)] = 1.0;
+	    if constexpr (ValidForPacks)
+	      VERIFY(md[Int(i), Int(j), Int(k)] == 1.0);
+
+	    if constexpr (ValidForArrays)
+	      {
+		std::array<Int, 3> ijk{Int(i), Int(j), Int(k)};
+		VERIFY(md[ijk] == 1.0);
+		VERIFY(md[std::span(ijk)] == 1.0);
+	      }
+	    storage[mapping(i, j, k)] = 0.0;
+	  }
+
+    if constexpr (!ValidForPacks)
+      static_assert(!indexable<MDSpan, Int, int, Int>);
+
+    if constexpr (!ValidForArrays)
+      {
+	static_assert(!indexable<MDSpan, std::array<Int, 3>>);
+	static_assert(!indexable<MDSpan, std::span<Int, 3>>);
+      }
+    return true;
+  }
+
+constexpr bool
+test_swap()
+{
+  using Extents = std::dextents<int, 2>;
+  auto e1 = Extents{3, 5};
+  auto e2 = Extents{7, 11};
+
+  std::array<double, 3*5> s1{};
+  std::array<double, 7*11> s2{};
+
+  auto md1 = std::mdspan(s1.data(), e1);
+  auto md2 = std::mdspan(s2.data(), e2);
+
+  std::swap(md1, md2);
+
+  VERIFY(md1.data_handle() == s2.data());
+  VERIFY(md2.data_handle() == s1.data());
+
+  VERIFY(md1.size() == s2.size());
+  VERIFY(md2.size() == s1.size());
+  return true;
+}
+
+namespace adl
+{
+  template<typename T>
+    struct SwappableAccessor
+    {
+      using element_type = T;
+      using reference = T&;
+      using data_handle_type = T*;
+      using offset_policy = SwappableAccessor;
+
+      reference
+      access(data_handle_type p, size_t i) const
+      { return p[i]; }
+
+      typename offset_policy::data_handle_type
+      offset(data_handle_type p, size_t i) const
+      { return p + i; }
+
+      friend void
+      swap(SwappableAccessor&, SwappableAccessor&)
+      { ++swap_count; }
+
+      static inline size_t swap_count = 0;
+    };
+}
+
+void
+test_swap_adl()
+{
+  using Extents = std::extents<int, dyn>;
+  using Layout = std::layout_left;
+  using Accessor = adl::SwappableAccessor<double>;
+  Accessor::swap_count = 0;
+
+  std::mdspan<double, Extents, Layout, Accessor> m1, m2;
+  swap(m1, m2);
+  VERIFY(Accessor::swap_count == 1);
+}
+
+template<bool Constructible, bool Assignable>
+constexpr void
+test_nothrow_movable()
+{
+  using Layout = std::layout_left;
+  using Extents = std::dextents<int, 3>;
+  using Accessor = OpaqueAccessor<int, Constructible, Assignable>;
+  using Handle = Accessor::data_handle_type;
+  static_assert(std::is_nothrow_move_assignable_v<Accessor>);
+  static_assert(std::is_nothrow_move_constructible_v<Accessor>);
+  static_assert(std::is_nothrow_move_assignable_v<Handle> == Assignable);
+  static_assert(std::is_nothrow_move_constructible_v<Handle> == Constructible);
+
+  using MDSpan = std::mdspan<int, Extents, Layout, Accessor>;
+  static_assert(std::is_nothrow_move_assignable_v<MDSpan> == Assignable);
+  static_assert(std::is_nothrow_move_constructible_v<MDSpan> == Constructible);
+}
+
+constexpr void
+test_nothrow_movable_all()
+{
+  using MDSpan = std::mdspan<double, std::dextents<int, 3>>;
+  static_assert(std::is_nothrow_move_assignable_v<MDSpan>);
+  static_assert(std::is_nothrow_move_constructible_v<MDSpan>);
+
+  test_nothrow_movable<true, true>();
+  test_nothrow_movable<true, false>();
+  test_nothrow_movable<false, true>();
+  test_nothrow_movable<false, false>();
+}
+
+template<typename Layout, bool Expected>
+  constexpr void
+  test_nothrow_is_methods()
+  {
+    using Extents = std::extents<int, dyn>;
+    using MDSpan = std::mdspan<double, Extents, Layout>;
+    static_assert(noexcept(MDSpan::is_always_unique()) == Expected);
+    static_assert(noexcept(MDSpan::is_always_exhaustive()) == Expected);
+    static_assert(noexcept(MDSpan::is_always_strided()) == Expected);
+
+    static_assert(noexcept(std::declval<MDSpan>().is_unique()) == Expected);
+    static_assert(noexcept(std::declval<MDSpan>().is_exhaustive()) == Expected);
+    static_assert(noexcept(std::declval<MDSpan>().is_strided()) == Expected);
+  }
+
+int
+main()
+{
+  test_typedefs_all();
+
+  test_rank_all();
+  test_extent_all();
+  static_assert(test_extent_all());
+
+  test_class_properties_all();
+  static_assert(test_class_properties_all());
+
+  test_empty_all();
+  static_assert(test_empty_all());
+
+  test_default_ctor();
+  static_assert(test_default_ctor());
+
+  test_nothrow_default_ctor<std::default_accessor, true>();
+  test_nothrow_default_ctor<ThrowingDefaultAccessor, false>();
+
+  test_from_other();
+  static_assert(test_from_other());
+
+  test_from_carray();
+  static_assert(test_from_carray());
+
+  test_from_pointer_and_shape();
+  static_assert(test_from_pointer_and_shape());
+
+  test_from_pointer_and_integral_constant();
+  static_assert(test_from_pointer_and_integral_constant());
+
+  test_from_extents();
+  static_assert(test_from_extents());
+
+  test_from_mapping();
+  static_assert(test_from_mapping());
+
+  test_from_accessor();
+  static_assert(test_from_accessor());
+
+  test_from_int_like<int, true, true>();
+  static_assert(test_from_int_like<int, true, true>());
+  test_from_int_like<IntLike, true, true>();
+  test_from_int_like<ThrowingInt, false, false>();
+  test_from_int_like<MutatingInt, true, false>();
+  test_from_int_like<RValueInt, true, false>();
+
+  test_from_opaque_accessor();
+  test_from_base_class_accessor();
+  test_from_mapping_like();
+  static_assert(test_from_mapping_like());
+
+  test_access<int, true, true>();
+  static_assert(test_access<int, true, true>());
+  test_access<IntLike, true, true>();
+  test_access<ThrowingInt, false, false>();
+  test_access<MutatingInt, true, false>();
+  test_access<RValueInt, true, false>();
+
+  test_swap();
+  static_assert(test_swap());
+  test_swap_adl();
+
+  test_nothrow_movable_all();
+  test_nothrow_is_methods<std::layout_right, true>();
+  test_nothrow_is_methods<ThrowingLayout, false>();
+  return 0;
+}
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/out_of_bounds_neg.cc b/libstdc++-v3/testsuite/23_containers/mdspan/out_of_bounds_neg.cc
new file mode 100644
index 0000000..dceae56
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/out_of_bounds_neg.cc
@@ -0,0 +1,24 @@
+// { dg-do compile { target c++23 } }
+#include<mdspan>
+
+#include "layout_like.h"
+
+template<typename Layout>
+constexpr bool
+test_invalid_multi_index()
+{
+
+  double data = 1.1;
+  auto m = typename Layout::mapping<std::extents<int, 1, 2, 3>>{};
+  auto md = std::mdspan(&data, m);
+
+  [[maybe_unused]] double x = md[0, 2, 2]; // { dg-error "expansion of" }
+  return true;
+};
+static_assert(test_invalid_multi_index<LayoutLike>()); // { dg-error "expansion of" }
+static_assert(test_invalid_multi_index<std::layout_left>()); // { dg-error "expansion of" }
+static_assert(test_invalid_multi_index<std::layout_right>()); // { dg-error "expansion of" }
+static_assert(test_invalid_multi_index<std::layout_stride>()); // { dg-error "expansion of" }
+
+// { dg-prune-output "non-constant condition" }
+// { dg-prune-output "__glibcxx_assert" }
diff --git a/libstdc++-v3/testsuite/23_containers/mdspan/version.cc b/libstdc++-v3/testsuite/23_containers/mdspan/version.cc
new file mode 100644
index 0000000..106ee40
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/mdspan/version.cc
@@ -0,0 +1,9 @@
+// { dg-do compile { target c++23 } }
+#include <mdspan>
+
+#ifndef __cpp_lib_mdspan
+#error "Feature test macro __cpp_lib_mdspan is missing for <mdspan>"
+#if __cpp_lib_mdspan < 202207
+#error "Feature test macro __cpp_lib_mdspan has the wrong value"
+#endif
+#endif
diff --git a/libstdc++-v3/testsuite/23_containers/span/120997.cc b/libstdc++-v3/testsuite/23_containers/span/120997.cc
new file mode 100644
index 0000000..fbf194c
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/span/120997.cc
@@ -0,0 +1,46 @@
+// { dg-do run { target c++26 } }
+
+#include <span>
+#include <testsuite_hooks.h>
+
+void
+test_first()
+{
+  bool arr[5];
+  std::span<const bool> s(arr);
+  std::span<const bool> s2 = s.first(5);
+  VERIFY( s2.data() == s.data() );
+  std::span<const bool> s3 = s.first<5>();
+  VERIFY( s3.data() == s.data() );
+}
+
+void
+test_last()
+{
+  bool arr[5];
+  std::span<const bool> s(arr);
+  std::span<const bool> s2 = s.last(5);
+  VERIFY( s2.data() == s.data() );
+  std::span<const bool> s3 = s.last<5>();
+  VERIFY( s3.data() == s.data() );
+}
+
+void
+test_subspan()
+{
+  bool arr[5];
+  std::span<const bool> s(arr);
+  std::span<const bool> s2 = s.subspan(0, 5);
+  VERIFY( s2.data() == s.data() );
+  std::span<const bool> s3 = s.subspan<0>();
+  VERIFY( s3.data() == s.data() );
+  std::span<const bool> s4 = s.subspan<0, 5>();
+  VERIFY( s4.data() == s.data() );
+}
+
+int main()
+{
+  test_first();
+  test_last();
+  test_subspan();
+}
diff --git a/libstdc++-v3/testsuite/23_containers/span/contiguous_range_neg.cc b/libstdc++-v3/testsuite/23_containers/span/contiguous_range_neg.cc
index c9e9112..890fdf8 100644
--- a/libstdc++-v3/testsuite/23_containers/span/contiguous_range_neg.cc
+++ b/libstdc++-v3/testsuite/23_containers/span/contiguous_range_neg.cc
@@ -25,6 +25,7 @@ main()
 {
   std::deque<int> d{};
   std::span<int, std::dynamic_extent> myspan(d); // { dg-error "no match" }
+  (void) myspan;
 }
 
 // { dg-prune-output "data" }
diff --git a/libstdc++-v3/testsuite/23_containers/span/deduction.cc b/libstdc++-v3/testsuite/23_containers/span/deduction.cc
index dce6ced..c66db90 100644
--- a/libstdc++-v3/testsuite/23_containers/span/deduction.cc
+++ b/libstdc++-v3/testsuite/23_containers/span/deduction.cc
@@ -80,4 +80,7 @@ test01()
 
   std::span s12(const_cast<const std::span<int>&>(s5));
   static_assert( is_dynamic_span<int>(s12) );
+
+  std::span s13(a.data(), std::integral_constant<size_t, 3>{});
+  static_assert( is_static_span<long, 3>(s13));
 }
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_map/const_container.cc b/libstdc++-v3/testsuite/23_containers/unordered_map/const_container.cc
new file mode 100644
index 0000000..e62f158
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_map/const_container.cc
@@ -0,0 +1,19 @@
+// { dg-do compile { target c++11 } }
+
+#include <unordered_map>
+
+#include <testsuite_hooks.h>
+
+// PR c++/116369
+const std::unordered_map<int, int> um
+  {
+    { 0, 1 },
+    { 2, 3 },
+    { 4, 5 }
+  };
+
+int main()
+{
+  VERIFY( um.size() == 3 );
+  VERIFY( um.find(0) != um.end() );
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multimap/const_container.cc b/libstdc++-v3/testsuite/23_containers/unordered_multimap/const_container.cc
new file mode 100644
index 0000000..3da1e33
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multimap/const_container.cc
@@ -0,0 +1,22 @@
+// { dg-do compile { target c++11 } }
+
+#include <unordered_map>
+
+#include <testsuite_hooks.h>
+
+// PR c++/116369
+const std::unordered_multimap<int, int> umm
+  {
+    { 0, 1 },
+    { 0, 1 },
+    { 2, 3 },
+    { 2, 3 },
+    { 4, 5 },
+    { 4, 5 }
+  };
+
+int main()
+{
+  VERIFY( umm.size() == 6 );
+  VERIFY( umm.find(0) != umm.end() );
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_multiset/const_container.cc b/libstdc++-v3/testsuite/23_containers/unordered_multiset/const_container.cc
new file mode 100644
index 0000000..841d25a
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_multiset/const_container.cc
@@ -0,0 +1,15 @@
+// { dg-do compile { target c++11 } }
+
+#include <unordered_set>
+
+#include <testsuite_hooks.h>
+
+// PR c++/116369
+const std::unordered_multiset<int> ums
+  { 0, 0, 1, 1, 2, 2 };
+
+int main()
+{
+  VERIFY( ums.size() == 6 );
+  VERIFY( ums.find(0) != ums.end() );
+}
diff --git a/libstdc++-v3/testsuite/23_containers/unordered_set/const_container.cc b/libstdc++-v3/testsuite/23_containers/unordered_set/const_container.cc
new file mode 100644
index 0000000..ffdbbad
--- /dev/null
+++ b/libstdc++-v3/testsuite/23_containers/unordered_set/const_container.cc
@@ -0,0 +1,14 @@
+// { dg-do compile { target c++11 } }
+
+#include <unordered_set>
+
+#include <testsuite_hooks.h>
+
+// PR c++/116369
+const std::unordered_set<int> us { 0, 1, 2 };
+
+int main()
+{
+  VERIFY( us.size() == 3 );
+  VERIFY( us.find(0) != us.end() );
+}
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/cons/from_range.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/cons/from_range.cc
index 339c06bd..516d888 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/bool/cons/from_range.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/cons/from_range.cc
@@ -42,14 +42,16 @@ do_test(Alloc alloc)
 }
 
 template<typename Range>
-void
+constexpr void
 do_test_a()
 {
   do_test<Range>(std::allocator<bool>());
-  do_test<Range>(__gnu_test::uneq_allocator<bool>(42));
+  if not consteval {
+    do_test<Range>(__gnu_test::uneq_allocator<bool>(42));
+  }
 }
 
-bool
+constexpr bool
 test_ranges()
 {
   using namespace __gnu_test;
@@ -71,9 +73,9 @@ test_ranges()
 
   // Not lvalue-convertible to bool
   struct C {
-    C(bool v) : val(v) { }
-    operator bool() && { return val; }
-    bool operator==(bool b) const { return b == val; }
+    constexpr C(bool v) : val(v) { }
+    constexpr operator bool() && { return val; }
+    constexpr bool operator==(bool b) const { return b == val; }
     bool val;
   };
   using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
@@ -82,16 +84,8 @@ test_ranges()
   return true;
 }
 
-constexpr bool
-test_constexpr()
-{
-  // XXX: this doesn't test the non-forward_range code paths are constexpr.
-  do_test<std::span<bool>>(std::allocator<bool>());
-  return true;
-}
-
 int main()
 {
   test_ranges();
-  static_assert( test_constexpr() );
+  static_assert( test_ranges() );
 }
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/assign/assign_range.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/assign/assign_range.cc
index 7e58700..ced7efe 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/assign/assign_range.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/assign/assign_range.cc
@@ -59,14 +59,14 @@ do_test()
 }
 
 template<typename Range>
-void
+constexpr void
 do_test_a()
 {
   do_test<Range, std::allocator<bool>>();
   do_test<Range, __gnu_test::SimpleAllocator<bool>>();
 }
 
-bool
+constexpr bool
 test_ranges()
 {
   using namespace __gnu_test;
@@ -88,9 +88,9 @@ test_ranges()
 
   // Not lvalue-convertible to bool
   struct C {
-    C(bool v) : val(v) { }
-    operator bool() && { return val; }
-    bool operator==(bool b) const { return b == val; }
+    constexpr C(bool v) : val(v) { }
+    constexpr operator bool() && { return val; }
+    constexpr bool operator==(bool b) const { return b == val; }
     bool val;
   };
   using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
@@ -99,16 +99,8 @@ test_ranges()
   return true;
 }
 
-constexpr bool
-test_constexpr()
-{
-  // XXX: this doesn't test the non-forward_range code paths are constexpr.
-  do_test<std::span<short>, std::allocator<bool>>();
-  return true;
-}
-
 int main()
 {
   test_ranges();
-  static_assert( test_constexpr() );
+  static_assert( test_ranges() );
 }
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/insert/append_range.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/insert/append_range.cc
index 43a698f..c2e2186 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/insert/append_range.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/insert/append_range.cc
@@ -38,14 +38,14 @@ do_test()
 }
 
 template<typename Range>
-void
+constexpr void
 do_test_a()
 {
   do_test<Range, std::allocator<bool>>();
   do_test<Range, __gnu_test::SimpleAllocator<bool>>();
 }
 
-bool
+constexpr bool
 test_ranges()
 {
   using namespace __gnu_test;
@@ -67,9 +67,9 @@ test_ranges()
 
   // Not lvalue-convertible to bool
   struct C {
-    C(bool v) : val(v) { }
-    operator bool() && { return val; }
-    bool operator==(bool b) const { return b == val; }
+    constexpr C(bool v) : val(v) { }
+    constexpr operator bool() && { return val; }
+    constexpr bool operator==(bool b) const { return b == val; }
     bool val;
   };
   using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
@@ -81,8 +81,7 @@ test_ranges()
 constexpr bool
 test_constexpr()
 {
-  // XXX: this doesn't test the non-forward_range code paths are constexpr.
-  do_test<std::span<short>, std::allocator<bool>>();
+  test_ranges();
 
   // Some basic tests for overlapping ranges in constant expressions.
   using I = std::vector<bool>::iterator;
diff --git a/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/insert/insert_range.cc b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/insert/insert_range.cc
index 5c65610..2ec91b0 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/insert/insert_range.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/bool/modifiers/insert/insert_range.cc
@@ -55,14 +55,14 @@ do_test()
 }
 
 template<typename Range>
-void
+constexpr void
 do_test_a()
 {
   do_test<Range, std::allocator<bool>>();
   do_test<Range, __gnu_test::SimpleAllocator<bool>>();
 }
 
-bool
+constexpr bool
 test_ranges()
 {
   using namespace __gnu_test;
@@ -84,9 +84,9 @@ test_ranges()
 
   // Not lvalue-convertible to bool
   struct C {
-    C(bool v) : val(v) { }
-    operator bool() && { return val; }
-    bool operator==(bool b) const { return b == val; }
+    constexpr C(bool v) : val(v) { }
+    constexpr operator bool() && { return val; }
+    constexpr bool operator==(bool b) const { return b == val; }
     bool val;
   };
   using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
@@ -95,16 +95,8 @@ test_ranges()
   return true;
 }
 
-constexpr bool
-test_constexpr()
-{
-  // XXX: this doesn't test the non-forward_range code paths are constexpr.
-  do_test<std::span<bool>, std::allocator<bool>>();
-  return true;
-}
-
 int main()
 {
   test_ranges();
-  static_assert( test_constexpr() );
+  static_assert( test_ranges() );
 }
diff --git a/libstdc++-v3/testsuite/23_containers/vector/cons/from_range.cc b/libstdc++-v3/testsuite/23_containers/vector/cons/from_range.cc
index 3784b9c..be3e699 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/cons/from_range.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/cons/from_range.cc
@@ -58,14 +58,16 @@ do_test(Alloc alloc)
 }
 
 template<typename Range>
-void
+constexpr void
 do_test_a()
 {
   do_test<Range>(std::allocator<int>());
-  do_test<Range>(__gnu_test::uneq_allocator<int>(42));
+  if not consteval {
+    do_test<Range>(__gnu_test::uneq_allocator<int>(42));
+  }
 }
 
-bool
+constexpr bool
 test_ranges()
 {
   using namespace __gnu_test;
@@ -87,9 +89,9 @@ test_ranges()
 
   // Not lvalue-convertible to int
   struct C {
-    C(int v) : val(v) { }
-    operator int() && { return val; }
-    bool operator==(int b) const { return b == val; }
+    constexpr C(int v) : val(v) { }
+    constexpr operator int() && { return val; }
+    constexpr bool operator==(int b) const { return b == val; }
     int val;
   };
   using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
@@ -98,14 +100,6 @@ test_ranges()
   return true;
 }
 
-constexpr bool
-test_constexpr()
-{
-  // XXX: this doesn't test the non-forward_range code paths are constexpr.
-  do_test<std::span<short>>(std::allocator<int>());
-  return true;
-}
-
 void
 test_pr120367()
 {
@@ -130,6 +124,6 @@ test_pr120367()
 int main()
 {
   test_ranges();
-  static_assert( test_constexpr() );
+  static_assert( test_ranges() );
   test_pr120367();
 }
diff --git a/libstdc++-v3/testsuite/23_containers/vector/debug/mutex_association.cc b/libstdc++-v3/testsuite/23_containers/vector/debug/mutex_association.cc
index ba2ede0..792ed45 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/debug/mutex_association.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/debug/mutex_association.cc
@@ -25,7 +25,7 @@ class container : public __gnu_debug::_Safe_sequence<container>
 {
 public:
   __gnu_cxx::__mutex&
-  get_mutex()
+  get_mutex() const
   { return this->_M_get_mutex(); }
 };
 
diff --git a/libstdc++-v3/testsuite/23_containers/vector/modifiers/append_range.cc b/libstdc++-v3/testsuite/23_containers/vector/modifiers/append_range.cc
index be097e2..f5b21df 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/modifiers/append_range.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/modifiers/append_range.cc
@@ -42,14 +42,14 @@ do_test()
 }
 
 template<typename Range>
-void
+constexpr void
 do_test_a()
 {
   do_test<Range, std::allocator<int>>();
   do_test<Range, __gnu_test::SimpleAllocator<int>>();
 }
 
-bool
+constexpr bool
 test_ranges()
 {
   using namespace __gnu_test;
@@ -71,9 +71,9 @@ test_ranges()
 
   // Not lvalue-convertible to int
   struct C {
-    C(int v) : val(v) { }
-    operator int() && { return val; }
-    bool operator==(int b) const { return b == val; }
+    constexpr C(int v) : val(v) { }
+    constexpr operator int() && { return val; }
+    constexpr bool operator==(int b) const { return b == val; }
     int val;
   };
   using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
@@ -82,7 +82,7 @@ test_ranges()
   return true;
 }
 
-void
+constexpr void
 test_overlapping()
 {
   using __gnu_test::test_input_range;
@@ -199,64 +199,14 @@ test_overlapping()
   }
 }
 
-constexpr bool
-test_constexpr()
+int main()
 {
-  // XXX: this doesn't test the non-forward_range code paths are constexpr.
-  do_test<std::span<short>, std::allocator<int>>();
-
-  // Some basic tests for overlapping ranges in constant expressions.
-  struct InputRange
-  {
-    struct Sent { const void* end; };
-
-    struct Iter
-    {
-      using value_type = int;
-      using difference_type = int;
-      constexpr explicit Iter(int* p) : ptr(p) { }
-      constexpr Iter& operator++() { ++ptr; return *this; }
-      constexpr Iter operator++(int) { auto i = *this; ++ptr; return i; }
-      constexpr int operator*() const { return *ptr; }
-      constexpr bool operator==(const Iter&) const = default;
-      constexpr bool operator==(const Sent& s) const { return ptr == s.end; }
-      int* ptr;
-    };
-
-    Iter iter;
-    Sent sent;
-
-    constexpr InputRange(int* f, int* l) : iter{f}, sent{l} { }
-    constexpr Iter begin() const { return iter; }
-    constexpr Sent end() const { return sent; }
+  auto test_all = [] {
+    test_ranges();
+    test_overlapping();
+    return true;
   };
-  static_assert( std::ranges::input_range<InputRange> );
-  static_assert( ! std::ranges::forward_range<InputRange> );
-
-  std::vector<int> vec(5);
-
-  // Test overlapping input ranges
-  vec.resize(vec.capacity());
-  vec.append_range(InputRange(vec.data(), vec.data() + 3)); // no capacity
-  vec.reserve(vec.capacity() + 2);
-  vec.append_range(InputRange(vec.data(), vec.data() + 4)); // some capacity
-  vec.reserve(vec.capacity() + 6);
-  vec.append_range(InputRange(vec.data(), vec.data() + 5)); // enough capacity
-
-  // Test overlapping forward ranges
-  vec.resize(vec.capacity());
-  vec.append_range(std::span<int>(vec));               // no capacity
-  vec.reserve(vec.size() + 2);
-  vec.append_range(std::span<int>(vec).subspan(1, 4)); // some capacity
-  vec.reserve(vec.size() + 6);
-  vec.append_range(std::span<int>(vec).subspan(1, 5)); // enough capacity
 
-  return true;
-}
-
-int main()
-{
-  test_ranges();
-  test_overlapping();
-  static_assert( test_constexpr() );
+  test_all();
+  static_assert( test_all() );
 }
diff --git a/libstdc++-v3/testsuite/23_containers/vector/modifiers/assign/assign_range.cc b/libstdc++-v3/testsuite/23_containers/vector/modifiers/assign/assign_range.cc
index db3b06c..26d33bc 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/modifiers/assign/assign_range.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/modifiers/assign/assign_range.cc
@@ -63,14 +63,14 @@ do_test()
 }
 
 template<typename Range>
-void
+constexpr void
 do_test_a()
 {
   do_test<Range, std::allocator<int>>();
   do_test<Range, __gnu_test::SimpleAllocator<int>>();
 }
 
-bool
+constexpr bool
 test_ranges()
 {
   using namespace __gnu_test;
@@ -92,9 +92,9 @@ test_ranges()
 
   // Not lvalue-convertible to int
   struct C {
-    C(int v) : val(v) { }
-    operator int() && { return val; }
-    bool operator==(int b) const { return b == val; }
+    constexpr C(int v) : val(v) { }
+    constexpr operator int() && { return val; }
+    constexpr bool operator==(int b) const { return b == val; }
     int val;
   };
   using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
@@ -103,16 +103,8 @@ test_ranges()
   return true;
 }
 
-constexpr bool
-test_constexpr()
-{
-  // XXX: this doesn't test the non-forward_range code paths are constexpr.
-  do_test<std::span<short>, std::allocator<int>>();
-  return true;
-}
-
 int main()
 {
   test_ranges();
-  static_assert( test_constexpr() );
+  static_assert( test_ranges() );
 }
diff --git a/libstdc++-v3/testsuite/23_containers/vector/modifiers/insert/insert_range.cc b/libstdc++-v3/testsuite/23_containers/vector/modifiers/insert/insert_range.cc
index 5907143..506bebb 100644
--- a/libstdc++-v3/testsuite/23_containers/vector/modifiers/insert/insert_range.cc
+++ b/libstdc++-v3/testsuite/23_containers/vector/modifiers/insert/insert_range.cc
@@ -59,14 +59,14 @@ do_test()
 }
 
 template<typename Range>
-void
+constexpr void
 do_test_a()
 {
   do_test<Range, std::allocator<int>>();
   do_test<Range, __gnu_test::SimpleAllocator<int>>();
 }
 
-bool
+constexpr bool
 test_ranges()
 {
   using namespace __gnu_test;
@@ -88,9 +88,9 @@ test_ranges()
 
   // Not lvalue-convertible to int
   struct C {
-    C(int v) : val(v) { }
-    operator int() && { return val; }
-    bool operator==(int b) const { return b == val; }
+    constexpr C(int v) : val(v) { }
+    constexpr operator int() && { return val; }
+    constexpr bool operator==(int b) const { return b == val; }
     int val;
   };
   using rvalue_input_range = test_range<C, input_iterator_wrapper_rval>;
@@ -99,16 +99,8 @@ test_ranges()
   return true;
 }
 
-constexpr bool
-test_constexpr()
-{
-  // XXX: this doesn't test the non-forward_range code paths are constexpr.
-  do_test<std::span<short>, std::allocator<int>>();
-  return true;
-}
-
 int main()
 {
   test_ranges();
-  static_assert( test_constexpr() );
+  static_assert( test_ranges() );
 }
diff --git a/libstdc++-v3/testsuite/24_iterators/operations/cxx20_iterators.cc b/libstdc++-v3/testsuite/24_iterators/operations/cxx20_iterators.cc
new file mode 100644
index 0000000..b613c37
--- /dev/null
+++ b/libstdc++-v3/testsuite/24_iterators/operations/cxx20_iterators.cc
@@ -0,0 +1,60 @@
+// { dg-do run { target c++20 } }
+
+#include <ranges>
+#include <testsuite_iterators.h>
+#include <testsuite_hooks.h>
+
+// Bug 102181 std::advance and std::views::iota<std::int64_t> don't work
+void
+test_pr102181()
+{
+#ifdef __SIZEOF_INT128__
+  using type = unsigned __int128;
+#else
+  using type = unsigned long;
+#endif
+  auto v = std::ranges::iota_view(type(0), type(10));
+  auto b = v.begin();
+  VERIFY( std::distance(b, std::next(b)) == 1 );
+  std::advance(b, std::iter_difference_t<decltype(b)>(1));
+  VERIFY( *b == 1 );
+  VERIFY( std::distance(b, v.end()) == 9 );
+}
+
+// https://stackoverflow.com/questions/68100775/rangesviewtransform-produces-an-inputiterator-preventing-the-use-of-stdpre
+void
+test_transform_view_iterator()
+{
+  int a[] = {0, 1, 2, 3};
+  __gnu_test::random_access_container<int> rr(a);
+  auto rx = std::ranges::views::transform(rr, std::identity{});
+  auto re = rx.end();
+  VERIFY( *std::prev(re) == 3 );
+  VERIFY( std::distance(rx.begin(), re) == 4 );
+
+  __gnu_test::bidirectional_container<int> br(a);
+  auto bx = std::ranges::views::transform(br, std::identity{});
+  auto be = bx.end();
+  VERIFY( *std::prev(be) == 3 );
+  VERIFY( std::distance(bx.begin(), be) == 4 );
+
+  __gnu_test::forward_container<int> fr(a);
+  auto fx = std::ranges::views::transform(br, std::identity{});
+  auto fb = fx.begin();
+  VERIFY( *std::next(fb) == 1 );
+  VERIFY( std::distance(fb, fx.end()) == 4 );
+
+  __gnu_test::test_input_range<int> ir(a);
+  auto ix = std::ranges::views::transform(ir, std::identity{});
+  auto ii = ix.begin();
+  std::advance(ii, 1);
+  VERIFY( *ii == 1 );
+  // N.B. cannot use std::distance or std::next here because there is no
+  // iterator_traits<decltype(ii)>::difference_type for this iterator.
+}
+
+int main()
+{
+  test_pr102181();
+  test_transform_view_iterator();
+}
diff --git a/libstdc++-v3/testsuite/24_iterators/random_access/string_vector_iterators.cc b/libstdc++-v3/testsuite/24_iterators/random_access/string_vector_iterators.cc
index a8dbe51..22803e7 100644
--- a/libstdc++-v3/testsuite/24_iterators/random_access/string_vector_iterators.cc
+++ b/libstdc++-v3/testsuite/24_iterators/random_access/string_vector_iterators.cc
@@ -17,238 +17,162 @@
 // along with this program; see the file COPYING3.  If not see
 // <http://www.gnu.org/licenses/>.
 
-
 #include <string>
 #include <vector>
 #include <testsuite_hooks.h>
 
-int 
+void
 string_stuff()
 {
-   int failures(0);
-
    std::string s("abcde");
 
    std::string::iterator i1(s.begin());
-   if (*i1 != 'a')
-      ++failures;
+   VERIFY( *i1 == 'a' );
 
    ++i1;
-   if (*i1 != 'b')
-      ++failures;
+   VERIFY( *i1 == 'b' );
 
-   if (*i1++ != 'b')
-       ++failures;
-   if (*i1 != 'c')
-      ++failures;
+   VERIFY( *i1++ == 'b' );
+   VERIFY( *i1 == 'c' );
 
    ++ ++i1;
-   if (*i1 != 'e')
-      ++failures;
+   VERIFY( *i1 == 'e' );
 
    --i1;
-   if (*i1 != 'd')
-      ++failures;
+   VERIFY( *i1 == 'd' );
 
-   if (*i1-- != 'd')
-      ++failures;
-   if (*i1 != 'c')
-      ++failures;
+   VERIFY( *i1-- == 'd' );
+   VERIFY( *i1 == 'c' );
 
    -- --i1;
-   if (*i1 != 'a')
-      ++failures;
+   VERIFY( *i1 == 'a' );
 
    std::string::iterator i2;
    i2 = s.end();
    std::iterator_traits<std::string::iterator>::difference_type d1;
    d1 = i2 - i1;
-   if (d1 != 5)
-      ++failures;
+   VERIFY( d1 == 5 );
 
    std::iterator_traits<std::string::iterator>::value_type v1;
    v1 = i1[0];
-   if (v1 != 'a')
-      ++failures;
+   VERIFY( v1 == 'a' );
 
    std::iterator_traits<std::string::iterator>::reference r1(i1[0]);
-   if (r1 != 'a')
-      ++failures;
+   VERIFY( r1 == 'a' );
    r1 = 'x';
-   if (r1 != 'x')
-      ++failures;
+   VERIFY( r1 == 'x' );
    r1 = 'a';
 
-   if ((i1 != i2) != true)
-      ++failures;
-   if ((i1 == i2) != false)
-      ++failures;
-   if ((i1 <  i2) != true)
-      ++failures;
-   if ((i1 >  i2) != false)
-      ++failures;
-   if ((i1 <= i2) != true)
-      ++failures;
-   if ((i1 >= i2) != false)
-      ++failures;
+   VERIFY( (i1 != i2) == true );
+   VERIFY( (i1 == i2) == false );
+   VERIFY( (i1 <  i2) == true );
+   VERIFY( (i1 >  i2) == false );
+   VERIFY( (i1 <= i2) == true );
+   VERIFY( (i1 >= i2) == false );
 
    std::string::iterator i3;
    i3 = i1;
-   if ((i3 == i1) != true)
-      ++failures;
+   VERIFY( (i3 == i1) == true );
 
    i3 += 5;
-   if ((i3 == i2) != true)
-      ++failures;
+   VERIFY( (i3 == i2) == true );
 
    i3 -= 5;
-   if ((i3 == i1) != true)
-      ++failures;
-
-   if (i3 + 5 != i2)
-      ++failures;
-
-   if (5 + i3 != i2)
-      ++failures;
+   VERIFY( (i3 == i1) == true );
 
-   if (i2 - 5 != i3)
-      ++failures;
+   VERIFY( i3 + 5 == i2 );
+   VERIFY( 5 + i3 == i2 );
+   VERIFY( i2 - 5 == i3 );
 
-   if (i1[0] != 'a')
-      ++failures;
+   VERIFY( i1[0] == 'a' );
 
    i1[4] = 'x';
-   if (i2[-1] != 'x')
-      ++failures;
+   VERIFY( i2[-1] == 'x' );
    i1[4] = 'e';
 
    i1[2] = 'x';
-   if (i2[-3] != 'x')
-      ++failures;
+   VERIFY( i2[-3] == 'x' );
    i1[2] = 'c';
 
    std::string::const_iterator ci1(s.begin());
-   if (*ci1 != 'a')
-      ++failures;
+   VERIFY( *ci1 == 'a' );
 
    ++ci1;
-   if (*ci1 != 'b')
-      ++failures;
+   VERIFY( *ci1 == 'b' );
 
-   if (*ci1++ != 'b')
-      ++failures;
-   if (*ci1 != 'c')
-      ++failures;
+   VERIFY( *ci1++ == 'b' );
+   VERIFY( *ci1 == 'c' );
 
    ++ ++ci1;
-   if (*ci1 != 'e')
-      ++failures;
+   VERIFY( *ci1 == 'e' );
 
    --ci1;
-   if (*ci1 != 'd')
-      ++failures;
+   VERIFY( *ci1 == 'd' );
 
-   if (*ci1-- != 'd')
-      ++failures;
-   if (*ci1 != 'c')
-      ++failures;
+   VERIFY( *ci1-- == 'd' );
+   VERIFY( *ci1 == 'c' );
 
    -- --ci1;
-   if (*ci1 != 'a')
-      ++failures;
+   VERIFY( *ci1 == 'a' );
 
    std::string::const_iterator ci2;
    ci2 = s.end();
    std::iterator_traits<std::string::const_iterator>::difference_type d2;
    d2 = ci2 - ci1;
-   if (d2 != 5)
-     ++failures;
+   VERIFY( d2 == 5 );
 
    std::iterator_traits<std::string::const_iterator>::value_type v2;
    v2 = ci1[0];
-   if (v2 != 'a')
-     ++failures;
+   VERIFY( v2 == 'a' );
 
    std::iterator_traits<std::string::const_iterator>::reference r2(ci1[0]);
-   if (r2 != 'a')
-      ++failures;
-
-   if ((ci1 != ci2) != true)
-      ++failures;
-   if ((ci1 == ci2) != false)
-      ++failures;
-   if ((ci1 <  ci2) != true)
-      ++failures;
-   if ((ci1 >  ci2) != false)
-      ++failures;
-   if ((ci1 <= ci2) != true)
-      ++failures;
-   if ((ci1 >= ci2) != false)
-      ++failures;
+   VERIFY( r2 == 'a' );
+
+   VERIFY( (ci1 != ci2) == true );
+   VERIFY( (ci1 == ci2) == false );
+   VERIFY( (ci1 <  ci2) == true );
+   VERIFY( (ci1 >  ci2) == false );
+   VERIFY( (ci1 <= ci2) == true );
+   VERIFY( (ci1 >= ci2) == false );
 
    std::string::const_iterator ci3;
    ci3 = ci1;
-   if ((ci3 == ci1) != true)
-      ++failures;
+   VERIFY( ci3 == ci1 );
 
    ci3 += 5;
-   if ((ci3 == ci2) != true)
-      ++failures;
+   VERIFY( ci3 == ci2 );
 
    ci3 -= 5;
-   if ((ci3 == ci1) != true)
-      ++failures;
-
-   if (ci3 + 5 != ci2)
-      ++failures;
-
-   if (5 + ci3 != ci2)
-      ++failures;
-
-   if (ci2 - 5 != ci3)
-      ++failures;
+   VERIFY( ci3 == ci1 );
 
-   if (ci1[2] != 'c')
-      ++failures;
+   VERIFY( ci3 + 5 == ci2 );
+   VERIFY( 5 + ci3 == ci2 );
+   VERIFY( ci2 - 5 == ci3 );
 
-   if (ci2[-1] != 'e')
-      ++failures;
+   VERIFY( ci1[2] == 'c' );
+   VERIFY( ci2[-1] == 'e' );
 
    // iterator and const_iterator
    std::string::const_iterator ci4(i1);
-   if ((ci4 == i1) != true)
-      ++failures;
-   if ((ci4 != i1) != false)
-      ++failures;
-   if ((ci4 < i1)  != false)
-     ++failures;
-   if ((ci4 > i1)  != false)
-     ++failures;
-   if ((ci4 <= i1) != true)
-     ++failures;
-   if ((ci4 >= i1) != true)
-     ++failures;
+   VERIFY( (ci4 == i1) == true );
+   VERIFY( (ci4 != i1) == false );
+   VERIFY( (ci4 < i1) == false );
+   VERIFY( (ci4 > i1) == false );
+   VERIFY( (ci4 <= i1) == true );
+   VERIFY( (ci4 >= i1) == true );
    ci4 = i2;
-   if ((i2 == ci4) != true)
-     ++failures;
-   if ((i2 < ci4)  != false)
-     ++failures;
-   if ((i2 > ci4)  != false)
-     ++failures;
-   if ((i2 <= ci4) != true)
-     ++failures;
-   if ((i2 >= ci4) != true)
-     ++failures;
+   VERIFY( (i2 == ci4) == true );
+   VERIFY( (i2 < ci4) == false );
+   VERIFY( (i2 > ci4) == false );
+   VERIFY( (i2 <= ci4) == true );
+   VERIFY( (i2 >= ci4) == true );
 
    const std::string cs("ABCDE");
    std::string::const_iterator ci5(cs.begin());
-   if (ci5[0] != 'A')
-      ++failures;
-
-   return failures;
+   VERIFY( ci5[0] == 'A' );
 }
 
-int 
+void
 vector_stuff()
 {
    int failures(0);
@@ -261,347 +185,190 @@ vector_stuff()
    v.push_back(int(5));
 
    std::vector<int>::iterator i1(v.begin());
-   if (*i1 != 1)
-      ++failures;
+   VERIFY( *i1 == 1 );
 
    ++i1;
-   if (*i1 != 2)
-      ++failures;
+   VERIFY( *i1 == 2 );
 
-   if (*i1++ != 2)
-      ++failures;
-   if (*i1 != 3)
-      ++failures;
+   VERIFY( *i1++ == 2 );
+   VERIFY( *i1 == 3 );
 
    ++ ++i1;
-   if (*i1 != 5)
-      ++failures;
+   VERIFY( *i1 == 5 );
 
    --i1;
-   if (*i1 != 4)
-      ++failures;
+   VERIFY( *i1 == 4 );
 
-   if (*i1-- != 4)
-      ++failures;
-   if (*i1 != 3)
-      ++failures;
+   VERIFY( *i1-- == 4 );
+   VERIFY( *i1 == 3 );
 
    -- --i1;
-   if (*i1 != 1)
-      ++failures;
+   VERIFY( *i1 == 1 );
 
    std::vector<int>::iterator i2;
    i2 = v.end();
    std::iterator_traits<std::vector<int>::iterator>::difference_type d1;
    d1 = i2 - i1;
-   if (d1 != 5)
-      ++failures;
+   VERIFY( d1 == 5 );
 
    std::iterator_traits<std::vector<int>::iterator>::value_type v1;
    v1 = i1[0];
-   if (v1 != 1)
-      ++failures;
+   VERIFY( v1 == 1 );
 
    std::iterator_traits<std::vector<int>::iterator>::reference r1(i1[0]);
-   if (r1 != 1)
-      ++failures;
+   VERIFY( r1 == 1 );
    r1 = 9;
-   if (r1 != 9)
-      ++failures;
+   VERIFY( r1 == 9 );
    r1 = 1;
 
-   if ((i1 != i2) != true)
-      ++failures;
-   if ((i1 == i2) != false)
-      ++failures;
-   if ((i1 <  i2) != true)
-      ++failures;
-   if ((i1 >  i2) != false)
-      ++failures;
-   if ((i1 <= i2) != true)
-      ++failures;
-   if ((i1 >= i2) != false)
-      ++failures;
+   VERIFY( (i1 != i2) == true );
+   VERIFY( (i1 == i2) == false );
+   VERIFY( (i1 <  i2) == true );
+   VERIFY( (i1 >  i2) == false );
+   VERIFY( (i1 <= i2) == true );
+   VERIFY( (i1 >= i2) == false );
 
    std::vector<int>::iterator i3;
    i3 = i1;
-   if ((i3 == i1) != true)
-      ++failures;
+   VERIFY( (i3 == i1) == true );
 
    i3 += 5;
-   if ((i3 == i2) != true)
-      ++failures;
+   VERIFY( (i3 == i2) == true );
 
    i3 -= 5;
-   if ((i3 == i1) != true)
-      ++failures;
-
-   if (i3 + 5 != i2)
-      ++failures;
-
-   if (5 + i3 != i2)
-      ++failures;
+   VERIFY( (i3 == i1) == true );
 
-   if (i2 - 5 != i3)
-      ++failures;
+   VERIFY( i3 + 5 == i2 );
+   VERIFY( 5 + i3 == i2 );
+   VERIFY( i2 - 5 == i3 );
 
-   if (i1[0] != 1)
-      ++failures;
+   VERIFY( i1[0] == 1 );
 
    i1[4] = 9;
-   if (i2[-1] != 9)
-      ++failures;
+   VERIFY( i2[-1] == 9 );
    i1[4] = 5;
 
    i1[2] = 9;
-   if (i2[-3] != 9)
-      ++failures;
+   VERIFY( i2[-3] == 9 );
    i1[2] = 3;
 
    std::vector<int>::const_iterator ci1(v.begin());
-   if (*ci1 != 1)
-      ++failures;
+   VERIFY( *ci1 == 1 );
 
    ++ci1;
-   if (*ci1 != 2)
-      ++failures;
+   VERIFY( *ci1 == 2 );
 
-   if (*ci1++ != 2)
-      ++failures;
-   if (*ci1 != 3)
-      ++failures;
+   VERIFY( *ci1++ == 2 );
+   VERIFY( *ci1 == 3 );
 
    ++ ++ci1;
-   if (*ci1 != 5)
-      ++failures;
+   VERIFY( *ci1 == 5 );
 
    --ci1;
-   if (*ci1 != 4)
-      ++failures;
+   VERIFY( *ci1 == 4 );
 
-   if (*ci1-- != 4)
-      ++failures;
-   if (*ci1 != 3)
-      ++failures;
+   VERIFY( *ci1-- == 4 );
+   VERIFY( *ci1 == 3 );
 
    -- --ci1;
-   if (*ci1 != 1)
-      ++failures;
+   VERIFY( *ci1 == 1 );
 
    std::vector<int>::const_iterator ci2;
    ci2 = v.end();
    std::iterator_traits<std::vector<int>::const_iterator>::difference_type d2;
    d2 = ci2 - ci1;
-   if (d2 != 5)
-      ++failures;
+   VERIFY( d2 == 5 );
 
    std::iterator_traits<std::vector<int>::const_iterator>::value_type v2;
    v2 = ci1[0];
-   if (v2 != 1)
-      ++failures;
+   VERIFY( v2 == 1 );
 
    std::iterator_traits<std::vector<int>::const_iterator>::reference
       r2(ci1[0]);
-   if (r2 != 1)
-      ++failures;
-
-   if ((ci1 != ci2) != true)
-      ++failures;
-   if ((ci1 == ci2) != false)
-      ++failures;
-   if ((ci1 <  ci2) != true)
-      ++failures;
-   if ((ci1 >  ci2) != false)
-      ++failures;
-   if ((ci1 <= ci2) != true)
-      ++failures;
-   if ((ci1 >= ci2) != false)
-      ++failures;
+   VERIFY( r2 == 1 );
+
+   VERIFY( (ci1 != ci2) == true );
+   VERIFY( (ci1 == ci2) == false );
+   VERIFY( (ci1 <  ci2) == true );
+   VERIFY( (ci1 >  ci2) == false );
+   VERIFY( (ci1 <= ci2) == true );
+   VERIFY( (ci1 >= ci2) == false );
 
    std::vector<int>::const_iterator ci3;
    ci3 = ci1;
-   if ((ci3 == ci1) != true)
-      ++failures;
+   VERIFY( (ci3 == ci1) == true );
 
    ci3 += 5;
-   if ((ci3 == ci2) != true)
-      ++failures;
+   VERIFY( (ci3 == ci2) == true );
 
    ci3 -= 5;
-   if ((ci3 == ci1) != true)
-      ++failures;
-
-   if (ci3 + 5 != ci2)
-      ++failures;
+   VERIFY( (ci3 == ci1) == true );
 
-   if (5 + ci3 != ci2)
-      ++failures;
+   VERIFY( ci3 + 5 == ci2 );
+   VERIFY( 5 + ci3 == ci2 );
+   VERIFY( ci2 - 5 == ci3 );
 
-   if (ci2 - 5 != ci3)
-      ++failures;
+   VERIFY( ci1[2] == 3 );
 
-   if (ci1[2] != 3)
-      ++failures;
-
-   if (ci2[-1] != 5)
-      ++failures;
+   VERIFY( ci2[-1] == 5 );
 
    // iterator to const_iterator
    std::vector<int>::const_iterator ci4(i1);
-   if ((ci4 == i1) != true)
-      ++failures;
-   if ((ci4 != i1) != false)
-      ++failures;
-   if ((ci4 < i1)  != false)
-     ++failures;
-   if ((ci4 > i1)  != false)
-     ++failures;
-   if ((ci4 <= i1) != true)
-     ++failures;
-   if ((ci4 >= i1) != true)
-     ++failures;
+   VERIFY( (ci4 == i1) == true );
+   VERIFY( (ci4 != i1) == false );
+   VERIFY( (ci4 < i1) == false );
+   VERIFY( (ci4 > i1) == false );
+   VERIFY( (ci4 <= i1) == true );
+   VERIFY( (ci4 >= i1) == true );
    ci4 = i2;
-   if ((i2 == ci4) != true)
-     ++failures;
-   if ((i2 < ci4)  != false)
-     ++failures;
-   if ((i2 > ci4)  != false)
-     ++failures;
-   if ((i2 <= ci4) != true)
-     ++failures;
-   if ((i2 >= ci4) != true)
-     ++failures;
+   VERIFY( (i2 == ci4) == true );
+   VERIFY( (i2 < ci4) == false );
+   VERIFY( (i2 > ci4) == false );
+   VERIFY( (i2 <= ci4) == true );
+   VERIFY( (i2 >= ci4) == true );
 
    const std::vector<int> cv(v);
    std::vector<int>::const_iterator ci5(cv.begin());
-   if (ci5[0] != 1)
-      ++failures;
+   VERIFY( ci5[0] == 1 );
 
    std::vector<std::string> vs;
    vs.push_back(std::string("abc"));
    std::vector<std::string>::iterator ivs(vs.begin());
-   if (ivs->c_str()[1] != 'b')
-      ++failures;
-
-   return failures;
+   VERIFY( ivs->c_str()[1] == 'b' );
 }
 
-int 
+void 
 reverse_stuff()
 {
-   int failures(0);
-
    std::string s("abcde");
 
    std::string::reverse_iterator ri(s.rbegin());
-   if (*ri != 'e')
-      ++failures;
+   VERIFY( *ri == 'e' );
 
    std::iterator_traits<std::string::reverse_iterator>::difference_type d;
    d = s.rend() - ri;
-   if (d != 5)
-      ++failures;
+   VERIFY( d == 5 );
 
    const std::string cs("abcde");
    std::string::const_reverse_iterator cri(cs.rend());
-   if (cri - 5 != cs.rbegin())
-      ++failures;
-
-   return failures;
-}
-
-// the following should be compiler errors
-// flag runtime errors in case they slip through the compiler
-int 
-wrong_stuff()
-{
-   int failures(0);
-
-#ifdef ITER24_F1
-   extern void f(std::vector<std::string*>::iterator);
-   std::vector<std::string*> vs[2];
-   f(vs);                       // address of array is not an iterator
-   failures++;
-#endif
-
-#ifdef ITER24_F2
-   std::string s;
-   char *i = s.begin();         // begin() doesn't return a pointer
-   failures++;
-#endif
-
-#ifdef ITER24_F3
-   std::string::const_iterator ci;
-   std::string::iterator i;
-   if (i - ci)                  // remove const_ is a warning
-      i++;
-   // failures++;  only a warning
-#endif
-
-#ifdef ITER24_F4
-   std::vector<char>::iterator iv;
-   std::string::iterator is(iv);// vector<char> is not string
-   failures++;
-#endif
-
-#ifdef ITER24_F5
-   std::vector<char>::iterator iv;
-   std::string::iterator is;
-   if (iv == is)                // vector<char> is not string
-      ++iv;
-   failures++;
-#endif
-
-#ifdef ITER24_F6
-   std::vector<char>::const_iterator ci;
-   std::vector<char>::iterator i = ci;  // remove const_ is a warning
-   ++i;
-   // failures++; only a warning
-#endif
-
-#ifdef ITER24_F7
-   std::vector<int> v(1);
-   std::vector<int>::const_iterator ci(v.begin());
-   *ci = 1;                     // cannot assign through const_iterator
-   failures++;
-#endif
-
-#ifdef ITER24_F8
-   std::vector<const int> v(1);
-   std::vector<const int>::reference r(v.begin()[0]);
-   r = 1;                       // cannot assign through reference to const
-   failures++;
-#endif
-
-   return failures;
+   VERIFY( cri - 5 == cs.rbegin() );
 }
 
 // libstdc++/6642
-int
+void
 test6642()
 {
    std::string s;
    std::string::iterator it = s.begin();
    std::string::const_iterator cit = s.begin();
-
-   return it - cit;
+   VERIFY( (it - cit) == 0 );
 }
 
 int
 main()
 {
-   int failures(0);
-
-   failures += string_stuff();
-
-   failures += vector_stuff();
-
-   failures += reverse_stuff();
-
-   failures += wrong_stuff();
-
-   failures += test6642();
-
-   VERIFY(failures == 0);
+   string_stuff();
+   vector_stuff();
+   reverse_stuff();
    return 0;
 }
diff --git a/libstdc++-v3/testsuite/24_iterators/range_generators/lwg3900.cc b/libstdc++-v3/testsuite/24_iterators/range_generators/lwg3900.cc
index 957879e..08fd5c2 100644
--- a/libstdc++-v3/testsuite/24_iterators/range_generators/lwg3900.cc
+++ b/libstdc++-v3/testsuite/24_iterators/range_generators/lwg3900.cc
@@ -13,4 +13,5 @@ bar(std::allocator_arg_t, std::pmr::memory_resource& mr) // { dg-error "here" }
 }
 
 // { dg-error "static assertion failed" "" { target *-*-* } 0 }
-// { dg-error "no matching function .*memory_resource&" "" { target *-*-* } 0 }
+// { dg-error "could not convert 'const std::pmr::memory_resource'" "" { target *-*-* } 0 }
+// { dg-error "no matching function \[^\n\r\]*memory_resource&" "" { target *-*-* } 0 }
diff --git a/libstdc++-v3/testsuite/25_algorithms/copy/debug/constexpr_neg.cc b/libstdc++-v3/testsuite/25_algorithms/copy/debug/constexpr_neg.cc
index 0e80977..3840524 100644
--- a/libstdc++-v3/testsuite/25_algorithms/copy/debug/constexpr_neg.cc
+++ b/libstdc++-v3/testsuite/25_algorithms/copy/debug/constexpr_neg.cc
@@ -33,7 +33,7 @@ test1()
 }
 
 static_assert(test1()); // { dg-error "non-constant condition" }
-// { dg-error "_Error_formatter::_M_error()" "" { target *-*-* } 0 }
+// { dg-error "_Error_formatter::(_M_error|_S_at)" "" { target *-*-* } 0 }
 
 constexpr bool
 test2()
diff --git a/libstdc++-v3/testsuite/25_algorithms/copy_backward/debug/constexpr_neg.cc b/libstdc++-v3/testsuite/25_algorithms/copy_backward/debug/constexpr_neg.cc
index 410c235..d5d84b1 100644
--- a/libstdc++-v3/testsuite/25_algorithms/copy_backward/debug/constexpr_neg.cc
+++ b/libstdc++-v3/testsuite/25_algorithms/copy_backward/debug/constexpr_neg.cc
@@ -35,4 +35,4 @@ test()
 
 static_assert(test()); // { dg-error "non-constant condition" }
 
-// { dg-prune-output "_Error_formatter::_M_error()" }
+// { dg-prune-output "_Error_formatter::(_M_error|_S_at)" }
diff --git a/libstdc++-v3/testsuite/25_algorithms/equal/debug/constexpr_neg.cc b/libstdc++-v3/testsuite/25_algorithms/equal/debug/constexpr_neg.cc
index cbc7509..6c1531d 100644
--- a/libstdc++-v3/testsuite/25_algorithms/equal/debug/constexpr_neg.cc
+++ b/libstdc++-v3/testsuite/25_algorithms/equal/debug/constexpr_neg.cc
@@ -32,7 +32,7 @@ test01()
 }
 
 static_assert(test01()); // { dg-error "non-constant condition" }
-// { dg-error "_Error_formatter::_M_error()" "" { target *-*-* } 0 }
+// { dg-error "_Error_formatter::(_M_error|_S_at)" "" { target *-*-* } 0 }
 
 constexpr bool
 test02()
diff --git a/libstdc++-v3/testsuite/25_algorithms/lower_bound/debug/constexpr_partitioned_neg.cc b/libstdc++-v3/testsuite/25_algorithms/lower_bound/debug/constexpr_partitioned_neg.cc
index c07145c..b44cb4b 100644
--- a/libstdc++-v3/testsuite/25_algorithms/lower_bound/debug/constexpr_partitioned_neg.cc
+++ b/libstdc++-v3/testsuite/25_algorithms/lower_bound/debug/constexpr_partitioned_neg.cc
@@ -43,5 +43,5 @@ test()
 
 static_assert(test()); // { dg-error "" }
 
-// { dg-prune-output "_Error_formatter::_M_error()" }
+// { dg-prune-output "_Error_formatter::(_M_error|_S_at)" }
 // { dg-prune-output "in 'constexpr'" }
diff --git a/libstdc++-v3/testsuite/25_algorithms/lower_bound/debug/constexpr_partitioned_pred_neg.cc b/libstdc++-v3/testsuite/25_algorithms/lower_bound/debug/constexpr_partitioned_pred_neg.cc
index 09ae26f..7835b30 100644
--- a/libstdc++-v3/testsuite/25_algorithms/lower_bound/debug/constexpr_partitioned_pred_neg.cc
+++ b/libstdc++-v3/testsuite/25_algorithms/lower_bound/debug/constexpr_partitioned_pred_neg.cc
@@ -33,4 +33,4 @@ test()
 
 static_assert(test()); // { dg-error "" }
 
-// { dg-prune-output "_Error_formatter::_M_error()" }
+// { dg-prune-output "_Error_formatter::(_M_error|_S_at)" }
diff --git a/libstdc++-v3/testsuite/25_algorithms/lower_bound/debug/constexpr_valid_range_neg.cc b/libstdc++-v3/testsuite/25_algorithms/lower_bound/debug/constexpr_valid_range_neg.cc
index 20eb026..911880b 100644
--- a/libstdc++-v3/testsuite/25_algorithms/lower_bound/debug/constexpr_valid_range_neg.cc
+++ b/libstdc++-v3/testsuite/25_algorithms/lower_bound/debug/constexpr_valid_range_neg.cc
@@ -46,5 +46,5 @@ test2()
 
 static_assert(test2()); // { dg-error "" }
 
-// { dg-prune-output "_Error_formatter::_M_error()" }
+// { dg-prune-output "_Error_formatter::(_M_error|_S_at)" }
 
diff --git a/libstdc++-v3/testsuite/25_algorithms/shift_left/constrained.cc b/libstdc++-v3/testsuite/25_algorithms/shift_left/constrained.cc
new file mode 100644
index 0000000..73d0614
--- /dev/null
+++ b/libstdc++-v3/testsuite/25_algorithms/shift_left/constrained.cc
@@ -0,0 +1,105 @@
+// Copyright (C) 2020-2025 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-do run { target c++23 } }
+// This test is based on shift_left/1.cc.
+
+#include <algorithm>
+#include <testsuite_hooks.h>
+#include <testsuite_iterators.h>
+
+using __gnu_test::test_range;
+using __gnu_test::forward_iterator_wrapper;
+using __gnu_test::bidirectional_iterator_wrapper;
+using __gnu_test::random_access_iterator_wrapper;
+
+struct X
+{
+  int a = -1;
+  bool moved_from = false;
+
+  X() = default;
+
+  X(int a)
+    : a(a)
+  { }
+
+  X(const X&) = delete;
+  X& operator=(const X&) = delete;
+
+  X(X&& other)
+  {
+    if (this != &other)
+    *this = std::move(other);
+  }
+
+  X&
+  operator=(X&& other)
+  {
+    a = other.a;
+    other.moved_from = true;
+    moved_from = false;
+    return *this;
+  }
+};
+
+template<int N, template<typename> typename Wrapper>
+void
+test01()
+{
+  for (int n = 0; n < N+5; n++)
+    {
+      X x[N];
+      for (int i = 0; i < N; i++)
+	x[i] = X{i};
+      test_range<X, Wrapper> rx(x);
+      auto [first,out] = std::ranges::shift_left(rx.begin(), rx.end(), n);
+      VERIFY( first == rx.begin() );
+      if (n < N)
+	{
+	  VERIFY( out.ptr == x+(N-n) );
+	  for (int i = 0; i < N-n; i++)
+	    {
+	      VERIFY( x[i].a == n+i );
+	      VERIFY( !x[i].moved_from );
+	    }
+	  for (int i = std::max(n, N-n); i < N; i++)
+	    VERIFY( x[i].moved_from );
+	}
+      else
+	{
+	  VERIFY( out.ptr == x );
+	  for (int i = 0; i < N; i++)
+	    {
+	      VERIFY( x[i].a == i );
+	      VERIFY( !x[i].moved_from );
+	    }
+	}
+    }
+}
+
+int
+main()
+{
+  test01<23, forward_iterator_wrapper>();
+  test01<23, bidirectional_iterator_wrapper>();
+  test01<23, random_access_iterator_wrapper>();
+
+  test01<24, forward_iterator_wrapper>();
+  test01<24, bidirectional_iterator_wrapper>();
+  test01<24, random_access_iterator_wrapper>();
+}
diff --git a/libstdc++-v3/testsuite/25_algorithms/shift_right/constrained.cc b/libstdc++-v3/testsuite/25_algorithms/shift_right/constrained.cc
new file mode 100644
index 0000000..0d53707
--- /dev/null
+++ b/libstdc++-v3/testsuite/25_algorithms/shift_right/constrained.cc
@@ -0,0 +1,104 @@
+// Copyright (C) 2020-2025 Free Software Foundation, Inc.
+//
+// This file is part of the GNU ISO C++ Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License along
+// with this library; see the file COPYING3.  If not see
+// <http://www.gnu.org/licenses/>.
+
+// { dg-do run { target c++23 } }
+// This test is based on shift_right/1.cc.
+
+#include <algorithm>
+#include <testsuite_hooks.h>
+#include <testsuite_iterators.h>
+
+using __gnu_test::test_range;
+using __gnu_test::forward_iterator_wrapper;
+using __gnu_test::bidirectional_iterator_wrapper;
+using __gnu_test::random_access_iterator_wrapper;
+
+struct X
+{
+  int a = -1;
+  bool moved_from = false;
+
+  X() = default;
+
+  X(int a)
+    : a(a)
+  { }
+
+  X(const X&) = delete;
+  X& operator=(const X&) = delete;
+
+  X(X&& other)
+  {
+    if (this != &other)
+    *this = std::move(other);
+  }
+
+  X&
+  operator=(X&& other)
+  {
+    a = other.a;
+    other.moved_from = true;
+    moved_from = false;
+    return *this;
+  }
+};
+
+template<int N, template<typename> typename Wrapper>
+void
+test01()
+{
+  for (int n = 0; n < N+5; n++)
+    {
+      X x[N];
+      for (int i = 0; i < N; i++)
+	x[i] = X(i);
+      test_range<X, Wrapper> rx(x);
+      auto [out,last] = std::ranges::shift_right(rx.begin(), rx.end(), n);
+      VERIFY( last == rx.end() );
+      if (n < N)
+	{
+	  VERIFY( out.ptr == x+n );
+	  for (int i = n; i < N; i++)
+	    VERIFY( x[i].a == i-n );
+	  for (int i = 0; i < std::min(n, N-n); i++)
+	    VERIFY( x[i].moved_from );
+	  for (int i = std::min(n, N-n); i < std::max(n, N-n); i++)
+	    VERIFY( !x[i].moved_from );
+	}
+      else
+	{
+	  VERIFY( out.ptr == x+N );
+	  for (int i = 0; i < N; i++)
+	    {
+	      VERIFY( x[i].a == i );
+	      VERIFY( !x[i].moved_from );
+	    }
+	}
+    }
+}
+
+int
+main()
+{
+  test01<23, forward_iterator_wrapper>();
+  test01<23, bidirectional_iterator_wrapper>();
+  test01<23, random_access_iterator_wrapper>();
+
+  test01<24, forward_iterator_wrapper>();
+  test01<24, bidirectional_iterator_wrapper>();
+  test01<24, random_access_iterator_wrapper>();
+}
diff --git a/libstdc++-v3/testsuite/25_algorithms/upper_bound/debug/constexpr_partitioned_neg.cc b/libstdc++-v3/testsuite/25_algorithms/upper_bound/debug/constexpr_partitioned_neg.cc
index ffe5d77..950d432 100644
--- a/libstdc++-v3/testsuite/25_algorithms/upper_bound/debug/constexpr_partitioned_neg.cc
+++ b/libstdc++-v3/testsuite/25_algorithms/upper_bound/debug/constexpr_partitioned_neg.cc
@@ -43,4 +43,4 @@ test()
 
 static_assert(test()); // { dg-error "" }
 
-// { dg-prune-output "_Error_formatter::_M_error()" }
+// { dg-prune-output "_Error_formatter::(_M_error|_S_at)" }
diff --git a/libstdc++-v3/testsuite/25_algorithms/upper_bound/debug/constexpr_partitioned_pred_neg.cc b/libstdc++-v3/testsuite/25_algorithms/upper_bound/debug/constexpr_partitioned_pred_neg.cc
index a1c2ce6..41eb00c 100644
--- a/libstdc++-v3/testsuite/25_algorithms/upper_bound/debug/constexpr_partitioned_pred_neg.cc
+++ b/libstdc++-v3/testsuite/25_algorithms/upper_bound/debug/constexpr_partitioned_pred_neg.cc
@@ -33,4 +33,4 @@ test()
 
 static_assert(test()); // { dg-error "" }
 
-// { dg-prune-output "_Error_formatter::_M_error()" }
+// { dg-prune-output "_Error_formatter::(_M_error|_S_at)" }
diff --git a/libstdc++-v3/testsuite/25_algorithms/upper_bound/debug/constexpr_valid_range_neg.cc b/libstdc++-v3/testsuite/25_algorithms/upper_bound/debug/constexpr_valid_range_neg.cc
index c7c9e3b..032c8d3 100644
--- a/libstdc++-v3/testsuite/25_algorithms/upper_bound/debug/constexpr_valid_range_neg.cc
+++ b/libstdc++-v3/testsuite/25_algorithms/upper_bound/debug/constexpr_valid_range_neg.cc
@@ -46,4 +46,4 @@ test2()
 
 static_assert(test2()); // { dg-error "" }
 
-// { dg-prune-output "_Error_formatter::_M_error()" }
+// { dg-prune-output "_Error_formatter::(_M_error|_S_at)" }
diff --git a/libstdc++-v3/testsuite/29_atomics/atomic/requirements/types_neg.cc b/libstdc++-v3/testsuite/29_atomics/atomic/requirements/types_neg.cc
index 5aa3243..cfe44d2 100644
--- a/libstdc++-v3/testsuite/29_atomics/atomic/requirements/types_neg.cc
+++ b/libstdc++-v3/testsuite/29_atomics/atomic/requirements/types_neg.cc
@@ -20,6 +20,7 @@
 #include <atomic>
 
 std::atomic<const int> a; // { dg-error "here" }
+// { dg-error "assignment to read-only type" "" { target *-*-* } 0 }
 
 struct MoveOnly
 {
@@ -40,3 +41,4 @@ struct NoMove
 std::atomic<NoMove> c; // { dg-error "here" }
 
 // { dg-error "static assertion failed" "" { target *-*-* } 0 }
+// { dg-error "use of deleted function" "" { target *-*-* } 0 }
diff --git a/libstdc++-v3/testsuite/30_threads/stop_token/stop_callback/destructible_neg.cc b/libstdc++-v3/testsuite/30_threads/stop_token/stop_callback/destructible_neg.cc
index 2c5125b..cdbb2f4 100644
--- a/libstdc++-v3/testsuite/30_threads/stop_token/stop_callback/destructible_neg.cc
+++ b/libstdc++-v3/testsuite/30_threads/stop_token/stop_callback/destructible_neg.cc
@@ -54,3 +54,4 @@ test02(std::stop_token& tok, G& g)
 }
 
 // { dg-error "static assertion failed" "" { target *-*-* } 0 }
+// { dg-error "private within this context" "" { target *-*-* } 0 }
diff --git a/libstdc++-v3/testsuite/30_threads/stop_token/stop_callback/invocable_neg.cc b/libstdc++-v3/testsuite/30_threads/stop_token/stop_callback/invocable_neg.cc
index 93b3186..2b2fce4 100644
--- a/libstdc++-v3/testsuite/30_threads/stop_token/stop_callback/invocable_neg.cc
+++ b/libstdc++-v3/testsuite/30_threads/stop_token/stop_callback/invocable_neg.cc
@@ -32,3 +32,4 @@ test01(std::stop_token& tok, F& f)
 }
 
 // { dg-error "static assertion failed" "" { target *-*-* } 0 }
+// { dg-error "no match for call" "" { target *-*-* } 0 }
diff --git a/libstdc++-v3/testsuite/experimental/memory/observer_ptr/make_observer.cc b/libstdc++-v3/testsuite/experimental/memory/observer_ptr/make_observer.cc
index 048735f..1de9cf0 100644
--- a/libstdc++-v3/testsuite/experimental/memory/observer_ptr/make_observer.cc
+++ b/libstdc++-v3/testsuite/experimental/memory/observer_ptr/make_observer.cc
@@ -20,12 +20,19 @@
 #include <experimental/memory>
 #include <testsuite_hooks.h>
 
-int main()
+constexpr bool test()
 {
   const int i = 42;
   auto o = std::experimental::make_observer(&i);
   static_assert( std::is_same<decltype(o),
-                 std::experimental::observer_ptr<const int>>(), "" );
+		   std::experimental::observer_ptr<const int>>(), "" );
   VERIFY( o && *o == 42 );
   VERIFY( o.get() == &i );
+  return true;
+}
+
+int main()
+{
+  test();
+  static_assert( test(), "LWG 4295 - make_observer should be constexpr" );
 }
diff --git a/libstdc++-v3/testsuite/experimental/memory/observer_ptr/relops/relops.cc b/libstdc++-v3/testsuite/experimental/memory/observer_ptr/relops/relops.cc
index 3e23e0b..d03dd5d 100644
--- a/libstdc++-v3/testsuite/experimental/memory/observer_ptr/relops/relops.cc
+++ b/libstdc++-v3/testsuite/experimental/memory/observer_ptr/relops/relops.cc
@@ -22,13 +22,13 @@
 
 using std::experimental::observer_ptr;
 
-void test01()
+constexpr void test01()
 {
   observer_ptr<int> a, b;
   VERIFY(a == b);
 }
 
-void test02()
+constexpr void test02()
 {
   int x[2]{};
   observer_ptr<int> a{&x[0]};
@@ -40,7 +40,7 @@ void test02()
   VERIFY(b > a);
 }
 
-void test03()
+constexpr void test03()
 {
   int x{};
   observer_ptr<int> a{&x};
@@ -48,9 +48,10 @@ void test03()
   VERIFY(a == b);
 }
 
-void test04()
+int x[2]{};
+
+constexpr void test04()
 {
-  static constexpr int x[2]{};
   constexpr observer_ptr<const int> a{&x[0]};
   constexpr observer_ptr<const int> b{&x[1]};
   VERIFY(a != b);
@@ -60,20 +61,25 @@ void test04()
   VERIFY(b > a);
 }
 
-void test05()
+constexpr void test05()
 {
-  static constexpr int x{};
-  constexpr observer_ptr<const int> a{&x};
-  constexpr observer_ptr<const int> b{&x};
+  constexpr observer_ptr<const int> a{&x[0]};
+  constexpr observer_ptr<const int> b{&x[0]};
   VERIFY(a == b);
 }
 
-
-int main()
+constexpr bool all_tests()
 {
   test01();
   test02();
   test03();
   test04();
   test05();
+  return true;
+}
+
+int main()
+{
+  all_tests();
+  static_assert( all_tests(), "LWG 4295 - relops should be constexpr" );
 }
diff --git a/libstdc++-v3/testsuite/experimental/memory/observer_ptr/swap/swap.cc b/libstdc++-v3/testsuite/experimental/memory/observer_ptr/swap/swap.cc
index 9e76788..84b8844 100644
--- a/libstdc++-v3/testsuite/experimental/memory/observer_ptr/swap/swap.cc
+++ b/libstdc++-v3/testsuite/experimental/memory/observer_ptr/swap/swap.cc
@@ -25,7 +25,7 @@ using std::experimental::observer_ptr;
 struct B {};
 struct D : B {};
 
-void test01()
+constexpr void test01()
 {
   observer_ptr<int> a, b;
   VERIFY(a == b);
@@ -33,7 +33,7 @@ void test01()
   VERIFY(a == b);
 }
 
-void test02()
+constexpr void test02()
 {
   int x{};
   observer_ptr<int> a;
@@ -45,7 +45,7 @@ void test02()
   VERIFY(!b);
 }
 
-void test03()
+constexpr void test03()
 {
   int x[2]{1,2};
   observer_ptr<int> a{&x[0]};
@@ -57,10 +57,16 @@ void test03()
   VERIFY(*b == 1);
 }
 
-
 int main()
 {
-  test01();
-  test02();
-  test03();
+  auto tests = [] {
+    test01();
+    test02();
+    test03();
+    return true;
+  };
+  tests();
+#if __cpp_lib_constexpr_algorithms >= 201806L // >= C++20
+  static_assert( tests(), "LWG 4295 - swap should be constexpr" );
+#endif
 }
diff --git a/libstdc++-v3/testsuite/ext/unicode/view.cc b/libstdc++-v3/testsuite/ext/unicode/view.cc
index 6f3c099..4ccf646 100644
--- a/libstdc++-v3/testsuite/ext/unicode/view.cc
+++ b/libstdc++-v3/testsuite/ext/unicode/view.cc
@@ -7,13 +7,24 @@
 namespace uc = std::__unicode;
 using namespace std::string_view_literals;
 
+template<std::ranges::range View>
+constexpr void
+compare(View v, std::basic_string_view<std::ranges::range_value_t<View>> s)
+{
+  long size = s.size();
+  VERIFY( std::ranges::distance(v) == size );
+  VERIFY( std::ranges::equal(v,  s) );
+  auto rev = std::views::reverse(v);
+  VERIFY( std::ranges::distance(rev) == size );
+  VERIFY( std::ranges::equal(rev,  s | std::views::reverse) );
+}
+
 constexpr void
 test_utf8_to_utf8()
 {
   const auto s8 = u8"£🇬🇧 €🇪🇺 æбçδé ♠♥♦♣ 🤡"sv;
   uc::_Utf8_view v(s8);
-  VERIFY( std::ranges::distance(v) == s8.size() );
-  VERIFY( std::ranges::equal(v,  s8) );
+  compare(v, s8);
 }
 
 constexpr void
@@ -22,8 +33,7 @@ test_utf8_to_utf16()
   const auto s8  = u8"£🇬🇧 €🇪🇺 æбçδé ♠♥♦♣ 🤡"sv;
   const std::u16string_view s16 = u"£🇬🇧 €🇪🇺 æбçδé ♠♥♦♣ 🤡";
   uc::_Utf16_view v(s8);
-  VERIFY( std::ranges::distance(v) == s16.size() );
-  VERIFY( std::ranges::equal(v,  s16) );
+  compare(v, s16);
 }
 
 constexpr void
@@ -32,36 +42,41 @@ test_utf8_to_utf32()
   const auto s8 = u8"£🇬🇧 €🇪🇺 æбçδé ♠♥♦♣ 🤡"sv;
   const auto s32 = U"£🇬🇧 €🇪🇺 æбçδé ♠♥♦♣ 🤡"sv;
   uc::_Utf32_view v(s8);
-  VERIFY( std::ranges::distance(v) == s32.size() );
-  VERIFY( std::ranges::equal(v,  s32) );
+  compare(v, s32);
 }
 
 constexpr void
 test_illformed_utf8()
 {
   uc::_Utf32_view v("\xa3 10.99 \xee \xdd"sv);
-  VERIFY( std::ranges::equal(v, U"\uFFFD 10.99 \uFFFD \uFFFD"sv) );
+  compare(v, U"\uFFFD 10.99 \uFFFD \uFFFD"sv);
 
   uc::_Utf16_view v2(" \xf8\x80\x80\x80 "sv);
-  VERIFY( std::ranges::distance(v2) == 6 );
-  VERIFY( std::ranges::equal(v2, U" \uFFFD\uFFFD\uFFFD\uFFFD "sv) );
+  compare(v2, u" \uFFFD\uFFFD\uFFFD\uFFFD "sv);
 
   // Examples of U+FFFD substitution from Unicode standard.
   uc::_Utf8_view v3("\xc0\xaf\xe0\x80\xbf\xf0\x81\x82\x41"sv); // Table 3-8
-  VERIFY( std::ranges::equal(v3, u8"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\x41"sv) );
+  compare(v3, u8"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\x41"sv);
   uc::_Utf8_view v4("\xed\xa0\x80\xed\xbf\xbf\xed\xaf\x41"sv); // Table 3-9
-  VERIFY( std::ranges::equal(v4, u8"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\x41"sv) );
+  compare(v4, u8"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\x41"sv);
   uc::_Utf8_view v5("\xf4\x91\x92\x93\xff\x41\x80\xbf\x42"sv); // Table 3-10
-  VERIFY( std::ranges::equal(v5, u8"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\x41\uFFFD\uFFFD\x42"sv) );
+  compare(v5, u8"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\x41\uFFFD\uFFFD\x42"sv);
   uc::_Utf8_view v6("\xe1\x80\xe2\xf0\x91\x92\xf1\xbf\x41"sv); // Table 3-11
-  VERIFY( std::ranges::equal(v6, u8"\uFFFD\uFFFD\uFFFD\uFFFD\x41"sv) );
+  compare(v6, u8"\uFFFD\uFFFD\uFFFD\uFFFD\x41"sv);
 
   uc::_Utf32_view v7("\xe1\x80"sv);
-  VERIFY( std::ranges::equal(v7, U"\uFFFD"sv) );
+  compare(v7,  U"\uFFFD"sv);
   uc::_Utf32_view v8("\xf1\x80"sv);
-  VERIFY( std::ranges::equal(v8, U"\uFFFD"sv) );
+  compare(v8,  U"\uFFFD"sv);
   uc::_Utf32_view v9("\xf1\x80\x80"sv);
-  VERIFY( std::ranges::equal(v9, U"\uFFFD"sv) );
+  compare(v9,  U"\uFFFD"sv);
+
+  uc::_Utf32_view v10("\xcf\x80\x80\x81\x82\x83 \x84\x85\x86\x87\x88 "sv);
+  compare(v10,  U"\u03C0\uFFFD\uFFFD\uFFFD\uFFFD \uFFFD\uFFFD\uFFFD\uFFFD\uFFFD "sv);
+  uc::_Utf16_view v11("\xcf\x80\x80\x81\x82\x83 \x84\x85\x86\x87\x88 "sv);
+  compare(v11,  u"\u03C0\uFFFD\uFFFD\uFFFD\uFFFD \uFFFD\uFFFD\uFFFD\uFFFD\uFFFD "sv);
+  uc::_Utf8_view v12("\xcf\x80\x80\x81\x82\x83 \x84\x85\x86\x87\x88 "sv);
+  compare(v12,  u8"\u03C0\uFFFD\uFFFD\uFFFD\uFFFD \uFFFD\uFFFD\uFFFD\uFFFD\uFFFD "sv);
 }
 
 constexpr void
@@ -69,27 +84,27 @@ test_illformed_utf16()
 {
   std::u16string_view s = u"\N{CLOWN FACE}";
   std::u16string_view r = u"\uFFFD";
-  VERIFY( std::ranges::equal(uc::_Utf16_view(s.substr(0, 1)), r) );
-  VERIFY( std::ranges::equal(uc::_Utf16_view(s.substr(1, 1)), r) );
+  compare(uc::_Utf16_view(s.substr(0, 1)), r);
+  compare(uc::_Utf16_view(s.substr(1, 1)), r);
   std::array s2{ s[0], s[0] };
-  VERIFY( std::ranges::equal(uc::_Utf16_view(s2), u"\uFFFD\uFFFD"sv) );
+  compare(uc::_Utf16_view(s2), u"\uFFFD\uFFFD"sv);
   std::array s3{ s[0], s[0], s[1] };
-  VERIFY( std::ranges::equal(uc::_Utf16_view(s3), u"\uFFFD\N{CLOWN FACE}"sv) );
+  compare(uc::_Utf16_view(s3), u"\uFFFD\N{CLOWN FACE}"sv);
   std::array s4{ s[1], s[0] };
-  VERIFY( std::ranges::equal(uc::_Utf16_view(s4), u"\uFFFD\uFFFD"sv) );
+  compare(uc::_Utf16_view(s4), u"\uFFFD\uFFFD"sv);
   std::array s5{ s[1], s[0], s[1] };
-  VERIFY( std::ranges::equal(uc::_Utf16_view(s5), u"\uFFFD\N{CLOWN FACE}"sv) );
+  compare(uc::_Utf16_view(s5), u"\uFFFD\N{CLOWN FACE}"sv);
 }
 
 constexpr void
 test_illformed_utf32()
 {
   std::u32string_view s = U"\x110000";
-  VERIFY( std::ranges::equal(uc::_Utf32_view(s), U"\uFFFD"sv) );
+  compare(uc::_Utf32_view(s), U"\uFFFD"sv);
   s = U"\xFFFFFF";
-  VERIFY( std::ranges::equal(uc::_Utf32_view(s), U"\uFFFD"sv) );
+  compare(uc::_Utf32_view(s), U"\uFFFD"sv);
   s = U"\xFFFFFFF0";
-  VERIFY( std::ranges::equal(uc::_Utf32_view(s), U"\uFFFD"sv) );
+  compare(uc::_Utf32_view(s), U"\uFFFD"sv);
 }
 
 constexpr void
@@ -110,6 +125,13 @@ test_past_the_end()
   iter++;
   VERIFY( iter == v.end() );
   VERIFY( *iter == U'4' );
+  std::ranges::advance(iter, -4);
+  VERIFY( *iter == U'1' );
+  // Incrementing before begin has well-defined behaviour.
+  iter--;
+  VERIFY( *iter == U'1' );
+  iter--;
+  VERIFY( *iter == U'1' );
 
   std::string_view empty;
   uc::_Utf32_view v2(empty);
@@ -119,6 +141,9 @@ test_past_the_end()
   iter++;
   VERIFY( iter2 == v2.end() );
   VERIFY( *iter2 == U'\0' );
+  iter--;
+  VERIFY( iter2 == v2.end() );
+  VERIFY( *iter2 == U'\0' );
 }
 
 int main()
diff --git a/libstdc++-v3/testsuite/ext/verify_neg.cc b/libstdc++-v3/testsuite/ext/verify_neg.cc
new file mode 100644
index 0000000..ce03374
--- /dev/null
+++ b/libstdc++-v3/testsuite/ext/verify_neg.cc
@@ -0,0 +1,28 @@
+// { dg-do compile { target c++11 } }
+
+#include <testsuite_hooks.h>
+
+struct X { explicit operator void*() const { return nullptr; } };
+
+void
+test_VERIFY(int i)
+{
+  // This should not be parsed as a function type bool(bool(i)):
+  VERIFY( bool(i) );
+
+  // This should not produce warnings about lambda in unevaluated context:
+  VERIFY( []{ return 1; }() );
+
+  // Only one expression allowed:
+  VERIFY(1, 2); // { dg-error "in expansion of macro" }
+  // { dg-error "compound expression in functional cast" "" { target *-*-* } 0 }
+
+  // A scoped enum is not contextually convertible to bool:
+  enum class E { E0 };
+  VERIFY( E::E0 ); // { dg-error "could not convert" }
+
+  // explicit conversion to void* is not contextually convertible to bool:
+  X x;
+  VERIFY( x ); // { dg-error "in expansion of macro" }
+  // { dg-error "invalid cast .* to type 'bool'" "" { target *-*-* } 0 }
+}
diff --git a/libstdc++-v3/testsuite/std/format/arguments/args_neg.cc b/libstdc++-v3/testsuite/std/format/arguments/args_neg.cc
index ded56fe..83c7b22 100644
--- a/libstdc++-v3/testsuite/std/format/arguments/args_neg.cc
+++ b/libstdc++-v3/testsuite/std/format/arguments/args_neg.cc
@@ -42,3 +42,4 @@ void test_const_arg()
 }
 
 // { dg-prune-output "no matching function for call to .*::basic_format_arg<" }
+// { dg-prune-output "use of deleted function" }
diff --git a/libstdc++-v3/testsuite/std/format/ranges/format_kind.cc b/libstdc++-v3/testsuite/std/format/ranges/format_kind.cc
index 14b9ff2..1450fba 100644
--- a/libstdc++-v3/testsuite/std/format/ranges/format_kind.cc
+++ b/libstdc++-v3/testsuite/std/format/ranges/format_kind.cc
@@ -75,7 +75,9 @@ void test_override()
   CustFormat<int, std::range_format::set> setf{1, 2, 3};
   VERIFY( std::format("{}", setf) == "{1, 2, 3}" );
 
-  // TODO test map once formatter for pair is implenented
+  CustFormat<std::pair<int, int>, std::range_format::map> mapf
+  {{1, 11}, {2, 22}, {3, 33}};
+  VERIFY( std::format("{}", mapf) == "{1: 11, 2: 22, 3: 33}" );
 
   CustFormat<char, std::range_format::string> stringf{'a', 'b', 'c', 'd'};
   VERIFY( std::format("{}", stringf) == "abcd" );
diff --git a/libstdc++-v3/testsuite/std/format/ranges/pr119861_neg.cc b/libstdc++-v3/testsuite/std/format/ranges/pr119861_neg.cc
new file mode 100644
index 0000000..9a6ed16
--- /dev/null
+++ b/libstdc++-v3/testsuite/std/format/ranges/pr119861_neg.cc
@@ -0,0 +1,52 @@
+// { dg-do compile { target c++23 } }
+
+#include <format>
+#include <vector>
+
+// only format_kind::sequence provides set_brackets and set_separator methods 
+
+template<std::range_format fk, typename T>
+struct MyCont : std::vector<T>
+{
+  using std::vector<T>::vector;
+};
+
+template<std::range_format fk, typename T>
+constexpr std::range_format std::format_kind<MyCont<fk, T>> = fk;
+
+void test_sequence()
+{
+  std::formatter<MyCont<std::range_format::sequence, int>, char> fmtter;
+  fmtter.set_brackets("{", "}");
+  fmtter.set_separator(",");
+}
+
+void test_map()
+{
+  std::formatter<MyCont<std::range_format::map, std::pair<int, int>>, char> fmtter;
+  fmtter.set_brackets("{", "}"); // { dg-error "here" }
+  fmtter.set_separator(","); // { dg-error "here" }
+}
+
+void test_set()
+{
+  std::formatter<MyCont<std::range_format::set, int>, char> fmtter;
+  fmtter.set_brackets("{", "}"); // { dg-error "here" }
+  fmtter.set_separator(","); // { dg-error "here" }
+}
+
+void test_string()
+{
+  std::formatter<MyCont<std::range_format::string, char>, char> fmtter;
+  fmtter.set_brackets("{", "}"); // { dg-error "here" }
+  fmtter.set_separator(","); // { dg-error "here" }
+}
+
+void test_debug_string()
+{
+  std::formatter<MyCont<std::range_format::debug_string, char>, char> fmtter;
+  fmtter.set_brackets("{", "}"); // { dg-error "here" }
+  fmtter.set_separator(","); // { dg-error "here" }
+}
+
+// { dg-error "no matching function for call to 'std::formatter<" "" { target *-*-* } 0 }
diff --git a/libstdc++-v3/testsuite/std/format/string_neg.cc b/libstdc++-v3/testsuite/std/format/string_neg.cc
index 09cc9a2..acae88e 100644
--- a/libstdc++-v3/testsuite/std/format/string_neg.cc
+++ b/libstdc++-v3/testsuite/std/format/string_neg.cc
@@ -8,3 +8,5 @@ auto s = std::format(" {9} "); // { dg-error "call to consteval function" }
 struct X { };
 std::format_string<X> str(""); // { dg-error "here" }
 // { dg-error "std::formatter must be specialized" "" { target *-*-* } 0 }
+
+// { dg-prune-output "use of deleted function" }
diff --git a/libstdc++-v3/testsuite/std/ranges/adaptors/join.cc b/libstdc++-v3/testsuite/std/ranges/adaptors/join.cc
index 2861115..a9395b4 100644
--- a/libstdc++-v3/testsuite/std/ranges/adaptors/join.cc
+++ b/libstdc++-v3/testsuite/std/ranges/adaptors/join.cc
@@ -233,6 +233,13 @@ test14()
   VERIFY( ranges::equal(v | views::join, (int[]){1, 2, 3}) );
 }
 
+void
+test15()
+{
+  // PR libstdc++/119962 - __maybe_present_t misses initialization
+  constexpr decltype(views::join(views::single(views::single(0))).begin()) it;
+}
+
 int
 main()
 {
@@ -250,4 +257,5 @@ main()
   test12();
   test13();
   test14();
+  test15();
 }
diff --git a/libstdc++-v3/testsuite/std/ranges/adaptors/join_with/1.cc b/libstdc++-v3/testsuite/std/ranges/adaptors/join_with/1.cc
index 8ab30a5..4d55c9d 100644
--- a/libstdc++-v3/testsuite/std/ranges/adaptors/join_with/1.cc
+++ b/libstdc++-v3/testsuite/std/ranges/adaptors/join_with/1.cc
@@ -94,6 +94,13 @@ test04()
   return true;
 }
 
+void
+test05()
+{
+  // PR libstdc++/119962 - __maybe_present_t misses initialization
+  constexpr decltype(views::join_with(views::single(views::single(0)), 0).begin()) it;
+}
+
 int
 main()
 {
@@ -105,4 +112,5 @@ main()
 #else
   VERIFY(test04());
 #endif
+  test05();
 }
diff --git a/libstdc++-v3/testsuite/std/ranges/adaptors/lazy_split.cc b/libstdc++-v3/testsuite/std/ranges/adaptors/lazy_split.cc
index 81fc60b..321ae27 100644
--- a/libstdc++-v3/testsuite/std/ranges/adaptors/lazy_split.cc
+++ b/libstdc++-v3/testsuite/std/ranges/adaptors/lazy_split.cc
@@ -232,6 +232,13 @@ test12()
   return true;
 }
 
+void
+test13()
+{
+  // PR libstdc++/119962 - __maybe_present_t misses initialization
+  constexpr decltype(views::lazy_split(views::single(0), 0).begin()) it;
+}
+
 int
 main()
 {
@@ -247,4 +254,5 @@ main()
   test10();
   test11();
   static_assert(test12());
+  test13();
 }
diff --git a/libstdc++-v3/testsuite/std/ranges/concat/1.cc b/libstdc++-v3/testsuite/std/ranges/concat/1.cc
index 1672191..f78ed08 100644
--- a/libstdc++-v3/testsuite/std/ranges/concat/1.cc
+++ b/libstdc++-v3/testsuite/std/ranges/concat/1.cc
@@ -99,6 +99,18 @@ test04()
   using type = decltype(v);
 }
 
+void
+test05()
+{
+  // PR libstdc++/120934 - views::concat is ill-formed depending on argument order
+  auto v1 = views::single(1);
+  std::vector<int> vec = {2, 3};
+  auto v2 = views::join(views::transform(vec, views::single));
+
+  static_assert( ranges::range<decltype(views::concat(v1, v2))> );
+  static_assert( ranges::range<decltype(views::concat(v2, v1))> );
+}
+
 int
 main()
 {
@@ -107,4 +119,5 @@ main()
   test02();
   test03();
   test04();
+  test05();
 }
diff --git a/libstdc++-v3/testsuite/std/ranges/iota/max_size_type.cc b/libstdc++-v3/testsuite/std/ranges/iota/max_size_type.cc
index 4739d9e..fbd783b 100644
--- a/libstdc++-v3/testsuite/std/ranges/iota/max_size_type.cc
+++ b/libstdc++-v3/testsuite/std/ranges/iota/max_size_type.cc
@@ -352,6 +352,9 @@ static_assert(numeric_limits<max_size_t>::is_specialized);
 static_assert(!numeric_limits<max_size_t>::is_signed);
 static_assert(numeric_limits<max_size_t>::is_integer);
 static_assert(numeric_limits<max_size_t>::is_exact);
+static_assert(numeric_limits<max_size_t>::is_bounded);
+static_assert(numeric_limits<max_size_t>::is_modulo);
+static_assert(numeric_limits<max_size_t>::radix == 2);
 // We can't unconditionally use numeric_limits here because __int128 is an
 // integral type only in GNU mode.
 #if __SIZEOF_INT128__
@@ -379,6 +382,9 @@ static_assert(numeric_limits<max_diff_t>::is_specialized);
 static_assert(numeric_limits<max_diff_t>::is_signed);
 static_assert(numeric_limits<max_diff_t>::is_integer);
 static_assert(numeric_limits<max_diff_t>::is_exact);
+static_assert(numeric_limits<max_diff_t>::is_bounded);
+static_assert(!numeric_limits<max_diff_t>::is_modulo);
+static_assert(numeric_limits<max_diff_t>::radix == 2);
 static_assert(numeric_limits<max_diff_t>::digits
 	      == numeric_limits<max_size_t>::digits - 1);
 static_assert(numeric_limits<max_diff_t>::digits10
@@ -400,6 +406,31 @@ static_assert(max_diff_t(max_size_t(1)
 			 << (numeric_limits<max_size_t>::digits-1))
 	      == numeric_limits<max_diff_t>::min());
 
+template <typename integer_class>
+constexpr bool verify_numeric_limits_values_not_meaningful_for = true
+	&& (numeric_limits<integer_class>::max_digits10 == 0)
+	&& (numeric_limits<integer_class>::min_exponent == 0)
+	&& (numeric_limits<integer_class>::min_exponent10 == 0)
+	&& (numeric_limits<integer_class>::max_exponent == 0)
+	&& (numeric_limits<integer_class>::max_exponent10 == 0)
+	&& !numeric_limits<integer_class>::is_iec559
+	&& !numeric_limits<integer_class>::has_infinity
+	&& !numeric_limits<integer_class>::has_quiet_NaN
+	&& !numeric_limits<integer_class>::has_signaling_NaN
+	&& !numeric_limits<integer_class>::has_denorm_loss
+	&& !numeric_limits<integer_class>::tinyness_before
+	&& (numeric_limits<integer_class>::has_denorm == std::denorm_absent)
+	&& (numeric_limits<integer_class>::round_style == std::round_toward_zero)
+	&& (numeric_limits<integer_class>::denorm_min() == 0)
+	&& (numeric_limits<integer_class>::epsilon() == 0)
+	&& (numeric_limits<integer_class>::round_error() == 0)
+	&& (numeric_limits<integer_class>::infinity() == 0)
+	&& (numeric_limits<integer_class>::quiet_NaN() == 0)
+	&& (numeric_limits<integer_class>::signaling_NaN() == 0);
+
+static_assert(verify_numeric_limits_values_not_meaningful_for<max_size_t>);
+static_assert(verify_numeric_limits_values_not_meaningful_for<max_diff_t>);
+
 // Verify that the types are structural types and can therefore be used
 // as NTTP types.
 template<max_size_t V> struct Su { static_assert(V*V == V+132); };
diff --git a/libstdc++-v3/testsuite/std/time/format/data_not_present_neg.cc b/libstdc++-v3/testsuite/std/time/format/data_not_present_neg.cc
index bb09451..cb8f916 100644
--- a/libstdc++-v3/testsuite/std/time/format/data_not_present_neg.cc
+++ b/libstdc++-v3/testsuite/std/time/format/data_not_present_neg.cc
@@ -119,7 +119,7 @@ auto tai = std::format("{:%Q}", tai_clock::now()); // { dg-error "call to conste
 auto file = std::format("{:%Q}", file_clock::now()); // { dg-error "call to consteval function" }
 
 const auto ltc = local_seconds(10s);
-#if _GLIBCXX_USE_CXX11_ABI || !_GLIBCXX_USE_DUAL_ABI
+#if _GLIBCXX_USE_CXX11_ABI
 const auto zt = zoned_time<seconds>("Europe/Sofia", local_seconds(10s));
 auto zt1 = std::format("{:%Q}", zt); // { dg-error "call to consteval function" "" { target cxx11_abi } }
 #endif
@@ -141,7 +141,7 @@ auto hms5 = std::format("{:%F}", HMS(1255s)); // { dg-error "call to consteval f
 auto hms6 = std::format("{:%Q}", HMS(1255s)); // { dg-error "call to consteval function" }
 auto hms7 = std::format("{:%Z}", HMS(1255s)); // { dg-error "call to consteval function" }
 
-#if _GLIBCXX_USE_CXX11_ABI || !_GLIBCXX_USE_DUAL_ABI
+#if _GLIBCXX_USE_CXX11_ABI
 auto li1 = std::format("{:%d}", local_info()); // { dg-error "call to consteval function" "" { target cxx11_abi } }
 auto li2 = std::format("{:%w}", local_info()); // { dg-error "call to consteval function" "" { target cxx11_abi } }
 auto li3 = std::format("{:%m}", local_info()); // { dg-error "call to consteval function" "" { target cxx11_abi } }
diff --git a/libstdc++-v3/testsuite/std/time/format/format.cc b/libstdc++-v3/testsuite/std/time/format/format.cc
index d6e3583..00affb9 100644
--- a/libstdc++-v3/testsuite/std/time/format/format.cc
+++ b/libstdc++-v3/testsuite/std/time/format/format.cc
@@ -78,6 +78,13 @@ test_bad_format_strings()
   VERIFY( not is_format_string_for("{:%OOy}", t) );
   VERIFY( not is_format_string_for("{:%OEy}", t) );
   VERIFY( not is_format_string_for("{:%EOy}", t) );
+
+  // weekday and month values for which ok() is false
+  VERIFY( not is_format_string_for("{:%a}", std::chrono::weekday(8)) );
+  VERIFY( not is_format_string_for("{:%A}", std::chrono::weekday(8)) );
+  VERIFY( not is_format_string_for("{:%b}", std::chrono::month(13)) );
+  VERIFY( not is_format_string_for("{:%h}", std::chrono::month(13)) );
+  VERIFY( not is_format_string_for("{:%B}", std::chrono::month(13)) );
 }
 
 template<typename I>
diff --git a/libstdc++-v3/testsuite/std/time/format/pr117214_custom_timeput.cc b/libstdc++-v3/testsuite/std/time/format/pr117214_custom_timeput.cc
new file mode 100644
index 0000000..03b9496
--- /dev/null
+++ b/libstdc++-v3/testsuite/std/time/format/pr117214_custom_timeput.cc
@@ -0,0 +1,37 @@
+// { dg-do run { target c++20 } }
+
+#include <chrono>
+#include <format>
+#include <locale>
+#include <testsuite_hooks.h>
+
+struct custom_time_put : std::time_put<char>
+{
+  iter_type
+  do_put(iter_type out, std::ios_base& io, char_type fill, const tm* t,
+	 char format, char modifier) const override
+  {
+    using Base = std::time_put<char>;
+
+    switch (format) {
+	case 'a': case 'A': case 'b': case 'h': case 'B': case 'p':
+	*out++ = '[';
+	*out++ = format;
+	*out++ = ']';
+    }
+    return Base::do_put(out, io, fill, t, format, modifier);
+  }
+};
+
+int main()
+{
+  using namespace std::chrono;
+  std::locale loc(std::locale::classic(), new custom_time_put);
+#define test(t, fmt, exp) VERIFY( std::format(loc, fmt, t) == exp )
+  test(Monday,  "{:L%a}", "[a]Mon");
+  test(Monday,  "{:L%A}", "[A]Monday");
+  test(January, "{:L%b}", "[b]Jan");
+  test(January, "{:L%h}", "[h]Jan");
+  test(January, "{:L%B}", "[B]January");
+  test(1h,      "{:L%p}", "[p]AM");
+}
diff --git a/libstdc++-v3/testsuite/std/time/month/io.cc b/libstdc++-v3/testsuite/std/time/month/io.cc
index 99ec073..edfa196 100644
--- a/libstdc++-v3/testsuite/std/time/month/io.cc
+++ b/libstdc++-v3/testsuite/std/time/month/io.cc
@@ -24,6 +24,9 @@ test_ostream()
   ss.imbue(std::locale(ISO_8859(15,fr_FR)));
   ss << month(1);
   VERIFY( ss.str() == "janv." );
+  ss.str("");
+  ss << month(0) << '|' << month(13);
+  VERIFY( ss.str() == "0 is not a valid month|13 is not a valid month" );
 }
 
 void
@@ -66,6 +69,10 @@ test_format()
   VERIFY( s == "Jan" );
   s = std::format(loc_fr, "{:L%b}", month(1));
   VERIFY( s == "janv." );
+  s = std::format(loc_fr, "{:L}", month(0));
+  VERIFY( s == "0 is not a valid month" );
+  s = std::format(loc_fr, "{:L}", month(13));
+  VERIFY( s == "13 is not a valid month" );
 
   std::string_view specs = "aAbBcCdDeFgGhHIjmMpqQrRSTuUVwWxXyYzZ";
   std::string_view my_specs = "bBhm";
diff --git a/libstdc++-v3/testsuite/std/time/weekday/io.cc b/libstdc++-v3/testsuite/std/time/weekday/io.cc
index a56cdae..90a9bcb 100644
--- a/libstdc++-v3/testsuite/std/time/weekday/io.cc
+++ b/libstdc++-v3/testsuite/std/time/weekday/io.cc
@@ -69,6 +69,8 @@ test_format()
   VERIFY( s == "Mon" );
   s = std::format(loc_fr, "{:L%a}", weekday(1));
   VERIFY( s == "lun." );
+  s = std::format(loc_fr, "{:L}", weekday(25));
+  VERIFY( s == "25 is not a valid weekday" );
 
   std::string_view specs = "aAbBcCdDeFgGhHIjmMpqQrRSTuUVwWxXyYzZ";
   std::string_view my_specs = "aAuw";
diff --git a/libstdc++-v3/testsuite/util/testsuite_allocator.h b/libstdc++-v3/testsuite/util/testsuite_allocator.h
index e5ffad2..ee95752 100644
--- a/libstdc++-v3/testsuite/util/testsuite_allocator.h
+++ b/libstdc++-v3/testsuite/util/testsuite_allocator.h
@@ -517,19 +517,24 @@ namespace __gnu_test
       constexpr SimpleAllocator() noexcept { }
 
       template <class T>
+	constexpr
         SimpleAllocator(const SimpleAllocator<T>&) { }
 
+      _GLIBCXX20_CONSTEXPR
       Tp *allocate(std::size_t n)
       { return std::allocator<Tp>().allocate(n); }
 
+      _GLIBCXX20_CONSTEXPR
       void deallocate(Tp *p, std::size_t n)
       { std::allocator<Tp>().deallocate(p, n); }
     };
 
   template <class T, class U>
+    constexpr
     bool operator==(const SimpleAllocator<T>&, const SimpleAllocator<U>&)
     { return true; }
   template <class T, class U>
+    constexpr
     bool operator!=(const SimpleAllocator<T>&, const SimpleAllocator<U>&)
     { return false; }
 
diff --git a/libstdc++-v3/testsuite/util/testsuite_containers.h b/libstdc++-v3/testsuite/util/testsuite_containers.h
index 37491a4..ab0107f 100644
--- a/libstdc++-v3/testsuite/util/testsuite_containers.h
+++ b/libstdc++-v3/testsuite/util/testsuite_containers.h
@@ -210,6 +210,9 @@ namespace __gnu_test
 	clit = container.cbegin(bn);
 	assert( ++clit == container.cend(bn) );
 
+	clit = container.begin(bn);
+	assert( ++clit == container.cend(bn) );
+
 	assert( container.begin(bn) != container.cend(bn) );
       }
     };
@@ -304,6 +307,9 @@ namespace __gnu_test
 	assert( container.cbegin() != container.cend() );
 	assert( container.cbegin() != container.end() );
 	assert( container.begin() != container.cend() );
+
+	auto cit = container.begin();
+	assert( cit == container.cbegin() );
       }
   };
 
diff --git a/libstdc++-v3/testsuite/util/testsuite_hooks.h b/libstdc++-v3/testsuite/util/testsuite_hooks.h
index faa01ba..bf34fd1 100644
--- a/libstdc++-v3/testsuite/util/testsuite_hooks.h
+++ b/libstdc++-v3/testsuite/util/testsuite_hooks.h
@@ -58,16 +58,13 @@
 # define _VERIFY_PRINT(S, F, L, P, C) __builtin_printf(S, F, L, P, C)
 #endif
 
-#define VERIFY(fn)                                                      \
-  do                                                                    \
-  {                                                                     \
-    if (! (fn))								\
-      {									\
-	_VERIFY_PRINT("%s:%d: %s: Assertion '%s' failed.\n",		\
-		      __FILE__, __LINE__, __PRETTY_FUNCTION__, #fn);	\
-	__builtin_abort();						\
-      }									\
-  } while (false)
+#define VERIFY(...)							\
+   ((void)((__VA_ARGS__)						\
+	     ? (void)(true ? true : bool(__VA_ARGS__))			\
+	     : (_VERIFY_PRINT("%s:%d: %s: Assertion '%s' failed.\n",	\
+			      __FILE__, __LINE__, __PRETTY_FUNCTION__,	\
+			      #__VA_ARGS__),				\
+		__builtin_abort())))
 
 #ifdef _GLIBCXX_HAVE_UNISTD_H
 # include <unistd.h>
diff --git a/libstdc++-v3/testsuite/util/testsuite_iterators.h b/libstdc++-v3/testsuite/util/testsuite_iterators.h
index 74a8739..acd412a 100644
--- a/libstdc++-v3/testsuite/util/testsuite_iterators.h
+++ b/libstdc++-v3/testsuite/util/testsuite_iterators.h
@@ -61,10 +61,12 @@ namespace __gnu_test
       T* first;
       T* last;
 
+      _GLIBCXX_CONSTEXPR
       BoundsContainer(T* _first, T* _last) : first(_first), last(_last)
       { }
 
-      std::size_t size() const { return last - first; }
+      _GLIBCXX_CONSTEXPR std::size_t
+      size() const { return last - first; }
     };
 
   // Simple container for holding state of a set of output iterators.
@@ -74,11 +76,13 @@ namespace __gnu_test
       T* incrementedto;
       bool* writtento;
 
+      _GLIBCXX20_CONSTEXPR
       OutputContainer(T* _first, T* _last)
       : BoundsContainer<T>(_first, _last), incrementedto(_first),
 	writtento(new bool[this->size()]())
       { }
 
+      _GLIBCXX20_CONSTEXPR
       ~OutputContainer()
       { delete[] writtento; }
     };
@@ -92,12 +96,14 @@ namespace __gnu_test
     public:
       OutputContainer<T>* SharedInfo;
 
+      _GLIBCXX_CONSTEXPR
       WritableObject(T* ptr_in, OutputContainer<T>* SharedInfo_in):
 	ptr(ptr_in), SharedInfo(SharedInfo_in)
       { }
 
 #if __cplusplus >= 201103L
       template<class U>
+      _GLIBCXX14_CONSTEXPR
       typename std::enable_if<std::is_assignable<T&, U>::value>::type
       operator=(U&& new_val) const
       {
@@ -107,6 +113,7 @@ namespace __gnu_test
       }
 #else
       template<class U>
+      _GLIBCXX14_CONSTEXPR
       void
       operator=(const U& new_val)
       {
@@ -128,6 +135,7 @@ namespace __gnu_test
   struct output_iterator_wrapper
   {
   protected:
+    _GLIBCXX_CONSTEXPR
     output_iterator_wrapper() : ptr(0), SharedInfo(0)
     { }
 
@@ -142,6 +150,7 @@ namespace __gnu_test
     T* ptr;
     ContainerType* SharedInfo;
 
+    _GLIBCXX14_CONSTEXPR
     output_iterator_wrapper(T* _ptr, ContainerType* SharedInfo_in)
     : ptr(_ptr), SharedInfo(SharedInfo_in)
     {
@@ -155,6 +164,7 @@ namespace __gnu_test
     operator=(const output_iterator_wrapper&) = default;
 #endif
 
+    _GLIBCXX14_CONSTEXPR
     WritableObject<T>
     operator*() const
     {
@@ -163,6 +173,7 @@ namespace __gnu_test
       return WritableObject<T>(ptr, SharedInfo);
     }
 
+    _GLIBCXX14_CONSTEXPR
     output_iterator_wrapper&
     operator++()
     {
@@ -173,6 +184,7 @@ namespace __gnu_test
       return *this;
     }
 
+    _GLIBCXX14_CONSTEXPR
     output_iterator_wrapper
     operator++(int)
     {
@@ -224,13 +236,19 @@ namespace __gnu_test
       struct deref_proxy
       {
 	T* ptr;
-	operator const T&() const { return *ptr; }
+
+	_GLIBCXX_CONSTEXPR
+	operator const T&() const
+	{ return *ptr; }
       } p;
 
-      deref_proxy operator*() const { return p; }
+      _GLIBCXX_CONSTEXPR
+      deref_proxy operator*() const
+      { return p; }
     };
 
   protected:
+    _GLIBCXX_CONSTEXPR
     input_iterator_wrapper() : ptr(0), SharedInfo(0)
     { }
 
@@ -245,6 +263,7 @@ namespace __gnu_test
     T* ptr;
     ContainerType* SharedInfo;
 
+    _GLIBCXX14_CONSTEXPR
     input_iterator_wrapper(T* _ptr, ContainerType* SharedInfo_in)
     : ptr(_ptr), SharedInfo(SharedInfo_in)
     { ITERATOR_VERIFY(ptr >= SharedInfo->first && ptr <= SharedInfo->last); }
@@ -256,6 +275,7 @@ namespace __gnu_test
     operator=(const input_iterator_wrapper&) = default;
 #endif
 
+    _GLIBCXX14_CONSTEXPR
     bool
     operator==(const input_iterator_wrapper& in) const
     {
@@ -264,26 +284,34 @@ namespace __gnu_test
       return ptr == in.ptr;
     }
 
+    _GLIBCXX14_CONSTEXPR
     bool
     operator!=(const input_iterator_wrapper& in) const
     {
       return !(*this == in);
     }
 
-    T&
-    operator*() const
+    _GLIBCXX_CONSTEXPR
+    T* base() const
+    {
+      return ptr;
+    }
+
+    _GLIBCXX14_CONSTEXPR
+    T& operator*() const
     {
       ITERATOR_VERIFY(SharedInfo && ptr < SharedInfo->last);
       ITERATOR_VERIFY(ptr >= SharedInfo->first);
       return *ptr;
     }
 
-    T*
-    operator->() const
+    _GLIBCXX14_CONSTEXPR
+    T* operator->() const
     {
       return &**this;
     }
 
+    _GLIBCXX14_CONSTEXPR
     input_iterator_wrapper&
     operator++()
     {
@@ -294,6 +322,7 @@ namespace __gnu_test
       return *this;
     }
 
+    _GLIBCXX14_CONSTEXPR
     post_inc_proxy
     operator++(int)
     {
@@ -334,10 +363,12 @@ namespace __gnu_test
     typedef BoundsContainer<T> ContainerType;
     typedef std::forward_iterator_tag iterator_category;
 
+    _GLIBCXX14_CONSTEXPR
     forward_iterator_wrapper(T* _ptr, ContainerType* SharedInfo_in)
     : input_iterator_wrapper<T>(_ptr, SharedInfo_in)
     { }
 
+    _GLIBCXX14_CONSTEXPR
     forward_iterator_wrapper()
     { }
 
@@ -348,17 +379,18 @@ namespace __gnu_test
     operator=(const forward_iterator_wrapper&) = default;
 #endif
 
-    T&
-    operator*() const
+    _GLIBCXX14_CONSTEXPR
+    T& operator*() const
     {
       ITERATOR_VERIFY(this->SharedInfo && this->ptr < this->SharedInfo->last);
       return *(this->ptr);
     }
 
-    T*
-    operator->() const
+    _GLIBCXX14_CONSTEXPR
+    T* operator->() const
     { return &**this; }
 
+    _GLIBCXX14_CONSTEXPR
     forward_iterator_wrapper&
     operator++()
     {
@@ -367,6 +399,7 @@ namespace __gnu_test
       return *this;
     }
 
+    _GLIBCXX14_CONSTEXPR
     forward_iterator_wrapper
     operator++(int)
     {
@@ -376,8 +409,8 @@ namespace __gnu_test
     }
 
 #if __cplusplus >= 201402L
-    bool
-    operator==(const forward_iterator_wrapper& it) const noexcept
+    constexpr
+    bool operator==(const forward_iterator_wrapper& it) const noexcept
     {
       // Since C++14 value-initialized forward iterators are comparable.
       if (this->SharedInfo == nullptr || it.SharedInfo == nullptr)
@@ -388,8 +421,8 @@ namespace __gnu_test
       return base_this == base_that;
     }
 
-    bool
-    operator!=(const forward_iterator_wrapper& it) const noexcept
+    constexpr
+    bool operator!=(const forward_iterator_wrapper& it) const noexcept
     {
       return !(*this == it);
     }
@@ -409,10 +442,12 @@ namespace __gnu_test
     typedef BoundsContainer<T> ContainerType;
     typedef std::bidirectional_iterator_tag iterator_category;
 
+    _GLIBCXX14_CONSTEXPR
     bidirectional_iterator_wrapper(T* _ptr, ContainerType* SharedInfo_in)
     : forward_iterator_wrapper<T>(_ptr, SharedInfo_in)
     { }
 
+    _GLIBCXX14_CONSTEXPR
     bidirectional_iterator_wrapper()
     : forward_iterator_wrapper<T>()
     { }
@@ -425,6 +460,7 @@ namespace __gnu_test
     operator=(const bidirectional_iterator_wrapper&) = default;
 #endif
 
+    _GLIBCXX14_CONSTEXPR
     bidirectional_iterator_wrapper&
     operator++()
     {
@@ -433,6 +469,7 @@ namespace __gnu_test
       return *this;
     }
 
+    _GLIBCXX14_CONSTEXPR
     bidirectional_iterator_wrapper
     operator++(int)
     {
@@ -441,6 +478,7 @@ namespace __gnu_test
       return tmp;
     }
 
+    _GLIBCXX14_CONSTEXPR
     bidirectional_iterator_wrapper&
     operator--()
     {
@@ -449,6 +487,7 @@ namespace __gnu_test
       return *this;
     }
 
+    _GLIBCXX14_CONSTEXPR
     bidirectional_iterator_wrapper
     operator--(int)
     {
@@ -472,10 +511,12 @@ namespace __gnu_test
     typedef BoundsContainer<T> ContainerType;
     typedef std::random_access_iterator_tag iterator_category;
 
+    _GLIBCXX14_CONSTEXPR
     random_access_iterator_wrapper(T* _ptr, ContainerType* SharedInfo_in)
     : bidirectional_iterator_wrapper<T>(_ptr, SharedInfo_in)
     { }
 
+    _GLIBCXX14_CONSTEXPR
     random_access_iterator_wrapper()
     : bidirectional_iterator_wrapper<T>()
     { }
@@ -488,6 +529,7 @@ namespace __gnu_test
     operator=(const random_access_iterator_wrapper&) = default;
 #endif
 
+    _GLIBCXX14_CONSTEXPR
     random_access_iterator_wrapper&
     operator++()
     {
@@ -496,6 +538,7 @@ namespace __gnu_test
       return *this;
     }
 
+    _GLIBCXX14_CONSTEXPR
     random_access_iterator_wrapper
     operator++(int)
     {
@@ -504,6 +547,7 @@ namespace __gnu_test
       return tmp;
     }
 
+    _GLIBCXX14_CONSTEXPR
     random_access_iterator_wrapper&
     operator--()
     {
@@ -512,6 +556,7 @@ namespace __gnu_test
       return *this;
     }
 
+    _GLIBCXX14_CONSTEXPR
     random_access_iterator_wrapper
     operator--(int)
     {
@@ -520,6 +565,7 @@ namespace __gnu_test
       return tmp;
     }
 
+    _GLIBCXX14_CONSTEXPR
     random_access_iterator_wrapper&
     operator+=(std::ptrdiff_t n)
     {
@@ -536,10 +582,12 @@ namespace __gnu_test
       return *this;
     }
 
+    _GLIBCXX14_CONSTEXPR
     random_access_iterator_wrapper&
     operator-=(std::ptrdiff_t n)
     { return *this += -n; }
 
+    _GLIBCXX14_CONSTEXPR
     random_access_iterator_wrapper
     operator-(std::ptrdiff_t n) const
     {
@@ -547,6 +595,7 @@ namespace __gnu_test
       return tmp -= n;
     }
 
+    _GLIBCXX14_CONSTEXPR
     std::ptrdiff_t
     operator-(const random_access_iterator_wrapper<T>& in) const
     {
@@ -554,42 +603,44 @@ namespace __gnu_test
       return this->ptr - in.ptr;
     }
 
-    T&
-    operator[](std::ptrdiff_t n) const
+    _GLIBCXX14_CONSTEXPR
+    T& operator[](std::ptrdiff_t n) const
     { return *(*this + n); }
 
-    bool
-    operator<(const random_access_iterator_wrapper<T>& in) const
+    _GLIBCXX14_CONSTEXPR
+    bool operator<(const random_access_iterator_wrapper<T>& in) const
     {
       ITERATOR_VERIFY(this->SharedInfo == in.SharedInfo);
       return this->ptr < in.ptr;
     }
 
-    bool
-    operator>(const random_access_iterator_wrapper<T>& in) const
+    _GLIBCXX14_CONSTEXPR
+    bool operator>(const random_access_iterator_wrapper<T>& in) const
     {
       return in < *this;
     }
 
-    bool
-    operator>=(const random_access_iterator_wrapper<T>& in) const
+    _GLIBCXX14_CONSTEXPR
+    bool operator>=(const random_access_iterator_wrapper<T>& in) const
     {
       return !(*this < in);
     }
 
-    bool
-    operator<=(const random_access_iterator_wrapper<T>& in) const
+    _GLIBCXX14_CONSTEXPR
+    bool operator<=(const random_access_iterator_wrapper<T>& in) const
     {
       return !(*this > in);
     }
   };
 
   template<typename T>
+    _GLIBCXX14_CONSTEXPR
     random_access_iterator_wrapper<T>
     operator+(random_access_iterator_wrapper<T> it, std::ptrdiff_t n)
     { return it += n; }
 
   template<typename T>
+    _GLIBCXX14_CONSTEXPR
     random_access_iterator_wrapper<T>
     operator+(std::ptrdiff_t n, random_access_iterator_wrapper<T> it)
     { return it += n; }
@@ -607,14 +658,17 @@ namespace __gnu_test
   {
     typename ItType<T>::ContainerType bounds;
 
+    _GLIBCXX_CONSTEXPR
     test_container(T* _first, T* _last) : bounds(_first, _last)
     { }
 
     template<std::size_t N>
       explicit
+      _GLIBCXX_CONSTEXPR
       test_container(T (&arr)[N]) : bounds(arr, arr+N)
       { }
 
+    _GLIBCXX14_CONSTEXPR
     ItType<T>
     it(int pos)
     {
@@ -622,6 +676,7 @@ namespace __gnu_test
       return ItType<T>(bounds.first + pos, &bounds);
     }
 
+    _GLIBCXX14_CONSTEXPR
     ItType<T>
     it(T* pos)
     {
@@ -629,18 +684,22 @@ namespace __gnu_test
       return ItType<T>(pos, &bounds);
     }
 
+    _GLIBCXX_CONSTEXPR
     const T&
     val(int pos)
     { return (bounds.first)[pos]; }
 
+    _GLIBCXX14_CONSTEXPR
     ItType<T>
     begin()
     { return it(bounds.first); }
 
+    _GLIBCXX14_CONSTEXPR
     ItType<T>
     end()
     { return it(bounds.last); }
 
+    _GLIBCXX_CONSTEXPR
     std::size_t
     size() const
     { return bounds.size(); }
@@ -680,6 +739,7 @@ namespace __gnu_test
       // Use an integer-class type to try and break the library code.
       using difference_type = std::ranges::__detail::__max_diff_type;
 
+      constexpr
       contiguous_iterator_wrapper&
       operator++()
       {
@@ -687,6 +747,7 @@ namespace __gnu_test
 	return *this;
       }
 
+      constexpr
       contiguous_iterator_wrapper&
       operator--()
       {
@@ -694,6 +755,7 @@ namespace __gnu_test
 	return *this;
       }
 
+      constexpr
       contiguous_iterator_wrapper
       operator++(int)
       {
@@ -702,6 +764,7 @@ namespace __gnu_test
 	return tmp;
       }
 
+      constexpr
       contiguous_iterator_wrapper
       operator--(int)
       {
@@ -710,6 +773,7 @@ namespace __gnu_test
 	return tmp;
       }
 
+      constexpr
       contiguous_iterator_wrapper&
       operator+=(difference_type n)
       {
@@ -718,23 +782,28 @@ namespace __gnu_test
 	return *this;
       }
 
-      friend contiguous_iterator_wrapper
+      friend constexpr
+      contiguous_iterator_wrapper
       operator+(contiguous_iterator_wrapper iter, difference_type n)
       { return iter += n; }
 
-      friend contiguous_iterator_wrapper
+      friend constexpr
+      contiguous_iterator_wrapper
       operator+(difference_type n, contiguous_iterator_wrapper iter)
       { return iter += n; }
 
+      constexpr
       contiguous_iterator_wrapper&
       operator-=(difference_type n)
       { return *this += -n; }
 
-      friend contiguous_iterator_wrapper
+      friend constexpr
+      contiguous_iterator_wrapper
       operator-(contiguous_iterator_wrapper iter, difference_type n)
       { return iter -= n; }
 
-      friend difference_type
+      friend constexpr
+      difference_type
       operator-(contiguous_iterator_wrapper l, contiguous_iterator_wrapper r)
       {
 	const random_access_iterator_wrapper<T>& lbase = l;
@@ -742,6 +811,7 @@ namespace __gnu_test
 	return static_cast<difference_type>(lbase - rbase);
       }
 
+      constexpr
       decltype(auto) operator[](difference_type n) const
       {
 	auto d = static_cast<std::ptrdiff_t>(n);
@@ -759,6 +829,7 @@ namespace __gnu_test
     {
       using input_iterator_wrapper<T>::input_iterator_wrapper;
 
+      constexpr
       input_iterator_wrapper_nocopy()
 	: input_iterator_wrapper<T>(nullptr, nullptr)
       { }
@@ -773,6 +844,7 @@ namespace __gnu_test
 
       using input_iterator_wrapper<T>::operator++;
 
+      constexpr
       input_iterator_wrapper_nocopy&
       operator++()
       {
@@ -789,6 +861,7 @@ namespace __gnu_test
 
       using input_iterator_wrapper<T>::operator++;
 
+      constexpr
       input_iterator_wrapper_rval&
       operator++()
       {
@@ -796,8 +869,8 @@ namespace __gnu_test
 	return *this;
       }
 
-      T&&
-      operator*() const
+      constexpr
+      T&& operator*() const
       { return std::move(input_iterator_wrapper<T>::operator*()); }
     };
 
@@ -815,7 +888,9 @@ namespace __gnu_test
 
 	using Iter<T>::operator++;
 
-	iterator& operator++() { Iter<T>::operator++(); return *this; }
+	constexpr
+	iterator& operator++()
+       	{ Iter<T>::operator++(); return *this; }
       };
 
       template<typename I>
@@ -823,21 +898,24 @@ namespace __gnu_test
 	{
 	  T* end;
 
-	  friend bool operator==(const sentinel& s, const I& i) noexcept
+	  friend constexpr bool
+	  operator==(const sentinel& s, const I& i) noexcept
 	  { return s.end == i.ptr; }
 
-	  friend auto operator-(const sentinel& s, const I& i) noexcept
+	  friend constexpr
+	  auto operator-(const sentinel& s, const I& i) noexcept
 	    requires std::random_access_iterator<I>
 	  { return std::iter_difference_t<I>(s.end - i.ptr); }
 
-	  friend auto operator-(const I& i, const sentinel& s) noexcept
+	  friend constexpr auto
+	  operator-(const I& i, const sentinel& s) noexcept
 	    requires std::random_access_iterator<I>
 	  { return std::iter_difference_t<I>(i.ptr - s.end); }
 	};
 
     protected:
-      auto
-      get_iterator(T* p)
+      constexpr
+      auto get_iterator(T* p)
       {
 	if constexpr (std::default_initializable<Iter<T>>)
 	  return Iter<T>(p, &bounds);
@@ -846,17 +924,19 @@ namespace __gnu_test
       }
 
     public:
+      constexpr
       test_range(T* first, T* last) : bounds(first, last)
       { }
 
       template<std::size_t N>
-	explicit
+	explicit constexpr
 	test_range(T (&arr)[N]) : test_range(arr, arr+N)
 	{ }
 
-      auto begin() & { return get_iterator(bounds.first); }
+      constexpr auto begin() &
+      { return get_iterator(bounds.first); }
 
-      auto end() &
+      constexpr auto end() &
       {
 	using I = decltype(get_iterator(bounds.last));
 	return sentinel<I>{bounds.last};
@@ -869,7 +949,9 @@ namespace __gnu_test
   template<typename T, template<typename> class Iter>
     struct test_range_nocopy : test_range<T, Iter>
     {
-      test_range_nocopy(T* first, T* last) : test_range<T, Iter>(first, last)
+      constexpr
+      test_range_nocopy(T* first, T* last)
+      : test_range<T, Iter>(first, last)
       {}
 
       test_range_nocopy(test_range_nocopy&&) = default;
@@ -904,6 +986,7 @@ namespace __gnu_test
     {
       using test_range<T, Iter>::test_range;
 
+      constexpr
       std::size_t size() const noexcept
       { return this->bounds.size(); }
     };
@@ -939,18 +1022,22 @@ namespace __gnu_test
 	{
 	  T* end;
 
-	  friend bool operator==(const sentinel& s, const I& i) noexcept
+	  friend constexpr
+	  bool operator==(const sentinel& s, const I& i) noexcept
 	  { return s.end == i.ptr; }
 
-	  friend std::iter_difference_t<I>
+	  friend constexpr
+	  std::iter_difference_t<I>
 	  operator-(const sentinel& s, const I& i) noexcept
 	  { return std::iter_difference_t<I>(s.end - i.ptr); }
 
-	  friend std::iter_difference_t<I>
+	  friend constexpr
+	  std::iter_difference_t<I>
 	  operator-(const I& i, const sentinel& s) noexcept
 	  { return std::iter_difference_t<I>(i.ptr - s.end); }
 	};
 
+      constexpr
       auto end() &
       {
 	using I = decltype(this->get_iterator(this->bounds.last));
diff --git a/maintainer-scripts/ChangeLog b/maintainer-scripts/ChangeLog
index 8b2ffca..a835391 100644
--- a/maintainer-scripts/ChangeLog
+++ b/maintainer-scripts/ChangeLog
@@ -1,3 +1,7 @@
+2025-07-04  Richard Biener  <rguenther@suse.de>
+
+	* crontab: Stop doing GCC 12 snapshots.
+
 2025-05-23  Richard Biener  <rguenther@suse.de>
 
 	* update_web_docs_git: Conditionalize libgdiagnostic processing
diff --git a/maintainer-scripts/crontab b/maintainer-scripts/crontab
index c880d7d..c17289f 100644
--- a/maintainer-scripts/crontab
+++ b/maintainer-scripts/crontab
@@ -1,7 +1,6 @@
 16  0 * * * sh /home/gccadmin/scripts/update_version_git
 50  0 * * * sh /home/gccadmin/scripts/update_web_docs_git
 55  0 * * * sh /home/gccadmin/scripts/update_web_docs_libstdcxx_git
-32 22 * * 3 sh /home/gccadmin/scripts/gcc_release -s 12:releases/gcc-12 -l -d /sourceware/snapshot-tmp/gcc all
 32 22 * * 4 sh /home/gccadmin/scripts/gcc_release -s 13:releases/gcc-13 -l -d /sourceware/snapshot-tmp/gcc all
 32 22 * * 5 sh /home/gccadmin/scripts/gcc_release -s 14:releases/gcc-14 -l -d /sourceware/snapshot-tmp/gcc all
 32 22 * * 6 sh /home/gccadmin/scripts/gcc_release -s 15:releases/gcc-15 -l -d /sourceware/snapshot-tmp/gcc all
diff --git a/zlib/CMakeLists.txt b/zlib/CMakeLists.txt
index 0fe939d..15ceebe 100644
--- a/zlib/CMakeLists.txt
+++ b/zlib/CMakeLists.txt
@@ -1,12 +1,11 @@
-cmake_minimum_required(VERSION 2.4.4)
+cmake_minimum_required(VERSION 2.4.4...3.15.0)
 set(CMAKE_ALLOW_LOOSE_LOOP_CONSTRUCTS ON)
 
 project(zlib C)
 
-set(VERSION "1.2.11")
+set(VERSION "1.3.1")
 
-option(ASM686 "Enable building i686 assembly implementation")
-option(AMD64 "Enable building amd64 assembly implementation")
+option(ZLIB_BUILD_EXAMPLES "Enable Zlib Examples" ON)
 
 set(INSTALL_BIN_DIR "${CMAKE_INSTALL_PREFIX}/bin" CACHE PATH "Installation directory for executables")
 set(INSTALL_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib" CACHE PATH "Installation directory for libraries")
@@ -129,39 +128,6 @@ if(NOT MINGW)
     )
 endif()
 
-if(CMAKE_COMPILER_IS_GNUCC)
-    if(ASM686)
-        set(ZLIB_ASMS contrib/asm686/match.S)
-    elseif (AMD64)
-        set(ZLIB_ASMS contrib/amd64/amd64-match.S)
-    endif ()
-
-	if(ZLIB_ASMS)
-		add_definitions(-DASMV)
-		set_source_files_properties(${ZLIB_ASMS} PROPERTIES LANGUAGE C COMPILE_FLAGS -DNO_UNDERLINE)
-	endif()
-endif()
-
-if(MSVC)
-    if(ASM686)
-		ENABLE_LANGUAGE(ASM_MASM)
-        set(ZLIB_ASMS
-			contrib/masmx86/inffas32.asm
-			contrib/masmx86/match686.asm
-		)
-    elseif (AMD64)
-		ENABLE_LANGUAGE(ASM_MASM)
-        set(ZLIB_ASMS
-			contrib/masmx64/gvmat64.asm
-			contrib/masmx64/inffasx64.asm
-		)
-    endif()
-
-	if(ZLIB_ASMS)
-		add_definitions(-DASMV -DASMINF)
-	endif()
-endif()
-
 # parse the full version number from zlib.h and include in ZLIB_FULL_VERSION
 file(READ ${CMAKE_CURRENT_SOURCE_DIR}/zlib.h _zlib_h_contents)
 string(REGEX REPLACE ".*#define[ \t]+ZLIB_VERSION[ \t]+\"([-0-9A-Za-z.]+)\".*"
@@ -183,8 +149,10 @@ if(MINGW)
     set(ZLIB_DLL_SRCS ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj)
 endif(MINGW)
 
-add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
-add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_ASMS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
+add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
+target_include_directories(zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
+add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
+target_include_directories(zlibstatic PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
 set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL)
 set_target_properties(zlib PROPERTIES SOVERSION 1)
 
@@ -202,7 +170,7 @@ endif()
 if(UNIX)
     # On unix-like platforms the library is almost always called libz
    set_target_properties(zlib zlibstatic PROPERTIES OUTPUT_NAME z)
-   if(NOT APPLE)
+   if(NOT APPLE AND NOT(CMAKE_SYSTEM_NAME STREQUAL AIX))
      set_target_properties(zlib PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"")
    endif()
 elseif(BUILD_SHARED_LIBS AND WIN32)
@@ -229,21 +197,22 @@ endif()
 #============================================================================
 # Example binaries
 #============================================================================
-
-add_executable(example test/example.c)
-target_link_libraries(example zlib)
-add_test(example example)
-
-add_executable(minigzip test/minigzip.c)
-target_link_libraries(minigzip zlib)
-
-if(HAVE_OFF64_T)
-    add_executable(example64 test/example.c)
-    target_link_libraries(example64 zlib)
-    set_target_properties(example64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64")
-    add_test(example64 example64)
-
-    add_executable(minigzip64 test/minigzip.c)
-    target_link_libraries(minigzip64 zlib)
-    set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64")
+if(ZLIB_BUILD_EXAMPLES)
+    add_executable(example test/example.c)
+    target_link_libraries(example zlib)
+    add_test(example example)
+
+    add_executable(minigzip test/minigzip.c)
+    target_link_libraries(minigzip zlib)
+
+    if(HAVE_OFF64_T)
+        add_executable(example64 test/example.c)
+        target_link_libraries(example64 zlib)
+        set_target_properties(example64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64")
+        add_test(example64 example64)
+
+        add_executable(minigzip64 test/minigzip.c)
+        target_link_libraries(minigzip64 zlib)
+        set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64")
+    endif()
 endif()
diff --git a/zlib/ChangeLog b/zlib/ChangeLog
index 0b7ddfd..b0aebbc 100644
--- a/zlib/ChangeLog
+++ b/zlib/ChangeLog
@@ -1,3 +1,8 @@
+2025-07-31  Sam James  <sam@gentoo.org>
+
+	* configure: Regenerate.
+	* configure.ac: Set version to 1.3.1.
+
 2023-10-22  Iain Sandoe  <iain@sandoe.co.uk>
 
 	* configure: Regenerate.
@@ -98,6 +103,109 @@
 
                 ChangeLog file for zlib
 
+Changes in 1.3.1 (22 Jan 2024)
+- Reject overflows of zip header fields in minizip
+- Fix bug in inflateSync() for data held in bit buffer
+- Add LIT_MEM define to use more memory for a small deflate speedup
+- Fix decision on the emission of Zip64 end records in minizip
+- Add bounds checking to ERR_MSG() macro, used by zError()
+- Neutralize zip file traversal attacks in miniunz
+- Fix a bug in ZLIB_DEBUG compiles in check_match()
+- Various portability and appearance improvements
+
+Changes in 1.3 (18 Aug 2023)
+- Remove K&R function definitions and zlib2ansi
+- Fix bug in deflateBound() for level 0 and memLevel 9
+- Fix bug when gzungetc() is used immediately after gzopen()
+- Fix bug when using gzflush() with a very small buffer
+- Fix crash when gzsetparams() attempted for transparent write
+- Fix test/example.c to work with FORCE_STORED
+- Rewrite of zran in examples (see zran.c version history)
+- Fix minizip to allow it to open an empty zip file
+- Fix reading disk number start on zip64 files in minizip
+- Fix logic error in minizip argument processing
+- Add minizip testing to Makefile
+- Read multiple bytes instead of byte-by-byte in minizip unzip.c
+- Add memory sanitizer to configure (--memory)
+- Various portability improvements
+- Various documentation improvements
+- Various spelling and typo corrections
+
+Changes in 1.2.13 (13 Oct 2022)
+- Fix configure issue that discarded provided CC definition
+- Correct incorrect inputs provided to the CRC functions
+- Repair prototypes and exporting of new CRC functions
+- Fix inflateBack to detect invalid input with distances too far
+- Have infback() deliver all of the available output up to any error
+- Fix a bug when getting a gzip header extra field with inflate()
+- Fix bug in block type selection when Z_FIXED used
+- Tighten deflateBound bounds
+- Remove deleted assembler code references
+- Various portability and appearance improvements
+
+Changes in 1.2.12 (27 Mar 2022)
+- Cygwin does not have _wopen(), so do not create gzopen_w() there
+- Permit a deflateParams() parameter change as soon as possible
+- Limit hash table inserts after switch from stored deflate
+- Fix bug when window full in deflate_stored()
+- Fix CLEAR_HASH macro to be usable as a single statement
+- Avoid a conversion error in gzseek when off_t type too small
+- Have Makefile return non-zero error code on test failure
+- Avoid some conversion warnings in gzread.c and gzwrite.c
+- Update use of errno for newer Windows CE versions
+- Small speedup to inflate [psumbera]
+- Return an error if the gzputs string length can't fit in an int
+- Add address checking in clang to -w option of configure
+- Don't compute check value for raw inflate if asked to validate
+- Handle case where inflateSync used when header never processed
+- Avoid the use of ptrdiff_t
+- Avoid an undefined behavior of memcpy() in gzappend()
+- Avoid undefined behaviors of memcpy() in gz*printf()
+- Avoid an undefined behavior of memcpy() in _tr_stored_block()
+- Make the names in functions declarations identical to definitions
+- Remove old assembler code in which bugs have manifested
+- Fix deflateEnd() to not report an error at start of raw deflate
+- Add legal disclaimer to README
+- Emphasize the need to continue decompressing gzip members
+- Correct the initialization requirements for deflateInit2()
+- Fix a bug that can crash deflate on some input when using Z_FIXED
+- Assure that the number of bits for deflatePrime() is valid
+- Use a structure to make globals in enough.c evident
+- Use a macro for the printf format of big_t in enough.c
+- Clean up code style in enough.c, update version
+- Use inline function instead of macro for index in enough.c
+- Clarify that prefix codes are counted in enough.c
+- Show all the codes for the maximum tables size in enough.c
+- Add gznorm.c example, which normalizes gzip files
+- Fix the zran.c example to work on a multiple-member gzip file
+- Add tables for crc32_combine(), to speed it up by a factor of 200
+- Add crc32_combine_gen() and crc32_combine_op() for fast combines
+- Speed up software CRC-32 computation by a factor of 1.5 to 3
+- Use atomic test and set, if available, for dynamic CRC tables
+- Don't bother computing check value after successful inflateSync()
+- Correct comment in crc32.c
+- Add use of the ARMv8 crc32 instructions when requested
+- Use ARM crc32 instructions if the ARM architecture has them
+- Explicitly note that the 32-bit check values are 32 bits
+- Avoid adding empty gzip member after gzflush with Z_FINISH
+- Fix memory leak on error in gzlog.c
+- Fix error in comment on the polynomial representation of a byte
+- Clarify gz* function interfaces, referring to parameter names
+- Change macro name in inflate.c to avoid collision in VxWorks
+- Correct typo in blast.c
+- Improve portability of contrib/minizip
+- Fix indentation in minizip's zip.c
+- Replace black/white with allow/block. (theresa-m)
+- minizip warning fix if MAXU32 already defined. (gvollant)
+- Fix unztell64() in minizip to work past 4GB. (Daniël Hörchner)
+- Clean up minizip to reduce warnings for testing
+- Add fallthrough comments for gcc
+- Eliminate use of ULL constants
+- Separate out address sanitizing from warnings in configure
+- Remove destructive aspects of make distclean
+- Check for cc masquerading as gcc or clang in configure
+- Fix crc32.c to compile local functions only if used
+
 Changes in 1.2.11 (15 Jan 2017)
 - Fix deflate stored bug when pulling last block from window
 - Permit immediate deflateParams changes before any deflate input
@@ -193,7 +301,7 @@ Changes in 1.2.7.1 (24 Mar 2013)
 - Fix types in contrib/minizip to match result of get_crc_table()
 - Simplify contrib/vstudio/vc10 with 'd' suffix
 - Add TOP support to win32/Makefile.msc
-- Suport i686 and amd64 assembler builds in CMakeLists.txt
+- Support i686 and amd64 assembler builds in CMakeLists.txt
 - Fix typos in the use of _LARGEFILE64_SOURCE in zconf.h
 - Add vc11 and vc12 build files to contrib/vstudio
 - Add gzvprintf() as an undocumented function in zlib
@@ -393,14 +501,14 @@ Changes in 1.2.5.1 (10 Sep 2011)
 - Use u4 type for crc_table to avoid conversion warnings
 - Apply casts in zlib.h to avoid conversion warnings
 - Add OF to prototypes for adler32_combine_ and crc32_combine_ [Miller]
-- Improve inflateSync() documentation to note indeterminancy
+- Improve inflateSync() documentation to note indeterminacy
 - Add deflatePending() function to return the amount of pending output
 - Correct the spelling of "specification" in FAQ [Randers-Pehrson]
 - Add a check in configure for stdarg.h, use for gzprintf()
 - Check that pointers fit in ints when gzprint() compiled old style
 - Add dummy name before $(SHAREDLIBV) in Makefile [Bar-Lev, Bowler]
 - Delete line in configure that adds -L. libz.a to LDFLAGS [Weigelt]
-- Add debug records in assmebler code [Londer]
+- Add debug records in assembler code [Londer]
 - Update RFC references to use http://tools.ietf.org/html/... [Li]
 - Add --archs option, use of libtool to configure for Mac OS X [Borstel]
 
@@ -608,7 +716,7 @@ Changes in 1.2.3.5 (8 Jan 2010)
 - Don't use _vsnprintf on later versions of MSVC [Lowman]
 - Add CMake build script and input file [Lowman]
 - Update contrib/minizip to 1.1 [Svensson, Vollant]
-- Moved nintendods directory from contrib to .
+- Moved nintendods directory from contrib to root
 - Replace gzio.c with a new set of routines with the same functionality
 - Add gzbuffer(), gzoffset(), gzclose_r(), gzclose_w() as part of above
 - Update contrib/minizip to 1.1b
@@ -782,7 +890,7 @@ Changes in 1.2.2.4 (11 July 2005)
 - Be more strict on incomplete code sets in inflate_table() and increase
   ENOUGH and MAXD -- this repairs a possible security vulnerability for
   invalid inflate input.  Thanks to Tavis Ormandy and Markus Oberhumer for
-  discovering the vulnerability and providing test cases.
+  discovering the vulnerability and providing test cases
 - Add ia64 support to configure for HP-UX [Smith]
 - Add error return to gzread() for format or i/o error [Levin]
 - Use malloc.h for OS/2 [Necasek]
@@ -818,7 +926,7 @@ Changes in 1.2.2.2 (30 December 2004)
 - Add Z_FIXED strategy option to deflateInit2() to force fixed trees
 - Add updated make_vms.com [Coghlan], update README
 - Create a new "examples" directory, move gzappend.c there, add zpipe.c,
-  fitblk.c, gzlog.[ch], gzjoin.c, and zlib_how.html.
+  fitblk.c, gzlog.[ch], gzjoin.c, and zlib_how.html
 - Add FAQ entry and comments in deflate.c on uninitialized memory access
 - Add Solaris 9 make options in configure [Gilbert]
 - Allow strerror() usage in gzio.c for STDC
@@ -889,7 +997,7 @@ Changes in 1.2.1.1 (9 January 2004)
 - Fix a big fat bug in inftrees.c that prevented decoding valid
   dynamic blocks with only literals and no distance codes --
   Thanks to "Hot Emu" for the bug report and sample file
-- Add a note to puff.c on no distance codes case.
+- Add a note to puff.c on no distance codes case
 
 Changes in 1.2.1 (17 November 2003)
 - Remove a tab in contrib/gzappend/gzappend.c
@@ -1067,7 +1175,7 @@ Changes in 1.2.0.1 (17 March 2003)
     - Include additional header file on VMS for off_t typedef
 - Try to use _vsnprintf where it supplants vsprintf [Vollant]
 - Add some casts in inffast.c
-- Enchance comments in zlib.h on what happens if gzprintf() tries to
+- Enhance comments in zlib.h on what happens if gzprintf() tries to
   write more than 4095 bytes before compression
 - Remove unused state from inflateBackEnd()
 - Remove exit(0) from minigzip.c, example.c
@@ -1133,14 +1241,14 @@ Changes in 1.2.0 (9 March 2003)
 - Add contrib/puff/ simple inflate for deflate format description
 
 Changes in 1.1.4 (11 March 2002)
-- ZFREE was repeated on same allocation on some error conditions.
+- ZFREE was repeated on same allocation on some error conditions
   This creates a security problem described in
   http://www.zlib.org/advisory-2002-03-11.txt
 - Returned incorrect error (Z_MEM_ERROR) on some invalid data
 - Avoid accesses before window for invalid distances with inflate window
-  less than 32K.
+  less than 32K
 - force windowBits > 8 to avoid a bug in the encoder for a window size
-  of 256 bytes. (A complete fix will be available in 1.1.5).
+  of 256 bytes. (A complete fix will be available in 1.1.5)
 
 Changes in 1.1.3 (9 July 1998)
 - fix "an inflate input buffer bug that shows up on rare but persistent
@@ -1214,7 +1322,7 @@ Changes in 1.1.1 (27 Feb 98)
 - remove block truncation heuristic which had very marginal effect for zlib
   (smaller lit_bufsize than in gzip 1.2.4) and degraded a little the
   compression ratio on some files. This also allows inlining _tr_tally for
-  matches in deflate_slow.
+  matches in deflate_slow
 - added msdos/Makefile.w32 for WIN32 Microsoft Visual C++ (Bob Frazier)
 
 Changes in 1.1.0 (24 Feb 98)
@@ -1245,7 +1353,7 @@ Changes in 1.0.9 (17 Feb 1998)
 - Avoid gcc 2.8.0 comparison bug a little differently than zlib 1.0.8
 - in inftrees.c, avoid cc -O bug on HP (Farshid Elahi)
 - in zconf.h move the ZLIB_DLL stuff earlier to avoid problems with
-  the declaration of FAR (Gilles VOllant)
+  the declaration of FAR (Gilles Vollant)
 - install libz.so* with mode 755 (executable) instead of 644 (Marc Lehmann)
 - read_buf buf parameter of type Bytef* instead of charf*
 - zmemcpy parameters are of type Bytef*, not charf* (Joseph Strout)
@@ -1259,7 +1367,7 @@ Changes in 1.0.8 (27 Jan 1998)
 - include sys/types.h to get off_t on some systems (Marc Lehmann & QingLong)
 - use constant arrays for the static trees in trees.c instead of computing
   them at run time (thanks to Ken Raeburn for this suggestion). To create
-  trees.h, compile with GEN_TREES_H and run "make test".
+  trees.h, compile with GEN_TREES_H and run "make test"
 - check return code of example in "make test" and display result
 - pass minigzip command line options to file_compress
 - simplifying code of inflateSync to avoid gcc 2.8 bug
@@ -1298,12 +1406,12 @@ Changes in 1.0.6 (19 Jan 1998)
 - add functions gzprintf, gzputc, gzgetc, gztell, gzeof, gzseek, gzrewind and
   gzsetparams (thanks to Roland Giersig and Kevin Ruland for some of this code)
 - Fix a deflate bug occurring only with compression level 0 (thanks to
-  Andy Buckler for finding this one).
-- In minigzip, pass transparently also the first byte for .Z files.
+  Andy Buckler for finding this one)
+- In minigzip, pass transparently also the first byte for .Z files
 - return Z_BUF_ERROR instead of Z_OK if output buffer full in uncompress()
 - check Z_FINISH in inflate (thanks to Marc Schluper)
 - Implement deflateCopy (thanks to Adam Costello)
-- make static libraries by default in configure, add --shared option.
+- make static libraries by default in configure, add --shared option
 - move MSDOS or Windows specific files to directory msdos
 - suppress the notion of partial flush to simplify the interface
   (but the symbol Z_PARTIAL_FLUSH is kept for compatibility with 1.0.4)
@@ -1315,7 +1423,7 @@ Changes in 1.0.6 (19 Jan 1998)
 - added Makefile.nt (thanks to Stephen Williams)
 - added the unsupported "contrib" directory:
    contrib/asm386/ by Gilles Vollant <info@winimage.com>
-        386 asm code replacing longest_match().
+        386 asm code replacing longest_match()
    contrib/iostream/ by Kevin Ruland <kevin@rodin.wustl.edu>
         A C++ I/O streams interface to the zlib gz* functions
    contrib/iostream2/  by Tyge Løvset <Tyge.Lovset@cmr.no>
@@ -1323,7 +1431,7 @@ Changes in 1.0.6 (19 Jan 1998)
    contrib/untgz/  by "Pedro A. Aranda Guti\irrez" <paag@tid.es>
         A very simple tar.gz file extractor using zlib
    contrib/visual-basic.txt by Carlos Rios <c_rios@sonda.cl>
-        How to use compress(), uncompress() and the gz* functions from VB.
+        How to use compress(), uncompress() and the gz* functions from VB
 - pass params -f (filtered data), -h (huffman only), -1 to -9 (compression
   level) in minigzip (thanks to Tom Lane)
 
@@ -1332,8 +1440,8 @@ Changes in 1.0.6 (19 Jan 1998)
 - add undocumented function inflateSyncPoint() (hack for Paul Mackerras)
 - add undocumented function zError to convert error code to string
   (for Tim Smithers)
-- Allow compilation of gzio with -DNO_DEFLATE to avoid the compression code.
-- Use default memcpy for Symantec MSDOS compiler.
+- Allow compilation of gzio with -DNO_DEFLATE to avoid the compression code
+- Use default memcpy for Symantec MSDOS compiler
 - Add EXPORT keyword for check_func (needed for Windows DLL)
 - add current directory to LD_LIBRARY_PATH for "make test"
 - create also a link for libz.so.1
@@ -1346,7 +1454,7 @@ Changes in 1.0.6 (19 Jan 1998)
 - allow compilation with ANSI keywords only enabled for TurboC in large model
 - avoid "versionString"[0] (Borland bug)
 - add NEED_DUMMY_RETURN for Borland
-- use variable z_verbose for tracing in debug mode (L. Peter Deutsch).
+- use variable z_verbose for tracing in debug mode (L. Peter Deutsch)
 - allow compilation with CC
 - defined STDC for OS/2 (David Charlap)
 - limit external names to 8 chars for MVS (Thomas Lund)
@@ -1356,7 +1464,7 @@ Changes in 1.0.6 (19 Jan 1998)
 - use _fdopen instead of fdopen for MSC >= 6.0 (Thomas Fanslau)
 - added makelcc.bat for lcc-win32 (Tom St Denis)
 - in Makefile.dj2, use copy and del instead of install and rm (Frank Donahoe)
-- Avoid expanded $Id: ChangeLog,v 1.5 2002/05/08 04:38:00 aoliva Exp $. Use "rcs -kb" or "cvs admin -kb" to avoid Id expansion.
+- Avoid expanded $Id$. Use "rcs -kb" or "cvs admin -kb" to avoid Id expansion
 - check for unistd.h in configure (for off_t)
 - remove useless check parameter in inflate_blocks_free
 - avoid useless assignment of s->check to itself in inflate_blocks_new
@@ -1377,7 +1485,7 @@ Changes in 1.0.5 (3 Jan 98)
 Changes in 1.0.4 (24 Jul 96)
 - In very rare conditions, deflate(s, Z_FINISH) could fail to produce an EOF
   bit, so the decompressor could decompress all the correct data but went
-  on to attempt decompressing extra garbage data. This affected minigzip too.
+  on to attempt decompressing extra garbage data. This affected minigzip too
 - zlibVersion and gzerror return const char* (needed for DLL)
 - port to RISCOS (no fdopen, no multiple dots, no unlink, no fileno)
 - use z_error only for DEBUG (avoid problem with DLLs)
@@ -1407,7 +1515,7 @@ Changes in 1.0.1 (20 May 96) [1.0 skipped to avoid confusion]
 - fix array overlay in deflate.c which sometimes caused bad compressed data
 - fix inflate bug with empty stored block
 - fix MSDOS medium model which was broken in 0.99
-- fix deflateParams() which could generate bad compressed data.
+- fix deflateParams() which could generate bad compressed data
 - Bytef is define'd instead of typedef'ed (work around Borland bug)
 - added an INDEX file
 - new makefiles for DJGPP (Makefile.dj2), 32-bit Borland (Makefile.b32),
@@ -1428,7 +1536,7 @@ Changes in 0.99 (27 Jan 96)
 - allow preset dictionary shared between compressor and decompressor
 - allow compression level 0 (no compression)
 - add deflateParams in zlib.h: allow dynamic change of compression level
-  and compression strategy.
+  and compression strategy
 - test large buffers and deflateParams in example.c
 - add optional "configure" to build zlib as a shared library
 - suppress Makefile.qnx, use configure instead
@@ -1467,33 +1575,33 @@ Changes in 0.99 (27 Jan 96)
 - fix typo in Make_vms.com (f$trnlnm -> f$getsyi)
 - in fcalloc, normalize pointer if size > 65520 bytes
 - don't use special fcalloc for 32 bit Borland C++
-- use STDC instead of __GO32__ to avoid redeclaring exit, calloc, etc...
+- use STDC instead of __GO32__ to avoid redeclaring exit, calloc, etc.
 - use Z_BINARY instead of BINARY
 - document that gzclose after gzdopen will close the file
-- allow "a" as mode in gzopen.
+- allow "a" as mode in gzopen
 - fix error checking in gzread
 - allow skipping .gz extra-field on pipes
 - added reference to Perl interface in README
 - put the crc table in FAR data (I dislike more and more the medium model :)
 - added get_crc_table
-- added a dimension to all arrays (Borland C can't count).
+- added a dimension to all arrays (Borland C can't count)
 - workaround Borland C bug in declaration of inflate_codes_new & inflate_fast
 - guard against multiple inclusion of *.h (for precompiled header on Mac)
-- Watcom C pretends to be Microsoft C small model even in 32 bit mode.
+- Watcom C pretends to be Microsoft C small model even in 32 bit mode
 - don't use unsized arrays to avoid silly warnings by Visual C++:
      warning C4746: 'inflate_mask' : unsized array treated as  '__far'
-     (what's wrong with far data in far model?).
+     (what's wrong with far data in far model?)
 - define enum out of inflate_blocks_state to allow compilation with C++
 
 Changes in 0.95 (16 Aug 95)
 - fix MSDOS small and medium model (now easier to adapt to any compiler)
 - inlined send_bits
 - fix the final (:-) bug for deflate with flush (output was correct but
-  not completely flushed in rare occasions).
+  not completely flushed in rare occasions)
 - default window size is same for compression and decompression
-  (it's now sufficient to set MAX_WBITS in zconf.h).
+  (it's now sufficient to set MAX_WBITS in zconf.h)
 - voidp -> voidpf and voidnp -> voidp (for consistency with other
-  typedefs and because voidnp was not near in large model).
+  typedefs and because voidnp was not near in large model)
 
 Changes in 0.94 (13 Aug 95)
 - support MSDOS medium model
@@ -1502,12 +1610,12 @@ Changes in 0.94 (13 Aug 95)
 - added support for VMS
 - allow a compression level in gzopen()
 - gzflush now calls fflush
-- For deflate with flush, flush even if no more input is provided.
+- For deflate with flush, flush even if no more input is provided
 - rename libgz.a as libz.a
 - avoid complex expression in infcodes.c triggering Turbo C bug
 - work around a problem with gcc on Alpha (in INSERT_STRING)
 - don't use inline functions (problem with some gcc versions)
-- allow renaming of Byte, uInt, etc... with #define.
+- allow renaming of Byte, uInt, etc... with #define
 - avoid warning about (unused) pointer before start of array in deflate.c
 - avoid various warnings in gzio.c, example.c, infblock.c, adler32.c, zutil.c
 - avoid reserved word 'new' in trees.c
@@ -1526,7 +1634,7 @@ Changes in 0.92 (3 May 95)
 - no memcpy on Pyramid
 - suppressed inftest.c
 - optimized fill_window, put longest_match inline for gcc
-- optimized inflate on stored blocks.
+- optimized inflate on stored blocks
 - untabify all sources to simplify patches
 
 Changes in 0.91 (2 May 95)
@@ -1544,7 +1652,7 @@ Changes in 0.9 (1 May 95)
 - let again gzread copy uncompressed data unchanged (was working in 0.71)
 - deflate(Z_FULL_FLUSH), inflateReset and inflateSync are now fully implemented
 - added a test of inflateSync in example.c
-- moved MAX_WBITS to zconf.h because users might want to change that.
+- moved MAX_WBITS to zconf.h because users might want to change that
 - document explicitly that zalloc(64K) on MSDOS must return a normalized
   pointer (zero offset)
 - added Makefiles for Microsoft C, Turbo C, Borland C++
@@ -1553,7 +1661,7 @@ Changes in 0.9 (1 May 95)
 Changes in 0.8 (29 April 95)
 - added fast inflate (inffast.c)
 - deflate(Z_FINISH) now returns Z_STREAM_END when done. Warning: this
-  is incompatible with previous versions of zlib which returned Z_OK.
+  is incompatible with previous versions of zlib which returned Z_OK
 - work around a TurboC compiler bug (bad code for b << 0, see infutil.h)
   (actually that was not a compiler bug, see 0.81 above)
 - gzread no longer reads one extra byte in certain cases
@@ -1563,50 +1671,50 @@ Changes in 0.8 (29 April 95)
 
 Changes in 0.71 (14 April 95)
 - Fixed more MSDOS compilation problems :( There is still a bug with
-  TurboC large model.
+  TurboC large model
 
 Changes in 0.7 (14 April 95)
-- Added full inflate support.
+- Added full inflate support
 - Simplified the crc32() interface. The pre- and post-conditioning
   (one's complement) is now done inside crc32(). WARNING: this is
-  incompatible with previous versions; see zlib.h for the new usage.
+  incompatible with previous versions; see zlib.h for the new usage
 
 Changes in 0.61 (12 April 95)
-- workaround for a bug in TurboC. example and minigzip now work on MSDOS.
+- workaround for a bug in TurboC. example and minigzip now work on MSDOS
 
 Changes in 0.6 (11 April 95)
 - added minigzip.c
 - added gzdopen to reopen a file descriptor as gzFile
-- added transparent reading of non-gziped files in gzread.
+- added transparent reading of non-gziped files in gzread
 - fixed bug in gzread (don't read crc as data)
-- fixed bug in destroy (gzio.c) (don't return Z_STREAM_END for gzclose).
+- fixed bug in destroy (gzio.c) (don't return Z_STREAM_END for gzclose)
 - don't allocate big arrays in the stack (for MSDOS)
 - fix some MSDOS compilation problems
 
 Changes in 0.5:
 - do real compression in deflate.c. Z_PARTIAL_FLUSH is supported but
-  not yet Z_FULL_FLUSH.
+  not yet Z_FULL_FLUSH
 - support decompression but only in a single step (forced Z_FINISH)
-- added opaque object for zalloc and zfree.
+- added opaque object for zalloc and zfree
 - added deflateReset and inflateReset
-- added a variable zlib_version for consistency checking.
-- renamed the 'filter' parameter of deflateInit2 as 'strategy'.
-  Added Z_FILTERED and Z_HUFFMAN_ONLY constants.
+- added a variable zlib_version for consistency checking
+- renamed the 'filter' parameter of deflateInit2 as 'strategy'
+  Added Z_FILTERED and Z_HUFFMAN_ONLY constants
 
 Changes in 0.4:
-- avoid "zip" everywhere, use zlib instead of ziplib.
+- avoid "zip" everywhere, use zlib instead of ziplib
 - suppress Z_BLOCK_FLUSH, interpret Z_PARTIAL_FLUSH as block flush
-  if compression method == 8.
+  if compression method == 8
 - added adler32 and crc32
 - renamed deflateOptions as deflateInit2, call one or the other but not both
-- added the method parameter for deflateInit2.
+- added the method parameter for deflateInit2
 - added inflateInit2
-- simplied considerably deflateInit and inflateInit by not supporting
+- simplified considerably deflateInit and inflateInit by not supporting
   user-provided history buffer. This is supported only in deflateInit2
-  and inflateInit2.
+  and inflateInit2
 
 Changes in 0.3:
 - prefix all macro names with Z_
-- use Z_FINISH instead of deflateEnd to finish compression.
+- use Z_FINISH instead of deflateEnd to finish compression
 - added Z_HUFFMAN_ONLY
 - added gzerror()
diff --git a/zlib/FAQ b/zlib/FAQ
index 99b7cf9..92f5d3e 100644
--- a/zlib/FAQ
+++ b/zlib/FAQ
@@ -4,7 +4,7 @@
 
 If your question is not there, please check the zlib home page
 http://zlib.net/ which may have more recent information.
-The lastest zlib FAQ is at http://zlib.net/zlib_faq.html
+The latest zlib FAQ is at http://zlib.net/zlib_faq.html
 
 
  1. Is zlib Y2K-compliant?
@@ -14,8 +14,7 @@ The lastest zlib FAQ is at http://zlib.net/zlib_faq.html
  2. Where can I get a Windows DLL version?
 
     The zlib sources can be compiled without change to produce a DLL.  See the
-    file win32/DLL_FAQ.txt in the zlib distribution.  Pointers to the
-    precompiled DLL are found in the zlib web site at http://zlib.net/ .
+    file win32/DLL_FAQ.txt in the zlib distribution.
 
  3. Where can I get a Visual Basic interface to zlib?
 
diff --git a/zlib/LICENSE b/zlib/LICENSE
new file mode 100644
index 0000000..ab8ee6f
--- /dev/null
+++ b/zlib/LICENSE
@@ -0,0 +1,22 @@
+Copyright notice:
+
+ (C) 1995-2022 Jean-loup Gailly and Mark Adler
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  Jean-loup Gailly        Mark Adler
+  jloup@gzip.org          madler@alumni.caltech.edu
diff --git a/zlib/README b/zlib/README
index 92e9d5c..3f5a990 100644
--- a/zlib/README
+++ b/zlib/README
@@ -3,7 +3,7 @@ shipped with GCC as convenience.
 
 ZLIB DATA COMPRESSION LIBRARY
 
-zlib 1.2.11 is a general purpose data compression library.  All the code is
+zlib 1.3.1 is a general purpose data compression library.  All the code is
 thread safe.  The data format used by the zlib library is described by RFCs
 (Request for Comments) 1950 to 1952 in the files
 http://tools.ietf.org/html/rfc1950 (zlib format), rfc1951 (deflate format) and
@@ -32,18 +32,17 @@ PLEASE read the zlib FAQ http://zlib.net/zlib_faq.html before asking for help.
 
 Mark Nelson <markn@ieee.org> wrote an article about zlib for the Jan.  1997
 issue of Dr.  Dobb's Journal; a copy of the article is available at
-http://marknelson.us/1997/01/01/zlib-engine/ .
+https://marknelson.us/posts/1997/01/01/zlib-engine.html .
 
-The changes made in version 1.2.11 are documented in the file ChangeLog.
+The changes made in version 1.3.1 are documented in the file ChangeLog.
 
 Unsupported third party contributions are provided in directory contrib/ .
 
-zlib is available in Java using the java.util.zip package, documented at
-http://java.sun.com/developer/technicalArticles/Programming/compression/ .
+zlib is available in Java using the java.util.zip package. Follow the API
+Documentation link at: https://docs.oracle.com/search/?q=java.util.zip .
 
-A Perl interface to zlib written by Paul Marquess <pmqs@cpan.org> is available
-at CPAN (Comprehensive Perl Archive Network) sites, including
-http://search.cpan.org/~pmqs/IO-Compress-Zlib/ .
+A Perl interface to zlib and bzip2 written by Paul Marquess <pmqs@cpan.org>
+can be found at https://github.com/pmqs/IO-Compress .
 
 A Python interface to zlib written by A.M. Kuchling <amk@amk.ca> is
 available in Python 1.5 and later versions, see
@@ -67,7 +66,7 @@ Notes for some targets:
 - zlib doesn't work with gcc 2.6.3 on a DEC 3000/300LX under OSF/1 2.1 it works
   when compiled with cc.
 
-- On Digital Unix 4.0D (formely OSF/1) on AlphaServer, the cc option -std1 is
+- On Digital Unix 4.0D (formerly OSF/1) on AlphaServer, the cc option -std1 is
   necessary to get gzprintf working correctly. This is done by configure.
 
 - zlib doesn't work on HP-UX 9.05 with some versions of /bin/cc. It works with
@@ -87,7 +86,7 @@ Acknowledgments:
 
 Copyright notice:
 
- (C) 1995-2017 Jean-loup Gailly and Mark Adler
+ (C) 1995-2024 Jean-loup Gailly and Mark Adler
 
   This software is provided 'as-is', without any express or implied
   warranty.  In no event will the authors be held liable for any damages
@@ -111,7 +110,10 @@ Copyright notice:
 If you use the zlib library in a product, we would appreciate *not* receiving
 lengthy legal documents to sign.  The sources are provided for free but without
 warranty of any kind.  The library has been entirely written by Jean-loup
-Gailly and Mark Adler; it does not include third-party code.
+Gailly and Mark Adler; it does not include third-party code.  We make all
+contributions to and distributions of this project solely in our personal
+capacity, and are not conveying any rights to any intellectual property of
+any third parties.
 
 If you redistribute modified sources, we would appreciate that you include in
 the file ChangeLog history information documenting your changes.  Please read
diff --git a/zlib/adler32.c b/zlib/adler32.c
index b3d35e2..04b81d2 100644
--- a/zlib/adler32.c
+++ b/zlib/adler32.c
@@ -3,12 +3,10 @@
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
-/* @(#) $Id: adler32.c,v 1.1.1.2 2002/03/11 21:53:23 tromey Exp $ */
+/* @(#) $Id$ */
 
 #include "zutil.h"
 
-local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2));
-
 #define BASE 65521U     /* largest prime smaller than 65536 */
 #define NMAX 5552
 /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */
@@ -60,11 +58,7 @@ local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2));
 #endif
 
 /* ========================================================================= */
-uLong ZEXPORT adler32_z(adler, buf, len)
-    uLong adler;
-    const Bytef *buf;
-    z_size_t len;
-{
+uLong ZEXPORT adler32_z(uLong adler, const Bytef *buf, z_size_t len) {
     unsigned long sum2;
     unsigned n;
 
@@ -131,20 +125,12 @@ uLong ZEXPORT adler32_z(adler, buf, len)
 }
 
 /* ========================================================================= */
-uLong ZEXPORT adler32(adler, buf, len)
-    uLong adler;
-    const Bytef *buf;
-    uInt len;
-{
+uLong ZEXPORT adler32(uLong adler, const Bytef *buf, uInt len) {
     return adler32_z(adler, buf, len);
 }
 
 /* ========================================================================= */
-local uLong adler32_combine_(adler1, adler2, len2)
-    uLong adler1;
-    uLong adler2;
-    z_off64_t len2;
-{
+local uLong adler32_combine_(uLong adler1, uLong adler2, z_off64_t len2) {
     unsigned long sum1;
     unsigned long sum2;
     unsigned rem;
@@ -169,18 +155,10 @@ local uLong adler32_combine_(adler1, adler2, len2)
 }
 
 /* ========================================================================= */
-uLong ZEXPORT adler32_combine(adler1, adler2, len2)
-    uLong adler1;
-    uLong adler2;
-    z_off_t len2;
-{
+uLong ZEXPORT adler32_combine(uLong adler1, uLong adler2, z_off_t len2) {
     return adler32_combine_(adler1, adler2, len2);
 }
 
-uLong ZEXPORT adler32_combine64(adler1, adler2, len2)
-    uLong adler1;
-    uLong adler2;
-    z_off64_t len2;
-{
+uLong ZEXPORT adler32_combine64(uLong adler1, uLong adler2, z_off64_t len2) {
     return adler32_combine_(adler1, adler2, len2);
 }
diff --git a/zlib/compress.c b/zlib/compress.c
index bb443c6..f43bacf7 100644
--- a/zlib/compress.c
+++ b/zlib/compress.c
@@ -3,7 +3,7 @@
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
-/* @(#) $Id: compress.c,v 1.1.1.2 2002/03/11 21:53:23 tromey Exp $ */
+/* @(#) $Id$ */
 
 #define ZLIB_INTERNAL
 #include "zlib.h"
@@ -19,13 +19,8 @@
    memory, Z_BUF_ERROR if there was not enough room in the output buffer,
    Z_STREAM_ERROR if the level parameter is invalid.
 */
-int ZEXPORT compress2 (dest, destLen, source, sourceLen, level)
-    Bytef *dest;
-    uLongf *destLen;
-    const Bytef *source;
-    uLong sourceLen;
-    int level;
-{
+int ZEXPORT compress2(Bytef *dest, uLongf *destLen, const Bytef *source,
+                      uLong sourceLen, int level) {
     z_stream stream;
     int err;
     const uInt max = (uInt)-1;
@@ -65,12 +60,8 @@ int ZEXPORT compress2 (dest, destLen, source, sourceLen, level)
 
 /* ===========================================================================
  */
-int ZEXPORT compress (dest, destLen, source, sourceLen)
-    Bytef *dest;
-    uLongf *destLen;
-    const Bytef *source;
-    uLong sourceLen;
-{
+int ZEXPORT compress(Bytef *dest, uLongf *destLen, const Bytef *source,
+                     uLong sourceLen) {
     return compress2(dest, destLen, source, sourceLen, Z_DEFAULT_COMPRESSION);
 }
 
@@ -78,9 +69,7 @@ int ZEXPORT compress (dest, destLen, source, sourceLen)
      If the default memLevel or windowBits for deflateInit() is changed, then
    this function needs to be updated.
  */
-uLong ZEXPORT compressBound (sourceLen)
-    uLong sourceLen;
-{
+uLong ZEXPORT compressBound(uLong sourceLen) {
     return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) +
            (sourceLen >> 25) + 13;
 }
diff --git a/zlib/configure b/zlib/configure
index 92c462d..202c15f 100755
--- a/zlib/configure
+++ b/zlib/configure
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for zlib 1.1.4.
+# Generated by GNU Autoconf 2.69 for zlib 1.3.1.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -587,8 +587,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='zlib'
 PACKAGE_TARNAME='zlib'
-PACKAGE_VERSION='1.1.4'
-PACKAGE_STRING='zlib 1.1.4'
+PACKAGE_VERSION='1.3.1'
+PACKAGE_STRING='zlib 1.3.1'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1330,7 +1330,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures zlib 1.1.4 to adapt to many kinds of systems.
+\`configure' configures zlib 1.3.1 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1401,7 +1401,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of zlib 1.1.4:";;
+     short | recursive ) echo "Configuration of zlib 1.3.1:";;
    esac
   cat <<\_ACEOF
 
@@ -1521,7 +1521,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-zlib configure 1.1.4
+zlib configure 1.3.1
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1889,7 +1889,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by zlib $as_me 1.1.4, which was
+It was created by zlib $as_me 1.3.1, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2916,7 +2916,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='zlib'
- VERSION='1.1.4'
+ VERSION='1.3.1'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -12256,7 +12256,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by zlib $as_me 1.1.4, which was
+This file was extended by zlib $as_me 1.3.1, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -12313,7 +12313,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-zlib config.status 1.1.4
+zlib config.status 1.3.1
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 
diff --git a/zlib/configure.ac b/zlib/configure.ac
index 9501cdf..736b760 100644
--- a/zlib/configure.ac
+++ b/zlib/configure.ac
@@ -1,6 +1,6 @@
 dnl Process this with autoconf to create configure
 
-AC_INIT([zlib], [1.1.4])
+AC_INIT([zlib], [1.3.1])
 AC_CONFIG_SRCDIR([zlib.h])
 
 if test -n "${with_target_subdir}"; then
diff --git a/zlib/contrib/README.contrib b/zlib/contrib/README.contrib
index a411d5c..5e5f950 100644
--- a/zlib/contrib/README.contrib
+++ b/zlib/contrib/README.contrib
@@ -1,4 +1,4 @@
-All files under this contrib directory are UNSUPPORTED. There were
+All files under this contrib directory are UNSUPPORTED. They were
 provided by users of zlib and were not tested by the authors of zlib.
 Use at your own risk. Please contact the authors of the contributions
 for help about these, not the zlib authors. Thanks.
@@ -8,14 +8,6 @@ ada/        by Dmitriy Anisimkov <anisimkov@yahoo.com>
         Support for Ada
         See http://zlib-ada.sourceforge.net/
 
-amd64/      by Mikhail Teterin <mi@ALDAN.algebra.com>
-        asm code for AMD64
-        See patch at http://www.freebsd.org/cgi/query-pr.cgi?pr=bin/96393
-
-asm686/     by Brian Raiter <breadbox@muppetlabs.com>
-        asm code for Pentium and PPro/PII, using the AT&T (GNU as) syntax
-        See http://www.muppetlabs.com/~breadbox/software/assembly.html
-
 blast/      by Mark Adler <madler@alumni.caltech.edu>
         Decompressor for output of PKWare Data Compression Library (DCL)
 
@@ -32,9 +24,6 @@ gcc_gvmat64/by Gilles Vollant <info@winimage.com>
 infback9/   by Mark Adler <madler@alumni.caltech.edu>
         Unsupported diffs to infback to decode the deflate64 format
 
-inflate86/  by Chris Anderson <christop@charm.net>
-        Tuned x86 gcc asm code to replace inflate_fast()
-
 iostream/   by Kevin Ruland <kevin@rodin.wustl.edu>
         A C++ I/O streams interface to the zlib gz* functions
 
@@ -45,16 +34,6 @@ iostream3/  by Ludwig Schwardt <schwardt@sun.ac.za>
             and Kevin Ruland <kevin@rodin.wustl.edu>
         Yet another C++ I/O streams interface
 
-masmx64/    by Gilles Vollant <info@winimage.com>
-        x86 64-bit (AMD64 and Intel EM64t) code for x64 assembler to
-        replace longest_match() and inflate_fast(),  also masm x86
-        64-bits translation of Chris Anderson inflate_fast()
-
-masmx86/    by Gilles Vollant <info@winimage.com>
-        x86 asm code to replace longest_match() and inflate_fast(),
-        for Visual C++ and MASM (32 bits).
-        Based on Brian Raiter (asm686) and Chris Anderson (inflate86)
-
 minizip/    by Gilles Vollant <info@winimage.com>
         Mini zip and unzip based on zlib
         Includes Zip64 support by Mathias Svensson <mathias@result42.com>
diff --git a/zlib/contrib/ada/readme.txt b/zlib/contrib/ada/readme.txt
index ce4d2ca..efdd639 100644
--- a/zlib/contrib/ada/readme.txt
+++ b/zlib/contrib/ada/readme.txt
@@ -8,8 +8,8 @@ It provides Ada-style access to the ZLib C library.
 
         Here are the main changes since ZLib.Ada 1.2:
 
-- Attension: ZLib.Read generic routine have a initialization requirement
-  for Read_Last parameter now. It is a bit incompartible with previous version,
+- Attention: ZLib.Read generic routine have a initialization requirement
+  for Read_Last parameter now. It is a bit incompatible with previous version,
   but extends functionality, we could use new parameters Allow_Read_Some and
   Flush now.
 
diff --git a/zlib/contrib/ada/test.adb b/zlib/contrib/ada/test.adb
index 90773ac..8b35031 100644
--- a/zlib/contrib/ada/test.adb
+++ b/zlib/contrib/ada/test.adb
@@ -65,12 +65,12 @@ procedure Test is
    Time_Stamp : Ada.Calendar.Time;
 
    procedure Generate_File;
-   --  Generate file of spetsified size with some random data.
+   --  Generate file of specified size with some random data.
    --  The random data is repeatable, for the good compression.
 
    procedure Compare_Streams
      (Left, Right : in out Root_Stream_Type'Class);
-   --  The procedure compearing data in 2 streams.
+   --  The procedure comparing data in 2 streams.
    --  It is for compare data before and after compression/decompression.
 
    procedure Compare_Files (Left, Right : String);
diff --git a/zlib/contrib/ada/zlib-streams.ads b/zlib/contrib/ada/zlib-streams.ads
index 8e26cd4..af13693 100644
--- a/zlib/contrib/ada/zlib-streams.ads
+++ b/zlib/contrib/ada/zlib-streams.ads
@@ -62,7 +62,7 @@ package ZLib.Streams is
                                     := Default_Buffer_Size;
       Write_Buffer_Size : in     Ada.Streams.Stream_Element_Offset
                                     := Default_Buffer_Size);
-   --  Create the Comression/Decompression stream.
+   --  Create the Compression/Decompression stream.
    --  If mode is In_Stream then Write operation is disabled.
    --  If mode is Out_Stream then Read operation is disabled.
 
diff --git a/zlib/contrib/ada/zlib.adb b/zlib/contrib/ada/zlib.adb
index 8b6fd68..c1abe791 100644
--- a/zlib/contrib/ada/zlib.adb
+++ b/zlib/contrib/ada/zlib.adb
@@ -204,7 +204,7 @@ package body ZLib is
       end if;
 
       --  We allow ZLib to make header only in case of default header type.
-      --  Otherwise we would either do header by ourselfs, or do not do
+      --  Otherwise we would either do header by ourselves, or do not do
       --  header at all.
 
       if Header = None or else Header = GZip then
diff --git a/zlib/contrib/ada/zlib.ads b/zlib/contrib/ada/zlib.ads
index 79ffc40..81aaf1b 100644
--- a/zlib/contrib/ada/zlib.ads
+++ b/zlib/contrib/ada/zlib.ads
@@ -114,7 +114,7 @@ package ZLib is
    -- Compression strategy constants --
    ------------------------------------
 
-   --  RLE stategy could be used only in version 1.2.0 and later.
+   --  RLE strategy could be used only in version 1.2.0 and later.
 
    Filtered         : constant Strategy_Type;
    Huffman_Only     : constant Strategy_Type;
diff --git a/zlib/contrib/blast/blast.h b/zlib/contrib/blast/blast.h
index 6cf65ed..ef8544c 100644
--- a/zlib/contrib/blast/blast.h
+++ b/zlib/contrib/blast/blast.h
@@ -57,7 +57,7 @@ int blast(blast_in infun, void *inhow, blast_out outfun, void *outhow,
  * use by the application to pass an input descriptor to infun(), if desired.
  *
  * If left and in are not NULL and *left is not zero when blast() is called,
- * then the *left bytes are *in are consumed for input before infun() is used.
+ * then the *left bytes at *in are consumed for input before infun() is used.
  *
  * The output function is invoked: err = outfun(how, buf, len), where the bytes
  * to be written are buf[0..len-1].  If err is not zero, then blast() returns
diff --git a/zlib/contrib/delphi/ZLib.pas b/zlib/contrib/delphi/ZLib.pas
index 060e199..93fa4c9 100644
--- a/zlib/contrib/delphi/ZLib.pas
+++ b/zlib/contrib/delphi/ZLib.pas
@@ -152,7 +152,7 @@ procedure DecompressToUserBuf(const InBuf: Pointer; InBytes: Integer;
   const OutBuf: Pointer; BufSize: Integer);
 
 const
-  zlib_version = '1.2.11';
+  zlib_version = '1.3.1';
 
 type
   EZlibError = class(Exception);
diff --git a/zlib/contrib/dotzlib/DotZLib.chm b/zlib/contrib/dotzlib/DotZLib.chm
index 0bc7df7..f214a44 100644
--- a/zlib/contrib/dotzlib/DotZLib.chm
+++ b/zlib/contrib/dotzlib/DotZLib.chm
diff --git a/zlib/contrib/dotzlib/DotZLib/ChecksumImpl.cs b/zlib/contrib/dotzlib/DotZLib/ChecksumImpl.cs
index 788b2fc..de88dcf 100644
--- a/zlib/contrib/dotzlib/DotZLib/ChecksumImpl.cs
+++ b/zlib/contrib/dotzlib/DotZLib/ChecksumImpl.cs
@@ -34,7 +34,7 @@ namespace DotZLib
         }

 

         /// <summary>

-        /// Initializes a new instance of the checksum generator basewith a specified value

+        /// Initializes a new instance of the checksum generator base with a specified value

         /// </summary>

         /// <param name="initialValue">The value to set the current checksum to</param>

         public ChecksumGeneratorBase(uint initialValue)

@@ -61,7 +61,7 @@ namespace DotZLib
         /// <exception cref="ArgumentException">The sum of offset and count is larger than the length of <c>data</c></exception>

         /// <exception cref="NullReferenceException"><c>data</c> is a null reference</exception>

         /// <exception cref="ArgumentOutOfRangeException">Offset or count is negative.</exception>

-        /// <remarks>All the other <c>Update</c> methods are implmeneted in terms of this one.

+        /// <remarks>All the other <c>Update</c> methods are implemented in terms of this one.

         /// This is therefore the only method a derived class has to implement</remarks>

         public abstract void Update(byte[] data, int offset, int count);

 

diff --git a/zlib/contrib/dotzlib/DotZLib/CodecBase.cs b/zlib/contrib/dotzlib/DotZLib/CodecBase.cs
index 42e6da3..c4bc8b8 100644
--- a/zlib/contrib/dotzlib/DotZLib/CodecBase.cs
+++ b/zlib/contrib/dotzlib/DotZLib/CodecBase.cs
@@ -139,7 +139,7 @@ namespace DotZLib
         /// <remarks>This must be implemented by a derived class</remarks>

         protected abstract void CleanUp();

 

-        // performs the release of the handles and calls the dereived CleanUp()

+        // performs the release of the handles and calls the derived CleanUp()

         private void CleanUp(bool isDisposing)

         {

             if (!_isDisposed)

@@ -160,7 +160,7 @@ namespace DotZLib
         #region Helper methods

 

         /// <summary>

-        /// Copies a number of bytes to the internal codec buffer - ready for proccesing

+        /// Copies a number of bytes to the internal codec buffer - ready for processing

         /// </summary>

         /// <param name="data">The byte array that contains the data to copy</param>

         /// <param name="startIndex">The index of the first byte to copy</param>

diff --git a/zlib/contrib/dotzlib/DotZLib/GZipStream.cs b/zlib/contrib/dotzlib/DotZLib/GZipStream.cs
index b161300..58091d3 100644
--- a/zlib/contrib/dotzlib/DotZLib/GZipStream.cs
+++ b/zlib/contrib/dotzlib/DotZLib/GZipStream.cs
@@ -246,7 +246,7 @@ namespace DotZLib
         }

 

         /// <summary>

-        ///  Not suppported.

+        ///  Not supported.

         /// </summary>

         /// <param name="offset"></param>

         /// <param name="origin"></param>

@@ -268,7 +268,7 @@ namespace DotZLib
         }

 

         /// <summary>

-        /// Gets/sets the current position in the <c>GZipStream</c>. Not suppported.

+        /// Gets/sets the current position in the <c>GZipStream</c>. Not supported.

         /// </summary>

         /// <remarks>In this implementation this property is not supported</remarks>

         /// <exception cref="NotSupportedException">Always thrown</exception>

@@ -285,7 +285,7 @@ namespace DotZLib
         }

 

         /// <summary>

-        /// Gets the size of the stream. Not suppported.

+        /// Gets the size of the stream. Not supported.

         /// </summary>

         /// <remarks>In this implementation this property is not supported</remarks>

         /// <exception cref="NotSupportedException">Always thrown</exception>

diff --git a/zlib/contrib/dotzlib/DotZLib/UnitTests.cs b/zlib/contrib/dotzlib/DotZLib/UnitTests.cs
index 44f7633..d4f0980 100644
--- a/zlib/contrib/dotzlib/DotZLib/UnitTests.cs
+++ b/zlib/contrib/dotzlib/DotZLib/UnitTests.cs
@@ -156,7 +156,7 @@ namespace DotZLibTests
         public void Info_Version()

         {

             Info info = new Info();

-            Assert.AreEqual("1.2.11", Info.Version);

+            Assert.AreEqual("1.3.1", Info.Version);

             Assert.AreEqual(32, info.SizeOfUInt);

             Assert.AreEqual(32, info.SizeOfULong);

             Assert.AreEqual(32, info.SizeOfPointer);

diff --git a/zlib/contrib/dotzlib/readme.txt b/zlib/contrib/dotzlib/readme.txt
index b239572..47454fc 100644
--- a/zlib/contrib/dotzlib/readme.txt
+++ b/zlib/contrib/dotzlib/readme.txt
@@ -36,7 +36,7 @@ Build instructions:
    in the same directory as the DotZLib.build file.

    You can define 2 properties on the nant command-line to control the build:

    debug={true|false} to toggle between release/debug builds (default=true).

-   nunit={true|false} to include or esclude unit tests (default=true).

+   nunit={true|false} to include or exclude unit tests (default=true).

    Also the target clean will remove binaries.

    Output file (DotZLib.dll) will be found in either ./DotZLib/bin/release

    or ./DotZLib/bin/debug, depending on whether you are building the release

diff --git a/zlib/contrib/infback9/infback9.c b/zlib/contrib/infback9/infback9.c
index 05fb3e3..742a392 100644
--- a/zlib/contrib/infback9/infback9.c
+++ b/zlib/contrib/infback9/infback9.c
@@ -16,12 +16,8 @@
 
    window is a user-supplied window and output buffer that is 64K bytes.
  */
-int ZEXPORT inflateBack9Init_(strm, window, version, stream_size)
-z_stream FAR *strm;
-unsigned char FAR *window;
-const char *version;
-int stream_size;
-{
+int ZEXPORT inflateBack9Init_(z_stream FAR *strm, unsigned char FAR *window,
+                              const char *version, int stream_size) {
     struct inflate_state FAR *state;
 
     if (version == Z_NULL || version[0] != ZLIB_VERSION[0] ||
@@ -51,8 +47,7 @@ int stream_size;
 #ifdef MAKEFIXED
 #include <stdio.h>
 
-void makefixed9(void)
-{
+void makefixed9(void) {
     unsigned sym, bits, low, size;
     code *next, *lenfix, *distfix;
     struct inflate_state state;
@@ -214,13 +209,8 @@ void makefixed9(void)
    inflateBack() can also return Z_STREAM_ERROR if the input parameters
    are not correct, i.e. strm is Z_NULL or the state was not initialized.
  */
-int ZEXPORT inflateBack9(strm, in, in_desc, out, out_desc)
-z_stream FAR *strm;
-in_func in;
-void FAR *in_desc;
-out_func out;
-void FAR *out_desc;
-{
+int ZEXPORT inflateBack9(z_stream FAR *strm, in_func in, void FAR *in_desc,
+                         out_func out, void FAR *out_desc) {
     struct inflate_state FAR *state;
     z_const unsigned char FAR *next;    /* next input */
     unsigned char FAR *put;     /* next output */
@@ -603,9 +593,7 @@ void FAR *out_desc;
     return ret;
 }
 
-int ZEXPORT inflateBack9End(strm)
-z_stream FAR *strm;
-{
+int ZEXPORT inflateBack9End(z_stream FAR *strm) {
     if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0)
         return Z_STREAM_ERROR;
     ZFREE(strm, strm->state);
diff --git a/zlib/contrib/infback9/infback9.h b/zlib/contrib/infback9/infback9.h
index 1073c0a..8371b4e 100644
--- a/zlib/contrib/infback9/infback9.h
+++ b/zlib/contrib/infback9/infback9.h
@@ -20,14 +20,14 @@
 extern "C" {
 #endif
 
-ZEXTERN int ZEXPORT inflateBack9 OF((z_stream FAR *strm,
-                                    in_func in, void FAR *in_desc,
-                                    out_func out, void FAR *out_desc));
-ZEXTERN int ZEXPORT inflateBack9End OF((z_stream FAR *strm));
-ZEXTERN int ZEXPORT inflateBack9Init_ OF((z_stream FAR *strm,
-                                         unsigned char FAR *window,
-                                         const char *version,
-                                         int stream_size));
+ZEXTERN int ZEXPORT inflateBack9(z_stream FAR *strm,
+                                 in_func in, void FAR *in_desc,
+                                 out_func out, void FAR *out_desc);
+ZEXTERN int ZEXPORT inflateBack9End(z_stream FAR *strm);
+ZEXTERN int ZEXPORT inflateBack9Init_(z_stream FAR *strm,
+                                      unsigned char FAR *window,
+                                      const char *version,
+                                      int stream_size);
 #define inflateBack9Init(strm, window) \
         inflateBack9Init_((strm), (window), \
         ZLIB_VERSION, sizeof(z_stream))
diff --git a/zlib/contrib/infback9/inftree9.c b/zlib/contrib/infback9/inftree9.c
index 5f4a767..ac707ed 100644
--- a/zlib/contrib/infback9/inftree9.c
+++ b/zlib/contrib/infback9/inftree9.c
@@ -1,5 +1,5 @@
 /* inftree9.c -- generate Huffman trees for efficient decoding
- * Copyright (C) 1995-2017 Mark Adler
+ * Copyright (C) 1995-2024 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -9,7 +9,7 @@
 #define MAXBITS 15
 
 const char inflate9_copyright[] =
-   " inflate9 1.2.11 Copyright 1995-2017 Mark Adler ";
+   " inflate9 1.3.1 Copyright 1995-2024 Mark Adler ";
 /*
   If you use the zlib library in a product, an acknowledgment is welcome
   in the documentation of your product. If for some reason you cannot
@@ -29,14 +29,9 @@ const char inflate9_copyright[] =
    table index bits.  It will differ if the request is greater than the
    longest code or if it is less than the shortest code.
  */
-int inflate_table9(type, lens, codes, table, bits, work)
-codetype type;
-unsigned short FAR *lens;
-unsigned codes;
-code FAR * FAR *table;
-unsigned FAR *bits;
-unsigned short FAR *work;
-{
+int inflate_table9(codetype type, unsigned short FAR *lens, unsigned codes,
+                   code FAR * FAR *table, unsigned FAR *bits,
+                   unsigned short FAR *work) {
     unsigned len;               /* a code's length in bits */
     unsigned sym;               /* index of code symbols */
     unsigned min, max;          /* minimum and maximum code lengths */
@@ -64,7 +59,7 @@ unsigned short FAR *work;
     static const unsigned short lext[31] = { /* Length codes 257..285 extra */
         128, 128, 128, 128, 128, 128, 128, 128, 129, 129, 129, 129,
         130, 130, 130, 130, 131, 131, 131, 131, 132, 132, 132, 132,
-        133, 133, 133, 133, 144, 77, 202};
+        133, 133, 133, 133, 144, 203, 77};
     static const unsigned short dbase[32] = { /* Distance codes 0..31 base */
         1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49,
         65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073,
diff --git a/zlib/contrib/infback9/inftree9.h b/zlib/contrib/infback9/inftree9.h
index 5ab21f0..ab2ea28 100644
--- a/zlib/contrib/infback9/inftree9.h
+++ b/zlib/contrib/infback9/inftree9.h
@@ -38,11 +38,11 @@ typedef struct {
 /* Maximum size of the dynamic table.  The maximum number of code structures is
    1446, which is the sum of 852 for literal/length codes and 594 for distance
    codes.  These values were found by exhaustive searches using the program
-   examples/enough.c found in the zlib distribtution.  The arguments to that
+   examples/enough.c found in the zlib distribution.  The arguments to that
    program are the number of symbols, the initial root table size, and the
    maximum bit length of a code.  "enough 286 9 15" for literal/length codes
-   returns returns 852, and "enough 32 6 15" for distance codes returns 594.
-   The initial root table size (9 or 6) is found in the fifth argument of the
+   returns 852, and "enough 32 6 15" for distance codes returns 594. The
+   initial root table size (9 or 6) is found in the fifth argument of the
    inflate_table() calls in infback9.c.  If the root table size is changed,
    then these maximum sizes would be need to be recalculated and updated. */
 #define ENOUGH_LENS 852
@@ -56,6 +56,6 @@ typedef enum {
     DISTS
 } codetype;
 
-extern int inflate_table9 OF((codetype type, unsigned short FAR *lens,
-                             unsigned codes, code FAR * FAR *table,
-                             unsigned FAR *bits, unsigned short FAR *work));
+extern int inflate_table9(codetype type, unsigned short FAR *lens,
+                          unsigned codes, code FAR * FAR *table,
+                          unsigned FAR *bits, unsigned short FAR *work);
diff --git a/zlib/contrib/iostream2/zstream.h b/zlib/contrib/iostream2/zstream.h
index be7829b..43d2332 100644
--- a/zlib/contrib/iostream2/zstream.h
+++ b/zlib/contrib/iostream2/zstream.h
@@ -21,7 +21,7 @@
 
 /*
  * zstream.h - C++ interface to the 'zlib' general purpose compression library
- * $Id: zstream.h,v 1.1 1999/05/04 19:30:27 tromey Exp $
+ * $Id: zstream.h 1.1 1997-06-25 12:00:56+02 tyge Exp tyge $
  */
 
 #include <strstream.h>
diff --git a/zlib/contrib/iostream3/zfstream.h b/zlib/contrib/iostream3/zfstream.h
index 8574479..3dabc0f 100644
--- a/zlib/contrib/iostream3/zfstream.h
+++ b/zlib/contrib/iostream3/zfstream.h
@@ -413,7 +413,7 @@ template<typename T1, typename T2>
   class gzomanip2
   {
   public:
-    // Allows insertor to peek at internals
+    // Allows inserter to peek at internals
     template <typename Ta, typename Tb>
       friend gzofstream&
       operator<<(gzofstream&,
@@ -452,7 +452,7 @@ template<typename T1, typename T2>
   : func(f), val1(v1), val2(v2)
   { }
 
-// Insertor applies underlying manipulator function to stream
+// Inserter applies underlying manipulator function to stream
 template<typename T1, typename T2>
   inline gzofstream&
   operator<<(gzofstream& s, const gzomanip2<T1,T2>& m)
diff --git a/zlib/contrib/minizip/Makefile b/zlib/contrib/minizip/Makefile
index 84eaad2..3d927ec 100644
--- a/zlib/contrib/minizip/Makefile
+++ b/zlib/contrib/minizip/Makefile
@@ -1,5 +1,5 @@
-CC=cc
-CFLAGS=-O -I../..
+CC?=cc
+CFLAGS := $(CFLAGS) -O -I../..
 
 UNZ_OBJS = miniunz.o unzip.o ioapi.o ../../libz.a
 ZIP_OBJS = minizip.o zip.o   ioapi.o ../../libz.a
@@ -16,10 +16,14 @@ minizip:  $(ZIP_OBJS)
 	$(CC) $(CFLAGS) -o $@ $(ZIP_OBJS)
 
 test:	miniunz minizip
-	./minizip test readme.txt
+	@rm -f test.*
+	@echo hello hello hello > test.txt
+	./minizip test test.txt
 	./miniunz -l test.zip
-	mv readme.txt readme.old
+	@mv test.txt test.old
 	./miniunz test.zip
+	@cmp test.txt test.old
+	@rm -f test.*
 
 clean:
-	/bin/rm -f *.o *~ minizip miniunz
+	/bin/rm -f *.o *~ minizip miniunz test.*
diff --git a/zlib/contrib/minizip/MiniZip64_Changes.txt b/zlib/contrib/minizip/MiniZip64_Changes.txt
index 13a1bd9..3759468 100644
--- a/zlib/contrib/minizip/MiniZip64_Changes.txt
+++ b/zlib/contrib/minizip/MiniZip64_Changes.txt
@@ -1,5 +1,5 @@
 
-MiniZip 1.1 was derrived from MiniZip at version 1.01f
+MiniZip 1.1 was derived from MiniZip at version 1.01f
 
 Change in 1.0 (Okt 2009)
  - **TODO - Add history**
diff --git a/zlib/contrib/minizip/configure.ac b/zlib/contrib/minizip/configure.ac
index 5b11970..15ec917 100644
--- a/zlib/contrib/minizip/configure.ac
+++ b/zlib/contrib/minizip/configure.ac
@@ -1,7 +1,7 @@
 #                                               -*- Autoconf -*-
 # Process this file with autoconf to produce a configure script.
 
-AC_INIT([minizip], [1.2.11], [bugzilla.redhat.com])
+AC_INIT([minizip], [1.3.1], [bugzilla.redhat.com])
 AC_CONFIG_SRCDIR([minizip.c])
 AM_INIT_AUTOMAKE([foreign])
 LT_INIT
diff --git a/zlib/contrib/minizip/crypt.h b/zlib/contrib/minizip/crypt.h
index 1e9e820..f4b93b7 100644
--- a/zlib/contrib/minizip/crypt.h
+++ b/zlib/contrib/minizip/crypt.h
@@ -32,12 +32,12 @@
 /***********************************************************************
  * Return the next byte in the pseudo-random sequence
  */
-static int decrypt_byte(unsigned long* pkeys, const z_crc_t* pcrc_32_tab)
-{
+static int decrypt_byte(unsigned long* pkeys, const z_crc_t* pcrc_32_tab) {
     unsigned temp;  /* POTENTIAL BUG:  temp*(temp^1) may overflow in an
                      * unpredictable manner on 16-bit systems; not a problem
                      * with any known compiler so far, though */
 
+    (void)pcrc_32_tab;
     temp = ((unsigned)(*(pkeys+2)) & 0xffff) | 2;
     return (int)(((temp * (temp ^ 1)) >> 8) & 0xff);
 }
@@ -45,8 +45,7 @@ static int decrypt_byte(unsigned long* pkeys, const z_crc_t* pcrc_32_tab)
 /***********************************************************************
  * Update the encryption keys with the next byte of plain text
  */
-static int update_keys(unsigned long* pkeys,const z_crc_t* pcrc_32_tab,int c)
-{
+static int update_keys(unsigned long* pkeys, const z_crc_t* pcrc_32_tab, int c) {
     (*(pkeys+0)) = CRC32((*(pkeys+0)), c);
     (*(pkeys+1)) += (*(pkeys+0)) & 0xff;
     (*(pkeys+1)) = (*(pkeys+1)) * 134775813L + 1;
@@ -62,8 +61,7 @@ static int update_keys(unsigned long* pkeys,const z_crc_t* pcrc_32_tab,int c)
  * Initialize the encryption keys and the random header according to
  * the given password.
  */
-static void init_keys(const char* passwd,unsigned long* pkeys,const z_crc_t* pcrc_32_tab)
-{
+static void init_keys(const char* passwd, unsigned long* pkeys, const z_crc_t* pcrc_32_tab) {
     *(pkeys+0) = 305419896L;
     *(pkeys+1) = 591751049L;
     *(pkeys+2) = 878082192L;
@@ -77,24 +75,23 @@ static void init_keys(const char* passwd,unsigned long* pkeys,const z_crc_t* pcr
     (update_keys(pkeys,pcrc_32_tab,c ^= decrypt_byte(pkeys,pcrc_32_tab)))
 
 #define zencode(pkeys,pcrc_32_tab,c,t) \
-    (t=decrypt_byte(pkeys,pcrc_32_tab), update_keys(pkeys,pcrc_32_tab,c), t^(c))
+    (t=decrypt_byte(pkeys,pcrc_32_tab), update_keys(pkeys,pcrc_32_tab,c), (Byte)t^(c))
 
 #ifdef INCLUDECRYPTINGCODE_IFCRYPTALLOWED
 
 #define RAND_HEAD_LEN  12
    /* "last resort" source for second part of crypt seed pattern */
 #  ifndef ZCR_SEED2
-#    define ZCR_SEED2 3141592654UL     /* use PI as default pattern */
+#    define ZCR_SEED2 3141592654UL      /* use PI as default pattern */
 #  endif
 
-static int crypthead(const char* passwd,      /* password string */
-                     unsigned char* buf,      /* where to write header */
-                     int bufSize,
-                     unsigned long* pkeys,
-                     const z_crc_t* pcrc_32_tab,
-                     unsigned long crcForCrypting)
-{
-    int n;                       /* index in random header */
+static unsigned crypthead(const char* passwd,       /* password string */
+                          unsigned char* buf,       /* where to write header */
+                          int bufSize,
+                          unsigned long* pkeys,
+                          const z_crc_t* pcrc_32_tab,
+                          unsigned long crcForCrypting) {
+    unsigned n;                  /* index in random header */
     int t;                       /* temporary */
     int c;                       /* random byte */
     unsigned char header[RAND_HEAD_LEN-2]; /* random header */
diff --git a/zlib/contrib/minizip/ioapi.c b/zlib/contrib/minizip/ioapi.c
index 7f5c191..782d324 100644
--- a/zlib/contrib/minizip/ioapi.c
+++ b/zlib/contrib/minizip/ioapi.c
@@ -14,7 +14,7 @@
         #define _CRT_SECURE_NO_WARNINGS
 #endif
 
-#if defined(__APPLE__) || defined(IOAPI_NO_64)
+#if defined(__APPLE__) || defined(IOAPI_NO_64) || defined(__HAIKU__) || defined(MINIZIP_FOPEN_NO_64)
 // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions
 #define FOPEN_FUNC(filename, mode) fopen(filename, mode)
 #define FTELLO_FUNC(stream) ftello(stream)
@@ -28,8 +28,7 @@
 
 #include "ioapi.h"
 
-voidpf call_zopen64 (const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode)
-{
+voidpf call_zopen64 (const zlib_filefunc64_32_def* pfilefunc, const void*filename, int mode) {
     if (pfilefunc->zfile_func64.zopen64_file != NULL)
         return (*(pfilefunc->zfile_func64.zopen64_file)) (pfilefunc->zfile_func64.opaque,filename,mode);
     else
@@ -38,8 +37,7 @@ voidpf call_zopen64 (const zlib_filefunc64_32_def* pfilefunc,const void*filename
     }
 }
 
-long call_zseek64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin)
-{
+long call_zseek64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin) {
     if (pfilefunc->zfile_func64.zseek64_file != NULL)
         return (*(pfilefunc->zfile_func64.zseek64_file)) (pfilefunc->zfile_func64.opaque,filestream,offset,origin);
     else
@@ -52,13 +50,12 @@ long call_zseek64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZP
     }
 }
 
-ZPOS64_T call_ztell64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream)
-{
+ZPOS64_T call_ztell64 (const zlib_filefunc64_32_def* pfilefunc, voidpf filestream) {
     if (pfilefunc->zfile_func64.zseek64_file != NULL)
         return (*(pfilefunc->zfile_func64.ztell64_file)) (pfilefunc->zfile_func64.opaque,filestream);
     else
     {
-        uLong tell_uLong = (*(pfilefunc->ztell32_file))(pfilefunc->zfile_func64.opaque,filestream);
+        uLong tell_uLong = (uLong)(*(pfilefunc->ztell32_file))(pfilefunc->zfile_func64.opaque,filestream);
         if ((tell_uLong) == MAXU32)
             return (ZPOS64_T)-1;
         else
@@ -66,11 +63,9 @@ ZPOS64_T call_ztell64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream
     }
 }
 
-void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32)
-{
+void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32, const zlib_filefunc_def* p_filefunc32) {
     p_filefunc64_32->zfile_func64.zopen64_file = NULL;
     p_filefunc64_32->zopen32_file = p_filefunc32->zopen_file;
-    p_filefunc64_32->zfile_func64.zerror_file = p_filefunc32->zerror_file;
     p_filefunc64_32->zfile_func64.zread_file = p_filefunc32->zread_file;
     p_filefunc64_32->zfile_func64.zwrite_file = p_filefunc32->zwrite_file;
     p_filefunc64_32->zfile_func64.ztell64_file = NULL;
@@ -84,18 +79,10 @@ void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filef
 
 
 
-static voidpf  ZCALLBACK fopen_file_func OF((voidpf opaque, const char* filename, int mode));
-static uLong   ZCALLBACK fread_file_func OF((voidpf opaque, voidpf stream, void* buf, uLong size));
-static uLong   ZCALLBACK fwrite_file_func OF((voidpf opaque, voidpf stream, const void* buf,uLong size));
-static ZPOS64_T ZCALLBACK ftell64_file_func OF((voidpf opaque, voidpf stream));
-static long    ZCALLBACK fseek64_file_func OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin));
-static int     ZCALLBACK fclose_file_func OF((voidpf opaque, voidpf stream));
-static int     ZCALLBACK ferror_file_func OF((voidpf opaque, voidpf stream));
-
-static voidpf ZCALLBACK fopen_file_func (voidpf opaque, const char* filename, int mode)
-{
+static voidpf ZCALLBACK fopen_file_func(voidpf opaque, const char* filename, int mode) {
     FILE* file = NULL;
     const char* mode_fopen = NULL;
+    (void)opaque;
     if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ)
         mode_fopen = "rb";
     else
@@ -110,10 +97,10 @@ static voidpf ZCALLBACK fopen_file_func (voidpf opaque, const char* filename, in
     return file;
 }
 
-static voidpf ZCALLBACK fopen64_file_func (voidpf opaque, const void* filename, int mode)
-{
+static voidpf ZCALLBACK fopen64_file_func(voidpf opaque, const void* filename, int mode) {
     FILE* file = NULL;
     const char* mode_fopen = NULL;
+    (void)opaque;
     if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ)
         mode_fopen = "rb";
     else
@@ -129,39 +116,39 @@ static voidpf ZCALLBACK fopen64_file_func (voidpf opaque, const void* filename,
 }
 
 
-static uLong ZCALLBACK fread_file_func (voidpf opaque, voidpf stream, void* buf, uLong size)
-{
+static uLong ZCALLBACK fread_file_func(voidpf opaque, voidpf stream, void* buf, uLong size) {
     uLong ret;
+    (void)opaque;
     ret = (uLong)fread(buf, 1, (size_t)size, (FILE *)stream);
     return ret;
 }
 
-static uLong ZCALLBACK fwrite_file_func (voidpf opaque, voidpf stream, const void* buf, uLong size)
-{
+static uLong ZCALLBACK fwrite_file_func(voidpf opaque, voidpf stream, const void* buf, uLong size) {
     uLong ret;
+    (void)opaque;
     ret = (uLong)fwrite(buf, 1, (size_t)size, (FILE *)stream);
     return ret;
 }
 
-static long ZCALLBACK ftell_file_func (voidpf opaque, voidpf stream)
-{
+static long ZCALLBACK ftell_file_func(voidpf opaque, voidpf stream) {
     long ret;
+    (void)opaque;
     ret = ftell((FILE *)stream);
     return ret;
 }
 
 
-static ZPOS64_T ZCALLBACK ftell64_file_func (voidpf opaque, voidpf stream)
-{
+static ZPOS64_T ZCALLBACK ftell64_file_func(voidpf opaque, voidpf stream) {
     ZPOS64_T ret;
-    ret = FTELLO_FUNC((FILE *)stream);
+    (void)opaque;
+    ret = (ZPOS64_T)FTELLO_FUNC((FILE *)stream);
     return ret;
 }
 
-static long ZCALLBACK fseek_file_func (voidpf  opaque, voidpf stream, uLong offset, int origin)
-{
+static long ZCALLBACK fseek_file_func(voidpf opaque, voidpf stream, uLong offset, int origin) {
     int fseek_origin=0;
     long ret;
+    (void)opaque;
     switch (origin)
     {
     case ZLIB_FILEFUNC_SEEK_CUR :
@@ -176,15 +163,15 @@ static long ZCALLBACK fseek_file_func (voidpf  opaque, voidpf stream, uLong offs
     default: return -1;
     }
     ret = 0;
-    if (fseek((FILE *)stream, offset, fseek_origin) != 0)
+    if (fseek((FILE *)stream, (long)offset, fseek_origin) != 0)
         ret = -1;
     return ret;
 }
 
-static long ZCALLBACK fseek64_file_func (voidpf  opaque, voidpf stream, ZPOS64_T offset, int origin)
-{
+static long ZCALLBACK fseek64_file_func(voidpf opaque, voidpf stream, ZPOS64_T offset, int origin) {
     int fseek_origin=0;
     long ret;
+    (void)opaque;
     switch (origin)
     {
     case ZLIB_FILEFUNC_SEEK_CUR :
@@ -200,30 +187,28 @@ static long ZCALLBACK fseek64_file_func (voidpf  opaque, voidpf stream, ZPOS64_T
     }
     ret = 0;
 
-    if(FSEEKO_FUNC((FILE *)stream, offset, fseek_origin) != 0)
+    if(FSEEKO_FUNC((FILE *)stream, (z_off64_t)offset, fseek_origin) != 0)
                         ret = -1;
 
     return ret;
 }
 
 
-static int ZCALLBACK fclose_file_func (voidpf opaque, voidpf stream)
-{
+static int ZCALLBACK fclose_file_func(voidpf opaque, voidpf stream) {
     int ret;
+    (void)opaque;
     ret = fclose((FILE *)stream);
     return ret;
 }
 
-static int ZCALLBACK ferror_file_func (voidpf opaque, voidpf stream)
-{
+static int ZCALLBACK ferror_file_func(voidpf opaque, voidpf stream) {
     int ret;
+    (void)opaque;
     ret = ferror((FILE *)stream);
     return ret;
 }
 
-void fill_fopen_filefunc (pzlib_filefunc_def)
-  zlib_filefunc_def* pzlib_filefunc_def;
-{
+void fill_fopen_filefunc(zlib_filefunc_def* pzlib_filefunc_def) {
     pzlib_filefunc_def->zopen_file = fopen_file_func;
     pzlib_filefunc_def->zread_file = fread_file_func;
     pzlib_filefunc_def->zwrite_file = fwrite_file_func;
@@ -234,8 +219,7 @@ void fill_fopen_filefunc (pzlib_filefunc_def)
     pzlib_filefunc_def->opaque = NULL;
 }
 
-void fill_fopen64_filefunc (zlib_filefunc64_def*  pzlib_filefunc_def)
-{
+void fill_fopen64_filefunc(zlib_filefunc64_def* pzlib_filefunc_def) {
     pzlib_filefunc_def->zopen64_file = fopen64_file_func;
     pzlib_filefunc_def->zread_file = fread_file_func;
     pzlib_filefunc_def->zwrite_file = fwrite_file_func;
diff --git a/zlib/contrib/minizip/ioapi.h b/zlib/contrib/minizip/ioapi.h
index 8dcbdb0..a2d2e6e 100644
--- a/zlib/contrib/minizip/ioapi.h
+++ b/zlib/contrib/minizip/ioapi.h
@@ -50,7 +50,7 @@
 #define ftello64 ftell
 #define fseeko64 fseek
 #else
-#ifdef __FreeBSD__
+#if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__HAIKU__) || defined(MINIZIP_FOPEN_NO_64)
 #define fopen64 fopen
 #define ftello64 ftello
 #define fseeko64 fseeko
@@ -82,7 +82,7 @@
 #include "mz64conf.h"
 #endif
 
-/* a type choosen by DEFINE */
+/* a type chosen by DEFINE */
 #ifdef HAVE_64BIT_INT_CUSTOM
 typedef  64BIT_INT_CUSTOM_TYPE ZPOS64_T;
 #else
@@ -91,8 +91,7 @@ typedef  64BIT_INT_CUSTOM_TYPE ZPOS64_T;
 typedef uint64_t ZPOS64_T;
 #else
 
-/* Maximum unsigned 32-bit value used as placeholder for zip64 */
-#define MAXU32 0xffffffff
+
 
 #if defined(_MSC_VER) || defined(__BORLANDC__)
 typedef unsigned __int64 ZPOS64_T;
@@ -102,7 +101,10 @@ typedef unsigned long long int ZPOS64_T;
 #endif
 #endif
 
-
+/* Maximum unsigned 32-bit value used as placeholder for zip64 */
+#ifndef MAXU32
+#define MAXU32 (0xffffffff)
+#endif
 
 #ifdef __cplusplus
 extern "C" {
@@ -132,17 +134,17 @@ extern "C" {
 
 
 
-typedef voidpf   (ZCALLBACK *open_file_func)      OF((voidpf opaque, const char* filename, int mode));
-typedef uLong    (ZCALLBACK *read_file_func)      OF((voidpf opaque, voidpf stream, void* buf, uLong size));
-typedef uLong    (ZCALLBACK *write_file_func)     OF((voidpf opaque, voidpf stream, const void* buf, uLong size));
-typedef int      (ZCALLBACK *close_file_func)     OF((voidpf opaque, voidpf stream));
-typedef int      (ZCALLBACK *testerror_file_func) OF((voidpf opaque, voidpf stream));
+typedef voidpf   (ZCALLBACK *open_file_func)      (voidpf opaque, const char* filename, int mode);
+typedef uLong    (ZCALLBACK *read_file_func)      (voidpf opaque, voidpf stream, void* buf, uLong size);
+typedef uLong    (ZCALLBACK *write_file_func)     (voidpf opaque, voidpf stream, const void* buf, uLong size);
+typedef int      (ZCALLBACK *close_file_func)     (voidpf opaque, voidpf stream);
+typedef int      (ZCALLBACK *testerror_file_func) (voidpf opaque, voidpf stream);
 
-typedef long     (ZCALLBACK *tell_file_func)      OF((voidpf opaque, voidpf stream));
-typedef long     (ZCALLBACK *seek_file_func)      OF((voidpf opaque, voidpf stream, uLong offset, int origin));
+typedef long     (ZCALLBACK *tell_file_func)      (voidpf opaque, voidpf stream);
+typedef long     (ZCALLBACK *seek_file_func)      (voidpf opaque, voidpf stream, uLong offset, int origin);
 
 
-/* here is the "old" 32 bits structure structure */
+/* here is the "old" 32 bits structure */
 typedef struct zlib_filefunc_def_s
 {
     open_file_func      zopen_file;
@@ -155,9 +157,9 @@ typedef struct zlib_filefunc_def_s
     voidpf              opaque;
 } zlib_filefunc_def;
 
-typedef ZPOS64_T (ZCALLBACK *tell64_file_func)    OF((voidpf opaque, voidpf stream));
-typedef long     (ZCALLBACK *seek64_file_func)    OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin));
-typedef voidpf   (ZCALLBACK *open64_file_func)    OF((voidpf opaque, const void* filename, int mode));
+typedef ZPOS64_T (ZCALLBACK *tell64_file_func)    (voidpf opaque, voidpf stream);
+typedef long     (ZCALLBACK *seek64_file_func)    (voidpf opaque, voidpf stream, ZPOS64_T offset, int origin);
+typedef voidpf   (ZCALLBACK *open64_file_func)    (voidpf opaque, const void* filename, int mode);
 
 typedef struct zlib_filefunc64_def_s
 {
@@ -171,8 +173,8 @@ typedef struct zlib_filefunc64_def_s
     voidpf              opaque;
 } zlib_filefunc64_def;
 
-void fill_fopen64_filefunc OF((zlib_filefunc64_def* pzlib_filefunc_def));
-void fill_fopen_filefunc OF((zlib_filefunc_def* pzlib_filefunc_def));
+void fill_fopen64_filefunc(zlib_filefunc64_def* pzlib_filefunc_def);
+void fill_fopen_filefunc(zlib_filefunc_def* pzlib_filefunc_def);
 
 /* now internal definition, only for zip.c and unzip.h */
 typedef struct zlib_filefunc64_32_def_s
@@ -191,11 +193,11 @@ typedef struct zlib_filefunc64_32_def_s
 #define ZCLOSE64(filefunc,filestream)             ((*((filefunc).zfile_func64.zclose_file))  ((filefunc).zfile_func64.opaque,filestream))
 #define ZERROR64(filefunc,filestream)             ((*((filefunc).zfile_func64.zerror_file))  ((filefunc).zfile_func64.opaque,filestream))
 
-voidpf call_zopen64 OF((const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode));
-long    call_zseek64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin));
-ZPOS64_T call_ztell64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream));
+voidpf call_zopen64(const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode);
+long call_zseek64(const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin);
+ZPOS64_T call_ztell64(const zlib_filefunc64_32_def* pfilefunc,voidpf filestream);
 
-void    fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32);
+void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32);
 
 #define ZOPEN64(filefunc,filename,mode)         (call_zopen64((&(filefunc)),(filename),(mode)))
 #define ZTELL64(filefunc,filestream)            (call_ztell64((&(filefunc)),(filestream)))
diff --git a/zlib/contrib/minizip/iowin32.c b/zlib/contrib/minizip/iowin32.c
index 274f39e..08536e9 100644
--- a/zlib/contrib/minizip/iowin32.c
+++ b/zlib/contrib/minizip/iowin32.c
@@ -28,19 +28,16 @@
 
 // see Include/shared/winapifamily.h in the Windows Kit
 #if defined(WINAPI_FAMILY_PARTITION) && (!(defined(IOWIN32_USING_WINRT_API)))
+
+#if !defined(WINAPI_FAMILY_ONE_PARTITION)
+#define WINAPI_FAMILY_ONE_PARTITION(PartitionSet, Partition) ((WINAPI_FAMILY & PartitionSet) == Partition)
+#endif
+
 #if WINAPI_FAMILY_ONE_PARTITION(WINAPI_FAMILY, WINAPI_PARTITION_APP)
 #define IOWIN32_USING_WINRT_API 1
 #endif
 #endif
 
-voidpf  ZCALLBACK win32_open_file_func  OF((voidpf opaque, const char* filename, int mode));
-uLong   ZCALLBACK win32_read_file_func  OF((voidpf opaque, voidpf stream, void* buf, uLong size));
-uLong   ZCALLBACK win32_write_file_func OF((voidpf opaque, voidpf stream, const void* buf, uLong size));
-ZPOS64_T ZCALLBACK win32_tell64_file_func  OF((voidpf opaque, voidpf stream));
-long    ZCALLBACK win32_seek64_file_func  OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin));
-int     ZCALLBACK win32_close_file_func OF((voidpf opaque, voidpf stream));
-int     ZCALLBACK win32_error_file_func OF((voidpf opaque, voidpf stream));
-
 typedef struct
 {
     HANDLE hf;
@@ -52,8 +49,7 @@ static void win32_translate_open_mode(int mode,
                                       DWORD* lpdwDesiredAccess,
                                       DWORD* lpdwCreationDisposition,
                                       DWORD* lpdwShareMode,
-                                      DWORD* lpdwFlagsAndAttributes)
-{
+                                      DWORD* lpdwFlagsAndAttributes) {
     *lpdwDesiredAccess = *lpdwShareMode = *lpdwFlagsAndAttributes = *lpdwCreationDisposition = 0;
 
     if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ)
@@ -74,8 +70,7 @@ static void win32_translate_open_mode(int mode,
     }
 }
 
-static voidpf win32_build_iowin(HANDLE hFile)
-{
+static voidpf win32_build_iowin(HANDLE hFile) {
     voidpf ret=NULL;
 
     if ((hFile != NULL) && (hFile != INVALID_HANDLE_VALUE))
@@ -93,8 +88,7 @@ static voidpf win32_build_iowin(HANDLE hFile)
     return ret;
 }
 
-voidpf ZCALLBACK win32_open64_file_func (voidpf opaque,const void* filename,int mode)
-{
+voidpf ZCALLBACK win32_open64_file_func(voidpf opaque, const void* filename, int mode) {
     const char* mode_fopen = NULL;
     DWORD dwDesiredAccess,dwCreationDisposition,dwShareMode,dwFlagsAndAttributes ;
     HANDLE hFile = NULL;
@@ -122,8 +116,7 @@ voidpf ZCALLBACK win32_open64_file_func (voidpf opaque,const void* filename,int
 }
 
 
-voidpf ZCALLBACK win32_open64_file_funcA (voidpf opaque,const void* filename,int mode)
-{
+voidpf ZCALLBACK win32_open64_file_funcA(voidpf opaque, const void* filename, int mode) {
     const char* mode_fopen = NULL;
     DWORD dwDesiredAccess,dwCreationDisposition,dwShareMode,dwFlagsAndAttributes ;
     HANDLE hFile = NULL;
@@ -146,8 +139,7 @@ voidpf ZCALLBACK win32_open64_file_funcA (voidpf opaque,const void* filename,int
 }
 
 
-voidpf ZCALLBACK win32_open64_file_funcW (voidpf opaque,const void* filename,int mode)
-{
+voidpf ZCALLBACK win32_open64_file_funcW(voidpf opaque, const void* filename, int mode) {
     const char* mode_fopen = NULL;
     DWORD dwDesiredAccess,dwCreationDisposition,dwShareMode,dwFlagsAndAttributes ;
     HANDLE hFile = NULL;
@@ -166,8 +158,7 @@ voidpf ZCALLBACK win32_open64_file_funcW (voidpf opaque,const void* filename,int
 }
 
 
-voidpf ZCALLBACK win32_open_file_func (voidpf opaque,const char* filename,int mode)
-{
+voidpf ZCALLBACK win32_open_file_func(voidpf opaque, const char* filename, int mode) {
     const char* mode_fopen = NULL;
     DWORD dwDesiredAccess,dwCreationDisposition,dwShareMode,dwFlagsAndAttributes ;
     HANDLE hFile = NULL;
@@ -195,8 +186,7 @@ voidpf ZCALLBACK win32_open_file_func (voidpf opaque,const char* filename,int mo
 }
 
 
-uLong ZCALLBACK win32_read_file_func (voidpf opaque, voidpf stream, void* buf,uLong size)
-{
+uLong ZCALLBACK win32_read_file_func(voidpf opaque, voidpf stream, void* buf,uLong size) {
     uLong ret=0;
     HANDLE hFile = NULL;
     if (stream!=NULL)
@@ -217,8 +207,7 @@ uLong ZCALLBACK win32_read_file_func (voidpf opaque, voidpf stream, void* buf,uL
 }
 
 
-uLong ZCALLBACK win32_write_file_func (voidpf opaque,voidpf stream,const void* buf,uLong size)
-{
+uLong ZCALLBACK win32_write_file_func(voidpf opaque, voidpf stream, const void* buf, uLong size) {
     uLong ret=0;
     HANDLE hFile = NULL;
     if (stream!=NULL)
@@ -238,8 +227,7 @@ uLong ZCALLBACK win32_write_file_func (voidpf opaque,voidpf stream,const void* b
     return ret;
 }
 
-static BOOL MySetFilePointerEx(HANDLE hFile, LARGE_INTEGER pos, LARGE_INTEGER *newPos,  DWORD dwMoveMethod)
-{
+static BOOL MySetFilePointerEx(HANDLE hFile, LARGE_INTEGER pos, LARGE_INTEGER *newPos, DWORD dwMoveMethod) {
 #ifdef IOWIN32_USING_WINRT_API
     return SetFilePointerEx(hFile, pos, newPos, dwMoveMethod);
 #else
@@ -258,8 +246,7 @@ static BOOL MySetFilePointerEx(HANDLE hFile, LARGE_INTEGER pos, LARGE_INTEGER *n
 #endif
 }
 
-long ZCALLBACK win32_tell_file_func (voidpf opaque,voidpf stream)
-{
+long ZCALLBACK win32_tell_file_func(voidpf opaque, voidpf stream) {
     long ret=-1;
     HANDLE hFile = NULL;
     if (stream!=NULL)
@@ -281,8 +268,7 @@ long ZCALLBACK win32_tell_file_func (voidpf opaque,voidpf stream)
     return ret;
 }
 
-ZPOS64_T ZCALLBACK win32_tell64_file_func (voidpf opaque, voidpf stream)
-{
+ZPOS64_T ZCALLBACK win32_tell64_file_func(voidpf opaque, voidpf stream) {
     ZPOS64_T ret= (ZPOS64_T)-1;
     HANDLE hFile = NULL;
     if (stream!=NULL)
@@ -306,8 +292,7 @@ ZPOS64_T ZCALLBACK win32_tell64_file_func (voidpf opaque, voidpf stream)
 }
 
 
-long ZCALLBACK win32_seek_file_func (voidpf opaque,voidpf stream,uLong offset,int origin)
-{
+long ZCALLBACK win32_seek_file_func(voidpf opaque, voidpf stream, uLong offset, int origin) {
     DWORD dwMoveMethod=0xFFFFFFFF;
     HANDLE hFile = NULL;
 
@@ -344,8 +329,7 @@ long ZCALLBACK win32_seek_file_func (voidpf opaque,voidpf stream,uLong offset,in
     return ret;
 }
 
-long ZCALLBACK win32_seek64_file_func (voidpf opaque, voidpf stream,ZPOS64_T offset,int origin)
-{
+long ZCALLBACK win32_seek64_file_func(voidpf opaque, voidpf stream, ZPOS64_T offset, int origin) {
     DWORD dwMoveMethod=0xFFFFFFFF;
     HANDLE hFile = NULL;
     long ret=-1;
@@ -383,8 +367,7 @@ long ZCALLBACK win32_seek64_file_func (voidpf opaque, voidpf stream,ZPOS64_T off
     return ret;
 }
 
-int ZCALLBACK win32_close_file_func (voidpf opaque, voidpf stream)
-{
+int ZCALLBACK win32_close_file_func(voidpf opaque, voidpf stream) {
     int ret=-1;
 
     if (stream!=NULL)
@@ -401,8 +384,7 @@ int ZCALLBACK win32_close_file_func (voidpf opaque, voidpf stream)
     return ret;
 }
 
-int ZCALLBACK win32_error_file_func (voidpf opaque,voidpf stream)
-{
+int ZCALLBACK win32_error_file_func(voidpf opaque, voidpf stream) {
     int ret=-1;
     if (stream!=NULL)
     {
@@ -411,8 +393,7 @@ int ZCALLBACK win32_error_file_func (voidpf opaque,voidpf stream)
     return ret;
 }
 
-void fill_win32_filefunc (zlib_filefunc_def* pzlib_filefunc_def)
-{
+void fill_win32_filefunc(zlib_filefunc_def* pzlib_filefunc_def) {
     pzlib_filefunc_def->zopen_file = win32_open_file_func;
     pzlib_filefunc_def->zread_file = win32_read_file_func;
     pzlib_filefunc_def->zwrite_file = win32_write_file_func;
@@ -423,8 +404,7 @@ void fill_win32_filefunc (zlib_filefunc_def* pzlib_filefunc_def)
     pzlib_filefunc_def->opaque = NULL;
 }
 
-void fill_win32_filefunc64(zlib_filefunc64_def* pzlib_filefunc_def)
-{
+void fill_win32_filefunc64(zlib_filefunc64_def* pzlib_filefunc_def) {
     pzlib_filefunc_def->zopen64_file = win32_open64_file_func;
     pzlib_filefunc_def->zread_file = win32_read_file_func;
     pzlib_filefunc_def->zwrite_file = win32_write_file_func;
@@ -436,8 +416,7 @@ void fill_win32_filefunc64(zlib_filefunc64_def* pzlib_filefunc_def)
 }
 
 
-void fill_win32_filefunc64A(zlib_filefunc64_def* pzlib_filefunc_def)
-{
+void fill_win32_filefunc64A(zlib_filefunc64_def* pzlib_filefunc_def) {
     pzlib_filefunc_def->zopen64_file = win32_open64_file_funcA;
     pzlib_filefunc_def->zread_file = win32_read_file_func;
     pzlib_filefunc_def->zwrite_file = win32_write_file_func;
@@ -449,8 +428,7 @@ void fill_win32_filefunc64A(zlib_filefunc64_def* pzlib_filefunc_def)
 }
 
 
-void fill_win32_filefunc64W(zlib_filefunc64_def* pzlib_filefunc_def)
-{
+void fill_win32_filefunc64W(zlib_filefunc64_def* pzlib_filefunc_def) {
     pzlib_filefunc_def->zopen64_file = win32_open64_file_funcW;
     pzlib_filefunc_def->zread_file = win32_read_file_func;
     pzlib_filefunc_def->zwrite_file = win32_write_file_func;
diff --git a/zlib/contrib/minizip/iowin32.h b/zlib/contrib/minizip/iowin32.h
index 0ca0969..a23a65d 100644
--- a/zlib/contrib/minizip/iowin32.h
+++ b/zlib/contrib/minizip/iowin32.h
@@ -18,10 +18,10 @@
 extern "C" {
 #endif
 
-void fill_win32_filefunc OF((zlib_filefunc_def* pzlib_filefunc_def));
-void fill_win32_filefunc64 OF((zlib_filefunc64_def* pzlib_filefunc_def));
-void fill_win32_filefunc64A OF((zlib_filefunc64_def* pzlib_filefunc_def));
-void fill_win32_filefunc64W OF((zlib_filefunc64_def* pzlib_filefunc_def));
+void fill_win32_filefunc(zlib_filefunc_def* pzlib_filefunc_def);
+void fill_win32_filefunc64(zlib_filefunc64_def* pzlib_filefunc_def);
+void fill_win32_filefunc64A(zlib_filefunc64_def* pzlib_filefunc_def);
+void fill_win32_filefunc64W(zlib_filefunc64_def* pzlib_filefunc_def);
 
 #ifdef __cplusplus
 }
diff --git a/zlib/contrib/minizip/miniunz.c b/zlib/contrib/minizip/miniunz.c
index 3d65401..d627c42 100644
--- a/zlib/contrib/minizip/miniunz.c
+++ b/zlib/contrib/minizip/miniunz.c
@@ -27,7 +27,7 @@
         #endif
 #endif
 
-#ifdef __APPLE__
+#if defined(__APPLE__) || defined(__HAIKU__) || defined(MINIZIP_FOPEN_NO_64)
 // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions
 #define FOPEN_FUNC(filename, mode) fopen(filename, mode)
 #define FTELLO_FUNC(stream) ftello(stream)
@@ -45,6 +45,7 @@
 #include <time.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <sys/stat.h>
 
 #ifdef _WIN32
 # include <direct.h>
@@ -78,13 +79,9 @@
 
 /* change_file_date : change the date/time of a file
     filename : the filename of the file where date/time must be modified
-    dosdate : the new date at the MSDos format (4 bytes)
+    dosdate : the new date at the MSDOS format (4 bytes)
     tmu_date : the SAME new date at the tm_unz format */
-void change_file_date(filename,dosdate,tmu_date)
-    const char *filename;
-    uLong dosdate;
-    tm_unz tmu_date;
-{
+static void change_file_date(const char *filename, uLong dosdate, tm_unz tmu_date) {
 #ifdef _WIN32
   HANDLE hFile;
   FILETIME ftm,ftLocal,ftCreate,ftLastAcc,ftLastWrite;
@@ -97,7 +94,8 @@ void change_file_date(filename,dosdate,tmu_date)
   SetFileTime(hFile,&ftm,&ftLastAcc,&ftm);
   CloseHandle(hFile);
 #else
-#ifdef unix || __APPLE__
+#if defined(unix) || defined(__APPLE__)
+  (void)dosdate;
   struct utimbuf ut;
   struct tm newdate;
   newdate.tm_sec = tmu_date.tm_sec;
@@ -113,6 +111,10 @@ void change_file_date(filename,dosdate,tmu_date)
 
   ut.actime=ut.modtime=mktime(&newdate);
   utime(filename,&ut);
+#else
+  (void)filename;
+  (void)dosdate;
+  (void)tmu_date;
 #endif
 #endif
 }
@@ -121,9 +123,7 @@ void change_file_date(filename,dosdate,tmu_date)
 /* mymkdir and change_file_date are not 100 % portable
    As I don't know well Unix, I wait feedback for the unix portion */
 
-int mymkdir(dirname)
-    const char* dirname;
-{
+static int mymkdir(const char* dirname) {
     int ret=0;
 #ifdef _WIN32
     ret = _mkdir(dirname);
@@ -131,18 +131,18 @@ int mymkdir(dirname)
     ret = mkdir (dirname,0775);
 #elif __APPLE__
     ret = mkdir (dirname,0775);
+#else
+    (void)dirname;
 #endif
     return ret;
 }
 
-int makedir (newdir)
-    char *newdir;
-{
+static int makedir(const char *newdir) {
   char *buffer ;
   char *p;
-  int  len = (int)strlen(newdir);
+  size_t len = strlen(newdir);
 
-  if (len <= 0)
+  if (len == 0)
     return 0;
 
   buffer = (char*)malloc(len+1);
@@ -185,14 +185,12 @@ int makedir (newdir)
   return 1;
 }
 
-void do_banner()
-{
-    printf("MiniUnz 1.01b, demo of zLib + Unz package written by Gilles Vollant\n");
+static void do_banner(void) {
+    printf("MiniUnz 1.1, demo of zLib + Unz package written by Gilles Vollant\n");
     printf("more info at http://www.winimage.com/zLibDll/unzip.html\n\n");
 }
 
-void do_help()
-{
+static void do_help(void) {
     printf("Usage : miniunz [-e] [-x] [-v] [-l] [-o] [-p password] file.zip [file_to_extr.] [-d extractdir]\n\n" \
            "  -e  Extract without pathname (junk paths)\n" \
            "  -x  Extract with pathname\n" \
@@ -200,11 +198,10 @@ void do_help()
            "  -l  list files\n" \
            "  -d  directory to extract into\n" \
            "  -o  overwrite files without prompting\n" \
-           "  -p  extract crypted file using password\n\n");
+           "  -p  extract encrypted file using password\n\n");
 }
 
-void Display64BitsSize(ZPOS64_T n, int size_char)
-{
+static void Display64BitsSize(ZPOS64_T n, int size_char) {
   /* to avoid compatibility problem , we do here the conversion */
   char number[21];
   int offset=19;
@@ -231,9 +228,7 @@ void Display64BitsSize(ZPOS64_T n, int size_char)
   printf("%s",&number[pos_string]);
 }
 
-int do_list(uf)
-    unzFile uf;
-{
+static int do_list(unzFile uf) {
     uLong i;
     unz_global_info64 gi;
     int err;
@@ -248,7 +243,7 @@ int do_list(uf)
         char filename_inzip[256];
         unz_file_info64 file_info;
         uLong ratio=0;
-        const char *string_method;
+        const char *string_method = "";
         char charCrypt=' ';
         err = unzGetCurrentFileInfo64(uf,&file_info,filename_inzip,sizeof(filename_inzip),NULL,0,NULL,0);
         if (err!=UNZ_OK)
@@ -259,7 +254,7 @@ int do_list(uf)
         if (file_info.uncompressed_size>0)
             ratio = (uLong)((file_info.compressed_size*100)/file_info.uncompressed_size);
 
-        /* display a '*' if the file is crypted */
+        /* display a '*' if the file is encrypted */
         if ((file_info.flag & 1) != 0)
             charCrypt='*';
 
@@ -309,12 +304,7 @@ int do_list(uf)
 }
 
 
-int do_extract_currentfile(uf,popt_extract_without_path,popt_overwrite,password)
-    unzFile uf;
-    const int* popt_extract_without_path;
-    int* popt_overwrite;
-    const char* password;
-{
+static int do_extract_currentfile(unzFile uf, const int* popt_extract_without_path, int* popt_overwrite, const char* password) {
     char filename_inzip[256];
     char* filename_withoutpath;
     char* p;
@@ -324,7 +314,6 @@ int do_extract_currentfile(uf,popt_extract_without_path,popt_overwrite,password)
     uInt size_buf;
 
     unz_file_info64 file_info;
-    uLong ratio=0;
     err = unzGetCurrentFileInfo64(uf,&file_info,filename_inzip,sizeof(filename_inzip),NULL,0,NULL,0);
 
     if (err!=UNZ_OK)
@@ -367,6 +356,20 @@ int do_extract_currentfile(uf,popt_extract_without_path,popt_overwrite,password)
         else
             write_filename = filename_withoutpath;
 
+        if (write_filename[0]!='\0')
+        {
+            const char* relative_check = write_filename;
+            while (relative_check[1]!='\0')
+            {
+                if (relative_check[0]=='.' && relative_check[1]=='.')
+                    write_filename = relative_check;
+                relative_check++;
+            }
+        }
+
+        while (write_filename[0]=='/' || write_filename[0]=='.')
+            write_filename++;
+
         err = unzOpenCurrentFilePassword(uf,password);
         if (err!=UNZ_OK)
         {
@@ -439,7 +442,7 @@ int do_extract_currentfile(uf,popt_extract_without_path,popt_overwrite,password)
                     break;
                 }
                 if (err>0)
-                    if (fwrite(buf,err,1,fout)!=1)
+                    if (fwrite(buf,(unsigned)err,1,fout)!=1)
                     {
                         printf("error in writing extracted file\n");
                         err=UNZ_ERRNO;
@@ -472,16 +475,10 @@ int do_extract_currentfile(uf,popt_extract_without_path,popt_overwrite,password)
 }
 
 
-int do_extract(uf,opt_extract_without_path,opt_overwrite,password)
-    unzFile uf;
-    int opt_extract_without_path;
-    int opt_overwrite;
-    const char* password;
-{
+static int do_extract(unzFile uf, int opt_extract_without_path, int opt_overwrite, const char* password) {
     uLong i;
     unz_global_info64 gi;
     int err;
-    FILE* fout=NULL;
 
     err = unzGetGlobalInfo64(uf,&gi);
     if (err!=UNZ_OK)
@@ -508,14 +505,7 @@ int do_extract(uf,opt_extract_without_path,opt_overwrite,password)
     return 0;
 }
 
-int do_extract_onefile(uf,filename,opt_extract_without_path,opt_overwrite,password)
-    unzFile uf;
-    const char* filename;
-    int opt_extract_without_path;
-    int opt_overwrite;
-    const char* password;
-{
-    int err = UNZ_OK;
+static int do_extract_onefile(unzFile uf, const char* filename, int opt_extract_without_path, int opt_overwrite, const char* password) {
     if (unzLocateFile(uf,filename,CASESENSITIVITY)!=UNZ_OK)
     {
         printf("file %s not found in the zipfile\n",filename);
@@ -531,10 +521,7 @@ int do_extract_onefile(uf,filename,opt_extract_without_path,opt_overwrite,passwo
 }
 
 
-int main(argc,argv)
-    int argc;
-    char *argv[];
-{
+int main(int argc, char *argv[]) {
     const char *zipfilename=NULL;
     const char *filename_to_extract=NULL;
     const char *password=NULL;
@@ -565,7 +552,7 @@ int main(argc,argv)
 
                 while ((*p)!='\0')
                 {
-                    char c=*(p++);;
+                    char c=*(p++);
                     if ((c=='l') || (c=='L'))
                         opt_do_list = 1;
                     if ((c=='v') || (c=='V'))
@@ -607,7 +594,7 @@ int main(argc,argv)
 #        endif
 
         strncpy(filename_try, zipfilename,MAXFILENAME-1);
-        /* strncpy doesnt append the trailing NULL, of the string is too long. */
+        /* strncpy doesn't append the trailing NULL, of the string is too long. */
         filename_try[ MAXFILENAME ] = '\0';
 
 #        ifdef USEWIN32IOAPI
diff --git a/zlib/contrib/minizip/minizip.c b/zlib/contrib/minizip/minizip.c
index 4288962..26ee8d0 100644
--- a/zlib/contrib/minizip/minizip.c
+++ b/zlib/contrib/minizip/minizip.c
@@ -28,7 +28,7 @@
         #endif
 #endif
 
-#ifdef __APPLE__
+#if defined(__APPLE__) || defined(__HAIKU__) || defined(MINIZIP_FOPEN_NO_64)
 // In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions
 #define FOPEN_FUNC(filename, mode) fopen(filename, mode)
 #define FTELLO_FUNC(stream) ftello(stream)
@@ -71,11 +71,9 @@
 #define MAXFILENAME (256)
 
 #ifdef _WIN32
-uLong filetime(f, tmzip, dt)
-    char *f;                /* name of file to get info on */
-    tm_zip *tmzip;             /* return value: access, modific. and creation times */
-    uLong *dt;             /* dostime */
-{
+/* f: name of file to get info on, tmzip: return value: access,
+   modification and creation times, dt: dostime */
+static int filetime(const char *f, tm_zip *tmzip, uLong *dt) {
   int ret = 0;
   {
       FILETIME ftLocal;
@@ -94,12 +92,11 @@ uLong filetime(f, tmzip, dt)
   return ret;
 }
 #else
-#ifdef unix || __APPLE__
-uLong filetime(f, tmzip, dt)
-    char *f;               /* name of file to get info on */
-    tm_zip *tmzip;         /* return value: access, modific. and creation times */
-    uLong *dt;             /* dostime */
-{
+#if defined(unix) || defined(__APPLE__)
+/* f: name of file to get info on, tmzip: return value: access,
+   modification and creation times, dt: dostime */
+static int filetime(const char *f, tm_zip *tmzip, uLong *dt) {
+  (void)dt;
   int ret=0;
   struct stat s;        /* results of stat() */
   struct tm* filedate;
@@ -108,12 +105,12 @@ uLong filetime(f, tmzip, dt)
   if (strcmp(f,"-")!=0)
   {
     char name[MAXFILENAME+1];
-    int len = strlen(f);
+    size_t len = strlen(f);
     if (len > MAXFILENAME)
       len = MAXFILENAME;
 
     strncpy(name, f,MAXFILENAME-1);
-    /* strncpy doesnt append the trailing NULL, of the string is too long. */
+    /* strncpy doesn't append the trailing NULL, of the string is too long. */
     name[ MAXFILENAME ] = '\0';
 
     if (name[len - 1] == '/')
@@ -137,11 +134,12 @@ uLong filetime(f, tmzip, dt)
   return ret;
 }
 #else
-uLong filetime(f, tmzip, dt)
-    char *f;                /* name of file to get info on */
-    tm_zip *tmzip;             /* return value: access, modific. and creation times */
-    uLong *dt;             /* dostime */
-{
+/* f: name of file to get info on, tmzip: return value: access,
+   modification and creation times, dt: dostime */
+static int filetime(const char *f, tm_zip *tmzip, uLong *dt) {
+    (void)f;
+    (void)tmzip;
+    (void)dt;
     return 0;
 }
 #endif
@@ -150,9 +148,7 @@ uLong filetime(f, tmzip, dt)
 
 
 
-int check_exist_file(filename)
-    const char* filename;
-{
+static int check_exist_file(const char* filename) {
     FILE* ftestexist;
     int ret = 1;
     ftestexist = FOPEN_FUNC(filename,"rb");
@@ -163,14 +159,12 @@ int check_exist_file(filename)
     return ret;
 }
 
-void do_banner()
-{
+static void do_banner(void) {
     printf("MiniZip 1.1, demo of zLib + MiniZip64 package, written by Gilles Vollant\n");
     printf("more info on MiniZip at http://www.winimage.com/zLibDll/minizip.html\n\n");
 }
 
-void do_help()
-{
+static void do_help(void) {
     printf("Usage : minizip [-o] [-a] [-0 to -9] [-p password] [-j] file.zip [files_to_add]\n\n" \
            "  -o  Overwrite existing file.zip\n" \
            "  -a  Append to existing file.zip\n" \
@@ -182,14 +176,13 @@ void do_help()
 
 /* calculate the CRC32 of a file,
    because to encrypt a file, we need known the CRC32 of the file before */
-int getFileCrc(const char* filenameinzip,void*buf,unsigned long size_buf,unsigned long* result_crc)
-{
+static int getFileCrc(const char* filenameinzip, void* buf, unsigned long size_buf, unsigned long* result_crc) {
    unsigned long calculate_crc=0;
    int err=ZIP_OK;
    FILE * fin = FOPEN_FUNC(filenameinzip,"rb");
 
    unsigned long size_read = 0;
-   unsigned long total_read = 0;
+   /* unsigned long total_read = 0; */
    if (fin==NULL)
    {
        err = ZIP_ERRNO;
@@ -199,7 +192,7 @@ int getFileCrc(const char* filenameinzip,void*buf,unsigned long size_buf,unsigne
         do
         {
             err = ZIP_OK;
-            size_read = (int)fread(buf,1,size_buf,fin);
+            size_read = fread(buf,1,size_buf,fin);
             if (size_read < size_buf)
                 if (feof(fin)==0)
             {
@@ -208,8 +201,8 @@ int getFileCrc(const char* filenameinzip,void*buf,unsigned long size_buf,unsigne
             }
 
             if (size_read>0)
-                calculate_crc = crc32(calculate_crc,buf,size_read);
-            total_read += size_read;
+                calculate_crc = crc32_z(calculate_crc,buf,size_read);
+            /* total_read += size_read; */
 
         } while ((err == ZIP_OK) && (size_read>0));
 
@@ -221,18 +214,17 @@ int getFileCrc(const char* filenameinzip,void*buf,unsigned long size_buf,unsigne
     return err;
 }
 
-int isLargeFile(const char* filename)
-{
+static int isLargeFile(const char* filename) {
   int largeFile = 0;
   ZPOS64_T pos = 0;
   FILE* pFile = FOPEN_FUNC(filename, "rb");
 
   if(pFile != NULL)
   {
-    int n = FSEEKO_FUNC(pFile, 0, SEEK_END);
-    pos = FTELLO_FUNC(pFile);
+    FSEEKO_FUNC(pFile, 0, SEEK_END);
+    pos = (ZPOS64_T)FTELLO_FUNC(pFile);
 
-                printf("File : %s is %lld bytes\n", filename, pos);
+                printf("File : %s is %llu bytes\n", filename, pos);
 
     if(pos >= 0xffffffff)
      largeFile = 1;
@@ -243,10 +235,7 @@ int isLargeFile(const char* filename)
  return largeFile;
 }
 
-int main(argc,argv)
-    int argc;
-    char *argv[];
-{
+int main(int argc, char *argv[]) {
     int i;
     int opt_overwrite=0;
     int opt_compress_level=Z_DEFAULT_COMPRESSION;
@@ -255,7 +244,7 @@ int main(argc,argv)
     char filename_try[MAXFILENAME+16];
     int zipok;
     int err=0;
-    int size_buf=0;
+    size_t size_buf=0;
     void* buf=NULL;
     const char* password=NULL;
 
@@ -276,7 +265,7 @@ int main(argc,argv)
 
                 while ((*p)!='\0')
                 {
-                    char c=*(p++);;
+                    char c=*(p++);
                     if ((c=='o') || (c=='O'))
                         opt_overwrite = 1;
                     if ((c=='a') || (c=='A'))
@@ -322,7 +311,7 @@ int main(argc,argv)
 
         zipok = 1 ;
         strncpy(filename_try, argv[zipfilenamearg],MAXFILENAME-1);
-        /* strncpy doesnt append the trailing NULL, of the string is too long. */
+        /* strncpy doesn't append the trailing NULL, of the string is too long. */
         filename_try[ MAXFILENAME ] = '\0';
 
         len=(int)strlen(filename_try);
@@ -392,11 +381,11 @@ int main(argc,argv)
                   ((argv[i][1]=='o') || (argv[i][1]=='O') ||
                    (argv[i][1]=='a') || (argv[i][1]=='A') ||
                    (argv[i][1]=='p') || (argv[i][1]=='P') ||
-                   ((argv[i][1]>='0') || (argv[i][1]<='9'))) &&
+                   ((argv[i][1]>='0') && (argv[i][1]<='9'))) &&
                   (strlen(argv[i]) == 2)))
             {
-                FILE * fin;
-                int size_read;
+                FILE * fin = NULL;
+                size_t size_read;
                 const char* filenameinzip = argv[i];
                 const char *savefilenameinzip;
                 zip_fileinfo zi;
@@ -472,7 +461,7 @@ int main(argc,argv)
                     do
                     {
                         err = ZIP_OK;
-                        size_read = (int)fread(buf,1,size_buf,fin);
+                        size_read = fread(buf,1,size_buf,fin);
                         if (size_read < size_buf)
                             if (feof(fin)==0)
                         {
@@ -482,7 +471,7 @@ int main(argc,argv)
 
                         if (size_read>0)
                         {
-                            err = zipWriteInFileInZip (zf,buf,size_read);
+                            err = zipWriteInFileInZip (zf,buf,(unsigned)size_read);
                             if (err<0)
                             {
                                 printf("error in writing %s in the zipfile\n",
diff --git a/zlib/contrib/minizip/mztools.c b/zlib/contrib/minizip/mztools.c
index 96891c2..c8d2375 100644
--- a/zlib/contrib/minizip/mztools.c
+++ b/zlib/contrib/minizip/mztools.c
@@ -27,13 +27,7 @@
   WRITE_16((unsigned char*)(buff) + 2, (n) >> 16); \
 } while(0)
 
-extern int ZEXPORT unzRepair(file, fileOut, fileOutTmp, nRecovered, bytesRecovered)
-const char* file;
-const char* fileOut;
-const char* fileOutTmp;
-uLong* nRecovered;
-uLong* bytesRecovered;
-{
+extern int ZEXPORT unzRepair(const char* file, const char* fileOut, const char* fileOutTmp, uLong* nRecovered, uLong* bytesRecovered) {
   int err = Z_OK;
   FILE* fpZip = fopen(file, "rb");
   FILE* fpOut = fopen(fileOut, "wb");
diff --git a/zlib/contrib/minizip/unzip.c b/zlib/contrib/minizip/unzip.c
index bcfb941..ea05b7d 100644
--- a/zlib/contrib/minizip/unzip.c
+++ b/zlib/contrib/minizip/unzip.c
@@ -49,12 +49,12 @@
         Copyright (C) 2007-2008 Even Rouault
 
 
-        Oct-2009 - Mathias Svensson - Removed cpl_* from symbol names (Even Rouault added them but since this is now moved to a new project (minizip64) I renamed them again).
+  Oct-2009 - Mathias Svensson - Removed cpl_* from symbol names (Even Rouault added them but since this is now moved to a new project (minizip64) I renamed them again).
   Oct-2009 - Mathias Svensson - Fixed problem if uncompressed size was > 4G and compressed size was <4G
                                 should only read the compressed/uncompressed size from the Zip64 format if
                                 the size from normal header was 0xFFFFFFFF
-  Oct-2009 - Mathias Svensson - Applied some bug fixes from paches recived from Gilles Vollant
-        Oct-2009 - Mathias Svensson - Applied support to unzip files with compression mathod BZIP2 (bzip2 lib is required)
+  Oct-2009 - Mathias Svensson - Applied some bug fixes from patches received from Gilles Vollant
+  Oct-2009 - Mathias Svensson - Applied support to unzip files with compression method BZIP2 (bzip2 lib is required)
                                 Patch created by Daniel Borca
 
   Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer
@@ -77,8 +77,6 @@
 
 #ifdef STDC
 #  include <stddef.h>
-#  include <string.h>
-#  include <stdlib.h>
 #endif
 #ifdef NO_ERRNO_H
     extern int errno;
@@ -111,9 +109,6 @@
 #ifndef ALLOC
 # define ALLOC(size) (malloc(size))
 #endif
-#ifndef TRYFREE
-# define TRYFREE(p) {if (p) free(p);}
-#endif
 
 #define SIZECENTRALDIRITEM (0x2e)
 #define SIZEZIPLOCALHEADER (0x1e)
@@ -122,7 +117,7 @@
 const char unz_copyright[] =
    " unzip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll";
 
-/* unz_file_info_interntal contain internal info about a file in zipfile*/
+/* unz_file_info64_internal contain internal info about a file in zipfile*/
 typedef struct unz_file_info64_internal_s
 {
     ZPOS64_T offset_curfile;/* relative offset of local header 8 bytes */
@@ -153,7 +148,7 @@ typedef struct
     ZPOS64_T rest_read_compressed; /* number of byte to be decompressed */
     ZPOS64_T rest_read_uncompressed;/*number of byte to be obtained after decomp*/
     zlib_filefunc64_32_def z_filefunc;
-    voidpf filestream;        /* io structore of the zipfile */
+    voidpf filestream;        /* io structure of the zipfile */
     uLong compression_method;   /* compression method (0==store) */
     ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/
     int   raw;
@@ -166,7 +161,7 @@ typedef struct
 {
     zlib_filefunc64_32_def z_filefunc;
     int is64bitOpenFunction;
-    voidpf filestream;        /* io structore of the zipfile */
+    voidpf filestream;        /* io structure of the zipfile */
     unz_global_info64 gi;       /* public global information */
     ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/
     ZPOS64_T num_file;             /* number of the current file in the zipfile*/
@@ -197,29 +192,24 @@ typedef struct
 #include "crypt.h"
 #endif
 
+
 /* ===========================================================================
-     Read a byte from a gz_stream; update next_in and avail_in. Return EOF
-   for end of file.
-   IN assertion: the stream s has been successfully opened for reading.
+   Reads a long in LSB order from the given gz_stream. Sets
 */
 
-
-local int unz64local_getByte OF((
-    const zlib_filefunc64_32_def* pzlib_filefunc_def,
-    voidpf filestream,
-    int *pi));
-
-local int unz64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi)
-{
-    unsigned char c;
-    int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1);
-    if (err==1)
+local int unz64local_getShort(const zlib_filefunc64_32_def* pzlib_filefunc_def,
+                              voidpf filestream,
+                              uLong *pX) {
+    unsigned char c[2];
+    int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,c,2);
+    if (err==2)
     {
-        *pi = (int)c;
+        *pX = c[0] | ((uLong)c[1] << 8);
         return UNZ_OK;
     }
     else
     {
+        *pX = 0;
         if (ZERROR64(*pzlib_filefunc_def,filestream))
             return UNZ_ERRNO;
         else
@@ -227,127 +217,50 @@ local int unz64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def, v
     }
 }
 
-
-/* ===========================================================================
-   Reads a long in LSB order from the given gz_stream. Sets
-*/
-local int unz64local_getShort OF((
-    const zlib_filefunc64_32_def* pzlib_filefunc_def,
-    voidpf filestream,
-    uLong *pX));
-
-local int unz64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def,
+local int unz64local_getLong(const zlib_filefunc64_32_def* pzlib_filefunc_def,
                              voidpf filestream,
-                             uLong *pX)
-{
-    uLong x ;
-    int i = 0;
-    int err;
-
-    err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x = (uLong)i;
-
-    if (err==UNZ_OK)
-        err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x |= ((uLong)i)<<8;
-
-    if (err==UNZ_OK)
-        *pX = x;
-    else
-        *pX = 0;
-    return err;
-}
-
-local int unz64local_getLong OF((
-    const zlib_filefunc64_32_def* pzlib_filefunc_def,
-    voidpf filestream,
-    uLong *pX));
-
-local int unz64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def,
-                            voidpf filestream,
-                            uLong *pX)
-{
-    uLong x ;
-    int i = 0;
-    int err;
-
-    err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x = (uLong)i;
-
-    if (err==UNZ_OK)
-        err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x |= ((uLong)i)<<8;
-
-    if (err==UNZ_OK)
-        err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x |= ((uLong)i)<<16;
-
-    if (err==UNZ_OK)
-        err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x += ((uLong)i)<<24;
-
-    if (err==UNZ_OK)
-        *pX = x;
+                             uLong *pX) {
+    unsigned char c[4];
+    int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,c,4);
+    if (err==4)
+    {
+        *pX = c[0] | ((uLong)c[1] << 8) | ((uLong)c[2] << 16) | ((uLong)c[3] << 24);
+        return UNZ_OK;
+    }
     else
+    {
         *pX = 0;
-    return err;
+        if (ZERROR64(*pzlib_filefunc_def,filestream))
+            return UNZ_ERRNO;
+        else
+            return UNZ_EOF;
+    }
 }
 
-local int unz64local_getLong64 OF((
-    const zlib_filefunc64_32_def* pzlib_filefunc_def,
-    voidpf filestream,
-    ZPOS64_T *pX));
-
-
-local int unz64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def,
-                            voidpf filestream,
-                            ZPOS64_T *pX)
-{
-    ZPOS64_T x ;
-    int i = 0;
-    int err;
-
-    err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x = (ZPOS64_T)i;
-
-    if (err==UNZ_OK)
-        err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x |= ((ZPOS64_T)i)<<8;
-
-    if (err==UNZ_OK)
-        err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x |= ((ZPOS64_T)i)<<16;
-
-    if (err==UNZ_OK)
-        err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x |= ((ZPOS64_T)i)<<24;
-
-    if (err==UNZ_OK)
-        err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x |= ((ZPOS64_T)i)<<32;
-
-    if (err==UNZ_OK)
-        err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x |= ((ZPOS64_T)i)<<40;
 
-    if (err==UNZ_OK)
-        err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x |= ((ZPOS64_T)i)<<48;
-
-    if (err==UNZ_OK)
-        err = unz64local_getByte(pzlib_filefunc_def,filestream,&i);
-    x |= ((ZPOS64_T)i)<<56;
-
-    if (err==UNZ_OK)
-        *pX = x;
+local int unz64local_getLong64(const zlib_filefunc64_32_def* pzlib_filefunc_def,
+                               voidpf filestream,
+                               ZPOS64_T *pX) {
+    unsigned char c[8];
+    int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,c,8);
+    if (err==8)
+    {
+        *pX = c[0] | ((ZPOS64_T)c[1] << 8) | ((ZPOS64_T)c[2] << 16) | ((ZPOS64_T)c[3] << 24)
+            | ((ZPOS64_T)c[4] << 32) | ((ZPOS64_T)c[5] << 40) | ((ZPOS64_T)c[6] << 48) | ((ZPOS64_T)c[7] << 56);
+        return UNZ_OK;
+    }
     else
+    {
         *pX = 0;
-    return err;
+        if (ZERROR64(*pzlib_filefunc_def,filestream))
+            return UNZ_ERRNO;
+        else
+            return UNZ_EOF;
+    }
 }
 
 /* My own strcmpi / strcasecmp */
-local int strcmpcasenosensitive_internal (const char* fileName1, const char* fileName2)
-{
+local int strcmpcasenosensitive_internal(const char* fileName1, const char* fileName2) {
     for (;;)
     {
         char c1=*(fileName1++);
@@ -379,19 +292,17 @@ local int strcmpcasenosensitive_internal (const char* fileName1, const char* fil
 #endif
 
 /*
-   Compare two filename (fileName1,fileName2).
-   If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp)
-   If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi
+   Compare two filenames (fileName1,fileName2).
+   If iCaseSensitivity = 1, comparison is case sensitive (like strcmp)
+   If iCaseSensitivity = 2, comparison is not case sensitive (like strcmpi
                                                                 or strcasecmp)
-   If iCaseSenisivity = 0, case sensitivity is defaut of your operating system
+   If iCaseSensitivity = 0, case sensitivity is default of your operating system
         (like 1 on Unix, 2 on Windows)
 
 */
 extern int ZEXPORT unzStringFileNameCompare (const char*  fileName1,
-                                                 const char*  fileName2,
-                                                 int iCaseSensitivity)
-
-{
+                                             const char*  fileName2,
+                                             int iCaseSensitivity) {
     if (iCaseSensitivity==0)
         iCaseSensitivity=CASESENSITIVITYDEFAULTVALUE;
 
@@ -405,21 +316,23 @@ extern int ZEXPORT unzStringFileNameCompare (const char*  fileName1,
 #define BUFREADCOMMENT (0x400)
 #endif
 
+#ifndef CENTRALDIRINVALID
+#define CENTRALDIRINVALID ((ZPOS64_T)(-1))
+#endif
+
 /*
   Locate the Central directory of a zipfile (at the end, just before
     the global comment)
 */
-local ZPOS64_T unz64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream));
-local ZPOS64_T unz64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)
-{
+local ZPOS64_T unz64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) {
     unsigned char* buf;
     ZPOS64_T uSizeFile;
     ZPOS64_T uBackRead;
     ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */
-    ZPOS64_T uPosFound=0;
+    ZPOS64_T uPosFound=CENTRALDIRINVALID;
 
     if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0)
-        return 0;
+        return CENTRALDIRINVALID;
 
 
     uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream);
@@ -429,7 +342,7 @@ local ZPOS64_T unz64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_f
 
     buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4);
     if (buf==NULL)
-        return 0;
+        return CENTRALDIRINVALID;
 
     uBackRead = 4;
     while (uBackRead<uMaxBack)
@@ -455,14 +368,14 @@ local ZPOS64_T unz64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_f
             if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) &&
                 ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06))
             {
-                uPosFound = uReadPos+i;
+                uPosFound = uReadPos+(unsigned)i;
                 break;
             }
 
-        if (uPosFound!=0)
+        if (uPosFound!=CENTRALDIRINVALID)
             break;
     }
-    TRYFREE(buf);
+    free(buf);
     return uPosFound;
 }
 
@@ -471,23 +384,18 @@ local ZPOS64_T unz64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_f
   Locate the Central directory 64 of a zipfile (at the end, just before
     the global comment)
 */
-local ZPOS64_T unz64local_SearchCentralDir64 OF((
-    const zlib_filefunc64_32_def* pzlib_filefunc_def,
-    voidpf filestream));
-
 local ZPOS64_T unz64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def,
-                                      voidpf filestream)
-{
+                                             voidpf filestream) {
     unsigned char* buf;
     ZPOS64_T uSizeFile;
     ZPOS64_T uBackRead;
     ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */
-    ZPOS64_T uPosFound=0;
+    ZPOS64_T uPosFound=CENTRALDIRINVALID;
     uLong uL;
                 ZPOS64_T relativeOffset;
 
     if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0)
-        return 0;
+        return CENTRALDIRINVALID;
 
 
     uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream);
@@ -497,7 +405,7 @@ local ZPOS64_T unz64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib
 
     buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4);
     if (buf==NULL)
-        return 0;
+        return CENTRALDIRINVALID;
 
     uBackRead = 4;
     while (uBackRead<uMaxBack)
@@ -523,51 +431,51 @@ local ZPOS64_T unz64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib
             if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) &&
                 ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07))
             {
-                uPosFound = uReadPos+i;
+                uPosFound = uReadPos+(unsigned)i;
                 break;
             }
 
-        if (uPosFound!=0)
+        if (uPosFound!=CENTRALDIRINVALID)
             break;
     }
-    TRYFREE(buf);
-    if (uPosFound == 0)
-        return 0;
+    free(buf);
+    if (uPosFound == CENTRALDIRINVALID)
+        return CENTRALDIRINVALID;
 
     /* Zip64 end of central directory locator */
     if (ZSEEK64(*pzlib_filefunc_def,filestream, uPosFound,ZLIB_FILEFUNC_SEEK_SET)!=0)
-        return 0;
+        return CENTRALDIRINVALID;
 
     /* the signature, already checked */
     if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK)
-        return 0;
+        return CENTRALDIRINVALID;
 
-    /* number of the disk with the start of the zip64 end of  central directory */
+    /* number of the disk with the start of the zip64 end of central directory */
     if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK)
-        return 0;
+        return CENTRALDIRINVALID;
     if (uL != 0)
-        return 0;
+        return CENTRALDIRINVALID;
 
     /* relative offset of the zip64 end of central directory record */
     if (unz64local_getLong64(pzlib_filefunc_def,filestream,&relativeOffset)!=UNZ_OK)
-        return 0;
+        return CENTRALDIRINVALID;
 
     /* total number of disks */
     if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK)
-        return 0;
+        return CENTRALDIRINVALID;
     if (uL != 1)
-        return 0;
+        return CENTRALDIRINVALID;
 
     /* Goto end of central directory record */
     if (ZSEEK64(*pzlib_filefunc_def,filestream, relativeOffset,ZLIB_FILEFUNC_SEEK_SET)!=0)
-        return 0;
+        return CENTRALDIRINVALID;
 
      /* the signature */
     if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK)
-        return 0;
+        return CENTRALDIRINVALID;
 
     if (uL != 0x06064b50)
-        return 0;
+        return CENTRALDIRINVALID;
 
     return relativeOffset;
 }
@@ -581,19 +489,18 @@ local ZPOS64_T unz64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib
      Else, the return value is a unzFile Handle, usable with other function
        of this unzip package.
 */
-local unzFile unzOpenInternal (const void *path,
-                               zlib_filefunc64_32_def* pzlib_filefunc64_32_def,
-                               int is64bitOpenFunction)
-{
+local unzFile unzOpenInternal(const void *path,
+                              zlib_filefunc64_32_def* pzlib_filefunc64_32_def,
+                              int is64bitOpenFunction) {
     unz64_s us;
     unz64_s *s;
     ZPOS64_T central_pos;
     uLong   uL;
 
-    uLong number_disk;          /* number of the current dist, used for
-                                   spaning ZIP, unsupported, always 0*/
-    uLong number_disk_with_CD;  /* number the the disk with central dir, used
-                                   for spaning ZIP, unsupported, always 0*/
+    uLong number_disk;          /* number of the current disk, used for
+                                   spanning ZIP, unsupported, always 0*/
+    uLong number_disk_with_CD;  /* number the disk with central dir, used
+                                   for spanning ZIP, unsupported, always 0*/
     ZPOS64_T number_entry_CD;      /* total number of entries in
                                    the central dir
                                    (same than number_entry on nospan) */
@@ -621,7 +528,7 @@ local unzFile unzOpenInternal (const void *path,
         return NULL;
 
     central_pos = unz64local_SearchCentralDir64(&us.z_filefunc,us.filestream);
-    if (central_pos)
+    if (central_pos!=CENTRALDIRINVALID)
     {
         uLong uS;
         ZPOS64_T uL64;
@@ -683,7 +590,7 @@ local unzFile unzOpenInternal (const void *path,
     else
     {
         central_pos = unz64local_SearchCentralDir(&us.z_filefunc,us.filestream);
-        if (central_pos==0)
+        if (central_pos==CENTRALDIRINVALID)
             err=UNZ_ERRNO;
 
         us.isZip64 = 0;
@@ -762,9 +669,8 @@ local unzFile unzOpenInternal (const void *path,
 }
 
 
-extern unzFile ZEXPORT unzOpen2 (const char *path,
-                                        zlib_filefunc_def* pzlib_filefunc32_def)
-{
+extern unzFile ZEXPORT unzOpen2(const char *path,
+                                zlib_filefunc_def* pzlib_filefunc32_def) {
     if (pzlib_filefunc32_def != NULL)
     {
         zlib_filefunc64_32_def zlib_filefunc64_32_def_fill;
@@ -775,9 +681,8 @@ extern unzFile ZEXPORT unzOpen2 (const char *path,
         return unzOpenInternal(path, NULL, 0);
 }
 
-extern unzFile ZEXPORT unzOpen2_64 (const void *path,
-                                     zlib_filefunc64_def* pzlib_filefunc_def)
-{
+extern unzFile ZEXPORT unzOpen2_64(const void *path,
+                                   zlib_filefunc64_def* pzlib_filefunc_def) {
     if (pzlib_filefunc_def != NULL)
     {
         zlib_filefunc64_32_def zlib_filefunc64_32_def_fill;
@@ -790,13 +695,11 @@ extern unzFile ZEXPORT unzOpen2_64 (const void *path,
         return unzOpenInternal(path, NULL, 1);
 }
 
-extern unzFile ZEXPORT unzOpen (const char *path)
-{
+extern unzFile ZEXPORT unzOpen(const char *path) {
     return unzOpenInternal(path, NULL, 0);
 }
 
-extern unzFile ZEXPORT unzOpen64 (const void *path)
-{
+extern unzFile ZEXPORT unzOpen64(const void *path) {
     return unzOpenInternal(path, NULL, 1);
 }
 
@@ -805,8 +708,7 @@ extern unzFile ZEXPORT unzOpen64 (const void *path)
   If there is files inside the .Zip opened with unzOpenCurrentFile (see later),
     these files MUST be closed with unzCloseCurrentFile before call unzClose.
   return UNZ_OK if there is no problem. */
-extern int ZEXPORT unzClose (unzFile file)
-{
+extern int ZEXPORT unzClose(unzFile file) {
     unz64_s* s;
     if (file==NULL)
         return UNZ_PARAMERROR;
@@ -816,7 +718,7 @@ extern int ZEXPORT unzClose (unzFile file)
         unzCloseCurrentFile(file);
 
     ZCLOSE64(s->z_filefunc, s->filestream);
-    TRYFREE(s);
+    free(s);
     return UNZ_OK;
 }
 
@@ -825,8 +727,7 @@ extern int ZEXPORT unzClose (unzFile file)
   Write info about the ZipFile in the *pglobal_info structure.
   No preparation of the structure is needed
   return UNZ_OK if there is no problem. */
-extern int ZEXPORT unzGetGlobalInfo64 (unzFile file, unz_global_info64* pglobal_info)
-{
+extern int ZEXPORT unzGetGlobalInfo64(unzFile file, unz_global_info64* pglobal_info) {
     unz64_s* s;
     if (file==NULL)
         return UNZ_PARAMERROR;
@@ -835,8 +736,7 @@ extern int ZEXPORT unzGetGlobalInfo64 (unzFile file, unz_global_info64* pglobal_
     return UNZ_OK;
 }
 
-extern int ZEXPORT unzGetGlobalInfo (unzFile file, unz_global_info* pglobal_info32)
-{
+extern int ZEXPORT unzGetGlobalInfo(unzFile file, unz_global_info* pglobal_info32) {
     unz64_s* s;
     if (file==NULL)
         return UNZ_PARAMERROR;
@@ -847,46 +747,33 @@ extern int ZEXPORT unzGetGlobalInfo (unzFile file, unz_global_info* pglobal_info
     return UNZ_OK;
 }
 /*
-   Translate date/time from Dos format to tm_unz (readable more easilty)
+   Translate date/time from Dos format to tm_unz (readable more easily)
 */
-local void unz64local_DosDateToTmuDate (ZPOS64_T ulDosDate, tm_unz* ptm)
-{
+local void unz64local_DosDateToTmuDate(ZPOS64_T ulDosDate, tm_unz* ptm) {
     ZPOS64_T uDate;
     uDate = (ZPOS64_T)(ulDosDate>>16);
-    ptm->tm_mday = (uInt)(uDate&0x1f) ;
-    ptm->tm_mon =  (uInt)((((uDate)&0x1E0)/0x20)-1) ;
-    ptm->tm_year = (uInt)(((uDate&0x0FE00)/0x0200)+1980) ;
+    ptm->tm_mday = (int)(uDate&0x1f) ;
+    ptm->tm_mon =  (int)((((uDate)&0x1E0)/0x20)-1) ;
+    ptm->tm_year = (int)(((uDate&0x0FE00)/0x0200)+1980) ;
 
-    ptm->tm_hour = (uInt) ((ulDosDate &0xF800)/0x800);
-    ptm->tm_min =  (uInt) ((ulDosDate&0x7E0)/0x20) ;
-    ptm->tm_sec =  (uInt) (2*(ulDosDate&0x1f)) ;
+    ptm->tm_hour = (int) ((ulDosDate &0xF800)/0x800);
+    ptm->tm_min =  (int) ((ulDosDate&0x7E0)/0x20) ;
+    ptm->tm_sec =  (int) (2*(ulDosDate&0x1f)) ;
 }
 
 /*
   Get Info about the current file in the zipfile, with internal only info
 */
-local int unz64local_GetCurrentFileInfoInternal OF((unzFile file,
-                                                  unz_file_info64 *pfile_info,
-                                                  unz_file_info64_internal
-                                                  *pfile_info_internal,
-                                                  char *szFileName,
-                                                  uLong fileNameBufferSize,
-                                                  void *extraField,
-                                                  uLong extraFieldBufferSize,
-                                                  char *szComment,
-                                                  uLong commentBufferSize));
-
-local int unz64local_GetCurrentFileInfoInternal (unzFile file,
-                                                  unz_file_info64 *pfile_info,
-                                                  unz_file_info64_internal
-                                                  *pfile_info_internal,
-                                                  char *szFileName,
-                                                  uLong fileNameBufferSize,
-                                                  void *extraField,
-                                                  uLong extraFieldBufferSize,
-                                                  char *szComment,
-                                                  uLong commentBufferSize)
-{
+local int unz64local_GetCurrentFileInfoInternal(unzFile file,
+                                                unz_file_info64 *pfile_info,
+                                                unz_file_info64_internal
+                                                *pfile_info_internal,
+                                                char *szFileName,
+                                                uLong fileNameBufferSize,
+                                                void *extraField,
+                                                uLong extraFieldBufferSize,
+                                                char *szComment,
+                                                uLong commentBufferSize) {
     unz64_s* s;
     unz_file_info64 file_info;
     unz_file_info64_internal file_info_internal;
@@ -993,7 +880,7 @@ local int unz64local_GetCurrentFileInfoInternal (unzFile file,
 
         if (lSeek!=0)
         {
-            if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0)
+            if (ZSEEK64(s->z_filefunc, s->filestream,(ZPOS64_T)lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0)
                 lSeek=0;
             else
                 err=UNZ_ERRNO;
@@ -1018,7 +905,7 @@ local int unz64local_GetCurrentFileInfoInternal (unzFile file,
 
         if (lSeek!=0)
         {
-            if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0)
+            if (ZSEEK64(s->z_filefunc, s->filestream,(ZPOS64_T)lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0)
                 lSeek=0;
             else
                 err=UNZ_ERRNO;
@@ -1038,33 +925,31 @@ local int unz64local_GetCurrentFileInfoInternal (unzFile file,
             /* ZIP64 extra fields */
             if (headerId == 0x0001)
             {
-                                                        uLong uL;
-
-                                                                if(file_info.uncompressed_size == MAXU32)
-                                                                {
-                                                                        if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.uncompressed_size) != UNZ_OK)
-                                                                                        err=UNZ_ERRNO;
-                                                                }
-
-                                                                if(file_info.compressed_size == MAXU32)
-                                                                {
-                                                                        if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.compressed_size) != UNZ_OK)
-                                                                                  err=UNZ_ERRNO;
-                                                                }
-
-                                                                if(file_info_internal.offset_curfile == MAXU32)
-                                                                {
-                                                                        /* Relative Header offset */
-                                                                        if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info_internal.offset_curfile) != UNZ_OK)
-                                                                                err=UNZ_ERRNO;
-                                                                }
-
-                                                                if(file_info.disk_num_start == MAXU32)
-                                                                {
-                                                                        /* Disk Start Number */
-                                                                        if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK)
-                                                                                err=UNZ_ERRNO;
-                                                                }
+                if(file_info.uncompressed_size == MAXU32)
+                {
+                    if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.uncompressed_size) != UNZ_OK)
+                        err=UNZ_ERRNO;
+                }
+
+                if(file_info.compressed_size == MAXU32)
+                {
+                    if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.compressed_size) != UNZ_OK)
+                        err=UNZ_ERRNO;
+                }
+
+                if(file_info_internal.offset_curfile == MAXU32)
+                {
+                    /* Relative Header offset */
+                    if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info_internal.offset_curfile) != UNZ_OK)
+                        err=UNZ_ERRNO;
+                }
+
+                if(file_info.disk_num_start == 0xffff)
+                {
+                    /* Disk Start Number */
+                    if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.disk_num_start) != UNZ_OK)
+                        err=UNZ_ERRNO;
+                }
 
             }
             else
@@ -1090,7 +975,7 @@ local int unz64local_GetCurrentFileInfoInternal (unzFile file,
 
         if (lSeek!=0)
         {
-            if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0)
+            if (ZSEEK64(s->z_filefunc, s->filestream,(ZPOS64_T)lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0)
                 lSeek=0;
             else
                 err=UNZ_ERRNO;
@@ -1121,24 +1006,22 @@ local int unz64local_GetCurrentFileInfoInternal (unzFile file,
   No preparation of the structure is needed
   return UNZ_OK if there is no problem.
 */
-extern int ZEXPORT unzGetCurrentFileInfo64 (unzFile file,
-                                          unz_file_info64 * pfile_info,
-                                          char * szFileName, uLong fileNameBufferSize,
-                                          void *extraField, uLong extraFieldBufferSize,
-                                          char* szComment,  uLong commentBufferSize)
-{
+extern int ZEXPORT unzGetCurrentFileInfo64(unzFile file,
+                                           unz_file_info64 * pfile_info,
+                                           char * szFileName, uLong fileNameBufferSize,
+                                           void *extraField, uLong extraFieldBufferSize,
+                                           char* szComment,  uLong commentBufferSize) {
     return unz64local_GetCurrentFileInfoInternal(file,pfile_info,NULL,
-                                                szFileName,fileNameBufferSize,
-                                                extraField,extraFieldBufferSize,
-                                                szComment,commentBufferSize);
+                                                 szFileName,fileNameBufferSize,
+                                                 extraField,extraFieldBufferSize,
+                                                 szComment,commentBufferSize);
 }
 
-extern int ZEXPORT unzGetCurrentFileInfo (unzFile file,
-                                          unz_file_info * pfile_info,
-                                          char * szFileName, uLong fileNameBufferSize,
-                                          void *extraField, uLong extraFieldBufferSize,
-                                          char* szComment,  uLong commentBufferSize)
-{
+extern int ZEXPORT unzGetCurrentFileInfo(unzFile file,
+                                         unz_file_info * pfile_info,
+                                         char * szFileName, uLong fileNameBufferSize,
+                                         void *extraField, uLong extraFieldBufferSize,
+                                         char* szComment,  uLong commentBufferSize) {
     int err;
     unz_file_info64 file_info64;
     err = unz64local_GetCurrentFileInfoInternal(file,&file_info64,NULL,
@@ -1162,7 +1045,7 @@ extern int ZEXPORT unzGetCurrentFileInfo (unzFile file,
         pfile_info->internal_fa = file_info64.internal_fa;
         pfile_info->external_fa = file_info64.external_fa;
 
-        pfile_info->tmu_date = file_info64.tmu_date,
+        pfile_info->tmu_date = file_info64.tmu_date;
 
 
         pfile_info->compressed_size = (uLong)file_info64.compressed_size;
@@ -1175,8 +1058,7 @@ extern int ZEXPORT unzGetCurrentFileInfo (unzFile file,
   Set the current file of the zipfile to the first file.
   return UNZ_OK if there is no problem
 */
-extern int ZEXPORT unzGoToFirstFile (unzFile file)
-{
+extern int ZEXPORT unzGoToFirstFile(unzFile file) {
     int err=UNZ_OK;
     unz64_s* s;
     if (file==NULL)
@@ -1196,8 +1078,7 @@ extern int ZEXPORT unzGoToFirstFile (unzFile file)
   return UNZ_OK if there is no problem
   return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest.
 */
-extern int ZEXPORT unzGoToNextFile (unzFile  file)
-{
+extern int ZEXPORT unzGoToNextFile(unzFile file) {
     unz64_s* s;
     int err;
 
@@ -1229,8 +1110,7 @@ extern int ZEXPORT unzGoToNextFile (unzFile  file)
   UNZ_OK if the file is found. It becomes the current file.
   UNZ_END_OF_LIST_OF_FILE if the file is not found
 */
-extern int ZEXPORT unzLocateFile (unzFile file, const char *szFileName, int iCaseSensitivity)
-{
+extern int ZEXPORT unzLocateFile(unzFile file, const char *szFileName, int iCaseSensitivity) {
     unz64_s* s;
     int err;
 
@@ -1305,8 +1185,7 @@ typedef struct unz_file_pos_s
 } unz_file_pos;
 */
 
-extern int ZEXPORT unzGetFilePos64(unzFile file, unz64_file_pos*  file_pos)
-{
+extern int ZEXPORT unzGetFilePos64(unzFile file, unz64_file_pos* file_pos) {
     unz64_s* s;
 
     if (file==NULL || file_pos==NULL)
@@ -1321,10 +1200,7 @@ extern int ZEXPORT unzGetFilePos64(unzFile file, unz64_file_pos*  file_pos)
     return UNZ_OK;
 }
 
-extern int ZEXPORT unzGetFilePos(
-    unzFile file,
-    unz_file_pos* file_pos)
-{
+extern int ZEXPORT unzGetFilePos(unzFile file, unz_file_pos* file_pos) {
     unz64_file_pos file_pos64;
     int err = unzGetFilePos64(file,&file_pos64);
     if (err==UNZ_OK)
@@ -1335,8 +1211,7 @@ extern int ZEXPORT unzGetFilePos(
     return err;
 }
 
-extern int ZEXPORT unzGoToFilePos64(unzFile file, const unz64_file_pos* file_pos)
-{
+extern int ZEXPORT unzGoToFilePos64(unzFile file, const unz64_file_pos* file_pos) {
     unz64_s* s;
     int err;
 
@@ -1357,10 +1232,7 @@ extern int ZEXPORT unzGoToFilePos64(unzFile file, const unz64_file_pos* file_pos
     return err;
 }
 
-extern int ZEXPORT unzGoToFilePos(
-    unzFile file,
-    unz_file_pos* file_pos)
-{
+extern int ZEXPORT unzGoToFilePos(unzFile file, unz_file_pos* file_pos) {
     unz64_file_pos file_pos64;
     if (file_pos == NULL)
         return UNZ_PARAMERROR;
@@ -1382,10 +1254,9 @@ extern int ZEXPORT unzGoToFilePos(
   store in *piSizeVar the size of extra info in local header
         (filename and size of extra field data)
 */
-local int unz64local_CheckCurrentFileCoherencyHeader (unz64_s* s, uInt* piSizeVar,
-                                                    ZPOS64_T * poffset_local_extrafield,
-                                                    uInt  * psize_local_extrafield)
-{
+local int unz64local_CheckCurrentFileCoherencyHeader(unz64_s* s, uInt* piSizeVar,
+                                                     ZPOS64_T * poffset_local_extrafield,
+                                                     uInt  * psize_local_extrafield) {
     uLong uMagic,uData,uFlags;
     uLong size_filename;
     uLong size_extra_field;
@@ -1469,9 +1340,8 @@ local int unz64local_CheckCurrentFileCoherencyHeader (unz64_s* s, uInt* piSizeVa
   Open for reading data the current file in the zipfile.
   If there is no error and the file is opened, the return value is UNZ_OK.
 */
-extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, int* method,
-                                            int* level, int raw, const char* password)
-{
+extern int ZEXPORT unzOpenCurrentFile3(unzFile file, int* method,
+                                       int* level, int raw, const char* password) {
     int err=UNZ_OK;
     uInt iSizeVar;
     unz64_s* s;
@@ -1509,7 +1379,7 @@ extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, int* method,
 
     if (pfile_in_zip_read_info->read_buffer==NULL)
     {
-        TRYFREE(pfile_in_zip_read_info);
+        free(pfile_in_zip_read_info);
         return UNZ_INTERNALERROR;
     }
 
@@ -1566,7 +1436,8 @@ extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, int* method,
         pfile_in_zip_read_info->stream_initialised=Z_BZIP2ED;
       else
       {
-        TRYFREE(pfile_in_zip_read_info);
+        free(pfile_in_zip_read_info->read_buffer);
+        free(pfile_in_zip_read_info);
         return err;
       }
 #else
@@ -1586,7 +1457,8 @@ extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, int* method,
         pfile_in_zip_read_info->stream_initialised=Z_DEFLATED;
       else
       {
-        TRYFREE(pfile_in_zip_read_info);
+        free(pfile_in_zip_read_info->read_buffer);
+        free(pfile_in_zip_read_info);
         return err;
       }
         /* windowBits is passed < 0 to tell that there is no zlib header.
@@ -1638,25 +1510,21 @@ extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, int* method,
     return UNZ_OK;
 }
 
-extern int ZEXPORT unzOpenCurrentFile (unzFile file)
-{
+extern int ZEXPORT unzOpenCurrentFile(unzFile file) {
     return unzOpenCurrentFile3(file, NULL, NULL, 0, NULL);
 }
 
-extern int ZEXPORT unzOpenCurrentFilePassword (unzFile file, const char*  password)
-{
+extern int ZEXPORT unzOpenCurrentFilePassword(unzFile file, const char* password) {
     return unzOpenCurrentFile3(file, NULL, NULL, 0, password);
 }
 
-extern int ZEXPORT unzOpenCurrentFile2 (unzFile file, int* method, int* level, int raw)
-{
+extern int ZEXPORT unzOpenCurrentFile2(unzFile file, int* method, int* level, int raw) {
     return unzOpenCurrentFile3(file, method, level, raw, NULL);
 }
 
 /** Addition for GDAL : START */
 
-extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64( unzFile file)
-{
+extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64(unzFile file) {
     unz64_s* s;
     file_in_zip64_read_info_s* pfile_in_zip_read_info;
     s=(unz64_s*)file;
@@ -1676,13 +1544,12 @@ extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64( unzFile file)
   buf contain buffer where data must be copied
   len the size of buf.
 
-  return the number of byte copied if somes bytes are copied
+  return the number of byte copied if some bytes are copied
   return 0 if the end of file was reached
   return <0 with error code if there is an error
     (UNZ_ERRNO for IO error, or zLib error for uncompress error)
 */
-extern int ZEXPORT unzReadCurrentFile  (unzFile file, voidp buf, unsigned len)
-{
+extern int ZEXPORT unzReadCurrentFile(unzFile file, voidp buf, unsigned len) {
     int err=UNZ_OK;
     uInt iRead = 0;
     unz64_s* s;
@@ -1767,7 +1634,7 @@ extern int ZEXPORT unzReadCurrentFile  (unzFile file, voidp buf, unsigned len)
 
             if ((pfile_in_zip_read_info->stream.avail_in == 0) &&
                 (pfile_in_zip_read_info->rest_read_compressed == 0))
-                return (iRead==0) ? UNZ_EOF : iRead;
+                return (iRead==0) ? UNZ_EOF : (int)iRead;
 
             if (pfile_in_zip_read_info->stream.avail_out <
                             pfile_in_zip_read_info->stream.avail_in)
@@ -1857,6 +1724,9 @@ extern int ZEXPORT unzReadCurrentFile  (unzFile file, voidp buf, unsigned len)
               err = Z_DATA_ERROR;
 
             uTotalOutAfter = pfile_in_zip_read_info->stream.total_out;
+            /* Detect overflow, because z_stream.total_out is uLong (32 bits) */
+            if (uTotalOutAfter<uTotalOutBefore)
+                uTotalOutAfter += 1LL << 32; /* Add maximum value of uLong + 1 */
             uOutThis = uTotalOutAfter-uTotalOutBefore;
 
             pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uOutThis;
@@ -1871,14 +1741,14 @@ extern int ZEXPORT unzReadCurrentFile  (unzFile file, voidp buf, unsigned len)
             iRead += (uInt)(uTotalOutAfter - uTotalOutBefore);
 
             if (err==Z_STREAM_END)
-                return (iRead==0) ? UNZ_EOF : iRead;
+                return (iRead==0) ? UNZ_EOF : (int)iRead;
             if (err!=Z_OK)
                 break;
         }
     }
 
     if (err==Z_OK)
-        return iRead;
+        return (int)iRead;
     return err;
 }
 
@@ -1886,8 +1756,7 @@ extern int ZEXPORT unzReadCurrentFile  (unzFile file, voidp buf, unsigned len)
 /*
   Give the current position in uncompressed data
 */
-extern z_off_t ZEXPORT unztell (unzFile file)
-{
+extern z_off_t ZEXPORT unztell(unzFile file) {
     unz64_s* s;
     file_in_zip64_read_info_s* pfile_in_zip_read_info;
     if (file==NULL)
@@ -1901,8 +1770,7 @@ extern z_off_t ZEXPORT unztell (unzFile file)
     return (z_off_t)pfile_in_zip_read_info->stream.total_out;
 }
 
-extern ZPOS64_T ZEXPORT unztell64 (unzFile file)
-{
+extern ZPOS64_T ZEXPORT unztell64(unzFile file) {
 
     unz64_s* s;
     file_in_zip64_read_info_s* pfile_in_zip_read_info;
@@ -1921,8 +1789,7 @@ extern ZPOS64_T ZEXPORT unztell64 (unzFile file)
 /*
   return 1 if the end of file was reached, 0 elsewhere
 */
-extern int ZEXPORT unzeof (unzFile file)
-{
+extern int ZEXPORT unzeof(unzFile file) {
     unz64_s* s;
     file_in_zip64_read_info_s* pfile_in_zip_read_info;
     if (file==NULL)
@@ -1953,8 +1820,7 @@ more info in the local-header version than in the central-header)
   the return value is the number of bytes copied in buf, or (if <0)
     the error code
 */
-extern int ZEXPORT unzGetLocalExtrafield (unzFile file, voidp buf, unsigned len)
-{
+extern int ZEXPORT unzGetLocalExtrafield(unzFile file, voidp buf, unsigned len) {
     unz64_s* s;
     file_in_zip64_read_info_s* pfile_in_zip_read_info;
     uInt read_now;
@@ -2001,8 +1867,7 @@ extern int ZEXPORT unzGetLocalExtrafield (unzFile file, voidp buf, unsigned len)
   Close the file in zip opened with unzOpenCurrentFile
   Return UNZ_CRCERROR if all the file was read but the CRC is not good
 */
-extern int ZEXPORT unzCloseCurrentFile (unzFile file)
-{
+extern int ZEXPORT unzCloseCurrentFile(unzFile file) {
     int err=UNZ_OK;
 
     unz64_s* s;
@@ -2024,7 +1889,7 @@ extern int ZEXPORT unzCloseCurrentFile (unzFile file)
     }
 
 
-    TRYFREE(pfile_in_zip_read_info->read_buffer);
+    free(pfile_in_zip_read_info->read_buffer);
     pfile_in_zip_read_info->read_buffer = NULL;
     if (pfile_in_zip_read_info->stream_initialised == Z_DEFLATED)
         inflateEnd(&pfile_in_zip_read_info->stream);
@@ -2035,7 +1900,7 @@ extern int ZEXPORT unzCloseCurrentFile (unzFile file)
 
 
     pfile_in_zip_read_info->stream_initialised = 0;
-    TRYFREE(pfile_in_zip_read_info);
+    free(pfile_in_zip_read_info);
 
     s->pfile_in_zip_read=NULL;
 
@@ -2048,8 +1913,7 @@ extern int ZEXPORT unzCloseCurrentFile (unzFile file)
   uSizeBuf is the size of the szComment buffer.
   return the number of byte copied or an error code <0
 */
-extern int ZEXPORT unzGetGlobalComment (unzFile file, char * szComment, uLong uSizeBuf)
-{
+extern int ZEXPORT unzGetGlobalComment(unzFile file, char * szComment, uLong uSizeBuf) {
     unz64_s* s;
     uLong uReadThis ;
     if (file==NULL)
@@ -2076,8 +1940,7 @@ extern int ZEXPORT unzGetGlobalComment (unzFile file, char * szComment, uLong uS
 }
 
 /* Additions by RX '2004 */
-extern ZPOS64_T ZEXPORT unzGetOffset64(unzFile file)
-{
+extern ZPOS64_T ZEXPORT unzGetOffset64(unzFile file) {
     unz64_s* s;
 
     if (file==NULL)
@@ -2091,8 +1954,7 @@ extern ZPOS64_T ZEXPORT unzGetOffset64(unzFile file)
     return s->pos_in_central_dir;
 }
 
-extern uLong ZEXPORT unzGetOffset (unzFile file)
-{
+extern uLong ZEXPORT unzGetOffset(unzFile file) {
     ZPOS64_T offset64;
 
     if (file==NULL)
@@ -2101,8 +1963,7 @@ extern uLong ZEXPORT unzGetOffset (unzFile file)
     return (uLong)offset64;
 }
 
-extern int ZEXPORT unzSetOffset64(unzFile file, ZPOS64_T pos)
-{
+extern int ZEXPORT unzSetOffset64(unzFile file, ZPOS64_T pos) {
     unz64_s* s;
     int err;
 
@@ -2119,7 +1980,6 @@ extern int ZEXPORT unzSetOffset64(unzFile file, ZPOS64_T pos)
     return err;
 }
 
-extern int ZEXPORT unzSetOffset (unzFile file, uLong pos)
-{
+extern int ZEXPORT unzSetOffset (unzFile file, uLong pos) {
     return unzSetOffset64(file,pos);
 }
diff --git a/zlib/contrib/minizip/unzip.h b/zlib/contrib/minizip/unzip.h
index 2104e39..5cfc9c6 100644
--- a/zlib/contrib/minizip/unzip.h
+++ b/zlib/contrib/minizip/unzip.h
@@ -83,12 +83,12 @@ typedef voidp unzFile;
 /* tm_unz contain date/time info */
 typedef struct tm_unz_s
 {
-    uInt tm_sec;            /* seconds after the minute - [0,59] */
-    uInt tm_min;            /* minutes after the hour - [0,59] */
-    uInt tm_hour;           /* hours since midnight - [0,23] */
-    uInt tm_mday;           /* day of the month - [1,31] */
-    uInt tm_mon;            /* months since January - [0,11] */
-    uInt tm_year;           /* years - [1980..2044] */
+    int tm_sec;             /* seconds after the minute - [0,59] */
+    int tm_min;             /* minutes after the hour - [0,59] */
+    int tm_hour;            /* hours since midnight - [0,23] */
+    int tm_mday;            /* day of the month - [1,31] */
+    int tm_mon;             /* months since January - [0,11] */
+    int tm_year;            /* years - [1980..2044] */
 } tm_unz;
 
 /* unz_global_info structure contain global data about the ZIPfile
@@ -150,21 +150,21 @@ typedef struct unz_file_info_s
     tm_unz tmu_date;
 } unz_file_info;
 
-extern int ZEXPORT unzStringFileNameCompare OF ((const char* fileName1,
-                                                 const char* fileName2,
-                                                 int iCaseSensitivity));
+extern int ZEXPORT unzStringFileNameCompare(const char* fileName1,
+                                            const char* fileName2,
+                                            int iCaseSensitivity);
 /*
-   Compare two filename (fileName1,fileName2).
-   If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp)
-   If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi
+   Compare two filenames (fileName1,fileName2).
+   If iCaseSensitivity = 1, comparison is case sensitive (like strcmp)
+   If iCaseSensitivity = 2, comparison is not case sensitive (like strcmpi
                                 or strcasecmp)
-   If iCaseSenisivity = 0, case sensitivity is defaut of your operating system
+   If iCaseSensitivity = 0, case sensitivity is default of your operating system
     (like 1 on Unix, 2 on Windows)
 */
 
 
-extern unzFile ZEXPORT unzOpen OF((const char *path));
-extern unzFile ZEXPORT unzOpen64 OF((const void *path));
+extern unzFile ZEXPORT unzOpen(const char *path);
+extern unzFile ZEXPORT unzOpen64(const void *path);
 /*
   Open a Zip file. path contain the full pathname (by example,
      on a Windows XP computer "c:\\zlib\\zlib113.zip" or on an Unix computer
@@ -181,41 +181,41 @@ extern unzFile ZEXPORT unzOpen64 OF((const void *path));
 */
 
 
-extern unzFile ZEXPORT unzOpen2 OF((const char *path,
-                                    zlib_filefunc_def* pzlib_filefunc_def));
+extern unzFile ZEXPORT unzOpen2(const char *path,
+                                zlib_filefunc_def* pzlib_filefunc_def);
 /*
    Open a Zip file, like unzOpen, but provide a set of file low level API
       for read/write the zip file (see ioapi.h)
 */
 
-extern unzFile ZEXPORT unzOpen2_64 OF((const void *path,
-                                    zlib_filefunc64_def* pzlib_filefunc_def));
+extern unzFile ZEXPORT unzOpen2_64(const void *path,
+                                   zlib_filefunc64_def* pzlib_filefunc_def);
 /*
    Open a Zip file, like unz64Open, but provide a set of file low level API
       for read/write the zip file (see ioapi.h)
 */
 
-extern int ZEXPORT unzClose OF((unzFile file));
+extern int ZEXPORT unzClose(unzFile file);
 /*
   Close a ZipFile opened with unzOpen.
   If there is files inside the .Zip opened with unzOpenCurrentFile (see later),
     these files MUST be closed with unzCloseCurrentFile before call unzClose.
   return UNZ_OK if there is no problem. */
 
-extern int ZEXPORT unzGetGlobalInfo OF((unzFile file,
-                                        unz_global_info *pglobal_info));
+extern int ZEXPORT unzGetGlobalInfo(unzFile file,
+                                    unz_global_info *pglobal_info);
 
-extern int ZEXPORT unzGetGlobalInfo64 OF((unzFile file,
-                                        unz_global_info64 *pglobal_info));
+extern int ZEXPORT unzGetGlobalInfo64(unzFile file,
+                                      unz_global_info64 *pglobal_info);
 /*
   Write info about the ZipFile in the *pglobal_info structure.
   No preparation of the structure is needed
   return UNZ_OK if there is no problem. */
 
 
-extern int ZEXPORT unzGetGlobalComment OF((unzFile file,
-                                           char *szComment,
-                                           uLong uSizeBuf));
+extern int ZEXPORT unzGetGlobalComment(unzFile file,
+                                       char *szComment,
+                                       uLong uSizeBuf);
 /*
   Get the global comment string of the ZipFile, in the szComment buffer.
   uSizeBuf is the size of the szComment buffer.
@@ -226,22 +226,22 @@ extern int ZEXPORT unzGetGlobalComment OF((unzFile file,
 /***************************************************************************/
 /* Unzip package allow you browse the directory of the zipfile */
 
-extern int ZEXPORT unzGoToFirstFile OF((unzFile file));
+extern int ZEXPORT unzGoToFirstFile(unzFile file);
 /*
   Set the current file of the zipfile to the first file.
   return UNZ_OK if there is no problem
 */
 
-extern int ZEXPORT unzGoToNextFile OF((unzFile file));
+extern int ZEXPORT unzGoToNextFile(unzFile file);
 /*
   Set the current file of the zipfile to the next file.
   return UNZ_OK if there is no problem
   return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest.
 */
 
-extern int ZEXPORT unzLocateFile OF((unzFile file,
-                     const char *szFileName,
-                     int iCaseSensitivity));
+extern int ZEXPORT unzLocateFile(unzFile file,
+                                 const char *szFileName,
+                                 int iCaseSensitivity);
 /*
   Try locate the file szFileName in the zipfile.
   For the iCaseSensitivity signification, see unzStringFileNameCompare
@@ -285,28 +285,28 @@ extern int ZEXPORT unzGoToFilePos64(
 
 /* ****************************************** */
 
-extern int ZEXPORT unzGetCurrentFileInfo64 OF((unzFile file,
-                         unz_file_info64 *pfile_info,
-                         char *szFileName,
-                         uLong fileNameBufferSize,
-                         void *extraField,
-                         uLong extraFieldBufferSize,
-                         char *szComment,
-                         uLong commentBufferSize));
-
-extern int ZEXPORT unzGetCurrentFileInfo OF((unzFile file,
-                         unz_file_info *pfile_info,
-                         char *szFileName,
-                         uLong fileNameBufferSize,
-                         void *extraField,
-                         uLong extraFieldBufferSize,
-                         char *szComment,
-                         uLong commentBufferSize));
+extern int ZEXPORT unzGetCurrentFileInfo64(unzFile file,
+                                           unz_file_info64 *pfile_info,
+                                           char *szFileName,
+                                           uLong fileNameBufferSize,
+                                           void *extraField,
+                                           uLong extraFieldBufferSize,
+                                           char *szComment,
+                                           uLong commentBufferSize);
+
+extern int ZEXPORT unzGetCurrentFileInfo(unzFile file,
+                                         unz_file_info *pfile_info,
+                                         char *szFileName,
+                                         uLong fileNameBufferSize,
+                                         void *extraField,
+                                         uLong extraFieldBufferSize,
+                                         char *szComment,
+                                         uLong commentBufferSize);
 /*
   Get Info about the current file
-  if pfile_info!=NULL, the *pfile_info structure will contain somes info about
+  if pfile_info!=NULL, the *pfile_info structure will contain some info about
         the current file
-  if szFileName!=NULL, the filemane string will be copied in szFileName
+  if szFileName!=NULL, the filename string will be copied in szFileName
             (fileNameBufferSize is the size of the buffer)
   if extraField!=NULL, the extra field information will be copied in extraField
             (extraFieldBufferSize is the size of the buffer).
@@ -318,7 +318,7 @@ extern int ZEXPORT unzGetCurrentFileInfo OF((unzFile file,
 
 /** Addition for GDAL : START */
 
-extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64 OF((unzFile file));
+extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64(unzFile file);
 
 /** Addition for GDAL : END */
 
@@ -328,24 +328,24 @@ extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64 OF((unzFile file));
    from it, and close it (you can close it before reading all the file)
    */
 
-extern int ZEXPORT unzOpenCurrentFile OF((unzFile file));
+extern int ZEXPORT unzOpenCurrentFile(unzFile file);
 /*
   Open for reading data the current file in the zipfile.
   If there is no error, the return value is UNZ_OK.
 */
 
-extern int ZEXPORT unzOpenCurrentFilePassword OF((unzFile file,
-                                                  const char* password));
+extern int ZEXPORT unzOpenCurrentFilePassword(unzFile file,
+                                              const char* password);
 /*
   Open for reading data the current file in the zipfile.
   password is a crypting password
   If there is no error, the return value is UNZ_OK.
 */
 
-extern int ZEXPORT unzOpenCurrentFile2 OF((unzFile file,
-                                           int* method,
-                                           int* level,
-                                           int raw));
+extern int ZEXPORT unzOpenCurrentFile2(unzFile file,
+                                       int* method,
+                                       int* level,
+                                       int raw);
 /*
   Same than unzOpenCurrentFile, but open for read raw the file (not uncompress)
     if raw==1
@@ -355,11 +355,11 @@ extern int ZEXPORT unzOpenCurrentFile2 OF((unzFile file,
          but you CANNOT set method parameter as NULL
 */
 
-extern int ZEXPORT unzOpenCurrentFile3 OF((unzFile file,
-                                           int* method,
-                                           int* level,
-                                           int raw,
-                                           const char* password));
+extern int ZEXPORT unzOpenCurrentFile3(unzFile file,
+                                       int* method,
+                                       int* level,
+                                       int raw,
+                                       const char* password);
 /*
   Same than unzOpenCurrentFile, but open for read raw the file (not uncompress)
     if raw==1
@@ -370,41 +370,41 @@ extern int ZEXPORT unzOpenCurrentFile3 OF((unzFile file,
 */
 
 
-extern int ZEXPORT unzCloseCurrentFile OF((unzFile file));
+extern int ZEXPORT unzCloseCurrentFile(unzFile file);
 /*
   Close the file in zip opened with unzOpenCurrentFile
   Return UNZ_CRCERROR if all the file was read but the CRC is not good
 */
 
-extern int ZEXPORT unzReadCurrentFile OF((unzFile file,
-                      voidp buf,
-                      unsigned len));
+extern int ZEXPORT unzReadCurrentFile(unzFile file,
+                                      voidp buf,
+                                      unsigned len);
 /*
   Read bytes from the current file (opened by unzOpenCurrentFile)
   buf contain buffer where data must be copied
   len the size of buf.
 
-  return the number of byte copied if somes bytes are copied
+  return the number of byte copied if some bytes are copied
   return 0 if the end of file was reached
   return <0 with error code if there is an error
     (UNZ_ERRNO for IO error, or zLib error for uncompress error)
 */
 
-extern z_off_t ZEXPORT unztell OF((unzFile file));
+extern z_off_t ZEXPORT unztell(unzFile file);
 
-extern ZPOS64_T ZEXPORT unztell64 OF((unzFile file));
+extern ZPOS64_T ZEXPORT unztell64(unzFile file);
 /*
   Give the current position in uncompressed data
 */
 
-extern int ZEXPORT unzeof OF((unzFile file));
+extern int ZEXPORT unzeof(unzFile file);
 /*
   return 1 if the end of file was reached, 0 elsewhere
 */
 
-extern int ZEXPORT unzGetLocalExtrafield OF((unzFile file,
-                                             voidp buf,
-                                             unsigned len));
+extern int ZEXPORT unzGetLocalExtrafield(unzFile file,
+                                         voidp buf,
+                                         unsigned len);
 /*
   Read extra field from the current file (opened by unzOpenCurrentFile)
   This is the local-header version of the extra field (sometimes, there is
diff --git a/zlib/contrib/minizip/zip.c b/zlib/contrib/minizip/zip.c
index 44e88a9..60bdffa 100644
--- a/zlib/contrib/minizip/zip.c
+++ b/zlib/contrib/minizip/zip.c
@@ -14,7 +14,7 @@
    Oct-2009 - Mathias Svensson - Added Zip64 Support when creating new file archives
    Oct-2009 - Mathias Svensson - Did some code cleanup and refactoring to get better overview of some functions.
    Oct-2009 - Mathias Svensson - Added zipRemoveExtraInfoBlock to strip extra field data from its ZIP64 data
-                                 It is used when recreting zip archive with RAW when deleting items from a zip.
+                                 It is used when recreating zip archive with RAW when deleting items from a zip.
                                  ZIP64 data is automatically added to items that needs it, and existing ZIP64 data need to be removed.
    Oct-2009 - Mathias Svensson - Added support for BZIP2 as compression mode (bzip2 lib is required)
    Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer
@@ -25,14 +25,13 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <stdint.h>
 #include <time.h>
 #include "zlib.h"
 #include "zip.h"
 
 #ifdef STDC
 #  include <stddef.h>
-#  include <string.h>
-#  include <stdlib.h>
 #endif
 #ifdef NO_ERRNO_H
     extern int errno;
@@ -47,7 +46,7 @@
 /* compile with -Dlocal if your debugger can't find static symbols */
 
 #ifndef VERSIONMADEBY
-# define VERSIONMADEBY   (0x0) /* platform depedent */
+# define VERSIONMADEBY   (0x0) /* platform dependent */
 #endif
 
 #ifndef Z_BUFSIZE
@@ -61,9 +60,6 @@
 #ifndef ALLOC
 # define ALLOC(size) (malloc(size))
 #endif
-#ifndef TRYFREE
-# define TRYFREE(p) {if (p) free(p);}
-#endif
 
 /*
 #define SIZECENTRALDIRITEM (0x2e)
@@ -138,37 +134,37 @@ typedef struct
     uInt pos_in_buffered_data;  /* last written byte in buffered_data */
 
     ZPOS64_T pos_local_header;     /* offset of the local header of the file
-                                     currenty writing */
+                                     currently writing */
     char* central_header;       /* central header data for the current file */
     uLong size_centralExtra;
     uLong size_centralheader;   /* size of the central header for cur file */
     uLong size_centralExtraFree; /* Extra bytes allocated to the centralheader but that are not used */
     uLong flag;                 /* flag of the file currently writing */
 
-    int  method;                /* compression method of file currenty wr.*/
+    int  method;                /* compression method of file currently wr.*/
     int  raw;                   /* 1 for directly writing raw data */
     Byte buffered_data[Z_BUFSIZE];/* buffer contain compressed data to be writ*/
     uLong dosDate;
     uLong crc32;
     int  encrypt;
-    int  zip64;               /* Add ZIP64 extened information in the extra field */
+    int  zip64;               /* Add ZIP64 extended information in the extra field */
     ZPOS64_T pos_zip64extrainfo;
     ZPOS64_T totalCompressedData;
     ZPOS64_T totalUncompressedData;
 #ifndef NOCRYPT
     unsigned long keys[3];     /* keys defining the pseudo-random sequence */
     const z_crc_t* pcrc_32_tab;
-    int crypt_header_size;
+    unsigned crypt_header_size;
 #endif
 } curfile64_info;
 
 typedef struct
 {
     zlib_filefunc64_32_def z_filefunc;
-    voidpf filestream;        /* io structore of the zipfile */
+    voidpf filestream;        /* io structure of the zipfile */
     linkedlist_data central_dir;/* datablock with central dir in construction*/
     int  in_opened_file_inzip;  /* 1 if a file in the zip is currently writ.*/
-    curfile64_info ci;            /* info on the file curretly writing */
+    curfile64_info ci;            /* info on the file currently writing */
 
     ZPOS64_T begin_pos;            /* position of the beginning of the zipfile */
     ZPOS64_T add_position_when_writing_offset;
@@ -186,8 +182,7 @@ typedef struct
 #include "crypt.h"
 #endif
 
-local linkedlist_datablock_internal* allocate_new_datablock()
-{
+local linkedlist_datablock_internal* allocate_new_datablock(void) {
     linkedlist_datablock_internal* ldi;
     ldi = (linkedlist_datablock_internal*)
                  ALLOC(sizeof(linkedlist_datablock_internal));
@@ -200,30 +195,26 @@ local linkedlist_datablock_internal* allocate_new_datablock()
     return ldi;
 }
 
-local void free_datablock(linkedlist_datablock_internal* ldi)
-{
+local void free_datablock(linkedlist_datablock_internal* ldi) {
     while (ldi!=NULL)
     {
         linkedlist_datablock_internal* ldinext = ldi->next_datablock;
-        TRYFREE(ldi);
+        free(ldi);
         ldi = ldinext;
     }
 }
 
-local void init_linkedlist(linkedlist_data* ll)
-{
+local void init_linkedlist(linkedlist_data* ll) {
     ll->first_block = ll->last_block = NULL;
 }
 
-local void free_linkedlist(linkedlist_data* ll)
-{
+local void free_linkedlist(linkedlist_data* ll) {
     free_datablock(ll->first_block);
     ll->first_block = ll->last_block = NULL;
 }
 
 
-local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len)
-{
+local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len) {
     linkedlist_datablock_internal* ldi;
     const unsigned char* from_copy;
 
@@ -238,7 +229,7 @@ local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len)
     }
 
     ldi = ll->last_block;
-    from_copy = (unsigned char*)buf;
+    from_copy = (const unsigned char*)buf;
 
     while (len>0)
     {
@@ -283,9 +274,7 @@ local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len)
    nbByte == 1, 2 ,4 or 8 (byte, short or long, ZPOS64_T)
 */
 
-local int zip64local_putValue OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte));
-local int zip64local_putValue (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte)
-{
+local int zip64local_putValue(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte) {
     unsigned char buf[8];
     int n;
     for (n = 0; n < nbByte; n++)
@@ -301,15 +290,13 @@ local int zip64local_putValue (const zlib_filefunc64_32_def* pzlib_filefunc_def,
         }
       }
 
-    if (ZWRITE64(*pzlib_filefunc_def,filestream,buf,nbByte)!=(uLong)nbByte)
+    if (ZWRITE64(*pzlib_filefunc_def,filestream,buf,(uLong)nbByte)!=(uLong)nbByte)
         return ZIP_ERRNO;
     else
         return ZIP_OK;
 }
 
-local void zip64local_putValue_inmemory OF((void* dest, ZPOS64_T x, int nbByte));
-local void zip64local_putValue_inmemory (void* dest, ZPOS64_T x, int nbByte)
-{
+local void zip64local_putValue_inmemory (void* dest, ZPOS64_T x, int nbByte) {
     unsigned char* buf=(unsigned char*)dest;
     int n;
     for (n = 0; n < nbByte; n++) {
@@ -329,25 +316,21 @@ local void zip64local_putValue_inmemory (void* dest, ZPOS64_T x, int nbByte)
 /****************************************************************************/
 
 
-local uLong zip64local_TmzDateToDosDate(const tm_zip* ptm)
-{
+local uLong zip64local_TmzDateToDosDate(const tm_zip* ptm) {
     uLong year = (uLong)ptm->tm_year;
     if (year>=1980)
         year-=1980;
     else if (year>=80)
         year-=80;
     return
-      (uLong) (((ptm->tm_mday) + (32 * (ptm->tm_mon+1)) + (512 * year)) << 16) |
-        ((ptm->tm_sec/2) + (32* ptm->tm_min) + (2048 * (uLong)ptm->tm_hour));
+      (uLong) (((uLong)(ptm->tm_mday) + (32 * (uLong)(ptm->tm_mon+1)) + (512 * year)) << 16) |
+        (((uLong)ptm->tm_sec/2) + (32 * (uLong)ptm->tm_min) + (2048 * (uLong)ptm->tm_hour));
 }
 
 
 /****************************************************************************/
 
-local int zip64local_getByte OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi));
-
-local int zip64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def,voidpf filestream,int* pi)
-{
+local int zip64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int* pi) {
     unsigned char c;
     int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1);
     if (err==1)
@@ -368,10 +351,7 @@ local int zip64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def,vo
 /* ===========================================================================
    Reads a long in LSB order from the given gz_stream. Sets
 */
-local int zip64local_getShort OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX));
-
-local int zip64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX)
-{
+local int zip64local_getShort(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) {
     uLong x ;
     int i = 0;
     int err;
@@ -390,10 +370,7 @@ local int zip64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def,
     return err;
 }
 
-local int zip64local_getLong OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX));
-
-local int zip64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX)
-{
+local int zip64local_getLong(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) {
     uLong x ;
     int i = 0;
     int err;
@@ -420,11 +397,8 @@ local int zip64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def,
     return err;
 }
 
-local int zip64local_getLong64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX));
 
-
-local int zip64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX)
-{
+local int zip64local_getLong64(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX) {
   ZPOS64_T x;
   int i = 0;
   int err;
@@ -475,10 +449,7 @@ local int zip64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def
   Locate the Central directory of a zipfile (at the end, just before
     the global comment)
 */
-local ZPOS64_T zip64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream));
-
-local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)
-{
+local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) {
   unsigned char* buf;
   ZPOS64_T uSizeFile;
   ZPOS64_T uBackRead;
@@ -522,14 +493,14 @@ local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_f
       if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) &&
         ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06))
       {
-        uPosFound = uReadPos+i;
+        uPosFound = uReadPos+(unsigned)i;
         break;
       }
 
-      if (uPosFound!=0)
-        break;
+    if (uPosFound!=0)
+      break;
   }
-  TRYFREE(buf);
+  free(buf);
   return uPosFound;
 }
 
@@ -537,10 +508,7 @@ local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_f
 Locate the End of Zip64 Central directory locator and from there find the CD of a zipfile (at the end, just before
 the global comment)
 */
-local ZPOS64_T zip64local_SearchCentralDir64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream));
-
-local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)
-{
+local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) {
   unsigned char* buf;
   ZPOS64_T uSizeFile;
   ZPOS64_T uBackRead;
@@ -586,7 +554,7 @@ local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib
       // Signature "0x07064b50" Zip64 end of central directory locater
       if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07))
       {
-        uPosFound = uReadPos+i;
+        uPosFound = uReadPos+(unsigned)i;
         break;
       }
     }
@@ -595,7 +563,7 @@ local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib
         break;
   }
 
-  TRYFREE(buf);
+  free(buf);
   if (uPosFound == 0)
     return 0;
 
@@ -607,7 +575,7 @@ local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib
   if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK)
     return 0;
 
-  /* number of the disk with the start of the zip64 end of  central directory */
+  /* number of the disk with the start of the zip64 end of central directory */
   if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK)
     return 0;
   if (uL != 0)
@@ -637,8 +605,7 @@ local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib
   return relativeOffset;
 }
 
-int LoadCentralDirectoryRecord(zip64_internal* pziinit)
-{
+local int LoadCentralDirectoryRecord(zip64_internal* pziinit) {
   int err=ZIP_OK;
   ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/
 
@@ -647,10 +614,10 @@ int LoadCentralDirectoryRecord(zip64_internal* pziinit)
   ZPOS64_T central_pos;
   uLong uL;
 
-  uLong number_disk;          /* number of the current dist, used for
-                              spaning ZIP, unsupported, always 0*/
-  uLong number_disk_with_CD;  /* number the the disk with central dir, used
-                              for spaning ZIP, unsupported, always 0*/
+  uLong number_disk;          /* number of the current disk, used for
+                              spanning ZIP, unsupported, always 0*/
+  uLong number_disk_with_CD;  /* number of the disk with central dir, used
+                              for spanning ZIP, unsupported, always 0*/
   ZPOS64_T number_entry;
   ZPOS64_T number_entry_CD;      /* total number of entries in
                                 the central dir
@@ -830,7 +797,7 @@ int LoadCentralDirectoryRecord(zip64_internal* pziinit)
 
       size_central_dir_to_read-=read_this;
     }
-    TRYFREE(buf_read);
+    free(buf_read);
   }
   pziinit->begin_pos = byte_before_the_zipfile;
   pziinit->number_entry = number_entry_CD;
@@ -846,8 +813,7 @@ int LoadCentralDirectoryRecord(zip64_internal* pziinit)
 
 
 /************************************************************/
-extern zipFile ZEXPORT zipOpen3 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_32_def* pzlib_filefunc64_32_def)
-{
+extern zipFile ZEXPORT zipOpen3(const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_32_def* pzlib_filefunc64_32_def) {
     zip64_internal ziinit;
     zip64_internal* zi;
     int err=ZIP_OK;
@@ -905,9 +871,9 @@ extern zipFile ZEXPORT zipOpen3 (const void *pathname, int append, zipcharpc* gl
     if (err != ZIP_OK)
     {
 #    ifndef NO_ADDFILEINEXISTINGZIP
-        TRYFREE(ziinit.globalcomment);
+        free(ziinit.globalcomment);
 #    endif /* !NO_ADDFILEINEXISTINGZIP*/
-        TRYFREE(zi);
+        free(zi);
         return NULL;
     }
     else
@@ -917,8 +883,7 @@ extern zipFile ZEXPORT zipOpen3 (const void *pathname, int append, zipcharpc* gl
     }
 }
 
-extern zipFile ZEXPORT zipOpen2 (const char *pathname, int append, zipcharpc* globalcomment, zlib_filefunc_def* pzlib_filefunc32_def)
-{
+extern zipFile ZEXPORT zipOpen2(const char *pathname, int append, zipcharpc* globalcomment, zlib_filefunc_def* pzlib_filefunc32_def) {
     if (pzlib_filefunc32_def != NULL)
     {
         zlib_filefunc64_32_def zlib_filefunc64_32_def_fill;
@@ -929,8 +894,7 @@ extern zipFile ZEXPORT zipOpen2 (const char *pathname, int append, zipcharpc* gl
         return zipOpen3(pathname, append, globalcomment, NULL);
 }
 
-extern zipFile ZEXPORT zipOpen2_64 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_def* pzlib_filefunc_def)
-{
+extern zipFile ZEXPORT zipOpen2_64(const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_def* pzlib_filefunc_def) {
     if (pzlib_filefunc_def != NULL)
     {
         zlib_filefunc64_32_def zlib_filefunc64_32_def_fill;
@@ -945,18 +909,15 @@ extern zipFile ZEXPORT zipOpen2_64 (const void *pathname, int append, zipcharpc*
 
 
 
-extern zipFile ZEXPORT zipOpen (const char* pathname, int append)
-{
+extern zipFile ZEXPORT zipOpen(const char* pathname, int append) {
     return zipOpen3((const void*)pathname,append,NULL,NULL);
 }
 
-extern zipFile ZEXPORT zipOpen64 (const void* pathname, int append)
-{
+extern zipFile ZEXPORT zipOpen64(const void* pathname, int append) {
     return zipOpen3(pathname,append,NULL,NULL);
 }
 
-int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_extrafield_local, const void* extrafield_local)
-{
+local int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_extrafield_local, const void* extrafield_local) {
   /* write the local header */
   int err;
   uInt size_filename = (uInt)strlen(filename);
@@ -1034,8 +995,8 @@ int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_ex
       // Remember position of Zip64 extended info for the local file header. (needed when we update size after done with file)
       zi->ci.pos_zip64extrainfo = ZTELL64(zi->z_filefunc,zi->filestream);
 
-      err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)HeaderID,2);
-      err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)DataSize,2);
+      err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)HeaderID,2);
+      err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)DataSize,2);
 
       err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)UncompressedSize,8);
       err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)CompressedSize,8);
@@ -1052,14 +1013,13 @@ int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_ex
  It is not done here because then we need to realloc a new buffer since parameters are 'const' and I want to minimize
  unnecessary allocations.
  */
-extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename, const zip_fileinfo* zipfi,
-                                         const void* extrafield_local, uInt size_extrafield_local,
-                                         const void* extrafield_global, uInt size_extrafield_global,
-                                         const char* comment, int method, int level, int raw,
-                                         int windowBits,int memLevel, int strategy,
-                                         const char* password, uLong crcForCrypting,
-                                         uLong versionMadeBy, uLong flagBase, int zip64)
-{
+extern int ZEXPORT zipOpenNewFileInZip4_64(zipFile file, const char* filename, const zip_fileinfo* zipfi,
+                                           const void* extrafield_local, uInt size_extrafield_local,
+                                           const void* extrafield_global, uInt size_extrafield_global,
+                                           const char* comment, int method, int level, int raw,
+                                           int windowBits,int memLevel, int strategy,
+                                           const char* password, uLong crcForCrypting,
+                                           uLong versionMadeBy, uLong flagBase, int zip64) {
     zip64_internal* zi;
     uInt size_filename;
     uInt size_comment;
@@ -1083,6 +1043,17 @@ extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename,
       return ZIP_PARAMERROR;
 #endif
 
+    // The filename and comment length must fit in 16 bits.
+    if ((filename!=NULL) && (strlen(filename)>0xffff))
+        return ZIP_PARAMERROR;
+    if ((comment!=NULL) && (strlen(comment)>0xffff))
+        return ZIP_PARAMERROR;
+    // The extra field length must fit in 16 bits. If the member also requires
+    // a Zip64 extra block, that will also need to fit within that 16-bit
+    // length, but that will be checked for later.
+    if ((size_extrafield_local>0xffff) || (size_extrafield_global>0xffff))
+        return ZIP_PARAMERROR;
+
     zi = (zip64_internal*)file;
 
     if (zi->in_opened_file_inzip == 1)
@@ -1262,35 +1233,33 @@ extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename,
     return err;
 }
 
-extern int ZEXPORT zipOpenNewFileInZip4 (zipFile file, const char* filename, const zip_fileinfo* zipfi,
-                                         const void* extrafield_local, uInt size_extrafield_local,
-                                         const void* extrafield_global, uInt size_extrafield_global,
-                                         const char* comment, int method, int level, int raw,
-                                         int windowBits,int memLevel, int strategy,
-                                         const char* password, uLong crcForCrypting,
-                                         uLong versionMadeBy, uLong flagBase)
-{
-    return zipOpenNewFileInZip4_64 (file, filename, zipfi,
-                                 extrafield_local, size_extrafield_local,
-                                 extrafield_global, size_extrafield_global,
-                                 comment, method, level, raw,
-                                 windowBits, memLevel, strategy,
-                                 password, crcForCrypting, versionMadeBy, flagBase, 0);
+extern int ZEXPORT zipOpenNewFileInZip4(zipFile file, const char* filename, const zip_fileinfo* zipfi,
+                                        const void* extrafield_local, uInt size_extrafield_local,
+                                        const void* extrafield_global, uInt size_extrafield_global,
+                                        const char* comment, int method, int level, int raw,
+                                        int windowBits,int memLevel, int strategy,
+                                        const char* password, uLong crcForCrypting,
+                                        uLong versionMadeBy, uLong flagBase) {
+    return zipOpenNewFileInZip4_64(file, filename, zipfi,
+                                   extrafield_local, size_extrafield_local,
+                                   extrafield_global, size_extrafield_global,
+                                   comment, method, level, raw,
+                                   windowBits, memLevel, strategy,
+                                   password, crcForCrypting, versionMadeBy, flagBase, 0);
 }
 
-extern int ZEXPORT zipOpenNewFileInZip3 (zipFile file, const char* filename, const zip_fileinfo* zipfi,
-                                         const void* extrafield_local, uInt size_extrafield_local,
-                                         const void* extrafield_global, uInt size_extrafield_global,
-                                         const char* comment, int method, int level, int raw,
-                                         int windowBits,int memLevel, int strategy,
-                                         const char* password, uLong crcForCrypting)
-{
-    return zipOpenNewFileInZip4_64 (file, filename, zipfi,
-                                 extrafield_local, size_extrafield_local,
-                                 extrafield_global, size_extrafield_global,
-                                 comment, method, level, raw,
-                                 windowBits, memLevel, strategy,
-                                 password, crcForCrypting, VERSIONMADEBY, 0, 0);
+extern int ZEXPORT zipOpenNewFileInZip3(zipFile file, const char* filename, const zip_fileinfo* zipfi,
+                                        const void* extrafield_local, uInt size_extrafield_local,
+                                        const void* extrafield_global, uInt size_extrafield_global,
+                                        const char* comment, int method, int level, int raw,
+                                        int windowBits,int memLevel, int strategy,
+                                        const char* password, uLong crcForCrypting) {
+    return zipOpenNewFileInZip4_64(file, filename, zipfi,
+                                   extrafield_local, size_extrafield_local,
+                                   extrafield_global, size_extrafield_global,
+                                   comment, method, level, raw,
+                                   windowBits, memLevel, strategy,
+                                   password, crcForCrypting, VERSIONMADEBY, 0, 0);
 }
 
 extern int ZEXPORT zipOpenNewFileInZip3_64(zipFile file, const char* filename, const zip_fileinfo* zipfi,
@@ -1298,70 +1267,64 @@ extern int ZEXPORT zipOpenNewFileInZip3_64(zipFile file, const char* filename, c
                                          const void* extrafield_global, uInt size_extrafield_global,
                                          const char* comment, int method, int level, int raw,
                                          int windowBits,int memLevel, int strategy,
-                                         const char* password, uLong crcForCrypting, int zip64)
-{
-    return zipOpenNewFileInZip4_64 (file, filename, zipfi,
-                                 extrafield_local, size_extrafield_local,
-                                 extrafield_global, size_extrafield_global,
-                                 comment, method, level, raw,
-                                 windowBits, memLevel, strategy,
-                                 password, crcForCrypting, VERSIONMADEBY, 0, zip64);
+                                         const char* password, uLong crcForCrypting, int zip64) {
+    return zipOpenNewFileInZip4_64(file, filename, zipfi,
+                                   extrafield_local, size_extrafield_local,
+                                   extrafield_global, size_extrafield_global,
+                                   comment, method, level, raw,
+                                   windowBits, memLevel, strategy,
+                                   password, crcForCrypting, VERSIONMADEBY, 0, zip64);
 }
 
 extern int ZEXPORT zipOpenNewFileInZip2(zipFile file, const char* filename, const zip_fileinfo* zipfi,
                                         const void* extrafield_local, uInt size_extrafield_local,
                                         const void* extrafield_global, uInt size_extrafield_global,
-                                        const char* comment, int method, int level, int raw)
-{
-    return zipOpenNewFileInZip4_64 (file, filename, zipfi,
-                                 extrafield_local, size_extrafield_local,
-                                 extrafield_global, size_extrafield_global,
-                                 comment, method, level, raw,
-                                 -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY,
-                                 NULL, 0, VERSIONMADEBY, 0, 0);
+                                        const char* comment, int method, int level, int raw) {
+    return zipOpenNewFileInZip4_64(file, filename, zipfi,
+                                   extrafield_local, size_extrafield_local,
+                                   extrafield_global, size_extrafield_global,
+                                   comment, method, level, raw,
+                                   -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY,
+                                   NULL, 0, VERSIONMADEBY, 0, 0);
 }
 
 extern int ZEXPORT zipOpenNewFileInZip2_64(zipFile file, const char* filename, const zip_fileinfo* zipfi,
-                                        const void* extrafield_local, uInt size_extrafield_local,
-                                        const void* extrafield_global, uInt size_extrafield_global,
-                                        const char* comment, int method, int level, int raw, int zip64)
-{
-    return zipOpenNewFileInZip4_64 (file, filename, zipfi,
-                                 extrafield_local, size_extrafield_local,
-                                 extrafield_global, size_extrafield_global,
-                                 comment, method, level, raw,
-                                 -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY,
-                                 NULL, 0, VERSIONMADEBY, 0, zip64);
+                                           const void* extrafield_local, uInt size_extrafield_local,
+                                           const void* extrafield_global, uInt size_extrafield_global,
+                                           const char* comment, int method, int level, int raw, int zip64) {
+    return zipOpenNewFileInZip4_64(file, filename, zipfi,
+                                   extrafield_local, size_extrafield_local,
+                                   extrafield_global, size_extrafield_global,
+                                   comment, method, level, raw,
+                                   -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY,
+                                   NULL, 0, VERSIONMADEBY, 0, zip64);
 }
 
-extern int ZEXPORT zipOpenNewFileInZip64 (zipFile file, const char* filename, const zip_fileinfo* zipfi,
-                                        const void* extrafield_local, uInt size_extrafield_local,
-                                        const void*extrafield_global, uInt size_extrafield_global,
-                                        const char* comment, int method, int level, int zip64)
-{
-    return zipOpenNewFileInZip4_64 (file, filename, zipfi,
-                                 extrafield_local, size_extrafield_local,
-                                 extrafield_global, size_extrafield_global,
-                                 comment, method, level, 0,
-                                 -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY,
-                                 NULL, 0, VERSIONMADEBY, 0, zip64);
+extern int ZEXPORT zipOpenNewFileInZip64(zipFile file, const char* filename, const zip_fileinfo* zipfi,
+                                         const void* extrafield_local, uInt size_extrafield_local,
+                                         const void*extrafield_global, uInt size_extrafield_global,
+                                         const char* comment, int method, int level, int zip64) {
+    return zipOpenNewFileInZip4_64(file, filename, zipfi,
+                                   extrafield_local, size_extrafield_local,
+                                   extrafield_global, size_extrafield_global,
+                                   comment, method, level, 0,
+                                   -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY,
+                                   NULL, 0, VERSIONMADEBY, 0, zip64);
 }
 
-extern int ZEXPORT zipOpenNewFileInZip (zipFile file, const char* filename, const zip_fileinfo* zipfi,
-                                        const void* extrafield_local, uInt size_extrafield_local,
-                                        const void*extrafield_global, uInt size_extrafield_global,
-                                        const char* comment, int method, int level)
-{
-    return zipOpenNewFileInZip4_64 (file, filename, zipfi,
-                                 extrafield_local, size_extrafield_local,
-                                 extrafield_global, size_extrafield_global,
-                                 comment, method, level, 0,
-                                 -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY,
-                                 NULL, 0, VERSIONMADEBY, 0, 0);
+extern int ZEXPORT zipOpenNewFileInZip(zipFile file, const char* filename, const zip_fileinfo* zipfi,
+                                       const void* extrafield_local, uInt size_extrafield_local,
+                                       const void*extrafield_global, uInt size_extrafield_global,
+                                       const char* comment, int method, int level) {
+    return zipOpenNewFileInZip4_64(file, filename, zipfi,
+                                   extrafield_local, size_extrafield_local,
+                                   extrafield_global, size_extrafield_global,
+                                   comment, method, level, 0,
+                                   -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY,
+                                   NULL, 0, VERSIONMADEBY, 0, 0);
 }
 
-local int zip64FlushWriteBuffer(zip64_internal* zi)
-{
+local int zip64FlushWriteBuffer(zip64_internal* zi) {
     int err=ZIP_OK;
 
     if (zi->ci.encrypt != 0)
@@ -1399,8 +1362,7 @@ local int zip64FlushWriteBuffer(zip64_internal* zi)
     return err;
 }
 
-extern int ZEXPORT zipWriteInFileInZip (zipFile file,const void* buf,unsigned int len)
-{
+extern int ZEXPORT zipWriteInFileInZip(zipFile file, const void* buf, unsigned int len) {
     zip64_internal* zi;
     int err=ZIP_OK;
 
@@ -1450,7 +1412,7 @@ extern int ZEXPORT zipWriteInFileInZip (zipFile file,const void* buf,unsigned in
     else
 #endif
     {
-      zi->ci.stream.next_in = (Bytef*)buf;
+      zi->ci.stream.next_in = (Bytef*)(uintptr_t)buf;
       zi->ci.stream.avail_in = len;
 
       while ((err==ZIP_OK) && (zi->ci.stream.avail_in>0))
@@ -1471,11 +1433,6 @@ extern int ZEXPORT zipWriteInFileInZip (zipFile file,const void* buf,unsigned in
           {
               uLong uTotalOutBefore = zi->ci.stream.total_out;
               err=deflate(&zi->ci.stream,  Z_NO_FLUSH);
-              if(uTotalOutBefore > zi->ci.stream.total_out)
-              {
-                int bBreak = 0;
-                bBreak++;
-              }
 
               zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ;
           }
@@ -1506,17 +1463,15 @@ extern int ZEXPORT zipWriteInFileInZip (zipFile file,const void* buf,unsigned in
     return err;
 }
 
-extern int ZEXPORT zipCloseFileInZipRaw (zipFile file, uLong uncompressed_size, uLong crc32)
-{
+extern int ZEXPORT zipCloseFileInZipRaw(zipFile file, uLong uncompressed_size, uLong crc32) {
     return zipCloseFileInZipRaw64 (file, uncompressed_size, crc32);
 }
 
-extern int ZEXPORT zipCloseFileInZipRaw64 (zipFile file, ZPOS64_T uncompressed_size, uLong crc32)
-{
+extern int ZEXPORT zipCloseFileInZipRaw64(zipFile file, ZPOS64_T uncompressed_size, uLong crc32) {
     zip64_internal* zi;
     ZPOS64_T compressed_size;
     uLong invalidValue = 0xffffffff;
-    short datasize = 0;
+    unsigned datasize = 0;
     int err=ZIP_OK;
 
     if (file == NULL)
@@ -1653,7 +1608,7 @@ extern int ZEXPORT zipCloseFileInZipRaw64 (zipFile file, ZPOS64_T uncompressed_s
 
       if((uLong)(datasize + 4) > zi->ci.size_centralExtraFree)
       {
-        // we can not write more data to the buffer that we have room for.
+        // we cannot write more data to the buffer that we have room for.
         return ZIP_BADZIPFILE;
       }
 
@@ -1747,13 +1702,11 @@ extern int ZEXPORT zipCloseFileInZipRaw64 (zipFile file, ZPOS64_T uncompressed_s
     return err;
 }
 
-extern int ZEXPORT zipCloseFileInZip (zipFile file)
-{
+extern int ZEXPORT zipCloseFileInZip(zipFile file) {
     return zipCloseFileInZipRaw (file,0,0);
 }
 
-int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eocd_pos_inzip)
-{
+local int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eocd_pos_inzip) {
   int err = ZIP_OK;
   ZPOS64_T pos = zip64eocd_pos_inzip - zi->add_position_when_writing_offset;
 
@@ -1774,8 +1727,7 @@ int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eo
     return err;
 }
 
-int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip)
-{
+local int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) {
   int err = ZIP_OK;
 
   uLong Zip64DataSize = 44;
@@ -1813,8 +1765,8 @@ int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centra
   }
   return err;
 }
-int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip)
-{
+
+local int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) {
   int err = ZIP_OK;
 
   /*signature*/
@@ -1861,8 +1813,7 @@ int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir,
    return err;
 }
 
-int Write_GlobalComment(zip64_internal* zi, const char* global_comment)
-{
+local int Write_GlobalComment(zip64_internal* zi, const char* global_comment) {
   int err = ZIP_OK;
   uInt size_global_comment = 0;
 
@@ -1879,8 +1830,7 @@ int Write_GlobalComment(zip64_internal* zi, const char* global_comment)
   return err;
 }
 
-extern int ZEXPORT zipClose (zipFile file, const char* global_comment)
-{
+extern int ZEXPORT zipClose(zipFile file, const char* global_comment) {
     zip64_internal* zi;
     int err = 0;
     uLong size_centraldir = 0;
@@ -1922,7 +1872,7 @@ extern int ZEXPORT zipClose (zipFile file, const char* global_comment)
     free_linkedlist(&(zi->central_dir));
 
     pos = centraldir_pos_inzip - zi->add_position_when_writing_offset;
-    if(pos >= 0xffffffff || zi->number_entry > 0xFFFF)
+    if(pos >= 0xffffffff || zi->number_entry >= 0xFFFF)
     {
       ZPOS64_T Zip64EOCDpos = ZTELL64(zi->z_filefunc,zi->filestream);
       Write_Zip64EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip);
@@ -1941,15 +1891,14 @@ extern int ZEXPORT zipClose (zipFile file, const char* global_comment)
             err = ZIP_ERRNO;
 
 #ifndef NO_ADDFILEINEXISTINGZIP
-    TRYFREE(zi->globalcomment);
+    free(zi->globalcomment);
 #endif
-    TRYFREE(zi);
+    free(zi);
 
     return err;
 }
 
-extern int ZEXPORT zipRemoveExtraInfoBlock (char* pData, int* dataLen, short sHeader)
-{
+extern int ZEXPORT zipRemoveExtraInfoBlock(char* pData, int* dataLen, short sHeader) {
   char* p = pData;
   int size = 0;
   char* pNewHeader;
@@ -1959,10 +1908,10 @@ extern int ZEXPORT zipRemoveExtraInfoBlock (char* pData, int* dataLen, short sHe
 
   int retVal = ZIP_OK;
 
-  if(pData == NULL || *dataLen < 4)
+  if(pData == NULL || dataLen == NULL || *dataLen < 4)
     return ZIP_PARAMERROR;
 
-  pNewHeader = (char*)ALLOC(*dataLen);
+  pNewHeader = (char*)ALLOC((unsigned)*dataLen);
   pTmp = pNewHeader;
 
   while(p < (pData + *dataLen))
@@ -2001,7 +1950,7 @@ extern int ZEXPORT zipRemoveExtraInfoBlock (char* pData, int* dataLen, short sHe
   else
     retVal = ZIP_ERRNO;
 
-  TRYFREE(pNewHeader);
+  free(pNewHeader);
 
   return retVal;
 }
diff --git a/zlib/contrib/minizip/zip.h b/zlib/contrib/minizip/zip.h
index 8aaebb6..3e230d3 100644
--- a/zlib/contrib/minizip/zip.h
+++ b/zlib/contrib/minizip/zip.h
@@ -88,12 +88,12 @@ typedef voidp zipFile;
 /* tm_zip contain date/time info */
 typedef struct tm_zip_s
 {
-    uInt tm_sec;            /* seconds after the minute - [0,59] */
-    uInt tm_min;            /* minutes after the hour - [0,59] */
-    uInt tm_hour;           /* hours since midnight - [0,23] */
-    uInt tm_mday;           /* day of the month - [1,31] */
-    uInt tm_mon;            /* months since January - [0,11] */
-    uInt tm_year;           /* years - [1980..2044] */
+    int tm_sec;             /* seconds after the minute - [0,59] */
+    int tm_min;             /* minutes after the hour - [0,59] */
+    int tm_hour;            /* hours since midnight - [0,23] */
+    int tm_mday;            /* day of the month - [1,31] */
+    int tm_mon;             /* months since January - [0,11] */
+    int tm_year;            /* years - [1980..2044] */
 } tm_zip;
 
 typedef struct
@@ -113,8 +113,8 @@ typedef const char* zipcharpc;
 #define APPEND_STATUS_CREATEAFTER   (1)
 #define APPEND_STATUS_ADDINZIP      (2)
 
-extern zipFile ZEXPORT zipOpen OF((const char *pathname, int append));
-extern zipFile ZEXPORT zipOpen64 OF((const void *pathname, int append));
+extern zipFile ZEXPORT zipOpen(const char *pathname, int append);
+extern zipFile ZEXPORT zipOpen64(const void *pathname, int append);
 /*
   Create a zipfile.
      pathname contain on Windows XP a filename like "c:\\zlib\\zlib113.zip" or on
@@ -131,50 +131,55 @@ extern zipFile ZEXPORT zipOpen64 OF((const void *pathname, int append));
 
 /* Note : there is no delete function into a zipfile.
    If you want delete file into a zipfile, you must open a zipfile, and create another
-   Of couse, you can use RAW reading and writing to copy the file you did not want delte
+   Of course, you can use RAW reading and writing to copy the file you did not want delete
 */
 
-extern zipFile ZEXPORT zipOpen2 OF((const char *pathname,
-                                   int append,
-                                   zipcharpc* globalcomment,
-                                   zlib_filefunc_def* pzlib_filefunc_def));
+extern zipFile ZEXPORT zipOpen2(const char *pathname,
+                                int append,
+                                zipcharpc* globalcomment,
+                                zlib_filefunc_def* pzlib_filefunc_def);
 
-extern zipFile ZEXPORT zipOpen2_64 OF((const void *pathname,
+extern zipFile ZEXPORT zipOpen2_64(const void *pathname,
                                    int append,
                                    zipcharpc* globalcomment,
-                                   zlib_filefunc64_def* pzlib_filefunc_def));
-
-extern int ZEXPORT zipOpenNewFileInZip OF((zipFile file,
-                       const char* filename,
-                       const zip_fileinfo* zipfi,
-                       const void* extrafield_local,
-                       uInt size_extrafield_local,
-                       const void* extrafield_global,
-                       uInt size_extrafield_global,
-                       const char* comment,
-                       int method,
-                       int level));
-
-extern int ZEXPORT zipOpenNewFileInZip64 OF((zipFile file,
-                       const char* filename,
-                       const zip_fileinfo* zipfi,
-                       const void* extrafield_local,
-                       uInt size_extrafield_local,
-                       const void* extrafield_global,
-                       uInt size_extrafield_global,
-                       const char* comment,
-                       int method,
-                       int level,
-                       int zip64));
+                                   zlib_filefunc64_def* pzlib_filefunc_def);
+
+extern zipFile ZEXPORT zipOpen3(const void *pathname,
+                                int append,
+                                zipcharpc* globalcomment,
+                                zlib_filefunc64_32_def* pzlib_filefunc64_32_def);
+
+extern int ZEXPORT zipOpenNewFileInZip(zipFile file,
+                                       const char* filename,
+                                       const zip_fileinfo* zipfi,
+                                       const void* extrafield_local,
+                                       uInt size_extrafield_local,
+                                       const void* extrafield_global,
+                                       uInt size_extrafield_global,
+                                       const char* comment,
+                                       int method,
+                                       int level);
+
+extern int ZEXPORT zipOpenNewFileInZip64(zipFile file,
+                                         const char* filename,
+                                         const zip_fileinfo* zipfi,
+                                         const void* extrafield_local,
+                                         uInt size_extrafield_local,
+                                         const void* extrafield_global,
+                                         uInt size_extrafield_global,
+                                         const char* comment,
+                                         int method,
+                                         int level,
+                                         int zip64);
 
 /*
   Open a file in the ZIP for writing.
   filename : the filename in zip (if NULL, '-' without quote will be used
   *zipfi contain supplemental information
   if extrafield_local!=NULL and size_extrafield_local>0, extrafield_local
-    contains the extrafield data the the local header
+    contains the extrafield data for the local header
   if extrafield_global!=NULL and size_extrafield_global>0, extrafield_global
-    contains the extrafield data the the local header
+    contains the extrafield data for the global header
   if comment != NULL, comment contain the comment string
   method contain the compression method (0 for store, Z_DEFLATED for deflate)
   level contain the level of compression (can be Z_DEFAULT_COMPRESSION)
@@ -184,70 +189,69 @@ extern int ZEXPORT zipOpenNewFileInZip64 OF((zipFile file,
 */
 
 
-extern int ZEXPORT zipOpenNewFileInZip2 OF((zipFile file,
-                                            const char* filename,
-                                            const zip_fileinfo* zipfi,
-                                            const void* extrafield_local,
-                                            uInt size_extrafield_local,
-                                            const void* extrafield_global,
-                                            uInt size_extrafield_global,
-                                            const char* comment,
-                                            int method,
-                                            int level,
-                                            int raw));
-
-
-extern int ZEXPORT zipOpenNewFileInZip2_64 OF((zipFile file,
-                                            const char* filename,
-                                            const zip_fileinfo* zipfi,
-                                            const void* extrafield_local,
-                                            uInt size_extrafield_local,
-                                            const void* extrafield_global,
-                                            uInt size_extrafield_global,
-                                            const char* comment,
-                                            int method,
-                                            int level,
-                                            int raw,
-                                            int zip64));
+extern int ZEXPORT zipOpenNewFileInZip2(zipFile file,
+                                        const char* filename,
+                                        const zip_fileinfo* zipfi,
+                                        const void* extrafield_local,
+                                        uInt size_extrafield_local,
+                                        const void* extrafield_global,
+                                        uInt size_extrafield_global,
+                                        const char* comment,
+                                        int method,
+                                        int level,
+                                        int raw);
+
+
+extern int ZEXPORT zipOpenNewFileInZip2_64(zipFile file,
+                                           const char* filename,
+                                           const zip_fileinfo* zipfi,
+                                           const void* extrafield_local,
+                                           uInt size_extrafield_local,
+                                           const void* extrafield_global,
+                                           uInt size_extrafield_global,
+                                           const char* comment,
+                                           int method,
+                                           int level,
+                                           int raw,
+                                           int zip64);
 /*
   Same than zipOpenNewFileInZip, except if raw=1, we write raw file
  */
 
-extern int ZEXPORT zipOpenNewFileInZip3 OF((zipFile file,
-                                            const char* filename,
-                                            const zip_fileinfo* zipfi,
-                                            const void* extrafield_local,
-                                            uInt size_extrafield_local,
-                                            const void* extrafield_global,
-                                            uInt size_extrafield_global,
-                                            const char* comment,
-                                            int method,
-                                            int level,
-                                            int raw,
-                                            int windowBits,
-                                            int memLevel,
-                                            int strategy,
-                                            const char* password,
-                                            uLong crcForCrypting));
-
-extern int ZEXPORT zipOpenNewFileInZip3_64 OF((zipFile file,
-                                            const char* filename,
-                                            const zip_fileinfo* zipfi,
-                                            const void* extrafield_local,
-                                            uInt size_extrafield_local,
-                                            const void* extrafield_global,
-                                            uInt size_extrafield_global,
-                                            const char* comment,
-                                            int method,
-                                            int level,
-                                            int raw,
-                                            int windowBits,
-                                            int memLevel,
-                                            int strategy,
-                                            const char* password,
-                                            uLong crcForCrypting,
-                                            int zip64
-                                            ));
+extern int ZEXPORT zipOpenNewFileInZip3(zipFile file,
+                                        const char* filename,
+                                        const zip_fileinfo* zipfi,
+                                        const void* extrafield_local,
+                                        uInt size_extrafield_local,
+                                        const void* extrafield_global,
+                                        uInt size_extrafield_global,
+                                        const char* comment,
+                                        int method,
+                                        int level,
+                                        int raw,
+                                        int windowBits,
+                                        int memLevel,
+                                        int strategy,
+                                        const char* password,
+                                        uLong crcForCrypting);
+
+extern int ZEXPORT zipOpenNewFileInZip3_64(zipFile file,
+                                           const char* filename,
+                                           const zip_fileinfo* zipfi,
+                                           const void* extrafield_local,
+                                           uInt size_extrafield_local,
+                                           const void* extrafield_global,
+                                           uInt size_extrafield_global,
+                                           const char* comment,
+                                           int method,
+                                           int level,
+                                           int raw,
+                                           int windowBits,
+                                           int memLevel,
+                                           int strategy,
+                                           const char* password,
+                                           uLong crcForCrypting,
+                                           int zip64);
 
 /*
   Same than zipOpenNewFileInZip2, except
@@ -256,47 +260,45 @@ extern int ZEXPORT zipOpenNewFileInZip3_64 OF((zipFile file,
     crcForCrypting : crc of file to compress (needed for crypting)
  */
 
-extern int ZEXPORT zipOpenNewFileInZip4 OF((zipFile file,
-                                            const char* filename,
-                                            const zip_fileinfo* zipfi,
-                                            const void* extrafield_local,
-                                            uInt size_extrafield_local,
-                                            const void* extrafield_global,
-                                            uInt size_extrafield_global,
-                                            const char* comment,
-                                            int method,
-                                            int level,
-                                            int raw,
-                                            int windowBits,
-                                            int memLevel,
-                                            int strategy,
-                                            const char* password,
-                                            uLong crcForCrypting,
-                                            uLong versionMadeBy,
-                                            uLong flagBase
-                                            ));
-
-
-extern int ZEXPORT zipOpenNewFileInZip4_64 OF((zipFile file,
-                                            const char* filename,
-                                            const zip_fileinfo* zipfi,
-                                            const void* extrafield_local,
-                                            uInt size_extrafield_local,
-                                            const void* extrafield_global,
-                                            uInt size_extrafield_global,
-                                            const char* comment,
-                                            int method,
-                                            int level,
-                                            int raw,
-                                            int windowBits,
-                                            int memLevel,
-                                            int strategy,
-                                            const char* password,
-                                            uLong crcForCrypting,
-                                            uLong versionMadeBy,
-                                            uLong flagBase,
-                                            int zip64
-                                            ));
+extern int ZEXPORT zipOpenNewFileInZip4(zipFile file,
+                                        const char* filename,
+                                        const zip_fileinfo* zipfi,
+                                        const void* extrafield_local,
+                                        uInt size_extrafield_local,
+                                        const void* extrafield_global,
+                                        uInt size_extrafield_global,
+                                        const char* comment,
+                                        int method,
+                                        int level,
+                                        int raw,
+                                        int windowBits,
+                                        int memLevel,
+                                        int strategy,
+                                        const char* password,
+                                        uLong crcForCrypting,
+                                        uLong versionMadeBy,
+                                        uLong flagBase);
+
+
+extern int ZEXPORT zipOpenNewFileInZip4_64(zipFile file,
+                                           const char* filename,
+                                           const zip_fileinfo* zipfi,
+                                           const void* extrafield_local,
+                                           uInt size_extrafield_local,
+                                           const void* extrafield_global,
+                                           uInt size_extrafield_global,
+                                           const char* comment,
+                                           int method,
+                                           int level,
+                                           int raw,
+                                           int windowBits,
+                                           int memLevel,
+                                           int strategy,
+                                           const char* password,
+                                           uLong crcForCrypting,
+                                           uLong versionMadeBy,
+                                           uLong flagBase,
+                                           int zip64);
 /*
   Same than zipOpenNewFileInZip4, except
     versionMadeBy : value for Version made by field
@@ -304,25 +306,25 @@ extern int ZEXPORT zipOpenNewFileInZip4_64 OF((zipFile file,
  */
 
 
-extern int ZEXPORT zipWriteInFileInZip OF((zipFile file,
-                       const void* buf,
-                       unsigned len));
+extern int ZEXPORT zipWriteInFileInZip(zipFile file,
+                                       const void* buf,
+                                       unsigned len);
 /*
   Write data in the zipfile
 */
 
-extern int ZEXPORT zipCloseFileInZip OF((zipFile file));
+extern int ZEXPORT zipCloseFileInZip(zipFile file);
 /*
   Close the current file in the zipfile
 */
 
-extern int ZEXPORT zipCloseFileInZipRaw OF((zipFile file,
-                                            uLong uncompressed_size,
-                                            uLong crc32));
+extern int ZEXPORT zipCloseFileInZipRaw(zipFile file,
+                                        uLong uncompressed_size,
+                                        uLong crc32);
 
-extern int ZEXPORT zipCloseFileInZipRaw64 OF((zipFile file,
-                                            ZPOS64_T uncompressed_size,
-                                            uLong crc32));
+extern int ZEXPORT zipCloseFileInZipRaw64(zipFile file,
+                                          ZPOS64_T uncompressed_size,
+                                          uLong crc32);
 
 /*
   Close the current file in the zipfile, for file opened with
@@ -330,14 +332,14 @@ extern int ZEXPORT zipCloseFileInZipRaw64 OF((zipFile file,
   uncompressed_size and crc32 are value for the uncompressed size
 */
 
-extern int ZEXPORT zipClose OF((zipFile file,
-                const char* global_comment));
+extern int ZEXPORT zipClose(zipFile file,
+                            const char* global_comment);
 /*
   Close the zipfile
 */
 
 
-extern int ZEXPORT zipRemoveExtraInfoBlock OF((char* pData, int* dataLen, short sHeader));
+extern int ZEXPORT zipRemoveExtraInfoBlock(char* pData, int* dataLen, short sHeader);
 /*
   zipRemoveExtraInfoBlock -  Added by Mathias Svensson
 
diff --git a/zlib/contrib/nuget/nuget.csproj b/zlib/contrib/nuget/nuget.csproj
new file mode 100644
index 0000000..68627f0
--- /dev/null
+++ b/zlib/contrib/nuget/nuget.csproj
@@ -0,0 +1,43 @@
+<Project Sdk="Microsoft.Build.NoTargets/3.5.6">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <PackageId>madler.zlib.redist</PackageId>
+    <PackageId Condition="$([MSBuild]::IsOSPlatform('Windows'))">$(PackageId).win</PackageId>
+    <PackageId Condition="$([MSBuild]::IsOSPlatform('Linux'))">$(PackageId).linux</PackageId>
+    <PackageId Condition="$([MSBuild]::IsOSPlatform('OSX'))">$(PackageId).osx</PackageId>
+    <Copyright>(C) 1995-2024 Jean-loup Gailly and Mark Adler</Copyright>
+    <version>1.3.1</version>
+    <PackageDescription>NuGet Package for consuming native builds of zlib into .NET without complexity.</PackageDescription>
+    <!--
+        Warns about not having any lib or ref assemblies (.NET Assemblies) in those directories.
+        Native only packages that is to be consumed in .NET should not require these.
+    -->
+    <NoWarn>NU5128</NoWarn>
+    <PackageOutputPath>$(MSBuildProjectDirectory)</PackageOutputPath>
+    <Authors>Jean-loup Gailly and Mark Adler</Authors>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Include="../../LICENSE" Pack="true" PackagePath="" />
+    <!-- Package up Windows builds. -->
+    <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="../vstudio/vc17/x86/ZlibDll$(Configuration)/zlibwapi.dll" Pack="true" PackagePath="runtimes/win-x86/native" />
+    <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="../vstudio/vc17/x64/ZlibDll$(Configuration)/zlibwapi.dll" Pack="true" PackagePath="runtimes/win-x64/native" />
+    <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="../vstudio/vc17/arm/ZlibDll$(Configuration)/zlibwapi.dll" Pack="true" PackagePath="runtimes/win-arm/native" />
+    <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="../vstudio/vc17/arm64/ZlibDll$(Configuration)/zlibwapi.dll" Pack="true" PackagePath="runtimes/win-arm64/native" />
+    <!-- Include debug symbols as well as we never know if they might actually be needed in the future. -->
+    <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="../vstudio/vc17/x86/ZlibDll$(Configuration)/zlibwapi.pdb" Pack="true" PackagePath="runtimes/win-x86/native" />
+    <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="../vstudio/vc17/x64/ZlibDll$(Configuration)/zlibwapi.pdb" Pack="true" PackagePath="runtimes/win-x64/native" />
+    <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="../vstudio/vc17/arm/ZlibDll$(Configuration)/zlibwapi.pdb" Pack="true" PackagePath="runtimes/win-arm/native" />
+    <None Condition="$([MSBuild]::IsOSPlatform('Windows'))" Include="../vstudio/vc17/arm64/ZlibDll$(Configuration)/zlibwapi.pdb" Pack="true" PackagePath="runtimes/win-arm64/native" />
+    <!-- Package up Linux builds. -->
+    <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="./linux-x86/libz.so" Pack="true" PackagePath="runtimes/linux-x86/native" />
+    <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="./linux-x64/libz.so" Pack="true" PackagePath="runtimes/linux-x64/native" />
+    <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="./linux-arm/libz.so" Pack="true" PackagePath="runtimes/linux-arm/native" />
+    <None Condition="$([MSBuild]::IsOSPlatform('Linux'))" Include="./linux-arm64/libz.so" Pack="true" PackagePath="runtimes/linux-arm64/native" />
+    <!-- Package up MacOS builds. -->
+    <None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="./osx-x64/libz.dylib" Pack="true" PackagePath="runtimes/osx-x64/native" />
+    <None Condition="$([MSBuild]::IsOSPlatform('OSX'))" Include="./osx-arm64/libz.dylib" Pack="true" PackagePath="runtimes/osx-arm64/native" />
+  </ItemGroup>
+
+</Project>
diff --git a/zlib/contrib/nuget/nuget.sln b/zlib/contrib/nuget/nuget.sln
new file mode 100644
index 0000000..46ee8de
--- /dev/null
+++ b/zlib/contrib/nuget/nuget.sln
@@ -0,0 +1,22 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.0.31903.59
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "nuget", "nuget.csproj", "{B1BD3984-EF8F-4E9D-9A94-EB784E5EB1E8}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{B1BD3984-EF8F-4E9D-9A94-EB784E5EB1E8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B1BD3984-EF8F-4E9D-9A94-EB784E5EB1E8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B1BD3984-EF8F-4E9D-9A94-EB784E5EB1E8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B1BD3984-EF8F-4E9D-9A94-EB784E5EB1E8}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+EndGlobal
diff --git a/zlib/contrib/pascal/zlibpas.pas b/zlib/contrib/pascal/zlibpas.pas
index a0dff11..0cf0e7b 100644
--- a/zlib/contrib/pascal/zlibpas.pas
+++ b/zlib/contrib/pascal/zlibpas.pas
@@ -10,7 +10,7 @@ unit zlibpas;
 interface
 
 const
-  ZLIB_VERSION = '1.2.11';
+  ZLIB_VERSION = '1.3.1';
   ZLIB_VERNUM  = $12a0;
 
 type
diff --git a/zlib/contrib/puff/README b/zlib/contrib/puff/README
index bbc4cb5..d8192c7 100644
--- a/zlib/contrib/puff/README
+++ b/zlib/contrib/puff/README
@@ -38,7 +38,7 @@ Then you can call puff() to decompress a deflate stream that is in memory in
 its entirety at source, to a sufficiently sized block of memory for the
 decompressed data at dest.  puff() is the only external symbol in puff.c  The
 only C library functions that puff.c needs are setjmp() and longjmp(), which
-are used to simplify error checking in the code to improve readabilty.  puff.c
+are used to simplify error checking in the code to improve readability.  puff.c
 does no memory allocation, and uses less than 2K bytes off of the stack.
 
 If destlen is not enough space for the uncompressed data, then inflate will
diff --git a/zlib/contrib/puff/puff.c b/zlib/contrib/puff/puff.c
index c6c90d7..d759825 100644
--- a/zlib/contrib/puff/puff.c
+++ b/zlib/contrib/puff/puff.c
@@ -43,7 +43,7 @@
  *                      - Use pointers instead of long to specify source and
  *                        destination sizes to avoid arbitrary 4 GB limits
  * 1.2  17 Mar 2002     - Add faster version of decode(), doubles speed (!),
- *                        but leave simple version for readabilty
+ *                        but leave simple version for readability
  *                      - Make sure invalid distances detected if pointers
  *                        are 16 bits
  *                      - Fix fixed codes table error
@@ -593,10 +593,10 @@ local int fixed(struct state *s)
  *   provided for each of the literal/length symbols, and for each of the
  *   distance symbols.
  *
- * - If a symbol is not used in the block, this is represented by a zero as
- *   as the code length.  This does not mean a zero-length code, but rather
- *   that no code should be created for this symbol.  There is no way in the
- *   deflate format to represent a zero-length code.
+ * - If a symbol is not used in the block, this is represented by a zero as the
+ *   code length.  This does not mean a zero-length code, but rather that no
+ *   code should be created for this symbol.  There is no way in the deflate
+ *   format to represent a zero-length code.
  *
  * - The maximum number of bits in a code is 15, so the possible lengths for
  *   any code are 1..15.
@@ -624,7 +624,7 @@ local int fixed(struct state *s)
  *   are themselves compressed using Huffman codes and run-length encoding.  In
  *   the list of code lengths, a 0 symbol means no code, a 1..15 symbol means
  *   that length, and the symbols 16, 17, and 18 are run-length instructions.
- *   Each of 16, 17, and 18 are follwed by extra bits to define the length of
+ *   Each of 16, 17, and 18 are followed by extra bits to define the length of
  *   the run.  16 copies the last length 3 to 6 times.  17 represents 3 to 10
  *   zero lengths, and 18 represents 11 to 138 zero lengths.  Unused symbols
  *   are common, hence the special coding for zero lengths.
diff --git a/zlib/contrib/puff/pufftest.c b/zlib/contrib/puff/pufftest.c
index 7764814..5f72ecc 100644
--- a/zlib/contrib/puff/pufftest.c
+++ b/zlib/contrib/puff/pufftest.c
@@ -143,7 +143,7 @@ int main(int argc, char **argv)
                                      len - sourcelen);
     }
 
-    /* if requested, inflate again and write decompressd data to stdout */
+    /* if requested, inflate again and write decompressed data to stdout */
     if (put && ret == 0) {
         if (fail)
             destlen >>= 1;
diff --git a/zlib/contrib/puff/zeros.raw b/zlib/contrib/puff/zeros.raw
index 637b7be..0a90e76 100644
--- a/zlib/contrib/puff/zeros.raw
+++ b/zlib/contrib/puff/zeros.raw
diff --git a/zlib/contrib/testzlib/testzlib.c b/zlib/contrib/testzlib/testzlib.c
index 5f659de..b3c0014 100644
--- a/zlib/contrib/testzlib/testzlib.c
+++ b/zlib/contrib/testzlib/testzlib.c
@@ -169,7 +169,7 @@ int main(int argc, char *argv[])
         printf("error reading %s\n",argv[1]);

         return 1;

     }

-    else printf("file %s read, %u bytes\n",argv[1],lFileSize);

+    else printf("file %s read, %ld bytes\n",argv[1],lFileSize);

 

     if (argc>=3)

         BlockSizeCompress=atol(argv[2]);

diff --git a/zlib/contrib/untgz/untgz.c b/zlib/contrib/untgz/untgz.c
index 2c391e5..7857921 100644
--- a/zlib/contrib/untgz/untgz.c
+++ b/zlib/contrib/untgz/untgz.c
@@ -4,6 +4,22 @@
  * written by Pedro A. Aranda Gutierrez <paag@tid.es>
  * adaptation to Unix by Jean-loup Gailly <jloup@gzip.org>
  * various fixes by Cosmin Truta <cosmint@cs.ubbcluj.ro>
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty.  In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ *    claim that you wrote the original software. If you use this software
+ *    in a product, an acknowledgment in the product documentation would be
+ *    appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ *    misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
  */
 
 #include <stdio.h>
@@ -14,15 +30,10 @@
 
 #include "zlib.h"
 
-#ifdef unix
-#  include <unistd.h>
-#else
+#ifdef _WIN32
 #  include <direct.h>
 #  include <io.h>
-#endif
-
-#ifdef WIN32
-#include <windows.h>
+#  include <windows.h>
 #  ifndef F_OK
 #    define F_OK  0
 #  endif
@@ -33,6 +44,8 @@
 #    define strdup(str)         _strdup(str)
 #  endif
 #else
+#  include <sys/stat.h>
+#  include <unistd.h>
 #  include <utime.h>
 #endif
 
@@ -102,28 +115,14 @@ struct attr_item
 
 enum { TGZ_EXTRACT, TGZ_LIST, TGZ_INVALID };
 
-char *TGZfname          OF((const char *));
-void TGZnotfound        OF((const char *));
-
-int getoct              OF((char *, int));
-char *strtime           OF((time_t *));
-int setfiletime         OF((char *, time_t));
-void push_attr          OF((struct attr_item **, char *, int, time_t));
-void restore_attr       OF((struct attr_item **));
-
-int ExprMatch           OF((char *, char *));
-
-int makedir             OF((char *));
-int matchname           OF((int, int, char **, char *));
-
-void error              OF((const char *));
-int tar                 OF((gzFile, int, int, int, char **));
-
-void help               OF((int));
-int main                OF((int, char **));
-
 char *prog;
 
+void error(const char *msg)
+{
+  fprintf(stderr, "%s: %s\n", prog, msg);
+  exit(1);
+}
+
 const char *TGZsuffix[] = { "\0", ".tar", ".tar.gz", ".taz", ".tgz", NULL };
 
 /* return the file name of the TGZ archive */
@@ -205,7 +204,7 @@ char *strtime (time_t *t)
 
 int setfiletime (char *fname,time_t ftime)
 {
-#ifdef WIN32
+#ifdef _WIN32
   static int isWinNT = -1;
   SYSTEMTIME st;
   FILETIME locft, modft;
@@ -590,12 +589,6 @@ void help(int exitval)
   exit(exitval);
 }
 
-void error(const char *msg)
-{
-  fprintf(stderr, "%s: %s\n", prog, msg);
-  exit(1);
-}
-
 
 /* ============================================================ */
 
@@ -608,7 +601,7 @@ int main(int argc,char **argv)
     int         action = TGZ_EXTRACT;
     int         arg = 1;
     char        *TGZfile;
-    gzFile      *f;
+    gzFile      f;
 
     prog = strrchr(argv[0],'\\');
     if (prog == NULL)
diff --git a/zlib/contrib/vstudio/readme.txt b/zlib/contrib/vstudio/readme.txt
index f67eae8..061bbc0 100644
--- a/zlib/contrib/vstudio/readme.txt
+++ b/zlib/contrib/vstudio/readme.txt
@@ -1,78 +1,81 @@
-Building instructions for the DLL versions of Zlib 1.2.11

-========================================================

-

-This directory contains projects that build zlib and minizip using

-Microsoft Visual C++ 9.0/10.0.

-

-You don't need to build these projects yourself. You can download the

-binaries from:

-  http://www.winimage.com/zLibDll

-

-More information can be found at this site.

-

-

-

-

-

-Build instructions for Visual Studio 2008 (32 bits or 64 bits)

---------------------------------------------------------------

-- Decompress current zlib, including all contrib/* files

-- Compile assembly code (with Visual Studio Command Prompt) by running:

-   bld_ml64.bat (in contrib\masmx64)

-   bld_ml32.bat (in contrib\masmx86)

-- Open contrib\vstudio\vc9\zlibvc.sln with Microsoft Visual C++ 2008

-- Or run: vcbuild /rebuild contrib\vstudio\vc9\zlibvc.sln "Release|Win32"

-

-Build instructions for Visual Studio 2010 (32 bits or 64 bits)

---------------------------------------------------------------

-- Decompress current zlib, including all contrib/* files

-- Open contrib\vstudio\vc10\zlibvc.sln with Microsoft Visual C++ 2010

-

-Build instructions for Visual Studio 2012 (32 bits or 64 bits)

---------------------------------------------------------------

-- Decompress current zlib, including all contrib/* files

-- Open contrib\vstudio\vc11\zlibvc.sln with Microsoft Visual C++ 2012

-

-Build instructions for Visual Studio 2013 (32 bits or 64 bits)

---------------------------------------------------------------

-- Decompress current zlib, including all contrib/* files

-- Open contrib\vstudio\vc12\zlibvc.sln with Microsoft Visual C++ 2013

-

-Build instructions for Visual Studio 2015 (32 bits or 64 bits)

---------------------------------------------------------------

-- Decompress current zlib, including all contrib/* files

-- Open contrib\vstudio\vc14\zlibvc.sln with Microsoft Visual C++ 2015

-

-

-Important

----------

-- To use zlibwapi.dll in your application, you must define the

-  macro ZLIB_WINAPI when compiling your application's source files.

-

-

-Additional notes

-----------------

-- This DLL, named zlibwapi.dll, is compatible to the old zlib.dll built

-  by Gilles Vollant from the zlib 1.1.x sources, and distributed at

-    http://www.winimage.com/zLibDll

-  It uses the WINAPI calling convention for the exported functions, and

-  includes the minizip functionality. If your application needs that

-  particular build of zlib.dll, you can rename zlibwapi.dll to zlib.dll.

-

-- The new DLL was renamed because there exist several incompatible

-  versions of zlib.dll on the Internet.

-

-- There is also an official DLL build of zlib, named zlib1.dll. This one

-  is exporting the functions using the CDECL convention. See the file

-  win32\DLL_FAQ.txt found in this zlib distribution.

-

-- There used to be a ZLIB_DLL macro in zlib 1.1.x, but now this symbol

-  has a slightly different effect. To avoid compatibility problems, do

-  not define it here.

-

-

-Gilles Vollant

-info@winimage.com

-

-Visual Studio 2013 and 2015 Projects from Sean Hunt

-seandhunt_7@yahoo.com

+Building instructions for the DLL versions of Zlib 1.3.1
+========================================================
+
+This directory contains projects that build zlib and minizip using
+Microsoft Visual C++ 9.0/10.0.
+
+You don't need to build these projects yourself. You can download the
+binaries from:
+  http://www.winimage.com/zLibDll
+
+More information can be found at this site.
+
+
+
+
+
+Build instructions for Visual Studio 2008 (32 bits or 64 bits)
+--------------------------------------------------------------
+- Decompress current zlib, including all contrib/* files
+- Open contrib\vstudio\vc9\zlibvc.sln with Microsoft Visual C++ 2008
+- Or run: vcbuild /rebuild contrib\vstudio\vc9\zlibvc.sln "Release|Win32"
+
+Build instructions for Visual Studio 2010 (32 bits or 64 bits)
+--------------------------------------------------------------
+- Decompress current zlib, including all contrib/* files
+- Open contrib\vstudio\vc10\zlibvc.sln with Microsoft Visual C++ 2010
+
+Build instructions for Visual Studio 2012 (32 bits or 64 bits)
+--------------------------------------------------------------
+- Decompress current zlib, including all contrib/* files
+- Open contrib\vstudio\vc11\zlibvc.sln with Microsoft Visual C++ 2012
+
+Build instructions for Visual Studio 2013 (32 bits or 64 bits)
+--------------------------------------------------------------
+- Decompress current zlib, including all contrib/* files
+- Open contrib\vstudio\vc12\zlibvc.sln with Microsoft Visual C++ 2013
+
+Build instructions for Visual Studio 2015 (32 bits or 64 bits)
+--------------------------------------------------------------
+- Decompress current zlib, including all contrib/* files
+- Open contrib\vstudio\vc14\zlibvc.sln with Microsoft Visual C++ 2015
+
+Build instructions for Visual Studio 2022 (64 bits)
+--------------------------------------------------------------
+- Decompress current zlib, including all contrib/* files
+- Open contrib\vstudio\vc143\zlibvc.sln with Microsoft Visual C++ 2022
+
+
+
+Important
+---------
+- To use zlibwapi.dll in your application, you must define the
+  macro ZLIB_WINAPI when compiling your application's source files.
+
+
+Additional notes
+----------------
+- This DLL, named zlibwapi.dll, is compatible to the old zlib.dll built
+  by Gilles Vollant from the zlib 1.1.x sources, and distributed at
+    http://www.winimage.com/zLibDll
+  It uses the WINAPI calling convention for the exported functions, and
+  includes the minizip functionality. If your application needs that
+  particular build of zlib.dll, you can rename zlibwapi.dll to zlib.dll.
+
+- The new DLL was renamed because there exist several incompatible
+  versions of zlib.dll on the Internet.
+
+- There is also an official DLL build of zlib, named zlib1.dll. This one
+  is exporting the functions using the CDECL convention. See the file
+  win32\DLL_FAQ.txt found in this zlib distribution.
+
+- There used to be a ZLIB_DLL macro in zlib 1.1.x, but now this symbol
+  has a slightly different effect. To avoid compatibility problems, do
+  not define it here.
+
+
+Gilles Vollant
+info@winimage.com
+
+Visual Studio 2013, 2015, and 2022 Projects from Sean Hunt
+seandhunt_7@yahoo.com
diff --git a/zlib/contrib/vstudio/vc10/miniunz.vcxproj.filters b/zlib/contrib/vstudio/vc10/miniunz.vcxproj.filters
index 0b2a3de..e53556a 100644
--- a/zlib/contrib/vstudio/vc10/miniunz.vcxproj.filters
+++ b/zlib/contrib/vstudio/vc10/miniunz.vcxproj.filters
@@ -3,7 +3,7 @@
   <ItemGroup>

     <Filter Include="Source Files">

       <UniqueIdentifier>{048af943-022b-4db6-beeb-a54c34774ee2}</UniqueIdentifier>

-      <Extensions>cpp;c;cxx;def;odl;idl;hpj;bat;asm</Extensions>

+      <Extensions>cpp;c;cxx;def;odl;idl;hpj;bat</Extensions>

     </Filter>

     <Filter Include="Header Files">

       <UniqueIdentifier>{c1d600d2-888f-4aea-b73e-8b0dd9befa0c}</UniqueIdentifier>

diff --git a/zlib/contrib/vstudio/vc10/minizip.vcxproj.filters b/zlib/contrib/vstudio/vc10/minizip.vcxproj.filters
index dd73cd3..bd18d71 100644
--- a/zlib/contrib/vstudio/vc10/minizip.vcxproj.filters
+++ b/zlib/contrib/vstudio/vc10/minizip.vcxproj.filters
@@ -3,7 +3,7 @@
   <ItemGroup>

     <Filter Include="Source Files">

       <UniqueIdentifier>{c0419b40-bf50-40da-b153-ff74215b79de}</UniqueIdentifier>

-      <Extensions>cpp;c;cxx;def;odl;idl;hpj;bat;asm</Extensions>

+      <Extensions>cpp;c;cxx;def;odl;idl;hpj;bat</Extensions>

     </Filter>

     <Filter Include="Header Files">

       <UniqueIdentifier>{bb87b070-735b-478e-92ce-7383abb2f36c}</UniqueIdentifier>

diff --git a/zlib/contrib/vstudio/vc10/testzlib.vcxproj b/zlib/contrib/vstudio/vc10/testzlib.vcxproj
index 9088d17..0e668f7 100644
--- a/zlib/contrib/vstudio/vc10/testzlib.vcxproj
+++ b/zlib/contrib/vstudio/vc10/testzlib.vcxproj
@@ -181,7 +181,7 @@
     <ClCompile>

       <Optimization>Disabled</Optimization>

       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <MinimalRebuild>true</MinimalRebuild>

       <BasicRuntimeChecks>Default</BasicRuntimeChecks>

       <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>

@@ -194,7 +194,7 @@
       <DebugInformationFormat>EditAndContinue</DebugInformationFormat>

     </ClCompile>

     <Link>

-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <OutputFile>$(OutDir)testzlib.exe</OutputFile>

       <GenerateDebugInformation>true</GenerateDebugInformation>

       <ProgramDatabaseFile>$(OutDir)testzlib.pdb</ProgramDatabaseFile>

@@ -241,7 +241,7 @@
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

       <OmitFramePointers>true</OmitFramePointers>

       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <BasicRuntimeChecks>Default</BasicRuntimeChecks>

       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>

@@ -254,7 +254,7 @@
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>

     </ClCompile>

     <Link>

-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <OutputFile>$(OutDir)testzlib.exe</OutputFile>

       <GenerateDebugInformation>true</GenerateDebugInformation>

       <SubSystem>Console</SubSystem>

@@ -269,14 +269,14 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

     <ClCompile>

       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <BasicRuntimeChecks>Default</BasicRuntimeChecks>

       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>

       <BufferSecurityCheck>false</BufferSecurityCheck>

       <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>

     </ClCompile>

     <Link>

-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

     </Link>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">

@@ -352,14 +352,14 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

     <ClCompile>

       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <BasicRuntimeChecks>Default</BasicRuntimeChecks>

       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>

       <BufferSecurityCheck>false</BufferSecurityCheck>

       <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>

     </ClCompile>

     <Link>

-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

     </Link>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">

@@ -398,14 +398,6 @@
     <ClCompile Include="..\..\..\crc32.c" />

     <ClCompile Include="..\..\..\deflate.c" />

     <ClCompile Include="..\..\..\infback.c" />

-    <ClCompile Include="..\..\masmx64\inffas8664.c">

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>

-    </ClCompile>

     <ClCompile Include="..\..\..\inffast.c" />

     <ClCompile Include="..\..\..\inflate.c" />

     <ClCompile Include="..\..\..\inftrees.c" />

diff --git a/zlib/contrib/vstudio/vc10/testzlib.vcxproj.filters b/zlib/contrib/vstudio/vc10/testzlib.vcxproj.filters
index 249daa8..3cf52ee 100644
--- a/zlib/contrib/vstudio/vc10/testzlib.vcxproj.filters
+++ b/zlib/contrib/vstudio/vc10/testzlib.vcxproj.filters
@@ -3,7 +3,7 @@
   <ItemGroup>

     <Filter Include="Source Files">

       <UniqueIdentifier>{c1f6a2e3-5da5-4955-8653-310d3efe05a9}</UniqueIdentifier>

-      <Extensions>cpp;c;cxx;def;odl;idl;hpj;bat;asm</Extensions>

+      <Extensions>cpp;c;cxx;def;odl;idl;hpj;bat</Extensions>

     </Filter>

     <Filter Include="Header Files">

       <UniqueIdentifier>{c2aaffdc-2c95-4d6f-8466-4bec5890af2c}</UniqueIdentifier>

@@ -30,9 +30,6 @@
     <ClCompile Include="..\..\..\infback.c">

       <Filter>Source Files</Filter>

     </ClCompile>

-    <ClCompile Include="..\..\masmx64\inffas8664.c">

-      <Filter>Source Files</Filter>

-    </ClCompile>

     <ClCompile Include="..\..\..\inffast.c">

       <Filter>Source Files</Filter>

     </ClCompile>

diff --git a/zlib/contrib/vstudio/vc10/testzlibdll.vcxproj.filters b/zlib/contrib/vstudio/vc10/testzlibdll.vcxproj.filters
index 53a8693..aeb550e 100644
--- a/zlib/contrib/vstudio/vc10/testzlibdll.vcxproj.filters
+++ b/zlib/contrib/vstudio/vc10/testzlibdll.vcxproj.filters
@@ -3,7 +3,7 @@
   <ItemGroup>

     <Filter Include="Source Files">

       <UniqueIdentifier>{fa61a89f-93fc-4c89-b29e-36224b7592f4}</UniqueIdentifier>

-      <Extensions>cpp;c;cxx;def;odl;idl;hpj;bat;asm</Extensions>

+      <Extensions>cpp;c;cxx;def;odl;idl;hpj;bat</Extensions>

     </Filter>

     <Filter Include="Header Files">

       <UniqueIdentifier>{d4b85da0-2ba2-4934-b57f-e2584e3848ee}</UniqueIdentifier>

diff --git a/zlib/contrib/vstudio/vc10/zlib.rc b/zlib/contrib/vstudio/vc10/zlib.rc
index fee177a..856bd11 100644
--- a/zlib/contrib/vstudio/vc10/zlib.rc
+++ b/zlib/contrib/vstudio/vc10/zlib.rc
@@ -2,8 +2,8 @@
 

 #define IDR_VERSION1  1

 IDR_VERSION1	VERSIONINFO	MOVEABLE IMPURE LOADONCALL DISCARDABLE

-  FILEVERSION	 1, 2, 11, 0

-  PRODUCTVERSION 1, 2, 11, 0

+  FILEVERSION	 1, 3, 1, 0

+  PRODUCTVERSION 1, 3, 1, 0

   FILEFLAGSMASK	VS_FFI_FILEFLAGSMASK

   FILEFLAGS	0

   FILEOS	VOS_DOS_WINDOWS32

@@ -17,12 +17,12 @@ BEGIN
 

     BEGIN

       VALUE "FileDescription", "zlib data compression and ZIP file I/O library\0"

-      VALUE "FileVersion",	"1.2.11\0"

+      VALUE "FileVersion",	"1.3.1\0"

       VALUE "InternalName",	"zlib\0"

       VALUE "OriginalFilename",	"zlibwapi.dll\0"

       VALUE "ProductName",	"ZLib.DLL\0"

       VALUE "Comments","DLL support by Alessandro Iacopetti & Gilles Vollant\0"

-      VALUE "LegalCopyright", "(C) 1995-2017 Jean-loup Gailly & Mark Adler\0"

+      VALUE "LegalCopyright", "(C) 1995-2024 Jean-loup Gailly & Mark Adler\0"

     END

   END

   BLOCK "VarFileInfo"

diff --git a/zlib/contrib/vstudio/vc10/zlibstat.vcxproj b/zlib/contrib/vstudio/vc10/zlibstat.vcxproj
index b9f2bbe..c7ed09e 100644
--- a/zlib/contrib/vstudio/vc10/zlibstat.vcxproj
+++ b/zlib/contrib/vstudio/vc10/zlibstat.vcxproj
@@ -160,7 +160,7 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

     <ClCompile>

       <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -182,16 +182,12 @@
       <OutputFile>$(OutDir)zlibstat.lib</OutputFile>

       <SuppressStartupBanner>true</SuppressStartupBanner>

     </Lib>

-    <PreBuildEvent>

-      <Command>cd ..\..\masmx86

-bld_ml32.bat</Command>

-    </PreBuildEvent>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ASMV;ASMINF;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -210,19 +206,15 @@ bld_ml32.bat</Command>
     </ResourceCompile>

     <Lib>

       <AdditionalOptions>/MACHINE:X86 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>

-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <OutputFile>$(OutDir)zlibstat.lib</OutputFile>

       <SuppressStartupBanner>true</SuppressStartupBanner>

     </Lib>

-    <PreBuildEvent>

-      <Command>cd ..\..\masmx86

-bld_ml32.bat</Command>

-    </PreBuildEvent>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -252,7 +244,7 @@ bld_ml32.bat</Command>
     </Midl>

     <ClCompile>

       <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -274,10 +266,6 @@ bld_ml32.bat</Command>
       <OutputFile>$(OutDir)zlibstat.lib</OutputFile>

       <SuppressStartupBanner>true</SuppressStartupBanner>

     </Lib>

-    <PreBuildEvent>

-      <Command>cd ..\..\masmx64

-bld_ml64.bat</Command>

-    </PreBuildEvent>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">

     <Midl>

@@ -285,7 +273,7 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -314,8 +302,8 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ASMV;ASMINF;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -334,14 +322,10 @@ bld_ml64.bat</Command>
     </ResourceCompile>

     <Lib>

       <AdditionalOptions>/MACHINE:AMD64 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>

-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <OutputFile>$(OutDir)zlibstat.lib</OutputFile>

       <SuppressStartupBanner>true</SuppressStartupBanner>

     </Lib>

-    <PreBuildEvent>

-      <Command>cd ..\..\masmx64

-bld_ml64.bat</Command>

-    </PreBuildEvent>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">

     <Midl>

@@ -349,7 +333,7 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -379,7 +363,7 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -409,7 +393,7 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -443,14 +427,6 @@ bld_ml64.bat</Command>
     <ClCompile Include="..\..\..\gzread.c" />

     <ClCompile Include="..\..\..\gzwrite.c" />

     <ClCompile Include="..\..\..\infback.c" />

-    <ClCompile Include="..\..\masmx64\inffas8664.c">

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>

-    </ClCompile>

     <ClCompile Include="..\..\..\inffast.c" />

     <ClCompile Include="..\..\..\inflate.c" />

     <ClCompile Include="..\..\..\inftrees.c" />

diff --git a/zlib/contrib/vstudio/vc10/zlibstat.vcxproj.filters b/zlib/contrib/vstudio/vc10/zlibstat.vcxproj.filters
index c8c7f7e..ba7e23d 100644
--- a/zlib/contrib/vstudio/vc10/zlibstat.vcxproj.filters
+++ b/zlib/contrib/vstudio/vc10/zlibstat.vcxproj.filters
@@ -33,9 +33,6 @@
     <ClCompile Include="..\..\..\infback.c">

       <Filter>Source Files</Filter>

     </ClCompile>

-    <ClCompile Include="..\..\masmx64\inffas8664.c">

-      <Filter>Source Files</Filter>

-    </ClCompile>

     <ClCompile Include="..\..\..\inffast.c">

       <Filter>Source Files</Filter>

     </ClCompile>

diff --git a/zlib/contrib/vstudio/vc10/zlibvc.def b/zlib/contrib/vstudio/vc10/zlibvc.def
index 54e683d..3234a02 100644
--- a/zlib/contrib/vstudio/vc10/zlibvc.def
+++ b/zlib/contrib/vstudio/vc10/zlibvc.def
@@ -1,7 +1,7 @@
 LIBRARY

 ; zlib data compression and ZIP file I/O library

 

-VERSION		1.2

+VERSION		1.3.1

 

 EXPORTS

         adler32                                  @1

@@ -151,3 +151,8 @@ EXPORTS
         deflateGetDictionary                    @173

         adler32_z                               @174

         crc32_z                                 @175

+

+; zlib1 v1.2.12 added:

+		crc32_combine_gen						@176

+		crc32_combine_gen64						@177

+		crc32_combine_op						@178

diff --git a/zlib/contrib/vstudio/vc10/zlibvc.vcxproj b/zlib/contrib/vstudio/vc10/zlibvc.vcxproj
index 6ff9ddb..19dfc35 100644
--- a/zlib/contrib/vstudio/vc10/zlibvc.vcxproj
+++ b/zlib/contrib/vstudio/vc10/zlibvc.vcxproj
@@ -197,8 +197,8 @@
     </Midl>

     <ClCompile>

       <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <ExceptionHandling>

       </ExceptionHandling>

       <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>

@@ -219,7 +219,7 @@
     </ResourceCompile>

     <Link>

       <AdditionalOptions>/MACHINE:I386 %(AdditionalOptions)</AdditionalOptions>

-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <SuppressStartupBanner>true</SuppressStartupBanner>

       <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>

       <GenerateDebugInformation>true</GenerateDebugInformation>

@@ -229,10 +229,6 @@
       <DataExecutionPrevention>

       </DataExecutionPrevention>

     </Link>

-    <PreBuildEvent>

-      <Command>cd ..\..\masmx86

-bld_ml32.bat</Command>

-    </PreBuildEvent>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">

     <Midl>

@@ -244,7 +240,7 @@ bld_ml32.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -288,8 +284,8 @@ bld_ml32.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -312,7 +308,7 @@ bld_ml32.bat</Command>
     </ResourceCompile>

     <Link>

       <AdditionalOptions>/MACHINE:I386 %(AdditionalOptions)</AdditionalOptions>

-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <SuppressStartupBanner>true</SuppressStartupBanner>

       <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>

       <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>

@@ -322,10 +318,6 @@ bld_ml32.bat</Command>
       <DataExecutionPrevention>

       </DataExecutionPrevention>

     </Link>

-    <PreBuildEvent>

-      <Command>cd ..\..\masmx86

-bld_ml32.bat</Command>

-    </PreBuildEvent>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

     <Midl>

@@ -337,8 +329,8 @@ bld_ml32.bat</Command>
     </Midl>

     <ClCompile>

       <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <ExceptionHandling>

       </ExceptionHandling>

       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>

@@ -358,7 +350,7 @@ bld_ml32.bat</Command>
       <Culture>0x040c</Culture>

     </ResourceCompile>

     <Link>

-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <SuppressStartupBanner>true</SuppressStartupBanner>

       <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>

       <GenerateDebugInformation>true</GenerateDebugInformation>

@@ -366,10 +358,6 @@ bld_ml32.bat</Command>
       <SubSystem>Windows</SubSystem>

       <TargetMachine>MachineX64</TargetMachine>

     </Link>

-    <PreBuildEvent>

-      <Command>cd ..\..\masmx64

-bld_ml64.bat</Command>

-    </PreBuildEvent>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">

     <Midl>

@@ -381,7 +369,7 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -424,7 +412,7 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -465,7 +453,7 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -510,8 +498,8 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -533,7 +521,7 @@ bld_ml64.bat</Command>
       <Culture>0x040c</Culture>

     </ResourceCompile>

     <Link>

-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <SuppressStartupBanner>true</SuppressStartupBanner>

       <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>

       <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>

@@ -541,10 +529,6 @@ bld_ml64.bat</Command>
       <SubSystem>Windows</SubSystem>

       <TargetMachine>MachineX64</TargetMachine>

     </Link>

-    <PreBuildEvent>

-      <Command>cd ..\..\masmx64

-bld_ml64.bat</Command>

-    </PreBuildEvent>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">

     <Midl>

@@ -556,7 +540,7 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -601,14 +585,6 @@ bld_ml64.bat</Command>
     <ClCompile Include="..\..\..\gzread.c" />

     <ClCompile Include="..\..\..\gzwrite.c" />

     <ClCompile Include="..\..\..\infback.c" />

-    <ClCompile Include="..\..\masmx64\inffas8664.c">

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>

-    </ClCompile>

     <ClCompile Include="..\..\..\inffast.c" />

     <ClCompile Include="..\..\..\inflate.c" />

     <ClCompile Include="..\..\..\inftrees.c" />

diff --git a/zlib/contrib/vstudio/vc10/zlibvc.vcxproj.filters b/zlib/contrib/vstudio/vc10/zlibvc.vcxproj.filters
index 180b71c..67c444a 100644
--- a/zlib/contrib/vstudio/vc10/zlibvc.vcxproj.filters
+++ b/zlib/contrib/vstudio/vc10/zlibvc.vcxproj.filters
@@ -42,9 +42,6 @@
     <ClCompile Include="..\..\..\infback.c">

       <Filter>Source Files</Filter>

     </ClCompile>

-    <ClCompile Include="..\..\masmx64\inffas8664.c">

-      <Filter>Source Files</Filter>

-    </ClCompile>

     <ClCompile Include="..\..\..\inffast.c">

       <Filter>Source Files</Filter>

     </ClCompile>

diff --git a/zlib/contrib/vstudio/vc11/testzlib.vcxproj b/zlib/contrib/vstudio/vc11/testzlib.vcxproj
index 6d55954..c6198c1 100644
--- a/zlib/contrib/vstudio/vc11/testzlib.vcxproj
+++ b/zlib/contrib/vstudio/vc11/testzlib.vcxproj
@@ -187,7 +187,7 @@
     <ClCompile>

       <Optimization>Disabled</Optimization>

       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <MinimalRebuild>true</MinimalRebuild>

       <BasicRuntimeChecks>Default</BasicRuntimeChecks>

       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>

@@ -200,7 +200,7 @@
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>

     </ClCompile>

     <Link>

-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <OutputFile>$(OutDir)testzlib.exe</OutputFile>

       <GenerateDebugInformation>true</GenerateDebugInformation>

       <ProgramDatabaseFile>$(OutDir)testzlib.pdb</ProgramDatabaseFile>

@@ -247,7 +247,7 @@
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

       <OmitFramePointers>true</OmitFramePointers>

       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <BasicRuntimeChecks>Default</BasicRuntimeChecks>

       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>

@@ -260,7 +260,7 @@
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>

     </ClCompile>

     <Link>

-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <OutputFile>$(OutDir)testzlib.exe</OutputFile>

       <GenerateDebugInformation>true</GenerateDebugInformation>

       <SubSystem>Console</SubSystem>

@@ -275,14 +275,14 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

     <ClCompile>

       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <BasicRuntimeChecks>Default</BasicRuntimeChecks>

       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>

       <BufferSecurityCheck>false</BufferSecurityCheck>

       <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>

     </ClCompile>

     <Link>

-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

     </Link>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">

@@ -358,14 +358,14 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

     <ClCompile>

       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <BasicRuntimeChecks>Default</BasicRuntimeChecks>

       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>

       <BufferSecurityCheck>false</BufferSecurityCheck>

       <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>

     </ClCompile>

     <Link>

-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

     </Link>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">

@@ -404,14 +404,6 @@
     <ClCompile Include="..\..\..\crc32.c" />

     <ClCompile Include="..\..\..\deflate.c" />

     <ClCompile Include="..\..\..\infback.c" />

-    <ClCompile Include="..\..\masmx64\inffas8664.c">

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>

-    </ClCompile>

     <ClCompile Include="..\..\..\inffast.c" />

     <ClCompile Include="..\..\..\inflate.c" />

     <ClCompile Include="..\..\..\inftrees.c" />

diff --git a/zlib/contrib/vstudio/vc11/zlib.rc b/zlib/contrib/vstudio/vc11/zlib.rc
index fee177a..856bd11 100644
--- a/zlib/contrib/vstudio/vc11/zlib.rc
+++ b/zlib/contrib/vstudio/vc11/zlib.rc
@@ -2,8 +2,8 @@
 

 #define IDR_VERSION1  1

 IDR_VERSION1	VERSIONINFO	MOVEABLE IMPURE LOADONCALL DISCARDABLE

-  FILEVERSION	 1, 2, 11, 0

-  PRODUCTVERSION 1, 2, 11, 0

+  FILEVERSION	 1, 3, 1, 0

+  PRODUCTVERSION 1, 3, 1, 0

   FILEFLAGSMASK	VS_FFI_FILEFLAGSMASK

   FILEFLAGS	0

   FILEOS	VOS_DOS_WINDOWS32

@@ -17,12 +17,12 @@ BEGIN
 

     BEGIN

       VALUE "FileDescription", "zlib data compression and ZIP file I/O library\0"

-      VALUE "FileVersion",	"1.2.11\0"

+      VALUE "FileVersion",	"1.3.1\0"

       VALUE "InternalName",	"zlib\0"

       VALUE "OriginalFilename",	"zlibwapi.dll\0"

       VALUE "ProductName",	"ZLib.DLL\0"

       VALUE "Comments","DLL support by Alessandro Iacopetti & Gilles Vollant\0"

-      VALUE "LegalCopyright", "(C) 1995-2017 Jean-loup Gailly & Mark Adler\0"

+      VALUE "LegalCopyright", "(C) 1995-2024 Jean-loup Gailly & Mark Adler\0"

     END

   END

   BLOCK "VarFileInfo"

diff --git a/zlib/contrib/vstudio/vc11/zlibstat.vcxproj b/zlib/contrib/vstudio/vc11/zlibstat.vcxproj
index 806b76a..86fb1c8 100644
--- a/zlib/contrib/vstudio/vc11/zlibstat.vcxproj
+++ b/zlib/contrib/vstudio/vc11/zlibstat.vcxproj
@@ -167,7 +167,7 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

     <ClCompile>

       <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -193,8 +193,8 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ASMV;ASMINF;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -213,7 +213,7 @@
     </ResourceCompile>

     <Lib>

       <AdditionalOptions>/MACHINE:X86 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>

-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <OutputFile>$(OutDir)zlibstat.lib</OutputFile>

       <SuppressStartupBanner>true</SuppressStartupBanner>

     </Lib>

@@ -221,7 +221,7 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -251,7 +251,7 @@
     </Midl>

     <ClCompile>

       <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -280,7 +280,7 @@
     </Midl>

     <ClCompile>

       <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -309,8 +309,8 @@
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ASMV;ASMINF;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -329,7 +329,7 @@
     </ResourceCompile>

     <Lib>

       <AdditionalOptions>/MACHINE:AMD64 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>

-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <OutputFile>$(OutDir)zlibstat.lib</OutputFile>

       <SuppressStartupBanner>true</SuppressStartupBanner>

     </Lib>

@@ -340,7 +340,7 @@
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -370,7 +370,7 @@
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -400,7 +400,7 @@
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -434,14 +434,6 @@
     <ClCompile Include="..\..\..\gzread.c" />

     <ClCompile Include="..\..\..\gzwrite.c" />

     <ClCompile Include="..\..\..\infback.c" />

-    <ClCompile Include="..\..\masmx64\inffas8664.c">

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>

-    </ClCompile>

     <ClCompile Include="..\..\..\inffast.c" />

     <ClCompile Include="..\..\..\inflate.c" />

     <ClCompile Include="..\..\..\inftrees.c" />

diff --git a/zlib/contrib/vstudio/vc11/zlibvc.def b/zlib/contrib/vstudio/vc11/zlibvc.def
index 54e683d..3234a02 100644
--- a/zlib/contrib/vstudio/vc11/zlibvc.def
+++ b/zlib/contrib/vstudio/vc11/zlibvc.def
@@ -1,7 +1,7 @@
 LIBRARY

 ; zlib data compression and ZIP file I/O library

 

-VERSION		1.2

+VERSION		1.3.1

 

 EXPORTS

         adler32                                  @1

@@ -151,3 +151,8 @@ EXPORTS
         deflateGetDictionary                    @173

         adler32_z                               @174

         crc32_z                                 @175

+

+; zlib1 v1.2.12 added:

+		crc32_combine_gen						@176

+		crc32_combine_gen64						@177

+		crc32_combine_op						@178

diff --git a/zlib/contrib/vstudio/vc11/zlibvc.vcxproj b/zlib/contrib/vstudio/vc11/zlibvc.vcxproj
index c65b95f..fc8cd9c 100644
--- a/zlib/contrib/vstudio/vc11/zlibvc.vcxproj
+++ b/zlib/contrib/vstudio/vc11/zlibvc.vcxproj
@@ -204,8 +204,8 @@
     </Midl>

     <ClCompile>

       <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <ExceptionHandling>

       </ExceptionHandling>

       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>

@@ -226,7 +226,7 @@
     </ResourceCompile>

     <Link>

       <AdditionalOptions>/MACHINE:I386 %(AdditionalOptions)</AdditionalOptions>

-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>

       <SuppressStartupBanner>true</SuppressStartupBanner>

       <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>

@@ -240,10 +240,6 @@
       </DataExecutionPrevention>

       <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>

     </Link>

-    <PreBuildEvent>

-      <Command>cd ..\..\masmx86

-bld_ml32.bat</Command>

-    </PreBuildEvent>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">

     <Midl>

@@ -255,7 +251,7 @@ bld_ml32.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -303,8 +299,8 @@ bld_ml32.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -327,7 +323,7 @@ bld_ml32.bat</Command>
     </ResourceCompile>

     <Link>

       <AdditionalOptions>/MACHINE:I386 %(AdditionalOptions)</AdditionalOptions>

-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>

       <SuppressStartupBanner>true</SuppressStartupBanner>

       <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>

@@ -341,10 +337,6 @@ bld_ml32.bat</Command>
       </DataExecutionPrevention>

       <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>

     </Link>

-    <PreBuildEvent>

-      <Command>cd ..\..\masmx86

-bld_ml32.bat</Command>

-    </PreBuildEvent>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

     <Midl>

@@ -356,8 +348,8 @@ bld_ml32.bat</Command>
     </Midl>

     <ClCompile>

       <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <ExceptionHandling>

       </ExceptionHandling>

       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>

@@ -377,7 +369,7 @@ bld_ml32.bat</Command>
       <Culture>0x040c</Culture>

     </ResourceCompile>

     <Link>

-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>

       <SuppressStartupBanner>true</SuppressStartupBanner>

       <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>

@@ -389,10 +381,6 @@ bld_ml32.bat</Command>
       <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>

       <TargetMachine>MachineX64</TargetMachine>

     </Link>

-    <PreBuildEvent>

-      <Command>cd ..\..\contrib\masmx64

-bld_ml64.bat</Command>

-    </PreBuildEvent>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">

     <Midl>

@@ -404,7 +392,7 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -447,7 +435,7 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -492,7 +480,7 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -537,8 +525,8 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

       </ExceptionHandling>

@@ -560,7 +548,7 @@ bld_ml64.bat</Command>
       <Culture>0x040c</Culture>

     </ResourceCompile>

     <Link>

-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>

       <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>

       <SuppressStartupBanner>true</SuppressStartupBanner>

       <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>

@@ -572,10 +560,6 @@ bld_ml64.bat</Command>
       <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>

       <TargetMachine>MachineX64</TargetMachine>

     </Link>

-    <PreBuildEvent>

-      <Command>cd ..\..\masmx64

-bld_ml64.bat</Command>

-    </PreBuildEvent>

   </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">

     <Midl>

@@ -587,7 +571,7 @@ bld_ml64.bat</Command>
     </Midl>

     <ClCompile>

       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>

-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

       <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <StringPooling>true</StringPooling>

       <ExceptionHandling>

@@ -632,14 +616,6 @@ bld_ml64.bat</Command>
     <ClCompile Include="..\..\..\gzread.c" />

     <ClCompile Include="..\..\..\gzwrite.c" />

     <ClCompile Include="..\..\..\infback.c" />

-    <ClCompile Include="..\..\masmx64\inffas8664.c">

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>

-    </ClCompile>

     <ClCompile Include="..\..\..\inffast.c" />

     <ClCompile Include="..\..\..\inflate.c" />

     <ClCompile Include="..\..\..\inftrees.c" />

diff --git a/zlib/contrib/vstudio/vc12/testzlib.vcxproj b/zlib/contrib/vstudio/vc12/testzlib.vcxproj
index 64b2cbe34..41303c0 100644
--- a/zlib/contrib/vstudio/vc12/testzlib.vcxproj
+++ b/zlib/contrib/vstudio/vc12/testzlib.vcxproj
@@ -190,7 +190,7 @@
     <ClCompile>
       <Optimization>Disabled</Optimization>
       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <MinimalRebuild>true</MinimalRebuild>
       <BasicRuntimeChecks>Default</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -203,7 +203,7 @@
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)testzlib.exe</OutputFile>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <ProgramDatabaseFile>$(OutDir)testzlib.pdb</ProgramDatabaseFile>
@@ -250,7 +250,7 @@
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
       <OmitFramePointers>true</OmitFramePointers>
       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <BasicRuntimeChecks>Default</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@@ -263,7 +263,7 @@
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)testzlib.exe</OutputFile>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <SubSystem>Console</SubSystem>
@@ -279,14 +279,14 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <ClCompile>
       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <BasicRuntimeChecks>Default</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
       <BufferSecurityCheck>false</BufferSecurityCheck>
       <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">
@@ -362,14 +362,14 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <ClCompile>
       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <BasicRuntimeChecks>Default</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
       <BufferSecurityCheck>false</BufferSecurityCheck>
       <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">
@@ -408,14 +408,6 @@
     <ClCompile Include="..\..\..\crc32.c" />
     <ClCompile Include="..\..\..\deflate.c" />
     <ClCompile Include="..\..\..\infback.c" />
-    <ClCompile Include="..\..\masmx64\inffas8664.c">
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
-    </ClCompile>
     <ClCompile Include="..\..\..\inffast.c" />
     <ClCompile Include="..\..\..\inflate.c" />
     <ClCompile Include="..\..\..\inftrees.c" />
diff --git a/zlib/contrib/vstudio/vc12/zlib.rc b/zlib/contrib/vstudio/vc12/zlib.rc
index c4e4b01..a55f341 100644
--- a/zlib/contrib/vstudio/vc12/zlib.rc
+++ b/zlib/contrib/vstudio/vc12/zlib.rc
@@ -2,8 +2,8 @@
 
 #define IDR_VERSION1  1
 IDR_VERSION1	VERSIONINFO	MOVEABLE IMPURE LOADONCALL DISCARDABLE
-  FILEVERSION	 1, 2, 11, 0
-  PRODUCTVERSION 1, 2, 11, 0
+  FILEVERSION	 1, 3, 1, 0
+  PRODUCTVERSION 1, 3, 1, 0
   FILEFLAGSMASK	VS_FFI_FILEFLAGSMASK
   FILEFLAGS	0
   FILEOS	VOS_DOS_WINDOWS32
@@ -17,12 +17,12 @@ BEGIN
 
     BEGIN
       VALUE "FileDescription", "zlib data compression and ZIP file I/O library\0"
-      VALUE "FileVersion",	"1.2.11\0"
+      VALUE "FileVersion",	"1.3.1\0"
       VALUE "InternalName",	"zlib\0"
       VALUE "OriginalFilename",	"zlibwapi.dll\0"
       VALUE "ProductName",	"ZLib.DLL\0"
       VALUE "Comments","DLL support by Alessandro Iacopetti & Gilles Vollant\0"
-      VALUE "LegalCopyright", "(C) 1995-2017 Jean-loup Gailly & Mark Adler\0"
+      VALUE "LegalCopyright", "(C) 1995-2024 Jean-loup Gailly & Mark Adler\0"
     END
   END
   BLOCK "VarFileInfo"
diff --git a/zlib/contrib/vstudio/vc12/zlibstat.vcxproj b/zlib/contrib/vstudio/vc12/zlibstat.vcxproj
index 3fdee7c..6629d8e 100644
--- a/zlib/contrib/vstudio/vc12/zlibstat.vcxproj
+++ b/zlib/contrib/vstudio/vc12/zlibstat.vcxproj
@@ -170,7 +170,7 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -196,8 +196,8 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ASMV;ASMINF;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -216,7 +216,7 @@
     </ResourceCompile>
     <Lib>
       <AdditionalOptions>/MACHINE:X86 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
       <SuppressStartupBanner>true</SuppressStartupBanner>
     </Lib>
@@ -224,7 +224,7 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -254,7 +254,7 @@
     </Midl>
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -283,7 +283,7 @@
     </Midl>
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -312,8 +312,8 @@
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ASMV;ASMINF;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -332,7 +332,7 @@
     </ResourceCompile>
     <Lib>
       <AdditionalOptions>/MACHINE:AMD64 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
       <SuppressStartupBanner>true</SuppressStartupBanner>
     </Lib>
@@ -343,7 +343,7 @@
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -373,7 +373,7 @@
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -403,7 +403,7 @@
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -437,14 +437,6 @@
     <ClCompile Include="..\..\..\gzread.c" />
     <ClCompile Include="..\..\..\gzwrite.c" />
     <ClCompile Include="..\..\..\infback.c" />
-    <ClCompile Include="..\..\masmx64\inffas8664.c">
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
-    </ClCompile>
     <ClCompile Include="..\..\..\inffast.c" />
     <ClCompile Include="..\..\..\inflate.c" />
     <ClCompile Include="..\..\..\inftrees.c" />
diff --git a/zlib/contrib/vstudio/vc12/zlibvc.def b/zlib/contrib/vstudio/vc12/zlibvc.def
index 54e683d..3234a02 100644
--- a/zlib/contrib/vstudio/vc12/zlibvc.def
+++ b/zlib/contrib/vstudio/vc12/zlibvc.def
@@ -1,7 +1,7 @@
 LIBRARY

 ; zlib data compression and ZIP file I/O library

 

-VERSION		1.2

+VERSION		1.3.1

 

 EXPORTS

         adler32                                  @1

@@ -151,3 +151,8 @@ EXPORTS
         deflateGetDictionary                    @173

         adler32_z                               @174

         crc32_z                                 @175

+

+; zlib1 v1.2.12 added:

+		crc32_combine_gen						@176

+		crc32_combine_gen64						@177

+		crc32_combine_op						@178

diff --git a/zlib/contrib/vstudio/vc12/zlibvc.vcxproj b/zlib/contrib/vstudio/vc12/zlibvc.vcxproj
index ab2b6c3..4e0de69 100644
--- a/zlib/contrib/vstudio/vc12/zlibvc.vcxproj
+++ b/zlib/contrib/vstudio/vc12/zlibvc.vcxproj
@@ -207,8 +207,8 @@
     </Midl>
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ExceptionHandling>
       </ExceptionHandling>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -229,7 +229,7 @@
     </ResourceCompile>
     <Link>
       <AdditionalOptions>/MACHINE:I386 %(AdditionalOptions)</AdditionalOptions>
-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
       <SuppressStartupBanner>true</SuppressStartupBanner>
       <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
@@ -243,10 +243,6 @@
       </DataExecutionPrevention>
       <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
     </Link>
-    <PreBuildEvent>
-      <Command>cd ..\..\masmx86
-bld_ml32.bat</Command>
-    </PreBuildEvent>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">
     <Midl>
@@ -258,7 +254,7 @@ bld_ml32.bat</Command>
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -306,8 +302,8 @@ bld_ml32.bat</Command>
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -330,7 +326,7 @@ bld_ml32.bat</Command>
     </ResourceCompile>
     <Link>
       <AdditionalOptions>/MACHINE:I386 %(AdditionalOptions)</AdditionalOptions>
-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
       <SuppressStartupBanner>true</SuppressStartupBanner>
       <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
@@ -345,10 +341,6 @@ bld_ml32.bat</Command>
       <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
       <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
     </Link>
-    <PreBuildEvent>
-      <Command>cd ..\..\masmx86
-bld_ml32.bat</Command>
-    </PreBuildEvent>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <Midl>
@@ -360,8 +352,8 @@ bld_ml32.bat</Command>
     </Midl>
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ExceptionHandling>
       </ExceptionHandling>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -381,7 +373,7 @@ bld_ml32.bat</Command>
       <Culture>0x040c</Culture>
     </ResourceCompile>
     <Link>
-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
       <SuppressStartupBanner>true</SuppressStartupBanner>
       <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
@@ -393,10 +385,6 @@ bld_ml32.bat</Command>
       <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
       <TargetMachine>MachineX64</TargetMachine>
     </Link>
-    <PreBuildEvent>
-      <Command>cd ..\..\contrib\masmx64
-bld_ml64.bat</Command>
-    </PreBuildEvent>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">
     <Midl>
@@ -408,7 +396,7 @@ bld_ml64.bat</Command>
     </Midl>
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -451,7 +439,7 @@ bld_ml64.bat</Command>
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -496,7 +484,7 @@ bld_ml64.bat</Command>
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -541,8 +529,8 @@ bld_ml64.bat</Command>
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -564,7 +552,7 @@ bld_ml64.bat</Command>
       <Culture>0x040c</Culture>
     </ResourceCompile>
     <Link>
-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
       <SuppressStartupBanner>true</SuppressStartupBanner>
       <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
@@ -576,10 +564,6 @@ bld_ml64.bat</Command>
       <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
       <TargetMachine>MachineX64</TargetMachine>
     </Link>
-    <PreBuildEvent>
-      <Command>cd ..\..\masmx64
-bld_ml64.bat</Command>
-    </PreBuildEvent>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">
     <Midl>
@@ -591,7 +575,7 @@ bld_ml64.bat</Command>
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -636,14 +620,6 @@ bld_ml64.bat</Command>
     <ClCompile Include="..\..\..\gzread.c" />
     <ClCompile Include="..\..\..\gzwrite.c" />
     <ClCompile Include="..\..\..\infback.c" />
-    <ClCompile Include="..\..\masmx64\inffas8664.c">
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
-    </ClCompile>
     <ClCompile Include="..\..\..\inffast.c" />
     <ClCompile Include="..\..\..\inflate.c" />
     <ClCompile Include="..\..\..\inftrees.c" />
diff --git a/zlib/contrib/vstudio/vc14/testzlib.vcxproj b/zlib/contrib/vstudio/vc14/testzlib.vcxproj
index 2c37125..5452049 100644
--- a/zlib/contrib/vstudio/vc14/testzlib.vcxproj
+++ b/zlib/contrib/vstudio/vc14/testzlib.vcxproj
@@ -190,7 +190,7 @@
     <ClCompile>
       <Optimization>Disabled</Optimization>
       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <MinimalRebuild>true</MinimalRebuild>
       <BasicRuntimeChecks>Default</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -203,7 +203,7 @@
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)testzlib.exe</OutputFile>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <ProgramDatabaseFile>$(OutDir)testzlib.pdb</ProgramDatabaseFile>
@@ -250,7 +250,7 @@
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
       <OmitFramePointers>true</OmitFramePointers>
       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <BasicRuntimeChecks>Default</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@@ -263,7 +263,7 @@
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)testzlib.exe</OutputFile>
       <GenerateDebugInformation>true</GenerateDebugInformation>
       <SubSystem>Console</SubSystem>
@@ -279,14 +279,14 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <ClCompile>
       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <BasicRuntimeChecks>Default</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
       <BufferSecurityCheck>false</BufferSecurityCheck>
       <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">
@@ -362,14 +362,14 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <ClCompile>
       <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>ASMV;ASMINF;WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <BasicRuntimeChecks>Default</BasicRuntimeChecks>
       <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
       <BufferSecurityCheck>false</BufferSecurityCheck>
       <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">
@@ -408,14 +408,6 @@
     <ClCompile Include="..\..\..\crc32.c" />
     <ClCompile Include="..\..\..\deflate.c" />
     <ClCompile Include="..\..\..\infback.c" />
-    <ClCompile Include="..\..\masmx64\inffas8664.c">
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
-    </ClCompile>
     <ClCompile Include="..\..\..\inffast.c" />
     <ClCompile Include="..\..\..\inflate.c" />
     <ClCompile Include="..\..\..\inftrees.c" />
diff --git a/zlib/contrib/vstudio/vc14/zlib.rc b/zlib/contrib/vstudio/vc14/zlib.rc
index c4e4b01..a55f341 100644
--- a/zlib/contrib/vstudio/vc14/zlib.rc
+++ b/zlib/contrib/vstudio/vc14/zlib.rc
@@ -2,8 +2,8 @@
 
 #define IDR_VERSION1  1
 IDR_VERSION1	VERSIONINFO	MOVEABLE IMPURE LOADONCALL DISCARDABLE
-  FILEVERSION	 1, 2, 11, 0
-  PRODUCTVERSION 1, 2, 11, 0
+  FILEVERSION	 1, 3, 1, 0
+  PRODUCTVERSION 1, 3, 1, 0
   FILEFLAGSMASK	VS_FFI_FILEFLAGSMASK
   FILEFLAGS	0
   FILEOS	VOS_DOS_WINDOWS32
@@ -17,12 +17,12 @@ BEGIN
 
     BEGIN
       VALUE "FileDescription", "zlib data compression and ZIP file I/O library\0"
-      VALUE "FileVersion",	"1.2.11\0"
+      VALUE "FileVersion",	"1.3.1\0"
       VALUE "InternalName",	"zlib\0"
       VALUE "OriginalFilename",	"zlibwapi.dll\0"
       VALUE "ProductName",	"ZLib.DLL\0"
       VALUE "Comments","DLL support by Alessandro Iacopetti & Gilles Vollant\0"
-      VALUE "LegalCopyright", "(C) 1995-2017 Jean-loup Gailly & Mark Adler\0"
+      VALUE "LegalCopyright", "(C) 1995-2024 Jean-loup Gailly & Mark Adler\0"
     END
   END
   BLOCK "VarFileInfo"
diff --git a/zlib/contrib/vstudio/vc14/zlibstat.vcxproj b/zlib/contrib/vstudio/vc14/zlibstat.vcxproj
index 3e4b986..85c1e89 100644
--- a/zlib/contrib/vstudio/vc14/zlibstat.vcxproj
+++ b/zlib/contrib/vstudio/vc14/zlibstat.vcxproj
@@ -170,7 +170,7 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -196,8 +196,8 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ASMV;ASMINF;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -216,7 +216,7 @@
     </ResourceCompile>
     <Lib>
       <AdditionalOptions>/MACHINE:X86 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
       <SuppressStartupBanner>true</SuppressStartupBanner>
     </Lib>
@@ -224,7 +224,7 @@
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -254,7 +254,7 @@
     </Midl>
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -283,7 +283,7 @@
     </Midl>
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -312,8 +312,8 @@
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ASMV;ASMINF;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -332,7 +332,7 @@
     </ResourceCompile>
     <Lib>
       <AdditionalOptions>/MACHINE:AMD64 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
       <SuppressStartupBanner>true</SuppressStartupBanner>
     </Lib>
@@ -343,7 +343,7 @@
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -373,7 +373,7 @@
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -403,7 +403,7 @@
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -437,14 +437,6 @@
     <ClCompile Include="..\..\..\gzread.c" />
     <ClCompile Include="..\..\..\gzwrite.c" />
     <ClCompile Include="..\..\..\infback.c" />
-    <ClCompile Include="..\..\masmx64\inffas8664.c">
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
-    </ClCompile>
     <ClCompile Include="..\..\..\inffast.c" />
     <ClCompile Include="..\..\..\inflate.c" />
     <ClCompile Include="..\..\..\inftrees.c" />
diff --git a/zlib/contrib/vstudio/vc14/zlibvc.def b/zlib/contrib/vstudio/vc14/zlibvc.def
index 54e683d..3234a02 100644
--- a/zlib/contrib/vstudio/vc14/zlibvc.def
+++ b/zlib/contrib/vstudio/vc14/zlibvc.def
@@ -1,7 +1,7 @@
 LIBRARY

 ; zlib data compression and ZIP file I/O library

 

-VERSION		1.2

+VERSION		1.3.1

 

 EXPORTS

         adler32                                  @1

@@ -151,3 +151,8 @@ EXPORTS
         deflateGetDictionary                    @173

         adler32_z                               @174

         crc32_z                                 @175

+

+; zlib1 v1.2.12 added:

+		crc32_combine_gen						@176

+		crc32_combine_gen64						@177

+		crc32_combine_op						@178

diff --git a/zlib/contrib/vstudio/vc14/zlibvc.vcxproj b/zlib/contrib/vstudio/vc14/zlibvc.vcxproj
index f8f673c..424ff55 100644
--- a/zlib/contrib/vstudio/vc14/zlibvc.vcxproj
+++ b/zlib/contrib/vstudio/vc14/zlibvc.vcxproj
@@ -207,8 +207,8 @@
     </Midl>
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ExceptionHandling>
       </ExceptionHandling>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -229,7 +229,7 @@
     </ResourceCompile>
     <Link>
       <AdditionalOptions>/MACHINE:I386 %(AdditionalOptions)</AdditionalOptions>
-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
       <SuppressStartupBanner>true</SuppressStartupBanner>
       <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
@@ -243,10 +243,6 @@
       </DataExecutionPrevention>
       <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
     </Link>
-    <PreBuildEvent>
-      <Command>cd ..\..\masmx86
-bld_ml32.bat</Command>
-    </PreBuildEvent>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">
     <Midl>
@@ -258,7 +254,7 @@ bld_ml32.bat</Command>
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -306,8 +302,8 @@ bld_ml32.bat</Command>
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -330,7 +326,7 @@ bld_ml32.bat</Command>
     </ResourceCompile>
     <Link>
       <AdditionalOptions>/MACHINE:I386 %(AdditionalOptions)</AdditionalOptions>
-      <AdditionalDependencies>..\..\masmx86\match686.obj;..\..\masmx86\inffas32.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
       <SuppressStartupBanner>true</SuppressStartupBanner>
       <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
@@ -345,10 +341,6 @@ bld_ml32.bat</Command>
       <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
       <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
     </Link>
-    <PreBuildEvent>
-      <Command>cd ..\..\masmx86
-bld_ml32.bat</Command>
-    </PreBuildEvent>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <Midl>
@@ -360,8 +352,8 @@ bld_ml32.bat</Command>
     </Midl>
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ExceptionHandling>
       </ExceptionHandling>
       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -381,7 +373,7 @@ bld_ml32.bat</Command>
       <Culture>0x040c</Culture>
     </ResourceCompile>
     <Link>
-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
       <SuppressStartupBanner>true</SuppressStartupBanner>
       <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
@@ -393,10 +385,6 @@ bld_ml32.bat</Command>
       <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
       <TargetMachine>MachineX64</TargetMachine>
     </Link>
-    <PreBuildEvent>
-      <Command>cd ..\..\contrib\masmx64
-bld_ml64.bat</Command>
-    </PreBuildEvent>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">
     <Midl>
@@ -408,7 +396,7 @@ bld_ml64.bat</Command>
     </Midl>
     <ClCompile>
       <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -451,7 +439,7 @@ bld_ml64.bat</Command>
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -496,7 +484,7 @@ bld_ml64.bat</Command>
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -541,8 +529,8 @@ bld_ml64.bat</Command>
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
       </ExceptionHandling>
@@ -564,7 +552,7 @@ bld_ml64.bat</Command>
       <Culture>0x040c</Culture>
     </ResourceCompile>
     <Link>
-      <AdditionalDependencies>..\..\masmx64\gvmat64.obj;..\..\masmx64\inffasx64.obj;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
       <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
       <SuppressStartupBanner>true</SuppressStartupBanner>
       <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
@@ -576,10 +564,6 @@ bld_ml64.bat</Command>
       <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
       <TargetMachine>MachineX64</TargetMachine>
     </Link>
-    <PreBuildEvent>
-      <Command>cd ..\..\masmx64
-bld_ml64.bat</Command>
-    </PreBuildEvent>
   </ItemDefinitionGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">
     <Midl>
@@ -591,7 +575,7 @@ bld_ml64.bat</Command>
     </Midl>
     <ClCompile>
       <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
-      <AdditionalIncludeDirectories>..\..\..;..\..\masmx86;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <StringPooling>true</StringPooling>
       <ExceptionHandling>
@@ -636,14 +620,6 @@ bld_ml64.bat</Command>
     <ClCompile Include="..\..\..\gzread.c" />
     <ClCompile Include="..\..\..\gzwrite.c" />
     <ClCompile Include="..\..\..\infback.c" />
-    <ClCompile Include="..\..\masmx64\inffas8664.c">
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Itanium'">true</ExcludedFromBuild>
-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
-    </ClCompile>
     <ClCompile Include="..\..\..\inffast.c" />
     <ClCompile Include="..\..\..\inflate.c" />
     <ClCompile Include="..\..\..\inftrees.c" />
diff --git a/zlib/contrib/vstudio/vc17/miniunz.vcxproj b/zlib/contrib/vstudio/vc17/miniunz.vcxproj
new file mode 100644
index 0000000..68ef1658
--- /dev/null
+++ b/zlib/contrib/vstudio/vc17/miniunz.vcxproj
@@ -0,0 +1,409 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|ARM">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|ARM64">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM">
+      <Configuration>Release</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM64">
+      <Configuration>Release</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{C52F9E7B-498A-42BE-8DB4-85A15694382A}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <_ProjectFileVersion>10.0.30128.1</_ProjectFileVersion>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">x86\MiniUnzip$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">x86\MiniUnzip$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">x86\MiniUnzip$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">x86\MiniUnzip$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">x64\MiniUnzip$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">x64\MiniUnzip$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">true</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">true</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">x64\MiniUnzip$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">x64\MiniUnzip$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">false</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">false</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">false</GenerateManifest>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
+    <OutDir>arm64\MiniUnzip$(Configuration)\</OutDir>
+    <IntDir>arm64\MiniUnzip$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
+    <OutDir>arm64\MiniUnzip$(Configuration)\</OutDir>
+    <IntDir>arm64\MiniUnzip$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <OutDir>arm\MiniUnzip$(Configuration)\</OutDir>
+    <IntDir>arm\MiniUnzip$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <OutDir>arm\MiniUnzip$(Configuration)\</OutDir>
+    <IntDir>arm\MiniUnzip$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x86\ZlibDllDebug\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)miniunz.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)miniunz.pdb</ProgramDatabaseFile>
+      <SubSystem>Console</SubSystem>
+      <RandomizedBaseAddress>false</RandomizedBaseAddress>
+      <DataExecutionPrevention>
+      </DataExecutionPrevention>
+      <TargetMachine>MachineX86</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x86\ZlibDllRelease\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)miniunz.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <RandomizedBaseAddress>false</RandomizedBaseAddress>
+      <DataExecutionPrevention>
+      </DataExecutionPrevention>
+      <TargetMachine>MachineX86</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Midl>
+      <TargetEnvironment>X64</TargetEnvironment>
+    </Midl>
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;_DEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllDebug\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)miniunz.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)miniunz.pdb</ProgramDatabaseFile>
+      <SubSystem>Console</SubSystem>
+      <TargetMachine>MachineX64</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
+    <Midl />
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;_DEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllDebug\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)miniunz.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)miniunz.pdb</ProgramDatabaseFile>
+      <SubSystem>Console</SubSystem>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <Midl />
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;_DEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllDebug\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)miniunz.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)miniunz.pdb</ProgramDatabaseFile>
+      <SubSystem>Console</SubSystem>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Midl>
+      <TargetEnvironment>X64</TargetEnvironment>
+    </Midl>
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;NDEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllRelease\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)miniunz.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <TargetMachine>MachineX64</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
+    <Midl />
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;NDEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllRelease\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)miniunz.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <Midl />
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;NDEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllRelease\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)miniunz.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\minizip\miniunz.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="zlibvc.vcxproj">
+      <Project>{8fd826f8-3739-44e6-8cc8-997122e53b8d}</Project>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/zlib/contrib/vstudio/vc17/minizip.vcxproj b/zlib/contrib/vstudio/vc17/minizip.vcxproj
new file mode 100644
index 0000000..dd3c52e
--- /dev/null
+++ b/zlib/contrib/vstudio/vc17/minizip.vcxproj
@@ -0,0 +1,405 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|ARM">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|ARM64">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM">
+      <Configuration>Release</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM64">
+      <Configuration>Release</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <_ProjectFileVersion>10.0.30128.1</_ProjectFileVersion>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">x86\MiniZip$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">x86\MiniZip$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">x86\MiniZip$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">x86\MiniZip$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">x64\$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">x64\$(Configuration)\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">true</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">true</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">x64\$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">x64\$(Configuration)\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">false</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">false</LinkIncremental>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
+    <OutDir>arm64\MiniZip$(Configuration)\</OutDir>
+    <IntDir>arm64\MiniZip$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
+    <OutDir>arm64\MiniZip$(Configuration)\</OutDir>
+    <IntDir>arm64\MiniZip$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <OutDir>arm\MiniZip$(Configuration)\</OutDir>
+    <IntDir>arm\MiniZip$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <OutDir>arm\MiniZip$(Configuration)\</OutDir>
+    <IntDir>arm\MiniZip$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x86\ZlibDllDebug\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)minizip.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)minizip.pdb</ProgramDatabaseFile>
+      <SubSystem>Console</SubSystem>
+      <RandomizedBaseAddress>false</RandomizedBaseAddress>
+      <DataExecutionPrevention>
+      </DataExecutionPrevention>
+      <TargetMachine>MachineX86</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x86\ZlibDllRelease\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)minizip.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <RandomizedBaseAddress>false</RandomizedBaseAddress>
+      <DataExecutionPrevention>
+      </DataExecutionPrevention>
+      <TargetMachine>MachineX86</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Midl>
+      <TargetEnvironment>X64</TargetEnvironment>
+    </Midl>
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;_DEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllDebug\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)minizip.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)minizip.pdb</ProgramDatabaseFile>
+      <SubSystem>Console</SubSystem>
+      <TargetMachine>MachineX64</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
+    <Midl />
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;_DEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllDebug\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)minizip.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)minizip.pdb</ProgramDatabaseFile>
+      <SubSystem>Console</SubSystem>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <Midl />
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;_DEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllDebug\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)minizip.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)minizip.pdb</ProgramDatabaseFile>
+      <SubSystem>Console</SubSystem>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Midl>
+      <TargetEnvironment>X64</TargetEnvironment>
+    </Midl>
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;NDEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllRelease\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)minizip.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <TargetMachine>MachineX64</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
+    <Midl />
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;NDEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllRelease\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)minizip.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <Midl />
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;NDEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllRelease\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)minizip.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\minizip\minizip.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="zlibvc.vcxproj">
+      <Project>{8fd826f8-3739-44e6-8cc8-997122e53b8d}</Project>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/zlib/contrib/vstudio/vc17/testzlib.vcxproj b/zlib/contrib/vstudio/vc17/testzlib.vcxproj
new file mode 100644
index 0000000..4cc99b3
--- /dev/null
+++ b/zlib/contrib/vstudio/vc17/testzlib.vcxproj
@@ -0,0 +1,473 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|ARM">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|ARM64">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="ReleaseWithoutAsm|ARM">
+      <Configuration>ReleaseWithoutAsm</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="ReleaseWithoutAsm|ARM64">
+      <Configuration>ReleaseWithoutAsm</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="ReleaseWithoutAsm|Win32">
+      <Configuration>ReleaseWithoutAsm</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="ReleaseWithoutAsm|x64">
+      <Configuration>ReleaseWithoutAsm</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM">
+      <Configuration>Release</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM64">
+      <Configuration>Release</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}</ProjectGuid>
+    <RootNamespace>testzlib</RootNamespace>
+    <Keyword>Win32Proj</Keyword>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <_ProjectFileVersion>10.0.30128.1</_ProjectFileVersion>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">x86\TestZlib$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">x86\TestZlib$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">x86\TestZlib$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">x86\TestZlib$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">false</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">x86\TestZlib$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">x86\TestZlib$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">x64\TestZlib$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">x64\TestZlib$(Configuration)\Tmp\</IntDir>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">x64\TestZlib$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">x64\TestZlib$(Configuration)\Tmp\</IntDir>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">x64\TestZlib$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">x64\TestZlib$(Configuration)\Tmp\</IntDir>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">false</GenerateManifest>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
+    <OutDir>arm64\TestZlib$(Configuration)\</OutDir>
+    <IntDir>arm64\TestZlib$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
+    <OutDir>arm64\TestZlib$(Configuration)\</OutDir>
+    <IntDir>arm64\TestZlib$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'">
+    <OutDir>arm64\TestZlib$(Configuration)\</OutDir>
+    <IntDir>arm64\TestZlib$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <OutDir>arm\TestZlib$(Configuration)\</OutDir>
+    <IntDir>arm\TestZlib$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <OutDir>arm\TestZlib$(Configuration)\</OutDir>
+    <IntDir>arm\TestZlib$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'">
+    <OutDir>arm\TestZlib$(Configuration)\</OutDir>
+    <IntDir>arm\TestZlib$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerOutput>AssemblyAndSourceCode</AssemblerOutput>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)testzlib.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)testzlib.pdb</ProgramDatabaseFile>
+      <SubSystem>Console</SubSystem>
+      <RandomizedBaseAddress>false</RandomizedBaseAddress>
+      <DataExecutionPrevention>
+      </DataExecutionPrevention>
+      <TargetMachine>MachineX86</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <OutputFile>$(OutDir)testzlib.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <RandomizedBaseAddress>false</RandomizedBaseAddress>
+      <DataExecutionPrevention>
+      </DataExecutionPrevention>
+      <TargetMachine>MachineX86</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)testzlib.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <RandomizedBaseAddress>false</RandomizedBaseAddress>
+      <DataExecutionPrevention>
+      </DataExecutionPrevention>
+      <TargetMachine>MachineX86</TargetMachine>
+      <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
+    <ClCompile>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <ClCompile>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">
+    <ClCompile>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'">
+    <ClCompile>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'">
+    <ClCompile>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
+    <ClCompile>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <ClCompile>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\adler32.c" />
+    <ClCompile Include="..\..\..\compress.c" />
+    <ClCompile Include="..\..\..\crc32.c" />
+    <ClCompile Include="..\..\..\deflate.c" />
+    <ClCompile Include="..\..\..\infback.c" />
+    <ClCompile Include="..\..\..\inffast.c" />
+    <ClCompile Include="..\..\..\inflate.c" />
+    <ClCompile Include="..\..\..\inftrees.c" />
+    <ClCompile Include="..\..\testzlib\testzlib.c" />
+    <ClCompile Include="..\..\..\trees.c" />
+    <ClCompile Include="..\..\..\uncompr.c" />
+    <ClCompile Include="..\..\..\zutil.c" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/zlib/contrib/vstudio/vc17/testzlibdll.vcxproj b/zlib/contrib/vstudio/vc17/testzlibdll.vcxproj
new file mode 100644
index 0000000..73bba55
--- /dev/null
+++ b/zlib/contrib/vstudio/vc17/testzlibdll.vcxproj
@@ -0,0 +1,409 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|ARM">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|ARM64">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM">
+      <Configuration>Release</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM64">
+      <Configuration>Release</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{C52F9E7B-498A-42BE-8DB4-85A15694366A}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>Unicode</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <CharacterSet>MultiByte</CharacterSet>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <_ProjectFileVersion>10.0.30128.1</_ProjectFileVersion>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">x86\TestZlibDll$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">x86\TestZlibDll$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">x86\TestZlibDll$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">x86\TestZlibDll$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">x64\TestZlibDll$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">x64\TestZlibDll$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">true</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">true</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">x64\TestZlibDll$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">x64\TestZlibDll$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">false</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">false</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">false</GenerateManifest>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
+    <OutDir>arm64\TestZlibDll$(Configuration)\</OutDir>
+    <IntDir>arm64\TestZlibDll$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
+    <OutDir>arm64\TestZlibDll$(Configuration)\</OutDir>
+    <IntDir>arm64\TestZlibDll$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <OutDir>arm\TestZlibDll$(Configuration)\</OutDir>
+    <IntDir>arm\TestZlibDll$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <OutDir>arm\TestZlibDll$(Configuration)\</OutDir>
+    <IntDir>arm\TestZlibDll$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x86\ZlibDllDebug\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)testzlibdll.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)testzlib.pdb</ProgramDatabaseFile>
+      <SubSystem>Console</SubSystem>
+      <RandomizedBaseAddress>false</RandomizedBaseAddress>
+      <DataExecutionPrevention>
+      </DataExecutionPrevention>
+      <TargetMachine>MachineX86</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x86\ZlibDllRelease\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)testzlibdll.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <RandomizedBaseAddress>false</RandomizedBaseAddress>
+      <DataExecutionPrevention>
+      </DataExecutionPrevention>
+      <TargetMachine>MachineX86</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Midl>
+      <TargetEnvironment>X64</TargetEnvironment>
+    </Midl>
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;_DEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllDebug\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)testzlibdll.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)testzlib.pdb</ProgramDatabaseFile>
+      <SubSystem>Console</SubSystem>
+      <TargetMachine>MachineX64</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
+    <Midl />
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;_DEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllDebug\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)testzlibdll.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)testzlib.pdb</ProgramDatabaseFile>
+      <SubSystem>Console</SubSystem>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <Midl />
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;_DEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllDebug\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)testzlibdll.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)testzlib.pdb</ProgramDatabaseFile>
+      <SubSystem>Console</SubSystem>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Midl>
+      <TargetEnvironment>X64</TargetEnvironment>
+    </Midl>
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;NDEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllRelease\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)testzlibdll.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <TargetMachine>MachineX64</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
+    <Midl />
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;NDEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllRelease\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)testzlibdll.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <Midl />
+    <ClCompile>
+      <Optimization>MaxSpeed</Optimization>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <OmitFramePointers>true</OmitFramePointers>
+      <AdditionalIncludeDirectories>..\..\..;..\..\minizip;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;ZLIB_WINAPI;NDEBUG;_CONSOLE;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Link>
+      <AdditionalDependencies>x64\ZlibDllRelease\zlibwapi.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)testzlibdll.exe</OutputFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <SubSystem>Console</SubSystem>
+      <OptimizeReferences>true</OptimizeReferences>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\testzlib\testzlib.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="zlibvc.vcxproj">
+      <Project>{8fd826f8-3739-44e6-8cc8-997122e53b8d}</Project>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/zlib/contrib/vstudio/vc17/zlib.rc b/zlib/contrib/vstudio/vc17/zlib.rc
new file mode 100644
index 0000000..a55f341
--- /dev/null
+++ b/zlib/contrib/vstudio/vc17/zlib.rc
@@ -0,0 +1,32 @@
+#include <windows.h>
+
+#define IDR_VERSION1  1
+IDR_VERSION1	VERSIONINFO	MOVEABLE IMPURE LOADONCALL DISCARDABLE
+  FILEVERSION	 1, 3, 1, 0
+  PRODUCTVERSION 1, 3, 1, 0
+  FILEFLAGSMASK	VS_FFI_FILEFLAGSMASK
+  FILEFLAGS	0
+  FILEOS	VOS_DOS_WINDOWS32
+  FILETYPE	VFT_DLL
+  FILESUBTYPE	0	// not used
+BEGIN
+  BLOCK "StringFileInfo"
+  BEGIN
+    BLOCK "040904E4"
+    //language ID = U.S. English, char set = Windows, Multilingual
+
+    BEGIN
+      VALUE "FileDescription", "zlib data compression and ZIP file I/O library\0"
+      VALUE "FileVersion",	"1.3.1\0"
+      VALUE "InternalName",	"zlib\0"
+      VALUE "OriginalFilename",	"zlibwapi.dll\0"
+      VALUE "ProductName",	"ZLib.DLL\0"
+      VALUE "Comments","DLL support by Alessandro Iacopetti & Gilles Vollant\0"
+      VALUE "LegalCopyright", "(C) 1995-2024 Jean-loup Gailly & Mark Adler\0"
+    END
+  END
+  BLOCK "VarFileInfo"
+  BEGIN
+    VALUE "Translation", 0x0409, 1252
+  END
+END
diff --git a/zlib/contrib/vstudio/vc17/zlibstat.vcxproj b/zlib/contrib/vstudio/vc17/zlibstat.vcxproj
new file mode 100644
index 0000000..b946ac2
--- /dev/null
+++ b/zlib/contrib/vstudio/vc17/zlibstat.vcxproj
@@ -0,0 +1,602 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|ARM">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|ARM64">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="ReleaseWithoutAsm|ARM">
+      <Configuration>ReleaseWithoutAsm</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="ReleaseWithoutAsm|ARM64">
+      <Configuration>ReleaseWithoutAsm</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="ReleaseWithoutAsm|Win32">
+      <Configuration>ReleaseWithoutAsm</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="ReleaseWithoutAsm|x64">
+      <Configuration>ReleaseWithoutAsm</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM">
+      <Configuration>Release</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM64">
+      <Configuration>Release</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}</ProjectGuid>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <_ProjectFileVersion>10.0.30128.1</_ProjectFileVersion>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">x86\ZlibStat$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">x86\ZlibStat$(Configuration)\Tmp\</IntDir>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">x86\ZlibStat$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">x86\ZlibStat$(Configuration)\Tmp\</IntDir>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">x86\ZlibStat$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">x86\ZlibStat$(Configuration)\Tmp\</IntDir>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">x64\ZlibStat$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">x64\ZlibStat$(Configuration)\Tmp\</IntDir>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">x64\ZlibStat$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">x64\ZlibStat$(Configuration)\Tmp\</IntDir>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">x64\ZlibStat$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">x64\ZlibStat$(Configuration)\Tmp\</IntDir>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" />
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
+    <OutDir>arm64\ZlibStat$(Configuration)\</OutDir>
+    <IntDir>arm64\ZlibStat$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
+    <OutDir>arm64\ZlibStat$(Configuration)\</OutDir>
+    <IntDir>arm64\ZlibStat$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'">
+    <OutDir>arm64\ZlibStat$(Configuration)\</OutDir>
+    <IntDir>arm64\ZlibStat$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <OutDir>arm\ZlibStat$(Configuration)\</OutDir>
+    <IntDir>arm\ZlibStat$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <OutDir>arm\ZlibStat$(Configuration)\</OutDir>
+    <IntDir>arm\ZlibStat$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'">
+    <OutDir>arm\ZlibStat$(Configuration)\</OutDir>
+    <IntDir>arm\ZlibStat$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibstat.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <DebugInformationFormat>OldStyle</DebugInformationFormat>
+    </ClCompile>
+    <ResourceCompile>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Lib>
+      <AdditionalOptions>/MACHINE:X86 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
+      <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibstat.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Lib>
+      <AdditionalOptions>/MACHINE:X86 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibstat.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Lib>
+      <AdditionalOptions>/MACHINE:X86 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
+      <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Midl>
+      <TargetEnvironment>X64</TargetEnvironment>
+    </Midl>
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibstat.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <DebugInformationFormat>OldStyle</DebugInformationFormat>
+    </ClCompile>
+    <ResourceCompile>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Lib>
+      <AdditionalOptions>/MACHINE:AMD64 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
+      <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
+    <Midl />
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibstat.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <DebugInformationFormat>OldStyle</DebugInformationFormat>
+    </ClCompile>
+    <ResourceCompile>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Lib>
+      <AdditionalOptions>/MACHINE:ARM64 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
+      <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <Midl />
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibstat.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <DebugInformationFormat>OldStyle</DebugInformationFormat>
+    </ClCompile>
+    <ResourceCompile>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Lib>
+      <AdditionalOptions>/MACHINE:ARM /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
+      <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Midl>
+      <TargetEnvironment>X64</TargetEnvironment>
+    </Midl>
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibstat.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Lib>
+      <AdditionalOptions>/MACHINE:AMD64 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
+    <Midl />
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibstat.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Lib>
+      <AdditionalOptions>/MACHINE:ARM64 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <Midl />
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibstat.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Lib>
+      <AdditionalOptions>/MACHINE:ARM /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">
+    <Midl>
+      <TargetEnvironment>X64</TargetEnvironment>
+    </Midl>
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibstat.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Lib>
+      <AdditionalOptions>/MACHINE:AMD64 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
+      <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'">
+    <Midl />
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibstat.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Lib>
+      <AdditionalOptions>/MACHINE:ARM64 /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
+      <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'">
+    <Midl />
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibstat.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Lib>
+      <AdditionalOptions>/MACHINE:ARM /NODEFAULTLIB %(AdditionalOptions)</AdditionalOptions>
+      <OutputFile>$(OutDir)zlibstat.lib</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\adler32.c" />
+    <ClCompile Include="..\..\..\compress.c" />
+    <ClCompile Include="..\..\..\crc32.c" />
+    <ClCompile Include="..\..\..\deflate.c" />
+    <ClCompile Include="..\..\..\gzclose.c" />
+    <ClCompile Include="..\..\..\gzlib.c" />
+    <ClCompile Include="..\..\..\gzread.c" />
+    <ClCompile Include="..\..\..\gzwrite.c" />
+    <ClCompile Include="..\..\..\infback.c" />
+    <ClCompile Include="..\..\..\inffast.c" />
+    <ClCompile Include="..\..\..\inflate.c" />
+    <ClCompile Include="..\..\..\inftrees.c" />
+    <ClCompile Include="..\..\minizip\ioapi.c" />
+    <ClCompile Include="..\..\..\trees.c" />
+    <ClCompile Include="..\..\..\uncompr.c" />
+    <ClCompile Include="..\..\minizip\unzip.c" />
+    <ClCompile Include="..\..\minizip\zip.c" />
+    <ClCompile Include="..\..\..\zutil.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ResourceCompile Include="zlib.rc" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="zlibvc.def" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/zlib/contrib/vstudio/vc17/zlibvc.def b/zlib/contrib/vstudio/vc17/zlibvc.def
new file mode 100644
index 0000000..53947cc
--- /dev/null
+++ b/zlib/contrib/vstudio/vc17/zlibvc.def
@@ -0,0 +1,158 @@
+LIBRARY
+; zlib data compression and ZIP file I/O library
+
+VERSION		1.3.1
+
+EXPORTS
+        adler32                                  @1
+        compress                                 @2
+        crc32                                    @3
+        deflate                                  @4
+        deflateCopy                              @5
+        deflateEnd                               @6
+        deflateInit2_                            @7
+        deflateInit_                             @8
+        deflateParams                            @9
+        deflateReset                             @10
+        deflateSetDictionary                     @11
+        gzclose                                  @12
+        gzdopen                                  @13
+        gzerror                                  @14
+        gzflush                                  @15
+        gzopen                                   @16
+        gzread                                   @17
+        gzwrite                                  @18
+        inflate                                  @19
+        inflateEnd                               @20
+        inflateInit2_                            @21
+        inflateInit_                             @22
+        inflateReset                             @23
+        inflateSetDictionary                     @24
+        inflateSync                              @25
+        uncompress                               @26
+        zlibVersion                              @27
+        gzprintf                                 @28
+        gzputc                                   @29
+        gzgetc                                   @30
+        gzseek                                   @31
+        gzrewind                                 @32
+        gztell                                   @33
+        gzeof                                    @34
+        gzsetparams                              @35
+        zError                                   @36
+        inflateSyncPoint                         @37
+        get_crc_table                            @38
+        compress2                                @39
+        gzputs                                   @40
+        gzgets                                   @41
+        inflateCopy                              @42
+        inflateBackInit_                         @43
+        inflateBack                              @44
+        inflateBackEnd                           @45
+        compressBound                            @46
+        deflateBound                             @47
+        gzclearerr                               @48
+        gzungetc                                 @49
+        zlibCompileFlags                         @50
+        deflatePrime                             @51
+        deflatePending                           @52
+
+        unzOpen                                  @61
+        unzClose                                 @62
+        unzGetGlobalInfo                         @63
+        unzGetCurrentFileInfo                    @64
+        unzGoToFirstFile                         @65
+        unzGoToNextFile                          @66
+        unzOpenCurrentFile                       @67
+        unzReadCurrentFile                       @68
+        unzOpenCurrentFile3                      @69
+        unztell                                  @70
+        unzeof                                   @71
+        unzCloseCurrentFile                      @72
+        unzGetGlobalComment                      @73
+        unzStringFileNameCompare                 @74
+        unzLocateFile                            @75
+        unzGetLocalExtrafield                    @76
+        unzOpen2                                 @77
+        unzOpenCurrentFile2                      @78
+        unzOpenCurrentFilePassword               @79
+
+        zipOpen                                  @80
+        zipOpenNewFileInZip                      @81
+        zipWriteInFileInZip                      @82
+        zipCloseFileInZip                        @83
+        zipClose                                 @84
+        zipOpenNewFileInZip2                     @86
+        zipCloseFileInZipRaw                     @87
+        zipOpen2                                 @88
+        zipOpenNewFileInZip3                     @89
+
+        unzGetFilePos                            @100
+        unzGoToFilePos                           @101
+
+        fill_win32_filefunc                      @110
+
+; zlibwapi v1.2.4 added:
+        fill_win32_filefunc64                   @111
+        fill_win32_filefunc64A                  @112
+        fill_win32_filefunc64W                  @113
+
+        unzOpen64                               @120
+        unzOpen2_64                             @121
+        unzGetGlobalInfo64                      @122
+        unzGetCurrentFileInfo64                 @124
+        unzGetCurrentFileZStreamPos64           @125
+        unztell64                               @126
+        unzGetFilePos64                         @127
+        unzGoToFilePos64                        @128
+
+        zipOpen64                               @130
+        zipOpen2_64                             @131
+        zipOpenNewFileInZip64                   @132
+        zipOpenNewFileInZip2_64                 @133
+        zipOpenNewFileInZip3_64                 @134
+        zipOpenNewFileInZip4_64                 @135
+        zipCloseFileInZipRaw64                  @136
+
+; zlib1 v1.2.4 added:
+        adler32_combine                         @140
+        crc32_combine                           @142
+        deflateSetHeader                        @144
+        deflateTune                             @145
+        gzbuffer                                @146
+        gzclose_r                               @147
+        gzclose_w                               @148
+        gzdirect                                @149
+        gzoffset                                @150
+        inflateGetHeader                        @156
+        inflateMark                             @157
+        inflatePrime                            @158
+        inflateReset2                           @159
+        inflateUndermine                        @160
+
+; zlib1 v1.2.6 added:
+        gzgetc_                                 @161
+        inflateResetKeep                        @163
+        deflateResetKeep                        @164
+
+; zlib1 v1.2.7 added:
+        gzopen_w                                @165
+
+; zlib1 v1.2.8 added:
+        inflateGetDictionary                    @166
+        gzvprintf                               @167
+
+; zlib1 v1.2.9 added:
+        inflateCodesUsed                        @168
+        inflateValidate                         @169
+        uncompress2                             @170
+        gzfread                                 @171
+        gzfwrite                                @172
+        deflateGetDictionary                    @173
+        adler32_z                               @174
+        crc32_z                                 @175
+
+; zlib1 v1.2.12 added:
+		crc32_combine_gen						@176
+		crc32_combine_gen64						@177
+		crc32_combine_op						@178
diff --git a/zlib/contrib/vstudio/vc17/zlibvc.sln b/zlib/contrib/vstudio/vc17/zlibvc.sln
new file mode 100644
index 0000000..67896b7
--- /dev/null
+++ b/zlib/contrib/vstudio/vc17/zlibvc.sln
@@ -0,0 +1,179 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.4.33015.44
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zlibvc", "zlibvc.vcxproj", "{8FD826F8-3739-44E6-8CC8-997122E53B8D}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zlibstat", "zlibstat.vcxproj", "{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "testzlib", "testzlib.vcxproj", "{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "testzlibdll", "testzlibdll.vcxproj", "{C52F9E7B-498A-42BE-8DB4-85A15694366A}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "minizip", "minizip.vcxproj", "{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "miniunz", "miniunz.vcxproj", "{C52F9E7B-498A-42BE-8DB4-85A15694382A}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|ARM = Debug|ARM
+		Debug|ARM64 = Debug|ARM64
+		Debug|Win32 = Debug|Win32
+		Debug|x64 = Debug|x64
+		Release|ARM = Release|ARM
+		Release|ARM64 = Release|ARM64
+		Release|Win32 = Release|Win32
+		Release|x64 = Release|x64
+		ReleaseWithoutAsm|ARM = ReleaseWithoutAsm|ARM
+		ReleaseWithoutAsm|ARM64 = ReleaseWithoutAsm|ARM64
+		ReleaseWithoutAsm|Win32 = ReleaseWithoutAsm|Win32
+		ReleaseWithoutAsm|x64 = ReleaseWithoutAsm|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Debug|ARM.ActiveCfg = Debug|ARM
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Debug|ARM.Build.0 = Debug|ARM
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Debug|ARM64.ActiveCfg = Debug|ARM64
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Debug|ARM64.Build.0 = Debug|ARM64
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Debug|Win32.ActiveCfg = Debug|Win32
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Debug|Win32.Build.0 = Debug|Win32
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Debug|x64.ActiveCfg = Debug|x64
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Debug|x64.Build.0 = Debug|x64
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Release|ARM.ActiveCfg = Release|ARM
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Release|ARM.Build.0 = Release|ARM
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Release|ARM64.ActiveCfg = Release|ARM64
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Release|ARM64.Build.0 = Release|ARM64
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Release|Win32.ActiveCfg = Release|Win32
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Release|Win32.Build.0 = Release|Win32
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Release|x64.ActiveCfg = Release|x64
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.Release|x64.Build.0 = Release|x64
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.ReleaseWithoutAsm|ARM.ActiveCfg = ReleaseWithoutAsm|ARM
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.ReleaseWithoutAsm|ARM.Build.0 = ReleaseWithoutAsm|ARM
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.ReleaseWithoutAsm|ARM64.ActiveCfg = ReleaseWithoutAsm|ARM64
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.ReleaseWithoutAsm|ARM64.Build.0 = ReleaseWithoutAsm|ARM64
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.ReleaseWithoutAsm|Win32.ActiveCfg = ReleaseWithoutAsm|Win32
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.ReleaseWithoutAsm|Win32.Build.0 = ReleaseWithoutAsm|Win32
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.ReleaseWithoutAsm|x64.ActiveCfg = ReleaseWithoutAsm|x64
+		{8FD826F8-3739-44E6-8CC8-997122E53B8D}.ReleaseWithoutAsm|x64.Build.0 = ReleaseWithoutAsm|x64
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|ARM.ActiveCfg = Debug|ARM
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|ARM.Build.0 = Debug|ARM
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|ARM64.ActiveCfg = Debug|ARM64
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|ARM64.Build.0 = Debug|ARM64
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|Win32.ActiveCfg = Debug|Win32
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|Win32.Build.0 = Debug|Win32
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|x64.ActiveCfg = Debug|x64
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|x64.Build.0 = Debug|x64
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|ARM.ActiveCfg = Release|ARM
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|ARM.Build.0 = Release|ARM
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|ARM64.ActiveCfg = Release|ARM64
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|ARM64.Build.0 = Release|ARM64
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|Win32.ActiveCfg = Release|Win32
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|Win32.Build.0 = Release|Win32
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|x64.ActiveCfg = Release|x64
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|x64.Build.0 = Release|x64
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.ReleaseWithoutAsm|ARM.ActiveCfg = ReleaseWithoutAsm|ARM
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.ReleaseWithoutAsm|ARM.Build.0 = ReleaseWithoutAsm|ARM
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.ReleaseWithoutAsm|ARM64.ActiveCfg = ReleaseWithoutAsm|ARM64
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.ReleaseWithoutAsm|ARM64.Build.0 = ReleaseWithoutAsm|ARM64
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.ReleaseWithoutAsm|Win32.ActiveCfg = ReleaseWithoutAsm|Win32
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.ReleaseWithoutAsm|Win32.Build.0 = ReleaseWithoutAsm|Win32
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.ReleaseWithoutAsm|x64.ActiveCfg = ReleaseWithoutAsm|x64
+		{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.ReleaseWithoutAsm|x64.Build.0 = ReleaseWithoutAsm|x64
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Debug|ARM.ActiveCfg = Debug|ARM
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Debug|ARM.Build.0 = Debug|ARM
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Debug|ARM64.ActiveCfg = Debug|ARM64
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Debug|ARM64.Build.0 = Debug|ARM64
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Debug|Win32.ActiveCfg = Debug|Win32
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Debug|Win32.Build.0 = Debug|Win32
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Debug|x64.ActiveCfg = Debug|x64
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Debug|x64.Build.0 = Debug|x64
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Release|ARM.ActiveCfg = Release|ARM
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Release|ARM.Build.0 = Release|ARM
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Release|ARM64.ActiveCfg = Release|ARM64
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Release|ARM64.Build.0 = Release|ARM64
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Release|Win32.ActiveCfg = Release|Win32
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Release|Win32.Build.0 = Release|Win32
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Release|x64.ActiveCfg = Release|x64
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.Release|x64.Build.0 = Release|x64
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|ARM.ActiveCfg = ReleaseWithoutAsm|ARM
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|ARM.Build.0 = ReleaseWithoutAsm|ARM
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|ARM64.ActiveCfg = ReleaseWithoutAsm|ARM64
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|ARM64.Build.0 = ReleaseWithoutAsm|ARM64
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|Win32.ActiveCfg = ReleaseWithoutAsm|Win32
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|Win32.Build.0 = ReleaseWithoutAsm|Win32
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|x64.ActiveCfg = ReleaseWithoutAsm|x64
+		{AA6666AA-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|x64.Build.0 = ReleaseWithoutAsm|x64
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Debug|ARM.ActiveCfg = Debug|ARM
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Debug|ARM.Build.0 = Debug|ARM
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Debug|ARM64.ActiveCfg = Debug|ARM64
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Debug|ARM64.Build.0 = Debug|ARM64
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Debug|Win32.ActiveCfg = Debug|Win32
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Debug|Win32.Build.0 = Debug|Win32
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Debug|x64.ActiveCfg = Debug|x64
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Debug|x64.Build.0 = Debug|x64
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Release|ARM.ActiveCfg = Release|ARM
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Release|ARM.Build.0 = Release|ARM
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Release|ARM64.ActiveCfg = Release|ARM64
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Release|ARM64.Build.0 = Release|ARM64
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Release|Win32.ActiveCfg = Release|Win32
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Release|Win32.Build.0 = Release|Win32
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Release|x64.ActiveCfg = Release|x64
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.Release|x64.Build.0 = Release|x64
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.ReleaseWithoutAsm|ARM.ActiveCfg = Release|ARM
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.ReleaseWithoutAsm|ARM.Build.0 = Release|ARM
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.ReleaseWithoutAsm|ARM64.ActiveCfg = Release|ARM64
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.ReleaseWithoutAsm|ARM64.Build.0 = Release|ARM64
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.ReleaseWithoutAsm|Win32.ActiveCfg = Release|Win32
+		{C52F9E7B-498A-42BE-8DB4-85A15694366A}.ReleaseWithoutAsm|x64.ActiveCfg = Release|x64
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Debug|ARM.ActiveCfg = Debug|ARM
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Debug|ARM.Build.0 = Debug|ARM
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Debug|ARM64.ActiveCfg = Debug|ARM64
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Debug|ARM64.Build.0 = Debug|ARM64
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Debug|Win32.ActiveCfg = Debug|Win32
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Debug|Win32.Build.0 = Debug|Win32
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Debug|x64.ActiveCfg = Debug|x64
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Debug|x64.Build.0 = Debug|x64
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Release|ARM.ActiveCfg = Release|ARM
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Release|ARM.Build.0 = Release|ARM
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Release|ARM64.ActiveCfg = Release|ARM64
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Release|ARM64.Build.0 = Release|ARM64
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Release|Win32.ActiveCfg = Release|Win32
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Release|Win32.Build.0 = Release|Win32
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Release|x64.ActiveCfg = Release|x64
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.Release|x64.Build.0 = Release|x64
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|ARM.ActiveCfg = Release|ARM
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|ARM.Build.0 = Release|ARM
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|ARM64.ActiveCfg = Release|ARM64
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|ARM64.Build.0 = Release|ARM64
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|Win32.ActiveCfg = Release|Win32
+		{48CDD9DC-E09F-4135-9C0C-4FE50C3C654B}.ReleaseWithoutAsm|x64.ActiveCfg = Release|x64
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Debug|ARM.ActiveCfg = Debug|ARM
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Debug|ARM.Build.0 = Debug|ARM
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Debug|ARM64.ActiveCfg = Debug|ARM64
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Debug|ARM64.Build.0 = Debug|ARM64
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Debug|Win32.ActiveCfg = Debug|Win32
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Debug|Win32.Build.0 = Debug|Win32
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Debug|x64.ActiveCfg = Debug|x64
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Debug|x64.Build.0 = Debug|x64
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Release|ARM.ActiveCfg = Release|ARM
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Release|ARM.Build.0 = Release|ARM
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Release|ARM64.ActiveCfg = Release|ARM64
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Release|ARM64.Build.0 = Release|ARM64
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Release|Win32.ActiveCfg = Release|Win32
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Release|Win32.Build.0 = Release|Win32
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Release|x64.ActiveCfg = Release|x64
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.Release|x64.Build.0 = Release|x64
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.ReleaseWithoutAsm|ARM.ActiveCfg = Release|ARM
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.ReleaseWithoutAsm|ARM.Build.0 = Release|ARM
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.ReleaseWithoutAsm|ARM64.ActiveCfg = Release|ARM64
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.ReleaseWithoutAsm|ARM64.Build.0 = Release|ARM64
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.ReleaseWithoutAsm|Win32.ActiveCfg = Release|Win32
+		{C52F9E7B-498A-42BE-8DB4-85A15694382A}.ReleaseWithoutAsm|x64.ActiveCfg = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {EAA58685-56D9-43F2-8703-FD2CB020745E}
+	EndGlobalSection
+EndGlobal
diff --git a/zlib/contrib/vstudio/vc17/zlibvc.vcxproj b/zlib/contrib/vstudio/vc17/zlibvc.vcxproj
new file mode 100644
index 0000000..10a7a90
--- /dev/null
+++ b/zlib/contrib/vstudio/vc17/zlibvc.vcxproj
@@ -0,0 +1,875 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|ARM">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|ARM64">
+      <Configuration>Debug</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="ReleaseWithoutAsm|ARM">
+      <Configuration>ReleaseWithoutAsm</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="ReleaseWithoutAsm|ARM64">
+      <Configuration>ReleaseWithoutAsm</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="ReleaseWithoutAsm|Win32">
+      <Configuration>ReleaseWithoutAsm</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="ReleaseWithoutAsm|x64">
+      <Configuration>ReleaseWithoutAsm</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM">
+      <Configuration>Release</Configuration>
+      <Platform>ARM</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|ARM64">
+      <Configuration>Release</Configuration>
+      <Platform>ARM64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{8FD826F8-3739-44E6-8CC8-997122E53B8D}</ProjectGuid>
+    <WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" Label="Configuration">
+    <ConfigurationType>DynamicLibrary</ConfigurationType>
+    <UseOfMfc>false</UseOfMfc>
+    <PlatformToolset>v143</PlatformToolset>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <_ProjectFileVersion>10.0.30128.1</_ProjectFileVersion>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">x86\ZlibDll$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">x86\ZlibDll$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">x86\ZlibDll$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">x86\ZlibDll$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">false</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">x86\ZlibDll$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">x86\ZlibDll$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">x64\ZlibDll$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">x64\ZlibDll$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">true</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">true</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">x64\ZlibDll$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">x64\ZlibDll$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">false</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'">false</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'">false</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'">false</GenerateManifest>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">x64\ZlibDll$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">x64\ZlibDll$(Configuration)\Tmp\</IntDir>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">false</LinkIncremental>
+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">false</LinkIncremental>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">false</GenerateManifest>
+    <GenerateManifest Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">false</GenerateManifest>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" />
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" />
+    <TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">zlibwapi</TargetName>
+    <TargetName Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">zlibwapi</TargetName>
+    <TargetName Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">zlibwapi</TargetName>
+    <TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">zlibwapi</TargetName>
+    <TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">zlibwapi</TargetName>
+    <TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">zlibwapi</TargetName>
+    <TargetName Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">zlibwapi</TargetName>
+    <TargetName Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'">zlibwapi</TargetName>
+    <TargetName Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'">zlibwapi</TargetName>
+    <TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">zlibwapi</TargetName>
+    <TargetName Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">zlibwapi</TargetName>
+    <TargetName Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">zlibwapi</TargetName>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
+    <OutDir>arm64\ZlibDll$(Configuration)\</OutDir>
+    <IntDir>arm64\ZlibDll$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <OutDir>arm\ZlibDll$(Configuration)\</OutDir>
+    <IntDir>arm\ZlibDll$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
+    <OutDir>arm64\ZlibDll$(Configuration)\</OutDir>
+    <IntDir>arm64\ZlibDll$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'">
+    <OutDir>arm64\ZlibDll$(Configuration)\</OutDir>
+    <IntDir>arm64\ZlibDll$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <OutDir>arm\ZlibDll$(Configuration)\</OutDir>
+    <IntDir>arm\ZlibDll$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'">
+    <OutDir>arm\ZlibDll$(Configuration)\</OutDir>
+    <IntDir>arm\ZlibDll$(Configuration)\Tmp\</IntDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Midl>
+      <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MkTypLibCompatible>true</MkTypLibCompatible>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <TargetEnvironment>Win32</TargetEnvironment>
+      <TypeLibraryName>$(OutDir)zlibvc.tlb</TypeLibraryName>
+    </Midl>
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibvc.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <BrowseInformation>
+      </BrowseInformation>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <ResourceCompile>
+      <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Link>
+      <AdditionalOptions>/MACHINE:I386 %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)zlibwapi.pdb</ProgramDatabaseFile>
+      <GenerateMapFile>true</GenerateMapFile>
+      <MapFileName>$(OutDir)zlibwapi.map</MapFileName>
+      <SubSystem>Windows</SubSystem>
+      <RandomizedBaseAddress>false</RandomizedBaseAddress>
+      <DataExecutionPrevention>
+      </DataExecutionPrevention>
+      <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|Win32'">
+    <Midl>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MkTypLibCompatible>true</MkTypLibCompatible>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <TargetEnvironment>Win32</TargetEnvironment>
+      <TypeLibraryName>$(OutDir)zlibvc.tlb</TypeLibraryName>
+    </Midl>
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibvc.pch</PrecompiledHeaderOutputFile>
+      <AssemblerOutput>All</AssemblerOutput>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <BrowseInformation>
+      </BrowseInformation>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Link>
+      <AdditionalOptions>/MACHINE:I386 %(AdditionalOptions)</AdditionalOptions>
+      <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
+      <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
+      <ProgramDatabaseFile>$(OutDir)zlibwapi.pdb</ProgramDatabaseFile>
+      <GenerateMapFile>true</GenerateMapFile>
+      <MapFileName>$(OutDir)zlibwapi.map</MapFileName>
+      <SubSystem>Windows</SubSystem>
+      <RandomizedBaseAddress>false</RandomizedBaseAddress>
+      <DataExecutionPrevention>
+      </DataExecutionPrevention>
+      <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Midl>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MkTypLibCompatible>true</MkTypLibCompatible>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <TargetEnvironment>Win32</TargetEnvironment>
+      <TypeLibraryName>$(OutDir)zlibvc.tlb</TypeLibraryName>
+    </Midl>
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibvc.pch</PrecompiledHeaderOutputFile>
+      <AssemblerOutput>All</AssemblerOutput>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <BrowseInformation>
+      </BrowseInformation>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Link>
+      <AdditionalOptions>/MACHINE:I386 %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
+      <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
+      <ProgramDatabaseFile>$(OutDir)zlibwapi.pdb</ProgramDatabaseFile>
+      <GenerateMapFile>true</GenerateMapFile>
+      <MapFileName>$(OutDir)zlibwapi.map</MapFileName>
+      <SubSystem>Windows</SubSystem>
+      <RandomizedBaseAddress>false</RandomizedBaseAddress>
+      <DataExecutionPrevention>
+      </DataExecutionPrevention>
+      <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
+      <ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Midl>
+      <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MkTypLibCompatible>true</MkTypLibCompatible>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <TargetEnvironment>X64</TargetEnvironment>
+      <TypeLibraryName>$(OutDir)zlibvc.tlb</TypeLibraryName>
+    </Midl>
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibvc.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <BrowseInformation>
+      </BrowseInformation>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <ResourceCompile>
+      <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)zlibwapi.pdb</ProgramDatabaseFile>
+      <GenerateMapFile>true</GenerateMapFile>
+      <MapFileName>$(OutDir)zlibwapi.map</MapFileName>
+      <SubSystem>Windows</SubSystem>
+      <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
+      <TargetMachine>MachineX64</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">
+    <Midl>
+      <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MkTypLibCompatible>true</MkTypLibCompatible>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <TypeLibraryName>$(OutDir)zlibvc.tlb</TypeLibraryName>
+    </Midl>
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibvc.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <BrowseInformation>
+      </BrowseInformation>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <ResourceCompile>
+      <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)zlibwapi.pdb</ProgramDatabaseFile>
+      <GenerateMapFile>true</GenerateMapFile>
+      <MapFileName>$(OutDir)zlibwapi.map</MapFileName>
+      <SubSystem>Windows</SubSystem>
+      <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
+    <Midl>
+      <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MkTypLibCompatible>true</MkTypLibCompatible>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <TypeLibraryName>$(OutDir)zlibvc.tlb</TypeLibraryName>
+    </Midl>
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibvc.pch</PrecompiledHeaderOutputFile>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <BrowseInformation>
+      </BrowseInformation>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <ResourceCompile>
+      <PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <ProgramDatabaseFile>$(OutDir)zlibwapi.pdb</ProgramDatabaseFile>
+      <GenerateMapFile>true</GenerateMapFile>
+      <MapFileName>$(OutDir)zlibwapi.map</MapFileName>
+      <SubSystem>Windows</SubSystem>
+      <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|x64'">
+    <Midl>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MkTypLibCompatible>true</MkTypLibCompatible>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <TargetEnvironment>X64</TargetEnvironment>
+      <TypeLibraryName>$(OutDir)zlibvc.tlb</TypeLibraryName>
+    </Midl>
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibvc.pch</PrecompiledHeaderOutputFile>
+      <AssemblerOutput>All</AssemblerOutput>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <BrowseInformation>
+      </BrowseInformation>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Link>
+      <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
+      <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
+      <ProgramDatabaseFile>$(OutDir)zlibwapi.pdb</ProgramDatabaseFile>
+      <GenerateMapFile>true</GenerateMapFile>
+      <MapFileName>$(OutDir)zlibwapi.map</MapFileName>
+      <SubSystem>Windows</SubSystem>
+      <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
+      <TargetMachine>MachineX64</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM64'">
+    <Midl>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MkTypLibCompatible>true</MkTypLibCompatible>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <TypeLibraryName>$(OutDir)zlibvc.tlb</TypeLibraryName>
+    </Midl>
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibvc.pch</PrecompiledHeaderOutputFile>
+      <AssemblerOutput>All</AssemblerOutput>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <BrowseInformation>
+      </BrowseInformation>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Link>
+      <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
+      <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
+      <ProgramDatabaseFile>$(OutDir)zlibwapi.pdb</ProgramDatabaseFile>
+      <GenerateMapFile>true</GenerateMapFile>
+      <MapFileName>$(OutDir)zlibwapi.map</MapFileName>
+      <SubSystem>Windows</SubSystem>
+      <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='ReleaseWithoutAsm|ARM'">
+    <Midl>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MkTypLibCompatible>true</MkTypLibCompatible>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <TypeLibraryName>$(OutDir)zlibvc.tlb</TypeLibraryName>
+    </Midl>
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibvc.pch</PrecompiledHeaderOutputFile>
+      <AssemblerOutput>All</AssemblerOutput>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <BrowseInformation>
+      </BrowseInformation>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Link>
+      <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
+      <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
+      <ProgramDatabaseFile>$(OutDir)zlibwapi.pdb</ProgramDatabaseFile>
+      <GenerateMapFile>true</GenerateMapFile>
+      <MapFileName>$(OutDir)zlibwapi.map</MapFileName>
+      <SubSystem>Windows</SubSystem>
+      <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Midl>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MkTypLibCompatible>true</MkTypLibCompatible>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <TargetEnvironment>X64</TargetEnvironment>
+      <TypeLibraryName>$(OutDir)zlibvc.tlb</TypeLibraryName>
+    </Midl>
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibvc.pch</PrecompiledHeaderOutputFile>
+      <AssemblerOutput>All</AssemblerOutput>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <BrowseInformation>
+      </BrowseInformation>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
+      <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
+      <ProgramDatabaseFile>$(OutDir)zlibwapi.pdb</ProgramDatabaseFile>
+      <GenerateMapFile>true</GenerateMapFile>
+      <MapFileName>$(OutDir)zlibwapi.map</MapFileName>
+      <SubSystem>Windows</SubSystem>
+      <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
+      <TargetMachine>MachineX64</TargetMachine>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">
+    <Midl>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MkTypLibCompatible>true</MkTypLibCompatible>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <TypeLibraryName>$(OutDir)zlibvc.tlb</TypeLibraryName>
+    </Midl>
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibvc.pch</PrecompiledHeaderOutputFile>
+      <AssemblerOutput>All</AssemblerOutput>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <BrowseInformation>
+      </BrowseInformation>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
+      <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
+      <ProgramDatabaseFile>$(OutDir)zlibwapi.pdb</ProgramDatabaseFile>
+      <GenerateMapFile>true</GenerateMapFile>
+      <MapFileName>$(OutDir)zlibwapi.map</MapFileName>
+      <SubSystem>Windows</SubSystem>
+      <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
+    <Midl>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MkTypLibCompatible>true</MkTypLibCompatible>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <TypeLibraryName>$(OutDir)zlibvc.tlb</TypeLibraryName>
+    </Midl>
+    <ClCompile>
+      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
+      <AdditionalIncludeDirectories>..\..\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN32;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <StringPooling>true</StringPooling>
+      <ExceptionHandling>
+      </ExceptionHandling>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <BufferSecurityCheck>false</BufferSecurityCheck>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <PrecompiledHeaderOutputFile>$(IntDir)zlibvc.pch</PrecompiledHeaderOutputFile>
+      <AssemblerOutput>All</AssemblerOutput>
+      <AssemblerListingLocation>$(IntDir)</AssemblerListingLocation>
+      <ObjectFileName>$(IntDir)</ObjectFileName>
+      <ProgramDataBaseFileName>$(OutDir)</ProgramDataBaseFileName>
+      <BrowseInformation>
+      </BrowseInformation>
+      <WarningLevel>Level3</WarningLevel>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+    </ClCompile>
+    <ResourceCompile>
+      <PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <Culture>0x040c</Culture>
+    </ResourceCompile>
+    <Link>
+      <AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
+      <OutputFile>$(OutDir)zlibwapi.dll</OutputFile>
+      <SuppressStartupBanner>true</SuppressStartupBanner>
+      <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
+      <ModuleDefinitionFile>.\zlibvc.def</ModuleDefinitionFile>
+      <ProgramDatabaseFile>$(OutDir)zlibwapi.pdb</ProgramDatabaseFile>
+      <GenerateMapFile>true</GenerateMapFile>
+      <MapFileName>$(OutDir)zlibwapi.map</MapFileName>
+      <SubSystem>Windows</SubSystem>
+      <ImportLibrary>$(OutDir)zlibwapi.lib</ImportLibrary>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\..\..\adler32.c" />
+    <ClCompile Include="..\..\..\compress.c" />
+    <ClCompile Include="..\..\..\crc32.c" />
+    <ClCompile Include="..\..\..\deflate.c" />
+    <ClCompile Include="..\..\..\gzclose.c" />
+    <ClCompile Include="..\..\..\gzlib.c" />
+    <ClCompile Include="..\..\..\gzread.c" />
+    <ClCompile Include="..\..\..\gzwrite.c" />
+    <ClCompile Include="..\..\..\infback.c" />
+    <ClCompile Include="..\..\..\inffast.c" />
+    <ClCompile Include="..\..\..\inflate.c" />
+    <ClCompile Include="..\..\..\inftrees.c" />
+    <ClCompile Include="..\..\minizip\ioapi.c" />
+    <ClCompile Include="..\..\minizip\iowin32.c" />
+    <ClCompile Include="..\..\..\trees.c" />
+    <ClCompile Include="..\..\..\uncompr.c" />
+    <ClCompile Include="..\..\minizip\unzip.c">
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">ZLIB_INTERNAL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|x64'">%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|x64'">ZLIB_INTERNAL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">ZLIB_INTERNAL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">ZLIB_INTERNAL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <ClCompile Include="..\..\minizip\zip.c">
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">ZLIB_INTERNAL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|x64'">%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|x64'">ZLIB_INTERNAL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">ZLIB_INTERNAL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">ZLIB_INTERNAL;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+    <ClCompile Include="..\..\..\zutil.c" />
+  </ItemGroup>
+  <ItemGroup>
+    <ResourceCompile Include="zlib.rc" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="zlibvc.def" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClInclude Include="..\..\..\deflate.h" />
+    <ClInclude Include="..\..\..\infblock.h" />
+    <ClInclude Include="..\..\..\infcodes.h" />
+    <ClInclude Include="..\..\..\inffast.h" />
+    <ClInclude Include="..\..\..\inftrees.h" />
+    <ClInclude Include="..\..\..\infutil.h" />
+    <ClInclude Include="..\..\..\zconf.h" />
+    <ClInclude Include="..\..\..\zlib.h" />
+    <ClInclude Include="..\..\..\zutil.h" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
\ No newline at end of file
diff --git a/zlib/contrib/vstudio/vc9/miniunz.vcproj b/zlib/contrib/vstudio/vc9/miniunz.vcproj
index 7da32b9..cc3d13a 100644
--- a/zlib/contrib/vstudio/vc9/miniunz.vcproj
+++ b/zlib/contrib/vstudio/vc9/miniunz.vcproj
@@ -542,7 +542,7 @@
 	<Files>

 		<Filter

 			Name="Source Files"

-			Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm"

+			Filter="cpp;c;cxx;def;odl;idl;hpj;bat"

 			>

 			<File

 				RelativePath="..\..\minizip\miniunz.c"

diff --git a/zlib/contrib/vstudio/vc9/minizip.vcproj b/zlib/contrib/vstudio/vc9/minizip.vcproj
index e57e07d..fecd9dd 100644
--- a/zlib/contrib/vstudio/vc9/minizip.vcproj
+++ b/zlib/contrib/vstudio/vc9/minizip.vcproj
@@ -539,7 +539,7 @@
 	<Files>

 		<Filter

 			Name="Source Files"

-			Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm"

+			Filter="cpp;c;cxx;def;odl;idl;hpj;bat"

 			>

 			<File

 				RelativePath="..\..\minizip\minizip.c"

diff --git a/zlib/contrib/vstudio/vc9/testzlib.vcproj b/zlib/contrib/vstudio/vc9/testzlib.vcproj
index 9cb0bf8..ff69640 100644
--- a/zlib/contrib/vstudio/vc9/testzlib.vcproj
+++ b/zlib/contrib/vstudio/vc9/testzlib.vcproj
@@ -48,7 +48,7 @@
 				Name="VCCLCompilerTool"

 				Optimization="0"

 				AdditionalIncludeDirectories="..\..\.."

-				PreprocessorDefinitions="ASMV;ASMINF;WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS"

+				PreprocessorDefinitions="WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS"

 				MinimalRebuild="true"

 				BasicRuntimeChecks="0"

 				RuntimeLibrary="1"

@@ -71,7 +71,6 @@
 			/>

 			<Tool

 				Name="VCLinkerTool"

-				AdditionalDependencies="..\..\masmx86\match686.obj ..\..\masmx86\inffas32.obj"

 				OutputFile="$(OutDir)/testzlib.exe"

 				LinkIncremental="2"

 				GenerateManifest="false"

@@ -128,7 +127,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				AdditionalIncludeDirectories="..\..\.."

-				PreprocessorDefinitions="ASMV;ASMINF;WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS"

+				PreprocessorDefinitions="WIN32;ZLIB_WINAPI;_DEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS"

 				BasicRuntimeChecks="0"

 				RuntimeLibrary="3"

 				BufferSecurityCheck="false"

@@ -145,7 +144,6 @@
 			/>

 			<Tool

 				Name="VCLinkerTool"

-				AdditionalDependencies="..\..\masmx64\gvmat64.obj ..\..\masmx64\inffasx64.obj"

 				GenerateManifest="false"

 			/>

 			<Tool

@@ -517,7 +515,7 @@
 				InlineFunctionExpansion="1"

 				OmitFramePointers="true"

 				AdditionalIncludeDirectories="..\..\.."

-				PreprocessorDefinitions="ASMV;ASMINF;WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS"

+				PreprocessorDefinitions="WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS"

 				StringPooling="true"

 				BasicRuntimeChecks="0"

 				RuntimeLibrary="0"

@@ -540,7 +538,6 @@
 			/>

 			<Tool

 				Name="VCLinkerTool"

-				AdditionalDependencies="..\..\masmx86\match686.obj ..\..\masmx86\inffas32.obj"

 				OutputFile="$(OutDir)/testzlib.exe"

 				LinkIncremental="1"

 				GenerateManifest="false"

@@ -600,7 +597,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				AdditionalIncludeDirectories="..\..\.."

-				PreprocessorDefinitions="ASMV;ASMINF;WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS"

+				PreprocessorDefinitions="WIN32;ZLIB_WINAPI;NDEBUG;_CONSOLE;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS"

 				BasicRuntimeChecks="0"

 				RuntimeLibrary="0"

 				BufferSecurityCheck="false"

@@ -617,7 +614,6 @@
 			/>

 			<Tool

 				Name="VCLinkerTool"

-				AdditionalDependencies="..\..\masmx64\gvmat64.obj ..\..\masmx64\inffasx64.obj"

 				GenerateManifest="false"

 			/>

 			<Tool

@@ -733,7 +729,7 @@
 	<Files>

 		<Filter

 			Name="Source Files"

-			Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm"

+			Filter="cpp;c;cxx;def;odl;idl;hpj;bat"

 			>

 			<File

 				RelativePath="..\..\..\adler32.c"

@@ -756,58 +752,6 @@
 				>

 			</File>

 			<File

-				RelativePath="..\..\masmx64\inffas8664.c"

-				>

-				<FileConfiguration

-					Name="Debug|Win32"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="Debug|Itanium"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="ReleaseWithoutAsm|Win32"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="ReleaseWithoutAsm|Itanium"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="Release|Win32"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="Release|Itanium"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-			</File>

-			<File

 				RelativePath="..\..\..\inffast.c"

 				>

 			</File>

diff --git a/zlib/contrib/vstudio/vc9/testzlibdll.vcproj b/zlib/contrib/vstudio/vc9/testzlibdll.vcproj
index b1ddde0..6448b49 100644
--- a/zlib/contrib/vstudio/vc9/testzlibdll.vcproj
+++ b/zlib/contrib/vstudio/vc9/testzlibdll.vcproj
@@ -542,7 +542,7 @@
 	<Files>

 		<Filter

 			Name="Source Files"

-			Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm"

+			Filter="cpp;c;cxx;def;odl;idl;hpj;bat"

 			>

 			<File

 				RelativePath="..\..\testzlib\testzlib.c"

diff --git a/zlib/contrib/vstudio/vc9/zlib.rc b/zlib/contrib/vstudio/vc9/zlib.rc
index fee177a..856bd11 100644
--- a/zlib/contrib/vstudio/vc9/zlib.rc
+++ b/zlib/contrib/vstudio/vc9/zlib.rc
@@ -2,8 +2,8 @@
 

 #define IDR_VERSION1  1

 IDR_VERSION1	VERSIONINFO	MOVEABLE IMPURE LOADONCALL DISCARDABLE

-  FILEVERSION	 1, 2, 11, 0

-  PRODUCTVERSION 1, 2, 11, 0

+  FILEVERSION	 1, 3, 1, 0

+  PRODUCTVERSION 1, 3, 1, 0

   FILEFLAGSMASK	VS_FFI_FILEFLAGSMASK

   FILEFLAGS	0

   FILEOS	VOS_DOS_WINDOWS32

@@ -17,12 +17,12 @@ BEGIN
 

     BEGIN

       VALUE "FileDescription", "zlib data compression and ZIP file I/O library\0"

-      VALUE "FileVersion",	"1.2.11\0"

+      VALUE "FileVersion",	"1.3.1\0"

       VALUE "InternalName",	"zlib\0"

       VALUE "OriginalFilename",	"zlibwapi.dll\0"

       VALUE "ProductName",	"ZLib.DLL\0"

       VALUE "Comments","DLL support by Alessandro Iacopetti & Gilles Vollant\0"

-      VALUE "LegalCopyright", "(C) 1995-2017 Jean-loup Gailly & Mark Adler\0"

+      VALUE "LegalCopyright", "(C) 1995-2024 Jean-loup Gailly & Mark Adler\0"

     END

   END

   BLOCK "VarFileInfo"

diff --git a/zlib/contrib/vstudio/vc9/zlibstat.vcproj b/zlib/contrib/vstudio/vc9/zlibstat.vcproj
index 61c76c7..1bf1563 100644
--- a/zlib/contrib/vstudio/vc9/zlibstat.vcproj
+++ b/zlib/contrib/vstudio/vc9/zlibstat.vcproj
@@ -47,7 +47,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				Optimization="0"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

+				AdditionalIncludeDirectories="..\..\.."

 				PreprocessorDefinitions="WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS"

 				ExceptionHandling="0"

 				RuntimeLibrary="1"

@@ -121,7 +121,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				Optimization="0"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

+				AdditionalIncludeDirectories="..\..\.."

 				PreprocessorDefinitions="ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64"

 				ExceptionHandling="0"

 				RuntimeLibrary="3"

@@ -195,7 +195,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				Optimization="0"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

+				AdditionalIncludeDirectories="..\..\.."

 				PreprocessorDefinitions="ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64"

 				ExceptionHandling="0"

 				RuntimeLibrary="3"

@@ -268,8 +268,8 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				InlineFunctionExpansion="1"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

-				PreprocessorDefinitions="WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ASMV;ASMINF"

+				AdditionalIncludeDirectories="..\..\.."

+				PreprocessorDefinitions="WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS"

 				StringPooling="true"

 				ExceptionHandling="0"

 				RuntimeLibrary="0"

@@ -295,7 +295,6 @@
 			<Tool

 				Name="VCLibrarianTool"

 				AdditionalOptions="/MACHINE:X86 /NODEFAULTLIB"

-				AdditionalDependencies="..\..\masmx86\match686.obj ..\..\masmx86\inffas32.obj "

 				OutputFile="$(OutDir)\zlibstat.lib"

 				SuppressStartupBanner="true"

 			/>

@@ -343,8 +342,8 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				InlineFunctionExpansion="1"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

-				PreprocessorDefinitions="ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ASMV;ASMINF;WIN64"

+				AdditionalIncludeDirectories="..\..\.."

+				PreprocessorDefinitions="ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64"

 				StringPooling="true"

 				ExceptionHandling="0"

 				RuntimeLibrary="2"

@@ -370,7 +369,6 @@
 			<Tool

 				Name="VCLibrarianTool"

 				AdditionalOptions="/MACHINE:AMD64 /NODEFAULTLIB"

-				AdditionalDependencies="..\..\masmx64\gvmat64.obj ..\..\masmx64\inffasx64.obj "

 				OutputFile="$(OutDir)\zlibstat.lib"

 				SuppressStartupBanner="true"

 			/>

@@ -418,7 +416,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				InlineFunctionExpansion="1"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

+				AdditionalIncludeDirectories="..\..\.."

 				PreprocessorDefinitions="ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64"

 				StringPooling="true"

 				ExceptionHandling="0"

@@ -491,7 +489,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				InlineFunctionExpansion="1"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

+				AdditionalIncludeDirectories="..\..\.."

 				PreprocessorDefinitions="WIN32;ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS"

 				StringPooling="true"

 				ExceptionHandling="0"

@@ -565,7 +563,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				InlineFunctionExpansion="1"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

+				AdditionalIncludeDirectories="..\..\.."

 				PreprocessorDefinitions="ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64"

 				StringPooling="true"

 				ExceptionHandling="0"

@@ -639,7 +637,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				InlineFunctionExpansion="1"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

+				AdditionalIncludeDirectories="..\..\.."

 				PreprocessorDefinitions="ZLIB_WINAPI;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;WIN64"

 				StringPooling="true"

 				ExceptionHandling="0"

@@ -733,58 +731,6 @@
 				>

 			</File>

 			<File

-				RelativePath="..\..\masmx64\inffas8664.c"

-				>

-				<FileConfiguration

-					Name="Debug|Win32"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="Debug|Itanium"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="Release|Win32"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="Release|Itanium"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="ReleaseWithoutAsm|Win32"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="ReleaseWithoutAsm|Itanium"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-			</File>

-			<File

 				RelativePath="..\..\..\inffast.c"

 				>

 			</File>

diff --git a/zlib/contrib/vstudio/vc9/zlibvc.def b/zlib/contrib/vstudio/vc9/zlibvc.def
index 54e683d..3234a02 100644
--- a/zlib/contrib/vstudio/vc9/zlibvc.def
+++ b/zlib/contrib/vstudio/vc9/zlibvc.def
@@ -1,7 +1,7 @@
 LIBRARY

 ; zlib data compression and ZIP file I/O library

 

-VERSION		1.2

+VERSION		1.3.1

 

 EXPORTS

         adler32                                  @1

@@ -151,3 +151,8 @@ EXPORTS
         deflateGetDictionary                    @173

         adler32_z                               @174

         crc32_z                                 @175

+

+; zlib1 v1.2.12 added:

+		crc32_combine_gen						@176

+		crc32_combine_gen64						@177

+		crc32_combine_op						@178

diff --git a/zlib/contrib/vstudio/vc9/zlibvc.vcproj b/zlib/contrib/vstudio/vc9/zlibvc.vcproj
index c9a8947..f11dd1f 100644
--- a/zlib/contrib/vstudio/vc9/zlibvc.vcproj
+++ b/zlib/contrib/vstudio/vc9/zlibvc.vcproj
@@ -53,8 +53,8 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				Optimization="0"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

-				PreprocessorDefinitions="WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF"

+				AdditionalIncludeDirectories="..\..\.."

+				PreprocessorDefinitions="WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI"

 				ExceptionHandling="0"

 				RuntimeLibrary="1"

 				BufferSecurityCheck="false"

@@ -81,7 +81,6 @@
 			<Tool

 				Name="VCLinkerTool"

 				AdditionalOptions="/MACHINE:I386"

-				AdditionalDependencies="..\..\masmx86\match686.obj ..\..\masmx86\inffas32.obj"

 				OutputFile="$(OutDir)\zlibwapi.dll"

 				LinkIncremental="2"

 				SuppressStartupBanner="true"

@@ -150,8 +149,8 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				Optimization="0"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

-				PreprocessorDefinitions="WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;WIN64"

+				AdditionalIncludeDirectories="..\..\.."

+				PreprocessorDefinitions="WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64"

 				ExceptionHandling="0"

 				RuntimeLibrary="3"

 				BufferSecurityCheck="false"

@@ -177,7 +176,6 @@
 			/>

 			<Tool

 				Name="VCLinkerTool"

-				AdditionalDependencies="..\..\masmx64\gvmat64.obj ..\..\masmx64\inffasx64.obj "

 				OutputFile="$(OutDir)\zlibwapi.dll"

 				LinkIncremental="2"

 				SuppressStartupBanner="true"

@@ -245,7 +243,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				Optimization="0"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

+				AdditionalIncludeDirectories="..\..\.."

 				PreprocessorDefinitions="WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64"

 				ExceptionHandling="0"

 				RuntimeLibrary="3"

@@ -340,7 +338,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				InlineFunctionExpansion="1"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

+				AdditionalIncludeDirectories="..\..\.."

 				PreprocessorDefinitions="WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI"

 				StringPooling="true"

 				ExceptionHandling="0"

@@ -440,7 +438,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				InlineFunctionExpansion="1"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

+				AdditionalIncludeDirectories="..\..\.."

 				PreprocessorDefinitions="WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64"

 				StringPooling="true"

 				ExceptionHandling="0"

@@ -538,7 +536,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				InlineFunctionExpansion="1"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

+				AdditionalIncludeDirectories="..\..\.."

 				PreprocessorDefinitions="WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64"

 				StringPooling="true"

 				ExceptionHandling="0"

@@ -636,8 +634,8 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				InlineFunctionExpansion="1"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

-				PreprocessorDefinitions="WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF"

+				AdditionalIncludeDirectories="..\..\.."

+				PreprocessorDefinitions="WIN32;_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI"

 				StringPooling="true"

 				ExceptionHandling="0"

 				RuntimeLibrary="0"

@@ -666,7 +664,6 @@
 			<Tool

 				Name="VCLinkerTool"

 				AdditionalOptions="/MACHINE:I386"

-				AdditionalDependencies="..\..\masmx86\match686.obj ..\..\masmx86\inffas32.obj "

 				OutputFile="$(OutDir)\zlibwapi.dll"

 				LinkIncremental="1"

 				SuppressStartupBanner="true"

@@ -737,8 +734,8 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				InlineFunctionExpansion="1"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

-				PreprocessorDefinitions="_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;ASMV;ASMINF;WIN64"

+				AdditionalIncludeDirectories="..\..\.."

+				PreprocessorDefinitions="_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64"

 				StringPooling="true"

 				ExceptionHandling="0"

 				RuntimeLibrary="2"

@@ -766,7 +763,6 @@
 			/>

 			<Tool

 				Name="VCLinkerTool"

-				AdditionalDependencies="..\..\masmx64\gvmat64.obj ..\..\masmx64\inffasx64.obj "

 				OutputFile="$(OutDir)\zlibwapi.dll"

 				LinkIncremental="1"

 				SuppressStartupBanner="true"

@@ -836,7 +832,7 @@
 			<Tool

 				Name="VCCLCompilerTool"

 				InlineFunctionExpansion="1"

-				AdditionalIncludeDirectories="..\..\..;..\..\masmx86"

+				AdditionalIncludeDirectories="..\..\.."

 				PreprocessorDefinitions="_CRT_NONSTDC_NO_DEPRECATE;_CRT_SECURE_NO_DEPRECATE;_CRT_NONSTDC_NO_WARNINGS;ZLIB_WINAPI;WIN64"

 				StringPooling="true"

 				ExceptionHandling="0"

@@ -950,58 +946,6 @@
 				>

 			</File>

 			<File

-				RelativePath="..\..\masmx64\inffas8664.c"

-				>

-				<FileConfiguration

-					Name="Debug|Win32"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="Debug|Itanium"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="ReleaseWithoutAsm|Win32"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="ReleaseWithoutAsm|Itanium"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="Release|Win32"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="Release|Itanium"

-					ExcludedFromBuild="true"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-					/>

-				</FileConfiguration>

-			</File>

-			<File

 				RelativePath="..\..\..\inffast.c"

 				>

 			</File>

diff --git a/zlib/crc32.c b/zlib/crc32.c
index affcd46..6c38f5c 100644
--- a/zlib/crc32.c
+++ b/zlib/crc32.c
@@ -1,24 +1,23 @@
 /* crc32.c -- compute the CRC-32 of a data stream
- * Copyright (C) 1995-2006, 2010, 2011, 2012, 2016 Mark Adler
+ * Copyright (C) 1995-2022 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  *
- * Thanks to Rodney Brown <rbrown64@csc.com.au> for his contribution of faster
- * CRC methods: exclusive-oring 32 bits of data at a time, and pre-computing
- * tables for updating the shift register in one step with three exclusive-ors
- * instead of four steps with four exclusive-ors.  This results in about a
- * factor of two increase in speed on a Power PC G4 (PPC7455) using gcc -O3.
+ * This interleaved implementation of a CRC makes use of pipelined multiple
+ * arithmetic-logic units, commonly found in modern CPU cores. It is due to
+ * Kadatch and Jenkins (2010). See doc/crc-doc.1.0.pdf in this distribution.
  */
 
-/* @(#) $Id: crc32.c,v 1.1.1.2 2002/03/11 21:53:23 tromey Exp $ */
+/* @(#) $Id$ */
 
 /*
   Note on the use of DYNAMIC_CRC_TABLE: there is no mutex or semaphore
   protection on the static variables used to control the first-use generation
-  of the crc tables.  Therefore, if you #define DYNAMIC_CRC_TABLE, you should
+  of the crc tables. Therefore, if you #define DYNAMIC_CRC_TABLE, you should
   first call get_crc_table() to initialize the tables before allowing more than
   one thread to use crc32().
 
-  DYNAMIC_CRC_TABLE and MAKECRCH can be #defined to write out crc32.h.
+  MAKECRCH can be #defined to write out crc32.h. A main() routine is also
+  produced, so that this one source file can be compiled to an executable.
  */
 
 #ifdef MAKECRCH
@@ -28,415 +27,1023 @@
 #  endif /* !DYNAMIC_CRC_TABLE */
 #endif /* MAKECRCH */
 
-#include "zutil.h"      /* for STDC and FAR definitions */
+#include "zutil.h"      /* for Z_U4, Z_U8, z_crc_t, and FAR definitions */
 
-/* Definitions for doing the crc four data bytes at a time. */
-#if !defined(NOBYFOUR) && defined(Z_U4)
-#  define BYFOUR
+ /*
+  A CRC of a message is computed on N braids of words in the message, where
+  each word consists of W bytes (4 or 8). If N is 3, for example, then three
+  running sparse CRCs are calculated respectively on each braid, at these
+  indices in the array of words: 0, 3, 6, ..., 1, 4, 7, ..., and 2, 5, 8, ...
+  This is done starting at a word boundary, and continues until as many blocks
+  of N * W bytes as are available have been processed. The results are combined
+  into a single CRC at the end. For this code, N must be in the range 1..6 and
+  W must be 4 or 8. The upper limit on N can be increased if desired by adding
+  more #if blocks, extending the patterns apparent in the code. In addition,
+  crc32.h would need to be regenerated, if the maximum N value is increased.
+
+  N and W are chosen empirically by benchmarking the execution time on a given
+  processor. The choices for N and W below were based on testing on Intel Kaby
+  Lake i7, AMD Ryzen 7, ARM Cortex-A57, Sparc64-VII, PowerPC POWER9, and MIPS64
+  Octeon II processors. The Intel, AMD, and ARM processors were all fastest
+  with N=5, W=8. The Sparc, PowerPC, and MIPS64 were all fastest at N=5, W=4.
+  They were all tested with either gcc or clang, all using the -O3 optimization
+  level. Your mileage may vary.
+ */
+
+/* Define N */
+#ifdef Z_TESTN
+#  define N Z_TESTN
+#else
+#  define N 5
+#endif
+#if N < 1 || N > 6
+#  error N must be in 1..6
 #endif
-#ifdef BYFOUR
-   local unsigned long crc32_little OF((unsigned long,
-                        const unsigned char FAR *, z_size_t));
-   local unsigned long crc32_big OF((unsigned long,
-                        const unsigned char FAR *, z_size_t));
-#  define TBLS 8
+
+/*
+  z_crc_t must be at least 32 bits. z_word_t must be at least as long as
+  z_crc_t. It is assumed here that z_word_t is either 32 bits or 64 bits, and
+  that bytes are eight bits.
+ */
+
+/*
+  Define W and the associated z_word_t type. If W is not defined, then a
+  braided calculation is not used, and the associated tables and code are not
+  compiled.
+ */
+#ifdef Z_TESTW
+#  if Z_TESTW-1 != -1
+#    define W Z_TESTW
+#  endif
 #else
-#  define TBLS 1
-#endif /* BYFOUR */
+#  ifdef MAKECRCH
+#    define W 8         /* required for MAKECRCH */
+#  else
+#    if defined(__x86_64__) || defined(__aarch64__)
+#      define W 8
+#    else
+#      define W 4
+#    endif
+#  endif
+#endif
+#ifdef W
+#  if W == 8 && defined(Z_U8)
+     typedef Z_U8 z_word_t;
+#  elif defined(Z_U4)
+#    undef W
+#    define W 4
+     typedef Z_U4 z_word_t;
+#  else
+#    undef W
+#  endif
+#endif
 
-/* Local functions for crc concatenation */
-local unsigned long gf2_matrix_times OF((unsigned long *mat,
-                                         unsigned long vec));
-local void gf2_matrix_square OF((unsigned long *square, unsigned long *mat));
-local uLong crc32_combine_ OF((uLong crc1, uLong crc2, z_off64_t len2));
+/* If available, use the ARM processor CRC32 instruction. */
+#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) && W == 8
+#  define ARMCRC32
+#endif
 
+#if defined(W) && (!defined(ARMCRC32) || defined(DYNAMIC_CRC_TABLE))
+/*
+  Swap the bytes in a z_word_t to convert between little and big endian. Any
+  self-respecting compiler will optimize this to a single machine byte-swap
+  instruction, if one is available. This assumes that word_t is either 32 bits
+  or 64 bits.
+ */
+local z_word_t byte_swap(z_word_t word) {
+#  if W == 8
+    return
+        (word & 0xff00000000000000) >> 56 |
+        (word & 0xff000000000000) >> 40 |
+        (word & 0xff0000000000) >> 24 |
+        (word & 0xff00000000) >> 8 |
+        (word & 0xff000000) << 8 |
+        (word & 0xff0000) << 24 |
+        (word & 0xff00) << 40 |
+        (word & 0xff) << 56;
+#  else   /* W == 4 */
+    return
+        (word & 0xff000000) >> 24 |
+        (word & 0xff0000) >> 8 |
+        (word & 0xff00) << 8 |
+        (word & 0xff) << 24;
+#  endif
+}
+#endif
 
 #ifdef DYNAMIC_CRC_TABLE
+/* =========================================================================
+ * Table of powers of x for combining CRC-32s, filled in by make_crc_table()
+ * below.
+ */
+   local z_crc_t FAR x2n_table[32];
+#else
+/* =========================================================================
+ * Tables for byte-wise and braided CRC-32 calculations, and a table of powers
+ * of x for combining CRC-32s, all made by make_crc_table().
+ */
+#  include "crc32.h"
+#endif
+
+/* CRC polynomial. */
+#define POLY 0xedb88320         /* p(x) reflected, with x^32 implied */
 
-local volatile int crc_table_empty = 1;
-local z_crc_t FAR crc_table[TBLS][256];
-local void make_crc_table OF((void));
+/*
+  Return a(x) multiplied by b(x) modulo p(x), where p(x) is the CRC polynomial,
+  reflected. For speed, this requires that a not be zero.
+ */
+local z_crc_t multmodp(z_crc_t a, z_crc_t b) {
+    z_crc_t m, p;
+
+    m = (z_crc_t)1 << 31;
+    p = 0;
+    for (;;) {
+        if (a & m) {
+            p ^= b;
+            if ((a & (m - 1)) == 0)
+                break;
+        }
+        m >>= 1;
+        b = b & 1 ? (b >> 1) ^ POLY : b >> 1;
+    }
+    return p;
+}
+
+/*
+  Return x^(n * 2^k) modulo p(x). Requires that x2n_table[] has been
+  initialized.
+ */
+local z_crc_t x2nmodp(z_off64_t n, unsigned k) {
+    z_crc_t p;
+
+    p = (z_crc_t)1 << 31;           /* x^0 == 1 */
+    while (n) {
+        if (n & 1)
+            p = multmodp(x2n_table[k & 31], p);
+        n >>= 1;
+        k++;
+    }
+    return p;
+}
+
+#ifdef DYNAMIC_CRC_TABLE
+/* =========================================================================
+ * Build the tables for byte-wise and braided CRC-32 calculations, and a table
+ * of powers of x for combining CRC-32s.
+ */
+local z_crc_t FAR crc_table[256];
+#ifdef W
+   local z_word_t FAR crc_big_table[256];
+   local z_crc_t FAR crc_braid_table[W][256];
+   local z_word_t FAR crc_braid_big_table[W][256];
+   local void braid(z_crc_t [][256], z_word_t [][256], int, int);
+#endif
 #ifdef MAKECRCH
-   local void write_table OF((FILE *, const z_crc_t FAR *));
+   local void write_table(FILE *, const z_crc_t FAR *, int);
+   local void write_table32hi(FILE *, const z_word_t FAR *, int);
+   local void write_table64(FILE *, const z_word_t FAR *, int);
 #endif /* MAKECRCH */
+
+/*
+  Define a once() function depending on the availability of atomics. If this is
+  compiled with DYNAMIC_CRC_TABLE defined, and if CRCs will be computed in
+  multiple threads, and if atomics are not available, then get_crc_table() must
+  be called to initialize the tables and must return before any threads are
+  allowed to compute or combine CRCs.
+ */
+
+/* Definition of once functionality. */
+typedef struct once_s once_t;
+
+/* Check for the availability of atomics. */
+#if defined(__STDC__) && __STDC_VERSION__ >= 201112L && \
+    !defined(__STDC_NO_ATOMICS__)
+
+#include <stdatomic.h>
+
+/* Structure for once(), which must be initialized with ONCE_INIT. */
+struct once_s {
+    atomic_flag begun;
+    atomic_int done;
+};
+#define ONCE_INIT {ATOMIC_FLAG_INIT, 0}
+
+/*
+  Run the provided init() function exactly once, even if multiple threads
+  invoke once() at the same time. The state must be a once_t initialized with
+  ONCE_INIT.
+ */
+local void once(once_t *state, void (*init)(void)) {
+    if (!atomic_load(&state->done)) {
+        if (atomic_flag_test_and_set(&state->begun))
+            while (!atomic_load(&state->done))
+                ;
+        else {
+            init();
+            atomic_store(&state->done, 1);
+        }
+    }
+}
+
+#else   /* no atomics */
+
+/* Structure for once(), which must be initialized with ONCE_INIT. */
+struct once_s {
+    volatile int begun;
+    volatile int done;
+};
+#define ONCE_INIT {0, 0}
+
+/* Test and set. Alas, not atomic, but tries to minimize the period of
+   vulnerability. */
+local int test_and_set(int volatile *flag) {
+    int was;
+
+    was = *flag;
+    *flag = 1;
+    return was;
+}
+
+/* Run the provided init() function once. This is not thread-safe. */
+local void once(once_t *state, void (*init)(void)) {
+    if (!state->done) {
+        if (test_and_set(&state->begun))
+            while (!state->done)
+                ;
+        else {
+            init();
+            state->done = 1;
+        }
+    }
+}
+
+#endif
+
+/* State for once(). */
+local once_t made = ONCE_INIT;
+
 /*
   Generate tables for a byte-wise 32-bit CRC calculation on the polynomial:
   x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x+1.
 
   Polynomials over GF(2) are represented in binary, one bit per coefficient,
-  with the lowest powers in the most significant bit.  Then adding polynomials
+  with the lowest powers in the most significant bit. Then adding polynomials
   is just exclusive-or, and multiplying a polynomial by x is a right shift by
-  one.  If we call the above polynomial p, and represent a byte as the
+  one. If we call the above polynomial p, and represent a byte as the
   polynomial q, also with the lowest power in the most significant bit (so the
-  byte 0xb1 is the polynomial x^7+x^3+x+1), then the CRC is (q*x^32) mod p,
+  byte 0xb1 is the polynomial x^7+x^3+x^2+1), then the CRC is (q*x^32) mod p,
   where a mod b means the remainder after dividing a by b.
 
   This calculation is done using the shift-register method of multiplying and
-  taking the remainder.  The register is initialized to zero, and for each
+  taking the remainder. The register is initialized to zero, and for each
   incoming bit, x^32 is added mod p to the register if the bit is a one (where
-  x^32 mod p is p+x^32 = x^26+...+1), and the register is multiplied mod p by
-  x (which is shifting right by one and adding x^32 mod p if the bit shifted
-  out is a one).  We start with the highest power (least significant bit) of
-  q and repeat for all eight bits of q.
-
-  The first table is simply the CRC of all possible eight bit values.  This is
-  all the information needed to generate CRCs on data a byte at a time for all
-  combinations of CRC register values and incoming bytes.  The remaining tables
-  allow for word-at-a-time CRC calculation for both big-endian and little-
-  endian machines, where a word is four bytes.
-*/
-local void make_crc_table()
-{
-    z_crc_t c;
-    int n, k;
-    z_crc_t poly;                       /* polynomial exclusive-or pattern */
-    /* terms of polynomial defining this crc (except x^32): */
-    static volatile int first = 1;      /* flag to limit concurrent making */
-    static const unsigned char p[] = {0,1,2,4,5,7,8,10,11,12,16,22,23,26};
-
-    /* See if another task is already doing this (not thread-safe, but better
-       than nothing -- significantly reduces duration of vulnerability in
-       case the advice about DYNAMIC_CRC_TABLE is ignored) */
-    if (first) {
-        first = 0;
-
-        /* make exclusive-or pattern from polynomial (0xedb88320UL) */
-        poly = 0;
-        for (n = 0; n < (int)(sizeof(p)/sizeof(unsigned char)); n++)
-            poly |= (z_crc_t)1 << (31 - p[n]);
-
-        /* generate a crc for every 8-bit value */
-        for (n = 0; n < 256; n++) {
-            c = (z_crc_t)n;
-            for (k = 0; k < 8; k++)
-                c = c & 1 ? poly ^ (c >> 1) : c >> 1;
-            crc_table[0][n] = c;
-        }
+  x^32 mod p is p+x^32 = x^26+...+1), and the register is multiplied mod p by x
+  (which is shifting right by one and adding x^32 mod p if the bit shifted out
+  is a one). We start with the highest power (least significant bit) of q and
+  repeat for all eight bits of q.
 
-#ifdef BYFOUR
-        /* generate crc for each value followed by one, two, and three zeros,
-           and then the byte reversal of those as well as the first table */
-        for (n = 0; n < 256; n++) {
-            c = crc_table[0][n];
-            crc_table[4][n] = ZSWAP32(c);
-            for (k = 1; k < 4; k++) {
-                c = crc_table[0][c & 0xff] ^ (c >> 8);
-                crc_table[k][n] = c;
-                crc_table[k + 4][n] = ZSWAP32(c);
-            }
-        }
-#endif /* BYFOUR */
+  The table is simply the CRC of all possible eight bit values. This is all the
+  information needed to generate CRCs on data a byte at a time for all
+  combinations of CRC register values and incoming bytes.
+ */
 
-        crc_table_empty = 0;
-    }
-    else {      /* not first */
-        /* wait for the other guy to finish (not efficient, but rare) */
-        while (crc_table_empty)
-            ;
+local void make_crc_table(void) {
+    unsigned i, j, n;
+    z_crc_t p;
+
+    /* initialize the CRC of bytes tables */
+    for (i = 0; i < 256; i++) {
+        p = i;
+        for (j = 0; j < 8; j++)
+            p = p & 1 ? (p >> 1) ^ POLY : p >> 1;
+        crc_table[i] = p;
+#ifdef W
+        crc_big_table[i] = byte_swap(p);
+#endif
     }
 
+    /* initialize the x^2^n mod p(x) table */
+    p = (z_crc_t)1 << 30;         /* x^1 */
+    x2n_table[0] = p;
+    for (n = 1; n < 32; n++)
+        x2n_table[n] = p = multmodp(p, p);
+
+#ifdef W
+    /* initialize the braiding tables -- needs x2n_table[] */
+    braid(crc_braid_table, crc_braid_big_table, N, W);
+#endif
+
 #ifdef MAKECRCH
-    /* write out CRC tables to crc32.h */
     {
+        /*
+          The crc32.h header file contains tables for both 32-bit and 64-bit
+          z_word_t's, and so requires a 64-bit type be available. In that case,
+          z_word_t must be defined to be 64-bits. This code then also generates
+          and writes out the tables for the case that z_word_t is 32 bits.
+         */
+#if !defined(W) || W != 8
+#  error Need a 64-bit integer type in order to generate crc32.h.
+#endif
         FILE *out;
+        int k, n;
+        z_crc_t ltl[8][256];
+        z_word_t big[8][256];
 
         out = fopen("crc32.h", "w");
         if (out == NULL) return;
-        fprintf(out, "/* crc32.h -- tables for rapid CRC calculation\n");
-        fprintf(out, " * Generated automatically by crc32.c\n */\n\n");
-        fprintf(out, "local const z_crc_t FAR ");
-        fprintf(out, "crc_table[TBLS][256] =\n{\n  {\n");
-        write_table(out, crc_table[0]);
-#  ifdef BYFOUR
-        fprintf(out, "#ifdef BYFOUR\n");
-        for (k = 1; k < 8; k++) {
-            fprintf(out, "  },\n  {\n");
-            write_table(out, crc_table[k]);
+
+        /* write out little-endian CRC table to crc32.h */
+        fprintf(out,
+            "/* crc32.h -- tables for rapid CRC calculation\n"
+            " * Generated automatically by crc32.c\n */\n"
+            "\n"
+            "local const z_crc_t FAR crc_table[] = {\n"
+            "    ");
+        write_table(out, crc_table, 256);
+        fprintf(out,
+            "};\n");
+
+        /* write out big-endian CRC table for 64-bit z_word_t to crc32.h */
+        fprintf(out,
+            "\n"
+            "#ifdef W\n"
+            "\n"
+            "#if W == 8\n"
+            "\n"
+            "local const z_word_t FAR crc_big_table[] = {\n"
+            "    ");
+        write_table64(out, crc_big_table, 256);
+        fprintf(out,
+            "};\n");
+
+        /* write out big-endian CRC table for 32-bit z_word_t to crc32.h */
+        fprintf(out,
+            "\n"
+            "#else /* W == 4 */\n"
+            "\n"
+            "local const z_word_t FAR crc_big_table[] = {\n"
+            "    ");
+        write_table32hi(out, crc_big_table, 256);
+        fprintf(out,
+            "};\n"
+            "\n"
+            "#endif\n");
+
+        /* write out braid tables for each value of N */
+        for (n = 1; n <= 6; n++) {
+            fprintf(out,
+            "\n"
+            "#if N == %d\n", n);
+
+            /* compute braid tables for this N and 64-bit word_t */
+            braid(ltl, big, n, 8);
+
+            /* write out braid tables for 64-bit z_word_t to crc32.h */
+            fprintf(out,
+            "\n"
+            "#if W == 8\n"
+            "\n"
+            "local const z_crc_t FAR crc_braid_table[][256] = {\n");
+            for (k = 0; k < 8; k++) {
+                fprintf(out, "   {");
+                write_table(out, ltl[k], 256);
+                fprintf(out, "}%s", k < 7 ? ",\n" : "");
+            }
+            fprintf(out,
+            "};\n"
+            "\n"
+            "local const z_word_t FAR crc_braid_big_table[][256] = {\n");
+            for (k = 0; k < 8; k++) {
+                fprintf(out, "   {");
+                write_table64(out, big[k], 256);
+                fprintf(out, "}%s", k < 7 ? ",\n" : "");
+            }
+            fprintf(out,
+            "};\n");
+
+            /* compute braid tables for this N and 32-bit word_t */
+            braid(ltl, big, n, 4);
+
+            /* write out braid tables for 32-bit z_word_t to crc32.h */
+            fprintf(out,
+            "\n"
+            "#else /* W == 4 */\n"
+            "\n"
+            "local const z_crc_t FAR crc_braid_table[][256] = {\n");
+            for (k = 0; k < 4; k++) {
+                fprintf(out, "   {");
+                write_table(out, ltl[k], 256);
+                fprintf(out, "}%s", k < 3 ? ",\n" : "");
+            }
+            fprintf(out,
+            "};\n"
+            "\n"
+            "local const z_word_t FAR crc_braid_big_table[][256] = {\n");
+            for (k = 0; k < 4; k++) {
+                fprintf(out, "   {");
+                write_table32hi(out, big[k], 256);
+                fprintf(out, "}%s", k < 3 ? ",\n" : "");
+            }
+            fprintf(out,
+            "};\n"
+            "\n"
+            "#endif\n"
+            "\n"
+            "#endif\n");
         }
-        fprintf(out, "#endif\n");
-#  endif /* BYFOUR */
-        fprintf(out, "  }\n};\n");
+        fprintf(out,
+            "\n"
+            "#endif\n");
+
+        /* write out zeros operator table to crc32.h */
+        fprintf(out,
+            "\n"
+            "local const z_crc_t FAR x2n_table[] = {\n"
+            "    ");
+        write_table(out, x2n_table, 32);
+        fprintf(out,
+            "};\n");
         fclose(out);
     }
 #endif /* MAKECRCH */
 }
 
 #ifdef MAKECRCH
-local void write_table(out, table)
-    FILE *out;
-    const z_crc_t FAR *table;
-{
+
+/*
+   Write the 32-bit values in table[0..k-1] to out, five per line in
+   hexadecimal separated by commas.
+ */
+local void write_table(FILE *out, const z_crc_t FAR *table, int k) {
     int n;
 
-    for (n = 0; n < 256; n++)
-        fprintf(out, "%s0x%08lxUL%s", n % 5 ? "" : "    ",
+    for (n = 0; n < k; n++)
+        fprintf(out, "%s0x%08lx%s", n == 0 || n % 5 ? "" : "    ",
                 (unsigned long)(table[n]),
-                n == 255 ? "\n" : (n % 5 == 4 ? ",\n" : ", "));
+                n == k - 1 ? "" : (n % 5 == 4 ? ",\n" : ", "));
+}
+
+/*
+   Write the high 32-bits of each value in table[0..k-1] to out, five per line
+   in hexadecimal separated by commas.
+ */
+local void write_table32hi(FILE *out, const z_word_t FAR *table, int k) {
+    int n;
+
+    for (n = 0; n < k; n++)
+        fprintf(out, "%s0x%08lx%s", n == 0 || n % 5 ? "" : "    ",
+                (unsigned long)(table[n] >> 32),
+                n == k - 1 ? "" : (n % 5 == 4 ? ",\n" : ", "));
+}
+
+/*
+  Write the 64-bit values in table[0..k-1] to out, three per line in
+  hexadecimal separated by commas. This assumes that if there is a 64-bit
+  type, then there is also a long long integer type, and it is at least 64
+  bits. If not, then the type cast and format string can be adjusted
+  accordingly.
+ */
+local void write_table64(FILE *out, const z_word_t FAR *table, int k) {
+    int n;
+
+    for (n = 0; n < k; n++)
+        fprintf(out, "%s0x%016llx%s", n == 0 || n % 3 ? "" : "    ",
+                (unsigned long long)(table[n]),
+                n == k - 1 ? "" : (n % 3 == 2 ? ",\n" : ", "));
+}
+
+/* Actually do the deed. */
+int main(void) {
+    make_crc_table();
+    return 0;
 }
+
 #endif /* MAKECRCH */
 
-#else /* !DYNAMIC_CRC_TABLE */
-/* ========================================================================
- * Tables of CRC-32s of all single-byte values, made by make_crc_table().
+#ifdef W
+/*
+  Generate the little and big-endian braid tables for the given n and z_word_t
+  size w. Each array must have room for w blocks of 256 elements.
  */
-#include "crc32.h"
+local void braid(z_crc_t ltl[][256], z_word_t big[][256], int n, int w) {
+    int k;
+    z_crc_t i, p, q;
+    for (k = 0; k < w; k++) {
+        p = x2nmodp((n * w + 3 - k) << 3, 0);
+        ltl[k][0] = 0;
+        big[w - 1 - k][0] = 0;
+        for (i = 1; i < 256; i++) {
+            ltl[k][i] = q = multmodp(i << 24, p);
+            big[w - 1 - k][i] = byte_swap(q);
+        }
+    }
+}
+#endif
+
 #endif /* DYNAMIC_CRC_TABLE */
 
 /* =========================================================================
- * This function can be used by asm versions of crc32()
+ * This function can be used by asm versions of crc32(), and to force the
+ * generation of the CRC tables in a threaded application.
  */
-const z_crc_t FAR * ZEXPORT get_crc_table()
-{
+const z_crc_t FAR * ZEXPORT get_crc_table(void) {
 #ifdef DYNAMIC_CRC_TABLE
-    if (crc_table_empty)
-        make_crc_table();
+    once(&made, make_crc_table);
 #endif /* DYNAMIC_CRC_TABLE */
     return (const z_crc_t FAR *)crc_table;
 }
 
-/* ========================================================================= */
-#define DO1 crc = crc_table[0][((int)crc ^ (*buf++)) & 0xff] ^ (crc >> 8)
-#define DO8 DO1; DO1; DO1; DO1; DO1; DO1; DO1; DO1
+/* =========================================================================
+ * Use ARM machine instructions if available. This will compute the CRC about
+ * ten times faster than the braided calculation. This code does not check for
+ * the presence of the CRC instruction at run time. __ARM_FEATURE_CRC32 will
+ * only be defined if the compilation specifies an ARM processor architecture
+ * that has the instructions. For example, compiling with -march=armv8.1-a or
+ * -march=armv8-a+crc, or -march=native if the compile machine has the crc32
+ * instructions.
+ */
+#ifdef ARMCRC32
 
-/* ========================================================================= */
-unsigned long ZEXPORT crc32_z(crc, buf, len)
-    unsigned long crc;
-    const unsigned char FAR *buf;
-    z_size_t len;
-{
-    if (buf == Z_NULL) return 0UL;
+/*
+   Constants empirically determined to maximize speed. These values are from
+   measurements on a Cortex-A57. Your mileage may vary.
+ */
+#define Z_BATCH 3990                /* number of words in a batch */
+#define Z_BATCH_ZEROS 0xa10d3d0c    /* computed from Z_BATCH = 3990 */
+#define Z_BATCH_MIN 800             /* fewest words in a final batch */
+
+unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf,
+                              z_size_t len) {
+    z_crc_t val;
+    z_word_t crc1, crc2;
+    const z_word_t *word;
+    z_word_t val0, val1, val2;
+    z_size_t last, last2, i;
+    z_size_t num;
+
+    /* Return initial CRC, if requested. */
+    if (buf == Z_NULL) return 0;
 
 #ifdef DYNAMIC_CRC_TABLE
-    if (crc_table_empty)
-        make_crc_table();
+    once(&made, make_crc_table);
 #endif /* DYNAMIC_CRC_TABLE */
 
-#ifdef BYFOUR
-    if (sizeof(void *) == sizeof(ptrdiff_t)) {
-        z_crc_t endian;
+    /* Pre-condition the CRC */
+    crc = (~crc) & 0xffffffff;
 
-        endian = 1;
-        if (*((unsigned char *)(&endian)))
-            return crc32_little(crc, buf, len);
-        else
-            return crc32_big(crc, buf, len);
+    /* Compute the CRC up to a word boundary. */
+    while (len && ((z_size_t)buf & 7) != 0) {
+        len--;
+        val = *buf++;
+        __asm__ volatile("crc32b %w0, %w0, %w1" : "+r"(crc) : "r"(val));
     }
-#endif /* BYFOUR */
-    crc = crc ^ 0xffffffffUL;
-    while (len >= 8) {
-        DO8;
-        len -= 8;
+
+    /* Prepare to compute the CRC on full 64-bit words word[0..num-1]. */
+    word = (z_word_t const *)buf;
+    num = len >> 3;
+    len &= 7;
+
+    /* Do three interleaved CRCs to realize the throughput of one crc32x
+       instruction per cycle. Each CRC is calculated on Z_BATCH words. The
+       three CRCs are combined into a single CRC after each set of batches. */
+    while (num >= 3 * Z_BATCH) {
+        crc1 = 0;
+        crc2 = 0;
+        for (i = 0; i < Z_BATCH; i++) {
+            val0 = word[i];
+            val1 = word[i + Z_BATCH];
+            val2 = word[i + 2 * Z_BATCH];
+            __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc) : "r"(val0));
+            __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc1) : "r"(val1));
+            __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc2) : "r"(val2));
+        }
+        word += 3 * Z_BATCH;
+        num -= 3 * Z_BATCH;
+        crc = multmodp(Z_BATCH_ZEROS, crc) ^ crc1;
+        crc = multmodp(Z_BATCH_ZEROS, crc) ^ crc2;
     }
-    if (len) do {
-        DO1;
-    } while (--len);
-    return crc ^ 0xffffffffUL;
-}
 
-/* ========================================================================= */
-unsigned long ZEXPORT crc32(crc, buf, len)
-    unsigned long crc;
-    const unsigned char FAR *buf;
-    uInt len;
-{
-    return crc32_z(crc, buf, len);
+    /* Do one last smaller batch with the remaining words, if there are enough
+       to pay for the combination of CRCs. */
+    last = num / 3;
+    if (last >= Z_BATCH_MIN) {
+        last2 = last << 1;
+        crc1 = 0;
+        crc2 = 0;
+        for (i = 0; i < last; i++) {
+            val0 = word[i];
+            val1 = word[i + last];
+            val2 = word[i + last2];
+            __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc) : "r"(val0));
+            __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc1) : "r"(val1));
+            __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc2) : "r"(val2));
+        }
+        word += 3 * last;
+        num -= 3 * last;
+        val = x2nmodp(last, 6);
+        crc = multmodp(val, crc) ^ crc1;
+        crc = multmodp(val, crc) ^ crc2;
+    }
+
+    /* Compute the CRC on any remaining words. */
+    for (i = 0; i < num; i++) {
+        val0 = word[i];
+        __asm__ volatile("crc32x %w0, %w0, %x1" : "+r"(crc) : "r"(val0));
+    }
+    word += num;
+
+    /* Complete the CRC on any remaining bytes. */
+    buf = (const unsigned char FAR *)word;
+    while (len) {
+        len--;
+        val = *buf++;
+        __asm__ volatile("crc32b %w0, %w0, %w1" : "+r"(crc) : "r"(val));
+    }
+
+    /* Return the CRC, post-conditioned. */
+    return crc ^ 0xffffffff;
 }
 
-#ifdef BYFOUR
+#else
+
+#ifdef W
 
 /*
-   This BYFOUR code accesses the passed unsigned char * buffer with a 32-bit
-   integer pointer type. This violates the strict aliasing rule, where a
-   compiler can assume, for optimization purposes, that two pointers to
-   fundamentally different types won't ever point to the same memory. This can
-   manifest as a problem only if one of the pointers is written to. This code
-   only reads from those pointers. So long as this code remains isolated in
-   this compilation unit, there won't be a problem. For this reason, this code
-   should not be copied and pasted into a compilation unit in which other code
-   writes to the buffer that is passed to these routines.
+  Return the CRC of the W bytes in the word_t data, taking the
+  least-significant byte of the word as the first byte of data, without any pre
+  or post conditioning. This is used to combine the CRCs of each braid.
  */
+local z_crc_t crc_word(z_word_t data) {
+    int k;
+    for (k = 0; k < W; k++)
+        data = (data >> 8) ^ crc_table[data & 0xff];
+    return (z_crc_t)data;
+}
 
-/* ========================================================================= */
-#define DOLIT4 c ^= *buf4++; \
-        c = crc_table[3][c & 0xff] ^ crc_table[2][(c >> 8) & 0xff] ^ \
-            crc_table[1][(c >> 16) & 0xff] ^ crc_table[0][c >> 24]
-#define DOLIT32 DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4
+local z_word_t crc_word_big(z_word_t data) {
+    int k;
+    for (k = 0; k < W; k++)
+        data = (data << 8) ^
+            crc_big_table[(data >> ((W - 1) << 3)) & 0xff];
+    return data;
+}
+
+#endif
 
 /* ========================================================================= */
-local unsigned long crc32_little(crc, buf, len)
-    unsigned long crc;
-    const unsigned char FAR *buf;
-    z_size_t len;
-{
-    register z_crc_t c;
-    register const z_crc_t FAR *buf4;
-
-    c = (z_crc_t)crc;
-    c = ~c;
-    while (len && ((ptrdiff_t)buf & 3)) {
-        c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8);
-        len--;
-    }
+unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf,
+                              z_size_t len) {
+    /* Return initial CRC, if requested. */
+    if (buf == Z_NULL) return 0;
 
-    buf4 = (const z_crc_t FAR *)(const void FAR *)buf;
-    while (len >= 32) {
-        DOLIT32;
-        len -= 32;
-    }
-    while (len >= 4) {
-        DOLIT4;
-        len -= 4;
-    }
-    buf = (const unsigned char FAR *)buf4;
+#ifdef DYNAMIC_CRC_TABLE
+    once(&made, make_crc_table);
+#endif /* DYNAMIC_CRC_TABLE */
 
-    if (len) do {
-        c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8);
-    } while (--len);
-    c = ~c;
-    return (unsigned long)c;
-}
+    /* Pre-condition the CRC */
+    crc = (~crc) & 0xffffffff;
 
-/* ========================================================================= */
-#define DOBIG4 c ^= *buf4++; \
-        c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \
-            crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24]
-#define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4
+#ifdef W
 
-/* ========================================================================= */
-local unsigned long crc32_big(crc, buf, len)
-    unsigned long crc;
-    const unsigned char FAR *buf;
-    z_size_t len;
-{
-    register z_crc_t c;
-    register const z_crc_t FAR *buf4;
-
-    c = ZSWAP32((z_crc_t)crc);
-    c = ~c;
-    while (len && ((ptrdiff_t)buf & 3)) {
-        c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8);
-        len--;
+    /* If provided enough bytes, do a braided CRC calculation. */
+    if (len >= N * W + W - 1) {
+        z_size_t blks;
+        z_word_t const *words;
+        unsigned endian;
+        int k;
+
+        /* Compute the CRC up to a z_word_t boundary. */
+        while (len && ((z_size_t)buf & (W - 1)) != 0) {
+            len--;
+            crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff];
+        }
+
+        /* Compute the CRC on as many N z_word_t blocks as are available. */
+        blks = len / (N * W);
+        len -= blks * N * W;
+        words = (z_word_t const *)buf;
+
+        /* Do endian check at execution time instead of compile time, since ARM
+           processors can change the endianness at execution time. If the
+           compiler knows what the endianness will be, it can optimize out the
+           check and the unused branch. */
+        endian = 1;
+        if (*(unsigned char *)&endian) {
+            /* Little endian. */
+
+            z_crc_t crc0;
+            z_word_t word0;
+#if N > 1
+            z_crc_t crc1;
+            z_word_t word1;
+#if N > 2
+            z_crc_t crc2;
+            z_word_t word2;
+#if N > 3
+            z_crc_t crc3;
+            z_word_t word3;
+#if N > 4
+            z_crc_t crc4;
+            z_word_t word4;
+#if N > 5
+            z_crc_t crc5;
+            z_word_t word5;
+#endif
+#endif
+#endif
+#endif
+#endif
+
+            /* Initialize the CRC for each braid. */
+            crc0 = crc;
+#if N > 1
+            crc1 = 0;
+#if N > 2
+            crc2 = 0;
+#if N > 3
+            crc3 = 0;
+#if N > 4
+            crc4 = 0;
+#if N > 5
+            crc5 = 0;
+#endif
+#endif
+#endif
+#endif
+#endif
+
+            /*
+              Process the first blks-1 blocks, computing the CRCs on each braid
+              independently.
+             */
+            while (--blks) {
+                /* Load the word for each braid into registers. */
+                word0 = crc0 ^ words[0];
+#if N > 1
+                word1 = crc1 ^ words[1];
+#if N > 2
+                word2 = crc2 ^ words[2];
+#if N > 3
+                word3 = crc3 ^ words[3];
+#if N > 4
+                word4 = crc4 ^ words[4];
+#if N > 5
+                word5 = crc5 ^ words[5];
+#endif
+#endif
+#endif
+#endif
+#endif
+                words += N;
+
+                /* Compute and update the CRC for each word. The loop should
+                   get unrolled. */
+                crc0 = crc_braid_table[0][word0 & 0xff];
+#if N > 1
+                crc1 = crc_braid_table[0][word1 & 0xff];
+#if N > 2
+                crc2 = crc_braid_table[0][word2 & 0xff];
+#if N > 3
+                crc3 = crc_braid_table[0][word3 & 0xff];
+#if N > 4
+                crc4 = crc_braid_table[0][word4 & 0xff];
+#if N > 5
+                crc5 = crc_braid_table[0][word5 & 0xff];
+#endif
+#endif
+#endif
+#endif
+#endif
+                for (k = 1; k < W; k++) {
+                    crc0 ^= crc_braid_table[k][(word0 >> (k << 3)) & 0xff];
+#if N > 1
+                    crc1 ^= crc_braid_table[k][(word1 >> (k << 3)) & 0xff];
+#if N > 2
+                    crc2 ^= crc_braid_table[k][(word2 >> (k << 3)) & 0xff];
+#if N > 3
+                    crc3 ^= crc_braid_table[k][(word3 >> (k << 3)) & 0xff];
+#if N > 4
+                    crc4 ^= crc_braid_table[k][(word4 >> (k << 3)) & 0xff];
+#if N > 5
+                    crc5 ^= crc_braid_table[k][(word5 >> (k << 3)) & 0xff];
+#endif
+#endif
+#endif
+#endif
+#endif
+                }
+            }
+
+            /*
+              Process the last block, combining the CRCs of the N braids at the
+              same time.
+             */
+            crc = crc_word(crc0 ^ words[0]);
+#if N > 1
+            crc = crc_word(crc1 ^ words[1] ^ crc);
+#if N > 2
+            crc = crc_word(crc2 ^ words[2] ^ crc);
+#if N > 3
+            crc = crc_word(crc3 ^ words[3] ^ crc);
+#if N > 4
+            crc = crc_word(crc4 ^ words[4] ^ crc);
+#if N > 5
+            crc = crc_word(crc5 ^ words[5] ^ crc);
+#endif
+#endif
+#endif
+#endif
+#endif
+            words += N;
+        }
+        else {
+            /* Big endian. */
+
+            z_word_t crc0, word0, comb;
+#if N > 1
+            z_word_t crc1, word1;
+#if N > 2
+            z_word_t crc2, word2;
+#if N > 3
+            z_word_t crc3, word3;
+#if N > 4
+            z_word_t crc4, word4;
+#if N > 5
+            z_word_t crc5, word5;
+#endif
+#endif
+#endif
+#endif
+#endif
+
+            /* Initialize the CRC for each braid. */
+            crc0 = byte_swap(crc);
+#if N > 1
+            crc1 = 0;
+#if N > 2
+            crc2 = 0;
+#if N > 3
+            crc3 = 0;
+#if N > 4
+            crc4 = 0;
+#if N > 5
+            crc5 = 0;
+#endif
+#endif
+#endif
+#endif
+#endif
+
+            /*
+              Process the first blks-1 blocks, computing the CRCs on each braid
+              independently.
+             */
+            while (--blks) {
+                /* Load the word for each braid into registers. */
+                word0 = crc0 ^ words[0];
+#if N > 1
+                word1 = crc1 ^ words[1];
+#if N > 2
+                word2 = crc2 ^ words[2];
+#if N > 3
+                word3 = crc3 ^ words[3];
+#if N > 4
+                word4 = crc4 ^ words[4];
+#if N > 5
+                word5 = crc5 ^ words[5];
+#endif
+#endif
+#endif
+#endif
+#endif
+                words += N;
+
+                /* Compute and update the CRC for each word. The loop should
+                   get unrolled. */
+                crc0 = crc_braid_big_table[0][word0 & 0xff];
+#if N > 1
+                crc1 = crc_braid_big_table[0][word1 & 0xff];
+#if N > 2
+                crc2 = crc_braid_big_table[0][word2 & 0xff];
+#if N > 3
+                crc3 = crc_braid_big_table[0][word3 & 0xff];
+#if N > 4
+                crc4 = crc_braid_big_table[0][word4 & 0xff];
+#if N > 5
+                crc5 = crc_braid_big_table[0][word5 & 0xff];
+#endif
+#endif
+#endif
+#endif
+#endif
+                for (k = 1; k < W; k++) {
+                    crc0 ^= crc_braid_big_table[k][(word0 >> (k << 3)) & 0xff];
+#if N > 1
+                    crc1 ^= crc_braid_big_table[k][(word1 >> (k << 3)) & 0xff];
+#if N > 2
+                    crc2 ^= crc_braid_big_table[k][(word2 >> (k << 3)) & 0xff];
+#if N > 3
+                    crc3 ^= crc_braid_big_table[k][(word3 >> (k << 3)) & 0xff];
+#if N > 4
+                    crc4 ^= crc_braid_big_table[k][(word4 >> (k << 3)) & 0xff];
+#if N > 5
+                    crc5 ^= crc_braid_big_table[k][(word5 >> (k << 3)) & 0xff];
+#endif
+#endif
+#endif
+#endif
+#endif
+                }
+            }
+
+            /*
+              Process the last block, combining the CRCs of the N braids at the
+              same time.
+             */
+            comb = crc_word_big(crc0 ^ words[0]);
+#if N > 1
+            comb = crc_word_big(crc1 ^ words[1] ^ comb);
+#if N > 2
+            comb = crc_word_big(crc2 ^ words[2] ^ comb);
+#if N > 3
+            comb = crc_word_big(crc3 ^ words[3] ^ comb);
+#if N > 4
+            comb = crc_word_big(crc4 ^ words[4] ^ comb);
+#if N > 5
+            comb = crc_word_big(crc5 ^ words[5] ^ comb);
+#endif
+#endif
+#endif
+#endif
+#endif
+            words += N;
+            crc = byte_swap(comb);
+        }
+
+        /*
+          Update the pointer to the remaining bytes to process.
+         */
+        buf = (unsigned char const *)words;
     }
 
-    buf4 = (const z_crc_t FAR *)(const void FAR *)buf;
-    while (len >= 32) {
-        DOBIG32;
-        len -= 32;
+#endif /* W */
+
+    /* Complete the computation of the CRC on any remaining bytes. */
+    while (len >= 8) {
+        len -= 8;
+        crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff];
+        crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff];
+        crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff];
+        crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff];
+        crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff];
+        crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff];
+        crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff];
+        crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff];
     }
-    while (len >= 4) {
-        DOBIG4;
-        len -= 4;
+    while (len) {
+        len--;
+        crc = (crc >> 8) ^ crc_table[(crc ^ *buf++) & 0xff];
     }
-    buf = (const unsigned char FAR *)buf4;
 
-    if (len) do {
-        c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8);
-    } while (--len);
-    c = ~c;
-    return (unsigned long)(ZSWAP32(c));
+    /* Return the CRC, post-conditioned. */
+    return crc ^ 0xffffffff;
 }
 
-#endif /* BYFOUR */
-
-#define GF2_DIM 32      /* dimension of GF(2) vectors (length of CRC) */
+#endif
 
 /* ========================================================================= */
-local unsigned long gf2_matrix_times(mat, vec)
-    unsigned long *mat;
-    unsigned long vec;
-{
-    unsigned long sum;
-
-    sum = 0;
-    while (vec) {
-        if (vec & 1)
-            sum ^= *mat;
-        vec >>= 1;
-        mat++;
-    }
-    return sum;
+unsigned long ZEXPORT crc32(unsigned long crc, const unsigned char FAR *buf,
+                            uInt len) {
+    return crc32_z(crc, buf, len);
 }
 
 /* ========================================================================= */
-local void gf2_matrix_square(square, mat)
-    unsigned long *square;
-    unsigned long *mat;
-{
-    int n;
-
-    for (n = 0; n < GF2_DIM; n++)
-        square[n] = gf2_matrix_times(mat, mat[n]);
+uLong ZEXPORT crc32_combine64(uLong crc1, uLong crc2, z_off64_t len2) {
+#ifdef DYNAMIC_CRC_TABLE
+    once(&made, make_crc_table);
+#endif /* DYNAMIC_CRC_TABLE */
+    return multmodp(x2nmodp(len2, 3), crc1) ^ (crc2 & 0xffffffff);
 }
 
 /* ========================================================================= */
-local uLong crc32_combine_(crc1, crc2, len2)
-    uLong crc1;
-    uLong crc2;
-    z_off64_t len2;
-{
-    int n;
-    unsigned long row;
-    unsigned long even[GF2_DIM];    /* even-power-of-two zeros operator */
-    unsigned long odd[GF2_DIM];     /* odd-power-of-two zeros operator */
-
-    /* degenerate case (also disallow negative lengths) */
-    if (len2 <= 0)
-        return crc1;
-
-    /* put operator for one zero bit in odd */
-    odd[0] = 0xedb88320UL;          /* CRC-32 polynomial */
-    row = 1;
-    for (n = 1; n < GF2_DIM; n++) {
-        odd[n] = row;
-        row <<= 1;
-    }
+uLong ZEXPORT crc32_combine(uLong crc1, uLong crc2, z_off_t len2) {
+    return crc32_combine64(crc1, crc2, (z_off64_t)len2);
+}
 
-    /* put operator for two zero bits in even */
-    gf2_matrix_square(even, odd);
-
-    /* put operator for four zero bits in odd */
-    gf2_matrix_square(odd, even);
-
-    /* apply len2 zeros to crc1 (first square will put the operator for one
-       zero byte, eight zero bits, in even) */
-    do {
-        /* apply zeros operator for this bit of len2 */
-        gf2_matrix_square(even, odd);
-        if (len2 & 1)
-            crc1 = gf2_matrix_times(even, crc1);
-        len2 >>= 1;
-
-        /* if no more bits set, then done */
-        if (len2 == 0)
-            break;
-
-        /* another iteration of the loop with odd and even swapped */
-        gf2_matrix_square(odd, even);
-        if (len2 & 1)
-            crc1 = gf2_matrix_times(odd, crc1);
-        len2 >>= 1;
-
-        /* if no more bits set, then done */
-    } while (len2 != 0);
-
-    /* return combined crc */
-    crc1 ^= crc2;
-    return crc1;
+/* ========================================================================= */
+uLong ZEXPORT crc32_combine_gen64(z_off64_t len2) {
+#ifdef DYNAMIC_CRC_TABLE
+    once(&made, make_crc_table);
+#endif /* DYNAMIC_CRC_TABLE */
+    return x2nmodp(len2, 3);
 }
 
 /* ========================================================================= */
-uLong ZEXPORT crc32_combine(crc1, crc2, len2)
-    uLong crc1;
-    uLong crc2;
-    z_off_t len2;
-{
-    return crc32_combine_(crc1, crc2, len2);
+uLong ZEXPORT crc32_combine_gen(z_off_t len2) {
+    return crc32_combine_gen64((z_off64_t)len2);
 }
 
-uLong ZEXPORT crc32_combine64(crc1, crc2, len2)
-    uLong crc1;
-    uLong crc2;
-    z_off64_t len2;
-{
-    return crc32_combine_(crc1, crc2, len2);
+/* ========================================================================= */
+uLong ZEXPORT crc32_combine_op(uLong crc1, uLong crc2, uLong op) {
+    return multmodp(op, crc1) ^ (crc2 & 0xffffffff);
 }
diff --git a/zlib/crc32.h b/zlib/crc32.h
index 9e0c778..137df68 100644
--- a/zlib/crc32.h
+++ b/zlib/crc32.h
@@ -2,440 +2,9445 @@
  * Generated automatically by crc32.c
  */
 
-local const z_crc_t FAR crc_table[TBLS][256] =
-{
-  {
-    0x00000000UL, 0x77073096UL, 0xee0e612cUL, 0x990951baUL, 0x076dc419UL,
-    0x706af48fUL, 0xe963a535UL, 0x9e6495a3UL, 0x0edb8832UL, 0x79dcb8a4UL,
-    0xe0d5e91eUL, 0x97d2d988UL, 0x09b64c2bUL, 0x7eb17cbdUL, 0xe7b82d07UL,
-    0x90bf1d91UL, 0x1db71064UL, 0x6ab020f2UL, 0xf3b97148UL, 0x84be41deUL,
-    0x1adad47dUL, 0x6ddde4ebUL, 0xf4d4b551UL, 0x83d385c7UL, 0x136c9856UL,
-    0x646ba8c0UL, 0xfd62f97aUL, 0x8a65c9ecUL, 0x14015c4fUL, 0x63066cd9UL,
-    0xfa0f3d63UL, 0x8d080df5UL, 0x3b6e20c8UL, 0x4c69105eUL, 0xd56041e4UL,
-    0xa2677172UL, 0x3c03e4d1UL, 0x4b04d447UL, 0xd20d85fdUL, 0xa50ab56bUL,
-    0x35b5a8faUL, 0x42b2986cUL, 0xdbbbc9d6UL, 0xacbcf940UL, 0x32d86ce3UL,
-    0x45df5c75UL, 0xdcd60dcfUL, 0xabd13d59UL, 0x26d930acUL, 0x51de003aUL,
-    0xc8d75180UL, 0xbfd06116UL, 0x21b4f4b5UL, 0x56b3c423UL, 0xcfba9599UL,
-    0xb8bda50fUL, 0x2802b89eUL, 0x5f058808UL, 0xc60cd9b2UL, 0xb10be924UL,
-    0x2f6f7c87UL, 0x58684c11UL, 0xc1611dabUL, 0xb6662d3dUL, 0x76dc4190UL,
-    0x01db7106UL, 0x98d220bcUL, 0xefd5102aUL, 0x71b18589UL, 0x06b6b51fUL,
-    0x9fbfe4a5UL, 0xe8b8d433UL, 0x7807c9a2UL, 0x0f00f934UL, 0x9609a88eUL,
-    0xe10e9818UL, 0x7f6a0dbbUL, 0x086d3d2dUL, 0x91646c97UL, 0xe6635c01UL,
-    0x6b6b51f4UL, 0x1c6c6162UL, 0x856530d8UL, 0xf262004eUL, 0x6c0695edUL,
-    0x1b01a57bUL, 0x8208f4c1UL, 0xf50fc457UL, 0x65b0d9c6UL, 0x12b7e950UL,
-    0x8bbeb8eaUL, 0xfcb9887cUL, 0x62dd1ddfUL, 0x15da2d49UL, 0x8cd37cf3UL,
-    0xfbd44c65UL, 0x4db26158UL, 0x3ab551ceUL, 0xa3bc0074UL, 0xd4bb30e2UL,
-    0x4adfa541UL, 0x3dd895d7UL, 0xa4d1c46dUL, 0xd3d6f4fbUL, 0x4369e96aUL,
-    0x346ed9fcUL, 0xad678846UL, 0xda60b8d0UL, 0x44042d73UL, 0x33031de5UL,
-    0xaa0a4c5fUL, 0xdd0d7cc9UL, 0x5005713cUL, 0x270241aaUL, 0xbe0b1010UL,
-    0xc90c2086UL, 0x5768b525UL, 0x206f85b3UL, 0xb966d409UL, 0xce61e49fUL,
-    0x5edef90eUL, 0x29d9c998UL, 0xb0d09822UL, 0xc7d7a8b4UL, 0x59b33d17UL,
-    0x2eb40d81UL, 0xb7bd5c3bUL, 0xc0ba6cadUL, 0xedb88320UL, 0x9abfb3b6UL,
-    0x03b6e20cUL, 0x74b1d29aUL, 0xead54739UL, 0x9dd277afUL, 0x04db2615UL,
-    0x73dc1683UL, 0xe3630b12UL, 0x94643b84UL, 0x0d6d6a3eUL, 0x7a6a5aa8UL,
-    0xe40ecf0bUL, 0x9309ff9dUL, 0x0a00ae27UL, 0x7d079eb1UL, 0xf00f9344UL,
-    0x8708a3d2UL, 0x1e01f268UL, 0x6906c2feUL, 0xf762575dUL, 0x806567cbUL,
-    0x196c3671UL, 0x6e6b06e7UL, 0xfed41b76UL, 0x89d32be0UL, 0x10da7a5aUL,
-    0x67dd4accUL, 0xf9b9df6fUL, 0x8ebeeff9UL, 0x17b7be43UL, 0x60b08ed5UL,
-    0xd6d6a3e8UL, 0xa1d1937eUL, 0x38d8c2c4UL, 0x4fdff252UL, 0xd1bb67f1UL,
-    0xa6bc5767UL, 0x3fb506ddUL, 0x48b2364bUL, 0xd80d2bdaUL, 0xaf0a1b4cUL,
-    0x36034af6UL, 0x41047a60UL, 0xdf60efc3UL, 0xa867df55UL, 0x316e8eefUL,
-    0x4669be79UL, 0xcb61b38cUL, 0xbc66831aUL, 0x256fd2a0UL, 0x5268e236UL,
-    0xcc0c7795UL, 0xbb0b4703UL, 0x220216b9UL, 0x5505262fUL, 0xc5ba3bbeUL,
-    0xb2bd0b28UL, 0x2bb45a92UL, 0x5cb36a04UL, 0xc2d7ffa7UL, 0xb5d0cf31UL,
-    0x2cd99e8bUL, 0x5bdeae1dUL, 0x9b64c2b0UL, 0xec63f226UL, 0x756aa39cUL,
-    0x026d930aUL, 0x9c0906a9UL, 0xeb0e363fUL, 0x72076785UL, 0x05005713UL,
-    0x95bf4a82UL, 0xe2b87a14UL, 0x7bb12baeUL, 0x0cb61b38UL, 0x92d28e9bUL,
-    0xe5d5be0dUL, 0x7cdcefb7UL, 0x0bdbdf21UL, 0x86d3d2d4UL, 0xf1d4e242UL,
-    0x68ddb3f8UL, 0x1fda836eUL, 0x81be16cdUL, 0xf6b9265bUL, 0x6fb077e1UL,
-    0x18b74777UL, 0x88085ae6UL, 0xff0f6a70UL, 0x66063bcaUL, 0x11010b5cUL,
-    0x8f659effUL, 0xf862ae69UL, 0x616bffd3UL, 0x166ccf45UL, 0xa00ae278UL,
-    0xd70dd2eeUL, 0x4e048354UL, 0x3903b3c2UL, 0xa7672661UL, 0xd06016f7UL,
-    0x4969474dUL, 0x3e6e77dbUL, 0xaed16a4aUL, 0xd9d65adcUL, 0x40df0b66UL,
-    0x37d83bf0UL, 0xa9bcae53UL, 0xdebb9ec5UL, 0x47b2cf7fUL, 0x30b5ffe9UL,
-    0xbdbdf21cUL, 0xcabac28aUL, 0x53b39330UL, 0x24b4a3a6UL, 0xbad03605UL,
-    0xcdd70693UL, 0x54de5729UL, 0x23d967bfUL, 0xb3667a2eUL, 0xc4614ab8UL,
-    0x5d681b02UL, 0x2a6f2b94UL, 0xb40bbe37UL, 0xc30c8ea1UL, 0x5a05df1bUL,
-    0x2d02ef8dUL
-#ifdef BYFOUR
-  },
-  {
-    0x00000000UL, 0x191b3141UL, 0x32366282UL, 0x2b2d53c3UL, 0x646cc504UL,
-    0x7d77f445UL, 0x565aa786UL, 0x4f4196c7UL, 0xc8d98a08UL, 0xd1c2bb49UL,
-    0xfaefe88aUL, 0xe3f4d9cbUL, 0xacb54f0cUL, 0xb5ae7e4dUL, 0x9e832d8eUL,
-    0x87981ccfUL, 0x4ac21251UL, 0x53d92310UL, 0x78f470d3UL, 0x61ef4192UL,
-    0x2eaed755UL, 0x37b5e614UL, 0x1c98b5d7UL, 0x05838496UL, 0x821b9859UL,
-    0x9b00a918UL, 0xb02dfadbUL, 0xa936cb9aUL, 0xe6775d5dUL, 0xff6c6c1cUL,
-    0xd4413fdfUL, 0xcd5a0e9eUL, 0x958424a2UL, 0x8c9f15e3UL, 0xa7b24620UL,
-    0xbea97761UL, 0xf1e8e1a6UL, 0xe8f3d0e7UL, 0xc3de8324UL, 0xdac5b265UL,
-    0x5d5daeaaUL, 0x44469febUL, 0x6f6bcc28UL, 0x7670fd69UL, 0x39316baeUL,
-    0x202a5aefUL, 0x0b07092cUL, 0x121c386dUL, 0xdf4636f3UL, 0xc65d07b2UL,
-    0xed705471UL, 0xf46b6530UL, 0xbb2af3f7UL, 0xa231c2b6UL, 0x891c9175UL,
-    0x9007a034UL, 0x179fbcfbUL, 0x0e848dbaUL, 0x25a9de79UL, 0x3cb2ef38UL,
-    0x73f379ffUL, 0x6ae848beUL, 0x41c51b7dUL, 0x58de2a3cUL, 0xf0794f05UL,
-    0xe9627e44UL, 0xc24f2d87UL, 0xdb541cc6UL, 0x94158a01UL, 0x8d0ebb40UL,
-    0xa623e883UL, 0xbf38d9c2UL, 0x38a0c50dUL, 0x21bbf44cUL, 0x0a96a78fUL,
-    0x138d96ceUL, 0x5ccc0009UL, 0x45d73148UL, 0x6efa628bUL, 0x77e153caUL,
-    0xbabb5d54UL, 0xa3a06c15UL, 0x888d3fd6UL, 0x91960e97UL, 0xded79850UL,
-    0xc7cca911UL, 0xece1fad2UL, 0xf5facb93UL, 0x7262d75cUL, 0x6b79e61dUL,
-    0x4054b5deUL, 0x594f849fUL, 0x160e1258UL, 0x0f152319UL, 0x243870daUL,
-    0x3d23419bUL, 0x65fd6ba7UL, 0x7ce65ae6UL, 0x57cb0925UL, 0x4ed03864UL,
-    0x0191aea3UL, 0x188a9fe2UL, 0x33a7cc21UL, 0x2abcfd60UL, 0xad24e1afUL,
-    0xb43fd0eeUL, 0x9f12832dUL, 0x8609b26cUL, 0xc94824abUL, 0xd05315eaUL,
-    0xfb7e4629UL, 0xe2657768UL, 0x2f3f79f6UL, 0x362448b7UL, 0x1d091b74UL,
-    0x04122a35UL, 0x4b53bcf2UL, 0x52488db3UL, 0x7965de70UL, 0x607eef31UL,
-    0xe7e6f3feUL, 0xfefdc2bfUL, 0xd5d0917cUL, 0xcccba03dUL, 0x838a36faUL,
-    0x9a9107bbUL, 0xb1bc5478UL, 0xa8a76539UL, 0x3b83984bUL, 0x2298a90aUL,
-    0x09b5fac9UL, 0x10aecb88UL, 0x5fef5d4fUL, 0x46f46c0eUL, 0x6dd93fcdUL,
-    0x74c20e8cUL, 0xf35a1243UL, 0xea412302UL, 0xc16c70c1UL, 0xd8774180UL,
-    0x9736d747UL, 0x8e2de606UL, 0xa500b5c5UL, 0xbc1b8484UL, 0x71418a1aUL,
-    0x685abb5bUL, 0x4377e898UL, 0x5a6cd9d9UL, 0x152d4f1eUL, 0x0c367e5fUL,
-    0x271b2d9cUL, 0x3e001cddUL, 0xb9980012UL, 0xa0833153UL, 0x8bae6290UL,
-    0x92b553d1UL, 0xddf4c516UL, 0xc4eff457UL, 0xefc2a794UL, 0xf6d996d5UL,
-    0xae07bce9UL, 0xb71c8da8UL, 0x9c31de6bUL, 0x852aef2aUL, 0xca6b79edUL,
-    0xd37048acUL, 0xf85d1b6fUL, 0xe1462a2eUL, 0x66de36e1UL, 0x7fc507a0UL,
-    0x54e85463UL, 0x4df36522UL, 0x02b2f3e5UL, 0x1ba9c2a4UL, 0x30849167UL,
-    0x299fa026UL, 0xe4c5aeb8UL, 0xfdde9ff9UL, 0xd6f3cc3aUL, 0xcfe8fd7bUL,
-    0x80a96bbcUL, 0x99b25afdUL, 0xb29f093eUL, 0xab84387fUL, 0x2c1c24b0UL,
-    0x350715f1UL, 0x1e2a4632UL, 0x07317773UL, 0x4870e1b4UL, 0x516bd0f5UL,
-    0x7a468336UL, 0x635db277UL, 0xcbfad74eUL, 0xd2e1e60fUL, 0xf9ccb5ccUL,
-    0xe0d7848dUL, 0xaf96124aUL, 0xb68d230bUL, 0x9da070c8UL, 0x84bb4189UL,
-    0x03235d46UL, 0x1a386c07UL, 0x31153fc4UL, 0x280e0e85UL, 0x674f9842UL,
-    0x7e54a903UL, 0x5579fac0UL, 0x4c62cb81UL, 0x8138c51fUL, 0x9823f45eUL,
-    0xb30ea79dUL, 0xaa1596dcUL, 0xe554001bUL, 0xfc4f315aUL, 0xd7626299UL,
-    0xce7953d8UL, 0x49e14f17UL, 0x50fa7e56UL, 0x7bd72d95UL, 0x62cc1cd4UL,
-    0x2d8d8a13UL, 0x3496bb52UL, 0x1fbbe891UL, 0x06a0d9d0UL, 0x5e7ef3ecUL,
-    0x4765c2adUL, 0x6c48916eUL, 0x7553a02fUL, 0x3a1236e8UL, 0x230907a9UL,
-    0x0824546aUL, 0x113f652bUL, 0x96a779e4UL, 0x8fbc48a5UL, 0xa4911b66UL,
-    0xbd8a2a27UL, 0xf2cbbce0UL, 0xebd08da1UL, 0xc0fdde62UL, 0xd9e6ef23UL,
-    0x14bce1bdUL, 0x0da7d0fcUL, 0x268a833fUL, 0x3f91b27eUL, 0x70d024b9UL,
-    0x69cb15f8UL, 0x42e6463bUL, 0x5bfd777aUL, 0xdc656bb5UL, 0xc57e5af4UL,
-    0xee530937UL, 0xf7483876UL, 0xb809aeb1UL, 0xa1129ff0UL, 0x8a3fcc33UL,
-    0x9324fd72UL
-  },
-  {
-    0x00000000UL, 0x01c26a37UL, 0x0384d46eUL, 0x0246be59UL, 0x0709a8dcUL,
-    0x06cbc2ebUL, 0x048d7cb2UL, 0x054f1685UL, 0x0e1351b8UL, 0x0fd13b8fUL,
-    0x0d9785d6UL, 0x0c55efe1UL, 0x091af964UL, 0x08d89353UL, 0x0a9e2d0aUL,
-    0x0b5c473dUL, 0x1c26a370UL, 0x1de4c947UL, 0x1fa2771eUL, 0x1e601d29UL,
-    0x1b2f0bacUL, 0x1aed619bUL, 0x18abdfc2UL, 0x1969b5f5UL, 0x1235f2c8UL,
-    0x13f798ffUL, 0x11b126a6UL, 0x10734c91UL, 0x153c5a14UL, 0x14fe3023UL,
-    0x16b88e7aUL, 0x177ae44dUL, 0x384d46e0UL, 0x398f2cd7UL, 0x3bc9928eUL,
-    0x3a0bf8b9UL, 0x3f44ee3cUL, 0x3e86840bUL, 0x3cc03a52UL, 0x3d025065UL,
-    0x365e1758UL, 0x379c7d6fUL, 0x35dac336UL, 0x3418a901UL, 0x3157bf84UL,
-    0x3095d5b3UL, 0x32d36beaUL, 0x331101ddUL, 0x246be590UL, 0x25a98fa7UL,
-    0x27ef31feUL, 0x262d5bc9UL, 0x23624d4cUL, 0x22a0277bUL, 0x20e69922UL,
-    0x2124f315UL, 0x2a78b428UL, 0x2bbade1fUL, 0x29fc6046UL, 0x283e0a71UL,
-    0x2d711cf4UL, 0x2cb376c3UL, 0x2ef5c89aUL, 0x2f37a2adUL, 0x709a8dc0UL,
-    0x7158e7f7UL, 0x731e59aeUL, 0x72dc3399UL, 0x7793251cUL, 0x76514f2bUL,
-    0x7417f172UL, 0x75d59b45UL, 0x7e89dc78UL, 0x7f4bb64fUL, 0x7d0d0816UL,
-    0x7ccf6221UL, 0x798074a4UL, 0x78421e93UL, 0x7a04a0caUL, 0x7bc6cafdUL,
-    0x6cbc2eb0UL, 0x6d7e4487UL, 0x6f38fadeUL, 0x6efa90e9UL, 0x6bb5866cUL,
-    0x6a77ec5bUL, 0x68315202UL, 0x69f33835UL, 0x62af7f08UL, 0x636d153fUL,
-    0x612bab66UL, 0x60e9c151UL, 0x65a6d7d4UL, 0x6464bde3UL, 0x662203baUL,
-    0x67e0698dUL, 0x48d7cb20UL, 0x4915a117UL, 0x4b531f4eUL, 0x4a917579UL,
-    0x4fde63fcUL, 0x4e1c09cbUL, 0x4c5ab792UL, 0x4d98dda5UL, 0x46c49a98UL,
-    0x4706f0afUL, 0x45404ef6UL, 0x448224c1UL, 0x41cd3244UL, 0x400f5873UL,
-    0x4249e62aUL, 0x438b8c1dUL, 0x54f16850UL, 0x55330267UL, 0x5775bc3eUL,
-    0x56b7d609UL, 0x53f8c08cUL, 0x523aaabbUL, 0x507c14e2UL, 0x51be7ed5UL,
-    0x5ae239e8UL, 0x5b2053dfUL, 0x5966ed86UL, 0x58a487b1UL, 0x5deb9134UL,
-    0x5c29fb03UL, 0x5e6f455aUL, 0x5fad2f6dUL, 0xe1351b80UL, 0xe0f771b7UL,
-    0xe2b1cfeeUL, 0xe373a5d9UL, 0xe63cb35cUL, 0xe7fed96bUL, 0xe5b86732UL,
-    0xe47a0d05UL, 0xef264a38UL, 0xeee4200fUL, 0xeca29e56UL, 0xed60f461UL,
-    0xe82fe2e4UL, 0xe9ed88d3UL, 0xebab368aUL, 0xea695cbdUL, 0xfd13b8f0UL,
-    0xfcd1d2c7UL, 0xfe976c9eUL, 0xff5506a9UL, 0xfa1a102cUL, 0xfbd87a1bUL,
-    0xf99ec442UL, 0xf85cae75UL, 0xf300e948UL, 0xf2c2837fUL, 0xf0843d26UL,
-    0xf1465711UL, 0xf4094194UL, 0xf5cb2ba3UL, 0xf78d95faUL, 0xf64fffcdUL,
-    0xd9785d60UL, 0xd8ba3757UL, 0xdafc890eUL, 0xdb3ee339UL, 0xde71f5bcUL,
-    0xdfb39f8bUL, 0xddf521d2UL, 0xdc374be5UL, 0xd76b0cd8UL, 0xd6a966efUL,
-    0xd4efd8b6UL, 0xd52db281UL, 0xd062a404UL, 0xd1a0ce33UL, 0xd3e6706aUL,
-    0xd2241a5dUL, 0xc55efe10UL, 0xc49c9427UL, 0xc6da2a7eUL, 0xc7184049UL,
-    0xc25756ccUL, 0xc3953cfbUL, 0xc1d382a2UL, 0xc011e895UL, 0xcb4dafa8UL,
-    0xca8fc59fUL, 0xc8c97bc6UL, 0xc90b11f1UL, 0xcc440774UL, 0xcd866d43UL,
-    0xcfc0d31aUL, 0xce02b92dUL, 0x91af9640UL, 0x906dfc77UL, 0x922b422eUL,
-    0x93e92819UL, 0x96a63e9cUL, 0x976454abUL, 0x9522eaf2UL, 0x94e080c5UL,
-    0x9fbcc7f8UL, 0x9e7eadcfUL, 0x9c381396UL, 0x9dfa79a1UL, 0x98b56f24UL,
-    0x99770513UL, 0x9b31bb4aUL, 0x9af3d17dUL, 0x8d893530UL, 0x8c4b5f07UL,
-    0x8e0de15eUL, 0x8fcf8b69UL, 0x8a809decUL, 0x8b42f7dbUL, 0x89044982UL,
-    0x88c623b5UL, 0x839a6488UL, 0x82580ebfUL, 0x801eb0e6UL, 0x81dcdad1UL,
-    0x8493cc54UL, 0x8551a663UL, 0x8717183aUL, 0x86d5720dUL, 0xa9e2d0a0UL,
-    0xa820ba97UL, 0xaa6604ceUL, 0xaba46ef9UL, 0xaeeb787cUL, 0xaf29124bUL,
-    0xad6fac12UL, 0xacadc625UL, 0xa7f18118UL, 0xa633eb2fUL, 0xa4755576UL,
-    0xa5b73f41UL, 0xa0f829c4UL, 0xa13a43f3UL, 0xa37cfdaaUL, 0xa2be979dUL,
-    0xb5c473d0UL, 0xb40619e7UL, 0xb640a7beUL, 0xb782cd89UL, 0xb2cddb0cUL,
-    0xb30fb13bUL, 0xb1490f62UL, 0xb08b6555UL, 0xbbd72268UL, 0xba15485fUL,
-    0xb853f606UL, 0xb9919c31UL, 0xbcde8ab4UL, 0xbd1ce083UL, 0xbf5a5edaUL,
-    0xbe9834edUL
-  },
-  {
-    0x00000000UL, 0xb8bc6765UL, 0xaa09c88bUL, 0x12b5afeeUL, 0x8f629757UL,
-    0x37def032UL, 0x256b5fdcUL, 0x9dd738b9UL, 0xc5b428efUL, 0x7d084f8aUL,
-    0x6fbde064UL, 0xd7018701UL, 0x4ad6bfb8UL, 0xf26ad8ddUL, 0xe0df7733UL,
-    0x58631056UL, 0x5019579fUL, 0xe8a530faUL, 0xfa109f14UL, 0x42acf871UL,
-    0xdf7bc0c8UL, 0x67c7a7adUL, 0x75720843UL, 0xcdce6f26UL, 0x95ad7f70UL,
-    0x2d111815UL, 0x3fa4b7fbUL, 0x8718d09eUL, 0x1acfe827UL, 0xa2738f42UL,
-    0xb0c620acUL, 0x087a47c9UL, 0xa032af3eUL, 0x188ec85bUL, 0x0a3b67b5UL,
-    0xb28700d0UL, 0x2f503869UL, 0x97ec5f0cUL, 0x8559f0e2UL, 0x3de59787UL,
-    0x658687d1UL, 0xdd3ae0b4UL, 0xcf8f4f5aUL, 0x7733283fUL, 0xeae41086UL,
-    0x525877e3UL, 0x40edd80dUL, 0xf851bf68UL, 0xf02bf8a1UL, 0x48979fc4UL,
-    0x5a22302aUL, 0xe29e574fUL, 0x7f496ff6UL, 0xc7f50893UL, 0xd540a77dUL,
-    0x6dfcc018UL, 0x359fd04eUL, 0x8d23b72bUL, 0x9f9618c5UL, 0x272a7fa0UL,
-    0xbafd4719UL, 0x0241207cUL, 0x10f48f92UL, 0xa848e8f7UL, 0x9b14583dUL,
-    0x23a83f58UL, 0x311d90b6UL, 0x89a1f7d3UL, 0x1476cf6aUL, 0xaccaa80fUL,
-    0xbe7f07e1UL, 0x06c36084UL, 0x5ea070d2UL, 0xe61c17b7UL, 0xf4a9b859UL,
-    0x4c15df3cUL, 0xd1c2e785UL, 0x697e80e0UL, 0x7bcb2f0eUL, 0xc377486bUL,
-    0xcb0d0fa2UL, 0x73b168c7UL, 0x6104c729UL, 0xd9b8a04cUL, 0x446f98f5UL,
-    0xfcd3ff90UL, 0xee66507eUL, 0x56da371bUL, 0x0eb9274dUL, 0xb6054028UL,
-    0xa4b0efc6UL, 0x1c0c88a3UL, 0x81dbb01aUL, 0x3967d77fUL, 0x2bd27891UL,
-    0x936e1ff4UL, 0x3b26f703UL, 0x839a9066UL, 0x912f3f88UL, 0x299358edUL,
-    0xb4446054UL, 0x0cf80731UL, 0x1e4da8dfUL, 0xa6f1cfbaUL, 0xfe92dfecUL,
-    0x462eb889UL, 0x549b1767UL, 0xec277002UL, 0x71f048bbUL, 0xc94c2fdeUL,
-    0xdbf98030UL, 0x6345e755UL, 0x6b3fa09cUL, 0xd383c7f9UL, 0xc1366817UL,
-    0x798a0f72UL, 0xe45d37cbUL, 0x5ce150aeUL, 0x4e54ff40UL, 0xf6e89825UL,
-    0xae8b8873UL, 0x1637ef16UL, 0x048240f8UL, 0xbc3e279dUL, 0x21e91f24UL,
-    0x99557841UL, 0x8be0d7afUL, 0x335cb0caUL, 0xed59b63bUL, 0x55e5d15eUL,
-    0x47507eb0UL, 0xffec19d5UL, 0x623b216cUL, 0xda874609UL, 0xc832e9e7UL,
-    0x708e8e82UL, 0x28ed9ed4UL, 0x9051f9b1UL, 0x82e4565fUL, 0x3a58313aUL,
-    0xa78f0983UL, 0x1f336ee6UL, 0x0d86c108UL, 0xb53aa66dUL, 0xbd40e1a4UL,
-    0x05fc86c1UL, 0x1749292fUL, 0xaff54e4aUL, 0x322276f3UL, 0x8a9e1196UL,
-    0x982bbe78UL, 0x2097d91dUL, 0x78f4c94bUL, 0xc048ae2eUL, 0xd2fd01c0UL,
-    0x6a4166a5UL, 0xf7965e1cUL, 0x4f2a3979UL, 0x5d9f9697UL, 0xe523f1f2UL,
-    0x4d6b1905UL, 0xf5d77e60UL, 0xe762d18eUL, 0x5fdeb6ebUL, 0xc2098e52UL,
-    0x7ab5e937UL, 0x680046d9UL, 0xd0bc21bcUL, 0x88df31eaUL, 0x3063568fUL,
-    0x22d6f961UL, 0x9a6a9e04UL, 0x07bda6bdUL, 0xbf01c1d8UL, 0xadb46e36UL,
-    0x15080953UL, 0x1d724e9aUL, 0xa5ce29ffUL, 0xb77b8611UL, 0x0fc7e174UL,
-    0x9210d9cdUL, 0x2aacbea8UL, 0x38191146UL, 0x80a57623UL, 0xd8c66675UL,
-    0x607a0110UL, 0x72cfaefeUL, 0xca73c99bUL, 0x57a4f122UL, 0xef189647UL,
-    0xfdad39a9UL, 0x45115eccUL, 0x764dee06UL, 0xcef18963UL, 0xdc44268dUL,
-    0x64f841e8UL, 0xf92f7951UL, 0x41931e34UL, 0x5326b1daUL, 0xeb9ad6bfUL,
-    0xb3f9c6e9UL, 0x0b45a18cUL, 0x19f00e62UL, 0xa14c6907UL, 0x3c9b51beUL,
-    0x842736dbUL, 0x96929935UL, 0x2e2efe50UL, 0x2654b999UL, 0x9ee8defcUL,
-    0x8c5d7112UL, 0x34e11677UL, 0xa9362eceUL, 0x118a49abUL, 0x033fe645UL,
-    0xbb838120UL, 0xe3e09176UL, 0x5b5cf613UL, 0x49e959fdUL, 0xf1553e98UL,
-    0x6c820621UL, 0xd43e6144UL, 0xc68bceaaUL, 0x7e37a9cfUL, 0xd67f4138UL,
-    0x6ec3265dUL, 0x7c7689b3UL, 0xc4caeed6UL, 0x591dd66fUL, 0xe1a1b10aUL,
-    0xf3141ee4UL, 0x4ba87981UL, 0x13cb69d7UL, 0xab770eb2UL, 0xb9c2a15cUL,
-    0x017ec639UL, 0x9ca9fe80UL, 0x241599e5UL, 0x36a0360bUL, 0x8e1c516eUL,
-    0x866616a7UL, 0x3eda71c2UL, 0x2c6fde2cUL, 0x94d3b949UL, 0x090481f0UL,
-    0xb1b8e695UL, 0xa30d497bUL, 0x1bb12e1eUL, 0x43d23e48UL, 0xfb6e592dUL,
-    0xe9dbf6c3UL, 0x516791a6UL, 0xccb0a91fUL, 0x740cce7aUL, 0x66b96194UL,
-    0xde0506f1UL
-  },
-  {
-    0x00000000UL, 0x96300777UL, 0x2c610eeeUL, 0xba510999UL, 0x19c46d07UL,
-    0x8ff46a70UL, 0x35a563e9UL, 0xa395649eUL, 0x3288db0eUL, 0xa4b8dc79UL,
-    0x1ee9d5e0UL, 0x88d9d297UL, 0x2b4cb609UL, 0xbd7cb17eUL, 0x072db8e7UL,
-    0x911dbf90UL, 0x6410b71dUL, 0xf220b06aUL, 0x4871b9f3UL, 0xde41be84UL,
-    0x7dd4da1aUL, 0xebe4dd6dUL, 0x51b5d4f4UL, 0xc785d383UL, 0x56986c13UL,
-    0xc0a86b64UL, 0x7af962fdUL, 0xecc9658aUL, 0x4f5c0114UL, 0xd96c0663UL,
-    0x633d0ffaUL, 0xf50d088dUL, 0xc8206e3bUL, 0x5e10694cUL, 0xe44160d5UL,
-    0x727167a2UL, 0xd1e4033cUL, 0x47d4044bUL, 0xfd850dd2UL, 0x6bb50aa5UL,
-    0xfaa8b535UL, 0x6c98b242UL, 0xd6c9bbdbUL, 0x40f9bcacUL, 0xe36cd832UL,
-    0x755cdf45UL, 0xcf0dd6dcUL, 0x593dd1abUL, 0xac30d926UL, 0x3a00de51UL,
-    0x8051d7c8UL, 0x1661d0bfUL, 0xb5f4b421UL, 0x23c4b356UL, 0x9995bacfUL,
-    0x0fa5bdb8UL, 0x9eb80228UL, 0x0888055fUL, 0xb2d90cc6UL, 0x24e90bb1UL,
-    0x877c6f2fUL, 0x114c6858UL, 0xab1d61c1UL, 0x3d2d66b6UL, 0x9041dc76UL,
-    0x0671db01UL, 0xbc20d298UL, 0x2a10d5efUL, 0x8985b171UL, 0x1fb5b606UL,
-    0xa5e4bf9fUL, 0x33d4b8e8UL, 0xa2c90778UL, 0x34f9000fUL, 0x8ea80996UL,
-    0x18980ee1UL, 0xbb0d6a7fUL, 0x2d3d6d08UL, 0x976c6491UL, 0x015c63e6UL,
-    0xf4516b6bUL, 0x62616c1cUL, 0xd8306585UL, 0x4e0062f2UL, 0xed95066cUL,
-    0x7ba5011bUL, 0xc1f40882UL, 0x57c40ff5UL, 0xc6d9b065UL, 0x50e9b712UL,
-    0xeab8be8bUL, 0x7c88b9fcUL, 0xdf1ddd62UL, 0x492dda15UL, 0xf37cd38cUL,
-    0x654cd4fbUL, 0x5861b24dUL, 0xce51b53aUL, 0x7400bca3UL, 0xe230bbd4UL,
-    0x41a5df4aUL, 0xd795d83dUL, 0x6dc4d1a4UL, 0xfbf4d6d3UL, 0x6ae96943UL,
-    0xfcd96e34UL, 0x468867adUL, 0xd0b860daUL, 0x732d0444UL, 0xe51d0333UL,
-    0x5f4c0aaaUL, 0xc97c0dddUL, 0x3c710550UL, 0xaa410227UL, 0x10100bbeUL,
-    0x86200cc9UL, 0x25b56857UL, 0xb3856f20UL, 0x09d466b9UL, 0x9fe461ceUL,
-    0x0ef9de5eUL, 0x98c9d929UL, 0x2298d0b0UL, 0xb4a8d7c7UL, 0x173db359UL,
-    0x810db42eUL, 0x3b5cbdb7UL, 0xad6cbac0UL, 0x2083b8edUL, 0xb6b3bf9aUL,
-    0x0ce2b603UL, 0x9ad2b174UL, 0x3947d5eaUL, 0xaf77d29dUL, 0x1526db04UL,
-    0x8316dc73UL, 0x120b63e3UL, 0x843b6494UL, 0x3e6a6d0dUL, 0xa85a6a7aUL,
-    0x0bcf0ee4UL, 0x9dff0993UL, 0x27ae000aUL, 0xb19e077dUL, 0x44930ff0UL,
-    0xd2a30887UL, 0x68f2011eUL, 0xfec20669UL, 0x5d5762f7UL, 0xcb676580UL,
-    0x71366c19UL, 0xe7066b6eUL, 0x761bd4feUL, 0xe02bd389UL, 0x5a7ada10UL,
-    0xcc4add67UL, 0x6fdfb9f9UL, 0xf9efbe8eUL, 0x43beb717UL, 0xd58eb060UL,
-    0xe8a3d6d6UL, 0x7e93d1a1UL, 0xc4c2d838UL, 0x52f2df4fUL, 0xf167bbd1UL,
-    0x6757bca6UL, 0xdd06b53fUL, 0x4b36b248UL, 0xda2b0dd8UL, 0x4c1b0aafUL,
-    0xf64a0336UL, 0x607a0441UL, 0xc3ef60dfUL, 0x55df67a8UL, 0xef8e6e31UL,
-    0x79be6946UL, 0x8cb361cbUL, 0x1a8366bcUL, 0xa0d26f25UL, 0x36e26852UL,
-    0x95770cccUL, 0x03470bbbUL, 0xb9160222UL, 0x2f260555UL, 0xbe3bbac5UL,
-    0x280bbdb2UL, 0x925ab42bUL, 0x046ab35cUL, 0xa7ffd7c2UL, 0x31cfd0b5UL,
-    0x8b9ed92cUL, 0x1daede5bUL, 0xb0c2649bUL, 0x26f263ecUL, 0x9ca36a75UL,
-    0x0a936d02UL, 0xa906099cUL, 0x3f360eebUL, 0x85670772UL, 0x13570005UL,
-    0x824abf95UL, 0x147ab8e2UL, 0xae2bb17bUL, 0x381bb60cUL, 0x9b8ed292UL,
-    0x0dbed5e5UL, 0xb7efdc7cUL, 0x21dfdb0bUL, 0xd4d2d386UL, 0x42e2d4f1UL,
-    0xf8b3dd68UL, 0x6e83da1fUL, 0xcd16be81UL, 0x5b26b9f6UL, 0xe177b06fUL,
-    0x7747b718UL, 0xe65a0888UL, 0x706a0fffUL, 0xca3b0666UL, 0x5c0b0111UL,
-    0xff9e658fUL, 0x69ae62f8UL, 0xd3ff6b61UL, 0x45cf6c16UL, 0x78e20aa0UL,
-    0xeed20dd7UL, 0x5483044eUL, 0xc2b30339UL, 0x612667a7UL, 0xf71660d0UL,
-    0x4d476949UL, 0xdb776e3eUL, 0x4a6ad1aeUL, 0xdc5ad6d9UL, 0x660bdf40UL,
-    0xf03bd837UL, 0x53aebca9UL, 0xc59ebbdeUL, 0x7fcfb247UL, 0xe9ffb530UL,
-    0x1cf2bdbdUL, 0x8ac2bacaUL, 0x3093b353UL, 0xa6a3b424UL, 0x0536d0baUL,
-    0x9306d7cdUL, 0x2957de54UL, 0xbf67d923UL, 0x2e7a66b3UL, 0xb84a61c4UL,
-    0x021b685dUL, 0x942b6f2aUL, 0x37be0bb4UL, 0xa18e0cc3UL, 0x1bdf055aUL,
-    0x8def022dUL
-  },
-  {
-    0x00000000UL, 0x41311b19UL, 0x82623632UL, 0xc3532d2bUL, 0x04c56c64UL,
-    0x45f4777dUL, 0x86a75a56UL, 0xc796414fUL, 0x088ad9c8UL, 0x49bbc2d1UL,
-    0x8ae8effaUL, 0xcbd9f4e3UL, 0x0c4fb5acUL, 0x4d7eaeb5UL, 0x8e2d839eUL,
-    0xcf1c9887UL, 0x5112c24aUL, 0x1023d953UL, 0xd370f478UL, 0x9241ef61UL,
-    0x55d7ae2eUL, 0x14e6b537UL, 0xd7b5981cUL, 0x96848305UL, 0x59981b82UL,
-    0x18a9009bUL, 0xdbfa2db0UL, 0x9acb36a9UL, 0x5d5d77e6UL, 0x1c6c6cffUL,
-    0xdf3f41d4UL, 0x9e0e5acdUL, 0xa2248495UL, 0xe3159f8cUL, 0x2046b2a7UL,
-    0x6177a9beUL, 0xa6e1e8f1UL, 0xe7d0f3e8UL, 0x2483dec3UL, 0x65b2c5daUL,
-    0xaaae5d5dUL, 0xeb9f4644UL, 0x28cc6b6fUL, 0x69fd7076UL, 0xae6b3139UL,
-    0xef5a2a20UL, 0x2c09070bUL, 0x6d381c12UL, 0xf33646dfUL, 0xb2075dc6UL,
-    0x715470edUL, 0x30656bf4UL, 0xf7f32abbUL, 0xb6c231a2UL, 0x75911c89UL,
-    0x34a00790UL, 0xfbbc9f17UL, 0xba8d840eUL, 0x79dea925UL, 0x38efb23cUL,
-    0xff79f373UL, 0xbe48e86aUL, 0x7d1bc541UL, 0x3c2ade58UL, 0x054f79f0UL,
-    0x447e62e9UL, 0x872d4fc2UL, 0xc61c54dbUL, 0x018a1594UL, 0x40bb0e8dUL,
-    0x83e823a6UL, 0xc2d938bfUL, 0x0dc5a038UL, 0x4cf4bb21UL, 0x8fa7960aUL,
-    0xce968d13UL, 0x0900cc5cUL, 0x4831d745UL, 0x8b62fa6eUL, 0xca53e177UL,
-    0x545dbbbaUL, 0x156ca0a3UL, 0xd63f8d88UL, 0x970e9691UL, 0x5098d7deUL,
-    0x11a9ccc7UL, 0xd2fae1ecUL, 0x93cbfaf5UL, 0x5cd76272UL, 0x1de6796bUL,
-    0xdeb55440UL, 0x9f844f59UL, 0x58120e16UL, 0x1923150fUL, 0xda703824UL,
-    0x9b41233dUL, 0xa76bfd65UL, 0xe65ae67cUL, 0x2509cb57UL, 0x6438d04eUL,
-    0xa3ae9101UL, 0xe29f8a18UL, 0x21cca733UL, 0x60fdbc2aUL, 0xafe124adUL,
-    0xeed03fb4UL, 0x2d83129fUL, 0x6cb20986UL, 0xab2448c9UL, 0xea1553d0UL,
-    0x29467efbUL, 0x687765e2UL, 0xf6793f2fUL, 0xb7482436UL, 0x741b091dUL,
-    0x352a1204UL, 0xf2bc534bUL, 0xb38d4852UL, 0x70de6579UL, 0x31ef7e60UL,
-    0xfef3e6e7UL, 0xbfc2fdfeUL, 0x7c91d0d5UL, 0x3da0cbccUL, 0xfa368a83UL,
-    0xbb07919aUL, 0x7854bcb1UL, 0x3965a7a8UL, 0x4b98833bUL, 0x0aa99822UL,
-    0xc9fab509UL, 0x88cbae10UL, 0x4f5def5fUL, 0x0e6cf446UL, 0xcd3fd96dUL,
-    0x8c0ec274UL, 0x43125af3UL, 0x022341eaUL, 0xc1706cc1UL, 0x804177d8UL,
-    0x47d73697UL, 0x06e62d8eUL, 0xc5b500a5UL, 0x84841bbcUL, 0x1a8a4171UL,
-    0x5bbb5a68UL, 0x98e87743UL, 0xd9d96c5aUL, 0x1e4f2d15UL, 0x5f7e360cUL,
-    0x9c2d1b27UL, 0xdd1c003eUL, 0x120098b9UL, 0x533183a0UL, 0x9062ae8bUL,
-    0xd153b592UL, 0x16c5f4ddUL, 0x57f4efc4UL, 0x94a7c2efUL, 0xd596d9f6UL,
-    0xe9bc07aeUL, 0xa88d1cb7UL, 0x6bde319cUL, 0x2aef2a85UL, 0xed796bcaUL,
-    0xac4870d3UL, 0x6f1b5df8UL, 0x2e2a46e1UL, 0xe136de66UL, 0xa007c57fUL,
-    0x6354e854UL, 0x2265f34dUL, 0xe5f3b202UL, 0xa4c2a91bUL, 0x67918430UL,
-    0x26a09f29UL, 0xb8aec5e4UL, 0xf99fdefdUL, 0x3accf3d6UL, 0x7bfde8cfUL,
-    0xbc6ba980UL, 0xfd5ab299UL, 0x3e099fb2UL, 0x7f3884abUL, 0xb0241c2cUL,
-    0xf1150735UL, 0x32462a1eUL, 0x73773107UL, 0xb4e17048UL, 0xf5d06b51UL,
-    0x3683467aUL, 0x77b25d63UL, 0x4ed7facbUL, 0x0fe6e1d2UL, 0xccb5ccf9UL,
-    0x8d84d7e0UL, 0x4a1296afUL, 0x0b238db6UL, 0xc870a09dUL, 0x8941bb84UL,
-    0x465d2303UL, 0x076c381aUL, 0xc43f1531UL, 0x850e0e28UL, 0x42984f67UL,
-    0x03a9547eUL, 0xc0fa7955UL, 0x81cb624cUL, 0x1fc53881UL, 0x5ef42398UL,
-    0x9da70eb3UL, 0xdc9615aaUL, 0x1b0054e5UL, 0x5a314ffcUL, 0x996262d7UL,
-    0xd85379ceUL, 0x174fe149UL, 0x567efa50UL, 0x952dd77bUL, 0xd41ccc62UL,
-    0x138a8d2dUL, 0x52bb9634UL, 0x91e8bb1fUL, 0xd0d9a006UL, 0xecf37e5eUL,
-    0xadc26547UL, 0x6e91486cUL, 0x2fa05375UL, 0xe836123aUL, 0xa9070923UL,
-    0x6a542408UL, 0x2b653f11UL, 0xe479a796UL, 0xa548bc8fUL, 0x661b91a4UL,
-    0x272a8abdUL, 0xe0bccbf2UL, 0xa18dd0ebUL, 0x62defdc0UL, 0x23efe6d9UL,
-    0xbde1bc14UL, 0xfcd0a70dUL, 0x3f838a26UL, 0x7eb2913fUL, 0xb924d070UL,
-    0xf815cb69UL, 0x3b46e642UL, 0x7a77fd5bUL, 0xb56b65dcUL, 0xf45a7ec5UL,
-    0x370953eeUL, 0x763848f7UL, 0xb1ae09b8UL, 0xf09f12a1UL, 0x33cc3f8aUL,
-    0x72fd2493UL
-  },
-  {
-    0x00000000UL, 0x376ac201UL, 0x6ed48403UL, 0x59be4602UL, 0xdca80907UL,
-    0xebc2cb06UL, 0xb27c8d04UL, 0x85164f05UL, 0xb851130eUL, 0x8f3bd10fUL,
-    0xd685970dUL, 0xe1ef550cUL, 0x64f91a09UL, 0x5393d808UL, 0x0a2d9e0aUL,
-    0x3d475c0bUL, 0x70a3261cUL, 0x47c9e41dUL, 0x1e77a21fUL, 0x291d601eUL,
-    0xac0b2f1bUL, 0x9b61ed1aUL, 0xc2dfab18UL, 0xf5b56919UL, 0xc8f23512UL,
-    0xff98f713UL, 0xa626b111UL, 0x914c7310UL, 0x145a3c15UL, 0x2330fe14UL,
-    0x7a8eb816UL, 0x4de47a17UL, 0xe0464d38UL, 0xd72c8f39UL, 0x8e92c93bUL,
-    0xb9f80b3aUL, 0x3cee443fUL, 0x0b84863eUL, 0x523ac03cUL, 0x6550023dUL,
-    0x58175e36UL, 0x6f7d9c37UL, 0x36c3da35UL, 0x01a91834UL, 0x84bf5731UL,
-    0xb3d59530UL, 0xea6bd332UL, 0xdd011133UL, 0x90e56b24UL, 0xa78fa925UL,
-    0xfe31ef27UL, 0xc95b2d26UL, 0x4c4d6223UL, 0x7b27a022UL, 0x2299e620UL,
-    0x15f32421UL, 0x28b4782aUL, 0x1fdeba2bUL, 0x4660fc29UL, 0x710a3e28UL,
-    0xf41c712dUL, 0xc376b32cUL, 0x9ac8f52eUL, 0xada2372fUL, 0xc08d9a70UL,
-    0xf7e75871UL, 0xae591e73UL, 0x9933dc72UL, 0x1c259377UL, 0x2b4f5176UL,
-    0x72f11774UL, 0x459bd575UL, 0x78dc897eUL, 0x4fb64b7fUL, 0x16080d7dUL,
-    0x2162cf7cUL, 0xa4748079UL, 0x931e4278UL, 0xcaa0047aUL, 0xfdcac67bUL,
-    0xb02ebc6cUL, 0x87447e6dUL, 0xdefa386fUL, 0xe990fa6eUL, 0x6c86b56bUL,
-    0x5bec776aUL, 0x02523168UL, 0x3538f369UL, 0x087faf62UL, 0x3f156d63UL,
-    0x66ab2b61UL, 0x51c1e960UL, 0xd4d7a665UL, 0xe3bd6464UL, 0xba032266UL,
-    0x8d69e067UL, 0x20cbd748UL, 0x17a11549UL, 0x4e1f534bUL, 0x7975914aUL,
-    0xfc63de4fUL, 0xcb091c4eUL, 0x92b75a4cUL, 0xa5dd984dUL, 0x989ac446UL,
-    0xaff00647UL, 0xf64e4045UL, 0xc1248244UL, 0x4432cd41UL, 0x73580f40UL,
-    0x2ae64942UL, 0x1d8c8b43UL, 0x5068f154UL, 0x67023355UL, 0x3ebc7557UL,
-    0x09d6b756UL, 0x8cc0f853UL, 0xbbaa3a52UL, 0xe2147c50UL, 0xd57ebe51UL,
-    0xe839e25aUL, 0xdf53205bUL, 0x86ed6659UL, 0xb187a458UL, 0x3491eb5dUL,
-    0x03fb295cUL, 0x5a456f5eUL, 0x6d2fad5fUL, 0x801b35e1UL, 0xb771f7e0UL,
-    0xeecfb1e2UL, 0xd9a573e3UL, 0x5cb33ce6UL, 0x6bd9fee7UL, 0x3267b8e5UL,
-    0x050d7ae4UL, 0x384a26efUL, 0x0f20e4eeUL, 0x569ea2ecUL, 0x61f460edUL,
-    0xe4e22fe8UL, 0xd388ede9UL, 0x8a36abebUL, 0xbd5c69eaUL, 0xf0b813fdUL,
-    0xc7d2d1fcUL, 0x9e6c97feUL, 0xa90655ffUL, 0x2c101afaUL, 0x1b7ad8fbUL,
-    0x42c49ef9UL, 0x75ae5cf8UL, 0x48e900f3UL, 0x7f83c2f2UL, 0x263d84f0UL,
-    0x115746f1UL, 0x944109f4UL, 0xa32bcbf5UL, 0xfa958df7UL, 0xcdff4ff6UL,
-    0x605d78d9UL, 0x5737bad8UL, 0x0e89fcdaUL, 0x39e33edbUL, 0xbcf571deUL,
-    0x8b9fb3dfUL, 0xd221f5ddUL, 0xe54b37dcUL, 0xd80c6bd7UL, 0xef66a9d6UL,
-    0xb6d8efd4UL, 0x81b22dd5UL, 0x04a462d0UL, 0x33cea0d1UL, 0x6a70e6d3UL,
-    0x5d1a24d2UL, 0x10fe5ec5UL, 0x27949cc4UL, 0x7e2adac6UL, 0x494018c7UL,
-    0xcc5657c2UL, 0xfb3c95c3UL, 0xa282d3c1UL, 0x95e811c0UL, 0xa8af4dcbUL,
-    0x9fc58fcaUL, 0xc67bc9c8UL, 0xf1110bc9UL, 0x740744ccUL, 0x436d86cdUL,
-    0x1ad3c0cfUL, 0x2db902ceUL, 0x4096af91UL, 0x77fc6d90UL, 0x2e422b92UL,
-    0x1928e993UL, 0x9c3ea696UL, 0xab546497UL, 0xf2ea2295UL, 0xc580e094UL,
-    0xf8c7bc9fUL, 0xcfad7e9eUL, 0x9613389cUL, 0xa179fa9dUL, 0x246fb598UL,
-    0x13057799UL, 0x4abb319bUL, 0x7dd1f39aUL, 0x3035898dUL, 0x075f4b8cUL,
-    0x5ee10d8eUL, 0x698bcf8fUL, 0xec9d808aUL, 0xdbf7428bUL, 0x82490489UL,
-    0xb523c688UL, 0x88649a83UL, 0xbf0e5882UL, 0xe6b01e80UL, 0xd1dadc81UL,
-    0x54cc9384UL, 0x63a65185UL, 0x3a181787UL, 0x0d72d586UL, 0xa0d0e2a9UL,
-    0x97ba20a8UL, 0xce0466aaUL, 0xf96ea4abUL, 0x7c78ebaeUL, 0x4b1229afUL,
-    0x12ac6fadUL, 0x25c6adacUL, 0x1881f1a7UL, 0x2feb33a6UL, 0x765575a4UL,
-    0x413fb7a5UL, 0xc429f8a0UL, 0xf3433aa1UL, 0xaafd7ca3UL, 0x9d97bea2UL,
-    0xd073c4b5UL, 0xe71906b4UL, 0xbea740b6UL, 0x89cd82b7UL, 0x0cdbcdb2UL,
-    0x3bb10fb3UL, 0x620f49b1UL, 0x55658bb0UL, 0x6822d7bbUL, 0x5f4815baUL,
-    0x06f653b8UL, 0x319c91b9UL, 0xb48adebcUL, 0x83e01cbdUL, 0xda5e5abfUL,
-    0xed3498beUL
-  },
-  {
-    0x00000000UL, 0x6567bcb8UL, 0x8bc809aaUL, 0xeeafb512UL, 0x5797628fUL,
-    0x32f0de37UL, 0xdc5f6b25UL, 0xb938d79dUL, 0xef28b4c5UL, 0x8a4f087dUL,
-    0x64e0bd6fUL, 0x018701d7UL, 0xb8bfd64aUL, 0xddd86af2UL, 0x3377dfe0UL,
-    0x56106358UL, 0x9f571950UL, 0xfa30a5e8UL, 0x149f10faUL, 0x71f8ac42UL,
-    0xc8c07bdfUL, 0xada7c767UL, 0x43087275UL, 0x266fcecdUL, 0x707fad95UL,
-    0x1518112dUL, 0xfbb7a43fUL, 0x9ed01887UL, 0x27e8cf1aUL, 0x428f73a2UL,
-    0xac20c6b0UL, 0xc9477a08UL, 0x3eaf32a0UL, 0x5bc88e18UL, 0xb5673b0aUL,
-    0xd00087b2UL, 0x6938502fUL, 0x0c5fec97UL, 0xe2f05985UL, 0x8797e53dUL,
-    0xd1878665UL, 0xb4e03addUL, 0x5a4f8fcfUL, 0x3f283377UL, 0x8610e4eaUL,
-    0xe3775852UL, 0x0dd8ed40UL, 0x68bf51f8UL, 0xa1f82bf0UL, 0xc49f9748UL,
-    0x2a30225aUL, 0x4f579ee2UL, 0xf66f497fUL, 0x9308f5c7UL, 0x7da740d5UL,
-    0x18c0fc6dUL, 0x4ed09f35UL, 0x2bb7238dUL, 0xc518969fUL, 0xa07f2a27UL,
-    0x1947fdbaUL, 0x7c204102UL, 0x928ff410UL, 0xf7e848a8UL, 0x3d58149bUL,
-    0x583fa823UL, 0xb6901d31UL, 0xd3f7a189UL, 0x6acf7614UL, 0x0fa8caacUL,
-    0xe1077fbeUL, 0x8460c306UL, 0xd270a05eUL, 0xb7171ce6UL, 0x59b8a9f4UL,
-    0x3cdf154cUL, 0x85e7c2d1UL, 0xe0807e69UL, 0x0e2fcb7bUL, 0x6b4877c3UL,
-    0xa20f0dcbUL, 0xc768b173UL, 0x29c70461UL, 0x4ca0b8d9UL, 0xf5986f44UL,
-    0x90ffd3fcUL, 0x7e5066eeUL, 0x1b37da56UL, 0x4d27b90eUL, 0x284005b6UL,
-    0xc6efb0a4UL, 0xa3880c1cUL, 0x1ab0db81UL, 0x7fd76739UL, 0x9178d22bUL,
-    0xf41f6e93UL, 0x03f7263bUL, 0x66909a83UL, 0x883f2f91UL, 0xed589329UL,
-    0x546044b4UL, 0x3107f80cUL, 0xdfa84d1eUL, 0xbacff1a6UL, 0xecdf92feUL,
-    0x89b82e46UL, 0x67179b54UL, 0x027027ecUL, 0xbb48f071UL, 0xde2f4cc9UL,
-    0x3080f9dbUL, 0x55e74563UL, 0x9ca03f6bUL, 0xf9c783d3UL, 0x176836c1UL,
-    0x720f8a79UL, 0xcb375de4UL, 0xae50e15cUL, 0x40ff544eUL, 0x2598e8f6UL,
-    0x73888baeUL, 0x16ef3716UL, 0xf8408204UL, 0x9d273ebcUL, 0x241fe921UL,
-    0x41785599UL, 0xafd7e08bUL, 0xcab05c33UL, 0x3bb659edUL, 0x5ed1e555UL,
-    0xb07e5047UL, 0xd519ecffUL, 0x6c213b62UL, 0x094687daUL, 0xe7e932c8UL,
-    0x828e8e70UL, 0xd49eed28UL, 0xb1f95190UL, 0x5f56e482UL, 0x3a31583aUL,
-    0x83098fa7UL, 0xe66e331fUL, 0x08c1860dUL, 0x6da63ab5UL, 0xa4e140bdUL,
-    0xc186fc05UL, 0x2f294917UL, 0x4a4ef5afUL, 0xf3762232UL, 0x96119e8aUL,
-    0x78be2b98UL, 0x1dd99720UL, 0x4bc9f478UL, 0x2eae48c0UL, 0xc001fdd2UL,
-    0xa566416aUL, 0x1c5e96f7UL, 0x79392a4fUL, 0x97969f5dUL, 0xf2f123e5UL,
-    0x05196b4dUL, 0x607ed7f5UL, 0x8ed162e7UL, 0xebb6de5fUL, 0x528e09c2UL,
-    0x37e9b57aUL, 0xd9460068UL, 0xbc21bcd0UL, 0xea31df88UL, 0x8f566330UL,
-    0x61f9d622UL, 0x049e6a9aUL, 0xbda6bd07UL, 0xd8c101bfUL, 0x366eb4adUL,
-    0x53090815UL, 0x9a4e721dUL, 0xff29cea5UL, 0x11867bb7UL, 0x74e1c70fUL,
-    0xcdd91092UL, 0xa8beac2aUL, 0x46111938UL, 0x2376a580UL, 0x7566c6d8UL,
-    0x10017a60UL, 0xfeaecf72UL, 0x9bc973caUL, 0x22f1a457UL, 0x479618efUL,
-    0xa939adfdUL, 0xcc5e1145UL, 0x06ee4d76UL, 0x6389f1ceUL, 0x8d2644dcUL,
-    0xe841f864UL, 0x51792ff9UL, 0x341e9341UL, 0xdab12653UL, 0xbfd69aebUL,
-    0xe9c6f9b3UL, 0x8ca1450bUL, 0x620ef019UL, 0x07694ca1UL, 0xbe519b3cUL,
-    0xdb362784UL, 0x35999296UL, 0x50fe2e2eUL, 0x99b95426UL, 0xfcdee89eUL,
-    0x12715d8cUL, 0x7716e134UL, 0xce2e36a9UL, 0xab498a11UL, 0x45e63f03UL,
-    0x208183bbUL, 0x7691e0e3UL, 0x13f65c5bUL, 0xfd59e949UL, 0x983e55f1UL,
-    0x2106826cUL, 0x44613ed4UL, 0xaace8bc6UL, 0xcfa9377eUL, 0x38417fd6UL,
-    0x5d26c36eUL, 0xb389767cUL, 0xd6eecac4UL, 0x6fd61d59UL, 0x0ab1a1e1UL,
-    0xe41e14f3UL, 0x8179a84bUL, 0xd769cb13UL, 0xb20e77abUL, 0x5ca1c2b9UL,
-    0x39c67e01UL, 0x80fea99cUL, 0xe5991524UL, 0x0b36a036UL, 0x6e511c8eUL,
-    0xa7166686UL, 0xc271da3eUL, 0x2cde6f2cUL, 0x49b9d394UL, 0xf0810409UL,
-    0x95e6b8b1UL, 0x7b490da3UL, 0x1e2eb11bUL, 0x483ed243UL, 0x2d596efbUL,
-    0xc3f6dbe9UL, 0xa6916751UL, 0x1fa9b0ccUL, 0x7ace0c74UL, 0x9461b966UL,
-    0xf10605deUL
+local const z_crc_t FAR crc_table[] = {
+    0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419,
+    0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4,
+    0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07,
+    0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
+    0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856,
+    0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
+    0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4,
+    0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
+    0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3,
+    0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a,
+    0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599,
+    0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
+    0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190,
+    0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f,
+    0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e,
+    0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
+    0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed,
+    0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
+    0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3,
+    0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
+    0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a,
+    0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5,
+    0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010,
+    0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+    0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17,
+    0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6,
+    0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615,
+    0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
+    0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344,
+    0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
+    0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a,
+    0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
+    0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1,
+    0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c,
+    0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef,
+    0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
+    0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe,
+    0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31,
+    0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c,
+    0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
+    0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b,
+    0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
+    0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1,
+    0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
+    0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278,
+    0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7,
+    0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66,
+    0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+    0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605,
+    0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8,
+    0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b,
+    0x2d02ef8d};
+
+#ifdef W
+
+#if W == 8
+
+local const z_word_t FAR crc_big_table[] = {
+    0x0000000000000000, 0x9630077700000000, 0x2c610eee00000000,
+    0xba51099900000000, 0x19c46d0700000000, 0x8ff46a7000000000,
+    0x35a563e900000000, 0xa395649e00000000, 0x3288db0e00000000,
+    0xa4b8dc7900000000, 0x1ee9d5e000000000, 0x88d9d29700000000,
+    0x2b4cb60900000000, 0xbd7cb17e00000000, 0x072db8e700000000,
+    0x911dbf9000000000, 0x6410b71d00000000, 0xf220b06a00000000,
+    0x4871b9f300000000, 0xde41be8400000000, 0x7dd4da1a00000000,
+    0xebe4dd6d00000000, 0x51b5d4f400000000, 0xc785d38300000000,
+    0x56986c1300000000, 0xc0a86b6400000000, 0x7af962fd00000000,
+    0xecc9658a00000000, 0x4f5c011400000000, 0xd96c066300000000,
+    0x633d0ffa00000000, 0xf50d088d00000000, 0xc8206e3b00000000,
+    0x5e10694c00000000, 0xe44160d500000000, 0x727167a200000000,
+    0xd1e4033c00000000, 0x47d4044b00000000, 0xfd850dd200000000,
+    0x6bb50aa500000000, 0xfaa8b53500000000, 0x6c98b24200000000,
+    0xd6c9bbdb00000000, 0x40f9bcac00000000, 0xe36cd83200000000,
+    0x755cdf4500000000, 0xcf0dd6dc00000000, 0x593dd1ab00000000,
+    0xac30d92600000000, 0x3a00de5100000000, 0x8051d7c800000000,
+    0x1661d0bf00000000, 0xb5f4b42100000000, 0x23c4b35600000000,
+    0x9995bacf00000000, 0x0fa5bdb800000000, 0x9eb8022800000000,
+    0x0888055f00000000, 0xb2d90cc600000000, 0x24e90bb100000000,
+    0x877c6f2f00000000, 0x114c685800000000, 0xab1d61c100000000,
+    0x3d2d66b600000000, 0x9041dc7600000000, 0x0671db0100000000,
+    0xbc20d29800000000, 0x2a10d5ef00000000, 0x8985b17100000000,
+    0x1fb5b60600000000, 0xa5e4bf9f00000000, 0x33d4b8e800000000,
+    0xa2c9077800000000, 0x34f9000f00000000, 0x8ea8099600000000,
+    0x18980ee100000000, 0xbb0d6a7f00000000, 0x2d3d6d0800000000,
+    0x976c649100000000, 0x015c63e600000000, 0xf4516b6b00000000,
+    0x62616c1c00000000, 0xd830658500000000, 0x4e0062f200000000,
+    0xed95066c00000000, 0x7ba5011b00000000, 0xc1f4088200000000,
+    0x57c40ff500000000, 0xc6d9b06500000000, 0x50e9b71200000000,
+    0xeab8be8b00000000, 0x7c88b9fc00000000, 0xdf1ddd6200000000,
+    0x492dda1500000000, 0xf37cd38c00000000, 0x654cd4fb00000000,
+    0x5861b24d00000000, 0xce51b53a00000000, 0x7400bca300000000,
+    0xe230bbd400000000, 0x41a5df4a00000000, 0xd795d83d00000000,
+    0x6dc4d1a400000000, 0xfbf4d6d300000000, 0x6ae9694300000000,
+    0xfcd96e3400000000, 0x468867ad00000000, 0xd0b860da00000000,
+    0x732d044400000000, 0xe51d033300000000, 0x5f4c0aaa00000000,
+    0xc97c0ddd00000000, 0x3c71055000000000, 0xaa41022700000000,
+    0x10100bbe00000000, 0x86200cc900000000, 0x25b5685700000000,
+    0xb3856f2000000000, 0x09d466b900000000, 0x9fe461ce00000000,
+    0x0ef9de5e00000000, 0x98c9d92900000000, 0x2298d0b000000000,
+    0xb4a8d7c700000000, 0x173db35900000000, 0x810db42e00000000,
+    0x3b5cbdb700000000, 0xad6cbac000000000, 0x2083b8ed00000000,
+    0xb6b3bf9a00000000, 0x0ce2b60300000000, 0x9ad2b17400000000,
+    0x3947d5ea00000000, 0xaf77d29d00000000, 0x1526db0400000000,
+    0x8316dc7300000000, 0x120b63e300000000, 0x843b649400000000,
+    0x3e6a6d0d00000000, 0xa85a6a7a00000000, 0x0bcf0ee400000000,
+    0x9dff099300000000, 0x27ae000a00000000, 0xb19e077d00000000,
+    0x44930ff000000000, 0xd2a3088700000000, 0x68f2011e00000000,
+    0xfec2066900000000, 0x5d5762f700000000, 0xcb67658000000000,
+    0x71366c1900000000, 0xe7066b6e00000000, 0x761bd4fe00000000,
+    0xe02bd38900000000, 0x5a7ada1000000000, 0xcc4add6700000000,
+    0x6fdfb9f900000000, 0xf9efbe8e00000000, 0x43beb71700000000,
+    0xd58eb06000000000, 0xe8a3d6d600000000, 0x7e93d1a100000000,
+    0xc4c2d83800000000, 0x52f2df4f00000000, 0xf167bbd100000000,
+    0x6757bca600000000, 0xdd06b53f00000000, 0x4b36b24800000000,
+    0xda2b0dd800000000, 0x4c1b0aaf00000000, 0xf64a033600000000,
+    0x607a044100000000, 0xc3ef60df00000000, 0x55df67a800000000,
+    0xef8e6e3100000000, 0x79be694600000000, 0x8cb361cb00000000,
+    0x1a8366bc00000000, 0xa0d26f2500000000, 0x36e2685200000000,
+    0x95770ccc00000000, 0x03470bbb00000000, 0xb916022200000000,
+    0x2f26055500000000, 0xbe3bbac500000000, 0x280bbdb200000000,
+    0x925ab42b00000000, 0x046ab35c00000000, 0xa7ffd7c200000000,
+    0x31cfd0b500000000, 0x8b9ed92c00000000, 0x1daede5b00000000,
+    0xb0c2649b00000000, 0x26f263ec00000000, 0x9ca36a7500000000,
+    0x0a936d0200000000, 0xa906099c00000000, 0x3f360eeb00000000,
+    0x8567077200000000, 0x1357000500000000, 0x824abf9500000000,
+    0x147ab8e200000000, 0xae2bb17b00000000, 0x381bb60c00000000,
+    0x9b8ed29200000000, 0x0dbed5e500000000, 0xb7efdc7c00000000,
+    0x21dfdb0b00000000, 0xd4d2d38600000000, 0x42e2d4f100000000,
+    0xf8b3dd6800000000, 0x6e83da1f00000000, 0xcd16be8100000000,
+    0x5b26b9f600000000, 0xe177b06f00000000, 0x7747b71800000000,
+    0xe65a088800000000, 0x706a0fff00000000, 0xca3b066600000000,
+    0x5c0b011100000000, 0xff9e658f00000000, 0x69ae62f800000000,
+    0xd3ff6b6100000000, 0x45cf6c1600000000, 0x78e20aa000000000,
+    0xeed20dd700000000, 0x5483044e00000000, 0xc2b3033900000000,
+    0x612667a700000000, 0xf71660d000000000, 0x4d47694900000000,
+    0xdb776e3e00000000, 0x4a6ad1ae00000000, 0xdc5ad6d900000000,
+    0x660bdf4000000000, 0xf03bd83700000000, 0x53aebca900000000,
+    0xc59ebbde00000000, 0x7fcfb24700000000, 0xe9ffb53000000000,
+    0x1cf2bdbd00000000, 0x8ac2baca00000000, 0x3093b35300000000,
+    0xa6a3b42400000000, 0x0536d0ba00000000, 0x9306d7cd00000000,
+    0x2957de5400000000, 0xbf67d92300000000, 0x2e7a66b300000000,
+    0xb84a61c400000000, 0x021b685d00000000, 0x942b6f2a00000000,
+    0x37be0bb400000000, 0xa18e0cc300000000, 0x1bdf055a00000000,
+    0x8def022d00000000};
+
+#else /* W == 4 */
+
+local const z_word_t FAR crc_big_table[] = {
+    0x00000000, 0x96300777, 0x2c610eee, 0xba510999, 0x19c46d07,
+    0x8ff46a70, 0x35a563e9, 0xa395649e, 0x3288db0e, 0xa4b8dc79,
+    0x1ee9d5e0, 0x88d9d297, 0x2b4cb609, 0xbd7cb17e, 0x072db8e7,
+    0x911dbf90, 0x6410b71d, 0xf220b06a, 0x4871b9f3, 0xde41be84,
+    0x7dd4da1a, 0xebe4dd6d, 0x51b5d4f4, 0xc785d383, 0x56986c13,
+    0xc0a86b64, 0x7af962fd, 0xecc9658a, 0x4f5c0114, 0xd96c0663,
+    0x633d0ffa, 0xf50d088d, 0xc8206e3b, 0x5e10694c, 0xe44160d5,
+    0x727167a2, 0xd1e4033c, 0x47d4044b, 0xfd850dd2, 0x6bb50aa5,
+    0xfaa8b535, 0x6c98b242, 0xd6c9bbdb, 0x40f9bcac, 0xe36cd832,
+    0x755cdf45, 0xcf0dd6dc, 0x593dd1ab, 0xac30d926, 0x3a00de51,
+    0x8051d7c8, 0x1661d0bf, 0xb5f4b421, 0x23c4b356, 0x9995bacf,
+    0x0fa5bdb8, 0x9eb80228, 0x0888055f, 0xb2d90cc6, 0x24e90bb1,
+    0x877c6f2f, 0x114c6858, 0xab1d61c1, 0x3d2d66b6, 0x9041dc76,
+    0x0671db01, 0xbc20d298, 0x2a10d5ef, 0x8985b171, 0x1fb5b606,
+    0xa5e4bf9f, 0x33d4b8e8, 0xa2c90778, 0x34f9000f, 0x8ea80996,
+    0x18980ee1, 0xbb0d6a7f, 0x2d3d6d08, 0x976c6491, 0x015c63e6,
+    0xf4516b6b, 0x62616c1c, 0xd8306585, 0x4e0062f2, 0xed95066c,
+    0x7ba5011b, 0xc1f40882, 0x57c40ff5, 0xc6d9b065, 0x50e9b712,
+    0xeab8be8b, 0x7c88b9fc, 0xdf1ddd62, 0x492dda15, 0xf37cd38c,
+    0x654cd4fb, 0x5861b24d, 0xce51b53a, 0x7400bca3, 0xe230bbd4,
+    0x41a5df4a, 0xd795d83d, 0x6dc4d1a4, 0xfbf4d6d3, 0x6ae96943,
+    0xfcd96e34, 0x468867ad, 0xd0b860da, 0x732d0444, 0xe51d0333,
+    0x5f4c0aaa, 0xc97c0ddd, 0x3c710550, 0xaa410227, 0x10100bbe,
+    0x86200cc9, 0x25b56857, 0xb3856f20, 0x09d466b9, 0x9fe461ce,
+    0x0ef9de5e, 0x98c9d929, 0x2298d0b0, 0xb4a8d7c7, 0x173db359,
+    0x810db42e, 0x3b5cbdb7, 0xad6cbac0, 0x2083b8ed, 0xb6b3bf9a,
+    0x0ce2b603, 0x9ad2b174, 0x3947d5ea, 0xaf77d29d, 0x1526db04,
+    0x8316dc73, 0x120b63e3, 0x843b6494, 0x3e6a6d0d, 0xa85a6a7a,
+    0x0bcf0ee4, 0x9dff0993, 0x27ae000a, 0xb19e077d, 0x44930ff0,
+    0xd2a30887, 0x68f2011e, 0xfec20669, 0x5d5762f7, 0xcb676580,
+    0x71366c19, 0xe7066b6e, 0x761bd4fe, 0xe02bd389, 0x5a7ada10,
+    0xcc4add67, 0x6fdfb9f9, 0xf9efbe8e, 0x43beb717, 0xd58eb060,
+    0xe8a3d6d6, 0x7e93d1a1, 0xc4c2d838, 0x52f2df4f, 0xf167bbd1,
+    0x6757bca6, 0xdd06b53f, 0x4b36b248, 0xda2b0dd8, 0x4c1b0aaf,
+    0xf64a0336, 0x607a0441, 0xc3ef60df, 0x55df67a8, 0xef8e6e31,
+    0x79be6946, 0x8cb361cb, 0x1a8366bc, 0xa0d26f25, 0x36e26852,
+    0x95770ccc, 0x03470bbb, 0xb9160222, 0x2f260555, 0xbe3bbac5,
+    0x280bbdb2, 0x925ab42b, 0x046ab35c, 0xa7ffd7c2, 0x31cfd0b5,
+    0x8b9ed92c, 0x1daede5b, 0xb0c2649b, 0x26f263ec, 0x9ca36a75,
+    0x0a936d02, 0xa906099c, 0x3f360eeb, 0x85670772, 0x13570005,
+    0x824abf95, 0x147ab8e2, 0xae2bb17b, 0x381bb60c, 0x9b8ed292,
+    0x0dbed5e5, 0xb7efdc7c, 0x21dfdb0b, 0xd4d2d386, 0x42e2d4f1,
+    0xf8b3dd68, 0x6e83da1f, 0xcd16be81, 0x5b26b9f6, 0xe177b06f,
+    0x7747b718, 0xe65a0888, 0x706a0fff, 0xca3b0666, 0x5c0b0111,
+    0xff9e658f, 0x69ae62f8, 0xd3ff6b61, 0x45cf6c16, 0x78e20aa0,
+    0xeed20dd7, 0x5483044e, 0xc2b30339, 0x612667a7, 0xf71660d0,
+    0x4d476949, 0xdb776e3e, 0x4a6ad1ae, 0xdc5ad6d9, 0x660bdf40,
+    0xf03bd837, 0x53aebca9, 0xc59ebbde, 0x7fcfb247, 0xe9ffb530,
+    0x1cf2bdbd, 0x8ac2baca, 0x3093b353, 0xa6a3b424, 0x0536d0ba,
+    0x9306d7cd, 0x2957de54, 0xbf67d923, 0x2e7a66b3, 0xb84a61c4,
+    0x021b685d, 0x942b6f2a, 0x37be0bb4, 0xa18e0cc3, 0x1bdf055a,
+    0x8def022d};
+
+#endif
+
+#if N == 1
+
+#if W == 8
+
+local const z_crc_t FAR crc_braid_table[][256] = {
+   {0x00000000, 0xccaa009e, 0x4225077d, 0x8e8f07e3, 0x844a0efa,
+    0x48e00e64, 0xc66f0987, 0x0ac50919, 0xd3e51bb5, 0x1f4f1b2b,
+    0x91c01cc8, 0x5d6a1c56, 0x57af154f, 0x9b0515d1, 0x158a1232,
+    0xd92012ac, 0x7cbb312b, 0xb01131b5, 0x3e9e3656, 0xf23436c8,
+    0xf8f13fd1, 0x345b3f4f, 0xbad438ac, 0x767e3832, 0xaf5e2a9e,
+    0x63f42a00, 0xed7b2de3, 0x21d12d7d, 0x2b142464, 0xe7be24fa,
+    0x69312319, 0xa59b2387, 0xf9766256, 0x35dc62c8, 0xbb53652b,
+    0x77f965b5, 0x7d3c6cac, 0xb1966c32, 0x3f196bd1, 0xf3b36b4f,
+    0x2a9379e3, 0xe639797d, 0x68b67e9e, 0xa41c7e00, 0xaed97719,
+    0x62737787, 0xecfc7064, 0x205670fa, 0x85cd537d, 0x496753e3,
+    0xc7e85400, 0x0b42549e, 0x01875d87, 0xcd2d5d19, 0x43a25afa,
+    0x8f085a64, 0x562848c8, 0x9a824856, 0x140d4fb5, 0xd8a74f2b,
+    0xd2624632, 0x1ec846ac, 0x9047414f, 0x5ced41d1, 0x299dc2ed,
+    0xe537c273, 0x6bb8c590, 0xa712c50e, 0xadd7cc17, 0x617dcc89,
+    0xeff2cb6a, 0x2358cbf4, 0xfa78d958, 0x36d2d9c6, 0xb85dde25,
+    0x74f7debb, 0x7e32d7a2, 0xb298d73c, 0x3c17d0df, 0xf0bdd041,
+    0x5526f3c6, 0x998cf358, 0x1703f4bb, 0xdba9f425, 0xd16cfd3c,
+    0x1dc6fda2, 0x9349fa41, 0x5fe3fadf, 0x86c3e873, 0x4a69e8ed,
+    0xc4e6ef0e, 0x084cef90, 0x0289e689, 0xce23e617, 0x40ace1f4,
+    0x8c06e16a, 0xd0eba0bb, 0x1c41a025, 0x92cea7c6, 0x5e64a758,
+    0x54a1ae41, 0x980baedf, 0x1684a93c, 0xda2ea9a2, 0x030ebb0e,
+    0xcfa4bb90, 0x412bbc73, 0x8d81bced, 0x8744b5f4, 0x4beeb56a,
+    0xc561b289, 0x09cbb217, 0xac509190, 0x60fa910e, 0xee7596ed,
+    0x22df9673, 0x281a9f6a, 0xe4b09ff4, 0x6a3f9817, 0xa6959889,
+    0x7fb58a25, 0xb31f8abb, 0x3d908d58, 0xf13a8dc6, 0xfbff84df,
+    0x37558441, 0xb9da83a2, 0x7570833c, 0x533b85da, 0x9f918544,
+    0x111e82a7, 0xddb48239, 0xd7718b20, 0x1bdb8bbe, 0x95548c5d,
+    0x59fe8cc3, 0x80de9e6f, 0x4c749ef1, 0xc2fb9912, 0x0e51998c,
+    0x04949095, 0xc83e900b, 0x46b197e8, 0x8a1b9776, 0x2f80b4f1,
+    0xe32ab46f, 0x6da5b38c, 0xa10fb312, 0xabcaba0b, 0x6760ba95,
+    0xe9efbd76, 0x2545bde8, 0xfc65af44, 0x30cfafda, 0xbe40a839,
+    0x72eaa8a7, 0x782fa1be, 0xb485a120, 0x3a0aa6c3, 0xf6a0a65d,
+    0xaa4de78c, 0x66e7e712, 0xe868e0f1, 0x24c2e06f, 0x2e07e976,
+    0xe2ade9e8, 0x6c22ee0b, 0xa088ee95, 0x79a8fc39, 0xb502fca7,
+    0x3b8dfb44, 0xf727fbda, 0xfde2f2c3, 0x3148f25d, 0xbfc7f5be,
+    0x736df520, 0xd6f6d6a7, 0x1a5cd639, 0x94d3d1da, 0x5879d144,
+    0x52bcd85d, 0x9e16d8c3, 0x1099df20, 0xdc33dfbe, 0x0513cd12,
+    0xc9b9cd8c, 0x4736ca6f, 0x8b9ccaf1, 0x8159c3e8, 0x4df3c376,
+    0xc37cc495, 0x0fd6c40b, 0x7aa64737, 0xb60c47a9, 0x3883404a,
+    0xf42940d4, 0xfeec49cd, 0x32464953, 0xbcc94eb0, 0x70634e2e,
+    0xa9435c82, 0x65e95c1c, 0xeb665bff, 0x27cc5b61, 0x2d095278,
+    0xe1a352e6, 0x6f2c5505, 0xa386559b, 0x061d761c, 0xcab77682,
+    0x44387161, 0x889271ff, 0x825778e6, 0x4efd7878, 0xc0727f9b,
+    0x0cd87f05, 0xd5f86da9, 0x19526d37, 0x97dd6ad4, 0x5b776a4a,
+    0x51b26353, 0x9d1863cd, 0x1397642e, 0xdf3d64b0, 0x83d02561,
+    0x4f7a25ff, 0xc1f5221c, 0x0d5f2282, 0x079a2b9b, 0xcb302b05,
+    0x45bf2ce6, 0x89152c78, 0x50353ed4, 0x9c9f3e4a, 0x121039a9,
+    0xdeba3937, 0xd47f302e, 0x18d530b0, 0x965a3753, 0x5af037cd,
+    0xff6b144a, 0x33c114d4, 0xbd4e1337, 0x71e413a9, 0x7b211ab0,
+    0xb78b1a2e, 0x39041dcd, 0xf5ae1d53, 0x2c8e0fff, 0xe0240f61,
+    0x6eab0882, 0xa201081c, 0xa8c40105, 0x646e019b, 0xeae10678,
+    0x264b06e6},
+   {0x00000000, 0xa6770bb4, 0x979f1129, 0x31e81a9d, 0xf44f2413,
+    0x52382fa7, 0x63d0353a, 0xc5a73e8e, 0x33ef4e67, 0x959845d3,
+    0xa4705f4e, 0x020754fa, 0xc7a06a74, 0x61d761c0, 0x503f7b5d,
+    0xf64870e9, 0x67de9cce, 0xc1a9977a, 0xf0418de7, 0x56368653,
+    0x9391b8dd, 0x35e6b369, 0x040ea9f4, 0xa279a240, 0x5431d2a9,
+    0xf246d91d, 0xc3aec380, 0x65d9c834, 0xa07ef6ba, 0x0609fd0e,
+    0x37e1e793, 0x9196ec27, 0xcfbd399c, 0x69ca3228, 0x582228b5,
+    0xfe552301, 0x3bf21d8f, 0x9d85163b, 0xac6d0ca6, 0x0a1a0712,
+    0xfc5277fb, 0x5a257c4f, 0x6bcd66d2, 0xcdba6d66, 0x081d53e8,
+    0xae6a585c, 0x9f8242c1, 0x39f54975, 0xa863a552, 0x0e14aee6,
+    0x3ffcb47b, 0x998bbfcf, 0x5c2c8141, 0xfa5b8af5, 0xcbb39068,
+    0x6dc49bdc, 0x9b8ceb35, 0x3dfbe081, 0x0c13fa1c, 0xaa64f1a8,
+    0x6fc3cf26, 0xc9b4c492, 0xf85cde0f, 0x5e2bd5bb, 0x440b7579,
+    0xe27c7ecd, 0xd3946450, 0x75e36fe4, 0xb044516a, 0x16335ade,
+    0x27db4043, 0x81ac4bf7, 0x77e43b1e, 0xd19330aa, 0xe07b2a37,
+    0x460c2183, 0x83ab1f0d, 0x25dc14b9, 0x14340e24, 0xb2430590,
+    0x23d5e9b7, 0x85a2e203, 0xb44af89e, 0x123df32a, 0xd79acda4,
+    0x71edc610, 0x4005dc8d, 0xe672d739, 0x103aa7d0, 0xb64dac64,
+    0x87a5b6f9, 0x21d2bd4d, 0xe47583c3, 0x42028877, 0x73ea92ea,
+    0xd59d995e, 0x8bb64ce5, 0x2dc14751, 0x1c295dcc, 0xba5e5678,
+    0x7ff968f6, 0xd98e6342, 0xe86679df, 0x4e11726b, 0xb8590282,
+    0x1e2e0936, 0x2fc613ab, 0x89b1181f, 0x4c162691, 0xea612d25,
+    0xdb8937b8, 0x7dfe3c0c, 0xec68d02b, 0x4a1fdb9f, 0x7bf7c102,
+    0xdd80cab6, 0x1827f438, 0xbe50ff8c, 0x8fb8e511, 0x29cfeea5,
+    0xdf879e4c, 0x79f095f8, 0x48188f65, 0xee6f84d1, 0x2bc8ba5f,
+    0x8dbfb1eb, 0xbc57ab76, 0x1a20a0c2, 0x8816eaf2, 0x2e61e146,
+    0x1f89fbdb, 0xb9fef06f, 0x7c59cee1, 0xda2ec555, 0xebc6dfc8,
+    0x4db1d47c, 0xbbf9a495, 0x1d8eaf21, 0x2c66b5bc, 0x8a11be08,
+    0x4fb68086, 0xe9c18b32, 0xd82991af, 0x7e5e9a1b, 0xefc8763c,
+    0x49bf7d88, 0x78576715, 0xde206ca1, 0x1b87522f, 0xbdf0599b,
+    0x8c184306, 0x2a6f48b2, 0xdc27385b, 0x7a5033ef, 0x4bb82972,
+    0xedcf22c6, 0x28681c48, 0x8e1f17fc, 0xbff70d61, 0x198006d5,
+    0x47abd36e, 0xe1dcd8da, 0xd034c247, 0x7643c9f3, 0xb3e4f77d,
+    0x1593fcc9, 0x247be654, 0x820cede0, 0x74449d09, 0xd23396bd,
+    0xe3db8c20, 0x45ac8794, 0x800bb91a, 0x267cb2ae, 0x1794a833,
+    0xb1e3a387, 0x20754fa0, 0x86024414, 0xb7ea5e89, 0x119d553d,
+    0xd43a6bb3, 0x724d6007, 0x43a57a9a, 0xe5d2712e, 0x139a01c7,
+    0xb5ed0a73, 0x840510ee, 0x22721b5a, 0xe7d525d4, 0x41a22e60,
+    0x704a34fd, 0xd63d3f49, 0xcc1d9f8b, 0x6a6a943f, 0x5b828ea2,
+    0xfdf58516, 0x3852bb98, 0x9e25b02c, 0xafcdaab1, 0x09baa105,
+    0xfff2d1ec, 0x5985da58, 0x686dc0c5, 0xce1acb71, 0x0bbdf5ff,
+    0xadcafe4b, 0x9c22e4d6, 0x3a55ef62, 0xabc30345, 0x0db408f1,
+    0x3c5c126c, 0x9a2b19d8, 0x5f8c2756, 0xf9fb2ce2, 0xc813367f,
+    0x6e643dcb, 0x982c4d22, 0x3e5b4696, 0x0fb35c0b, 0xa9c457bf,
+    0x6c636931, 0xca146285, 0xfbfc7818, 0x5d8b73ac, 0x03a0a617,
+    0xa5d7ada3, 0x943fb73e, 0x3248bc8a, 0xf7ef8204, 0x519889b0,
+    0x6070932d, 0xc6079899, 0x304fe870, 0x9638e3c4, 0xa7d0f959,
+    0x01a7f2ed, 0xc400cc63, 0x6277c7d7, 0x539fdd4a, 0xf5e8d6fe,
+    0x647e3ad9, 0xc209316d, 0xf3e12bf0, 0x55962044, 0x90311eca,
+    0x3646157e, 0x07ae0fe3, 0xa1d90457, 0x579174be, 0xf1e67f0a,
+    0xc00e6597, 0x66796e23, 0xa3de50ad, 0x05a95b19, 0x34414184,
+    0x92364a30},
+   {0x00000000, 0xcb5cd3a5, 0x4dc8a10b, 0x869472ae, 0x9b914216,
+    0x50cd91b3, 0xd659e31d, 0x1d0530b8, 0xec53826d, 0x270f51c8,
+    0xa19b2366, 0x6ac7f0c3, 0x77c2c07b, 0xbc9e13de, 0x3a0a6170,
+    0xf156b2d5, 0x03d6029b, 0xc88ad13e, 0x4e1ea390, 0x85427035,
+    0x9847408d, 0x531b9328, 0xd58fe186, 0x1ed33223, 0xef8580f6,
+    0x24d95353, 0xa24d21fd, 0x6911f258, 0x7414c2e0, 0xbf481145,
+    0x39dc63eb, 0xf280b04e, 0x07ac0536, 0xccf0d693, 0x4a64a43d,
+    0x81387798, 0x9c3d4720, 0x57619485, 0xd1f5e62b, 0x1aa9358e,
+    0xebff875b, 0x20a354fe, 0xa6372650, 0x6d6bf5f5, 0x706ec54d,
+    0xbb3216e8, 0x3da66446, 0xf6fab7e3, 0x047a07ad, 0xcf26d408,
+    0x49b2a6a6, 0x82ee7503, 0x9feb45bb, 0x54b7961e, 0xd223e4b0,
+    0x197f3715, 0xe82985c0, 0x23755665, 0xa5e124cb, 0x6ebdf76e,
+    0x73b8c7d6, 0xb8e41473, 0x3e7066dd, 0xf52cb578, 0x0f580a6c,
+    0xc404d9c9, 0x4290ab67, 0x89cc78c2, 0x94c9487a, 0x5f959bdf,
+    0xd901e971, 0x125d3ad4, 0xe30b8801, 0x28575ba4, 0xaec3290a,
+    0x659ffaaf, 0x789aca17, 0xb3c619b2, 0x35526b1c, 0xfe0eb8b9,
+    0x0c8e08f7, 0xc7d2db52, 0x4146a9fc, 0x8a1a7a59, 0x971f4ae1,
+    0x5c439944, 0xdad7ebea, 0x118b384f, 0xe0dd8a9a, 0x2b81593f,
+    0xad152b91, 0x6649f834, 0x7b4cc88c, 0xb0101b29, 0x36846987,
+    0xfdd8ba22, 0x08f40f5a, 0xc3a8dcff, 0x453cae51, 0x8e607df4,
+    0x93654d4c, 0x58399ee9, 0xdeadec47, 0x15f13fe2, 0xe4a78d37,
+    0x2ffb5e92, 0xa96f2c3c, 0x6233ff99, 0x7f36cf21, 0xb46a1c84,
+    0x32fe6e2a, 0xf9a2bd8f, 0x0b220dc1, 0xc07ede64, 0x46eaacca,
+    0x8db67f6f, 0x90b34fd7, 0x5bef9c72, 0xdd7beedc, 0x16273d79,
+    0xe7718fac, 0x2c2d5c09, 0xaab92ea7, 0x61e5fd02, 0x7ce0cdba,
+    0xb7bc1e1f, 0x31286cb1, 0xfa74bf14, 0x1eb014d8, 0xd5ecc77d,
+    0x5378b5d3, 0x98246676, 0x852156ce, 0x4e7d856b, 0xc8e9f7c5,
+    0x03b52460, 0xf2e396b5, 0x39bf4510, 0xbf2b37be, 0x7477e41b,
+    0x6972d4a3, 0xa22e0706, 0x24ba75a8, 0xefe6a60d, 0x1d661643,
+    0xd63ac5e6, 0x50aeb748, 0x9bf264ed, 0x86f75455, 0x4dab87f0,
+    0xcb3ff55e, 0x006326fb, 0xf135942e, 0x3a69478b, 0xbcfd3525,
+    0x77a1e680, 0x6aa4d638, 0xa1f8059d, 0x276c7733, 0xec30a496,
+    0x191c11ee, 0xd240c24b, 0x54d4b0e5, 0x9f886340, 0x828d53f8,
+    0x49d1805d, 0xcf45f2f3, 0x04192156, 0xf54f9383, 0x3e134026,
+    0xb8873288, 0x73dbe12d, 0x6eded195, 0xa5820230, 0x2316709e,
+    0xe84aa33b, 0x1aca1375, 0xd196c0d0, 0x5702b27e, 0x9c5e61db,
+    0x815b5163, 0x4a0782c6, 0xcc93f068, 0x07cf23cd, 0xf6999118,
+    0x3dc542bd, 0xbb513013, 0x700de3b6, 0x6d08d30e, 0xa65400ab,
+    0x20c07205, 0xeb9ca1a0, 0x11e81eb4, 0xdab4cd11, 0x5c20bfbf,
+    0x977c6c1a, 0x8a795ca2, 0x41258f07, 0xc7b1fda9, 0x0ced2e0c,
+    0xfdbb9cd9, 0x36e74f7c, 0xb0733dd2, 0x7b2fee77, 0x662adecf,
+    0xad760d6a, 0x2be27fc4, 0xe0beac61, 0x123e1c2f, 0xd962cf8a,
+    0x5ff6bd24, 0x94aa6e81, 0x89af5e39, 0x42f38d9c, 0xc467ff32,
+    0x0f3b2c97, 0xfe6d9e42, 0x35314de7, 0xb3a53f49, 0x78f9ecec,
+    0x65fcdc54, 0xaea00ff1, 0x28347d5f, 0xe368aefa, 0x16441b82,
+    0xdd18c827, 0x5b8cba89, 0x90d0692c, 0x8dd55994, 0x46898a31,
+    0xc01df89f, 0x0b412b3a, 0xfa1799ef, 0x314b4a4a, 0xb7df38e4,
+    0x7c83eb41, 0x6186dbf9, 0xaada085c, 0x2c4e7af2, 0xe712a957,
+    0x15921919, 0xdececabc, 0x585ab812, 0x93066bb7, 0x8e035b0f,
+    0x455f88aa, 0xc3cbfa04, 0x089729a1, 0xf9c19b74, 0x329d48d1,
+    0xb4093a7f, 0x7f55e9da, 0x6250d962, 0xa90c0ac7, 0x2f987869,
+    0xe4c4abcc},
+   {0x00000000, 0x3d6029b0, 0x7ac05360, 0x47a07ad0, 0xf580a6c0,
+    0xc8e08f70, 0x8f40f5a0, 0xb220dc10, 0x30704bc1, 0x0d106271,
+    0x4ab018a1, 0x77d03111, 0xc5f0ed01, 0xf890c4b1, 0xbf30be61,
+    0x825097d1, 0x60e09782, 0x5d80be32, 0x1a20c4e2, 0x2740ed52,
+    0x95603142, 0xa80018f2, 0xefa06222, 0xd2c04b92, 0x5090dc43,
+    0x6df0f5f3, 0x2a508f23, 0x1730a693, 0xa5107a83, 0x98705333,
+    0xdfd029e3, 0xe2b00053, 0xc1c12f04, 0xfca106b4, 0xbb017c64,
+    0x866155d4, 0x344189c4, 0x0921a074, 0x4e81daa4, 0x73e1f314,
+    0xf1b164c5, 0xccd14d75, 0x8b7137a5, 0xb6111e15, 0x0431c205,
+    0x3951ebb5, 0x7ef19165, 0x4391b8d5, 0xa121b886, 0x9c419136,
+    0xdbe1ebe6, 0xe681c256, 0x54a11e46, 0x69c137f6, 0x2e614d26,
+    0x13016496, 0x9151f347, 0xac31daf7, 0xeb91a027, 0xd6f18997,
+    0x64d15587, 0x59b17c37, 0x1e1106e7, 0x23712f57, 0x58f35849,
+    0x659371f9, 0x22330b29, 0x1f532299, 0xad73fe89, 0x9013d739,
+    0xd7b3ade9, 0xead38459, 0x68831388, 0x55e33a38, 0x124340e8,
+    0x2f236958, 0x9d03b548, 0xa0639cf8, 0xe7c3e628, 0xdaa3cf98,
+    0x3813cfcb, 0x0573e67b, 0x42d39cab, 0x7fb3b51b, 0xcd93690b,
+    0xf0f340bb, 0xb7533a6b, 0x8a3313db, 0x0863840a, 0x3503adba,
+    0x72a3d76a, 0x4fc3feda, 0xfde322ca, 0xc0830b7a, 0x872371aa,
+    0xba43581a, 0x9932774d, 0xa4525efd, 0xe3f2242d, 0xde920d9d,
+    0x6cb2d18d, 0x51d2f83d, 0x167282ed, 0x2b12ab5d, 0xa9423c8c,
+    0x9422153c, 0xd3826fec, 0xeee2465c, 0x5cc29a4c, 0x61a2b3fc,
+    0x2602c92c, 0x1b62e09c, 0xf9d2e0cf, 0xc4b2c97f, 0x8312b3af,
+    0xbe729a1f, 0x0c52460f, 0x31326fbf, 0x7692156f, 0x4bf23cdf,
+    0xc9a2ab0e, 0xf4c282be, 0xb362f86e, 0x8e02d1de, 0x3c220dce,
+    0x0142247e, 0x46e25eae, 0x7b82771e, 0xb1e6b092, 0x8c869922,
+    0xcb26e3f2, 0xf646ca42, 0x44661652, 0x79063fe2, 0x3ea64532,
+    0x03c66c82, 0x8196fb53, 0xbcf6d2e3, 0xfb56a833, 0xc6368183,
+    0x74165d93, 0x49767423, 0x0ed60ef3, 0x33b62743, 0xd1062710,
+    0xec660ea0, 0xabc67470, 0x96a65dc0, 0x248681d0, 0x19e6a860,
+    0x5e46d2b0, 0x6326fb00, 0xe1766cd1, 0xdc164561, 0x9bb63fb1,
+    0xa6d61601, 0x14f6ca11, 0x2996e3a1, 0x6e369971, 0x5356b0c1,
+    0x70279f96, 0x4d47b626, 0x0ae7ccf6, 0x3787e546, 0x85a73956,
+    0xb8c710e6, 0xff676a36, 0xc2074386, 0x4057d457, 0x7d37fde7,
+    0x3a978737, 0x07f7ae87, 0xb5d77297, 0x88b75b27, 0xcf1721f7,
+    0xf2770847, 0x10c70814, 0x2da721a4, 0x6a075b74, 0x576772c4,
+    0xe547aed4, 0xd8278764, 0x9f87fdb4, 0xa2e7d404, 0x20b743d5,
+    0x1dd76a65, 0x5a7710b5, 0x67173905, 0xd537e515, 0xe857cca5,
+    0xaff7b675, 0x92979fc5, 0xe915e8db, 0xd475c16b, 0x93d5bbbb,
+    0xaeb5920b, 0x1c954e1b, 0x21f567ab, 0x66551d7b, 0x5b3534cb,
+    0xd965a31a, 0xe4058aaa, 0xa3a5f07a, 0x9ec5d9ca, 0x2ce505da,
+    0x11852c6a, 0x562556ba, 0x6b457f0a, 0x89f57f59, 0xb49556e9,
+    0xf3352c39, 0xce550589, 0x7c75d999, 0x4115f029, 0x06b58af9,
+    0x3bd5a349, 0xb9853498, 0x84e51d28, 0xc34567f8, 0xfe254e48,
+    0x4c059258, 0x7165bbe8, 0x36c5c138, 0x0ba5e888, 0x28d4c7df,
+    0x15b4ee6f, 0x521494bf, 0x6f74bd0f, 0xdd54611f, 0xe03448af,
+    0xa794327f, 0x9af41bcf, 0x18a48c1e, 0x25c4a5ae, 0x6264df7e,
+    0x5f04f6ce, 0xed242ade, 0xd044036e, 0x97e479be, 0xaa84500e,
+    0x4834505d, 0x755479ed, 0x32f4033d, 0x0f942a8d, 0xbdb4f69d,
+    0x80d4df2d, 0xc774a5fd, 0xfa148c4d, 0x78441b9c, 0x4524322c,
+    0x028448fc, 0x3fe4614c, 0x8dc4bd5c, 0xb0a494ec, 0xf704ee3c,
+    0xca64c78c},
+   {0x00000000, 0xb8bc6765, 0xaa09c88b, 0x12b5afee, 0x8f629757,
+    0x37def032, 0x256b5fdc, 0x9dd738b9, 0xc5b428ef, 0x7d084f8a,
+    0x6fbde064, 0xd7018701, 0x4ad6bfb8, 0xf26ad8dd, 0xe0df7733,
+    0x58631056, 0x5019579f, 0xe8a530fa, 0xfa109f14, 0x42acf871,
+    0xdf7bc0c8, 0x67c7a7ad, 0x75720843, 0xcdce6f26, 0x95ad7f70,
+    0x2d111815, 0x3fa4b7fb, 0x8718d09e, 0x1acfe827, 0xa2738f42,
+    0xb0c620ac, 0x087a47c9, 0xa032af3e, 0x188ec85b, 0x0a3b67b5,
+    0xb28700d0, 0x2f503869, 0x97ec5f0c, 0x8559f0e2, 0x3de59787,
+    0x658687d1, 0xdd3ae0b4, 0xcf8f4f5a, 0x7733283f, 0xeae41086,
+    0x525877e3, 0x40edd80d, 0xf851bf68, 0xf02bf8a1, 0x48979fc4,
+    0x5a22302a, 0xe29e574f, 0x7f496ff6, 0xc7f50893, 0xd540a77d,
+    0x6dfcc018, 0x359fd04e, 0x8d23b72b, 0x9f9618c5, 0x272a7fa0,
+    0xbafd4719, 0x0241207c, 0x10f48f92, 0xa848e8f7, 0x9b14583d,
+    0x23a83f58, 0x311d90b6, 0x89a1f7d3, 0x1476cf6a, 0xaccaa80f,
+    0xbe7f07e1, 0x06c36084, 0x5ea070d2, 0xe61c17b7, 0xf4a9b859,
+    0x4c15df3c, 0xd1c2e785, 0x697e80e0, 0x7bcb2f0e, 0xc377486b,
+    0xcb0d0fa2, 0x73b168c7, 0x6104c729, 0xd9b8a04c, 0x446f98f5,
+    0xfcd3ff90, 0xee66507e, 0x56da371b, 0x0eb9274d, 0xb6054028,
+    0xa4b0efc6, 0x1c0c88a3, 0x81dbb01a, 0x3967d77f, 0x2bd27891,
+    0x936e1ff4, 0x3b26f703, 0x839a9066, 0x912f3f88, 0x299358ed,
+    0xb4446054, 0x0cf80731, 0x1e4da8df, 0xa6f1cfba, 0xfe92dfec,
+    0x462eb889, 0x549b1767, 0xec277002, 0x71f048bb, 0xc94c2fde,
+    0xdbf98030, 0x6345e755, 0x6b3fa09c, 0xd383c7f9, 0xc1366817,
+    0x798a0f72, 0xe45d37cb, 0x5ce150ae, 0x4e54ff40, 0xf6e89825,
+    0xae8b8873, 0x1637ef16, 0x048240f8, 0xbc3e279d, 0x21e91f24,
+    0x99557841, 0x8be0d7af, 0x335cb0ca, 0xed59b63b, 0x55e5d15e,
+    0x47507eb0, 0xffec19d5, 0x623b216c, 0xda874609, 0xc832e9e7,
+    0x708e8e82, 0x28ed9ed4, 0x9051f9b1, 0x82e4565f, 0x3a58313a,
+    0xa78f0983, 0x1f336ee6, 0x0d86c108, 0xb53aa66d, 0xbd40e1a4,
+    0x05fc86c1, 0x1749292f, 0xaff54e4a, 0x322276f3, 0x8a9e1196,
+    0x982bbe78, 0x2097d91d, 0x78f4c94b, 0xc048ae2e, 0xd2fd01c0,
+    0x6a4166a5, 0xf7965e1c, 0x4f2a3979, 0x5d9f9697, 0xe523f1f2,
+    0x4d6b1905, 0xf5d77e60, 0xe762d18e, 0x5fdeb6eb, 0xc2098e52,
+    0x7ab5e937, 0x680046d9, 0xd0bc21bc, 0x88df31ea, 0x3063568f,
+    0x22d6f961, 0x9a6a9e04, 0x07bda6bd, 0xbf01c1d8, 0xadb46e36,
+    0x15080953, 0x1d724e9a, 0xa5ce29ff, 0xb77b8611, 0x0fc7e174,
+    0x9210d9cd, 0x2aacbea8, 0x38191146, 0x80a57623, 0xd8c66675,
+    0x607a0110, 0x72cfaefe, 0xca73c99b, 0x57a4f122, 0xef189647,
+    0xfdad39a9, 0x45115ecc, 0x764dee06, 0xcef18963, 0xdc44268d,
+    0x64f841e8, 0xf92f7951, 0x41931e34, 0x5326b1da, 0xeb9ad6bf,
+    0xb3f9c6e9, 0x0b45a18c, 0x19f00e62, 0xa14c6907, 0x3c9b51be,
+    0x842736db, 0x96929935, 0x2e2efe50, 0x2654b999, 0x9ee8defc,
+    0x8c5d7112, 0x34e11677, 0xa9362ece, 0x118a49ab, 0x033fe645,
+    0xbb838120, 0xe3e09176, 0x5b5cf613, 0x49e959fd, 0xf1553e98,
+    0x6c820621, 0xd43e6144, 0xc68bceaa, 0x7e37a9cf, 0xd67f4138,
+    0x6ec3265d, 0x7c7689b3, 0xc4caeed6, 0x591dd66f, 0xe1a1b10a,
+    0xf3141ee4, 0x4ba87981, 0x13cb69d7, 0xab770eb2, 0xb9c2a15c,
+    0x017ec639, 0x9ca9fe80, 0x241599e5, 0x36a0360b, 0x8e1c516e,
+    0x866616a7, 0x3eda71c2, 0x2c6fde2c, 0x94d3b949, 0x090481f0,
+    0xb1b8e695, 0xa30d497b, 0x1bb12e1e, 0x43d23e48, 0xfb6e592d,
+    0xe9dbf6c3, 0x516791a6, 0xccb0a91f, 0x740cce7a, 0x66b96194,
+    0xde0506f1},
+   {0x00000000, 0x01c26a37, 0x0384d46e, 0x0246be59, 0x0709a8dc,
+    0x06cbc2eb, 0x048d7cb2, 0x054f1685, 0x0e1351b8, 0x0fd13b8f,
+    0x0d9785d6, 0x0c55efe1, 0x091af964, 0x08d89353, 0x0a9e2d0a,
+    0x0b5c473d, 0x1c26a370, 0x1de4c947, 0x1fa2771e, 0x1e601d29,
+    0x1b2f0bac, 0x1aed619b, 0x18abdfc2, 0x1969b5f5, 0x1235f2c8,
+    0x13f798ff, 0x11b126a6, 0x10734c91, 0x153c5a14, 0x14fe3023,
+    0x16b88e7a, 0x177ae44d, 0x384d46e0, 0x398f2cd7, 0x3bc9928e,
+    0x3a0bf8b9, 0x3f44ee3c, 0x3e86840b, 0x3cc03a52, 0x3d025065,
+    0x365e1758, 0x379c7d6f, 0x35dac336, 0x3418a901, 0x3157bf84,
+    0x3095d5b3, 0x32d36bea, 0x331101dd, 0x246be590, 0x25a98fa7,
+    0x27ef31fe, 0x262d5bc9, 0x23624d4c, 0x22a0277b, 0x20e69922,
+    0x2124f315, 0x2a78b428, 0x2bbade1f, 0x29fc6046, 0x283e0a71,
+    0x2d711cf4, 0x2cb376c3, 0x2ef5c89a, 0x2f37a2ad, 0x709a8dc0,
+    0x7158e7f7, 0x731e59ae, 0x72dc3399, 0x7793251c, 0x76514f2b,
+    0x7417f172, 0x75d59b45, 0x7e89dc78, 0x7f4bb64f, 0x7d0d0816,
+    0x7ccf6221, 0x798074a4, 0x78421e93, 0x7a04a0ca, 0x7bc6cafd,
+    0x6cbc2eb0, 0x6d7e4487, 0x6f38fade, 0x6efa90e9, 0x6bb5866c,
+    0x6a77ec5b, 0x68315202, 0x69f33835, 0x62af7f08, 0x636d153f,
+    0x612bab66, 0x60e9c151, 0x65a6d7d4, 0x6464bde3, 0x662203ba,
+    0x67e0698d, 0x48d7cb20, 0x4915a117, 0x4b531f4e, 0x4a917579,
+    0x4fde63fc, 0x4e1c09cb, 0x4c5ab792, 0x4d98dda5, 0x46c49a98,
+    0x4706f0af, 0x45404ef6, 0x448224c1, 0x41cd3244, 0x400f5873,
+    0x4249e62a, 0x438b8c1d, 0x54f16850, 0x55330267, 0x5775bc3e,
+    0x56b7d609, 0x53f8c08c, 0x523aaabb, 0x507c14e2, 0x51be7ed5,
+    0x5ae239e8, 0x5b2053df, 0x5966ed86, 0x58a487b1, 0x5deb9134,
+    0x5c29fb03, 0x5e6f455a, 0x5fad2f6d, 0xe1351b80, 0xe0f771b7,
+    0xe2b1cfee, 0xe373a5d9, 0xe63cb35c, 0xe7fed96b, 0xe5b86732,
+    0xe47a0d05, 0xef264a38, 0xeee4200f, 0xeca29e56, 0xed60f461,
+    0xe82fe2e4, 0xe9ed88d3, 0xebab368a, 0xea695cbd, 0xfd13b8f0,
+    0xfcd1d2c7, 0xfe976c9e, 0xff5506a9, 0xfa1a102c, 0xfbd87a1b,
+    0xf99ec442, 0xf85cae75, 0xf300e948, 0xf2c2837f, 0xf0843d26,
+    0xf1465711, 0xf4094194, 0xf5cb2ba3, 0xf78d95fa, 0xf64fffcd,
+    0xd9785d60, 0xd8ba3757, 0xdafc890e, 0xdb3ee339, 0xde71f5bc,
+    0xdfb39f8b, 0xddf521d2, 0xdc374be5, 0xd76b0cd8, 0xd6a966ef,
+    0xd4efd8b6, 0xd52db281, 0xd062a404, 0xd1a0ce33, 0xd3e6706a,
+    0xd2241a5d, 0xc55efe10, 0xc49c9427, 0xc6da2a7e, 0xc7184049,
+    0xc25756cc, 0xc3953cfb, 0xc1d382a2, 0xc011e895, 0xcb4dafa8,
+    0xca8fc59f, 0xc8c97bc6, 0xc90b11f1, 0xcc440774, 0xcd866d43,
+    0xcfc0d31a, 0xce02b92d, 0x91af9640, 0x906dfc77, 0x922b422e,
+    0x93e92819, 0x96a63e9c, 0x976454ab, 0x9522eaf2, 0x94e080c5,
+    0x9fbcc7f8, 0x9e7eadcf, 0x9c381396, 0x9dfa79a1, 0x98b56f24,
+    0x99770513, 0x9b31bb4a, 0x9af3d17d, 0x8d893530, 0x8c4b5f07,
+    0x8e0de15e, 0x8fcf8b69, 0x8a809dec, 0x8b42f7db, 0x89044982,
+    0x88c623b5, 0x839a6488, 0x82580ebf, 0x801eb0e6, 0x81dcdad1,
+    0x8493cc54, 0x8551a663, 0x8717183a, 0x86d5720d, 0xa9e2d0a0,
+    0xa820ba97, 0xaa6604ce, 0xaba46ef9, 0xaeeb787c, 0xaf29124b,
+    0xad6fac12, 0xacadc625, 0xa7f18118, 0xa633eb2f, 0xa4755576,
+    0xa5b73f41, 0xa0f829c4, 0xa13a43f3, 0xa37cfdaa, 0xa2be979d,
+    0xb5c473d0, 0xb40619e7, 0xb640a7be, 0xb782cd89, 0xb2cddb0c,
+    0xb30fb13b, 0xb1490f62, 0xb08b6555, 0xbbd72268, 0xba15485f,
+    0xb853f606, 0xb9919c31, 0xbcde8ab4, 0xbd1ce083, 0xbf5a5eda,
+    0xbe9834ed},
+   {0x00000000, 0x191b3141, 0x32366282, 0x2b2d53c3, 0x646cc504,
+    0x7d77f445, 0x565aa786, 0x4f4196c7, 0xc8d98a08, 0xd1c2bb49,
+    0xfaefe88a, 0xe3f4d9cb, 0xacb54f0c, 0xb5ae7e4d, 0x9e832d8e,
+    0x87981ccf, 0x4ac21251, 0x53d92310, 0x78f470d3, 0x61ef4192,
+    0x2eaed755, 0x37b5e614, 0x1c98b5d7, 0x05838496, 0x821b9859,
+    0x9b00a918, 0xb02dfadb, 0xa936cb9a, 0xe6775d5d, 0xff6c6c1c,
+    0xd4413fdf, 0xcd5a0e9e, 0x958424a2, 0x8c9f15e3, 0xa7b24620,
+    0xbea97761, 0xf1e8e1a6, 0xe8f3d0e7, 0xc3de8324, 0xdac5b265,
+    0x5d5daeaa, 0x44469feb, 0x6f6bcc28, 0x7670fd69, 0x39316bae,
+    0x202a5aef, 0x0b07092c, 0x121c386d, 0xdf4636f3, 0xc65d07b2,
+    0xed705471, 0xf46b6530, 0xbb2af3f7, 0xa231c2b6, 0x891c9175,
+    0x9007a034, 0x179fbcfb, 0x0e848dba, 0x25a9de79, 0x3cb2ef38,
+    0x73f379ff, 0x6ae848be, 0x41c51b7d, 0x58de2a3c, 0xf0794f05,
+    0xe9627e44, 0xc24f2d87, 0xdb541cc6, 0x94158a01, 0x8d0ebb40,
+    0xa623e883, 0xbf38d9c2, 0x38a0c50d, 0x21bbf44c, 0x0a96a78f,
+    0x138d96ce, 0x5ccc0009, 0x45d73148, 0x6efa628b, 0x77e153ca,
+    0xbabb5d54, 0xa3a06c15, 0x888d3fd6, 0x91960e97, 0xded79850,
+    0xc7cca911, 0xece1fad2, 0xf5facb93, 0x7262d75c, 0x6b79e61d,
+    0x4054b5de, 0x594f849f, 0x160e1258, 0x0f152319, 0x243870da,
+    0x3d23419b, 0x65fd6ba7, 0x7ce65ae6, 0x57cb0925, 0x4ed03864,
+    0x0191aea3, 0x188a9fe2, 0x33a7cc21, 0x2abcfd60, 0xad24e1af,
+    0xb43fd0ee, 0x9f12832d, 0x8609b26c, 0xc94824ab, 0xd05315ea,
+    0xfb7e4629, 0xe2657768, 0x2f3f79f6, 0x362448b7, 0x1d091b74,
+    0x04122a35, 0x4b53bcf2, 0x52488db3, 0x7965de70, 0x607eef31,
+    0xe7e6f3fe, 0xfefdc2bf, 0xd5d0917c, 0xcccba03d, 0x838a36fa,
+    0x9a9107bb, 0xb1bc5478, 0xa8a76539, 0x3b83984b, 0x2298a90a,
+    0x09b5fac9, 0x10aecb88, 0x5fef5d4f, 0x46f46c0e, 0x6dd93fcd,
+    0x74c20e8c, 0xf35a1243, 0xea412302, 0xc16c70c1, 0xd8774180,
+    0x9736d747, 0x8e2de606, 0xa500b5c5, 0xbc1b8484, 0x71418a1a,
+    0x685abb5b, 0x4377e898, 0x5a6cd9d9, 0x152d4f1e, 0x0c367e5f,
+    0x271b2d9c, 0x3e001cdd, 0xb9980012, 0xa0833153, 0x8bae6290,
+    0x92b553d1, 0xddf4c516, 0xc4eff457, 0xefc2a794, 0xf6d996d5,
+    0xae07bce9, 0xb71c8da8, 0x9c31de6b, 0x852aef2a, 0xca6b79ed,
+    0xd37048ac, 0xf85d1b6f, 0xe1462a2e, 0x66de36e1, 0x7fc507a0,
+    0x54e85463, 0x4df36522, 0x02b2f3e5, 0x1ba9c2a4, 0x30849167,
+    0x299fa026, 0xe4c5aeb8, 0xfdde9ff9, 0xd6f3cc3a, 0xcfe8fd7b,
+    0x80a96bbc, 0x99b25afd, 0xb29f093e, 0xab84387f, 0x2c1c24b0,
+    0x350715f1, 0x1e2a4632, 0x07317773, 0x4870e1b4, 0x516bd0f5,
+    0x7a468336, 0x635db277, 0xcbfad74e, 0xd2e1e60f, 0xf9ccb5cc,
+    0xe0d7848d, 0xaf96124a, 0xb68d230b, 0x9da070c8, 0x84bb4189,
+    0x03235d46, 0x1a386c07, 0x31153fc4, 0x280e0e85, 0x674f9842,
+    0x7e54a903, 0x5579fac0, 0x4c62cb81, 0x8138c51f, 0x9823f45e,
+    0xb30ea79d, 0xaa1596dc, 0xe554001b, 0xfc4f315a, 0xd7626299,
+    0xce7953d8, 0x49e14f17, 0x50fa7e56, 0x7bd72d95, 0x62cc1cd4,
+    0x2d8d8a13, 0x3496bb52, 0x1fbbe891, 0x06a0d9d0, 0x5e7ef3ec,
+    0x4765c2ad, 0x6c48916e, 0x7553a02f, 0x3a1236e8, 0x230907a9,
+    0x0824546a, 0x113f652b, 0x96a779e4, 0x8fbc48a5, 0xa4911b66,
+    0xbd8a2a27, 0xf2cbbce0, 0xebd08da1, 0xc0fdde62, 0xd9e6ef23,
+    0x14bce1bd, 0x0da7d0fc, 0x268a833f, 0x3f91b27e, 0x70d024b9,
+    0x69cb15f8, 0x42e6463b, 0x5bfd777a, 0xdc656bb5, 0xc57e5af4,
+    0xee530937, 0xf7483876, 0xb809aeb1, 0xa1129ff0, 0x8a3fcc33,
+    0x9324fd72},
+   {0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419,
+    0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4,
+    0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07,
+    0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
+    0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856,
+    0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
+    0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4,
+    0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
+    0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3,
+    0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a,
+    0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599,
+    0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
+    0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190,
+    0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f,
+    0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e,
+    0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
+    0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed,
+    0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
+    0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3,
+    0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
+    0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a,
+    0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5,
+    0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010,
+    0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+    0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17,
+    0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6,
+    0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615,
+    0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
+    0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344,
+    0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
+    0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a,
+    0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
+    0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1,
+    0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c,
+    0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef,
+    0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
+    0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe,
+    0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31,
+    0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c,
+    0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
+    0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b,
+    0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
+    0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1,
+    0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
+    0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278,
+    0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7,
+    0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66,
+    0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+    0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605,
+    0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8,
+    0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b,
+    0x2d02ef8d}};
+
+local const z_word_t FAR crc_braid_big_table[][256] = {
+   {0x0000000000000000, 0x9630077700000000, 0x2c610eee00000000,
+    0xba51099900000000, 0x19c46d0700000000, 0x8ff46a7000000000,
+    0x35a563e900000000, 0xa395649e00000000, 0x3288db0e00000000,
+    0xa4b8dc7900000000, 0x1ee9d5e000000000, 0x88d9d29700000000,
+    0x2b4cb60900000000, 0xbd7cb17e00000000, 0x072db8e700000000,
+    0x911dbf9000000000, 0x6410b71d00000000, 0xf220b06a00000000,
+    0x4871b9f300000000, 0xde41be8400000000, 0x7dd4da1a00000000,
+    0xebe4dd6d00000000, 0x51b5d4f400000000, 0xc785d38300000000,
+    0x56986c1300000000, 0xc0a86b6400000000, 0x7af962fd00000000,
+    0xecc9658a00000000, 0x4f5c011400000000, 0xd96c066300000000,
+    0x633d0ffa00000000, 0xf50d088d00000000, 0xc8206e3b00000000,
+    0x5e10694c00000000, 0xe44160d500000000, 0x727167a200000000,
+    0xd1e4033c00000000, 0x47d4044b00000000, 0xfd850dd200000000,
+    0x6bb50aa500000000, 0xfaa8b53500000000, 0x6c98b24200000000,
+    0xd6c9bbdb00000000, 0x40f9bcac00000000, 0xe36cd83200000000,
+    0x755cdf4500000000, 0xcf0dd6dc00000000, 0x593dd1ab00000000,
+    0xac30d92600000000, 0x3a00de5100000000, 0x8051d7c800000000,
+    0x1661d0bf00000000, 0xb5f4b42100000000, 0x23c4b35600000000,
+    0x9995bacf00000000, 0x0fa5bdb800000000, 0x9eb8022800000000,
+    0x0888055f00000000, 0xb2d90cc600000000, 0x24e90bb100000000,
+    0x877c6f2f00000000, 0x114c685800000000, 0xab1d61c100000000,
+    0x3d2d66b600000000, 0x9041dc7600000000, 0x0671db0100000000,
+    0xbc20d29800000000, 0x2a10d5ef00000000, 0x8985b17100000000,
+    0x1fb5b60600000000, 0xa5e4bf9f00000000, 0x33d4b8e800000000,
+    0xa2c9077800000000, 0x34f9000f00000000, 0x8ea8099600000000,
+    0x18980ee100000000, 0xbb0d6a7f00000000, 0x2d3d6d0800000000,
+    0x976c649100000000, 0x015c63e600000000, 0xf4516b6b00000000,
+    0x62616c1c00000000, 0xd830658500000000, 0x4e0062f200000000,
+    0xed95066c00000000, 0x7ba5011b00000000, 0xc1f4088200000000,
+    0x57c40ff500000000, 0xc6d9b06500000000, 0x50e9b71200000000,
+    0xeab8be8b00000000, 0x7c88b9fc00000000, 0xdf1ddd6200000000,
+    0x492dda1500000000, 0xf37cd38c00000000, 0x654cd4fb00000000,
+    0x5861b24d00000000, 0xce51b53a00000000, 0x7400bca300000000,
+    0xe230bbd400000000, 0x41a5df4a00000000, 0xd795d83d00000000,
+    0x6dc4d1a400000000, 0xfbf4d6d300000000, 0x6ae9694300000000,
+    0xfcd96e3400000000, 0x468867ad00000000, 0xd0b860da00000000,
+    0x732d044400000000, 0xe51d033300000000, 0x5f4c0aaa00000000,
+    0xc97c0ddd00000000, 0x3c71055000000000, 0xaa41022700000000,
+    0x10100bbe00000000, 0x86200cc900000000, 0x25b5685700000000,
+    0xb3856f2000000000, 0x09d466b900000000, 0x9fe461ce00000000,
+    0x0ef9de5e00000000, 0x98c9d92900000000, 0x2298d0b000000000,
+    0xb4a8d7c700000000, 0x173db35900000000, 0x810db42e00000000,
+    0x3b5cbdb700000000, 0xad6cbac000000000, 0x2083b8ed00000000,
+    0xb6b3bf9a00000000, 0x0ce2b60300000000, 0x9ad2b17400000000,
+    0x3947d5ea00000000, 0xaf77d29d00000000, 0x1526db0400000000,
+    0x8316dc7300000000, 0x120b63e300000000, 0x843b649400000000,
+    0x3e6a6d0d00000000, 0xa85a6a7a00000000, 0x0bcf0ee400000000,
+    0x9dff099300000000, 0x27ae000a00000000, 0xb19e077d00000000,
+    0x44930ff000000000, 0xd2a3088700000000, 0x68f2011e00000000,
+    0xfec2066900000000, 0x5d5762f700000000, 0xcb67658000000000,
+    0x71366c1900000000, 0xe7066b6e00000000, 0x761bd4fe00000000,
+    0xe02bd38900000000, 0x5a7ada1000000000, 0xcc4add6700000000,
+    0x6fdfb9f900000000, 0xf9efbe8e00000000, 0x43beb71700000000,
+    0xd58eb06000000000, 0xe8a3d6d600000000, 0x7e93d1a100000000,
+    0xc4c2d83800000000, 0x52f2df4f00000000, 0xf167bbd100000000,
+    0x6757bca600000000, 0xdd06b53f00000000, 0x4b36b24800000000,
+    0xda2b0dd800000000, 0x4c1b0aaf00000000, 0xf64a033600000000,
+    0x607a044100000000, 0xc3ef60df00000000, 0x55df67a800000000,
+    0xef8e6e3100000000, 0x79be694600000000, 0x8cb361cb00000000,
+    0x1a8366bc00000000, 0xa0d26f2500000000, 0x36e2685200000000,
+    0x95770ccc00000000, 0x03470bbb00000000, 0xb916022200000000,
+    0x2f26055500000000, 0xbe3bbac500000000, 0x280bbdb200000000,
+    0x925ab42b00000000, 0x046ab35c00000000, 0xa7ffd7c200000000,
+    0x31cfd0b500000000, 0x8b9ed92c00000000, 0x1daede5b00000000,
+    0xb0c2649b00000000, 0x26f263ec00000000, 0x9ca36a7500000000,
+    0x0a936d0200000000, 0xa906099c00000000, 0x3f360eeb00000000,
+    0x8567077200000000, 0x1357000500000000, 0x824abf9500000000,
+    0x147ab8e200000000, 0xae2bb17b00000000, 0x381bb60c00000000,
+    0x9b8ed29200000000, 0x0dbed5e500000000, 0xb7efdc7c00000000,
+    0x21dfdb0b00000000, 0xd4d2d38600000000, 0x42e2d4f100000000,
+    0xf8b3dd6800000000, 0x6e83da1f00000000, 0xcd16be8100000000,
+    0x5b26b9f600000000, 0xe177b06f00000000, 0x7747b71800000000,
+    0xe65a088800000000, 0x706a0fff00000000, 0xca3b066600000000,
+    0x5c0b011100000000, 0xff9e658f00000000, 0x69ae62f800000000,
+    0xd3ff6b6100000000, 0x45cf6c1600000000, 0x78e20aa000000000,
+    0xeed20dd700000000, 0x5483044e00000000, 0xc2b3033900000000,
+    0x612667a700000000, 0xf71660d000000000, 0x4d47694900000000,
+    0xdb776e3e00000000, 0x4a6ad1ae00000000, 0xdc5ad6d900000000,
+    0x660bdf4000000000, 0xf03bd83700000000, 0x53aebca900000000,
+    0xc59ebbde00000000, 0x7fcfb24700000000, 0xe9ffb53000000000,
+    0x1cf2bdbd00000000, 0x8ac2baca00000000, 0x3093b35300000000,
+    0xa6a3b42400000000, 0x0536d0ba00000000, 0x9306d7cd00000000,
+    0x2957de5400000000, 0xbf67d92300000000, 0x2e7a66b300000000,
+    0xb84a61c400000000, 0x021b685d00000000, 0x942b6f2a00000000,
+    0x37be0bb400000000, 0xa18e0cc300000000, 0x1bdf055a00000000,
+    0x8def022d00000000},
+   {0x0000000000000000, 0x41311b1900000000, 0x8262363200000000,
+    0xc3532d2b00000000, 0x04c56c6400000000, 0x45f4777d00000000,
+    0x86a75a5600000000, 0xc796414f00000000, 0x088ad9c800000000,
+    0x49bbc2d100000000, 0x8ae8effa00000000, 0xcbd9f4e300000000,
+    0x0c4fb5ac00000000, 0x4d7eaeb500000000, 0x8e2d839e00000000,
+    0xcf1c988700000000, 0x5112c24a00000000, 0x1023d95300000000,
+    0xd370f47800000000, 0x9241ef6100000000, 0x55d7ae2e00000000,
+    0x14e6b53700000000, 0xd7b5981c00000000, 0x9684830500000000,
+    0x59981b8200000000, 0x18a9009b00000000, 0xdbfa2db000000000,
+    0x9acb36a900000000, 0x5d5d77e600000000, 0x1c6c6cff00000000,
+    0xdf3f41d400000000, 0x9e0e5acd00000000, 0xa224849500000000,
+    0xe3159f8c00000000, 0x2046b2a700000000, 0x6177a9be00000000,
+    0xa6e1e8f100000000, 0xe7d0f3e800000000, 0x2483dec300000000,
+    0x65b2c5da00000000, 0xaaae5d5d00000000, 0xeb9f464400000000,
+    0x28cc6b6f00000000, 0x69fd707600000000, 0xae6b313900000000,
+    0xef5a2a2000000000, 0x2c09070b00000000, 0x6d381c1200000000,
+    0xf33646df00000000, 0xb2075dc600000000, 0x715470ed00000000,
+    0x30656bf400000000, 0xf7f32abb00000000, 0xb6c231a200000000,
+    0x75911c8900000000, 0x34a0079000000000, 0xfbbc9f1700000000,
+    0xba8d840e00000000, 0x79dea92500000000, 0x38efb23c00000000,
+    0xff79f37300000000, 0xbe48e86a00000000, 0x7d1bc54100000000,
+    0x3c2ade5800000000, 0x054f79f000000000, 0x447e62e900000000,
+    0x872d4fc200000000, 0xc61c54db00000000, 0x018a159400000000,
+    0x40bb0e8d00000000, 0x83e823a600000000, 0xc2d938bf00000000,
+    0x0dc5a03800000000, 0x4cf4bb2100000000, 0x8fa7960a00000000,
+    0xce968d1300000000, 0x0900cc5c00000000, 0x4831d74500000000,
+    0x8b62fa6e00000000, 0xca53e17700000000, 0x545dbbba00000000,
+    0x156ca0a300000000, 0xd63f8d8800000000, 0x970e969100000000,
+    0x5098d7de00000000, 0x11a9ccc700000000, 0xd2fae1ec00000000,
+    0x93cbfaf500000000, 0x5cd7627200000000, 0x1de6796b00000000,
+    0xdeb5544000000000, 0x9f844f5900000000, 0x58120e1600000000,
+    0x1923150f00000000, 0xda70382400000000, 0x9b41233d00000000,
+    0xa76bfd6500000000, 0xe65ae67c00000000, 0x2509cb5700000000,
+    0x6438d04e00000000, 0xa3ae910100000000, 0xe29f8a1800000000,
+    0x21cca73300000000, 0x60fdbc2a00000000, 0xafe124ad00000000,
+    0xeed03fb400000000, 0x2d83129f00000000, 0x6cb2098600000000,
+    0xab2448c900000000, 0xea1553d000000000, 0x29467efb00000000,
+    0x687765e200000000, 0xf6793f2f00000000, 0xb748243600000000,
+    0x741b091d00000000, 0x352a120400000000, 0xf2bc534b00000000,
+    0xb38d485200000000, 0x70de657900000000, 0x31ef7e6000000000,
+    0xfef3e6e700000000, 0xbfc2fdfe00000000, 0x7c91d0d500000000,
+    0x3da0cbcc00000000, 0xfa368a8300000000, 0xbb07919a00000000,
+    0x7854bcb100000000, 0x3965a7a800000000, 0x4b98833b00000000,
+    0x0aa9982200000000, 0xc9fab50900000000, 0x88cbae1000000000,
+    0x4f5def5f00000000, 0x0e6cf44600000000, 0xcd3fd96d00000000,
+    0x8c0ec27400000000, 0x43125af300000000, 0x022341ea00000000,
+    0xc1706cc100000000, 0x804177d800000000, 0x47d7369700000000,
+    0x06e62d8e00000000, 0xc5b500a500000000, 0x84841bbc00000000,
+    0x1a8a417100000000, 0x5bbb5a6800000000, 0x98e8774300000000,
+    0xd9d96c5a00000000, 0x1e4f2d1500000000, 0x5f7e360c00000000,
+    0x9c2d1b2700000000, 0xdd1c003e00000000, 0x120098b900000000,
+    0x533183a000000000, 0x9062ae8b00000000, 0xd153b59200000000,
+    0x16c5f4dd00000000, 0x57f4efc400000000, 0x94a7c2ef00000000,
+    0xd596d9f600000000, 0xe9bc07ae00000000, 0xa88d1cb700000000,
+    0x6bde319c00000000, 0x2aef2a8500000000, 0xed796bca00000000,
+    0xac4870d300000000, 0x6f1b5df800000000, 0x2e2a46e100000000,
+    0xe136de6600000000, 0xa007c57f00000000, 0x6354e85400000000,
+    0x2265f34d00000000, 0xe5f3b20200000000, 0xa4c2a91b00000000,
+    0x6791843000000000, 0x26a09f2900000000, 0xb8aec5e400000000,
+    0xf99fdefd00000000, 0x3accf3d600000000, 0x7bfde8cf00000000,
+    0xbc6ba98000000000, 0xfd5ab29900000000, 0x3e099fb200000000,
+    0x7f3884ab00000000, 0xb0241c2c00000000, 0xf115073500000000,
+    0x32462a1e00000000, 0x7377310700000000, 0xb4e1704800000000,
+    0xf5d06b5100000000, 0x3683467a00000000, 0x77b25d6300000000,
+    0x4ed7facb00000000, 0x0fe6e1d200000000, 0xccb5ccf900000000,
+    0x8d84d7e000000000, 0x4a1296af00000000, 0x0b238db600000000,
+    0xc870a09d00000000, 0x8941bb8400000000, 0x465d230300000000,
+    0x076c381a00000000, 0xc43f153100000000, 0x850e0e2800000000,
+    0x42984f6700000000, 0x03a9547e00000000, 0xc0fa795500000000,
+    0x81cb624c00000000, 0x1fc5388100000000, 0x5ef4239800000000,
+    0x9da70eb300000000, 0xdc9615aa00000000, 0x1b0054e500000000,
+    0x5a314ffc00000000, 0x996262d700000000, 0xd85379ce00000000,
+    0x174fe14900000000, 0x567efa5000000000, 0x952dd77b00000000,
+    0xd41ccc6200000000, 0x138a8d2d00000000, 0x52bb963400000000,
+    0x91e8bb1f00000000, 0xd0d9a00600000000, 0xecf37e5e00000000,
+    0xadc2654700000000, 0x6e91486c00000000, 0x2fa0537500000000,
+    0xe836123a00000000, 0xa907092300000000, 0x6a54240800000000,
+    0x2b653f1100000000, 0xe479a79600000000, 0xa548bc8f00000000,
+    0x661b91a400000000, 0x272a8abd00000000, 0xe0bccbf200000000,
+    0xa18dd0eb00000000, 0x62defdc000000000, 0x23efe6d900000000,
+    0xbde1bc1400000000, 0xfcd0a70d00000000, 0x3f838a2600000000,
+    0x7eb2913f00000000, 0xb924d07000000000, 0xf815cb6900000000,
+    0x3b46e64200000000, 0x7a77fd5b00000000, 0xb56b65dc00000000,
+    0xf45a7ec500000000, 0x370953ee00000000, 0x763848f700000000,
+    0xb1ae09b800000000, 0xf09f12a100000000, 0x33cc3f8a00000000,
+    0x72fd249300000000},
+   {0x0000000000000000, 0x376ac20100000000, 0x6ed4840300000000,
+    0x59be460200000000, 0xdca8090700000000, 0xebc2cb0600000000,
+    0xb27c8d0400000000, 0x85164f0500000000, 0xb851130e00000000,
+    0x8f3bd10f00000000, 0xd685970d00000000, 0xe1ef550c00000000,
+    0x64f91a0900000000, 0x5393d80800000000, 0x0a2d9e0a00000000,
+    0x3d475c0b00000000, 0x70a3261c00000000, 0x47c9e41d00000000,
+    0x1e77a21f00000000, 0x291d601e00000000, 0xac0b2f1b00000000,
+    0x9b61ed1a00000000, 0xc2dfab1800000000, 0xf5b5691900000000,
+    0xc8f2351200000000, 0xff98f71300000000, 0xa626b11100000000,
+    0x914c731000000000, 0x145a3c1500000000, 0x2330fe1400000000,
+    0x7a8eb81600000000, 0x4de47a1700000000, 0xe0464d3800000000,
+    0xd72c8f3900000000, 0x8e92c93b00000000, 0xb9f80b3a00000000,
+    0x3cee443f00000000, 0x0b84863e00000000, 0x523ac03c00000000,
+    0x6550023d00000000, 0x58175e3600000000, 0x6f7d9c3700000000,
+    0x36c3da3500000000, 0x01a9183400000000, 0x84bf573100000000,
+    0xb3d5953000000000, 0xea6bd33200000000, 0xdd01113300000000,
+    0x90e56b2400000000, 0xa78fa92500000000, 0xfe31ef2700000000,
+    0xc95b2d2600000000, 0x4c4d622300000000, 0x7b27a02200000000,
+    0x2299e62000000000, 0x15f3242100000000, 0x28b4782a00000000,
+    0x1fdeba2b00000000, 0x4660fc2900000000, 0x710a3e2800000000,
+    0xf41c712d00000000, 0xc376b32c00000000, 0x9ac8f52e00000000,
+    0xada2372f00000000, 0xc08d9a7000000000, 0xf7e7587100000000,
+    0xae591e7300000000, 0x9933dc7200000000, 0x1c25937700000000,
+    0x2b4f517600000000, 0x72f1177400000000, 0x459bd57500000000,
+    0x78dc897e00000000, 0x4fb64b7f00000000, 0x16080d7d00000000,
+    0x2162cf7c00000000, 0xa474807900000000, 0x931e427800000000,
+    0xcaa0047a00000000, 0xfdcac67b00000000, 0xb02ebc6c00000000,
+    0x87447e6d00000000, 0xdefa386f00000000, 0xe990fa6e00000000,
+    0x6c86b56b00000000, 0x5bec776a00000000, 0x0252316800000000,
+    0x3538f36900000000, 0x087faf6200000000, 0x3f156d6300000000,
+    0x66ab2b6100000000, 0x51c1e96000000000, 0xd4d7a66500000000,
+    0xe3bd646400000000, 0xba03226600000000, 0x8d69e06700000000,
+    0x20cbd74800000000, 0x17a1154900000000, 0x4e1f534b00000000,
+    0x7975914a00000000, 0xfc63de4f00000000, 0xcb091c4e00000000,
+    0x92b75a4c00000000, 0xa5dd984d00000000, 0x989ac44600000000,
+    0xaff0064700000000, 0xf64e404500000000, 0xc124824400000000,
+    0x4432cd4100000000, 0x73580f4000000000, 0x2ae6494200000000,
+    0x1d8c8b4300000000, 0x5068f15400000000, 0x6702335500000000,
+    0x3ebc755700000000, 0x09d6b75600000000, 0x8cc0f85300000000,
+    0xbbaa3a5200000000, 0xe2147c5000000000, 0xd57ebe5100000000,
+    0xe839e25a00000000, 0xdf53205b00000000, 0x86ed665900000000,
+    0xb187a45800000000, 0x3491eb5d00000000, 0x03fb295c00000000,
+    0x5a456f5e00000000, 0x6d2fad5f00000000, 0x801b35e100000000,
+    0xb771f7e000000000, 0xeecfb1e200000000, 0xd9a573e300000000,
+    0x5cb33ce600000000, 0x6bd9fee700000000, 0x3267b8e500000000,
+    0x050d7ae400000000, 0x384a26ef00000000, 0x0f20e4ee00000000,
+    0x569ea2ec00000000, 0x61f460ed00000000, 0xe4e22fe800000000,
+    0xd388ede900000000, 0x8a36abeb00000000, 0xbd5c69ea00000000,
+    0xf0b813fd00000000, 0xc7d2d1fc00000000, 0x9e6c97fe00000000,
+    0xa90655ff00000000, 0x2c101afa00000000, 0x1b7ad8fb00000000,
+    0x42c49ef900000000, 0x75ae5cf800000000, 0x48e900f300000000,
+    0x7f83c2f200000000, 0x263d84f000000000, 0x115746f100000000,
+    0x944109f400000000, 0xa32bcbf500000000, 0xfa958df700000000,
+    0xcdff4ff600000000, 0x605d78d900000000, 0x5737bad800000000,
+    0x0e89fcda00000000, 0x39e33edb00000000, 0xbcf571de00000000,
+    0x8b9fb3df00000000, 0xd221f5dd00000000, 0xe54b37dc00000000,
+    0xd80c6bd700000000, 0xef66a9d600000000, 0xb6d8efd400000000,
+    0x81b22dd500000000, 0x04a462d000000000, 0x33cea0d100000000,
+    0x6a70e6d300000000, 0x5d1a24d200000000, 0x10fe5ec500000000,
+    0x27949cc400000000, 0x7e2adac600000000, 0x494018c700000000,
+    0xcc5657c200000000, 0xfb3c95c300000000, 0xa282d3c100000000,
+    0x95e811c000000000, 0xa8af4dcb00000000, 0x9fc58fca00000000,
+    0xc67bc9c800000000, 0xf1110bc900000000, 0x740744cc00000000,
+    0x436d86cd00000000, 0x1ad3c0cf00000000, 0x2db902ce00000000,
+    0x4096af9100000000, 0x77fc6d9000000000, 0x2e422b9200000000,
+    0x1928e99300000000, 0x9c3ea69600000000, 0xab54649700000000,
+    0xf2ea229500000000, 0xc580e09400000000, 0xf8c7bc9f00000000,
+    0xcfad7e9e00000000, 0x9613389c00000000, 0xa179fa9d00000000,
+    0x246fb59800000000, 0x1305779900000000, 0x4abb319b00000000,
+    0x7dd1f39a00000000, 0x3035898d00000000, 0x075f4b8c00000000,
+    0x5ee10d8e00000000, 0x698bcf8f00000000, 0xec9d808a00000000,
+    0xdbf7428b00000000, 0x8249048900000000, 0xb523c68800000000,
+    0x88649a8300000000, 0xbf0e588200000000, 0xe6b01e8000000000,
+    0xd1dadc8100000000, 0x54cc938400000000, 0x63a6518500000000,
+    0x3a18178700000000, 0x0d72d58600000000, 0xa0d0e2a900000000,
+    0x97ba20a800000000, 0xce0466aa00000000, 0xf96ea4ab00000000,
+    0x7c78ebae00000000, 0x4b1229af00000000, 0x12ac6fad00000000,
+    0x25c6adac00000000, 0x1881f1a700000000, 0x2feb33a600000000,
+    0x765575a400000000, 0x413fb7a500000000, 0xc429f8a000000000,
+    0xf3433aa100000000, 0xaafd7ca300000000, 0x9d97bea200000000,
+    0xd073c4b500000000, 0xe71906b400000000, 0xbea740b600000000,
+    0x89cd82b700000000, 0x0cdbcdb200000000, 0x3bb10fb300000000,
+    0x620f49b100000000, 0x55658bb000000000, 0x6822d7bb00000000,
+    0x5f4815ba00000000, 0x06f653b800000000, 0x319c91b900000000,
+    0xb48adebc00000000, 0x83e01cbd00000000, 0xda5e5abf00000000,
+    0xed3498be00000000},
+   {0x0000000000000000, 0x6567bcb800000000, 0x8bc809aa00000000,
+    0xeeafb51200000000, 0x5797628f00000000, 0x32f0de3700000000,
+    0xdc5f6b2500000000, 0xb938d79d00000000, 0xef28b4c500000000,
+    0x8a4f087d00000000, 0x64e0bd6f00000000, 0x018701d700000000,
+    0xb8bfd64a00000000, 0xddd86af200000000, 0x3377dfe000000000,
+    0x5610635800000000, 0x9f57195000000000, 0xfa30a5e800000000,
+    0x149f10fa00000000, 0x71f8ac4200000000, 0xc8c07bdf00000000,
+    0xada7c76700000000, 0x4308727500000000, 0x266fcecd00000000,
+    0x707fad9500000000, 0x1518112d00000000, 0xfbb7a43f00000000,
+    0x9ed0188700000000, 0x27e8cf1a00000000, 0x428f73a200000000,
+    0xac20c6b000000000, 0xc9477a0800000000, 0x3eaf32a000000000,
+    0x5bc88e1800000000, 0xb5673b0a00000000, 0xd00087b200000000,
+    0x6938502f00000000, 0x0c5fec9700000000, 0xe2f0598500000000,
+    0x8797e53d00000000, 0xd187866500000000, 0xb4e03add00000000,
+    0x5a4f8fcf00000000, 0x3f28337700000000, 0x8610e4ea00000000,
+    0xe377585200000000, 0x0dd8ed4000000000, 0x68bf51f800000000,
+    0xa1f82bf000000000, 0xc49f974800000000, 0x2a30225a00000000,
+    0x4f579ee200000000, 0xf66f497f00000000, 0x9308f5c700000000,
+    0x7da740d500000000, 0x18c0fc6d00000000, 0x4ed09f3500000000,
+    0x2bb7238d00000000, 0xc518969f00000000, 0xa07f2a2700000000,
+    0x1947fdba00000000, 0x7c20410200000000, 0x928ff41000000000,
+    0xf7e848a800000000, 0x3d58149b00000000, 0x583fa82300000000,
+    0xb6901d3100000000, 0xd3f7a18900000000, 0x6acf761400000000,
+    0x0fa8caac00000000, 0xe1077fbe00000000, 0x8460c30600000000,
+    0xd270a05e00000000, 0xb7171ce600000000, 0x59b8a9f400000000,
+    0x3cdf154c00000000, 0x85e7c2d100000000, 0xe0807e6900000000,
+    0x0e2fcb7b00000000, 0x6b4877c300000000, 0xa20f0dcb00000000,
+    0xc768b17300000000, 0x29c7046100000000, 0x4ca0b8d900000000,
+    0xf5986f4400000000, 0x90ffd3fc00000000, 0x7e5066ee00000000,
+    0x1b37da5600000000, 0x4d27b90e00000000, 0x284005b600000000,
+    0xc6efb0a400000000, 0xa3880c1c00000000, 0x1ab0db8100000000,
+    0x7fd7673900000000, 0x9178d22b00000000, 0xf41f6e9300000000,
+    0x03f7263b00000000, 0x66909a8300000000, 0x883f2f9100000000,
+    0xed58932900000000, 0x546044b400000000, 0x3107f80c00000000,
+    0xdfa84d1e00000000, 0xbacff1a600000000, 0xecdf92fe00000000,
+    0x89b82e4600000000, 0x67179b5400000000, 0x027027ec00000000,
+    0xbb48f07100000000, 0xde2f4cc900000000, 0x3080f9db00000000,
+    0x55e7456300000000, 0x9ca03f6b00000000, 0xf9c783d300000000,
+    0x176836c100000000, 0x720f8a7900000000, 0xcb375de400000000,
+    0xae50e15c00000000, 0x40ff544e00000000, 0x2598e8f600000000,
+    0x73888bae00000000, 0x16ef371600000000, 0xf840820400000000,
+    0x9d273ebc00000000, 0x241fe92100000000, 0x4178559900000000,
+    0xafd7e08b00000000, 0xcab05c3300000000, 0x3bb659ed00000000,
+    0x5ed1e55500000000, 0xb07e504700000000, 0xd519ecff00000000,
+    0x6c213b6200000000, 0x094687da00000000, 0xe7e932c800000000,
+    0x828e8e7000000000, 0xd49eed2800000000, 0xb1f9519000000000,
+    0x5f56e48200000000, 0x3a31583a00000000, 0x83098fa700000000,
+    0xe66e331f00000000, 0x08c1860d00000000, 0x6da63ab500000000,
+    0xa4e140bd00000000, 0xc186fc0500000000, 0x2f29491700000000,
+    0x4a4ef5af00000000, 0xf376223200000000, 0x96119e8a00000000,
+    0x78be2b9800000000, 0x1dd9972000000000, 0x4bc9f47800000000,
+    0x2eae48c000000000, 0xc001fdd200000000, 0xa566416a00000000,
+    0x1c5e96f700000000, 0x79392a4f00000000, 0x97969f5d00000000,
+    0xf2f123e500000000, 0x05196b4d00000000, 0x607ed7f500000000,
+    0x8ed162e700000000, 0xebb6de5f00000000, 0x528e09c200000000,
+    0x37e9b57a00000000, 0xd946006800000000, 0xbc21bcd000000000,
+    0xea31df8800000000, 0x8f56633000000000, 0x61f9d62200000000,
+    0x049e6a9a00000000, 0xbda6bd0700000000, 0xd8c101bf00000000,
+    0x366eb4ad00000000, 0x5309081500000000, 0x9a4e721d00000000,
+    0xff29cea500000000, 0x11867bb700000000, 0x74e1c70f00000000,
+    0xcdd9109200000000, 0xa8beac2a00000000, 0x4611193800000000,
+    0x2376a58000000000, 0x7566c6d800000000, 0x10017a6000000000,
+    0xfeaecf7200000000, 0x9bc973ca00000000, 0x22f1a45700000000,
+    0x479618ef00000000, 0xa939adfd00000000, 0xcc5e114500000000,
+    0x06ee4d7600000000, 0x6389f1ce00000000, 0x8d2644dc00000000,
+    0xe841f86400000000, 0x51792ff900000000, 0x341e934100000000,
+    0xdab1265300000000, 0xbfd69aeb00000000, 0xe9c6f9b300000000,
+    0x8ca1450b00000000, 0x620ef01900000000, 0x07694ca100000000,
+    0xbe519b3c00000000, 0xdb36278400000000, 0x3599929600000000,
+    0x50fe2e2e00000000, 0x99b9542600000000, 0xfcdee89e00000000,
+    0x12715d8c00000000, 0x7716e13400000000, 0xce2e36a900000000,
+    0xab498a1100000000, 0x45e63f0300000000, 0x208183bb00000000,
+    0x7691e0e300000000, 0x13f65c5b00000000, 0xfd59e94900000000,
+    0x983e55f100000000, 0x2106826c00000000, 0x44613ed400000000,
+    0xaace8bc600000000, 0xcfa9377e00000000, 0x38417fd600000000,
+    0x5d26c36e00000000, 0xb389767c00000000, 0xd6eecac400000000,
+    0x6fd61d5900000000, 0x0ab1a1e100000000, 0xe41e14f300000000,
+    0x8179a84b00000000, 0xd769cb1300000000, 0xb20e77ab00000000,
+    0x5ca1c2b900000000, 0x39c67e0100000000, 0x80fea99c00000000,
+    0xe599152400000000, 0x0b36a03600000000, 0x6e511c8e00000000,
+    0xa716668600000000, 0xc271da3e00000000, 0x2cde6f2c00000000,
+    0x49b9d39400000000, 0xf081040900000000, 0x95e6b8b100000000,
+    0x7b490da300000000, 0x1e2eb11b00000000, 0x483ed24300000000,
+    0x2d596efb00000000, 0xc3f6dbe900000000, 0xa691675100000000,
+    0x1fa9b0cc00000000, 0x7ace0c7400000000, 0x9461b96600000000,
+    0xf10605de00000000},
+   {0x0000000000000000, 0xb029603d00000000, 0x6053c07a00000000,
+    0xd07aa04700000000, 0xc0a680f500000000, 0x708fe0c800000000,
+    0xa0f5408f00000000, 0x10dc20b200000000, 0xc14b703000000000,
+    0x7162100d00000000, 0xa118b04a00000000, 0x1131d07700000000,
+    0x01edf0c500000000, 0xb1c490f800000000, 0x61be30bf00000000,
+    0xd197508200000000, 0x8297e06000000000, 0x32be805d00000000,
+    0xe2c4201a00000000, 0x52ed402700000000, 0x4231609500000000,
+    0xf21800a800000000, 0x2262a0ef00000000, 0x924bc0d200000000,
+    0x43dc905000000000, 0xf3f5f06d00000000, 0x238f502a00000000,
+    0x93a6301700000000, 0x837a10a500000000, 0x3353709800000000,
+    0xe329d0df00000000, 0x5300b0e200000000, 0x042fc1c100000000,
+    0xb406a1fc00000000, 0x647c01bb00000000, 0xd455618600000000,
+    0xc489413400000000, 0x74a0210900000000, 0xa4da814e00000000,
+    0x14f3e17300000000, 0xc564b1f100000000, 0x754dd1cc00000000,
+    0xa537718b00000000, 0x151e11b600000000, 0x05c2310400000000,
+    0xb5eb513900000000, 0x6591f17e00000000, 0xd5b8914300000000,
+    0x86b821a100000000, 0x3691419c00000000, 0xe6ebe1db00000000,
+    0x56c281e600000000, 0x461ea15400000000, 0xf637c16900000000,
+    0x264d612e00000000, 0x9664011300000000, 0x47f3519100000000,
+    0xf7da31ac00000000, 0x27a091eb00000000, 0x9789f1d600000000,
+    0x8755d16400000000, 0x377cb15900000000, 0xe706111e00000000,
+    0x572f712300000000, 0x4958f35800000000, 0xf971936500000000,
+    0x290b332200000000, 0x9922531f00000000, 0x89fe73ad00000000,
+    0x39d7139000000000, 0xe9adb3d700000000, 0x5984d3ea00000000,
+    0x8813836800000000, 0x383ae35500000000, 0xe840431200000000,
+    0x5869232f00000000, 0x48b5039d00000000, 0xf89c63a000000000,
+    0x28e6c3e700000000, 0x98cfa3da00000000, 0xcbcf133800000000,
+    0x7be6730500000000, 0xab9cd34200000000, 0x1bb5b37f00000000,
+    0x0b6993cd00000000, 0xbb40f3f000000000, 0x6b3a53b700000000,
+    0xdb13338a00000000, 0x0a84630800000000, 0xbaad033500000000,
+    0x6ad7a37200000000, 0xdafec34f00000000, 0xca22e3fd00000000,
+    0x7a0b83c000000000, 0xaa71238700000000, 0x1a5843ba00000000,
+    0x4d77329900000000, 0xfd5e52a400000000, 0x2d24f2e300000000,
+    0x9d0d92de00000000, 0x8dd1b26c00000000, 0x3df8d25100000000,
+    0xed82721600000000, 0x5dab122b00000000, 0x8c3c42a900000000,
+    0x3c15229400000000, 0xec6f82d300000000, 0x5c46e2ee00000000,
+    0x4c9ac25c00000000, 0xfcb3a26100000000, 0x2cc9022600000000,
+    0x9ce0621b00000000, 0xcfe0d2f900000000, 0x7fc9b2c400000000,
+    0xafb3128300000000, 0x1f9a72be00000000, 0x0f46520c00000000,
+    0xbf6f323100000000, 0x6f15927600000000, 0xdf3cf24b00000000,
+    0x0eaba2c900000000, 0xbe82c2f400000000, 0x6ef862b300000000,
+    0xded1028e00000000, 0xce0d223c00000000, 0x7e24420100000000,
+    0xae5ee24600000000, 0x1e77827b00000000, 0x92b0e6b100000000,
+    0x2299868c00000000, 0xf2e326cb00000000, 0x42ca46f600000000,
+    0x5216664400000000, 0xe23f067900000000, 0x3245a63e00000000,
+    0x826cc60300000000, 0x53fb968100000000, 0xe3d2f6bc00000000,
+    0x33a856fb00000000, 0x838136c600000000, 0x935d167400000000,
+    0x2374764900000000, 0xf30ed60e00000000, 0x4327b63300000000,
+    0x102706d100000000, 0xa00e66ec00000000, 0x7074c6ab00000000,
+    0xc05da69600000000, 0xd081862400000000, 0x60a8e61900000000,
+    0xb0d2465e00000000, 0x00fb266300000000, 0xd16c76e100000000,
+    0x614516dc00000000, 0xb13fb69b00000000, 0x0116d6a600000000,
+    0x11caf61400000000, 0xa1e3962900000000, 0x7199366e00000000,
+    0xc1b0565300000000, 0x969f277000000000, 0x26b6474d00000000,
+    0xf6cce70a00000000, 0x46e5873700000000, 0x5639a78500000000,
+    0xe610c7b800000000, 0x366a67ff00000000, 0x864307c200000000,
+    0x57d4574000000000, 0xe7fd377d00000000, 0x3787973a00000000,
+    0x87aef70700000000, 0x9772d7b500000000, 0x275bb78800000000,
+    0xf72117cf00000000, 0x470877f200000000, 0x1408c71000000000,
+    0xa421a72d00000000, 0x745b076a00000000, 0xc472675700000000,
+    0xd4ae47e500000000, 0x648727d800000000, 0xb4fd879f00000000,
+    0x04d4e7a200000000, 0xd543b72000000000, 0x656ad71d00000000,
+    0xb510775a00000000, 0x0539176700000000, 0x15e537d500000000,
+    0xa5cc57e800000000, 0x75b6f7af00000000, 0xc59f979200000000,
+    0xdbe815e900000000, 0x6bc175d400000000, 0xbbbbd59300000000,
+    0x0b92b5ae00000000, 0x1b4e951c00000000, 0xab67f52100000000,
+    0x7b1d556600000000, 0xcb34355b00000000, 0x1aa365d900000000,
+    0xaa8a05e400000000, 0x7af0a5a300000000, 0xcad9c59e00000000,
+    0xda05e52c00000000, 0x6a2c851100000000, 0xba56255600000000,
+    0x0a7f456b00000000, 0x597ff58900000000, 0xe95695b400000000,
+    0x392c35f300000000, 0x890555ce00000000, 0x99d9757c00000000,
+    0x29f0154100000000, 0xf98ab50600000000, 0x49a3d53b00000000,
+    0x983485b900000000, 0x281de58400000000, 0xf86745c300000000,
+    0x484e25fe00000000, 0x5892054c00000000, 0xe8bb657100000000,
+    0x38c1c53600000000, 0x88e8a50b00000000, 0xdfc7d42800000000,
+    0x6feeb41500000000, 0xbf94145200000000, 0x0fbd746f00000000,
+    0x1f6154dd00000000, 0xaf4834e000000000, 0x7f3294a700000000,
+    0xcf1bf49a00000000, 0x1e8ca41800000000, 0xaea5c42500000000,
+    0x7edf646200000000, 0xcef6045f00000000, 0xde2a24ed00000000,
+    0x6e0344d000000000, 0xbe79e49700000000, 0x0e5084aa00000000,
+    0x5d50344800000000, 0xed79547500000000, 0x3d03f43200000000,
+    0x8d2a940f00000000, 0x9df6b4bd00000000, 0x2ddfd48000000000,
+    0xfda574c700000000, 0x4d8c14fa00000000, 0x9c1b447800000000,
+    0x2c32244500000000, 0xfc48840200000000, 0x4c61e43f00000000,
+    0x5cbdc48d00000000, 0xec94a4b000000000, 0x3cee04f700000000,
+    0x8cc764ca00000000},
+   {0x0000000000000000, 0xa5d35ccb00000000, 0x0ba1c84d00000000,
+    0xae72948600000000, 0x1642919b00000000, 0xb391cd5000000000,
+    0x1de359d600000000, 0xb830051d00000000, 0x6d8253ec00000000,
+    0xc8510f2700000000, 0x66239ba100000000, 0xc3f0c76a00000000,
+    0x7bc0c27700000000, 0xde139ebc00000000, 0x70610a3a00000000,
+    0xd5b256f100000000, 0x9b02d60300000000, 0x3ed18ac800000000,
+    0x90a31e4e00000000, 0x3570428500000000, 0x8d40479800000000,
+    0x28931b5300000000, 0x86e18fd500000000, 0x2332d31e00000000,
+    0xf68085ef00000000, 0x5353d92400000000, 0xfd214da200000000,
+    0x58f2116900000000, 0xe0c2147400000000, 0x451148bf00000000,
+    0xeb63dc3900000000, 0x4eb080f200000000, 0x3605ac0700000000,
+    0x93d6f0cc00000000, 0x3da4644a00000000, 0x9877388100000000,
+    0x20473d9c00000000, 0x8594615700000000, 0x2be6f5d100000000,
+    0x8e35a91a00000000, 0x5b87ffeb00000000, 0xfe54a32000000000,
+    0x502637a600000000, 0xf5f56b6d00000000, 0x4dc56e7000000000,
+    0xe81632bb00000000, 0x4664a63d00000000, 0xe3b7faf600000000,
+    0xad077a0400000000, 0x08d426cf00000000, 0xa6a6b24900000000,
+    0x0375ee8200000000, 0xbb45eb9f00000000, 0x1e96b75400000000,
+    0xb0e423d200000000, 0x15377f1900000000, 0xc08529e800000000,
+    0x6556752300000000, 0xcb24e1a500000000, 0x6ef7bd6e00000000,
+    0xd6c7b87300000000, 0x7314e4b800000000, 0xdd66703e00000000,
+    0x78b52cf500000000, 0x6c0a580f00000000, 0xc9d904c400000000,
+    0x67ab904200000000, 0xc278cc8900000000, 0x7a48c99400000000,
+    0xdf9b955f00000000, 0x71e901d900000000, 0xd43a5d1200000000,
+    0x01880be300000000, 0xa45b572800000000, 0x0a29c3ae00000000,
+    0xaffa9f6500000000, 0x17ca9a7800000000, 0xb219c6b300000000,
+    0x1c6b523500000000, 0xb9b80efe00000000, 0xf7088e0c00000000,
+    0x52dbd2c700000000, 0xfca9464100000000, 0x597a1a8a00000000,
+    0xe14a1f9700000000, 0x4499435c00000000, 0xeaebd7da00000000,
+    0x4f388b1100000000, 0x9a8adde000000000, 0x3f59812b00000000,
+    0x912b15ad00000000, 0x34f8496600000000, 0x8cc84c7b00000000,
+    0x291b10b000000000, 0x8769843600000000, 0x22bad8fd00000000,
+    0x5a0ff40800000000, 0xffdca8c300000000, 0x51ae3c4500000000,
+    0xf47d608e00000000, 0x4c4d659300000000, 0xe99e395800000000,
+    0x47ecadde00000000, 0xe23ff11500000000, 0x378da7e400000000,
+    0x925efb2f00000000, 0x3c2c6fa900000000, 0x99ff336200000000,
+    0x21cf367f00000000, 0x841c6ab400000000, 0x2a6efe3200000000,
+    0x8fbda2f900000000, 0xc10d220b00000000, 0x64de7ec000000000,
+    0xcaacea4600000000, 0x6f7fb68d00000000, 0xd74fb39000000000,
+    0x729cef5b00000000, 0xdcee7bdd00000000, 0x793d271600000000,
+    0xac8f71e700000000, 0x095c2d2c00000000, 0xa72eb9aa00000000,
+    0x02fde56100000000, 0xbacde07c00000000, 0x1f1ebcb700000000,
+    0xb16c283100000000, 0x14bf74fa00000000, 0xd814b01e00000000,
+    0x7dc7ecd500000000, 0xd3b5785300000000, 0x7666249800000000,
+    0xce56218500000000, 0x6b857d4e00000000, 0xc5f7e9c800000000,
+    0x6024b50300000000, 0xb596e3f200000000, 0x1045bf3900000000,
+    0xbe372bbf00000000, 0x1be4777400000000, 0xa3d4726900000000,
+    0x06072ea200000000, 0xa875ba2400000000, 0x0da6e6ef00000000,
+    0x4316661d00000000, 0xe6c53ad600000000, 0x48b7ae5000000000,
+    0xed64f29b00000000, 0x5554f78600000000, 0xf087ab4d00000000,
+    0x5ef53fcb00000000, 0xfb26630000000000, 0x2e9435f100000000,
+    0x8b47693a00000000, 0x2535fdbc00000000, 0x80e6a17700000000,
+    0x38d6a46a00000000, 0x9d05f8a100000000, 0x33776c2700000000,
+    0x96a430ec00000000, 0xee111c1900000000, 0x4bc240d200000000,
+    0xe5b0d45400000000, 0x4063889f00000000, 0xf8538d8200000000,
+    0x5d80d14900000000, 0xf3f245cf00000000, 0x5621190400000000,
+    0x83934ff500000000, 0x2640133e00000000, 0x883287b800000000,
+    0x2de1db7300000000, 0x95d1de6e00000000, 0x300282a500000000,
+    0x9e70162300000000, 0x3ba34ae800000000, 0x7513ca1a00000000,
+    0xd0c096d100000000, 0x7eb2025700000000, 0xdb615e9c00000000,
+    0x63515b8100000000, 0xc682074a00000000, 0x68f093cc00000000,
+    0xcd23cf0700000000, 0x189199f600000000, 0xbd42c53d00000000,
+    0x133051bb00000000, 0xb6e30d7000000000, 0x0ed3086d00000000,
+    0xab0054a600000000, 0x0572c02000000000, 0xa0a19ceb00000000,
+    0xb41ee81100000000, 0x11cdb4da00000000, 0xbfbf205c00000000,
+    0x1a6c7c9700000000, 0xa25c798a00000000, 0x078f254100000000,
+    0xa9fdb1c700000000, 0x0c2eed0c00000000, 0xd99cbbfd00000000,
+    0x7c4fe73600000000, 0xd23d73b000000000, 0x77ee2f7b00000000,
+    0xcfde2a6600000000, 0x6a0d76ad00000000, 0xc47fe22b00000000,
+    0x61acbee000000000, 0x2f1c3e1200000000, 0x8acf62d900000000,
+    0x24bdf65f00000000, 0x816eaa9400000000, 0x395eaf8900000000,
+    0x9c8df34200000000, 0x32ff67c400000000, 0x972c3b0f00000000,
+    0x429e6dfe00000000, 0xe74d313500000000, 0x493fa5b300000000,
+    0xececf97800000000, 0x54dcfc6500000000, 0xf10fa0ae00000000,
+    0x5f7d342800000000, 0xfaae68e300000000, 0x821b441600000000,
+    0x27c818dd00000000, 0x89ba8c5b00000000, 0x2c69d09000000000,
+    0x9459d58d00000000, 0x318a894600000000, 0x9ff81dc000000000,
+    0x3a2b410b00000000, 0xef9917fa00000000, 0x4a4a4b3100000000,
+    0xe438dfb700000000, 0x41eb837c00000000, 0xf9db866100000000,
+    0x5c08daaa00000000, 0xf27a4e2c00000000, 0x57a912e700000000,
+    0x1919921500000000, 0xbccacede00000000, 0x12b85a5800000000,
+    0xb76b069300000000, 0x0f5b038e00000000, 0xaa885f4500000000,
+    0x04facbc300000000, 0xa129970800000000, 0x749bc1f900000000,
+    0xd1489d3200000000, 0x7f3a09b400000000, 0xdae9557f00000000,
+    0x62d9506200000000, 0xc70a0ca900000000, 0x6978982f00000000,
+    0xccabc4e400000000},
+   {0x0000000000000000, 0xb40b77a600000000, 0x29119f9700000000,
+    0x9d1ae83100000000, 0x13244ff400000000, 0xa72f385200000000,
+    0x3a35d06300000000, 0x8e3ea7c500000000, 0x674eef3300000000,
+    0xd345989500000000, 0x4e5f70a400000000, 0xfa54070200000000,
+    0x746aa0c700000000, 0xc061d76100000000, 0x5d7b3f5000000000,
+    0xe97048f600000000, 0xce9cde6700000000, 0x7a97a9c100000000,
+    0xe78d41f000000000, 0x5386365600000000, 0xddb8919300000000,
+    0x69b3e63500000000, 0xf4a90e0400000000, 0x40a279a200000000,
+    0xa9d2315400000000, 0x1dd946f200000000, 0x80c3aec300000000,
+    0x34c8d96500000000, 0xbaf67ea000000000, 0x0efd090600000000,
+    0x93e7e13700000000, 0x27ec969100000000, 0x9c39bdcf00000000,
+    0x2832ca6900000000, 0xb528225800000000, 0x012355fe00000000,
+    0x8f1df23b00000000, 0x3b16859d00000000, 0xa60c6dac00000000,
+    0x12071a0a00000000, 0xfb7752fc00000000, 0x4f7c255a00000000,
+    0xd266cd6b00000000, 0x666dbacd00000000, 0xe8531d0800000000,
+    0x5c586aae00000000, 0xc142829f00000000, 0x7549f53900000000,
+    0x52a563a800000000, 0xe6ae140e00000000, 0x7bb4fc3f00000000,
+    0xcfbf8b9900000000, 0x41812c5c00000000, 0xf58a5bfa00000000,
+    0x6890b3cb00000000, 0xdc9bc46d00000000, 0x35eb8c9b00000000,
+    0x81e0fb3d00000000, 0x1cfa130c00000000, 0xa8f164aa00000000,
+    0x26cfc36f00000000, 0x92c4b4c900000000, 0x0fde5cf800000000,
+    0xbbd52b5e00000000, 0x79750b4400000000, 0xcd7e7ce200000000,
+    0x506494d300000000, 0xe46fe37500000000, 0x6a5144b000000000,
+    0xde5a331600000000, 0x4340db2700000000, 0xf74bac8100000000,
+    0x1e3be47700000000, 0xaa3093d100000000, 0x372a7be000000000,
+    0x83210c4600000000, 0x0d1fab8300000000, 0xb914dc2500000000,
+    0x240e341400000000, 0x900543b200000000, 0xb7e9d52300000000,
+    0x03e2a28500000000, 0x9ef84ab400000000, 0x2af33d1200000000,
+    0xa4cd9ad700000000, 0x10c6ed7100000000, 0x8ddc054000000000,
+    0x39d772e600000000, 0xd0a73a1000000000, 0x64ac4db600000000,
+    0xf9b6a58700000000, 0x4dbdd22100000000, 0xc38375e400000000,
+    0x7788024200000000, 0xea92ea7300000000, 0x5e999dd500000000,
+    0xe54cb68b00000000, 0x5147c12d00000000, 0xcc5d291c00000000,
+    0x78565eba00000000, 0xf668f97f00000000, 0x42638ed900000000,
+    0xdf7966e800000000, 0x6b72114e00000000, 0x820259b800000000,
+    0x36092e1e00000000, 0xab13c62f00000000, 0x1f18b18900000000,
+    0x9126164c00000000, 0x252d61ea00000000, 0xb83789db00000000,
+    0x0c3cfe7d00000000, 0x2bd068ec00000000, 0x9fdb1f4a00000000,
+    0x02c1f77b00000000, 0xb6ca80dd00000000, 0x38f4271800000000,
+    0x8cff50be00000000, 0x11e5b88f00000000, 0xa5eecf2900000000,
+    0x4c9e87df00000000, 0xf895f07900000000, 0x658f184800000000,
+    0xd1846fee00000000, 0x5fbac82b00000000, 0xebb1bf8d00000000,
+    0x76ab57bc00000000, 0xc2a0201a00000000, 0xf2ea168800000000,
+    0x46e1612e00000000, 0xdbfb891f00000000, 0x6ff0feb900000000,
+    0xe1ce597c00000000, 0x55c52eda00000000, 0xc8dfc6eb00000000,
+    0x7cd4b14d00000000, 0x95a4f9bb00000000, 0x21af8e1d00000000,
+    0xbcb5662c00000000, 0x08be118a00000000, 0x8680b64f00000000,
+    0x328bc1e900000000, 0xaf9129d800000000, 0x1b9a5e7e00000000,
+    0x3c76c8ef00000000, 0x887dbf4900000000, 0x1567577800000000,
+    0xa16c20de00000000, 0x2f52871b00000000, 0x9b59f0bd00000000,
+    0x0643188c00000000, 0xb2486f2a00000000, 0x5b3827dc00000000,
+    0xef33507a00000000, 0x7229b84b00000000, 0xc622cfed00000000,
+    0x481c682800000000, 0xfc171f8e00000000, 0x610df7bf00000000,
+    0xd506801900000000, 0x6ed3ab4700000000, 0xdad8dce100000000,
+    0x47c234d000000000, 0xf3c9437600000000, 0x7df7e4b300000000,
+    0xc9fc931500000000, 0x54e67b2400000000, 0xe0ed0c8200000000,
+    0x099d447400000000, 0xbd9633d200000000, 0x208cdbe300000000,
+    0x9487ac4500000000, 0x1ab90b8000000000, 0xaeb27c2600000000,
+    0x33a8941700000000, 0x87a3e3b100000000, 0xa04f752000000000,
+    0x1444028600000000, 0x895eeab700000000, 0x3d559d1100000000,
+    0xb36b3ad400000000, 0x07604d7200000000, 0x9a7aa54300000000,
+    0x2e71d2e500000000, 0xc7019a1300000000, 0x730aedb500000000,
+    0xee10058400000000, 0x5a1b722200000000, 0xd425d5e700000000,
+    0x602ea24100000000, 0xfd344a7000000000, 0x493f3dd600000000,
+    0x8b9f1dcc00000000, 0x3f946a6a00000000, 0xa28e825b00000000,
+    0x1685f5fd00000000, 0x98bb523800000000, 0x2cb0259e00000000,
+    0xb1aacdaf00000000, 0x05a1ba0900000000, 0xecd1f2ff00000000,
+    0x58da855900000000, 0xc5c06d6800000000, 0x71cb1ace00000000,
+    0xfff5bd0b00000000, 0x4bfecaad00000000, 0xd6e4229c00000000,
+    0x62ef553a00000000, 0x4503c3ab00000000, 0xf108b40d00000000,
+    0x6c125c3c00000000, 0xd8192b9a00000000, 0x56278c5f00000000,
+    0xe22cfbf900000000, 0x7f3613c800000000, 0xcb3d646e00000000,
+    0x224d2c9800000000, 0x96465b3e00000000, 0x0b5cb30f00000000,
+    0xbf57c4a900000000, 0x3169636c00000000, 0x856214ca00000000,
+    0x1878fcfb00000000, 0xac738b5d00000000, 0x17a6a00300000000,
+    0xa3add7a500000000, 0x3eb73f9400000000, 0x8abc483200000000,
+    0x0482eff700000000, 0xb089985100000000, 0x2d93706000000000,
+    0x999807c600000000, 0x70e84f3000000000, 0xc4e3389600000000,
+    0x59f9d0a700000000, 0xedf2a70100000000, 0x63cc00c400000000,
+    0xd7c7776200000000, 0x4add9f5300000000, 0xfed6e8f500000000,
+    0xd93a7e6400000000, 0x6d3109c200000000, 0xf02be1f300000000,
+    0x4420965500000000, 0xca1e319000000000, 0x7e15463600000000,
+    0xe30fae0700000000, 0x5704d9a100000000, 0xbe74915700000000,
+    0x0a7fe6f100000000, 0x97650ec000000000, 0x236e796600000000,
+    0xad50dea300000000, 0x195ba90500000000, 0x8441413400000000,
+    0x304a369200000000},
+   {0x0000000000000000, 0x9e00aacc00000000, 0x7d07254200000000,
+    0xe3078f8e00000000, 0xfa0e4a8400000000, 0x640ee04800000000,
+    0x87096fc600000000, 0x1909c50a00000000, 0xb51be5d300000000,
+    0x2b1b4f1f00000000, 0xc81cc09100000000, 0x561c6a5d00000000,
+    0x4f15af5700000000, 0xd115059b00000000, 0x32128a1500000000,
+    0xac1220d900000000, 0x2b31bb7c00000000, 0xb53111b000000000,
+    0x56369e3e00000000, 0xc83634f200000000, 0xd13ff1f800000000,
+    0x4f3f5b3400000000, 0xac38d4ba00000000, 0x32387e7600000000,
+    0x9e2a5eaf00000000, 0x002af46300000000, 0xe32d7bed00000000,
+    0x7d2dd12100000000, 0x6424142b00000000, 0xfa24bee700000000,
+    0x1923316900000000, 0x87239ba500000000, 0x566276f900000000,
+    0xc862dc3500000000, 0x2b6553bb00000000, 0xb565f97700000000,
+    0xac6c3c7d00000000, 0x326c96b100000000, 0xd16b193f00000000,
+    0x4f6bb3f300000000, 0xe379932a00000000, 0x7d7939e600000000,
+    0x9e7eb66800000000, 0x007e1ca400000000, 0x1977d9ae00000000,
+    0x8777736200000000, 0x6470fcec00000000, 0xfa70562000000000,
+    0x7d53cd8500000000, 0xe353674900000000, 0x0054e8c700000000,
+    0x9e54420b00000000, 0x875d870100000000, 0x195d2dcd00000000,
+    0xfa5aa24300000000, 0x645a088f00000000, 0xc848285600000000,
+    0x5648829a00000000, 0xb54f0d1400000000, 0x2b4fa7d800000000,
+    0x324662d200000000, 0xac46c81e00000000, 0x4f41479000000000,
+    0xd141ed5c00000000, 0xedc29d2900000000, 0x73c237e500000000,
+    0x90c5b86b00000000, 0x0ec512a700000000, 0x17ccd7ad00000000,
+    0x89cc7d6100000000, 0x6acbf2ef00000000, 0xf4cb582300000000,
+    0x58d978fa00000000, 0xc6d9d23600000000, 0x25de5db800000000,
+    0xbbdef77400000000, 0xa2d7327e00000000, 0x3cd798b200000000,
+    0xdfd0173c00000000, 0x41d0bdf000000000, 0xc6f3265500000000,
+    0x58f38c9900000000, 0xbbf4031700000000, 0x25f4a9db00000000,
+    0x3cfd6cd100000000, 0xa2fdc61d00000000, 0x41fa499300000000,
+    0xdffae35f00000000, 0x73e8c38600000000, 0xede8694a00000000,
+    0x0eefe6c400000000, 0x90ef4c0800000000, 0x89e6890200000000,
+    0x17e623ce00000000, 0xf4e1ac4000000000, 0x6ae1068c00000000,
+    0xbba0ebd000000000, 0x25a0411c00000000, 0xc6a7ce9200000000,
+    0x58a7645e00000000, 0x41aea15400000000, 0xdfae0b9800000000,
+    0x3ca9841600000000, 0xa2a92eda00000000, 0x0ebb0e0300000000,
+    0x90bba4cf00000000, 0x73bc2b4100000000, 0xedbc818d00000000,
+    0xf4b5448700000000, 0x6ab5ee4b00000000, 0x89b261c500000000,
+    0x17b2cb0900000000, 0x909150ac00000000, 0x0e91fa6000000000,
+    0xed9675ee00000000, 0x7396df2200000000, 0x6a9f1a2800000000,
+    0xf49fb0e400000000, 0x17983f6a00000000, 0x899895a600000000,
+    0x258ab57f00000000, 0xbb8a1fb300000000, 0x588d903d00000000,
+    0xc68d3af100000000, 0xdf84fffb00000000, 0x4184553700000000,
+    0xa283dab900000000, 0x3c83707500000000, 0xda853b5300000000,
+    0x4485919f00000000, 0xa7821e1100000000, 0x3982b4dd00000000,
+    0x208b71d700000000, 0xbe8bdb1b00000000, 0x5d8c549500000000,
+    0xc38cfe5900000000, 0x6f9ede8000000000, 0xf19e744c00000000,
+    0x1299fbc200000000, 0x8c99510e00000000, 0x9590940400000000,
+    0x0b903ec800000000, 0xe897b14600000000, 0x76971b8a00000000,
+    0xf1b4802f00000000, 0x6fb42ae300000000, 0x8cb3a56d00000000,
+    0x12b30fa100000000, 0x0bbacaab00000000, 0x95ba606700000000,
+    0x76bdefe900000000, 0xe8bd452500000000, 0x44af65fc00000000,
+    0xdaafcf3000000000, 0x39a840be00000000, 0xa7a8ea7200000000,
+    0xbea12f7800000000, 0x20a185b400000000, 0xc3a60a3a00000000,
+    0x5da6a0f600000000, 0x8ce74daa00000000, 0x12e7e76600000000,
+    0xf1e068e800000000, 0x6fe0c22400000000, 0x76e9072e00000000,
+    0xe8e9ade200000000, 0x0bee226c00000000, 0x95ee88a000000000,
+    0x39fca87900000000, 0xa7fc02b500000000, 0x44fb8d3b00000000,
+    0xdafb27f700000000, 0xc3f2e2fd00000000, 0x5df2483100000000,
+    0xbef5c7bf00000000, 0x20f56d7300000000, 0xa7d6f6d600000000,
+    0x39d65c1a00000000, 0xdad1d39400000000, 0x44d1795800000000,
+    0x5dd8bc5200000000, 0xc3d8169e00000000, 0x20df991000000000,
+    0xbedf33dc00000000, 0x12cd130500000000, 0x8ccdb9c900000000,
+    0x6fca364700000000, 0xf1ca9c8b00000000, 0xe8c3598100000000,
+    0x76c3f34d00000000, 0x95c47cc300000000, 0x0bc4d60f00000000,
+    0x3747a67a00000000, 0xa9470cb600000000, 0x4a40833800000000,
+    0xd44029f400000000, 0xcd49ecfe00000000, 0x5349463200000000,
+    0xb04ec9bc00000000, 0x2e4e637000000000, 0x825c43a900000000,
+    0x1c5ce96500000000, 0xff5b66eb00000000, 0x615bcc2700000000,
+    0x7852092d00000000, 0xe652a3e100000000, 0x05552c6f00000000,
+    0x9b5586a300000000, 0x1c761d0600000000, 0x8276b7ca00000000,
+    0x6171384400000000, 0xff71928800000000, 0xe678578200000000,
+    0x7878fd4e00000000, 0x9b7f72c000000000, 0x057fd80c00000000,
+    0xa96df8d500000000, 0x376d521900000000, 0xd46add9700000000,
+    0x4a6a775b00000000, 0x5363b25100000000, 0xcd63189d00000000,
+    0x2e64971300000000, 0xb0643ddf00000000, 0x6125d08300000000,
+    0xff257a4f00000000, 0x1c22f5c100000000, 0x82225f0d00000000,
+    0x9b2b9a0700000000, 0x052b30cb00000000, 0xe62cbf4500000000,
+    0x782c158900000000, 0xd43e355000000000, 0x4a3e9f9c00000000,
+    0xa939101200000000, 0x3739bade00000000, 0x2e307fd400000000,
+    0xb030d51800000000, 0x53375a9600000000, 0xcd37f05a00000000,
+    0x4a146bff00000000, 0xd414c13300000000, 0x37134ebd00000000,
+    0xa913e47100000000, 0xb01a217b00000000, 0x2e1a8bb700000000,
+    0xcd1d043900000000, 0x531daef500000000, 0xff0f8e2c00000000,
+    0x610f24e000000000, 0x8208ab6e00000000, 0x1c0801a200000000,
+    0x0501c4a800000000, 0x9b016e6400000000, 0x7806e1ea00000000,
+    0xe6064b2600000000}};
+
+#else /* W == 4 */
+
+local const z_crc_t FAR crc_braid_table[][256] = {
+   {0x00000000, 0xb8bc6765, 0xaa09c88b, 0x12b5afee, 0x8f629757,
+    0x37def032, 0x256b5fdc, 0x9dd738b9, 0xc5b428ef, 0x7d084f8a,
+    0x6fbde064, 0xd7018701, 0x4ad6bfb8, 0xf26ad8dd, 0xe0df7733,
+    0x58631056, 0x5019579f, 0xe8a530fa, 0xfa109f14, 0x42acf871,
+    0xdf7bc0c8, 0x67c7a7ad, 0x75720843, 0xcdce6f26, 0x95ad7f70,
+    0x2d111815, 0x3fa4b7fb, 0x8718d09e, 0x1acfe827, 0xa2738f42,
+    0xb0c620ac, 0x087a47c9, 0xa032af3e, 0x188ec85b, 0x0a3b67b5,
+    0xb28700d0, 0x2f503869, 0x97ec5f0c, 0x8559f0e2, 0x3de59787,
+    0x658687d1, 0xdd3ae0b4, 0xcf8f4f5a, 0x7733283f, 0xeae41086,
+    0x525877e3, 0x40edd80d, 0xf851bf68, 0xf02bf8a1, 0x48979fc4,
+    0x5a22302a, 0xe29e574f, 0x7f496ff6, 0xc7f50893, 0xd540a77d,
+    0x6dfcc018, 0x359fd04e, 0x8d23b72b, 0x9f9618c5, 0x272a7fa0,
+    0xbafd4719, 0x0241207c, 0x10f48f92, 0xa848e8f7, 0x9b14583d,
+    0x23a83f58, 0x311d90b6, 0x89a1f7d3, 0x1476cf6a, 0xaccaa80f,
+    0xbe7f07e1, 0x06c36084, 0x5ea070d2, 0xe61c17b7, 0xf4a9b859,
+    0x4c15df3c, 0xd1c2e785, 0x697e80e0, 0x7bcb2f0e, 0xc377486b,
+    0xcb0d0fa2, 0x73b168c7, 0x6104c729, 0xd9b8a04c, 0x446f98f5,
+    0xfcd3ff90, 0xee66507e, 0x56da371b, 0x0eb9274d, 0xb6054028,
+    0xa4b0efc6, 0x1c0c88a3, 0x81dbb01a, 0x3967d77f, 0x2bd27891,
+    0x936e1ff4, 0x3b26f703, 0x839a9066, 0x912f3f88, 0x299358ed,
+    0xb4446054, 0x0cf80731, 0x1e4da8df, 0xa6f1cfba, 0xfe92dfec,
+    0x462eb889, 0x549b1767, 0xec277002, 0x71f048bb, 0xc94c2fde,
+    0xdbf98030, 0x6345e755, 0x6b3fa09c, 0xd383c7f9, 0xc1366817,
+    0x798a0f72, 0xe45d37cb, 0x5ce150ae, 0x4e54ff40, 0xf6e89825,
+    0xae8b8873, 0x1637ef16, 0x048240f8, 0xbc3e279d, 0x21e91f24,
+    0x99557841, 0x8be0d7af, 0x335cb0ca, 0xed59b63b, 0x55e5d15e,
+    0x47507eb0, 0xffec19d5, 0x623b216c, 0xda874609, 0xc832e9e7,
+    0x708e8e82, 0x28ed9ed4, 0x9051f9b1, 0x82e4565f, 0x3a58313a,
+    0xa78f0983, 0x1f336ee6, 0x0d86c108, 0xb53aa66d, 0xbd40e1a4,
+    0x05fc86c1, 0x1749292f, 0xaff54e4a, 0x322276f3, 0x8a9e1196,
+    0x982bbe78, 0x2097d91d, 0x78f4c94b, 0xc048ae2e, 0xd2fd01c0,
+    0x6a4166a5, 0xf7965e1c, 0x4f2a3979, 0x5d9f9697, 0xe523f1f2,
+    0x4d6b1905, 0xf5d77e60, 0xe762d18e, 0x5fdeb6eb, 0xc2098e52,
+    0x7ab5e937, 0x680046d9, 0xd0bc21bc, 0x88df31ea, 0x3063568f,
+    0x22d6f961, 0x9a6a9e04, 0x07bda6bd, 0xbf01c1d8, 0xadb46e36,
+    0x15080953, 0x1d724e9a, 0xa5ce29ff, 0xb77b8611, 0x0fc7e174,
+    0x9210d9cd, 0x2aacbea8, 0x38191146, 0x80a57623, 0xd8c66675,
+    0x607a0110, 0x72cfaefe, 0xca73c99b, 0x57a4f122, 0xef189647,
+    0xfdad39a9, 0x45115ecc, 0x764dee06, 0xcef18963, 0xdc44268d,
+    0x64f841e8, 0xf92f7951, 0x41931e34, 0x5326b1da, 0xeb9ad6bf,
+    0xb3f9c6e9, 0x0b45a18c, 0x19f00e62, 0xa14c6907, 0x3c9b51be,
+    0x842736db, 0x96929935, 0x2e2efe50, 0x2654b999, 0x9ee8defc,
+    0x8c5d7112, 0x34e11677, 0xa9362ece, 0x118a49ab, 0x033fe645,
+    0xbb838120, 0xe3e09176, 0x5b5cf613, 0x49e959fd, 0xf1553e98,
+    0x6c820621, 0xd43e6144, 0xc68bceaa, 0x7e37a9cf, 0xd67f4138,
+    0x6ec3265d, 0x7c7689b3, 0xc4caeed6, 0x591dd66f, 0xe1a1b10a,
+    0xf3141ee4, 0x4ba87981, 0x13cb69d7, 0xab770eb2, 0xb9c2a15c,
+    0x017ec639, 0x9ca9fe80, 0x241599e5, 0x36a0360b, 0x8e1c516e,
+    0x866616a7, 0x3eda71c2, 0x2c6fde2c, 0x94d3b949, 0x090481f0,
+    0xb1b8e695, 0xa30d497b, 0x1bb12e1e, 0x43d23e48, 0xfb6e592d,
+    0xe9dbf6c3, 0x516791a6, 0xccb0a91f, 0x740cce7a, 0x66b96194,
+    0xde0506f1},
+   {0x00000000, 0x01c26a37, 0x0384d46e, 0x0246be59, 0x0709a8dc,
+    0x06cbc2eb, 0x048d7cb2, 0x054f1685, 0x0e1351b8, 0x0fd13b8f,
+    0x0d9785d6, 0x0c55efe1, 0x091af964, 0x08d89353, 0x0a9e2d0a,
+    0x0b5c473d, 0x1c26a370, 0x1de4c947, 0x1fa2771e, 0x1e601d29,
+    0x1b2f0bac, 0x1aed619b, 0x18abdfc2, 0x1969b5f5, 0x1235f2c8,
+    0x13f798ff, 0x11b126a6, 0x10734c91, 0x153c5a14, 0x14fe3023,
+    0x16b88e7a, 0x177ae44d, 0x384d46e0, 0x398f2cd7, 0x3bc9928e,
+    0x3a0bf8b9, 0x3f44ee3c, 0x3e86840b, 0x3cc03a52, 0x3d025065,
+    0x365e1758, 0x379c7d6f, 0x35dac336, 0x3418a901, 0x3157bf84,
+    0x3095d5b3, 0x32d36bea, 0x331101dd, 0x246be590, 0x25a98fa7,
+    0x27ef31fe, 0x262d5bc9, 0x23624d4c, 0x22a0277b, 0x20e69922,
+    0x2124f315, 0x2a78b428, 0x2bbade1f, 0x29fc6046, 0x283e0a71,
+    0x2d711cf4, 0x2cb376c3, 0x2ef5c89a, 0x2f37a2ad, 0x709a8dc0,
+    0x7158e7f7, 0x731e59ae, 0x72dc3399, 0x7793251c, 0x76514f2b,
+    0x7417f172, 0x75d59b45, 0x7e89dc78, 0x7f4bb64f, 0x7d0d0816,
+    0x7ccf6221, 0x798074a4, 0x78421e93, 0x7a04a0ca, 0x7bc6cafd,
+    0x6cbc2eb0, 0x6d7e4487, 0x6f38fade, 0x6efa90e9, 0x6bb5866c,
+    0x6a77ec5b, 0x68315202, 0x69f33835, 0x62af7f08, 0x636d153f,
+    0x612bab66, 0x60e9c151, 0x65a6d7d4, 0x6464bde3, 0x662203ba,
+    0x67e0698d, 0x48d7cb20, 0x4915a117, 0x4b531f4e, 0x4a917579,
+    0x4fde63fc, 0x4e1c09cb, 0x4c5ab792, 0x4d98dda5, 0x46c49a98,
+    0x4706f0af, 0x45404ef6, 0x448224c1, 0x41cd3244, 0x400f5873,
+    0x4249e62a, 0x438b8c1d, 0x54f16850, 0x55330267, 0x5775bc3e,
+    0x56b7d609, 0x53f8c08c, 0x523aaabb, 0x507c14e2, 0x51be7ed5,
+    0x5ae239e8, 0x5b2053df, 0x5966ed86, 0x58a487b1, 0x5deb9134,
+    0x5c29fb03, 0x5e6f455a, 0x5fad2f6d, 0xe1351b80, 0xe0f771b7,
+    0xe2b1cfee, 0xe373a5d9, 0xe63cb35c, 0xe7fed96b, 0xe5b86732,
+    0xe47a0d05, 0xef264a38, 0xeee4200f, 0xeca29e56, 0xed60f461,
+    0xe82fe2e4, 0xe9ed88d3, 0xebab368a, 0xea695cbd, 0xfd13b8f0,
+    0xfcd1d2c7, 0xfe976c9e, 0xff5506a9, 0xfa1a102c, 0xfbd87a1b,
+    0xf99ec442, 0xf85cae75, 0xf300e948, 0xf2c2837f, 0xf0843d26,
+    0xf1465711, 0xf4094194, 0xf5cb2ba3, 0xf78d95fa, 0xf64fffcd,
+    0xd9785d60, 0xd8ba3757, 0xdafc890e, 0xdb3ee339, 0xde71f5bc,
+    0xdfb39f8b, 0xddf521d2, 0xdc374be5, 0xd76b0cd8, 0xd6a966ef,
+    0xd4efd8b6, 0xd52db281, 0xd062a404, 0xd1a0ce33, 0xd3e6706a,
+    0xd2241a5d, 0xc55efe10, 0xc49c9427, 0xc6da2a7e, 0xc7184049,
+    0xc25756cc, 0xc3953cfb, 0xc1d382a2, 0xc011e895, 0xcb4dafa8,
+    0xca8fc59f, 0xc8c97bc6, 0xc90b11f1, 0xcc440774, 0xcd866d43,
+    0xcfc0d31a, 0xce02b92d, 0x91af9640, 0x906dfc77, 0x922b422e,
+    0x93e92819, 0x96a63e9c, 0x976454ab, 0x9522eaf2, 0x94e080c5,
+    0x9fbcc7f8, 0x9e7eadcf, 0x9c381396, 0x9dfa79a1, 0x98b56f24,
+    0x99770513, 0x9b31bb4a, 0x9af3d17d, 0x8d893530, 0x8c4b5f07,
+    0x8e0de15e, 0x8fcf8b69, 0x8a809dec, 0x8b42f7db, 0x89044982,
+    0x88c623b5, 0x839a6488, 0x82580ebf, 0x801eb0e6, 0x81dcdad1,
+    0x8493cc54, 0x8551a663, 0x8717183a, 0x86d5720d, 0xa9e2d0a0,
+    0xa820ba97, 0xaa6604ce, 0xaba46ef9, 0xaeeb787c, 0xaf29124b,
+    0xad6fac12, 0xacadc625, 0xa7f18118, 0xa633eb2f, 0xa4755576,
+    0xa5b73f41, 0xa0f829c4, 0xa13a43f3, 0xa37cfdaa, 0xa2be979d,
+    0xb5c473d0, 0xb40619e7, 0xb640a7be, 0xb782cd89, 0xb2cddb0c,
+    0xb30fb13b, 0xb1490f62, 0xb08b6555, 0xbbd72268, 0xba15485f,
+    0xb853f606, 0xb9919c31, 0xbcde8ab4, 0xbd1ce083, 0xbf5a5eda,
+    0xbe9834ed},
+   {0x00000000, 0x191b3141, 0x32366282, 0x2b2d53c3, 0x646cc504,
+    0x7d77f445, 0x565aa786, 0x4f4196c7, 0xc8d98a08, 0xd1c2bb49,
+    0xfaefe88a, 0xe3f4d9cb, 0xacb54f0c, 0xb5ae7e4d, 0x9e832d8e,
+    0x87981ccf, 0x4ac21251, 0x53d92310, 0x78f470d3, 0x61ef4192,
+    0x2eaed755, 0x37b5e614, 0x1c98b5d7, 0x05838496, 0x821b9859,
+    0x9b00a918, 0xb02dfadb, 0xa936cb9a, 0xe6775d5d, 0xff6c6c1c,
+    0xd4413fdf, 0xcd5a0e9e, 0x958424a2, 0x8c9f15e3, 0xa7b24620,
+    0xbea97761, 0xf1e8e1a6, 0xe8f3d0e7, 0xc3de8324, 0xdac5b265,
+    0x5d5daeaa, 0x44469feb, 0x6f6bcc28, 0x7670fd69, 0x39316bae,
+    0x202a5aef, 0x0b07092c, 0x121c386d, 0xdf4636f3, 0xc65d07b2,
+    0xed705471, 0xf46b6530, 0xbb2af3f7, 0xa231c2b6, 0x891c9175,
+    0x9007a034, 0x179fbcfb, 0x0e848dba, 0x25a9de79, 0x3cb2ef38,
+    0x73f379ff, 0x6ae848be, 0x41c51b7d, 0x58de2a3c, 0xf0794f05,
+    0xe9627e44, 0xc24f2d87, 0xdb541cc6, 0x94158a01, 0x8d0ebb40,
+    0xa623e883, 0xbf38d9c2, 0x38a0c50d, 0x21bbf44c, 0x0a96a78f,
+    0x138d96ce, 0x5ccc0009, 0x45d73148, 0x6efa628b, 0x77e153ca,
+    0xbabb5d54, 0xa3a06c15, 0x888d3fd6, 0x91960e97, 0xded79850,
+    0xc7cca911, 0xece1fad2, 0xf5facb93, 0x7262d75c, 0x6b79e61d,
+    0x4054b5de, 0x594f849f, 0x160e1258, 0x0f152319, 0x243870da,
+    0x3d23419b, 0x65fd6ba7, 0x7ce65ae6, 0x57cb0925, 0x4ed03864,
+    0x0191aea3, 0x188a9fe2, 0x33a7cc21, 0x2abcfd60, 0xad24e1af,
+    0xb43fd0ee, 0x9f12832d, 0x8609b26c, 0xc94824ab, 0xd05315ea,
+    0xfb7e4629, 0xe2657768, 0x2f3f79f6, 0x362448b7, 0x1d091b74,
+    0x04122a35, 0x4b53bcf2, 0x52488db3, 0x7965de70, 0x607eef31,
+    0xe7e6f3fe, 0xfefdc2bf, 0xd5d0917c, 0xcccba03d, 0x838a36fa,
+    0x9a9107bb, 0xb1bc5478, 0xa8a76539, 0x3b83984b, 0x2298a90a,
+    0x09b5fac9, 0x10aecb88, 0x5fef5d4f, 0x46f46c0e, 0x6dd93fcd,
+    0x74c20e8c, 0xf35a1243, 0xea412302, 0xc16c70c1, 0xd8774180,
+    0x9736d747, 0x8e2de606, 0xa500b5c5, 0xbc1b8484, 0x71418a1a,
+    0x685abb5b, 0x4377e898, 0x5a6cd9d9, 0x152d4f1e, 0x0c367e5f,
+    0x271b2d9c, 0x3e001cdd, 0xb9980012, 0xa0833153, 0x8bae6290,
+    0x92b553d1, 0xddf4c516, 0xc4eff457, 0xefc2a794, 0xf6d996d5,
+    0xae07bce9, 0xb71c8da8, 0x9c31de6b, 0x852aef2a, 0xca6b79ed,
+    0xd37048ac, 0xf85d1b6f, 0xe1462a2e, 0x66de36e1, 0x7fc507a0,
+    0x54e85463, 0x4df36522, 0x02b2f3e5, 0x1ba9c2a4, 0x30849167,
+    0x299fa026, 0xe4c5aeb8, 0xfdde9ff9, 0xd6f3cc3a, 0xcfe8fd7b,
+    0x80a96bbc, 0x99b25afd, 0xb29f093e, 0xab84387f, 0x2c1c24b0,
+    0x350715f1, 0x1e2a4632, 0x07317773, 0x4870e1b4, 0x516bd0f5,
+    0x7a468336, 0x635db277, 0xcbfad74e, 0xd2e1e60f, 0xf9ccb5cc,
+    0xe0d7848d, 0xaf96124a, 0xb68d230b, 0x9da070c8, 0x84bb4189,
+    0x03235d46, 0x1a386c07, 0x31153fc4, 0x280e0e85, 0x674f9842,
+    0x7e54a903, 0x5579fac0, 0x4c62cb81, 0x8138c51f, 0x9823f45e,
+    0xb30ea79d, 0xaa1596dc, 0xe554001b, 0xfc4f315a, 0xd7626299,
+    0xce7953d8, 0x49e14f17, 0x50fa7e56, 0x7bd72d95, 0x62cc1cd4,
+    0x2d8d8a13, 0x3496bb52, 0x1fbbe891, 0x06a0d9d0, 0x5e7ef3ec,
+    0x4765c2ad, 0x6c48916e, 0x7553a02f, 0x3a1236e8, 0x230907a9,
+    0x0824546a, 0x113f652b, 0x96a779e4, 0x8fbc48a5, 0xa4911b66,
+    0xbd8a2a27, 0xf2cbbce0, 0xebd08da1, 0xc0fdde62, 0xd9e6ef23,
+    0x14bce1bd, 0x0da7d0fc, 0x268a833f, 0x3f91b27e, 0x70d024b9,
+    0x69cb15f8, 0x42e6463b, 0x5bfd777a, 0xdc656bb5, 0xc57e5af4,
+    0xee530937, 0xf7483876, 0xb809aeb1, 0xa1129ff0, 0x8a3fcc33,
+    0x9324fd72},
+   {0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419,
+    0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4,
+    0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07,
+    0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
+    0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856,
+    0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
+    0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4,
+    0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
+    0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3,
+    0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a,
+    0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599,
+    0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
+    0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190,
+    0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f,
+    0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e,
+    0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
+    0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed,
+    0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
+    0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3,
+    0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
+    0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a,
+    0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5,
+    0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010,
+    0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+    0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17,
+    0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6,
+    0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615,
+    0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
+    0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344,
+    0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
+    0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a,
+    0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
+    0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1,
+    0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c,
+    0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef,
+    0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
+    0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe,
+    0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31,
+    0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c,
+    0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
+    0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b,
+    0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
+    0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1,
+    0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
+    0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278,
+    0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7,
+    0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66,
+    0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+    0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605,
+    0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8,
+    0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b,
+    0x2d02ef8d}};
+
+local const z_word_t FAR crc_braid_big_table[][256] = {
+   {0x00000000, 0x96300777, 0x2c610eee, 0xba510999, 0x19c46d07,
+    0x8ff46a70, 0x35a563e9, 0xa395649e, 0x3288db0e, 0xa4b8dc79,
+    0x1ee9d5e0, 0x88d9d297, 0x2b4cb609, 0xbd7cb17e, 0x072db8e7,
+    0x911dbf90, 0x6410b71d, 0xf220b06a, 0x4871b9f3, 0xde41be84,
+    0x7dd4da1a, 0xebe4dd6d, 0x51b5d4f4, 0xc785d383, 0x56986c13,
+    0xc0a86b64, 0x7af962fd, 0xecc9658a, 0x4f5c0114, 0xd96c0663,
+    0x633d0ffa, 0xf50d088d, 0xc8206e3b, 0x5e10694c, 0xe44160d5,
+    0x727167a2, 0xd1e4033c, 0x47d4044b, 0xfd850dd2, 0x6bb50aa5,
+    0xfaa8b535, 0x6c98b242, 0xd6c9bbdb, 0x40f9bcac, 0xe36cd832,
+    0x755cdf45, 0xcf0dd6dc, 0x593dd1ab, 0xac30d926, 0x3a00de51,
+    0x8051d7c8, 0x1661d0bf, 0xb5f4b421, 0x23c4b356, 0x9995bacf,
+    0x0fa5bdb8, 0x9eb80228, 0x0888055f, 0xb2d90cc6, 0x24e90bb1,
+    0x877c6f2f, 0x114c6858, 0xab1d61c1, 0x3d2d66b6, 0x9041dc76,
+    0x0671db01, 0xbc20d298, 0x2a10d5ef, 0x8985b171, 0x1fb5b606,
+    0xa5e4bf9f, 0x33d4b8e8, 0xa2c90778, 0x34f9000f, 0x8ea80996,
+    0x18980ee1, 0xbb0d6a7f, 0x2d3d6d08, 0x976c6491, 0x015c63e6,
+    0xf4516b6b, 0x62616c1c, 0xd8306585, 0x4e0062f2, 0xed95066c,
+    0x7ba5011b, 0xc1f40882, 0x57c40ff5, 0xc6d9b065, 0x50e9b712,
+    0xeab8be8b, 0x7c88b9fc, 0xdf1ddd62, 0x492dda15, 0xf37cd38c,
+    0x654cd4fb, 0x5861b24d, 0xce51b53a, 0x7400bca3, 0xe230bbd4,
+    0x41a5df4a, 0xd795d83d, 0x6dc4d1a4, 0xfbf4d6d3, 0x6ae96943,
+    0xfcd96e34, 0x468867ad, 0xd0b860da, 0x732d0444, 0xe51d0333,
+    0x5f4c0aaa, 0xc97c0ddd, 0x3c710550, 0xaa410227, 0x10100bbe,
+    0x86200cc9, 0x25b56857, 0xb3856f20, 0x09d466b9, 0x9fe461ce,
+    0x0ef9de5e, 0x98c9d929, 0x2298d0b0, 0xb4a8d7c7, 0x173db359,
+    0x810db42e, 0x3b5cbdb7, 0xad6cbac0, 0x2083b8ed, 0xb6b3bf9a,
+    0x0ce2b603, 0x9ad2b174, 0x3947d5ea, 0xaf77d29d, 0x1526db04,
+    0x8316dc73, 0x120b63e3, 0x843b6494, 0x3e6a6d0d, 0xa85a6a7a,
+    0x0bcf0ee4, 0x9dff0993, 0x27ae000a, 0xb19e077d, 0x44930ff0,
+    0xd2a30887, 0x68f2011e, 0xfec20669, 0x5d5762f7, 0xcb676580,
+    0x71366c19, 0xe7066b6e, 0x761bd4fe, 0xe02bd389, 0x5a7ada10,
+    0xcc4add67, 0x6fdfb9f9, 0xf9efbe8e, 0x43beb717, 0xd58eb060,
+    0xe8a3d6d6, 0x7e93d1a1, 0xc4c2d838, 0x52f2df4f, 0xf167bbd1,
+    0x6757bca6, 0xdd06b53f, 0x4b36b248, 0xda2b0dd8, 0x4c1b0aaf,
+    0xf64a0336, 0x607a0441, 0xc3ef60df, 0x55df67a8, 0xef8e6e31,
+    0x79be6946, 0x8cb361cb, 0x1a8366bc, 0xa0d26f25, 0x36e26852,
+    0x95770ccc, 0x03470bbb, 0xb9160222, 0x2f260555, 0xbe3bbac5,
+    0x280bbdb2, 0x925ab42b, 0x046ab35c, 0xa7ffd7c2, 0x31cfd0b5,
+    0x8b9ed92c, 0x1daede5b, 0xb0c2649b, 0x26f263ec, 0x9ca36a75,
+    0x0a936d02, 0xa906099c, 0x3f360eeb, 0x85670772, 0x13570005,
+    0x824abf95, 0x147ab8e2, 0xae2bb17b, 0x381bb60c, 0x9b8ed292,
+    0x0dbed5e5, 0xb7efdc7c, 0x21dfdb0b, 0xd4d2d386, 0x42e2d4f1,
+    0xf8b3dd68, 0x6e83da1f, 0xcd16be81, 0x5b26b9f6, 0xe177b06f,
+    0x7747b718, 0xe65a0888, 0x706a0fff, 0xca3b0666, 0x5c0b0111,
+    0xff9e658f, 0x69ae62f8, 0xd3ff6b61, 0x45cf6c16, 0x78e20aa0,
+    0xeed20dd7, 0x5483044e, 0xc2b30339, 0x612667a7, 0xf71660d0,
+    0x4d476949, 0xdb776e3e, 0x4a6ad1ae, 0xdc5ad6d9, 0x660bdf40,
+    0xf03bd837, 0x53aebca9, 0xc59ebbde, 0x7fcfb247, 0xe9ffb530,
+    0x1cf2bdbd, 0x8ac2baca, 0x3093b353, 0xa6a3b424, 0x0536d0ba,
+    0x9306d7cd, 0x2957de54, 0xbf67d923, 0x2e7a66b3, 0xb84a61c4,
+    0x021b685d, 0x942b6f2a, 0x37be0bb4, 0xa18e0cc3, 0x1bdf055a,
+    0x8def022d},
+   {0x00000000, 0x41311b19, 0x82623632, 0xc3532d2b, 0x04c56c64,
+    0x45f4777d, 0x86a75a56, 0xc796414f, 0x088ad9c8, 0x49bbc2d1,
+    0x8ae8effa, 0xcbd9f4e3, 0x0c4fb5ac, 0x4d7eaeb5, 0x8e2d839e,
+    0xcf1c9887, 0x5112c24a, 0x1023d953, 0xd370f478, 0x9241ef61,
+    0x55d7ae2e, 0x14e6b537, 0xd7b5981c, 0x96848305, 0x59981b82,
+    0x18a9009b, 0xdbfa2db0, 0x9acb36a9, 0x5d5d77e6, 0x1c6c6cff,
+    0xdf3f41d4, 0x9e0e5acd, 0xa2248495, 0xe3159f8c, 0x2046b2a7,
+    0x6177a9be, 0xa6e1e8f1, 0xe7d0f3e8, 0x2483dec3, 0x65b2c5da,
+    0xaaae5d5d, 0xeb9f4644, 0x28cc6b6f, 0x69fd7076, 0xae6b3139,
+    0xef5a2a20, 0x2c09070b, 0x6d381c12, 0xf33646df, 0xb2075dc6,
+    0x715470ed, 0x30656bf4, 0xf7f32abb, 0xb6c231a2, 0x75911c89,
+    0x34a00790, 0xfbbc9f17, 0xba8d840e, 0x79dea925, 0x38efb23c,
+    0xff79f373, 0xbe48e86a, 0x7d1bc541, 0x3c2ade58, 0x054f79f0,
+    0x447e62e9, 0x872d4fc2, 0xc61c54db, 0x018a1594, 0x40bb0e8d,
+    0x83e823a6, 0xc2d938bf, 0x0dc5a038, 0x4cf4bb21, 0x8fa7960a,
+    0xce968d13, 0x0900cc5c, 0x4831d745, 0x8b62fa6e, 0xca53e177,
+    0x545dbbba, 0x156ca0a3, 0xd63f8d88, 0x970e9691, 0x5098d7de,
+    0x11a9ccc7, 0xd2fae1ec, 0x93cbfaf5, 0x5cd76272, 0x1de6796b,
+    0xdeb55440, 0x9f844f59, 0x58120e16, 0x1923150f, 0xda703824,
+    0x9b41233d, 0xa76bfd65, 0xe65ae67c, 0x2509cb57, 0x6438d04e,
+    0xa3ae9101, 0xe29f8a18, 0x21cca733, 0x60fdbc2a, 0xafe124ad,
+    0xeed03fb4, 0x2d83129f, 0x6cb20986, 0xab2448c9, 0xea1553d0,
+    0x29467efb, 0x687765e2, 0xf6793f2f, 0xb7482436, 0x741b091d,
+    0x352a1204, 0xf2bc534b, 0xb38d4852, 0x70de6579, 0x31ef7e60,
+    0xfef3e6e7, 0xbfc2fdfe, 0x7c91d0d5, 0x3da0cbcc, 0xfa368a83,
+    0xbb07919a, 0x7854bcb1, 0x3965a7a8, 0x4b98833b, 0x0aa99822,
+    0xc9fab509, 0x88cbae10, 0x4f5def5f, 0x0e6cf446, 0xcd3fd96d,
+    0x8c0ec274, 0x43125af3, 0x022341ea, 0xc1706cc1, 0x804177d8,
+    0x47d73697, 0x06e62d8e, 0xc5b500a5, 0x84841bbc, 0x1a8a4171,
+    0x5bbb5a68, 0x98e87743, 0xd9d96c5a, 0x1e4f2d15, 0x5f7e360c,
+    0x9c2d1b27, 0xdd1c003e, 0x120098b9, 0x533183a0, 0x9062ae8b,
+    0xd153b592, 0x16c5f4dd, 0x57f4efc4, 0x94a7c2ef, 0xd596d9f6,
+    0xe9bc07ae, 0xa88d1cb7, 0x6bde319c, 0x2aef2a85, 0xed796bca,
+    0xac4870d3, 0x6f1b5df8, 0x2e2a46e1, 0xe136de66, 0xa007c57f,
+    0x6354e854, 0x2265f34d, 0xe5f3b202, 0xa4c2a91b, 0x67918430,
+    0x26a09f29, 0xb8aec5e4, 0xf99fdefd, 0x3accf3d6, 0x7bfde8cf,
+    0xbc6ba980, 0xfd5ab299, 0x3e099fb2, 0x7f3884ab, 0xb0241c2c,
+    0xf1150735, 0x32462a1e, 0x73773107, 0xb4e17048, 0xf5d06b51,
+    0x3683467a, 0x77b25d63, 0x4ed7facb, 0x0fe6e1d2, 0xccb5ccf9,
+    0x8d84d7e0, 0x4a1296af, 0x0b238db6, 0xc870a09d, 0x8941bb84,
+    0x465d2303, 0x076c381a, 0xc43f1531, 0x850e0e28, 0x42984f67,
+    0x03a9547e, 0xc0fa7955, 0x81cb624c, 0x1fc53881, 0x5ef42398,
+    0x9da70eb3, 0xdc9615aa, 0x1b0054e5, 0x5a314ffc, 0x996262d7,
+    0xd85379ce, 0x174fe149, 0x567efa50, 0x952dd77b, 0xd41ccc62,
+    0x138a8d2d, 0x52bb9634, 0x91e8bb1f, 0xd0d9a006, 0xecf37e5e,
+    0xadc26547, 0x6e91486c, 0x2fa05375, 0xe836123a, 0xa9070923,
+    0x6a542408, 0x2b653f11, 0xe479a796, 0xa548bc8f, 0x661b91a4,
+    0x272a8abd, 0xe0bccbf2, 0xa18dd0eb, 0x62defdc0, 0x23efe6d9,
+    0xbde1bc14, 0xfcd0a70d, 0x3f838a26, 0x7eb2913f, 0xb924d070,
+    0xf815cb69, 0x3b46e642, 0x7a77fd5b, 0xb56b65dc, 0xf45a7ec5,
+    0x370953ee, 0x763848f7, 0xb1ae09b8, 0xf09f12a1, 0x33cc3f8a,
+    0x72fd2493},
+   {0x00000000, 0x376ac201, 0x6ed48403, 0x59be4602, 0xdca80907,
+    0xebc2cb06, 0xb27c8d04, 0x85164f05, 0xb851130e, 0x8f3bd10f,
+    0xd685970d, 0xe1ef550c, 0x64f91a09, 0x5393d808, 0x0a2d9e0a,
+    0x3d475c0b, 0x70a3261c, 0x47c9e41d, 0x1e77a21f, 0x291d601e,
+    0xac0b2f1b, 0x9b61ed1a, 0xc2dfab18, 0xf5b56919, 0xc8f23512,
+    0xff98f713, 0xa626b111, 0x914c7310, 0x145a3c15, 0x2330fe14,
+    0x7a8eb816, 0x4de47a17, 0xe0464d38, 0xd72c8f39, 0x8e92c93b,
+    0xb9f80b3a, 0x3cee443f, 0x0b84863e, 0x523ac03c, 0x6550023d,
+    0x58175e36, 0x6f7d9c37, 0x36c3da35, 0x01a91834, 0x84bf5731,
+    0xb3d59530, 0xea6bd332, 0xdd011133, 0x90e56b24, 0xa78fa925,
+    0xfe31ef27, 0xc95b2d26, 0x4c4d6223, 0x7b27a022, 0x2299e620,
+    0x15f32421, 0x28b4782a, 0x1fdeba2b, 0x4660fc29, 0x710a3e28,
+    0xf41c712d, 0xc376b32c, 0x9ac8f52e, 0xada2372f, 0xc08d9a70,
+    0xf7e75871, 0xae591e73, 0x9933dc72, 0x1c259377, 0x2b4f5176,
+    0x72f11774, 0x459bd575, 0x78dc897e, 0x4fb64b7f, 0x16080d7d,
+    0x2162cf7c, 0xa4748079, 0x931e4278, 0xcaa0047a, 0xfdcac67b,
+    0xb02ebc6c, 0x87447e6d, 0xdefa386f, 0xe990fa6e, 0x6c86b56b,
+    0x5bec776a, 0x02523168, 0x3538f369, 0x087faf62, 0x3f156d63,
+    0x66ab2b61, 0x51c1e960, 0xd4d7a665, 0xe3bd6464, 0xba032266,
+    0x8d69e067, 0x20cbd748, 0x17a11549, 0x4e1f534b, 0x7975914a,
+    0xfc63de4f, 0xcb091c4e, 0x92b75a4c, 0xa5dd984d, 0x989ac446,
+    0xaff00647, 0xf64e4045, 0xc1248244, 0x4432cd41, 0x73580f40,
+    0x2ae64942, 0x1d8c8b43, 0x5068f154, 0x67023355, 0x3ebc7557,
+    0x09d6b756, 0x8cc0f853, 0xbbaa3a52, 0xe2147c50, 0xd57ebe51,
+    0xe839e25a, 0xdf53205b, 0x86ed6659, 0xb187a458, 0x3491eb5d,
+    0x03fb295c, 0x5a456f5e, 0x6d2fad5f, 0x801b35e1, 0xb771f7e0,
+    0xeecfb1e2, 0xd9a573e3, 0x5cb33ce6, 0x6bd9fee7, 0x3267b8e5,
+    0x050d7ae4, 0x384a26ef, 0x0f20e4ee, 0x569ea2ec, 0x61f460ed,
+    0xe4e22fe8, 0xd388ede9, 0x8a36abeb, 0xbd5c69ea, 0xf0b813fd,
+    0xc7d2d1fc, 0x9e6c97fe, 0xa90655ff, 0x2c101afa, 0x1b7ad8fb,
+    0x42c49ef9, 0x75ae5cf8, 0x48e900f3, 0x7f83c2f2, 0x263d84f0,
+    0x115746f1, 0x944109f4, 0xa32bcbf5, 0xfa958df7, 0xcdff4ff6,
+    0x605d78d9, 0x5737bad8, 0x0e89fcda, 0x39e33edb, 0xbcf571de,
+    0x8b9fb3df, 0xd221f5dd, 0xe54b37dc, 0xd80c6bd7, 0xef66a9d6,
+    0xb6d8efd4, 0x81b22dd5, 0x04a462d0, 0x33cea0d1, 0x6a70e6d3,
+    0x5d1a24d2, 0x10fe5ec5, 0x27949cc4, 0x7e2adac6, 0x494018c7,
+    0xcc5657c2, 0xfb3c95c3, 0xa282d3c1, 0x95e811c0, 0xa8af4dcb,
+    0x9fc58fca, 0xc67bc9c8, 0xf1110bc9, 0x740744cc, 0x436d86cd,
+    0x1ad3c0cf, 0x2db902ce, 0x4096af91, 0x77fc6d90, 0x2e422b92,
+    0x1928e993, 0x9c3ea696, 0xab546497, 0xf2ea2295, 0xc580e094,
+    0xf8c7bc9f, 0xcfad7e9e, 0x9613389c, 0xa179fa9d, 0x246fb598,
+    0x13057799, 0x4abb319b, 0x7dd1f39a, 0x3035898d, 0x075f4b8c,
+    0x5ee10d8e, 0x698bcf8f, 0xec9d808a, 0xdbf7428b, 0x82490489,
+    0xb523c688, 0x88649a83, 0xbf0e5882, 0xe6b01e80, 0xd1dadc81,
+    0x54cc9384, 0x63a65185, 0x3a181787, 0x0d72d586, 0xa0d0e2a9,
+    0x97ba20a8, 0xce0466aa, 0xf96ea4ab, 0x7c78ebae, 0x4b1229af,
+    0x12ac6fad, 0x25c6adac, 0x1881f1a7, 0x2feb33a6, 0x765575a4,
+    0x413fb7a5, 0xc429f8a0, 0xf3433aa1, 0xaafd7ca3, 0x9d97bea2,
+    0xd073c4b5, 0xe71906b4, 0xbea740b6, 0x89cd82b7, 0x0cdbcdb2,
+    0x3bb10fb3, 0x620f49b1, 0x55658bb0, 0x6822d7bb, 0x5f4815ba,
+    0x06f653b8, 0x319c91b9, 0xb48adebc, 0x83e01cbd, 0xda5e5abf,
+    0xed3498be},
+   {0x00000000, 0x6567bcb8, 0x8bc809aa, 0xeeafb512, 0x5797628f,
+    0x32f0de37, 0xdc5f6b25, 0xb938d79d, 0xef28b4c5, 0x8a4f087d,
+    0x64e0bd6f, 0x018701d7, 0xb8bfd64a, 0xddd86af2, 0x3377dfe0,
+    0x56106358, 0x9f571950, 0xfa30a5e8, 0x149f10fa, 0x71f8ac42,
+    0xc8c07bdf, 0xada7c767, 0x43087275, 0x266fcecd, 0x707fad95,
+    0x1518112d, 0xfbb7a43f, 0x9ed01887, 0x27e8cf1a, 0x428f73a2,
+    0xac20c6b0, 0xc9477a08, 0x3eaf32a0, 0x5bc88e18, 0xb5673b0a,
+    0xd00087b2, 0x6938502f, 0x0c5fec97, 0xe2f05985, 0x8797e53d,
+    0xd1878665, 0xb4e03add, 0x5a4f8fcf, 0x3f283377, 0x8610e4ea,
+    0xe3775852, 0x0dd8ed40, 0x68bf51f8, 0xa1f82bf0, 0xc49f9748,
+    0x2a30225a, 0x4f579ee2, 0xf66f497f, 0x9308f5c7, 0x7da740d5,
+    0x18c0fc6d, 0x4ed09f35, 0x2bb7238d, 0xc518969f, 0xa07f2a27,
+    0x1947fdba, 0x7c204102, 0x928ff410, 0xf7e848a8, 0x3d58149b,
+    0x583fa823, 0xb6901d31, 0xd3f7a189, 0x6acf7614, 0x0fa8caac,
+    0xe1077fbe, 0x8460c306, 0xd270a05e, 0xb7171ce6, 0x59b8a9f4,
+    0x3cdf154c, 0x85e7c2d1, 0xe0807e69, 0x0e2fcb7b, 0x6b4877c3,
+    0xa20f0dcb, 0xc768b173, 0x29c70461, 0x4ca0b8d9, 0xf5986f44,
+    0x90ffd3fc, 0x7e5066ee, 0x1b37da56, 0x4d27b90e, 0x284005b6,
+    0xc6efb0a4, 0xa3880c1c, 0x1ab0db81, 0x7fd76739, 0x9178d22b,
+    0xf41f6e93, 0x03f7263b, 0x66909a83, 0x883f2f91, 0xed589329,
+    0x546044b4, 0x3107f80c, 0xdfa84d1e, 0xbacff1a6, 0xecdf92fe,
+    0x89b82e46, 0x67179b54, 0x027027ec, 0xbb48f071, 0xde2f4cc9,
+    0x3080f9db, 0x55e74563, 0x9ca03f6b, 0xf9c783d3, 0x176836c1,
+    0x720f8a79, 0xcb375de4, 0xae50e15c, 0x40ff544e, 0x2598e8f6,
+    0x73888bae, 0x16ef3716, 0xf8408204, 0x9d273ebc, 0x241fe921,
+    0x41785599, 0xafd7e08b, 0xcab05c33, 0x3bb659ed, 0x5ed1e555,
+    0xb07e5047, 0xd519ecff, 0x6c213b62, 0x094687da, 0xe7e932c8,
+    0x828e8e70, 0xd49eed28, 0xb1f95190, 0x5f56e482, 0x3a31583a,
+    0x83098fa7, 0xe66e331f, 0x08c1860d, 0x6da63ab5, 0xa4e140bd,
+    0xc186fc05, 0x2f294917, 0x4a4ef5af, 0xf3762232, 0x96119e8a,
+    0x78be2b98, 0x1dd99720, 0x4bc9f478, 0x2eae48c0, 0xc001fdd2,
+    0xa566416a, 0x1c5e96f7, 0x79392a4f, 0x97969f5d, 0xf2f123e5,
+    0x05196b4d, 0x607ed7f5, 0x8ed162e7, 0xebb6de5f, 0x528e09c2,
+    0x37e9b57a, 0xd9460068, 0xbc21bcd0, 0xea31df88, 0x8f566330,
+    0x61f9d622, 0x049e6a9a, 0xbda6bd07, 0xd8c101bf, 0x366eb4ad,
+    0x53090815, 0x9a4e721d, 0xff29cea5, 0x11867bb7, 0x74e1c70f,
+    0xcdd91092, 0xa8beac2a, 0x46111938, 0x2376a580, 0x7566c6d8,
+    0x10017a60, 0xfeaecf72, 0x9bc973ca, 0x22f1a457, 0x479618ef,
+    0xa939adfd, 0xcc5e1145, 0x06ee4d76, 0x6389f1ce, 0x8d2644dc,
+    0xe841f864, 0x51792ff9, 0x341e9341, 0xdab12653, 0xbfd69aeb,
+    0xe9c6f9b3, 0x8ca1450b, 0x620ef019, 0x07694ca1, 0xbe519b3c,
+    0xdb362784, 0x35999296, 0x50fe2e2e, 0x99b95426, 0xfcdee89e,
+    0x12715d8c, 0x7716e134, 0xce2e36a9, 0xab498a11, 0x45e63f03,
+    0x208183bb, 0x7691e0e3, 0x13f65c5b, 0xfd59e949, 0x983e55f1,
+    0x2106826c, 0x44613ed4, 0xaace8bc6, 0xcfa9377e, 0x38417fd6,
+    0x5d26c36e, 0xb389767c, 0xd6eecac4, 0x6fd61d59, 0x0ab1a1e1,
+    0xe41e14f3, 0x8179a84b, 0xd769cb13, 0xb20e77ab, 0x5ca1c2b9,
+    0x39c67e01, 0x80fea99c, 0xe5991524, 0x0b36a036, 0x6e511c8e,
+    0xa7166686, 0xc271da3e, 0x2cde6f2c, 0x49b9d394, 0xf0810409,
+    0x95e6b8b1, 0x7b490da3, 0x1e2eb11b, 0x483ed243, 0x2d596efb,
+    0xc3f6dbe9, 0xa6916751, 0x1fa9b0cc, 0x7ace0c74, 0x9461b966,
+    0xf10605de}};
+
+#endif
+
+#endif
+
+#if N == 2
+
+#if W == 8
+
+local const z_crc_t FAR crc_braid_table[][256] = {
+   {0x00000000, 0xae689191, 0x87a02563, 0x29c8b4f2, 0xd4314c87,
+    0x7a59dd16, 0x539169e4, 0xfdf9f875, 0x73139f4f, 0xdd7b0ede,
+    0xf4b3ba2c, 0x5adb2bbd, 0xa722d3c8, 0x094a4259, 0x2082f6ab,
+    0x8eea673a, 0xe6273e9e, 0x484faf0f, 0x61871bfd, 0xcfef8a6c,
+    0x32167219, 0x9c7ee388, 0xb5b6577a, 0x1bdec6eb, 0x9534a1d1,
+    0x3b5c3040, 0x129484b2, 0xbcfc1523, 0x4105ed56, 0xef6d7cc7,
+    0xc6a5c835, 0x68cd59a4, 0x173f7b7d, 0xb957eaec, 0x909f5e1e,
+    0x3ef7cf8f, 0xc30e37fa, 0x6d66a66b, 0x44ae1299, 0xeac68308,
+    0x642ce432, 0xca4475a3, 0xe38cc151, 0x4de450c0, 0xb01da8b5,
+    0x1e753924, 0x37bd8dd6, 0x99d51c47, 0xf11845e3, 0x5f70d472,
+    0x76b86080, 0xd8d0f111, 0x25290964, 0x8b4198f5, 0xa2892c07,
+    0x0ce1bd96, 0x820bdaac, 0x2c634b3d, 0x05abffcf, 0xabc36e5e,
+    0x563a962b, 0xf85207ba, 0xd19ab348, 0x7ff222d9, 0x2e7ef6fa,
+    0x8016676b, 0xa9ded399, 0x07b64208, 0xfa4fba7d, 0x54272bec,
+    0x7def9f1e, 0xd3870e8f, 0x5d6d69b5, 0xf305f824, 0xdacd4cd6,
+    0x74a5dd47, 0x895c2532, 0x2734b4a3, 0x0efc0051, 0xa09491c0,
+    0xc859c864, 0x663159f5, 0x4ff9ed07, 0xe1917c96, 0x1c6884e3,
+    0xb2001572, 0x9bc8a180, 0x35a03011, 0xbb4a572b, 0x1522c6ba,
+    0x3cea7248, 0x9282e3d9, 0x6f7b1bac, 0xc1138a3d, 0xe8db3ecf,
+    0x46b3af5e, 0x39418d87, 0x97291c16, 0xbee1a8e4, 0x10893975,
+    0xed70c100, 0x43185091, 0x6ad0e463, 0xc4b875f2, 0x4a5212c8,
+    0xe43a8359, 0xcdf237ab, 0x639aa63a, 0x9e635e4f, 0x300bcfde,
+    0x19c37b2c, 0xb7abeabd, 0xdf66b319, 0x710e2288, 0x58c6967a,
+    0xf6ae07eb, 0x0b57ff9e, 0xa53f6e0f, 0x8cf7dafd, 0x229f4b6c,
+    0xac752c56, 0x021dbdc7, 0x2bd50935, 0x85bd98a4, 0x784460d1,
+    0xd62cf140, 0xffe445b2, 0x518cd423, 0x5cfdedf4, 0xf2957c65,
+    0xdb5dc897, 0x75355906, 0x88cca173, 0x26a430e2, 0x0f6c8410,
+    0xa1041581, 0x2fee72bb, 0x8186e32a, 0xa84e57d8, 0x0626c649,
+    0xfbdf3e3c, 0x55b7afad, 0x7c7f1b5f, 0xd2178ace, 0xbadad36a,
+    0x14b242fb, 0x3d7af609, 0x93126798, 0x6eeb9fed, 0xc0830e7c,
+    0xe94bba8e, 0x47232b1f, 0xc9c94c25, 0x67a1ddb4, 0x4e696946,
+    0xe001f8d7, 0x1df800a2, 0xb3909133, 0x9a5825c1, 0x3430b450,
+    0x4bc29689, 0xe5aa0718, 0xcc62b3ea, 0x620a227b, 0x9ff3da0e,
+    0x319b4b9f, 0x1853ff6d, 0xb63b6efc, 0x38d109c6, 0x96b99857,
+    0xbf712ca5, 0x1119bd34, 0xece04541, 0x4288d4d0, 0x6b406022,
+    0xc528f1b3, 0xade5a817, 0x038d3986, 0x2a458d74, 0x842d1ce5,
+    0x79d4e490, 0xd7bc7501, 0xfe74c1f3, 0x501c5062, 0xdef63758,
+    0x709ea6c9, 0x5956123b, 0xf73e83aa, 0x0ac77bdf, 0xa4afea4e,
+    0x8d675ebc, 0x230fcf2d, 0x72831b0e, 0xdceb8a9f, 0xf5233e6d,
+    0x5b4baffc, 0xa6b25789, 0x08dac618, 0x211272ea, 0x8f7ae37b,
+    0x01908441, 0xaff815d0, 0x8630a122, 0x285830b3, 0xd5a1c8c6,
+    0x7bc95957, 0x5201eda5, 0xfc697c34, 0x94a42590, 0x3accb401,
+    0x130400f3, 0xbd6c9162, 0x40956917, 0xeefdf886, 0xc7354c74,
+    0x695ddde5, 0xe7b7badf, 0x49df2b4e, 0x60179fbc, 0xce7f0e2d,
+    0x3386f658, 0x9dee67c9, 0xb426d33b, 0x1a4e42aa, 0x65bc6073,
+    0xcbd4f1e2, 0xe21c4510, 0x4c74d481, 0xb18d2cf4, 0x1fe5bd65,
+    0x362d0997, 0x98459806, 0x16afff3c, 0xb8c76ead, 0x910fda5f,
+    0x3f674bce, 0xc29eb3bb, 0x6cf6222a, 0x453e96d8, 0xeb560749,
+    0x839b5eed, 0x2df3cf7c, 0x043b7b8e, 0xaa53ea1f, 0x57aa126a,
+    0xf9c283fb, 0xd00a3709, 0x7e62a698, 0xf088c1a2, 0x5ee05033,
+    0x7728e4c1, 0xd9407550, 0x24b98d25, 0x8ad11cb4, 0xa319a846,
+    0x0d7139d7},
+   {0x00000000, 0xb9fbdbe8, 0xa886b191, 0x117d6a79, 0x8a7c6563,
+    0x3387be8b, 0x22fad4f2, 0x9b010f1a, 0xcf89cc87, 0x7672176f,
+    0x670f7d16, 0xdef4a6fe, 0x45f5a9e4, 0xfc0e720c, 0xed731875,
+    0x5488c39d, 0x44629f4f, 0xfd9944a7, 0xece42ede, 0x551ff536,
+    0xce1efa2c, 0x77e521c4, 0x66984bbd, 0xdf639055, 0x8beb53c8,
+    0x32108820, 0x236de259, 0x9a9639b1, 0x019736ab, 0xb86ced43,
+    0xa911873a, 0x10ea5cd2, 0x88c53e9e, 0x313ee576, 0x20438f0f,
+    0x99b854e7, 0x02b95bfd, 0xbb428015, 0xaa3fea6c, 0x13c43184,
+    0x474cf219, 0xfeb729f1, 0xefca4388, 0x56319860, 0xcd30977a,
+    0x74cb4c92, 0x65b626eb, 0xdc4dfd03, 0xcca7a1d1, 0x755c7a39,
+    0x64211040, 0xdddacba8, 0x46dbc4b2, 0xff201f5a, 0xee5d7523,
+    0x57a6aecb, 0x032e6d56, 0xbad5b6be, 0xaba8dcc7, 0x1253072f,
+    0x89520835, 0x30a9d3dd, 0x21d4b9a4, 0x982f624c, 0xcafb7b7d,
+    0x7300a095, 0x627dcaec, 0xdb861104, 0x40871e1e, 0xf97cc5f6,
+    0xe801af8f, 0x51fa7467, 0x0572b7fa, 0xbc896c12, 0xadf4066b,
+    0x140fdd83, 0x8f0ed299, 0x36f50971, 0x27886308, 0x9e73b8e0,
+    0x8e99e432, 0x37623fda, 0x261f55a3, 0x9fe48e4b, 0x04e58151,
+    0xbd1e5ab9, 0xac6330c0, 0x1598eb28, 0x411028b5, 0xf8ebf35d,
+    0xe9969924, 0x506d42cc, 0xcb6c4dd6, 0x7297963e, 0x63eafc47,
+    0xda1127af, 0x423e45e3, 0xfbc59e0b, 0xeab8f472, 0x53432f9a,
+    0xc8422080, 0x71b9fb68, 0x60c49111, 0xd93f4af9, 0x8db78964,
+    0x344c528c, 0x253138f5, 0x9ccae31d, 0x07cbec07, 0xbe3037ef,
+    0xaf4d5d96, 0x16b6867e, 0x065cdaac, 0xbfa70144, 0xaeda6b3d,
+    0x1721b0d5, 0x8c20bfcf, 0x35db6427, 0x24a60e5e, 0x9d5dd5b6,
+    0xc9d5162b, 0x702ecdc3, 0x6153a7ba, 0xd8a87c52, 0x43a97348,
+    0xfa52a8a0, 0xeb2fc2d9, 0x52d41931, 0x4e87f0bb, 0xf77c2b53,
+    0xe601412a, 0x5ffa9ac2, 0xc4fb95d8, 0x7d004e30, 0x6c7d2449,
+    0xd586ffa1, 0x810e3c3c, 0x38f5e7d4, 0x29888dad, 0x90735645,
+    0x0b72595f, 0xb28982b7, 0xa3f4e8ce, 0x1a0f3326, 0x0ae56ff4,
+    0xb31eb41c, 0xa263de65, 0x1b98058d, 0x80990a97, 0x3962d17f,
+    0x281fbb06, 0x91e460ee, 0xc56ca373, 0x7c97789b, 0x6dea12e2,
+    0xd411c90a, 0x4f10c610, 0xf6eb1df8, 0xe7967781, 0x5e6dac69,
+    0xc642ce25, 0x7fb915cd, 0x6ec47fb4, 0xd73fa45c, 0x4c3eab46,
+    0xf5c570ae, 0xe4b81ad7, 0x5d43c13f, 0x09cb02a2, 0xb030d94a,
+    0xa14db333, 0x18b668db, 0x83b767c1, 0x3a4cbc29, 0x2b31d650,
+    0x92ca0db8, 0x8220516a, 0x3bdb8a82, 0x2aa6e0fb, 0x935d3b13,
+    0x085c3409, 0xb1a7efe1, 0xa0da8598, 0x19215e70, 0x4da99ded,
+    0xf4524605, 0xe52f2c7c, 0x5cd4f794, 0xc7d5f88e, 0x7e2e2366,
+    0x6f53491f, 0xd6a892f7, 0x847c8bc6, 0x3d87502e, 0x2cfa3a57,
+    0x9501e1bf, 0x0e00eea5, 0xb7fb354d, 0xa6865f34, 0x1f7d84dc,
+    0x4bf54741, 0xf20e9ca9, 0xe373f6d0, 0x5a882d38, 0xc1892222,
+    0x7872f9ca, 0x690f93b3, 0xd0f4485b, 0xc01e1489, 0x79e5cf61,
+    0x6898a518, 0xd1637ef0, 0x4a6271ea, 0xf399aa02, 0xe2e4c07b,
+    0x5b1f1b93, 0x0f97d80e, 0xb66c03e6, 0xa711699f, 0x1eeab277,
+    0x85ebbd6d, 0x3c106685, 0x2d6d0cfc, 0x9496d714, 0x0cb9b558,
+    0xb5426eb0, 0xa43f04c9, 0x1dc4df21, 0x86c5d03b, 0x3f3e0bd3,
+    0x2e4361aa, 0x97b8ba42, 0xc33079df, 0x7acba237, 0x6bb6c84e,
+    0xd24d13a6, 0x494c1cbc, 0xf0b7c754, 0xe1caad2d, 0x583176c5,
+    0x48db2a17, 0xf120f1ff, 0xe05d9b86, 0x59a6406e, 0xc2a74f74,
+    0x7b5c949c, 0x6a21fee5, 0xd3da250d, 0x8752e690, 0x3ea93d78,
+    0x2fd45701, 0x962f8ce9, 0x0d2e83f3, 0xb4d5581b, 0xa5a83262,
+    0x1c53e98a},
+   {0x00000000, 0x9d0fe176, 0xe16ec4ad, 0x7c6125db, 0x19ac8f1b,
+    0x84a36e6d, 0xf8c24bb6, 0x65cdaac0, 0x33591e36, 0xae56ff40,
+    0xd237da9b, 0x4f383bed, 0x2af5912d, 0xb7fa705b, 0xcb9b5580,
+    0x5694b4f6, 0x66b23c6c, 0xfbbddd1a, 0x87dcf8c1, 0x1ad319b7,
+    0x7f1eb377, 0xe2115201, 0x9e7077da, 0x037f96ac, 0x55eb225a,
+    0xc8e4c32c, 0xb485e6f7, 0x298a0781, 0x4c47ad41, 0xd1484c37,
+    0xad2969ec, 0x3026889a, 0xcd6478d8, 0x506b99ae, 0x2c0abc75,
+    0xb1055d03, 0xd4c8f7c3, 0x49c716b5, 0x35a6336e, 0xa8a9d218,
+    0xfe3d66ee, 0x63328798, 0x1f53a243, 0x825c4335, 0xe791e9f5,
+    0x7a9e0883, 0x06ff2d58, 0x9bf0cc2e, 0xabd644b4, 0x36d9a5c2,
+    0x4ab88019, 0xd7b7616f, 0xb27acbaf, 0x2f752ad9, 0x53140f02,
+    0xce1bee74, 0x988f5a82, 0x0580bbf4, 0x79e19e2f, 0xe4ee7f59,
+    0x8123d599, 0x1c2c34ef, 0x604d1134, 0xfd42f042, 0x41b9f7f1,
+    0xdcb61687, 0xa0d7335c, 0x3dd8d22a, 0x581578ea, 0xc51a999c,
+    0xb97bbc47, 0x24745d31, 0x72e0e9c7, 0xefef08b1, 0x938e2d6a,
+    0x0e81cc1c, 0x6b4c66dc, 0xf64387aa, 0x8a22a271, 0x172d4307,
+    0x270bcb9d, 0xba042aeb, 0xc6650f30, 0x5b6aee46, 0x3ea74486,
+    0xa3a8a5f0, 0xdfc9802b, 0x42c6615d, 0x1452d5ab, 0x895d34dd,
+    0xf53c1106, 0x6833f070, 0x0dfe5ab0, 0x90f1bbc6, 0xec909e1d,
+    0x719f7f6b, 0x8cdd8f29, 0x11d26e5f, 0x6db34b84, 0xf0bcaaf2,
+    0x95710032, 0x087ee144, 0x741fc49f, 0xe91025e9, 0xbf84911f,
+    0x228b7069, 0x5eea55b2, 0xc3e5b4c4, 0xa6281e04, 0x3b27ff72,
+    0x4746daa9, 0xda493bdf, 0xea6fb345, 0x77605233, 0x0b0177e8,
+    0x960e969e, 0xf3c33c5e, 0x6eccdd28, 0x12adf8f3, 0x8fa21985,
+    0xd936ad73, 0x44394c05, 0x385869de, 0xa55788a8, 0xc09a2268,
+    0x5d95c31e, 0x21f4e6c5, 0xbcfb07b3, 0x8373efe2, 0x1e7c0e94,
+    0x621d2b4f, 0xff12ca39, 0x9adf60f9, 0x07d0818f, 0x7bb1a454,
+    0xe6be4522, 0xb02af1d4, 0x2d2510a2, 0x51443579, 0xcc4bd40f,
+    0xa9867ecf, 0x34899fb9, 0x48e8ba62, 0xd5e75b14, 0xe5c1d38e,
+    0x78ce32f8, 0x04af1723, 0x99a0f655, 0xfc6d5c95, 0x6162bde3,
+    0x1d039838, 0x800c794e, 0xd698cdb8, 0x4b972cce, 0x37f60915,
+    0xaaf9e863, 0xcf3442a3, 0x523ba3d5, 0x2e5a860e, 0xb3556778,
+    0x4e17973a, 0xd318764c, 0xaf795397, 0x3276b2e1, 0x57bb1821,
+    0xcab4f957, 0xb6d5dc8c, 0x2bda3dfa, 0x7d4e890c, 0xe041687a,
+    0x9c204da1, 0x012facd7, 0x64e20617, 0xf9ede761, 0x858cc2ba,
+    0x188323cc, 0x28a5ab56, 0xb5aa4a20, 0xc9cb6ffb, 0x54c48e8d,
+    0x3109244d, 0xac06c53b, 0xd067e0e0, 0x4d680196, 0x1bfcb560,
+    0x86f35416, 0xfa9271cd, 0x679d90bb, 0x02503a7b, 0x9f5fdb0d,
+    0xe33efed6, 0x7e311fa0, 0xc2ca1813, 0x5fc5f965, 0x23a4dcbe,
+    0xbeab3dc8, 0xdb669708, 0x4669767e, 0x3a0853a5, 0xa707b2d3,
+    0xf1930625, 0x6c9ce753, 0x10fdc288, 0x8df223fe, 0xe83f893e,
+    0x75306848, 0x09514d93, 0x945eace5, 0xa478247f, 0x3977c509,
+    0x4516e0d2, 0xd81901a4, 0xbdd4ab64, 0x20db4a12, 0x5cba6fc9,
+    0xc1b58ebf, 0x97213a49, 0x0a2edb3f, 0x764ffee4, 0xeb401f92,
+    0x8e8db552, 0x13825424, 0x6fe371ff, 0xf2ec9089, 0x0fae60cb,
+    0x92a181bd, 0xeec0a466, 0x73cf4510, 0x1602efd0, 0x8b0d0ea6,
+    0xf76c2b7d, 0x6a63ca0b, 0x3cf77efd, 0xa1f89f8b, 0xdd99ba50,
+    0x40965b26, 0x255bf1e6, 0xb8541090, 0xc435354b, 0x593ad43d,
+    0x691c5ca7, 0xf413bdd1, 0x8872980a, 0x157d797c, 0x70b0d3bc,
+    0xedbf32ca, 0x91de1711, 0x0cd1f667, 0x5a454291, 0xc74aa3e7,
+    0xbb2b863c, 0x2624674a, 0x43e9cd8a, 0xdee62cfc, 0xa2870927,
+    0x3f88e851},
+   {0x00000000, 0xdd96d985, 0x605cb54b, 0xbdca6cce, 0xc0b96a96,
+    0x1d2fb313, 0xa0e5dfdd, 0x7d730658, 0x5a03d36d, 0x87950ae8,
+    0x3a5f6626, 0xe7c9bfa3, 0x9abab9fb, 0x472c607e, 0xfae60cb0,
+    0x2770d535, 0xb407a6da, 0x69917f5f, 0xd45b1391, 0x09cdca14,
+    0x74becc4c, 0xa92815c9, 0x14e27907, 0xc974a082, 0xee0475b7,
+    0x3392ac32, 0x8e58c0fc, 0x53ce1979, 0x2ebd1f21, 0xf32bc6a4,
+    0x4ee1aa6a, 0x937773ef, 0xb37e4bf5, 0x6ee89270, 0xd322febe,
+    0x0eb4273b, 0x73c72163, 0xae51f8e6, 0x139b9428, 0xce0d4dad,
+    0xe97d9898, 0x34eb411d, 0x89212dd3, 0x54b7f456, 0x29c4f20e,
+    0xf4522b8b, 0x49984745, 0x940e9ec0, 0x0779ed2f, 0xdaef34aa,
+    0x67255864, 0xbab381e1, 0xc7c087b9, 0x1a565e3c, 0xa79c32f2,
+    0x7a0aeb77, 0x5d7a3e42, 0x80ece7c7, 0x3d268b09, 0xe0b0528c,
+    0x9dc354d4, 0x40558d51, 0xfd9fe19f, 0x2009381a, 0xbd8d91ab,
+    0x601b482e, 0xddd124e0, 0x0047fd65, 0x7d34fb3d, 0xa0a222b8,
+    0x1d684e76, 0xc0fe97f3, 0xe78e42c6, 0x3a189b43, 0x87d2f78d,
+    0x5a442e08, 0x27372850, 0xfaa1f1d5, 0x476b9d1b, 0x9afd449e,
+    0x098a3771, 0xd41ceef4, 0x69d6823a, 0xb4405bbf, 0xc9335de7,
+    0x14a58462, 0xa96fe8ac, 0x74f93129, 0x5389e41c, 0x8e1f3d99,
+    0x33d55157, 0xee4388d2, 0x93308e8a, 0x4ea6570f, 0xf36c3bc1,
+    0x2efae244, 0x0ef3da5e, 0xd36503db, 0x6eaf6f15, 0xb339b690,
+    0xce4ab0c8, 0x13dc694d, 0xae160583, 0x7380dc06, 0x54f00933,
+    0x8966d0b6, 0x34acbc78, 0xe93a65fd, 0x944963a5, 0x49dfba20,
+    0xf415d6ee, 0x29830f6b, 0xbaf47c84, 0x6762a501, 0xdaa8c9cf,
+    0x073e104a, 0x7a4d1612, 0xa7dbcf97, 0x1a11a359, 0xc7877adc,
+    0xe0f7afe9, 0x3d61766c, 0x80ab1aa2, 0x5d3dc327, 0x204ec57f,
+    0xfdd81cfa, 0x40127034, 0x9d84a9b1, 0xa06a2517, 0x7dfcfc92,
+    0xc036905c, 0x1da049d9, 0x60d34f81, 0xbd459604, 0x008ffaca,
+    0xdd19234f, 0xfa69f67a, 0x27ff2fff, 0x9a354331, 0x47a39ab4,
+    0x3ad09cec, 0xe7464569, 0x5a8c29a7, 0x871af022, 0x146d83cd,
+    0xc9fb5a48, 0x74313686, 0xa9a7ef03, 0xd4d4e95b, 0x094230de,
+    0xb4885c10, 0x691e8595, 0x4e6e50a0, 0x93f88925, 0x2e32e5eb,
+    0xf3a43c6e, 0x8ed73a36, 0x5341e3b3, 0xee8b8f7d, 0x331d56f8,
+    0x13146ee2, 0xce82b767, 0x7348dba9, 0xaede022c, 0xd3ad0474,
+    0x0e3bddf1, 0xb3f1b13f, 0x6e6768ba, 0x4917bd8f, 0x9481640a,
+    0x294b08c4, 0xf4ddd141, 0x89aed719, 0x54380e9c, 0xe9f26252,
+    0x3464bbd7, 0xa713c838, 0x7a8511bd, 0xc74f7d73, 0x1ad9a4f6,
+    0x67aaa2ae, 0xba3c7b2b, 0x07f617e5, 0xda60ce60, 0xfd101b55,
+    0x2086c2d0, 0x9d4cae1e, 0x40da779b, 0x3da971c3, 0xe03fa846,
+    0x5df5c488, 0x80631d0d, 0x1de7b4bc, 0xc0716d39, 0x7dbb01f7,
+    0xa02dd872, 0xdd5ede2a, 0x00c807af, 0xbd026b61, 0x6094b2e4,
+    0x47e467d1, 0x9a72be54, 0x27b8d29a, 0xfa2e0b1f, 0x875d0d47,
+    0x5acbd4c2, 0xe701b80c, 0x3a976189, 0xa9e01266, 0x7476cbe3,
+    0xc9bca72d, 0x142a7ea8, 0x695978f0, 0xb4cfa175, 0x0905cdbb,
+    0xd493143e, 0xf3e3c10b, 0x2e75188e, 0x93bf7440, 0x4e29adc5,
+    0x335aab9d, 0xeecc7218, 0x53061ed6, 0x8e90c753, 0xae99ff49,
+    0x730f26cc, 0xcec54a02, 0x13539387, 0x6e2095df, 0xb3b64c5a,
+    0x0e7c2094, 0xd3eaf911, 0xf49a2c24, 0x290cf5a1, 0x94c6996f,
+    0x495040ea, 0x342346b2, 0xe9b59f37, 0x547ff3f9, 0x89e92a7c,
+    0x1a9e5993, 0xc7088016, 0x7ac2ecd8, 0xa754355d, 0xda273305,
+    0x07b1ea80, 0xba7b864e, 0x67ed5fcb, 0x409d8afe, 0x9d0b537b,
+    0x20c13fb5, 0xfd57e630, 0x8024e068, 0x5db239ed, 0xe0785523,
+    0x3dee8ca6},
+   {0x00000000, 0x9ba54c6f, 0xec3b9e9f, 0x779ed2f0, 0x03063b7f,
+    0x98a37710, 0xef3da5e0, 0x7498e98f, 0x060c76fe, 0x9da93a91,
+    0xea37e861, 0x7192a40e, 0x050a4d81, 0x9eaf01ee, 0xe931d31e,
+    0x72949f71, 0x0c18edfc, 0x97bda193, 0xe0237363, 0x7b863f0c,
+    0x0f1ed683, 0x94bb9aec, 0xe325481c, 0x78800473, 0x0a149b02,
+    0x91b1d76d, 0xe62f059d, 0x7d8a49f2, 0x0912a07d, 0x92b7ec12,
+    0xe5293ee2, 0x7e8c728d, 0x1831dbf8, 0x83949797, 0xf40a4567,
+    0x6faf0908, 0x1b37e087, 0x8092ace8, 0xf70c7e18, 0x6ca93277,
+    0x1e3dad06, 0x8598e169, 0xf2063399, 0x69a37ff6, 0x1d3b9679,
+    0x869eda16, 0xf10008e6, 0x6aa54489, 0x14293604, 0x8f8c7a6b,
+    0xf812a89b, 0x63b7e4f4, 0x172f0d7b, 0x8c8a4114, 0xfb1493e4,
+    0x60b1df8b, 0x122540fa, 0x89800c95, 0xfe1ede65, 0x65bb920a,
+    0x11237b85, 0x8a8637ea, 0xfd18e51a, 0x66bda975, 0x3063b7f0,
+    0xabc6fb9f, 0xdc58296f, 0x47fd6500, 0x33658c8f, 0xa8c0c0e0,
+    0xdf5e1210, 0x44fb5e7f, 0x366fc10e, 0xadca8d61, 0xda545f91,
+    0x41f113fe, 0x3569fa71, 0xaeccb61e, 0xd95264ee, 0x42f72881,
+    0x3c7b5a0c, 0xa7de1663, 0xd040c493, 0x4be588fc, 0x3f7d6173,
+    0xa4d82d1c, 0xd346ffec, 0x48e3b383, 0x3a772cf2, 0xa1d2609d,
+    0xd64cb26d, 0x4de9fe02, 0x3971178d, 0xa2d45be2, 0xd54a8912,
+    0x4eefc57d, 0x28526c08, 0xb3f72067, 0xc469f297, 0x5fccbef8,
+    0x2b545777, 0xb0f11b18, 0xc76fc9e8, 0x5cca8587, 0x2e5e1af6,
+    0xb5fb5699, 0xc2658469, 0x59c0c806, 0x2d582189, 0xb6fd6de6,
+    0xc163bf16, 0x5ac6f379, 0x244a81f4, 0xbfefcd9b, 0xc8711f6b,
+    0x53d45304, 0x274cba8b, 0xbce9f6e4, 0xcb772414, 0x50d2687b,
+    0x2246f70a, 0xb9e3bb65, 0xce7d6995, 0x55d825fa, 0x2140cc75,
+    0xbae5801a, 0xcd7b52ea, 0x56de1e85, 0x60c76fe0, 0xfb62238f,
+    0x8cfcf17f, 0x1759bd10, 0x63c1549f, 0xf86418f0, 0x8ffaca00,
+    0x145f866f, 0x66cb191e, 0xfd6e5571, 0x8af08781, 0x1155cbee,
+    0x65cd2261, 0xfe686e0e, 0x89f6bcfe, 0x1253f091, 0x6cdf821c,
+    0xf77ace73, 0x80e41c83, 0x1b4150ec, 0x6fd9b963, 0xf47cf50c,
+    0x83e227fc, 0x18476b93, 0x6ad3f4e2, 0xf176b88d, 0x86e86a7d,
+    0x1d4d2612, 0x69d5cf9d, 0xf27083f2, 0x85ee5102, 0x1e4b1d6d,
+    0x78f6b418, 0xe353f877, 0x94cd2a87, 0x0f6866e8, 0x7bf08f67,
+    0xe055c308, 0x97cb11f8, 0x0c6e5d97, 0x7efac2e6, 0xe55f8e89,
+    0x92c15c79, 0x09641016, 0x7dfcf999, 0xe659b5f6, 0x91c76706,
+    0x0a622b69, 0x74ee59e4, 0xef4b158b, 0x98d5c77b, 0x03708b14,
+    0x77e8629b, 0xec4d2ef4, 0x9bd3fc04, 0x0076b06b, 0x72e22f1a,
+    0xe9476375, 0x9ed9b185, 0x057cfdea, 0x71e41465, 0xea41580a,
+    0x9ddf8afa, 0x067ac695, 0x50a4d810, 0xcb01947f, 0xbc9f468f,
+    0x273a0ae0, 0x53a2e36f, 0xc807af00, 0xbf997df0, 0x243c319f,
+    0x56a8aeee, 0xcd0de281, 0xba933071, 0x21367c1e, 0x55ae9591,
+    0xce0bd9fe, 0xb9950b0e, 0x22304761, 0x5cbc35ec, 0xc7197983,
+    0xb087ab73, 0x2b22e71c, 0x5fba0e93, 0xc41f42fc, 0xb381900c,
+    0x2824dc63, 0x5ab04312, 0xc1150f7d, 0xb68bdd8d, 0x2d2e91e2,
+    0x59b6786d, 0xc2133402, 0xb58de6f2, 0x2e28aa9d, 0x489503e8,
+    0xd3304f87, 0xa4ae9d77, 0x3f0bd118, 0x4b933897, 0xd03674f8,
+    0xa7a8a608, 0x3c0dea67, 0x4e997516, 0xd53c3979, 0xa2a2eb89,
+    0x3907a7e6, 0x4d9f4e69, 0xd63a0206, 0xa1a4d0f6, 0x3a019c99,
+    0x448dee14, 0xdf28a27b, 0xa8b6708b, 0x33133ce4, 0x478bd56b,
+    0xdc2e9904, 0xabb04bf4, 0x3015079b, 0x428198ea, 0xd924d485,
+    0xaeba0675, 0x351f4a1a, 0x4187a395, 0xda22effa, 0xadbc3d0a,
+    0x36197165},
+   {0x00000000, 0xc18edfc0, 0x586cb9c1, 0x99e26601, 0xb0d97382,
+    0x7157ac42, 0xe8b5ca43, 0x293b1583, 0xbac3e145, 0x7b4d3e85,
+    0xe2af5884, 0x23218744, 0x0a1a92c7, 0xcb944d07, 0x52762b06,
+    0x93f8f4c6, 0xaef6c4cb, 0x6f781b0b, 0xf69a7d0a, 0x3714a2ca,
+    0x1e2fb749, 0xdfa16889, 0x46430e88, 0x87cdd148, 0x1435258e,
+    0xd5bbfa4e, 0x4c599c4f, 0x8dd7438f, 0xa4ec560c, 0x656289cc,
+    0xfc80efcd, 0x3d0e300d, 0x869c8fd7, 0x47125017, 0xdef03616,
+    0x1f7ee9d6, 0x3645fc55, 0xf7cb2395, 0x6e294594, 0xafa79a54,
+    0x3c5f6e92, 0xfdd1b152, 0x6433d753, 0xa5bd0893, 0x8c861d10,
+    0x4d08c2d0, 0xd4eaa4d1, 0x15647b11, 0x286a4b1c, 0xe9e494dc,
+    0x7006f2dd, 0xb1882d1d, 0x98b3389e, 0x593de75e, 0xc0df815f,
+    0x01515e9f, 0x92a9aa59, 0x53277599, 0xcac51398, 0x0b4bcc58,
+    0x2270d9db, 0xe3fe061b, 0x7a1c601a, 0xbb92bfda, 0xd64819ef,
+    0x17c6c62f, 0x8e24a02e, 0x4faa7fee, 0x66916a6d, 0xa71fb5ad,
+    0x3efdd3ac, 0xff730c6c, 0x6c8bf8aa, 0xad05276a, 0x34e7416b,
+    0xf5699eab, 0xdc528b28, 0x1ddc54e8, 0x843e32e9, 0x45b0ed29,
+    0x78bedd24, 0xb93002e4, 0x20d264e5, 0xe15cbb25, 0xc867aea6,
+    0x09e97166, 0x900b1767, 0x5185c8a7, 0xc27d3c61, 0x03f3e3a1,
+    0x9a1185a0, 0x5b9f5a60, 0x72a44fe3, 0xb32a9023, 0x2ac8f622,
+    0xeb4629e2, 0x50d49638, 0x915a49f8, 0x08b82ff9, 0xc936f039,
+    0xe00de5ba, 0x21833a7a, 0xb8615c7b, 0x79ef83bb, 0xea17777d,
+    0x2b99a8bd, 0xb27bcebc, 0x73f5117c, 0x5ace04ff, 0x9b40db3f,
+    0x02a2bd3e, 0xc32c62fe, 0xfe2252f3, 0x3fac8d33, 0xa64eeb32,
+    0x67c034f2, 0x4efb2171, 0x8f75feb1, 0x169798b0, 0xd7194770,
+    0x44e1b3b6, 0x856f6c76, 0x1c8d0a77, 0xdd03d5b7, 0xf438c034,
+    0x35b61ff4, 0xac5479f5, 0x6ddaa635, 0x77e1359f, 0xb66fea5f,
+    0x2f8d8c5e, 0xee03539e, 0xc738461d, 0x06b699dd, 0x9f54ffdc,
+    0x5eda201c, 0xcd22d4da, 0x0cac0b1a, 0x954e6d1b, 0x54c0b2db,
+    0x7dfba758, 0xbc757898, 0x25971e99, 0xe419c159, 0xd917f154,
+    0x18992e94, 0x817b4895, 0x40f59755, 0x69ce82d6, 0xa8405d16,
+    0x31a23b17, 0xf02ce4d7, 0x63d41011, 0xa25acfd1, 0x3bb8a9d0,
+    0xfa367610, 0xd30d6393, 0x1283bc53, 0x8b61da52, 0x4aef0592,
+    0xf17dba48, 0x30f36588, 0xa9110389, 0x689fdc49, 0x41a4c9ca,
+    0x802a160a, 0x19c8700b, 0xd846afcb, 0x4bbe5b0d, 0x8a3084cd,
+    0x13d2e2cc, 0xd25c3d0c, 0xfb67288f, 0x3ae9f74f, 0xa30b914e,
+    0x62854e8e, 0x5f8b7e83, 0x9e05a143, 0x07e7c742, 0xc6691882,
+    0xef520d01, 0x2edcd2c1, 0xb73eb4c0, 0x76b06b00, 0xe5489fc6,
+    0x24c64006, 0xbd242607, 0x7caaf9c7, 0x5591ec44, 0x941f3384,
+    0x0dfd5585, 0xcc738a45, 0xa1a92c70, 0x6027f3b0, 0xf9c595b1,
+    0x384b4a71, 0x11705ff2, 0xd0fe8032, 0x491ce633, 0x889239f3,
+    0x1b6acd35, 0xdae412f5, 0x430674f4, 0x8288ab34, 0xabb3beb7,
+    0x6a3d6177, 0xf3df0776, 0x3251d8b6, 0x0f5fe8bb, 0xced1377b,
+    0x5733517a, 0x96bd8eba, 0xbf869b39, 0x7e0844f9, 0xe7ea22f8,
+    0x2664fd38, 0xb59c09fe, 0x7412d63e, 0xedf0b03f, 0x2c7e6fff,
+    0x05457a7c, 0xc4cba5bc, 0x5d29c3bd, 0x9ca71c7d, 0x2735a3a7,
+    0xe6bb7c67, 0x7f591a66, 0xbed7c5a6, 0x97ecd025, 0x56620fe5,
+    0xcf8069e4, 0x0e0eb624, 0x9df642e2, 0x5c789d22, 0xc59afb23,
+    0x041424e3, 0x2d2f3160, 0xeca1eea0, 0x754388a1, 0xb4cd5761,
+    0x89c3676c, 0x484db8ac, 0xd1afdead, 0x1021016d, 0x391a14ee,
+    0xf894cb2e, 0x6176ad2f, 0xa0f872ef, 0x33008629, 0xf28e59e9,
+    0x6b6c3fe8, 0xaae2e028, 0x83d9f5ab, 0x42572a6b, 0xdbb54c6a,
+    0x1a3b93aa},
+   {0x00000000, 0xefc26b3e, 0x04f5d03d, 0xeb37bb03, 0x09eba07a,
+    0xe629cb44, 0x0d1e7047, 0xe2dc1b79, 0x13d740f4, 0xfc152bca,
+    0x172290c9, 0xf8e0fbf7, 0x1a3ce08e, 0xf5fe8bb0, 0x1ec930b3,
+    0xf10b5b8d, 0x27ae81e8, 0xc86cead6, 0x235b51d5, 0xcc993aeb,
+    0x2e452192, 0xc1874aac, 0x2ab0f1af, 0xc5729a91, 0x3479c11c,
+    0xdbbbaa22, 0x308c1121, 0xdf4e7a1f, 0x3d926166, 0xd2500a58,
+    0x3967b15b, 0xd6a5da65, 0x4f5d03d0, 0xa09f68ee, 0x4ba8d3ed,
+    0xa46ab8d3, 0x46b6a3aa, 0xa974c894, 0x42437397, 0xad8118a9,
+    0x5c8a4324, 0xb348281a, 0x587f9319, 0xb7bdf827, 0x5561e35e,
+    0xbaa38860, 0x51943363, 0xbe56585d, 0x68f38238, 0x8731e906,
+    0x6c065205, 0x83c4393b, 0x61182242, 0x8eda497c, 0x65edf27f,
+    0x8a2f9941, 0x7b24c2cc, 0x94e6a9f2, 0x7fd112f1, 0x901379cf,
+    0x72cf62b6, 0x9d0d0988, 0x763ab28b, 0x99f8d9b5, 0x9eba07a0,
+    0x71786c9e, 0x9a4fd79d, 0x758dbca3, 0x9751a7da, 0x7893cce4,
+    0x93a477e7, 0x7c661cd9, 0x8d6d4754, 0x62af2c6a, 0x89989769,
+    0x665afc57, 0x8486e72e, 0x6b448c10, 0x80733713, 0x6fb15c2d,
+    0xb9148648, 0x56d6ed76, 0xbde15675, 0x52233d4b, 0xb0ff2632,
+    0x5f3d4d0c, 0xb40af60f, 0x5bc89d31, 0xaac3c6bc, 0x4501ad82,
+    0xae361681, 0x41f47dbf, 0xa32866c6, 0x4cea0df8, 0xa7ddb6fb,
+    0x481fddc5, 0xd1e70470, 0x3e256f4e, 0xd512d44d, 0x3ad0bf73,
+    0xd80ca40a, 0x37cecf34, 0xdcf97437, 0x333b1f09, 0xc2304484,
+    0x2df22fba, 0xc6c594b9, 0x2907ff87, 0xcbdbe4fe, 0x24198fc0,
+    0xcf2e34c3, 0x20ec5ffd, 0xf6498598, 0x198beea6, 0xf2bc55a5,
+    0x1d7e3e9b, 0xffa225e2, 0x10604edc, 0xfb57f5df, 0x14959ee1,
+    0xe59ec56c, 0x0a5cae52, 0xe16b1551, 0x0ea97e6f, 0xec756516,
+    0x03b70e28, 0xe880b52b, 0x0742de15, 0xe6050901, 0x09c7623f,
+    0xe2f0d93c, 0x0d32b202, 0xefeea97b, 0x002cc245, 0xeb1b7946,
+    0x04d91278, 0xf5d249f5, 0x1a1022cb, 0xf12799c8, 0x1ee5f2f6,
+    0xfc39e98f, 0x13fb82b1, 0xf8cc39b2, 0x170e528c, 0xc1ab88e9,
+    0x2e69e3d7, 0xc55e58d4, 0x2a9c33ea, 0xc8402893, 0x278243ad,
+    0xccb5f8ae, 0x23779390, 0xd27cc81d, 0x3dbea323, 0xd6891820,
+    0x394b731e, 0xdb976867, 0x34550359, 0xdf62b85a, 0x30a0d364,
+    0xa9580ad1, 0x469a61ef, 0xadaddaec, 0x426fb1d2, 0xa0b3aaab,
+    0x4f71c195, 0xa4467a96, 0x4b8411a8, 0xba8f4a25, 0x554d211b,
+    0xbe7a9a18, 0x51b8f126, 0xb364ea5f, 0x5ca68161, 0xb7913a62,
+    0x5853515c, 0x8ef68b39, 0x6134e007, 0x8a035b04, 0x65c1303a,
+    0x871d2b43, 0x68df407d, 0x83e8fb7e, 0x6c2a9040, 0x9d21cbcd,
+    0x72e3a0f3, 0x99d41bf0, 0x761670ce, 0x94ca6bb7, 0x7b080089,
+    0x903fbb8a, 0x7ffdd0b4, 0x78bf0ea1, 0x977d659f, 0x7c4ade9c,
+    0x9388b5a2, 0x7154aedb, 0x9e96c5e5, 0x75a17ee6, 0x9a6315d8,
+    0x6b684e55, 0x84aa256b, 0x6f9d9e68, 0x805ff556, 0x6283ee2f,
+    0x8d418511, 0x66763e12, 0x89b4552c, 0x5f118f49, 0xb0d3e477,
+    0x5be45f74, 0xb426344a, 0x56fa2f33, 0xb938440d, 0x520fff0e,
+    0xbdcd9430, 0x4cc6cfbd, 0xa304a483, 0x48331f80, 0xa7f174be,
+    0x452d6fc7, 0xaaef04f9, 0x41d8bffa, 0xae1ad4c4, 0x37e20d71,
+    0xd820664f, 0x3317dd4c, 0xdcd5b672, 0x3e09ad0b, 0xd1cbc635,
+    0x3afc7d36, 0xd53e1608, 0x24354d85, 0xcbf726bb, 0x20c09db8,
+    0xcf02f686, 0x2ddeedff, 0xc21c86c1, 0x292b3dc2, 0xc6e956fc,
+    0x104c8c99, 0xff8ee7a7, 0x14b95ca4, 0xfb7b379a, 0x19a72ce3,
+    0xf66547dd, 0x1d52fcde, 0xf29097e0, 0x039bcc6d, 0xec59a753,
+    0x076e1c50, 0xe8ac776e, 0x0a706c17, 0xe5b20729, 0x0e85bc2a,
+    0xe147d714},
+   {0x00000000, 0x177b1443, 0x2ef62886, 0x398d3cc5, 0x5dec510c,
+    0x4a97454f, 0x731a798a, 0x64616dc9, 0xbbd8a218, 0xaca3b65b,
+    0x952e8a9e, 0x82559edd, 0xe634f314, 0xf14fe757, 0xc8c2db92,
+    0xdfb9cfd1, 0xacc04271, 0xbbbb5632, 0x82366af7, 0x954d7eb4,
+    0xf12c137d, 0xe657073e, 0xdfda3bfb, 0xc8a12fb8, 0x1718e069,
+    0x0063f42a, 0x39eec8ef, 0x2e95dcac, 0x4af4b165, 0x5d8fa526,
+    0x640299e3, 0x73798da0, 0x82f182a3, 0x958a96e0, 0xac07aa25,
+    0xbb7cbe66, 0xdf1dd3af, 0xc866c7ec, 0xf1ebfb29, 0xe690ef6a,
+    0x392920bb, 0x2e5234f8, 0x17df083d, 0x00a41c7e, 0x64c571b7,
+    0x73be65f4, 0x4a335931, 0x5d484d72, 0x2e31c0d2, 0x394ad491,
+    0x00c7e854, 0x17bcfc17, 0x73dd91de, 0x64a6859d, 0x5d2bb958,
+    0x4a50ad1b, 0x95e962ca, 0x82927689, 0xbb1f4a4c, 0xac645e0f,
+    0xc80533c6, 0xdf7e2785, 0xe6f31b40, 0xf1880f03, 0xde920307,
+    0xc9e91744, 0xf0642b81, 0xe71f3fc2, 0x837e520b, 0x94054648,
+    0xad887a8d, 0xbaf36ece, 0x654aa11f, 0x7231b55c, 0x4bbc8999,
+    0x5cc79dda, 0x38a6f013, 0x2fdde450, 0x1650d895, 0x012bccd6,
+    0x72524176, 0x65295535, 0x5ca469f0, 0x4bdf7db3, 0x2fbe107a,
+    0x38c50439, 0x014838fc, 0x16332cbf, 0xc98ae36e, 0xdef1f72d,
+    0xe77ccbe8, 0xf007dfab, 0x9466b262, 0x831da621, 0xba909ae4,
+    0xadeb8ea7, 0x5c6381a4, 0x4b1895e7, 0x7295a922, 0x65eebd61,
+    0x018fd0a8, 0x16f4c4eb, 0x2f79f82e, 0x3802ec6d, 0xe7bb23bc,
+    0xf0c037ff, 0xc94d0b3a, 0xde361f79, 0xba5772b0, 0xad2c66f3,
+    0x94a15a36, 0x83da4e75, 0xf0a3c3d5, 0xe7d8d796, 0xde55eb53,
+    0xc92eff10, 0xad4f92d9, 0xba34869a, 0x83b9ba5f, 0x94c2ae1c,
+    0x4b7b61cd, 0x5c00758e, 0x658d494b, 0x72f65d08, 0x169730c1,
+    0x01ec2482, 0x38611847, 0x2f1a0c04, 0x6655004f, 0x712e140c,
+    0x48a328c9, 0x5fd83c8a, 0x3bb95143, 0x2cc24500, 0x154f79c5,
+    0x02346d86, 0xdd8da257, 0xcaf6b614, 0xf37b8ad1, 0xe4009e92,
+    0x8061f35b, 0x971ae718, 0xae97dbdd, 0xb9eccf9e, 0xca95423e,
+    0xddee567d, 0xe4636ab8, 0xf3187efb, 0x97791332, 0x80020771,
+    0xb98f3bb4, 0xaef42ff7, 0x714de026, 0x6636f465, 0x5fbbc8a0,
+    0x48c0dce3, 0x2ca1b12a, 0x3bdaa569, 0x025799ac, 0x152c8def,
+    0xe4a482ec, 0xf3df96af, 0xca52aa6a, 0xdd29be29, 0xb948d3e0,
+    0xae33c7a3, 0x97befb66, 0x80c5ef25, 0x5f7c20f4, 0x480734b7,
+    0x718a0872, 0x66f11c31, 0x029071f8, 0x15eb65bb, 0x2c66597e,
+    0x3b1d4d3d, 0x4864c09d, 0x5f1fd4de, 0x6692e81b, 0x71e9fc58,
+    0x15889191, 0x02f385d2, 0x3b7eb917, 0x2c05ad54, 0xf3bc6285,
+    0xe4c776c6, 0xdd4a4a03, 0xca315e40, 0xae503389, 0xb92b27ca,
+    0x80a61b0f, 0x97dd0f4c, 0xb8c70348, 0xafbc170b, 0x96312bce,
+    0x814a3f8d, 0xe52b5244, 0xf2504607, 0xcbdd7ac2, 0xdca66e81,
+    0x031fa150, 0x1464b513, 0x2de989d6, 0x3a929d95, 0x5ef3f05c,
+    0x4988e41f, 0x7005d8da, 0x677ecc99, 0x14074139, 0x037c557a,
+    0x3af169bf, 0x2d8a7dfc, 0x49eb1035, 0x5e900476, 0x671d38b3,
+    0x70662cf0, 0xafdfe321, 0xb8a4f762, 0x8129cba7, 0x9652dfe4,
+    0xf233b22d, 0xe548a66e, 0xdcc59aab, 0xcbbe8ee8, 0x3a3681eb,
+    0x2d4d95a8, 0x14c0a96d, 0x03bbbd2e, 0x67dad0e7, 0x70a1c4a4,
+    0x492cf861, 0x5e57ec22, 0x81ee23f3, 0x969537b0, 0xaf180b75,
+    0xb8631f36, 0xdc0272ff, 0xcb7966bc, 0xf2f45a79, 0xe58f4e3a,
+    0x96f6c39a, 0x818dd7d9, 0xb800eb1c, 0xaf7bff5f, 0xcb1a9296,
+    0xdc6186d5, 0xe5ecba10, 0xf297ae53, 0x2d2e6182, 0x3a5575c1,
+    0x03d84904, 0x14a35d47, 0x70c2308e, 0x67b924cd, 0x5e341808,
+    0x494f0c4b}};
+
+local const z_word_t FAR crc_braid_big_table[][256] = {
+   {0x0000000000000000, 0x43147b1700000000, 0x8628f62e00000000,
+    0xc53c8d3900000000, 0x0c51ec5d00000000, 0x4f45974a00000000,
+    0x8a791a7300000000, 0xc96d616400000000, 0x18a2d8bb00000000,
+    0x5bb6a3ac00000000, 0x9e8a2e9500000000, 0xdd9e558200000000,
+    0x14f334e600000000, 0x57e74ff100000000, 0x92dbc2c800000000,
+    0xd1cfb9df00000000, 0x7142c0ac00000000, 0x3256bbbb00000000,
+    0xf76a368200000000, 0xb47e4d9500000000, 0x7d132cf100000000,
+    0x3e0757e600000000, 0xfb3bdadf00000000, 0xb82fa1c800000000,
+    0x69e0181700000000, 0x2af4630000000000, 0xefc8ee3900000000,
+    0xacdc952e00000000, 0x65b1f44a00000000, 0x26a58f5d00000000,
+    0xe399026400000000, 0xa08d797300000000, 0xa382f18200000000,
+    0xe0968a9500000000, 0x25aa07ac00000000, 0x66be7cbb00000000,
+    0xafd31ddf00000000, 0xecc766c800000000, 0x29fbebf100000000,
+    0x6aef90e600000000, 0xbb20293900000000, 0xf834522e00000000,
+    0x3d08df1700000000, 0x7e1ca40000000000, 0xb771c56400000000,
+    0xf465be7300000000, 0x3159334a00000000, 0x724d485d00000000,
+    0xd2c0312e00000000, 0x91d44a3900000000, 0x54e8c70000000000,
+    0x17fcbc1700000000, 0xde91dd7300000000, 0x9d85a66400000000,
+    0x58b92b5d00000000, 0x1bad504a00000000, 0xca62e99500000000,
+    0x8976928200000000, 0x4c4a1fbb00000000, 0x0f5e64ac00000000,
+    0xc63305c800000000, 0x85277edf00000000, 0x401bf3e600000000,
+    0x030f88f100000000, 0x070392de00000000, 0x4417e9c900000000,
+    0x812b64f000000000, 0xc23f1fe700000000, 0x0b527e8300000000,
+    0x4846059400000000, 0x8d7a88ad00000000, 0xce6ef3ba00000000,
+    0x1fa14a6500000000, 0x5cb5317200000000, 0x9989bc4b00000000,
+    0xda9dc75c00000000, 0x13f0a63800000000, 0x50e4dd2f00000000,
+    0x95d8501600000000, 0xd6cc2b0100000000, 0x7641527200000000,
+    0x3555296500000000, 0xf069a45c00000000, 0xb37ddf4b00000000,
+    0x7a10be2f00000000, 0x3904c53800000000, 0xfc38480100000000,
+    0xbf2c331600000000, 0x6ee38ac900000000, 0x2df7f1de00000000,
+    0xe8cb7ce700000000, 0xabdf07f000000000, 0x62b2669400000000,
+    0x21a61d8300000000, 0xe49a90ba00000000, 0xa78eebad00000000,
+    0xa481635c00000000, 0xe795184b00000000, 0x22a9957200000000,
+    0x61bdee6500000000, 0xa8d08f0100000000, 0xebc4f41600000000,
+    0x2ef8792f00000000, 0x6dec023800000000, 0xbc23bbe700000000,
+    0xff37c0f000000000, 0x3a0b4dc900000000, 0x791f36de00000000,
+    0xb07257ba00000000, 0xf3662cad00000000, 0x365aa19400000000,
+    0x754eda8300000000, 0xd5c3a3f000000000, 0x96d7d8e700000000,
+    0x53eb55de00000000, 0x10ff2ec900000000, 0xd9924fad00000000,
+    0x9a8634ba00000000, 0x5fbab98300000000, 0x1caec29400000000,
+    0xcd617b4b00000000, 0x8e75005c00000000, 0x4b498d6500000000,
+    0x085df67200000000, 0xc130971600000000, 0x8224ec0100000000,
+    0x4718613800000000, 0x040c1a2f00000000, 0x4f00556600000000,
+    0x0c142e7100000000, 0xc928a34800000000, 0x8a3cd85f00000000,
+    0x4351b93b00000000, 0x0045c22c00000000, 0xc5794f1500000000,
+    0x866d340200000000, 0x57a28ddd00000000, 0x14b6f6ca00000000,
+    0xd18a7bf300000000, 0x929e00e400000000, 0x5bf3618000000000,
+    0x18e71a9700000000, 0xdddb97ae00000000, 0x9ecfecb900000000,
+    0x3e4295ca00000000, 0x7d56eedd00000000, 0xb86a63e400000000,
+    0xfb7e18f300000000, 0x3213799700000000, 0x7107028000000000,
+    0xb43b8fb900000000, 0xf72ff4ae00000000, 0x26e04d7100000000,
+    0x65f4366600000000, 0xa0c8bb5f00000000, 0xe3dcc04800000000,
+    0x2ab1a12c00000000, 0x69a5da3b00000000, 0xac99570200000000,
+    0xef8d2c1500000000, 0xec82a4e400000000, 0xaf96dff300000000,
+    0x6aaa52ca00000000, 0x29be29dd00000000, 0xe0d348b900000000,
+    0xa3c733ae00000000, 0x66fbbe9700000000, 0x25efc58000000000,
+    0xf4207c5f00000000, 0xb734074800000000, 0x72088a7100000000,
+    0x311cf16600000000, 0xf871900200000000, 0xbb65eb1500000000,
+    0x7e59662c00000000, 0x3d4d1d3b00000000, 0x9dc0644800000000,
+    0xded41f5f00000000, 0x1be8926600000000, 0x58fce97100000000,
+    0x9191881500000000, 0xd285f30200000000, 0x17b97e3b00000000,
+    0x54ad052c00000000, 0x8562bcf300000000, 0xc676c7e400000000,
+    0x034a4add00000000, 0x405e31ca00000000, 0x893350ae00000000,
+    0xca272bb900000000, 0x0f1ba68000000000, 0x4c0fdd9700000000,
+    0x4803c7b800000000, 0x0b17bcaf00000000, 0xce2b319600000000,
+    0x8d3f4a8100000000, 0x44522be500000000, 0x074650f200000000,
+    0xc27addcb00000000, 0x816ea6dc00000000, 0x50a11f0300000000,
+    0x13b5641400000000, 0xd689e92d00000000, 0x959d923a00000000,
+    0x5cf0f35e00000000, 0x1fe4884900000000, 0xdad8057000000000,
+    0x99cc7e6700000000, 0x3941071400000000, 0x7a557c0300000000,
+    0xbf69f13a00000000, 0xfc7d8a2d00000000, 0x3510eb4900000000,
+    0x7604905e00000000, 0xb3381d6700000000, 0xf02c667000000000,
+    0x21e3dfaf00000000, 0x62f7a4b800000000, 0xa7cb298100000000,
+    0xe4df529600000000, 0x2db233f200000000, 0x6ea648e500000000,
+    0xab9ac5dc00000000, 0xe88ebecb00000000, 0xeb81363a00000000,
+    0xa8954d2d00000000, 0x6da9c01400000000, 0x2ebdbb0300000000,
+    0xe7d0da6700000000, 0xa4c4a17000000000, 0x61f82c4900000000,
+    0x22ec575e00000000, 0xf323ee8100000000, 0xb037959600000000,
+    0x750b18af00000000, 0x361f63b800000000, 0xff7202dc00000000,
+    0xbc6679cb00000000, 0x795af4f200000000, 0x3a4e8fe500000000,
+    0x9ac3f69600000000, 0xd9d78d8100000000, 0x1ceb00b800000000,
+    0x5fff7baf00000000, 0x96921acb00000000, 0xd58661dc00000000,
+    0x10baece500000000, 0x53ae97f200000000, 0x82612e2d00000000,
+    0xc175553a00000000, 0x0449d80300000000, 0x475da31400000000,
+    0x8e30c27000000000, 0xcd24b96700000000, 0x0818345e00000000,
+    0x4b0c4f4900000000},
+   {0x0000000000000000, 0x3e6bc2ef00000000, 0x3dd0f50400000000,
+    0x03bb37eb00000000, 0x7aa0eb0900000000, 0x44cb29e600000000,
+    0x47701e0d00000000, 0x791bdce200000000, 0xf440d71300000000,
+    0xca2b15fc00000000, 0xc990221700000000, 0xf7fbe0f800000000,
+    0x8ee03c1a00000000, 0xb08bfef500000000, 0xb330c91e00000000,
+    0x8d5b0bf100000000, 0xe881ae2700000000, 0xd6ea6cc800000000,
+    0xd5515b2300000000, 0xeb3a99cc00000000, 0x9221452e00000000,
+    0xac4a87c100000000, 0xaff1b02a00000000, 0x919a72c500000000,
+    0x1cc1793400000000, 0x22aabbdb00000000, 0x21118c3000000000,
+    0x1f7a4edf00000000, 0x6661923d00000000, 0x580a50d200000000,
+    0x5bb1673900000000, 0x65daa5d600000000, 0xd0035d4f00000000,
+    0xee689fa000000000, 0xedd3a84b00000000, 0xd3b86aa400000000,
+    0xaaa3b64600000000, 0x94c874a900000000, 0x9773434200000000,
+    0xa91881ad00000000, 0x24438a5c00000000, 0x1a2848b300000000,
+    0x19937f5800000000, 0x27f8bdb700000000, 0x5ee3615500000000,
+    0x6088a3ba00000000, 0x6333945100000000, 0x5d5856be00000000,
+    0x3882f36800000000, 0x06e9318700000000, 0x0552066c00000000,
+    0x3b39c48300000000, 0x4222186100000000, 0x7c49da8e00000000,
+    0x7ff2ed6500000000, 0x41992f8a00000000, 0xccc2247b00000000,
+    0xf2a9e69400000000, 0xf112d17f00000000, 0xcf79139000000000,
+    0xb662cf7200000000, 0x88090d9d00000000, 0x8bb23a7600000000,
+    0xb5d9f89900000000, 0xa007ba9e00000000, 0x9e6c787100000000,
+    0x9dd74f9a00000000, 0xa3bc8d7500000000, 0xdaa7519700000000,
+    0xe4cc937800000000, 0xe777a49300000000, 0xd91c667c00000000,
+    0x54476d8d00000000, 0x6a2caf6200000000, 0x6997988900000000,
+    0x57fc5a6600000000, 0x2ee7868400000000, 0x108c446b00000000,
+    0x1337738000000000, 0x2d5cb16f00000000, 0x488614b900000000,
+    0x76edd65600000000, 0x7556e1bd00000000, 0x4b3d235200000000,
+    0x3226ffb000000000, 0x0c4d3d5f00000000, 0x0ff60ab400000000,
+    0x319dc85b00000000, 0xbcc6c3aa00000000, 0x82ad014500000000,
+    0x811636ae00000000, 0xbf7df44100000000, 0xc66628a300000000,
+    0xf80dea4c00000000, 0xfbb6dda700000000, 0xc5dd1f4800000000,
+    0x7004e7d100000000, 0x4e6f253e00000000, 0x4dd412d500000000,
+    0x73bfd03a00000000, 0x0aa40cd800000000, 0x34cfce3700000000,
+    0x3774f9dc00000000, 0x091f3b3300000000, 0x844430c200000000,
+    0xba2ff22d00000000, 0xb994c5c600000000, 0x87ff072900000000,
+    0xfee4dbcb00000000, 0xc08f192400000000, 0xc3342ecf00000000,
+    0xfd5fec2000000000, 0x988549f600000000, 0xa6ee8b1900000000,
+    0xa555bcf200000000, 0x9b3e7e1d00000000, 0xe225a2ff00000000,
+    0xdc4e601000000000, 0xdff557fb00000000, 0xe19e951400000000,
+    0x6cc59ee500000000, 0x52ae5c0a00000000, 0x51156be100000000,
+    0x6f7ea90e00000000, 0x166575ec00000000, 0x280eb70300000000,
+    0x2bb580e800000000, 0x15de420700000000, 0x010905e600000000,
+    0x3f62c70900000000, 0x3cd9f0e200000000, 0x02b2320d00000000,
+    0x7ba9eeef00000000, 0x45c22c0000000000, 0x46791beb00000000,
+    0x7812d90400000000, 0xf549d2f500000000, 0xcb22101a00000000,
+    0xc89927f100000000, 0xf6f2e51e00000000, 0x8fe939fc00000000,
+    0xb182fb1300000000, 0xb239ccf800000000, 0x8c520e1700000000,
+    0xe988abc100000000, 0xd7e3692e00000000, 0xd4585ec500000000,
+    0xea339c2a00000000, 0x932840c800000000, 0xad43822700000000,
+    0xaef8b5cc00000000, 0x9093772300000000, 0x1dc87cd200000000,
+    0x23a3be3d00000000, 0x201889d600000000, 0x1e734b3900000000,
+    0x676897db00000000, 0x5903553400000000, 0x5ab862df00000000,
+    0x64d3a03000000000, 0xd10a58a900000000, 0xef619a4600000000,
+    0xecdaadad00000000, 0xd2b16f4200000000, 0xabaab3a000000000,
+    0x95c1714f00000000, 0x967a46a400000000, 0xa811844b00000000,
+    0x254a8fba00000000, 0x1b214d5500000000, 0x189a7abe00000000,
+    0x26f1b85100000000, 0x5fea64b300000000, 0x6181a65c00000000,
+    0x623a91b700000000, 0x5c51535800000000, 0x398bf68e00000000,
+    0x07e0346100000000, 0x045b038a00000000, 0x3a30c16500000000,
+    0x432b1d8700000000, 0x7d40df6800000000, 0x7efbe88300000000,
+    0x40902a6c00000000, 0xcdcb219d00000000, 0xf3a0e37200000000,
+    0xf01bd49900000000, 0xce70167600000000, 0xb76bca9400000000,
+    0x8900087b00000000, 0x8abb3f9000000000, 0xb4d0fd7f00000000,
+    0xa10ebf7800000000, 0x9f657d9700000000, 0x9cde4a7c00000000,
+    0xa2b5889300000000, 0xdbae547100000000, 0xe5c5969e00000000,
+    0xe67ea17500000000, 0xd815639a00000000, 0x554e686b00000000,
+    0x6b25aa8400000000, 0x689e9d6f00000000, 0x56f55f8000000000,
+    0x2fee836200000000, 0x1185418d00000000, 0x123e766600000000,
+    0x2c55b48900000000, 0x498f115f00000000, 0x77e4d3b000000000,
+    0x745fe45b00000000, 0x4a3426b400000000, 0x332ffa5600000000,
+    0x0d4438b900000000, 0x0eff0f5200000000, 0x3094cdbd00000000,
+    0xbdcfc64c00000000, 0x83a404a300000000, 0x801f334800000000,
+    0xbe74f1a700000000, 0xc76f2d4500000000, 0xf904efaa00000000,
+    0xfabfd84100000000, 0xc4d41aae00000000, 0x710de23700000000,
+    0x4f6620d800000000, 0x4cdd173300000000, 0x72b6d5dc00000000,
+    0x0bad093e00000000, 0x35c6cbd100000000, 0x367dfc3a00000000,
+    0x08163ed500000000, 0x854d352400000000, 0xbb26f7cb00000000,
+    0xb89dc02000000000, 0x86f602cf00000000, 0xffedde2d00000000,
+    0xc1861cc200000000, 0xc23d2b2900000000, 0xfc56e9c600000000,
+    0x998c4c1000000000, 0xa7e78eff00000000, 0xa45cb91400000000,
+    0x9a377bfb00000000, 0xe32ca71900000000, 0xdd4765f600000000,
+    0xdefc521d00000000, 0xe09790f200000000, 0x6dcc9b0300000000,
+    0x53a759ec00000000, 0x501c6e0700000000, 0x6e77ace800000000,
+    0x176c700a00000000, 0x2907b2e500000000, 0x2abc850e00000000,
+    0x14d747e100000000},
+   {0x0000000000000000, 0xc0df8ec100000000, 0xc1b96c5800000000,
+    0x0166e29900000000, 0x8273d9b000000000, 0x42ac577100000000,
+    0x43cab5e800000000, 0x83153b2900000000, 0x45e1c3ba00000000,
+    0x853e4d7b00000000, 0x8458afe200000000, 0x4487212300000000,
+    0xc7921a0a00000000, 0x074d94cb00000000, 0x062b765200000000,
+    0xc6f4f89300000000, 0xcbc4f6ae00000000, 0x0b1b786f00000000,
+    0x0a7d9af600000000, 0xcaa2143700000000, 0x49b72f1e00000000,
+    0x8968a1df00000000, 0x880e434600000000, 0x48d1cd8700000000,
+    0x8e25351400000000, 0x4efabbd500000000, 0x4f9c594c00000000,
+    0x8f43d78d00000000, 0x0c56eca400000000, 0xcc89626500000000,
+    0xcdef80fc00000000, 0x0d300e3d00000000, 0xd78f9c8600000000,
+    0x1750124700000000, 0x1636f0de00000000, 0xd6e97e1f00000000,
+    0x55fc453600000000, 0x9523cbf700000000, 0x9445296e00000000,
+    0x549aa7af00000000, 0x926e5f3c00000000, 0x52b1d1fd00000000,
+    0x53d7336400000000, 0x9308bda500000000, 0x101d868c00000000,
+    0xd0c2084d00000000, 0xd1a4ead400000000, 0x117b641500000000,
+    0x1c4b6a2800000000, 0xdc94e4e900000000, 0xddf2067000000000,
+    0x1d2d88b100000000, 0x9e38b39800000000, 0x5ee73d5900000000,
+    0x5f81dfc000000000, 0x9f5e510100000000, 0x59aaa99200000000,
+    0x9975275300000000, 0x9813c5ca00000000, 0x58cc4b0b00000000,
+    0xdbd9702200000000, 0x1b06fee300000000, 0x1a601c7a00000000,
+    0xdabf92bb00000000, 0xef1948d600000000, 0x2fc6c61700000000,
+    0x2ea0248e00000000, 0xee7faa4f00000000, 0x6d6a916600000000,
+    0xadb51fa700000000, 0xacd3fd3e00000000, 0x6c0c73ff00000000,
+    0xaaf88b6c00000000, 0x6a2705ad00000000, 0x6b41e73400000000,
+    0xab9e69f500000000, 0x288b52dc00000000, 0xe854dc1d00000000,
+    0xe9323e8400000000, 0x29edb04500000000, 0x24ddbe7800000000,
+    0xe40230b900000000, 0xe564d22000000000, 0x25bb5ce100000000,
+    0xa6ae67c800000000, 0x6671e90900000000, 0x67170b9000000000,
+    0xa7c8855100000000, 0x613c7dc200000000, 0xa1e3f30300000000,
+    0xa085119a00000000, 0x605a9f5b00000000, 0xe34fa47200000000,
+    0x23902ab300000000, 0x22f6c82a00000000, 0xe22946eb00000000,
+    0x3896d45000000000, 0xf8495a9100000000, 0xf92fb80800000000,
+    0x39f036c900000000, 0xbae50de000000000, 0x7a3a832100000000,
+    0x7b5c61b800000000, 0xbb83ef7900000000, 0x7d7717ea00000000,
+    0xbda8992b00000000, 0xbcce7bb200000000, 0x7c11f57300000000,
+    0xff04ce5a00000000, 0x3fdb409b00000000, 0x3ebda20200000000,
+    0xfe622cc300000000, 0xf35222fe00000000, 0x338dac3f00000000,
+    0x32eb4ea600000000, 0xf234c06700000000, 0x7121fb4e00000000,
+    0xb1fe758f00000000, 0xb098971600000000, 0x704719d700000000,
+    0xb6b3e14400000000, 0x766c6f8500000000, 0x770a8d1c00000000,
+    0xb7d503dd00000000, 0x34c038f400000000, 0xf41fb63500000000,
+    0xf57954ac00000000, 0x35a6da6d00000000, 0x9f35e17700000000,
+    0x5fea6fb600000000, 0x5e8c8d2f00000000, 0x9e5303ee00000000,
+    0x1d4638c700000000, 0xdd99b60600000000, 0xdcff549f00000000,
+    0x1c20da5e00000000, 0xdad422cd00000000, 0x1a0bac0c00000000,
+    0x1b6d4e9500000000, 0xdbb2c05400000000, 0x58a7fb7d00000000,
+    0x987875bc00000000, 0x991e972500000000, 0x59c119e400000000,
+    0x54f117d900000000, 0x942e991800000000, 0x95487b8100000000,
+    0x5597f54000000000, 0xd682ce6900000000, 0x165d40a800000000,
+    0x173ba23100000000, 0xd7e42cf000000000, 0x1110d46300000000,
+    0xd1cf5aa200000000, 0xd0a9b83b00000000, 0x107636fa00000000,
+    0x93630dd300000000, 0x53bc831200000000, 0x52da618b00000000,
+    0x9205ef4a00000000, 0x48ba7df100000000, 0x8865f33000000000,
+    0x890311a900000000, 0x49dc9f6800000000, 0xcac9a44100000000,
+    0x0a162a8000000000, 0x0b70c81900000000, 0xcbaf46d800000000,
+    0x0d5bbe4b00000000, 0xcd84308a00000000, 0xcce2d21300000000,
+    0x0c3d5cd200000000, 0x8f2867fb00000000, 0x4ff7e93a00000000,
+    0x4e910ba300000000, 0x8e4e856200000000, 0x837e8b5f00000000,
+    0x43a1059e00000000, 0x42c7e70700000000, 0x821869c600000000,
+    0x010d52ef00000000, 0xc1d2dc2e00000000, 0xc0b43eb700000000,
+    0x006bb07600000000, 0xc69f48e500000000, 0x0640c62400000000,
+    0x072624bd00000000, 0xc7f9aa7c00000000, 0x44ec915500000000,
+    0x84331f9400000000, 0x8555fd0d00000000, 0x458a73cc00000000,
+    0x702ca9a100000000, 0xb0f3276000000000, 0xb195c5f900000000,
+    0x714a4b3800000000, 0xf25f701100000000, 0x3280fed000000000,
+    0x33e61c4900000000, 0xf339928800000000, 0x35cd6a1b00000000,
+    0xf512e4da00000000, 0xf474064300000000, 0x34ab888200000000,
+    0xb7beb3ab00000000, 0x77613d6a00000000, 0x7607dff300000000,
+    0xb6d8513200000000, 0xbbe85f0f00000000, 0x7b37d1ce00000000,
+    0x7a51335700000000, 0xba8ebd9600000000, 0x399b86bf00000000,
+    0xf944087e00000000, 0xf822eae700000000, 0x38fd642600000000,
+    0xfe099cb500000000, 0x3ed6127400000000, 0x3fb0f0ed00000000,
+    0xff6f7e2c00000000, 0x7c7a450500000000, 0xbca5cbc400000000,
+    0xbdc3295d00000000, 0x7d1ca79c00000000, 0xa7a3352700000000,
+    0x677cbbe600000000, 0x661a597f00000000, 0xa6c5d7be00000000,
+    0x25d0ec9700000000, 0xe50f625600000000, 0xe46980cf00000000,
+    0x24b60e0e00000000, 0xe242f69d00000000, 0x229d785c00000000,
+    0x23fb9ac500000000, 0xe324140400000000, 0x60312f2d00000000,
+    0xa0eea1ec00000000, 0xa188437500000000, 0x6157cdb400000000,
+    0x6c67c38900000000, 0xacb84d4800000000, 0xaddeafd100000000,
+    0x6d01211000000000, 0xee141a3900000000, 0x2ecb94f800000000,
+    0x2fad766100000000, 0xef72f8a000000000, 0x2986003300000000,
+    0xe9598ef200000000, 0xe83f6c6b00000000, 0x28e0e2aa00000000,
+    0xabf5d98300000000, 0x6b2a574200000000, 0x6a4cb5db00000000,
+    0xaa933b1a00000000},
+   {0x0000000000000000, 0x6f4ca59b00000000, 0x9f9e3bec00000000,
+    0xf0d29e7700000000, 0x7f3b060300000000, 0x1077a39800000000,
+    0xe0a53def00000000, 0x8fe9987400000000, 0xfe760c0600000000,
+    0x913aa99d00000000, 0x61e837ea00000000, 0x0ea4927100000000,
+    0x814d0a0500000000, 0xee01af9e00000000, 0x1ed331e900000000,
+    0x719f947200000000, 0xfced180c00000000, 0x93a1bd9700000000,
+    0x637323e000000000, 0x0c3f867b00000000, 0x83d61e0f00000000,
+    0xec9abb9400000000, 0x1c4825e300000000, 0x7304807800000000,
+    0x029b140a00000000, 0x6dd7b19100000000, 0x9d052fe600000000,
+    0xf2498a7d00000000, 0x7da0120900000000, 0x12ecb79200000000,
+    0xe23e29e500000000, 0x8d728c7e00000000, 0xf8db311800000000,
+    0x9797948300000000, 0x67450af400000000, 0x0809af6f00000000,
+    0x87e0371b00000000, 0xe8ac928000000000, 0x187e0cf700000000,
+    0x7732a96c00000000, 0x06ad3d1e00000000, 0x69e1988500000000,
+    0x993306f200000000, 0xf67fa36900000000, 0x79963b1d00000000,
+    0x16da9e8600000000, 0xe60800f100000000, 0x8944a56a00000000,
+    0x0436291400000000, 0x6b7a8c8f00000000, 0x9ba812f800000000,
+    0xf4e4b76300000000, 0x7b0d2f1700000000, 0x14418a8c00000000,
+    0xe49314fb00000000, 0x8bdfb16000000000, 0xfa40251200000000,
+    0x950c808900000000, 0x65de1efe00000000, 0x0a92bb6500000000,
+    0x857b231100000000, 0xea37868a00000000, 0x1ae518fd00000000,
+    0x75a9bd6600000000, 0xf0b7633000000000, 0x9ffbc6ab00000000,
+    0x6f2958dc00000000, 0x0065fd4700000000, 0x8f8c653300000000,
+    0xe0c0c0a800000000, 0x10125edf00000000, 0x7f5efb4400000000,
+    0x0ec16f3600000000, 0x618dcaad00000000, 0x915f54da00000000,
+    0xfe13f14100000000, 0x71fa693500000000, 0x1eb6ccae00000000,
+    0xee6452d900000000, 0x8128f74200000000, 0x0c5a7b3c00000000,
+    0x6316dea700000000, 0x93c440d000000000, 0xfc88e54b00000000,
+    0x73617d3f00000000, 0x1c2dd8a400000000, 0xecff46d300000000,
+    0x83b3e34800000000, 0xf22c773a00000000, 0x9d60d2a100000000,
+    0x6db24cd600000000, 0x02fee94d00000000, 0x8d17713900000000,
+    0xe25bd4a200000000, 0x12894ad500000000, 0x7dc5ef4e00000000,
+    0x086c522800000000, 0x6720f7b300000000, 0x97f269c400000000,
+    0xf8becc5f00000000, 0x7757542b00000000, 0x181bf1b000000000,
+    0xe8c96fc700000000, 0x8785ca5c00000000, 0xf61a5e2e00000000,
+    0x9956fbb500000000, 0x698465c200000000, 0x06c8c05900000000,
+    0x8921582d00000000, 0xe66dfdb600000000, 0x16bf63c100000000,
+    0x79f3c65a00000000, 0xf4814a2400000000, 0x9bcdefbf00000000,
+    0x6b1f71c800000000, 0x0453d45300000000, 0x8bba4c2700000000,
+    0xe4f6e9bc00000000, 0x142477cb00000000, 0x7b68d25000000000,
+    0x0af7462200000000, 0x65bbe3b900000000, 0x95697dce00000000,
+    0xfa25d85500000000, 0x75cc402100000000, 0x1a80e5ba00000000,
+    0xea527bcd00000000, 0x851ede5600000000, 0xe06fc76000000000,
+    0x8f2362fb00000000, 0x7ff1fc8c00000000, 0x10bd591700000000,
+    0x9f54c16300000000, 0xf01864f800000000, 0x00cafa8f00000000,
+    0x6f865f1400000000, 0x1e19cb6600000000, 0x71556efd00000000,
+    0x8187f08a00000000, 0xeecb551100000000, 0x6122cd6500000000,
+    0x0e6e68fe00000000, 0xfebcf68900000000, 0x91f0531200000000,
+    0x1c82df6c00000000, 0x73ce7af700000000, 0x831ce48000000000,
+    0xec50411b00000000, 0x63b9d96f00000000, 0x0cf57cf400000000,
+    0xfc27e28300000000, 0x936b471800000000, 0xe2f4d36a00000000,
+    0x8db876f100000000, 0x7d6ae88600000000, 0x12264d1d00000000,
+    0x9dcfd56900000000, 0xf28370f200000000, 0x0251ee8500000000,
+    0x6d1d4b1e00000000, 0x18b4f67800000000, 0x77f853e300000000,
+    0x872acd9400000000, 0xe866680f00000000, 0x678ff07b00000000,
+    0x08c355e000000000, 0xf811cb9700000000, 0x975d6e0c00000000,
+    0xe6c2fa7e00000000, 0x898e5fe500000000, 0x795cc19200000000,
+    0x1610640900000000, 0x99f9fc7d00000000, 0xf6b559e600000000,
+    0x0667c79100000000, 0x692b620a00000000, 0xe459ee7400000000,
+    0x8b154bef00000000, 0x7bc7d59800000000, 0x148b700300000000,
+    0x9b62e87700000000, 0xf42e4dec00000000, 0x04fcd39b00000000,
+    0x6bb0760000000000, 0x1a2fe27200000000, 0x756347e900000000,
+    0x85b1d99e00000000, 0xeafd7c0500000000, 0x6514e47100000000,
+    0x0a5841ea00000000, 0xfa8adf9d00000000, 0x95c67a0600000000,
+    0x10d8a45000000000, 0x7f9401cb00000000, 0x8f469fbc00000000,
+    0xe00a3a2700000000, 0x6fe3a25300000000, 0x00af07c800000000,
+    0xf07d99bf00000000, 0x9f313c2400000000, 0xeeaea85600000000,
+    0x81e20dcd00000000, 0x713093ba00000000, 0x1e7c362100000000,
+    0x9195ae5500000000, 0xfed90bce00000000, 0x0e0b95b900000000,
+    0x6147302200000000, 0xec35bc5c00000000, 0x837919c700000000,
+    0x73ab87b000000000, 0x1ce7222b00000000, 0x930eba5f00000000,
+    0xfc421fc400000000, 0x0c9081b300000000, 0x63dc242800000000,
+    0x1243b05a00000000, 0x7d0f15c100000000, 0x8ddd8bb600000000,
+    0xe2912e2d00000000, 0x6d78b65900000000, 0x023413c200000000,
+    0xf2e68db500000000, 0x9daa282e00000000, 0xe803954800000000,
+    0x874f30d300000000, 0x779daea400000000, 0x18d10b3f00000000,
+    0x9738934b00000000, 0xf87436d000000000, 0x08a6a8a700000000,
+    0x67ea0d3c00000000, 0x1675994e00000000, 0x79393cd500000000,
+    0x89eba2a200000000, 0xe6a7073900000000, 0x694e9f4d00000000,
+    0x06023ad600000000, 0xf6d0a4a100000000, 0x999c013a00000000,
+    0x14ee8d4400000000, 0x7ba228df00000000, 0x8b70b6a800000000,
+    0xe43c133300000000, 0x6bd58b4700000000, 0x04992edc00000000,
+    0xf44bb0ab00000000, 0x9b07153000000000, 0xea98814200000000,
+    0x85d424d900000000, 0x7506baae00000000, 0x1a4a1f3500000000,
+    0x95a3874100000000, 0xfaef22da00000000, 0x0a3dbcad00000000,
+    0x6571193600000000},
+   {0x0000000000000000, 0x85d996dd00000000, 0x4bb55c6000000000,
+    0xce6ccabd00000000, 0x966ab9c000000000, 0x13b32f1d00000000,
+    0xdddfe5a000000000, 0x5806737d00000000, 0x6dd3035a00000000,
+    0xe80a958700000000, 0x26665f3a00000000, 0xa3bfc9e700000000,
+    0xfbb9ba9a00000000, 0x7e602c4700000000, 0xb00ce6fa00000000,
+    0x35d5702700000000, 0xdaa607b400000000, 0x5f7f916900000000,
+    0x91135bd400000000, 0x14cacd0900000000, 0x4cccbe7400000000,
+    0xc91528a900000000, 0x0779e21400000000, 0x82a074c900000000,
+    0xb77504ee00000000, 0x32ac923300000000, 0xfcc0588e00000000,
+    0x7919ce5300000000, 0x211fbd2e00000000, 0xa4c62bf300000000,
+    0x6aaae14e00000000, 0xef73779300000000, 0xf54b7eb300000000,
+    0x7092e86e00000000, 0xbefe22d300000000, 0x3b27b40e00000000,
+    0x6321c77300000000, 0xe6f851ae00000000, 0x28949b1300000000,
+    0xad4d0dce00000000, 0x98987de900000000, 0x1d41eb3400000000,
+    0xd32d218900000000, 0x56f4b75400000000, 0x0ef2c42900000000,
+    0x8b2b52f400000000, 0x4547984900000000, 0xc09e0e9400000000,
+    0x2fed790700000000, 0xaa34efda00000000, 0x6458256700000000,
+    0xe181b3ba00000000, 0xb987c0c700000000, 0x3c5e561a00000000,
+    0xf2329ca700000000, 0x77eb0a7a00000000, 0x423e7a5d00000000,
+    0xc7e7ec8000000000, 0x098b263d00000000, 0x8c52b0e000000000,
+    0xd454c39d00000000, 0x518d554000000000, 0x9fe19ffd00000000,
+    0x1a38092000000000, 0xab918dbd00000000, 0x2e481b6000000000,
+    0xe024d1dd00000000, 0x65fd470000000000, 0x3dfb347d00000000,
+    0xb822a2a000000000, 0x764e681d00000000, 0xf397fec000000000,
+    0xc6428ee700000000, 0x439b183a00000000, 0x8df7d28700000000,
+    0x082e445a00000000, 0x5028372700000000, 0xd5f1a1fa00000000,
+    0x1b9d6b4700000000, 0x9e44fd9a00000000, 0x71378a0900000000,
+    0xf4ee1cd400000000, 0x3a82d66900000000, 0xbf5b40b400000000,
+    0xe75d33c900000000, 0x6284a51400000000, 0xace86fa900000000,
+    0x2931f97400000000, 0x1ce4895300000000, 0x993d1f8e00000000,
+    0x5751d53300000000, 0xd28843ee00000000, 0x8a8e309300000000,
+    0x0f57a64e00000000, 0xc13b6cf300000000, 0x44e2fa2e00000000,
+    0x5edaf30e00000000, 0xdb0365d300000000, 0x156faf6e00000000,
+    0x90b639b300000000, 0xc8b04ace00000000, 0x4d69dc1300000000,
+    0x830516ae00000000, 0x06dc807300000000, 0x3309f05400000000,
+    0xb6d0668900000000, 0x78bcac3400000000, 0xfd653ae900000000,
+    0xa563499400000000, 0x20badf4900000000, 0xeed615f400000000,
+    0x6b0f832900000000, 0x847cf4ba00000000, 0x01a5626700000000,
+    0xcfc9a8da00000000, 0x4a103e0700000000, 0x12164d7a00000000,
+    0x97cfdba700000000, 0x59a3111a00000000, 0xdc7a87c700000000,
+    0xe9aff7e000000000, 0x6c76613d00000000, 0xa21aab8000000000,
+    0x27c33d5d00000000, 0x7fc54e2000000000, 0xfa1cd8fd00000000,
+    0x3470124000000000, 0xb1a9849d00000000, 0x17256aa000000000,
+    0x92fcfc7d00000000, 0x5c9036c000000000, 0xd949a01d00000000,
+    0x814fd36000000000, 0x049645bd00000000, 0xcafa8f0000000000,
+    0x4f2319dd00000000, 0x7af669fa00000000, 0xff2fff2700000000,
+    0x3143359a00000000, 0xb49aa34700000000, 0xec9cd03a00000000,
+    0x694546e700000000, 0xa7298c5a00000000, 0x22f01a8700000000,
+    0xcd836d1400000000, 0x485afbc900000000, 0x8636317400000000,
+    0x03efa7a900000000, 0x5be9d4d400000000, 0xde30420900000000,
+    0x105c88b400000000, 0x95851e6900000000, 0xa0506e4e00000000,
+    0x2589f89300000000, 0xebe5322e00000000, 0x6e3ca4f300000000,
+    0x363ad78e00000000, 0xb3e3415300000000, 0x7d8f8bee00000000,
+    0xf8561d3300000000, 0xe26e141300000000, 0x67b782ce00000000,
+    0xa9db487300000000, 0x2c02deae00000000, 0x7404add300000000,
+    0xf1dd3b0e00000000, 0x3fb1f1b300000000, 0xba68676e00000000,
+    0x8fbd174900000000, 0x0a64819400000000, 0xc4084b2900000000,
+    0x41d1ddf400000000, 0x19d7ae8900000000, 0x9c0e385400000000,
+    0x5262f2e900000000, 0xd7bb643400000000, 0x38c813a700000000,
+    0xbd11857a00000000, 0x737d4fc700000000, 0xf6a4d91a00000000,
+    0xaea2aa6700000000, 0x2b7b3cba00000000, 0xe517f60700000000,
+    0x60ce60da00000000, 0x551b10fd00000000, 0xd0c2862000000000,
+    0x1eae4c9d00000000, 0x9b77da4000000000, 0xc371a93d00000000,
+    0x46a83fe000000000, 0x88c4f55d00000000, 0x0d1d638000000000,
+    0xbcb4e71d00000000, 0x396d71c000000000, 0xf701bb7d00000000,
+    0x72d82da000000000, 0x2ade5edd00000000, 0xaf07c80000000000,
+    0x616b02bd00000000, 0xe4b2946000000000, 0xd167e44700000000,
+    0x54be729a00000000, 0x9ad2b82700000000, 0x1f0b2efa00000000,
+    0x470d5d8700000000, 0xc2d4cb5a00000000, 0x0cb801e700000000,
+    0x8961973a00000000, 0x6612e0a900000000, 0xe3cb767400000000,
+    0x2da7bcc900000000, 0xa87e2a1400000000, 0xf078596900000000,
+    0x75a1cfb400000000, 0xbbcd050900000000, 0x3e1493d400000000,
+    0x0bc1e3f300000000, 0x8e18752e00000000, 0x4074bf9300000000,
+    0xc5ad294e00000000, 0x9dab5a3300000000, 0x1872ccee00000000,
+    0xd61e065300000000, 0x53c7908e00000000, 0x49ff99ae00000000,
+    0xcc260f7300000000, 0x024ac5ce00000000, 0x8793531300000000,
+    0xdf95206e00000000, 0x5a4cb6b300000000, 0x94207c0e00000000,
+    0x11f9ead300000000, 0x242c9af400000000, 0xa1f50c2900000000,
+    0x6f99c69400000000, 0xea40504900000000, 0xb246233400000000,
+    0x379fb5e900000000, 0xf9f37f5400000000, 0x7c2ae98900000000,
+    0x93599e1a00000000, 0x168008c700000000, 0xd8ecc27a00000000,
+    0x5d3554a700000000, 0x053327da00000000, 0x80eab10700000000,
+    0x4e867bba00000000, 0xcb5fed6700000000, 0xfe8a9d4000000000,
+    0x7b530b9d00000000, 0xb53fc12000000000, 0x30e657fd00000000,
+    0x68e0248000000000, 0xed39b25d00000000, 0x235578e000000000,
+    0xa68cee3d00000000},
+   {0x0000000000000000, 0x76e10f9d00000000, 0xadc46ee100000000,
+    0xdb25617c00000000, 0x1b8fac1900000000, 0x6d6ea38400000000,
+    0xb64bc2f800000000, 0xc0aacd6500000000, 0x361e593300000000,
+    0x40ff56ae00000000, 0x9bda37d200000000, 0xed3b384f00000000,
+    0x2d91f52a00000000, 0x5b70fab700000000, 0x80559bcb00000000,
+    0xf6b4945600000000, 0x6c3cb26600000000, 0x1addbdfb00000000,
+    0xc1f8dc8700000000, 0xb719d31a00000000, 0x77b31e7f00000000,
+    0x015211e200000000, 0xda77709e00000000, 0xac967f0300000000,
+    0x5a22eb5500000000, 0x2cc3e4c800000000, 0xf7e685b400000000,
+    0x81078a2900000000, 0x41ad474c00000000, 0x374c48d100000000,
+    0xec6929ad00000000, 0x9a88263000000000, 0xd87864cd00000000,
+    0xae996b5000000000, 0x75bc0a2c00000000, 0x035d05b100000000,
+    0xc3f7c8d400000000, 0xb516c74900000000, 0x6e33a63500000000,
+    0x18d2a9a800000000, 0xee663dfe00000000, 0x9887326300000000,
+    0x43a2531f00000000, 0x35435c8200000000, 0xf5e991e700000000,
+    0x83089e7a00000000, 0x582dff0600000000, 0x2eccf09b00000000,
+    0xb444d6ab00000000, 0xc2a5d93600000000, 0x1980b84a00000000,
+    0x6f61b7d700000000, 0xafcb7ab200000000, 0xd92a752f00000000,
+    0x020f145300000000, 0x74ee1bce00000000, 0x825a8f9800000000,
+    0xf4bb800500000000, 0x2f9ee17900000000, 0x597feee400000000,
+    0x99d5238100000000, 0xef342c1c00000000, 0x34114d6000000000,
+    0x42f042fd00000000, 0xf1f7b94100000000, 0x8716b6dc00000000,
+    0x5c33d7a000000000, 0x2ad2d83d00000000, 0xea78155800000000,
+    0x9c991ac500000000, 0x47bc7bb900000000, 0x315d742400000000,
+    0xc7e9e07200000000, 0xb108efef00000000, 0x6a2d8e9300000000,
+    0x1ccc810e00000000, 0xdc664c6b00000000, 0xaa8743f600000000,
+    0x71a2228a00000000, 0x07432d1700000000, 0x9dcb0b2700000000,
+    0xeb2a04ba00000000, 0x300f65c600000000, 0x46ee6a5b00000000,
+    0x8644a73e00000000, 0xf0a5a8a300000000, 0x2b80c9df00000000,
+    0x5d61c64200000000, 0xabd5521400000000, 0xdd345d8900000000,
+    0x06113cf500000000, 0x70f0336800000000, 0xb05afe0d00000000,
+    0xc6bbf19000000000, 0x1d9e90ec00000000, 0x6b7f9f7100000000,
+    0x298fdd8c00000000, 0x5f6ed21100000000, 0x844bb36d00000000,
+    0xf2aabcf000000000, 0x3200719500000000, 0x44e17e0800000000,
+    0x9fc41f7400000000, 0xe92510e900000000, 0x1f9184bf00000000,
+    0x69708b2200000000, 0xb255ea5e00000000, 0xc4b4e5c300000000,
+    0x041e28a600000000, 0x72ff273b00000000, 0xa9da464700000000,
+    0xdf3b49da00000000, 0x45b36fea00000000, 0x3352607700000000,
+    0xe877010b00000000, 0x9e960e9600000000, 0x5e3cc3f300000000,
+    0x28ddcc6e00000000, 0xf3f8ad1200000000, 0x8519a28f00000000,
+    0x73ad36d900000000, 0x054c394400000000, 0xde69583800000000,
+    0xa88857a500000000, 0x68229ac000000000, 0x1ec3955d00000000,
+    0xc5e6f42100000000, 0xb307fbbc00000000, 0xe2ef738300000000,
+    0x940e7c1e00000000, 0x4f2b1d6200000000, 0x39ca12ff00000000,
+    0xf960df9a00000000, 0x8f81d00700000000, 0x54a4b17b00000000,
+    0x2245bee600000000, 0xd4f12ab000000000, 0xa210252d00000000,
+    0x7935445100000000, 0x0fd44bcc00000000, 0xcf7e86a900000000,
+    0xb99f893400000000, 0x62bae84800000000, 0x145be7d500000000,
+    0x8ed3c1e500000000, 0xf832ce7800000000, 0x2317af0400000000,
+    0x55f6a09900000000, 0x955c6dfc00000000, 0xe3bd626100000000,
+    0x3898031d00000000, 0x4e790c8000000000, 0xb8cd98d600000000,
+    0xce2c974b00000000, 0x1509f63700000000, 0x63e8f9aa00000000,
+    0xa34234cf00000000, 0xd5a33b5200000000, 0x0e865a2e00000000,
+    0x786755b300000000, 0x3a97174e00000000, 0x4c7618d300000000,
+    0x975379af00000000, 0xe1b2763200000000, 0x2118bb5700000000,
+    0x57f9b4ca00000000, 0x8cdcd5b600000000, 0xfa3dda2b00000000,
+    0x0c894e7d00000000, 0x7a6841e000000000, 0xa14d209c00000000,
+    0xd7ac2f0100000000, 0x1706e26400000000, 0x61e7edf900000000,
+    0xbac28c8500000000, 0xcc23831800000000, 0x56aba52800000000,
+    0x204aaab500000000, 0xfb6fcbc900000000, 0x8d8ec45400000000,
+    0x4d24093100000000, 0x3bc506ac00000000, 0xe0e067d000000000,
+    0x9601684d00000000, 0x60b5fc1b00000000, 0x1654f38600000000,
+    0xcd7192fa00000000, 0xbb909d6700000000, 0x7b3a500200000000,
+    0x0ddb5f9f00000000, 0xd6fe3ee300000000, 0xa01f317e00000000,
+    0x1318cac200000000, 0x65f9c55f00000000, 0xbedca42300000000,
+    0xc83dabbe00000000, 0x089766db00000000, 0x7e76694600000000,
+    0xa553083a00000000, 0xd3b207a700000000, 0x250693f100000000,
+    0x53e79c6c00000000, 0x88c2fd1000000000, 0xfe23f28d00000000,
+    0x3e893fe800000000, 0x4868307500000000, 0x934d510900000000,
+    0xe5ac5e9400000000, 0x7f2478a400000000, 0x09c5773900000000,
+    0xd2e0164500000000, 0xa40119d800000000, 0x64abd4bd00000000,
+    0x124adb2000000000, 0xc96fba5c00000000, 0xbf8eb5c100000000,
+    0x493a219700000000, 0x3fdb2e0a00000000, 0xe4fe4f7600000000,
+    0x921f40eb00000000, 0x52b58d8e00000000, 0x2454821300000000,
+    0xff71e36f00000000, 0x8990ecf200000000, 0xcb60ae0f00000000,
+    0xbd81a19200000000, 0x66a4c0ee00000000, 0x1045cf7300000000,
+    0xd0ef021600000000, 0xa60e0d8b00000000, 0x7d2b6cf700000000,
+    0x0bca636a00000000, 0xfd7ef73c00000000, 0x8b9ff8a100000000,
+    0x50ba99dd00000000, 0x265b964000000000, 0xe6f15b2500000000,
+    0x901054b800000000, 0x4b3535c400000000, 0x3dd43a5900000000,
+    0xa75c1c6900000000, 0xd1bd13f400000000, 0x0a98728800000000,
+    0x7c797d1500000000, 0xbcd3b07000000000, 0xca32bfed00000000,
+    0x1117de9100000000, 0x67f6d10c00000000, 0x9142455a00000000,
+    0xe7a34ac700000000, 0x3c862bbb00000000, 0x4a67242600000000,
+    0x8acde94300000000, 0xfc2ce6de00000000, 0x270987a200000000,
+    0x51e8883f00000000},
+   {0x0000000000000000, 0xe8dbfbb900000000, 0x91b186a800000000,
+    0x796a7d1100000000, 0x63657c8a00000000, 0x8bbe873300000000,
+    0xf2d4fa2200000000, 0x1a0f019b00000000, 0x87cc89cf00000000,
+    0x6f17727600000000, 0x167d0f6700000000, 0xfea6f4de00000000,
+    0xe4a9f54500000000, 0x0c720efc00000000, 0x751873ed00000000,
+    0x9dc3885400000000, 0x4f9f624400000000, 0xa74499fd00000000,
+    0xde2ee4ec00000000, 0x36f51f5500000000, 0x2cfa1ece00000000,
+    0xc421e57700000000, 0xbd4b986600000000, 0x559063df00000000,
+    0xc853eb8b00000000, 0x2088103200000000, 0x59e26d2300000000,
+    0xb139969a00000000, 0xab36970100000000, 0x43ed6cb800000000,
+    0x3a8711a900000000, 0xd25cea1000000000, 0x9e3ec58800000000,
+    0x76e53e3100000000, 0x0f8f432000000000, 0xe754b89900000000,
+    0xfd5bb90200000000, 0x158042bb00000000, 0x6cea3faa00000000,
+    0x8431c41300000000, 0x19f24c4700000000, 0xf129b7fe00000000,
+    0x8843caef00000000, 0x6098315600000000, 0x7a9730cd00000000,
+    0x924ccb7400000000, 0xeb26b66500000000, 0x03fd4ddc00000000,
+    0xd1a1a7cc00000000, 0x397a5c7500000000, 0x4010216400000000,
+    0xa8cbdadd00000000, 0xb2c4db4600000000, 0x5a1f20ff00000000,
+    0x23755dee00000000, 0xcbaea65700000000, 0x566d2e0300000000,
+    0xbeb6d5ba00000000, 0xc7dca8ab00000000, 0x2f07531200000000,
+    0x3508528900000000, 0xddd3a93000000000, 0xa4b9d42100000000,
+    0x4c622f9800000000, 0x7d7bfbca00000000, 0x95a0007300000000,
+    0xecca7d6200000000, 0x041186db00000000, 0x1e1e874000000000,
+    0xf6c57cf900000000, 0x8faf01e800000000, 0x6774fa5100000000,
+    0xfab7720500000000, 0x126c89bc00000000, 0x6b06f4ad00000000,
+    0x83dd0f1400000000, 0x99d20e8f00000000, 0x7109f53600000000,
+    0x0863882700000000, 0xe0b8739e00000000, 0x32e4998e00000000,
+    0xda3f623700000000, 0xa3551f2600000000, 0x4b8ee49f00000000,
+    0x5181e50400000000, 0xb95a1ebd00000000, 0xc03063ac00000000,
+    0x28eb981500000000, 0xb528104100000000, 0x5df3ebf800000000,
+    0x249996e900000000, 0xcc426d5000000000, 0xd64d6ccb00000000,
+    0x3e96977200000000, 0x47fcea6300000000, 0xaf2711da00000000,
+    0xe3453e4200000000, 0x0b9ec5fb00000000, 0x72f4b8ea00000000,
+    0x9a2f435300000000, 0x802042c800000000, 0x68fbb97100000000,
+    0x1191c46000000000, 0xf94a3fd900000000, 0x6489b78d00000000,
+    0x8c524c3400000000, 0xf538312500000000, 0x1de3ca9c00000000,
+    0x07eccb0700000000, 0xef3730be00000000, 0x965d4daf00000000,
+    0x7e86b61600000000, 0xacda5c0600000000, 0x4401a7bf00000000,
+    0x3d6bdaae00000000, 0xd5b0211700000000, 0xcfbf208c00000000,
+    0x2764db3500000000, 0x5e0ea62400000000, 0xb6d55d9d00000000,
+    0x2b16d5c900000000, 0xc3cd2e7000000000, 0xbaa7536100000000,
+    0x527ca8d800000000, 0x4873a94300000000, 0xa0a852fa00000000,
+    0xd9c22feb00000000, 0x3119d45200000000, 0xbbf0874e00000000,
+    0x532b7cf700000000, 0x2a4101e600000000, 0xc29afa5f00000000,
+    0xd895fbc400000000, 0x304e007d00000000, 0x49247d6c00000000,
+    0xa1ff86d500000000, 0x3c3c0e8100000000, 0xd4e7f53800000000,
+    0xad8d882900000000, 0x4556739000000000, 0x5f59720b00000000,
+    0xb78289b200000000, 0xcee8f4a300000000, 0x26330f1a00000000,
+    0xf46fe50a00000000, 0x1cb41eb300000000, 0x65de63a200000000,
+    0x8d05981b00000000, 0x970a998000000000, 0x7fd1623900000000,
+    0x06bb1f2800000000, 0xee60e49100000000, 0x73a36cc500000000,
+    0x9b78977c00000000, 0xe212ea6d00000000, 0x0ac911d400000000,
+    0x10c6104f00000000, 0xf81debf600000000, 0x817796e700000000,
+    0x69ac6d5e00000000, 0x25ce42c600000000, 0xcd15b97f00000000,
+    0xb47fc46e00000000, 0x5ca43fd700000000, 0x46ab3e4c00000000,
+    0xae70c5f500000000, 0xd71ab8e400000000, 0x3fc1435d00000000,
+    0xa202cb0900000000, 0x4ad930b000000000, 0x33b34da100000000,
+    0xdb68b61800000000, 0xc167b78300000000, 0x29bc4c3a00000000,
+    0x50d6312b00000000, 0xb80dca9200000000, 0x6a51208200000000,
+    0x828adb3b00000000, 0xfbe0a62a00000000, 0x133b5d9300000000,
+    0x09345c0800000000, 0xe1efa7b100000000, 0x9885daa000000000,
+    0x705e211900000000, 0xed9da94d00000000, 0x054652f400000000,
+    0x7c2c2fe500000000, 0x94f7d45c00000000, 0x8ef8d5c700000000,
+    0x66232e7e00000000, 0x1f49536f00000000, 0xf792a8d600000000,
+    0xc68b7c8400000000, 0x2e50873d00000000, 0x573afa2c00000000,
+    0xbfe1019500000000, 0xa5ee000e00000000, 0x4d35fbb700000000,
+    0x345f86a600000000, 0xdc847d1f00000000, 0x4147f54b00000000,
+    0xa99c0ef200000000, 0xd0f673e300000000, 0x382d885a00000000,
+    0x222289c100000000, 0xcaf9727800000000, 0xb3930f6900000000,
+    0x5b48f4d000000000, 0x89141ec000000000, 0x61cfe57900000000,
+    0x18a5986800000000, 0xf07e63d100000000, 0xea71624a00000000,
+    0x02aa99f300000000, 0x7bc0e4e200000000, 0x931b1f5b00000000,
+    0x0ed8970f00000000, 0xe6036cb600000000, 0x9f6911a700000000,
+    0x77b2ea1e00000000, 0x6dbdeb8500000000, 0x8566103c00000000,
+    0xfc0c6d2d00000000, 0x14d7969400000000, 0x58b5b90c00000000,
+    0xb06e42b500000000, 0xc9043fa400000000, 0x21dfc41d00000000,
+    0x3bd0c58600000000, 0xd30b3e3f00000000, 0xaa61432e00000000,
+    0x42bab89700000000, 0xdf7930c300000000, 0x37a2cb7a00000000,
+    0x4ec8b66b00000000, 0xa6134dd200000000, 0xbc1c4c4900000000,
+    0x54c7b7f000000000, 0x2dadcae100000000, 0xc576315800000000,
+    0x172adb4800000000, 0xfff120f100000000, 0x869b5de000000000,
+    0x6e40a65900000000, 0x744fa7c200000000, 0x9c945c7b00000000,
+    0xe5fe216a00000000, 0x0d25dad300000000, 0x90e6528700000000,
+    0x783da93e00000000, 0x0157d42f00000000, 0xe98c2f9600000000,
+    0xf3832e0d00000000, 0x1b58d5b400000000, 0x6232a8a500000000,
+    0x8ae9531c00000000},
+   {0x0000000000000000, 0x919168ae00000000, 0x6325a08700000000,
+    0xf2b4c82900000000, 0x874c31d400000000, 0x16dd597a00000000,
+    0xe469915300000000, 0x75f8f9fd00000000, 0x4f9f137300000000,
+    0xde0e7bdd00000000, 0x2cbab3f400000000, 0xbd2bdb5a00000000,
+    0xc8d322a700000000, 0x59424a0900000000, 0xabf6822000000000,
+    0x3a67ea8e00000000, 0x9e3e27e600000000, 0x0faf4f4800000000,
+    0xfd1b876100000000, 0x6c8aefcf00000000, 0x1972163200000000,
+    0x88e37e9c00000000, 0x7a57b6b500000000, 0xebc6de1b00000000,
+    0xd1a1349500000000, 0x40305c3b00000000, 0xb284941200000000,
+    0x2315fcbc00000000, 0x56ed054100000000, 0xc77c6def00000000,
+    0x35c8a5c600000000, 0xa459cd6800000000, 0x7d7b3f1700000000,
+    0xecea57b900000000, 0x1e5e9f9000000000, 0x8fcff73e00000000,
+    0xfa370ec300000000, 0x6ba6666d00000000, 0x9912ae4400000000,
+    0x0883c6ea00000000, 0x32e42c6400000000, 0xa37544ca00000000,
+    0x51c18ce300000000, 0xc050e44d00000000, 0xb5a81db000000000,
+    0x2439751e00000000, 0xd68dbd3700000000, 0x471cd59900000000,
+    0xe34518f100000000, 0x72d4705f00000000, 0x8060b87600000000,
+    0x11f1d0d800000000, 0x6409292500000000, 0xf598418b00000000,
+    0x072c89a200000000, 0x96bde10c00000000, 0xacda0b8200000000,
+    0x3d4b632c00000000, 0xcfffab0500000000, 0x5e6ec3ab00000000,
+    0x2b963a5600000000, 0xba0752f800000000, 0x48b39ad100000000,
+    0xd922f27f00000000, 0xfaf67e2e00000000, 0x6b67168000000000,
+    0x99d3dea900000000, 0x0842b60700000000, 0x7dba4ffa00000000,
+    0xec2b275400000000, 0x1e9fef7d00000000, 0x8f0e87d300000000,
+    0xb5696d5d00000000, 0x24f805f300000000, 0xd64ccdda00000000,
+    0x47dda57400000000, 0x32255c8900000000, 0xa3b4342700000000,
+    0x5100fc0e00000000, 0xc09194a000000000, 0x64c859c800000000,
+    0xf559316600000000, 0x07edf94f00000000, 0x967c91e100000000,
+    0xe384681c00000000, 0x721500b200000000, 0x80a1c89b00000000,
+    0x1130a03500000000, 0x2b574abb00000000, 0xbac6221500000000,
+    0x4872ea3c00000000, 0xd9e3829200000000, 0xac1b7b6f00000000,
+    0x3d8a13c100000000, 0xcf3edbe800000000, 0x5eafb34600000000,
+    0x878d413900000000, 0x161c299700000000, 0xe4a8e1be00000000,
+    0x7539891000000000, 0x00c170ed00000000, 0x9150184300000000,
+    0x63e4d06a00000000, 0xf275b8c400000000, 0xc812524a00000000,
+    0x59833ae400000000, 0xab37f2cd00000000, 0x3aa69a6300000000,
+    0x4f5e639e00000000, 0xdecf0b3000000000, 0x2c7bc31900000000,
+    0xbdeaabb700000000, 0x19b366df00000000, 0x88220e7100000000,
+    0x7a96c65800000000, 0xeb07aef600000000, 0x9eff570b00000000,
+    0x0f6e3fa500000000, 0xfddaf78c00000000, 0x6c4b9f2200000000,
+    0x562c75ac00000000, 0xc7bd1d0200000000, 0x3509d52b00000000,
+    0xa498bd8500000000, 0xd160447800000000, 0x40f12cd600000000,
+    0xb245e4ff00000000, 0x23d48c5100000000, 0xf4edfd5c00000000,
+    0x657c95f200000000, 0x97c85ddb00000000, 0x0659357500000000,
+    0x73a1cc8800000000, 0xe230a42600000000, 0x10846c0f00000000,
+    0x811504a100000000, 0xbb72ee2f00000000, 0x2ae3868100000000,
+    0xd8574ea800000000, 0x49c6260600000000, 0x3c3edffb00000000,
+    0xadafb75500000000, 0x5f1b7f7c00000000, 0xce8a17d200000000,
+    0x6ad3daba00000000, 0xfb42b21400000000, 0x09f67a3d00000000,
+    0x9867129300000000, 0xed9feb6e00000000, 0x7c0e83c000000000,
+    0x8eba4be900000000, 0x1f2b234700000000, 0x254cc9c900000000,
+    0xb4dda16700000000, 0x4669694e00000000, 0xd7f801e000000000,
+    0xa200f81d00000000, 0x339190b300000000, 0xc125589a00000000,
+    0x50b4303400000000, 0x8996c24b00000000, 0x1807aae500000000,
+    0xeab362cc00000000, 0x7b220a6200000000, 0x0edaf39f00000000,
+    0x9f4b9b3100000000, 0x6dff531800000000, 0xfc6e3bb600000000,
+    0xc609d13800000000, 0x5798b99600000000, 0xa52c71bf00000000,
+    0x34bd191100000000, 0x4145e0ec00000000, 0xd0d4884200000000,
+    0x2260406b00000000, 0xb3f128c500000000, 0x17a8e5ad00000000,
+    0x86398d0300000000, 0x748d452a00000000, 0xe51c2d8400000000,
+    0x90e4d47900000000, 0x0175bcd700000000, 0xf3c174fe00000000,
+    0x62501c5000000000, 0x5837f6de00000000, 0xc9a69e7000000000,
+    0x3b12565900000000, 0xaa833ef700000000, 0xdf7bc70a00000000,
+    0x4eeaafa400000000, 0xbc5e678d00000000, 0x2dcf0f2300000000,
+    0x0e1b837200000000, 0x9f8aebdc00000000, 0x6d3e23f500000000,
+    0xfcaf4b5b00000000, 0x8957b2a600000000, 0x18c6da0800000000,
+    0xea72122100000000, 0x7be37a8f00000000, 0x4184900100000000,
+    0xd015f8af00000000, 0x22a1308600000000, 0xb330582800000000,
+    0xc6c8a1d500000000, 0x5759c97b00000000, 0xa5ed015200000000,
+    0x347c69fc00000000, 0x9025a49400000000, 0x01b4cc3a00000000,
+    0xf300041300000000, 0x62916cbd00000000, 0x1769954000000000,
+    0x86f8fdee00000000, 0x744c35c700000000, 0xe5dd5d6900000000,
+    0xdfbab7e700000000, 0x4e2bdf4900000000, 0xbc9f176000000000,
+    0x2d0e7fce00000000, 0x58f6863300000000, 0xc967ee9d00000000,
+    0x3bd326b400000000, 0xaa424e1a00000000, 0x7360bc6500000000,
+    0xe2f1d4cb00000000, 0x10451ce200000000, 0x81d4744c00000000,
+    0xf42c8db100000000, 0x65bde51f00000000, 0x97092d3600000000,
+    0x0698459800000000, 0x3cffaf1600000000, 0xad6ec7b800000000,
+    0x5fda0f9100000000, 0xce4b673f00000000, 0xbbb39ec200000000,
+    0x2a22f66c00000000, 0xd8963e4500000000, 0x490756eb00000000,
+    0xed5e9b8300000000, 0x7ccff32d00000000, 0x8e7b3b0400000000,
+    0x1fea53aa00000000, 0x6a12aa5700000000, 0xfb83c2f900000000,
+    0x09370ad000000000, 0x98a6627e00000000, 0xa2c188f000000000,
+    0x3350e05e00000000, 0xc1e4287700000000, 0x507540d900000000,
+    0x258db92400000000, 0xb41cd18a00000000, 0x46a819a300000000,
+    0xd739710d00000000}};
+
+#else /* W == 4 */
+
+local const z_crc_t FAR crc_braid_table[][256] = {
+   {0x00000000, 0xccaa009e, 0x4225077d, 0x8e8f07e3, 0x844a0efa,
+    0x48e00e64, 0xc66f0987, 0x0ac50919, 0xd3e51bb5, 0x1f4f1b2b,
+    0x91c01cc8, 0x5d6a1c56, 0x57af154f, 0x9b0515d1, 0x158a1232,
+    0xd92012ac, 0x7cbb312b, 0xb01131b5, 0x3e9e3656, 0xf23436c8,
+    0xf8f13fd1, 0x345b3f4f, 0xbad438ac, 0x767e3832, 0xaf5e2a9e,
+    0x63f42a00, 0xed7b2de3, 0x21d12d7d, 0x2b142464, 0xe7be24fa,
+    0x69312319, 0xa59b2387, 0xf9766256, 0x35dc62c8, 0xbb53652b,
+    0x77f965b5, 0x7d3c6cac, 0xb1966c32, 0x3f196bd1, 0xf3b36b4f,
+    0x2a9379e3, 0xe639797d, 0x68b67e9e, 0xa41c7e00, 0xaed97719,
+    0x62737787, 0xecfc7064, 0x205670fa, 0x85cd537d, 0x496753e3,
+    0xc7e85400, 0x0b42549e, 0x01875d87, 0xcd2d5d19, 0x43a25afa,
+    0x8f085a64, 0x562848c8, 0x9a824856, 0x140d4fb5, 0xd8a74f2b,
+    0xd2624632, 0x1ec846ac, 0x9047414f, 0x5ced41d1, 0x299dc2ed,
+    0xe537c273, 0x6bb8c590, 0xa712c50e, 0xadd7cc17, 0x617dcc89,
+    0xeff2cb6a, 0x2358cbf4, 0xfa78d958, 0x36d2d9c6, 0xb85dde25,
+    0x74f7debb, 0x7e32d7a2, 0xb298d73c, 0x3c17d0df, 0xf0bdd041,
+    0x5526f3c6, 0x998cf358, 0x1703f4bb, 0xdba9f425, 0xd16cfd3c,
+    0x1dc6fda2, 0x9349fa41, 0x5fe3fadf, 0x86c3e873, 0x4a69e8ed,
+    0xc4e6ef0e, 0x084cef90, 0x0289e689, 0xce23e617, 0x40ace1f4,
+    0x8c06e16a, 0xd0eba0bb, 0x1c41a025, 0x92cea7c6, 0x5e64a758,
+    0x54a1ae41, 0x980baedf, 0x1684a93c, 0xda2ea9a2, 0x030ebb0e,
+    0xcfa4bb90, 0x412bbc73, 0x8d81bced, 0x8744b5f4, 0x4beeb56a,
+    0xc561b289, 0x09cbb217, 0xac509190, 0x60fa910e, 0xee7596ed,
+    0x22df9673, 0x281a9f6a, 0xe4b09ff4, 0x6a3f9817, 0xa6959889,
+    0x7fb58a25, 0xb31f8abb, 0x3d908d58, 0xf13a8dc6, 0xfbff84df,
+    0x37558441, 0xb9da83a2, 0x7570833c, 0x533b85da, 0x9f918544,
+    0x111e82a7, 0xddb48239, 0xd7718b20, 0x1bdb8bbe, 0x95548c5d,
+    0x59fe8cc3, 0x80de9e6f, 0x4c749ef1, 0xc2fb9912, 0x0e51998c,
+    0x04949095, 0xc83e900b, 0x46b197e8, 0x8a1b9776, 0x2f80b4f1,
+    0xe32ab46f, 0x6da5b38c, 0xa10fb312, 0xabcaba0b, 0x6760ba95,
+    0xe9efbd76, 0x2545bde8, 0xfc65af44, 0x30cfafda, 0xbe40a839,
+    0x72eaa8a7, 0x782fa1be, 0xb485a120, 0x3a0aa6c3, 0xf6a0a65d,
+    0xaa4de78c, 0x66e7e712, 0xe868e0f1, 0x24c2e06f, 0x2e07e976,
+    0xe2ade9e8, 0x6c22ee0b, 0xa088ee95, 0x79a8fc39, 0xb502fca7,
+    0x3b8dfb44, 0xf727fbda, 0xfde2f2c3, 0x3148f25d, 0xbfc7f5be,
+    0x736df520, 0xd6f6d6a7, 0x1a5cd639, 0x94d3d1da, 0x5879d144,
+    0x52bcd85d, 0x9e16d8c3, 0x1099df20, 0xdc33dfbe, 0x0513cd12,
+    0xc9b9cd8c, 0x4736ca6f, 0x8b9ccaf1, 0x8159c3e8, 0x4df3c376,
+    0xc37cc495, 0x0fd6c40b, 0x7aa64737, 0xb60c47a9, 0x3883404a,
+    0xf42940d4, 0xfeec49cd, 0x32464953, 0xbcc94eb0, 0x70634e2e,
+    0xa9435c82, 0x65e95c1c, 0xeb665bff, 0x27cc5b61, 0x2d095278,
+    0xe1a352e6, 0x6f2c5505, 0xa386559b, 0x061d761c, 0xcab77682,
+    0x44387161, 0x889271ff, 0x825778e6, 0x4efd7878, 0xc0727f9b,
+    0x0cd87f05, 0xd5f86da9, 0x19526d37, 0x97dd6ad4, 0x5b776a4a,
+    0x51b26353, 0x9d1863cd, 0x1397642e, 0xdf3d64b0, 0x83d02561,
+    0x4f7a25ff, 0xc1f5221c, 0x0d5f2282, 0x079a2b9b, 0xcb302b05,
+    0x45bf2ce6, 0x89152c78, 0x50353ed4, 0x9c9f3e4a, 0x121039a9,
+    0xdeba3937, 0xd47f302e, 0x18d530b0, 0x965a3753, 0x5af037cd,
+    0xff6b144a, 0x33c114d4, 0xbd4e1337, 0x71e413a9, 0x7b211ab0,
+    0xb78b1a2e, 0x39041dcd, 0xf5ae1d53, 0x2c8e0fff, 0xe0240f61,
+    0x6eab0882, 0xa201081c, 0xa8c40105, 0x646e019b, 0xeae10678,
+    0x264b06e6},
+   {0x00000000, 0xa6770bb4, 0x979f1129, 0x31e81a9d, 0xf44f2413,
+    0x52382fa7, 0x63d0353a, 0xc5a73e8e, 0x33ef4e67, 0x959845d3,
+    0xa4705f4e, 0x020754fa, 0xc7a06a74, 0x61d761c0, 0x503f7b5d,
+    0xf64870e9, 0x67de9cce, 0xc1a9977a, 0xf0418de7, 0x56368653,
+    0x9391b8dd, 0x35e6b369, 0x040ea9f4, 0xa279a240, 0x5431d2a9,
+    0xf246d91d, 0xc3aec380, 0x65d9c834, 0xa07ef6ba, 0x0609fd0e,
+    0x37e1e793, 0x9196ec27, 0xcfbd399c, 0x69ca3228, 0x582228b5,
+    0xfe552301, 0x3bf21d8f, 0x9d85163b, 0xac6d0ca6, 0x0a1a0712,
+    0xfc5277fb, 0x5a257c4f, 0x6bcd66d2, 0xcdba6d66, 0x081d53e8,
+    0xae6a585c, 0x9f8242c1, 0x39f54975, 0xa863a552, 0x0e14aee6,
+    0x3ffcb47b, 0x998bbfcf, 0x5c2c8141, 0xfa5b8af5, 0xcbb39068,
+    0x6dc49bdc, 0x9b8ceb35, 0x3dfbe081, 0x0c13fa1c, 0xaa64f1a8,
+    0x6fc3cf26, 0xc9b4c492, 0xf85cde0f, 0x5e2bd5bb, 0x440b7579,
+    0xe27c7ecd, 0xd3946450, 0x75e36fe4, 0xb044516a, 0x16335ade,
+    0x27db4043, 0x81ac4bf7, 0x77e43b1e, 0xd19330aa, 0xe07b2a37,
+    0x460c2183, 0x83ab1f0d, 0x25dc14b9, 0x14340e24, 0xb2430590,
+    0x23d5e9b7, 0x85a2e203, 0xb44af89e, 0x123df32a, 0xd79acda4,
+    0x71edc610, 0x4005dc8d, 0xe672d739, 0x103aa7d0, 0xb64dac64,
+    0x87a5b6f9, 0x21d2bd4d, 0xe47583c3, 0x42028877, 0x73ea92ea,
+    0xd59d995e, 0x8bb64ce5, 0x2dc14751, 0x1c295dcc, 0xba5e5678,
+    0x7ff968f6, 0xd98e6342, 0xe86679df, 0x4e11726b, 0xb8590282,
+    0x1e2e0936, 0x2fc613ab, 0x89b1181f, 0x4c162691, 0xea612d25,
+    0xdb8937b8, 0x7dfe3c0c, 0xec68d02b, 0x4a1fdb9f, 0x7bf7c102,
+    0xdd80cab6, 0x1827f438, 0xbe50ff8c, 0x8fb8e511, 0x29cfeea5,
+    0xdf879e4c, 0x79f095f8, 0x48188f65, 0xee6f84d1, 0x2bc8ba5f,
+    0x8dbfb1eb, 0xbc57ab76, 0x1a20a0c2, 0x8816eaf2, 0x2e61e146,
+    0x1f89fbdb, 0xb9fef06f, 0x7c59cee1, 0xda2ec555, 0xebc6dfc8,
+    0x4db1d47c, 0xbbf9a495, 0x1d8eaf21, 0x2c66b5bc, 0x8a11be08,
+    0x4fb68086, 0xe9c18b32, 0xd82991af, 0x7e5e9a1b, 0xefc8763c,
+    0x49bf7d88, 0x78576715, 0xde206ca1, 0x1b87522f, 0xbdf0599b,
+    0x8c184306, 0x2a6f48b2, 0xdc27385b, 0x7a5033ef, 0x4bb82972,
+    0xedcf22c6, 0x28681c48, 0x8e1f17fc, 0xbff70d61, 0x198006d5,
+    0x47abd36e, 0xe1dcd8da, 0xd034c247, 0x7643c9f3, 0xb3e4f77d,
+    0x1593fcc9, 0x247be654, 0x820cede0, 0x74449d09, 0xd23396bd,
+    0xe3db8c20, 0x45ac8794, 0x800bb91a, 0x267cb2ae, 0x1794a833,
+    0xb1e3a387, 0x20754fa0, 0x86024414, 0xb7ea5e89, 0x119d553d,
+    0xd43a6bb3, 0x724d6007, 0x43a57a9a, 0xe5d2712e, 0x139a01c7,
+    0xb5ed0a73, 0x840510ee, 0x22721b5a, 0xe7d525d4, 0x41a22e60,
+    0x704a34fd, 0xd63d3f49, 0xcc1d9f8b, 0x6a6a943f, 0x5b828ea2,
+    0xfdf58516, 0x3852bb98, 0x9e25b02c, 0xafcdaab1, 0x09baa105,
+    0xfff2d1ec, 0x5985da58, 0x686dc0c5, 0xce1acb71, 0x0bbdf5ff,
+    0xadcafe4b, 0x9c22e4d6, 0x3a55ef62, 0xabc30345, 0x0db408f1,
+    0x3c5c126c, 0x9a2b19d8, 0x5f8c2756, 0xf9fb2ce2, 0xc813367f,
+    0x6e643dcb, 0x982c4d22, 0x3e5b4696, 0x0fb35c0b, 0xa9c457bf,
+    0x6c636931, 0xca146285, 0xfbfc7818, 0x5d8b73ac, 0x03a0a617,
+    0xa5d7ada3, 0x943fb73e, 0x3248bc8a, 0xf7ef8204, 0x519889b0,
+    0x6070932d, 0xc6079899, 0x304fe870, 0x9638e3c4, 0xa7d0f959,
+    0x01a7f2ed, 0xc400cc63, 0x6277c7d7, 0x539fdd4a, 0xf5e8d6fe,
+    0x647e3ad9, 0xc209316d, 0xf3e12bf0, 0x55962044, 0x90311eca,
+    0x3646157e, 0x07ae0fe3, 0xa1d90457, 0x579174be, 0xf1e67f0a,
+    0xc00e6597, 0x66796e23, 0xa3de50ad, 0x05a95b19, 0x34414184,
+    0x92364a30},
+   {0x00000000, 0xcb5cd3a5, 0x4dc8a10b, 0x869472ae, 0x9b914216,
+    0x50cd91b3, 0xd659e31d, 0x1d0530b8, 0xec53826d, 0x270f51c8,
+    0xa19b2366, 0x6ac7f0c3, 0x77c2c07b, 0xbc9e13de, 0x3a0a6170,
+    0xf156b2d5, 0x03d6029b, 0xc88ad13e, 0x4e1ea390, 0x85427035,
+    0x9847408d, 0x531b9328, 0xd58fe186, 0x1ed33223, 0xef8580f6,
+    0x24d95353, 0xa24d21fd, 0x6911f258, 0x7414c2e0, 0xbf481145,
+    0x39dc63eb, 0xf280b04e, 0x07ac0536, 0xccf0d693, 0x4a64a43d,
+    0x81387798, 0x9c3d4720, 0x57619485, 0xd1f5e62b, 0x1aa9358e,
+    0xebff875b, 0x20a354fe, 0xa6372650, 0x6d6bf5f5, 0x706ec54d,
+    0xbb3216e8, 0x3da66446, 0xf6fab7e3, 0x047a07ad, 0xcf26d408,
+    0x49b2a6a6, 0x82ee7503, 0x9feb45bb, 0x54b7961e, 0xd223e4b0,
+    0x197f3715, 0xe82985c0, 0x23755665, 0xa5e124cb, 0x6ebdf76e,
+    0x73b8c7d6, 0xb8e41473, 0x3e7066dd, 0xf52cb578, 0x0f580a6c,
+    0xc404d9c9, 0x4290ab67, 0x89cc78c2, 0x94c9487a, 0x5f959bdf,
+    0xd901e971, 0x125d3ad4, 0xe30b8801, 0x28575ba4, 0xaec3290a,
+    0x659ffaaf, 0x789aca17, 0xb3c619b2, 0x35526b1c, 0xfe0eb8b9,
+    0x0c8e08f7, 0xc7d2db52, 0x4146a9fc, 0x8a1a7a59, 0x971f4ae1,
+    0x5c439944, 0xdad7ebea, 0x118b384f, 0xe0dd8a9a, 0x2b81593f,
+    0xad152b91, 0x6649f834, 0x7b4cc88c, 0xb0101b29, 0x36846987,
+    0xfdd8ba22, 0x08f40f5a, 0xc3a8dcff, 0x453cae51, 0x8e607df4,
+    0x93654d4c, 0x58399ee9, 0xdeadec47, 0x15f13fe2, 0xe4a78d37,
+    0x2ffb5e92, 0xa96f2c3c, 0x6233ff99, 0x7f36cf21, 0xb46a1c84,
+    0x32fe6e2a, 0xf9a2bd8f, 0x0b220dc1, 0xc07ede64, 0x46eaacca,
+    0x8db67f6f, 0x90b34fd7, 0x5bef9c72, 0xdd7beedc, 0x16273d79,
+    0xe7718fac, 0x2c2d5c09, 0xaab92ea7, 0x61e5fd02, 0x7ce0cdba,
+    0xb7bc1e1f, 0x31286cb1, 0xfa74bf14, 0x1eb014d8, 0xd5ecc77d,
+    0x5378b5d3, 0x98246676, 0x852156ce, 0x4e7d856b, 0xc8e9f7c5,
+    0x03b52460, 0xf2e396b5, 0x39bf4510, 0xbf2b37be, 0x7477e41b,
+    0x6972d4a3, 0xa22e0706, 0x24ba75a8, 0xefe6a60d, 0x1d661643,
+    0xd63ac5e6, 0x50aeb748, 0x9bf264ed, 0x86f75455, 0x4dab87f0,
+    0xcb3ff55e, 0x006326fb, 0xf135942e, 0x3a69478b, 0xbcfd3525,
+    0x77a1e680, 0x6aa4d638, 0xa1f8059d, 0x276c7733, 0xec30a496,
+    0x191c11ee, 0xd240c24b, 0x54d4b0e5, 0x9f886340, 0x828d53f8,
+    0x49d1805d, 0xcf45f2f3, 0x04192156, 0xf54f9383, 0x3e134026,
+    0xb8873288, 0x73dbe12d, 0x6eded195, 0xa5820230, 0x2316709e,
+    0xe84aa33b, 0x1aca1375, 0xd196c0d0, 0x5702b27e, 0x9c5e61db,
+    0x815b5163, 0x4a0782c6, 0xcc93f068, 0x07cf23cd, 0xf6999118,
+    0x3dc542bd, 0xbb513013, 0x700de3b6, 0x6d08d30e, 0xa65400ab,
+    0x20c07205, 0xeb9ca1a0, 0x11e81eb4, 0xdab4cd11, 0x5c20bfbf,
+    0x977c6c1a, 0x8a795ca2, 0x41258f07, 0xc7b1fda9, 0x0ced2e0c,
+    0xfdbb9cd9, 0x36e74f7c, 0xb0733dd2, 0x7b2fee77, 0x662adecf,
+    0xad760d6a, 0x2be27fc4, 0xe0beac61, 0x123e1c2f, 0xd962cf8a,
+    0x5ff6bd24, 0x94aa6e81, 0x89af5e39, 0x42f38d9c, 0xc467ff32,
+    0x0f3b2c97, 0xfe6d9e42, 0x35314de7, 0xb3a53f49, 0x78f9ecec,
+    0x65fcdc54, 0xaea00ff1, 0x28347d5f, 0xe368aefa, 0x16441b82,
+    0xdd18c827, 0x5b8cba89, 0x90d0692c, 0x8dd55994, 0x46898a31,
+    0xc01df89f, 0x0b412b3a, 0xfa1799ef, 0x314b4a4a, 0xb7df38e4,
+    0x7c83eb41, 0x6186dbf9, 0xaada085c, 0x2c4e7af2, 0xe712a957,
+    0x15921919, 0xdececabc, 0x585ab812, 0x93066bb7, 0x8e035b0f,
+    0x455f88aa, 0xc3cbfa04, 0x089729a1, 0xf9c19b74, 0x329d48d1,
+    0xb4093a7f, 0x7f55e9da, 0x6250d962, 0xa90c0ac7, 0x2f987869,
+    0xe4c4abcc},
+   {0x00000000, 0x3d6029b0, 0x7ac05360, 0x47a07ad0, 0xf580a6c0,
+    0xc8e08f70, 0x8f40f5a0, 0xb220dc10, 0x30704bc1, 0x0d106271,
+    0x4ab018a1, 0x77d03111, 0xc5f0ed01, 0xf890c4b1, 0xbf30be61,
+    0x825097d1, 0x60e09782, 0x5d80be32, 0x1a20c4e2, 0x2740ed52,
+    0x95603142, 0xa80018f2, 0xefa06222, 0xd2c04b92, 0x5090dc43,
+    0x6df0f5f3, 0x2a508f23, 0x1730a693, 0xa5107a83, 0x98705333,
+    0xdfd029e3, 0xe2b00053, 0xc1c12f04, 0xfca106b4, 0xbb017c64,
+    0x866155d4, 0x344189c4, 0x0921a074, 0x4e81daa4, 0x73e1f314,
+    0xf1b164c5, 0xccd14d75, 0x8b7137a5, 0xb6111e15, 0x0431c205,
+    0x3951ebb5, 0x7ef19165, 0x4391b8d5, 0xa121b886, 0x9c419136,
+    0xdbe1ebe6, 0xe681c256, 0x54a11e46, 0x69c137f6, 0x2e614d26,
+    0x13016496, 0x9151f347, 0xac31daf7, 0xeb91a027, 0xd6f18997,
+    0x64d15587, 0x59b17c37, 0x1e1106e7, 0x23712f57, 0x58f35849,
+    0x659371f9, 0x22330b29, 0x1f532299, 0xad73fe89, 0x9013d739,
+    0xd7b3ade9, 0xead38459, 0x68831388, 0x55e33a38, 0x124340e8,
+    0x2f236958, 0x9d03b548, 0xa0639cf8, 0xe7c3e628, 0xdaa3cf98,
+    0x3813cfcb, 0x0573e67b, 0x42d39cab, 0x7fb3b51b, 0xcd93690b,
+    0xf0f340bb, 0xb7533a6b, 0x8a3313db, 0x0863840a, 0x3503adba,
+    0x72a3d76a, 0x4fc3feda, 0xfde322ca, 0xc0830b7a, 0x872371aa,
+    0xba43581a, 0x9932774d, 0xa4525efd, 0xe3f2242d, 0xde920d9d,
+    0x6cb2d18d, 0x51d2f83d, 0x167282ed, 0x2b12ab5d, 0xa9423c8c,
+    0x9422153c, 0xd3826fec, 0xeee2465c, 0x5cc29a4c, 0x61a2b3fc,
+    0x2602c92c, 0x1b62e09c, 0xf9d2e0cf, 0xc4b2c97f, 0x8312b3af,
+    0xbe729a1f, 0x0c52460f, 0x31326fbf, 0x7692156f, 0x4bf23cdf,
+    0xc9a2ab0e, 0xf4c282be, 0xb362f86e, 0x8e02d1de, 0x3c220dce,
+    0x0142247e, 0x46e25eae, 0x7b82771e, 0xb1e6b092, 0x8c869922,
+    0xcb26e3f2, 0xf646ca42, 0x44661652, 0x79063fe2, 0x3ea64532,
+    0x03c66c82, 0x8196fb53, 0xbcf6d2e3, 0xfb56a833, 0xc6368183,
+    0x74165d93, 0x49767423, 0x0ed60ef3, 0x33b62743, 0xd1062710,
+    0xec660ea0, 0xabc67470, 0x96a65dc0, 0x248681d0, 0x19e6a860,
+    0x5e46d2b0, 0x6326fb00, 0xe1766cd1, 0xdc164561, 0x9bb63fb1,
+    0xa6d61601, 0x14f6ca11, 0x2996e3a1, 0x6e369971, 0x5356b0c1,
+    0x70279f96, 0x4d47b626, 0x0ae7ccf6, 0x3787e546, 0x85a73956,
+    0xb8c710e6, 0xff676a36, 0xc2074386, 0x4057d457, 0x7d37fde7,
+    0x3a978737, 0x07f7ae87, 0xb5d77297, 0x88b75b27, 0xcf1721f7,
+    0xf2770847, 0x10c70814, 0x2da721a4, 0x6a075b74, 0x576772c4,
+    0xe547aed4, 0xd8278764, 0x9f87fdb4, 0xa2e7d404, 0x20b743d5,
+    0x1dd76a65, 0x5a7710b5, 0x67173905, 0xd537e515, 0xe857cca5,
+    0xaff7b675, 0x92979fc5, 0xe915e8db, 0xd475c16b, 0x93d5bbbb,
+    0xaeb5920b, 0x1c954e1b, 0x21f567ab, 0x66551d7b, 0x5b3534cb,
+    0xd965a31a, 0xe4058aaa, 0xa3a5f07a, 0x9ec5d9ca, 0x2ce505da,
+    0x11852c6a, 0x562556ba, 0x6b457f0a, 0x89f57f59, 0xb49556e9,
+    0xf3352c39, 0xce550589, 0x7c75d999, 0x4115f029, 0x06b58af9,
+    0x3bd5a349, 0xb9853498, 0x84e51d28, 0xc34567f8, 0xfe254e48,
+    0x4c059258, 0x7165bbe8, 0x36c5c138, 0x0ba5e888, 0x28d4c7df,
+    0x15b4ee6f, 0x521494bf, 0x6f74bd0f, 0xdd54611f, 0xe03448af,
+    0xa794327f, 0x9af41bcf, 0x18a48c1e, 0x25c4a5ae, 0x6264df7e,
+    0x5f04f6ce, 0xed242ade, 0xd044036e, 0x97e479be, 0xaa84500e,
+    0x4834505d, 0x755479ed, 0x32f4033d, 0x0f942a8d, 0xbdb4f69d,
+    0x80d4df2d, 0xc774a5fd, 0xfa148c4d, 0x78441b9c, 0x4524322c,
+    0x028448fc, 0x3fe4614c, 0x8dc4bd5c, 0xb0a494ec, 0xf704ee3c,
+    0xca64c78c}};
+
+local const z_word_t FAR crc_braid_big_table[][256] = {
+   {0x00000000, 0xb029603d, 0x6053c07a, 0xd07aa047, 0xc0a680f5,
+    0x708fe0c8, 0xa0f5408f, 0x10dc20b2, 0xc14b7030, 0x7162100d,
+    0xa118b04a, 0x1131d077, 0x01edf0c5, 0xb1c490f8, 0x61be30bf,
+    0xd1975082, 0x8297e060, 0x32be805d, 0xe2c4201a, 0x52ed4027,
+    0x42316095, 0xf21800a8, 0x2262a0ef, 0x924bc0d2, 0x43dc9050,
+    0xf3f5f06d, 0x238f502a, 0x93a63017, 0x837a10a5, 0x33537098,
+    0xe329d0df, 0x5300b0e2, 0x042fc1c1, 0xb406a1fc, 0x647c01bb,
+    0xd4556186, 0xc4894134, 0x74a02109, 0xa4da814e, 0x14f3e173,
+    0xc564b1f1, 0x754dd1cc, 0xa537718b, 0x151e11b6, 0x05c23104,
+    0xb5eb5139, 0x6591f17e, 0xd5b89143, 0x86b821a1, 0x3691419c,
+    0xe6ebe1db, 0x56c281e6, 0x461ea154, 0xf637c169, 0x264d612e,
+    0x96640113, 0x47f35191, 0xf7da31ac, 0x27a091eb, 0x9789f1d6,
+    0x8755d164, 0x377cb159, 0xe706111e, 0x572f7123, 0x4958f358,
+    0xf9719365, 0x290b3322, 0x9922531f, 0x89fe73ad, 0x39d71390,
+    0xe9adb3d7, 0x5984d3ea, 0x88138368, 0x383ae355, 0xe8404312,
+    0x5869232f, 0x48b5039d, 0xf89c63a0, 0x28e6c3e7, 0x98cfa3da,
+    0xcbcf1338, 0x7be67305, 0xab9cd342, 0x1bb5b37f, 0x0b6993cd,
+    0xbb40f3f0, 0x6b3a53b7, 0xdb13338a, 0x0a846308, 0xbaad0335,
+    0x6ad7a372, 0xdafec34f, 0xca22e3fd, 0x7a0b83c0, 0xaa712387,
+    0x1a5843ba, 0x4d773299, 0xfd5e52a4, 0x2d24f2e3, 0x9d0d92de,
+    0x8dd1b26c, 0x3df8d251, 0xed827216, 0x5dab122b, 0x8c3c42a9,
+    0x3c152294, 0xec6f82d3, 0x5c46e2ee, 0x4c9ac25c, 0xfcb3a261,
+    0x2cc90226, 0x9ce0621b, 0xcfe0d2f9, 0x7fc9b2c4, 0xafb31283,
+    0x1f9a72be, 0x0f46520c, 0xbf6f3231, 0x6f159276, 0xdf3cf24b,
+    0x0eaba2c9, 0xbe82c2f4, 0x6ef862b3, 0xded1028e, 0xce0d223c,
+    0x7e244201, 0xae5ee246, 0x1e77827b, 0x92b0e6b1, 0x2299868c,
+    0xf2e326cb, 0x42ca46f6, 0x52166644, 0xe23f0679, 0x3245a63e,
+    0x826cc603, 0x53fb9681, 0xe3d2f6bc, 0x33a856fb, 0x838136c6,
+    0x935d1674, 0x23747649, 0xf30ed60e, 0x4327b633, 0x102706d1,
+    0xa00e66ec, 0x7074c6ab, 0xc05da696, 0xd0818624, 0x60a8e619,
+    0xb0d2465e, 0x00fb2663, 0xd16c76e1, 0x614516dc, 0xb13fb69b,
+    0x0116d6a6, 0x11caf614, 0xa1e39629, 0x7199366e, 0xc1b05653,
+    0x969f2770, 0x26b6474d, 0xf6cce70a, 0x46e58737, 0x5639a785,
+    0xe610c7b8, 0x366a67ff, 0x864307c2, 0x57d45740, 0xe7fd377d,
+    0x3787973a, 0x87aef707, 0x9772d7b5, 0x275bb788, 0xf72117cf,
+    0x470877f2, 0x1408c710, 0xa421a72d, 0x745b076a, 0xc4726757,
+    0xd4ae47e5, 0x648727d8, 0xb4fd879f, 0x04d4e7a2, 0xd543b720,
+    0x656ad71d, 0xb510775a, 0x05391767, 0x15e537d5, 0xa5cc57e8,
+    0x75b6f7af, 0xc59f9792, 0xdbe815e9, 0x6bc175d4, 0xbbbbd593,
+    0x0b92b5ae, 0x1b4e951c, 0xab67f521, 0x7b1d5566, 0xcb34355b,
+    0x1aa365d9, 0xaa8a05e4, 0x7af0a5a3, 0xcad9c59e, 0xda05e52c,
+    0x6a2c8511, 0xba562556, 0x0a7f456b, 0x597ff589, 0xe95695b4,
+    0x392c35f3, 0x890555ce, 0x99d9757c, 0x29f01541, 0xf98ab506,
+    0x49a3d53b, 0x983485b9, 0x281de584, 0xf86745c3, 0x484e25fe,
+    0x5892054c, 0xe8bb6571, 0x38c1c536, 0x88e8a50b, 0xdfc7d428,
+    0x6feeb415, 0xbf941452, 0x0fbd746f, 0x1f6154dd, 0xaf4834e0,
+    0x7f3294a7, 0xcf1bf49a, 0x1e8ca418, 0xaea5c425, 0x7edf6462,
+    0xcef6045f, 0xde2a24ed, 0x6e0344d0, 0xbe79e497, 0x0e5084aa,
+    0x5d503448, 0xed795475, 0x3d03f432, 0x8d2a940f, 0x9df6b4bd,
+    0x2ddfd480, 0xfda574c7, 0x4d8c14fa, 0x9c1b4478, 0x2c322445,
+    0xfc488402, 0x4c61e43f, 0x5cbdc48d, 0xec94a4b0, 0x3cee04f7,
+    0x8cc764ca},
+   {0x00000000, 0xa5d35ccb, 0x0ba1c84d, 0xae729486, 0x1642919b,
+    0xb391cd50, 0x1de359d6, 0xb830051d, 0x6d8253ec, 0xc8510f27,
+    0x66239ba1, 0xc3f0c76a, 0x7bc0c277, 0xde139ebc, 0x70610a3a,
+    0xd5b256f1, 0x9b02d603, 0x3ed18ac8, 0x90a31e4e, 0x35704285,
+    0x8d404798, 0x28931b53, 0x86e18fd5, 0x2332d31e, 0xf68085ef,
+    0x5353d924, 0xfd214da2, 0x58f21169, 0xe0c21474, 0x451148bf,
+    0xeb63dc39, 0x4eb080f2, 0x3605ac07, 0x93d6f0cc, 0x3da4644a,
+    0x98773881, 0x20473d9c, 0x85946157, 0x2be6f5d1, 0x8e35a91a,
+    0x5b87ffeb, 0xfe54a320, 0x502637a6, 0xf5f56b6d, 0x4dc56e70,
+    0xe81632bb, 0x4664a63d, 0xe3b7faf6, 0xad077a04, 0x08d426cf,
+    0xa6a6b249, 0x0375ee82, 0xbb45eb9f, 0x1e96b754, 0xb0e423d2,
+    0x15377f19, 0xc08529e8, 0x65567523, 0xcb24e1a5, 0x6ef7bd6e,
+    0xd6c7b873, 0x7314e4b8, 0xdd66703e, 0x78b52cf5, 0x6c0a580f,
+    0xc9d904c4, 0x67ab9042, 0xc278cc89, 0x7a48c994, 0xdf9b955f,
+    0x71e901d9, 0xd43a5d12, 0x01880be3, 0xa45b5728, 0x0a29c3ae,
+    0xaffa9f65, 0x17ca9a78, 0xb219c6b3, 0x1c6b5235, 0xb9b80efe,
+    0xf7088e0c, 0x52dbd2c7, 0xfca94641, 0x597a1a8a, 0xe14a1f97,
+    0x4499435c, 0xeaebd7da, 0x4f388b11, 0x9a8adde0, 0x3f59812b,
+    0x912b15ad, 0x34f84966, 0x8cc84c7b, 0x291b10b0, 0x87698436,
+    0x22bad8fd, 0x5a0ff408, 0xffdca8c3, 0x51ae3c45, 0xf47d608e,
+    0x4c4d6593, 0xe99e3958, 0x47ecadde, 0xe23ff115, 0x378da7e4,
+    0x925efb2f, 0x3c2c6fa9, 0x99ff3362, 0x21cf367f, 0x841c6ab4,
+    0x2a6efe32, 0x8fbda2f9, 0xc10d220b, 0x64de7ec0, 0xcaacea46,
+    0x6f7fb68d, 0xd74fb390, 0x729cef5b, 0xdcee7bdd, 0x793d2716,
+    0xac8f71e7, 0x095c2d2c, 0xa72eb9aa, 0x02fde561, 0xbacde07c,
+    0x1f1ebcb7, 0xb16c2831, 0x14bf74fa, 0xd814b01e, 0x7dc7ecd5,
+    0xd3b57853, 0x76662498, 0xce562185, 0x6b857d4e, 0xc5f7e9c8,
+    0x6024b503, 0xb596e3f2, 0x1045bf39, 0xbe372bbf, 0x1be47774,
+    0xa3d47269, 0x06072ea2, 0xa875ba24, 0x0da6e6ef, 0x4316661d,
+    0xe6c53ad6, 0x48b7ae50, 0xed64f29b, 0x5554f786, 0xf087ab4d,
+    0x5ef53fcb, 0xfb266300, 0x2e9435f1, 0x8b47693a, 0x2535fdbc,
+    0x80e6a177, 0x38d6a46a, 0x9d05f8a1, 0x33776c27, 0x96a430ec,
+    0xee111c19, 0x4bc240d2, 0xe5b0d454, 0x4063889f, 0xf8538d82,
+    0x5d80d149, 0xf3f245cf, 0x56211904, 0x83934ff5, 0x2640133e,
+    0x883287b8, 0x2de1db73, 0x95d1de6e, 0x300282a5, 0x9e701623,
+    0x3ba34ae8, 0x7513ca1a, 0xd0c096d1, 0x7eb20257, 0xdb615e9c,
+    0x63515b81, 0xc682074a, 0x68f093cc, 0xcd23cf07, 0x189199f6,
+    0xbd42c53d, 0x133051bb, 0xb6e30d70, 0x0ed3086d, 0xab0054a6,
+    0x0572c020, 0xa0a19ceb, 0xb41ee811, 0x11cdb4da, 0xbfbf205c,
+    0x1a6c7c97, 0xa25c798a, 0x078f2541, 0xa9fdb1c7, 0x0c2eed0c,
+    0xd99cbbfd, 0x7c4fe736, 0xd23d73b0, 0x77ee2f7b, 0xcfde2a66,
+    0x6a0d76ad, 0xc47fe22b, 0x61acbee0, 0x2f1c3e12, 0x8acf62d9,
+    0x24bdf65f, 0x816eaa94, 0x395eaf89, 0x9c8df342, 0x32ff67c4,
+    0x972c3b0f, 0x429e6dfe, 0xe74d3135, 0x493fa5b3, 0xececf978,
+    0x54dcfc65, 0xf10fa0ae, 0x5f7d3428, 0xfaae68e3, 0x821b4416,
+    0x27c818dd, 0x89ba8c5b, 0x2c69d090, 0x9459d58d, 0x318a8946,
+    0x9ff81dc0, 0x3a2b410b, 0xef9917fa, 0x4a4a4b31, 0xe438dfb7,
+    0x41eb837c, 0xf9db8661, 0x5c08daaa, 0xf27a4e2c, 0x57a912e7,
+    0x19199215, 0xbccacede, 0x12b85a58, 0xb76b0693, 0x0f5b038e,
+    0xaa885f45, 0x04facbc3, 0xa1299708, 0x749bc1f9, 0xd1489d32,
+    0x7f3a09b4, 0xdae9557f, 0x62d95062, 0xc70a0ca9, 0x6978982f,
+    0xccabc4e4},
+   {0x00000000, 0xb40b77a6, 0x29119f97, 0x9d1ae831, 0x13244ff4,
+    0xa72f3852, 0x3a35d063, 0x8e3ea7c5, 0x674eef33, 0xd3459895,
+    0x4e5f70a4, 0xfa540702, 0x746aa0c7, 0xc061d761, 0x5d7b3f50,
+    0xe97048f6, 0xce9cde67, 0x7a97a9c1, 0xe78d41f0, 0x53863656,
+    0xddb89193, 0x69b3e635, 0xf4a90e04, 0x40a279a2, 0xa9d23154,
+    0x1dd946f2, 0x80c3aec3, 0x34c8d965, 0xbaf67ea0, 0x0efd0906,
+    0x93e7e137, 0x27ec9691, 0x9c39bdcf, 0x2832ca69, 0xb5282258,
+    0x012355fe, 0x8f1df23b, 0x3b16859d, 0xa60c6dac, 0x12071a0a,
+    0xfb7752fc, 0x4f7c255a, 0xd266cd6b, 0x666dbacd, 0xe8531d08,
+    0x5c586aae, 0xc142829f, 0x7549f539, 0x52a563a8, 0xe6ae140e,
+    0x7bb4fc3f, 0xcfbf8b99, 0x41812c5c, 0xf58a5bfa, 0x6890b3cb,
+    0xdc9bc46d, 0x35eb8c9b, 0x81e0fb3d, 0x1cfa130c, 0xa8f164aa,
+    0x26cfc36f, 0x92c4b4c9, 0x0fde5cf8, 0xbbd52b5e, 0x79750b44,
+    0xcd7e7ce2, 0x506494d3, 0xe46fe375, 0x6a5144b0, 0xde5a3316,
+    0x4340db27, 0xf74bac81, 0x1e3be477, 0xaa3093d1, 0x372a7be0,
+    0x83210c46, 0x0d1fab83, 0xb914dc25, 0x240e3414, 0x900543b2,
+    0xb7e9d523, 0x03e2a285, 0x9ef84ab4, 0x2af33d12, 0xa4cd9ad7,
+    0x10c6ed71, 0x8ddc0540, 0x39d772e6, 0xd0a73a10, 0x64ac4db6,
+    0xf9b6a587, 0x4dbdd221, 0xc38375e4, 0x77880242, 0xea92ea73,
+    0x5e999dd5, 0xe54cb68b, 0x5147c12d, 0xcc5d291c, 0x78565eba,
+    0xf668f97f, 0x42638ed9, 0xdf7966e8, 0x6b72114e, 0x820259b8,
+    0x36092e1e, 0xab13c62f, 0x1f18b189, 0x9126164c, 0x252d61ea,
+    0xb83789db, 0x0c3cfe7d, 0x2bd068ec, 0x9fdb1f4a, 0x02c1f77b,
+    0xb6ca80dd, 0x38f42718, 0x8cff50be, 0x11e5b88f, 0xa5eecf29,
+    0x4c9e87df, 0xf895f079, 0x658f1848, 0xd1846fee, 0x5fbac82b,
+    0xebb1bf8d, 0x76ab57bc, 0xc2a0201a, 0xf2ea1688, 0x46e1612e,
+    0xdbfb891f, 0x6ff0feb9, 0xe1ce597c, 0x55c52eda, 0xc8dfc6eb,
+    0x7cd4b14d, 0x95a4f9bb, 0x21af8e1d, 0xbcb5662c, 0x08be118a,
+    0x8680b64f, 0x328bc1e9, 0xaf9129d8, 0x1b9a5e7e, 0x3c76c8ef,
+    0x887dbf49, 0x15675778, 0xa16c20de, 0x2f52871b, 0x9b59f0bd,
+    0x0643188c, 0xb2486f2a, 0x5b3827dc, 0xef33507a, 0x7229b84b,
+    0xc622cfed, 0x481c6828, 0xfc171f8e, 0x610df7bf, 0xd5068019,
+    0x6ed3ab47, 0xdad8dce1, 0x47c234d0, 0xf3c94376, 0x7df7e4b3,
+    0xc9fc9315, 0x54e67b24, 0xe0ed0c82, 0x099d4474, 0xbd9633d2,
+    0x208cdbe3, 0x9487ac45, 0x1ab90b80, 0xaeb27c26, 0x33a89417,
+    0x87a3e3b1, 0xa04f7520, 0x14440286, 0x895eeab7, 0x3d559d11,
+    0xb36b3ad4, 0x07604d72, 0x9a7aa543, 0x2e71d2e5, 0xc7019a13,
+    0x730aedb5, 0xee100584, 0x5a1b7222, 0xd425d5e7, 0x602ea241,
+    0xfd344a70, 0x493f3dd6, 0x8b9f1dcc, 0x3f946a6a, 0xa28e825b,
+    0x1685f5fd, 0x98bb5238, 0x2cb0259e, 0xb1aacdaf, 0x05a1ba09,
+    0xecd1f2ff, 0x58da8559, 0xc5c06d68, 0x71cb1ace, 0xfff5bd0b,
+    0x4bfecaad, 0xd6e4229c, 0x62ef553a, 0x4503c3ab, 0xf108b40d,
+    0x6c125c3c, 0xd8192b9a, 0x56278c5f, 0xe22cfbf9, 0x7f3613c8,
+    0xcb3d646e, 0x224d2c98, 0x96465b3e, 0x0b5cb30f, 0xbf57c4a9,
+    0x3169636c, 0x856214ca, 0x1878fcfb, 0xac738b5d, 0x17a6a003,
+    0xa3add7a5, 0x3eb73f94, 0x8abc4832, 0x0482eff7, 0xb0899851,
+    0x2d937060, 0x999807c6, 0x70e84f30, 0xc4e33896, 0x59f9d0a7,
+    0xedf2a701, 0x63cc00c4, 0xd7c77762, 0x4add9f53, 0xfed6e8f5,
+    0xd93a7e64, 0x6d3109c2, 0xf02be1f3, 0x44209655, 0xca1e3190,
+    0x7e154636, 0xe30fae07, 0x5704d9a1, 0xbe749157, 0x0a7fe6f1,
+    0x97650ec0, 0x236e7966, 0xad50dea3, 0x195ba905, 0x84414134,
+    0x304a3692},
+   {0x00000000, 0x9e00aacc, 0x7d072542, 0xe3078f8e, 0xfa0e4a84,
+    0x640ee048, 0x87096fc6, 0x1909c50a, 0xb51be5d3, 0x2b1b4f1f,
+    0xc81cc091, 0x561c6a5d, 0x4f15af57, 0xd115059b, 0x32128a15,
+    0xac1220d9, 0x2b31bb7c, 0xb53111b0, 0x56369e3e, 0xc83634f2,
+    0xd13ff1f8, 0x4f3f5b34, 0xac38d4ba, 0x32387e76, 0x9e2a5eaf,
+    0x002af463, 0xe32d7bed, 0x7d2dd121, 0x6424142b, 0xfa24bee7,
+    0x19233169, 0x87239ba5, 0x566276f9, 0xc862dc35, 0x2b6553bb,
+    0xb565f977, 0xac6c3c7d, 0x326c96b1, 0xd16b193f, 0x4f6bb3f3,
+    0xe379932a, 0x7d7939e6, 0x9e7eb668, 0x007e1ca4, 0x1977d9ae,
+    0x87777362, 0x6470fcec, 0xfa705620, 0x7d53cd85, 0xe3536749,
+    0x0054e8c7, 0x9e54420b, 0x875d8701, 0x195d2dcd, 0xfa5aa243,
+    0x645a088f, 0xc8482856, 0x5648829a, 0xb54f0d14, 0x2b4fa7d8,
+    0x324662d2, 0xac46c81e, 0x4f414790, 0xd141ed5c, 0xedc29d29,
+    0x73c237e5, 0x90c5b86b, 0x0ec512a7, 0x17ccd7ad, 0x89cc7d61,
+    0x6acbf2ef, 0xf4cb5823, 0x58d978fa, 0xc6d9d236, 0x25de5db8,
+    0xbbdef774, 0xa2d7327e, 0x3cd798b2, 0xdfd0173c, 0x41d0bdf0,
+    0xc6f32655, 0x58f38c99, 0xbbf40317, 0x25f4a9db, 0x3cfd6cd1,
+    0xa2fdc61d, 0x41fa4993, 0xdffae35f, 0x73e8c386, 0xede8694a,
+    0x0eefe6c4, 0x90ef4c08, 0x89e68902, 0x17e623ce, 0xf4e1ac40,
+    0x6ae1068c, 0xbba0ebd0, 0x25a0411c, 0xc6a7ce92, 0x58a7645e,
+    0x41aea154, 0xdfae0b98, 0x3ca98416, 0xa2a92eda, 0x0ebb0e03,
+    0x90bba4cf, 0x73bc2b41, 0xedbc818d, 0xf4b54487, 0x6ab5ee4b,
+    0x89b261c5, 0x17b2cb09, 0x909150ac, 0x0e91fa60, 0xed9675ee,
+    0x7396df22, 0x6a9f1a28, 0xf49fb0e4, 0x17983f6a, 0x899895a6,
+    0x258ab57f, 0xbb8a1fb3, 0x588d903d, 0xc68d3af1, 0xdf84fffb,
+    0x41845537, 0xa283dab9, 0x3c837075, 0xda853b53, 0x4485919f,
+    0xa7821e11, 0x3982b4dd, 0x208b71d7, 0xbe8bdb1b, 0x5d8c5495,
+    0xc38cfe59, 0x6f9ede80, 0xf19e744c, 0x1299fbc2, 0x8c99510e,
+    0x95909404, 0x0b903ec8, 0xe897b146, 0x76971b8a, 0xf1b4802f,
+    0x6fb42ae3, 0x8cb3a56d, 0x12b30fa1, 0x0bbacaab, 0x95ba6067,
+    0x76bdefe9, 0xe8bd4525, 0x44af65fc, 0xdaafcf30, 0x39a840be,
+    0xa7a8ea72, 0xbea12f78, 0x20a185b4, 0xc3a60a3a, 0x5da6a0f6,
+    0x8ce74daa, 0x12e7e766, 0xf1e068e8, 0x6fe0c224, 0x76e9072e,
+    0xe8e9ade2, 0x0bee226c, 0x95ee88a0, 0x39fca879, 0xa7fc02b5,
+    0x44fb8d3b, 0xdafb27f7, 0xc3f2e2fd, 0x5df24831, 0xbef5c7bf,
+    0x20f56d73, 0xa7d6f6d6, 0x39d65c1a, 0xdad1d394, 0x44d17958,
+    0x5dd8bc52, 0xc3d8169e, 0x20df9910, 0xbedf33dc, 0x12cd1305,
+    0x8ccdb9c9, 0x6fca3647, 0xf1ca9c8b, 0xe8c35981, 0x76c3f34d,
+    0x95c47cc3, 0x0bc4d60f, 0x3747a67a, 0xa9470cb6, 0x4a408338,
+    0xd44029f4, 0xcd49ecfe, 0x53494632, 0xb04ec9bc, 0x2e4e6370,
+    0x825c43a9, 0x1c5ce965, 0xff5b66eb, 0x615bcc27, 0x7852092d,
+    0xe652a3e1, 0x05552c6f, 0x9b5586a3, 0x1c761d06, 0x8276b7ca,
+    0x61713844, 0xff719288, 0xe6785782, 0x7878fd4e, 0x9b7f72c0,
+    0x057fd80c, 0xa96df8d5, 0x376d5219, 0xd46add97, 0x4a6a775b,
+    0x5363b251, 0xcd63189d, 0x2e649713, 0xb0643ddf, 0x6125d083,
+    0xff257a4f, 0x1c22f5c1, 0x82225f0d, 0x9b2b9a07, 0x052b30cb,
+    0xe62cbf45, 0x782c1589, 0xd43e3550, 0x4a3e9f9c, 0xa9391012,
+    0x3739bade, 0x2e307fd4, 0xb030d518, 0x53375a96, 0xcd37f05a,
+    0x4a146bff, 0xd414c133, 0x37134ebd, 0xa913e471, 0xb01a217b,
+    0x2e1a8bb7, 0xcd1d0439, 0x531daef5, 0xff0f8e2c, 0x610f24e0,
+    0x8208ab6e, 0x1c0801a2, 0x0501c4a8, 0x9b016e64, 0x7806e1ea,
+    0xe6064b26}};
+
 #endif
-  }
-};
+
+#endif
+
+#if N == 3
+
+#if W == 8
+
+local const z_crc_t FAR crc_braid_table[][256] = {
+   {0x00000000, 0x81256527, 0xd93bcc0f, 0x581ea928, 0x69069e5f,
+    0xe823fb78, 0xb03d5250, 0x31183777, 0xd20d3cbe, 0x53285999,
+    0x0b36f0b1, 0x8a139596, 0xbb0ba2e1, 0x3a2ec7c6, 0x62306eee,
+    0xe3150bc9, 0x7f6b7f3d, 0xfe4e1a1a, 0xa650b332, 0x2775d615,
+    0x166de162, 0x97488445, 0xcf562d6d, 0x4e73484a, 0xad664383,
+    0x2c4326a4, 0x745d8f8c, 0xf578eaab, 0xc460dddc, 0x4545b8fb,
+    0x1d5b11d3, 0x9c7e74f4, 0xfed6fe7a, 0x7ff39b5d, 0x27ed3275,
+    0xa6c85752, 0x97d06025, 0x16f50502, 0x4eebac2a, 0xcfcec90d,
+    0x2cdbc2c4, 0xadfea7e3, 0xf5e00ecb, 0x74c56bec, 0x45dd5c9b,
+    0xc4f839bc, 0x9ce69094, 0x1dc3f5b3, 0x81bd8147, 0x0098e460,
+    0x58864d48, 0xd9a3286f, 0xe8bb1f18, 0x699e7a3f, 0x3180d317,
+    0xb0a5b630, 0x53b0bdf9, 0xd295d8de, 0x8a8b71f6, 0x0bae14d1,
+    0x3ab623a6, 0xbb934681, 0xe38defa9, 0x62a88a8e, 0x26dcfab5,
+    0xa7f99f92, 0xffe736ba, 0x7ec2539d, 0x4fda64ea, 0xceff01cd,
+    0x96e1a8e5, 0x17c4cdc2, 0xf4d1c60b, 0x75f4a32c, 0x2dea0a04,
+    0xaccf6f23, 0x9dd75854, 0x1cf23d73, 0x44ec945b, 0xc5c9f17c,
+    0x59b78588, 0xd892e0af, 0x808c4987, 0x01a92ca0, 0x30b11bd7,
+    0xb1947ef0, 0xe98ad7d8, 0x68afb2ff, 0x8bbab936, 0x0a9fdc11,
+    0x52817539, 0xd3a4101e, 0xe2bc2769, 0x6399424e, 0x3b87eb66,
+    0xbaa28e41, 0xd80a04cf, 0x592f61e8, 0x0131c8c0, 0x8014ade7,
+    0xb10c9a90, 0x3029ffb7, 0x6837569f, 0xe91233b8, 0x0a073871,
+    0x8b225d56, 0xd33cf47e, 0x52199159, 0x6301a62e, 0xe224c309,
+    0xba3a6a21, 0x3b1f0f06, 0xa7617bf2, 0x26441ed5, 0x7e5ab7fd,
+    0xff7fd2da, 0xce67e5ad, 0x4f42808a, 0x175c29a2, 0x96794c85,
+    0x756c474c, 0xf449226b, 0xac578b43, 0x2d72ee64, 0x1c6ad913,
+    0x9d4fbc34, 0xc551151c, 0x4474703b, 0x4db9f56a, 0xcc9c904d,
+    0x94823965, 0x15a75c42, 0x24bf6b35, 0xa59a0e12, 0xfd84a73a,
+    0x7ca1c21d, 0x9fb4c9d4, 0x1e91acf3, 0x468f05db, 0xc7aa60fc,
+    0xf6b2578b, 0x779732ac, 0x2f899b84, 0xaeacfea3, 0x32d28a57,
+    0xb3f7ef70, 0xebe94658, 0x6acc237f, 0x5bd41408, 0xdaf1712f,
+    0x82efd807, 0x03cabd20, 0xe0dfb6e9, 0x61fad3ce, 0x39e47ae6,
+    0xb8c11fc1, 0x89d928b6, 0x08fc4d91, 0x50e2e4b9, 0xd1c7819e,
+    0xb36f0b10, 0x324a6e37, 0x6a54c71f, 0xeb71a238, 0xda69954f,
+    0x5b4cf068, 0x03525940, 0x82773c67, 0x616237ae, 0xe0475289,
+    0xb859fba1, 0x397c9e86, 0x0864a9f1, 0x8941ccd6, 0xd15f65fe,
+    0x507a00d9, 0xcc04742d, 0x4d21110a, 0x153fb822, 0x941add05,
+    0xa502ea72, 0x24278f55, 0x7c39267d, 0xfd1c435a, 0x1e094893,
+    0x9f2c2db4, 0xc732849c, 0x4617e1bb, 0x770fd6cc, 0xf62ab3eb,
+    0xae341ac3, 0x2f117fe4, 0x6b650fdf, 0xea406af8, 0xb25ec3d0,
+    0x337ba6f7, 0x02639180, 0x8346f4a7, 0xdb585d8f, 0x5a7d38a8,
+    0xb9683361, 0x384d5646, 0x6053ff6e, 0xe1769a49, 0xd06ead3e,
+    0x514bc819, 0x09556131, 0x88700416, 0x140e70e2, 0x952b15c5,
+    0xcd35bced, 0x4c10d9ca, 0x7d08eebd, 0xfc2d8b9a, 0xa43322b2,
+    0x25164795, 0xc6034c5c, 0x4726297b, 0x1f388053, 0x9e1de574,
+    0xaf05d203, 0x2e20b724, 0x763e1e0c, 0xf71b7b2b, 0x95b3f1a5,
+    0x14969482, 0x4c883daa, 0xcdad588d, 0xfcb56ffa, 0x7d900add,
+    0x258ea3f5, 0xa4abc6d2, 0x47becd1b, 0xc69ba83c, 0x9e850114,
+    0x1fa06433, 0x2eb85344, 0xaf9d3663, 0xf7839f4b, 0x76a6fa6c,
+    0xead88e98, 0x6bfdebbf, 0x33e34297, 0xb2c627b0, 0x83de10c7,
+    0x02fb75e0, 0x5ae5dcc8, 0xdbc0b9ef, 0x38d5b226, 0xb9f0d701,
+    0xe1ee7e29, 0x60cb1b0e, 0x51d32c79, 0xd0f6495e, 0x88e8e076,
+    0x09cd8551},
+   {0x00000000, 0x9b73ead4, 0xed96d3e9, 0x76e5393d, 0x005ca193,
+    0x9b2f4b47, 0xedca727a, 0x76b998ae, 0x00b94326, 0x9bcaa9f2,
+    0xed2f90cf, 0x765c7a1b, 0x00e5e2b5, 0x9b960861, 0xed73315c,
+    0x7600db88, 0x0172864c, 0x9a016c98, 0xece455a5, 0x7797bf71,
+    0x012e27df, 0x9a5dcd0b, 0xecb8f436, 0x77cb1ee2, 0x01cbc56a,
+    0x9ab82fbe, 0xec5d1683, 0x772efc57, 0x019764f9, 0x9ae48e2d,
+    0xec01b710, 0x77725dc4, 0x02e50c98, 0x9996e64c, 0xef73df71,
+    0x740035a5, 0x02b9ad0b, 0x99ca47df, 0xef2f7ee2, 0x745c9436,
+    0x025c4fbe, 0x992fa56a, 0xefca9c57, 0x74b97683, 0x0200ee2d,
+    0x997304f9, 0xef963dc4, 0x74e5d710, 0x03978ad4, 0x98e46000,
+    0xee01593d, 0x7572b3e9, 0x03cb2b47, 0x98b8c193, 0xee5df8ae,
+    0x752e127a, 0x032ec9f2, 0x985d2326, 0xeeb81a1b, 0x75cbf0cf,
+    0x03726861, 0x980182b5, 0xeee4bb88, 0x7597515c, 0x05ca1930,
+    0x9eb9f3e4, 0xe85ccad9, 0x732f200d, 0x0596b8a3, 0x9ee55277,
+    0xe8006b4a, 0x7373819e, 0x05735a16, 0x9e00b0c2, 0xe8e589ff,
+    0x7396632b, 0x052ffb85, 0x9e5c1151, 0xe8b9286c, 0x73cac2b8,
+    0x04b89f7c, 0x9fcb75a8, 0xe92e4c95, 0x725da641, 0x04e43eef,
+    0x9f97d43b, 0xe972ed06, 0x720107d2, 0x0401dc5a, 0x9f72368e,
+    0xe9970fb3, 0x72e4e567, 0x045d7dc9, 0x9f2e971d, 0xe9cbae20,
+    0x72b844f4, 0x072f15a8, 0x9c5cff7c, 0xeab9c641, 0x71ca2c95,
+    0x0773b43b, 0x9c005eef, 0xeae567d2, 0x71968d06, 0x0796568e,
+    0x9ce5bc5a, 0xea008567, 0x71736fb3, 0x07caf71d, 0x9cb91dc9,
+    0xea5c24f4, 0x712fce20, 0x065d93e4, 0x9d2e7930, 0xebcb400d,
+    0x70b8aad9, 0x06013277, 0x9d72d8a3, 0xeb97e19e, 0x70e40b4a,
+    0x06e4d0c2, 0x9d973a16, 0xeb72032b, 0x7001e9ff, 0x06b87151,
+    0x9dcb9b85, 0xeb2ea2b8, 0x705d486c, 0x0b943260, 0x90e7d8b4,
+    0xe602e189, 0x7d710b5d, 0x0bc893f3, 0x90bb7927, 0xe65e401a,
+    0x7d2daace, 0x0b2d7146, 0x905e9b92, 0xe6bba2af, 0x7dc8487b,
+    0x0b71d0d5, 0x90023a01, 0xe6e7033c, 0x7d94e9e8, 0x0ae6b42c,
+    0x91955ef8, 0xe77067c5, 0x7c038d11, 0x0aba15bf, 0x91c9ff6b,
+    0xe72cc656, 0x7c5f2c82, 0x0a5ff70a, 0x912c1dde, 0xe7c924e3,
+    0x7cbace37, 0x0a035699, 0x9170bc4d, 0xe7958570, 0x7ce66fa4,
+    0x09713ef8, 0x9202d42c, 0xe4e7ed11, 0x7f9407c5, 0x092d9f6b,
+    0x925e75bf, 0xe4bb4c82, 0x7fc8a656, 0x09c87dde, 0x92bb970a,
+    0xe45eae37, 0x7f2d44e3, 0x0994dc4d, 0x92e73699, 0xe4020fa4,
+    0x7f71e570, 0x0803b8b4, 0x93705260, 0xe5956b5d, 0x7ee68189,
+    0x085f1927, 0x932cf3f3, 0xe5c9cace, 0x7eba201a, 0x08bafb92,
+    0x93c91146, 0xe52c287b, 0x7e5fc2af, 0x08e65a01, 0x9395b0d5,
+    0xe57089e8, 0x7e03633c, 0x0e5e2b50, 0x952dc184, 0xe3c8f8b9,
+    0x78bb126d, 0x0e028ac3, 0x95716017, 0xe394592a, 0x78e7b3fe,
+    0x0ee76876, 0x959482a2, 0xe371bb9f, 0x7802514b, 0x0ebbc9e5,
+    0x95c82331, 0xe32d1a0c, 0x785ef0d8, 0x0f2cad1c, 0x945f47c8,
+    0xe2ba7ef5, 0x79c99421, 0x0f700c8f, 0x9403e65b, 0xe2e6df66,
+    0x799535b2, 0x0f95ee3a, 0x94e604ee, 0xe2033dd3, 0x7970d707,
+    0x0fc94fa9, 0x94baa57d, 0xe25f9c40, 0x792c7694, 0x0cbb27c8,
+    0x97c8cd1c, 0xe12df421, 0x7a5e1ef5, 0x0ce7865b, 0x97946c8f,
+    0xe17155b2, 0x7a02bf66, 0x0c0264ee, 0x97718e3a, 0xe194b707,
+    0x7ae75dd3, 0x0c5ec57d, 0x972d2fa9, 0xe1c81694, 0x7abbfc40,
+    0x0dc9a184, 0x96ba4b50, 0xe05f726d, 0x7b2c98b9, 0x0d950017,
+    0x96e6eac3, 0xe003d3fe, 0x7b70392a, 0x0d70e2a2, 0x96030876,
+    0xe0e6314b, 0x7b95db9f, 0x0d2c4331, 0x965fa9e5, 0xe0ba90d8,
+    0x7bc97a0c},
+   {0x00000000, 0x172864c0, 0x2e50c980, 0x3978ad40, 0x5ca19300,
+    0x4b89f7c0, 0x72f15a80, 0x65d93e40, 0xb9432600, 0xae6b42c0,
+    0x9713ef80, 0x803b8b40, 0xe5e2b500, 0xf2cad1c0, 0xcbb27c80,
+    0xdc9a1840, 0xa9f74a41, 0xbedf2e81, 0x87a783c1, 0x908fe701,
+    0xf556d941, 0xe27ebd81, 0xdb0610c1, 0xcc2e7401, 0x10b46c41,
+    0x079c0881, 0x3ee4a5c1, 0x29ccc101, 0x4c15ff41, 0x5b3d9b81,
+    0x624536c1, 0x756d5201, 0x889f92c3, 0x9fb7f603, 0xa6cf5b43,
+    0xb1e73f83, 0xd43e01c3, 0xc3166503, 0xfa6ec843, 0xed46ac83,
+    0x31dcb4c3, 0x26f4d003, 0x1f8c7d43, 0x08a41983, 0x6d7d27c3,
+    0x7a554303, 0x432dee43, 0x54058a83, 0x2168d882, 0x3640bc42,
+    0x0f381102, 0x181075c2, 0x7dc94b82, 0x6ae12f42, 0x53998202,
+    0x44b1e6c2, 0x982bfe82, 0x8f039a42, 0xb67b3702, 0xa15353c2,
+    0xc48a6d82, 0xd3a20942, 0xeadaa402, 0xfdf2c0c2, 0xca4e23c7,
+    0xdd664707, 0xe41eea47, 0xf3368e87, 0x96efb0c7, 0x81c7d407,
+    0xb8bf7947, 0xaf971d87, 0x730d05c7, 0x64256107, 0x5d5dcc47,
+    0x4a75a887, 0x2fac96c7, 0x3884f207, 0x01fc5f47, 0x16d43b87,
+    0x63b96986, 0x74910d46, 0x4de9a006, 0x5ac1c4c6, 0x3f18fa86,
+    0x28309e46, 0x11483306, 0x066057c6, 0xdafa4f86, 0xcdd22b46,
+    0xf4aa8606, 0xe382e2c6, 0x865bdc86, 0x9173b846, 0xa80b1506,
+    0xbf2371c6, 0x42d1b104, 0x55f9d5c4, 0x6c817884, 0x7ba91c44,
+    0x1e702204, 0x095846c4, 0x3020eb84, 0x27088f44, 0xfb929704,
+    0xecbaf3c4, 0xd5c25e84, 0xc2ea3a44, 0xa7330404, 0xb01b60c4,
+    0x8963cd84, 0x9e4ba944, 0xeb26fb45, 0xfc0e9f85, 0xc57632c5,
+    0xd25e5605, 0xb7876845, 0xa0af0c85, 0x99d7a1c5, 0x8effc505,
+    0x5265dd45, 0x454db985, 0x7c3514c5, 0x6b1d7005, 0x0ec44e45,
+    0x19ec2a85, 0x209487c5, 0x37bce305, 0x4fed41cf, 0x58c5250f,
+    0x61bd884f, 0x7695ec8f, 0x134cd2cf, 0x0464b60f, 0x3d1c1b4f,
+    0x2a347f8f, 0xf6ae67cf, 0xe186030f, 0xd8feae4f, 0xcfd6ca8f,
+    0xaa0ff4cf, 0xbd27900f, 0x845f3d4f, 0x9377598f, 0xe61a0b8e,
+    0xf1326f4e, 0xc84ac20e, 0xdf62a6ce, 0xbabb988e, 0xad93fc4e,
+    0x94eb510e, 0x83c335ce, 0x5f592d8e, 0x4871494e, 0x7109e40e,
+    0x662180ce, 0x03f8be8e, 0x14d0da4e, 0x2da8770e, 0x3a8013ce,
+    0xc772d30c, 0xd05ab7cc, 0xe9221a8c, 0xfe0a7e4c, 0x9bd3400c,
+    0x8cfb24cc, 0xb583898c, 0xa2abed4c, 0x7e31f50c, 0x691991cc,
+    0x50613c8c, 0x4749584c, 0x2290660c, 0x35b802cc, 0x0cc0af8c,
+    0x1be8cb4c, 0x6e85994d, 0x79adfd8d, 0x40d550cd, 0x57fd340d,
+    0x32240a4d, 0x250c6e8d, 0x1c74c3cd, 0x0b5ca70d, 0xd7c6bf4d,
+    0xc0eedb8d, 0xf99676cd, 0xeebe120d, 0x8b672c4d, 0x9c4f488d,
+    0xa537e5cd, 0xb21f810d, 0x85a36208, 0x928b06c8, 0xabf3ab88,
+    0xbcdbcf48, 0xd902f108, 0xce2a95c8, 0xf7523888, 0xe07a5c48,
+    0x3ce04408, 0x2bc820c8, 0x12b08d88, 0x0598e948, 0x6041d708,
+    0x7769b3c8, 0x4e111e88, 0x59397a48, 0x2c542849, 0x3b7c4c89,
+    0x0204e1c9, 0x152c8509, 0x70f5bb49, 0x67dddf89, 0x5ea572c9,
+    0x498d1609, 0x95170e49, 0x823f6a89, 0xbb47c7c9, 0xac6fa309,
+    0xc9b69d49, 0xde9ef989, 0xe7e654c9, 0xf0ce3009, 0x0d3cf0cb,
+    0x1a14940b, 0x236c394b, 0x34445d8b, 0x519d63cb, 0x46b5070b,
+    0x7fcdaa4b, 0x68e5ce8b, 0xb47fd6cb, 0xa357b20b, 0x9a2f1f4b,
+    0x8d077b8b, 0xe8de45cb, 0xfff6210b, 0xc68e8c4b, 0xd1a6e88b,
+    0xa4cbba8a, 0xb3e3de4a, 0x8a9b730a, 0x9db317ca, 0xf86a298a,
+    0xef424d4a, 0xd63ae00a, 0xc11284ca, 0x1d889c8a, 0x0aa0f84a,
+    0x33d8550a, 0x24f031ca, 0x41290f8a, 0x56016b4a, 0x6f79c60a,
+    0x7851a2ca},
+   {0x00000000, 0x9fda839e, 0xe4c4017d, 0x7b1e82e3, 0x12f904bb,
+    0x8d238725, 0xf63d05c6, 0x69e78658, 0x25f20976, 0xba288ae8,
+    0xc136080b, 0x5eec8b95, 0x370b0dcd, 0xa8d18e53, 0xd3cf0cb0,
+    0x4c158f2e, 0x4be412ec, 0xd43e9172, 0xaf201391, 0x30fa900f,
+    0x591d1657, 0xc6c795c9, 0xbdd9172a, 0x220394b4, 0x6e161b9a,
+    0xf1cc9804, 0x8ad21ae7, 0x15089979, 0x7cef1f21, 0xe3359cbf,
+    0x982b1e5c, 0x07f19dc2, 0x97c825d8, 0x0812a646, 0x730c24a5,
+    0xecd6a73b, 0x85312163, 0x1aeba2fd, 0x61f5201e, 0xfe2fa380,
+    0xb23a2cae, 0x2de0af30, 0x56fe2dd3, 0xc924ae4d, 0xa0c32815,
+    0x3f19ab8b, 0x44072968, 0xdbddaaf6, 0xdc2c3734, 0x43f6b4aa,
+    0x38e83649, 0xa732b5d7, 0xced5338f, 0x510fb011, 0x2a1132f2,
+    0xb5cbb16c, 0xf9de3e42, 0x6604bddc, 0x1d1a3f3f, 0x82c0bca1,
+    0xeb273af9, 0x74fdb967, 0x0fe33b84, 0x9039b81a, 0xf4e14df1,
+    0x6b3bce6f, 0x10254c8c, 0x8fffcf12, 0xe618494a, 0x79c2cad4,
+    0x02dc4837, 0x9d06cba9, 0xd1134487, 0x4ec9c719, 0x35d745fa,
+    0xaa0dc664, 0xc3ea403c, 0x5c30c3a2, 0x272e4141, 0xb8f4c2df,
+    0xbf055f1d, 0x20dfdc83, 0x5bc15e60, 0xc41bddfe, 0xadfc5ba6,
+    0x3226d838, 0x49385adb, 0xd6e2d945, 0x9af7566b, 0x052dd5f5,
+    0x7e335716, 0xe1e9d488, 0x880e52d0, 0x17d4d14e, 0x6cca53ad,
+    0xf310d033, 0x63296829, 0xfcf3ebb7, 0x87ed6954, 0x1837eaca,
+    0x71d06c92, 0xee0aef0c, 0x95146def, 0x0aceee71, 0x46db615f,
+    0xd901e2c1, 0xa21f6022, 0x3dc5e3bc, 0x542265e4, 0xcbf8e67a,
+    0xb0e66499, 0x2f3ce707, 0x28cd7ac5, 0xb717f95b, 0xcc097bb8,
+    0x53d3f826, 0x3a347e7e, 0xa5eefde0, 0xdef07f03, 0x412afc9d,
+    0x0d3f73b3, 0x92e5f02d, 0xe9fb72ce, 0x7621f150, 0x1fc67708,
+    0x801cf496, 0xfb027675, 0x64d8f5eb, 0x32b39da3, 0xad691e3d,
+    0xd6779cde, 0x49ad1f40, 0x204a9918, 0xbf901a86, 0xc48e9865,
+    0x5b541bfb, 0x174194d5, 0x889b174b, 0xf38595a8, 0x6c5f1636,
+    0x05b8906e, 0x9a6213f0, 0xe17c9113, 0x7ea6128d, 0x79578f4f,
+    0xe68d0cd1, 0x9d938e32, 0x02490dac, 0x6bae8bf4, 0xf474086a,
+    0x8f6a8a89, 0x10b00917, 0x5ca58639, 0xc37f05a7, 0xb8618744,
+    0x27bb04da, 0x4e5c8282, 0xd186011c, 0xaa9883ff, 0x35420061,
+    0xa57bb87b, 0x3aa13be5, 0x41bfb906, 0xde653a98, 0xb782bcc0,
+    0x28583f5e, 0x5346bdbd, 0xcc9c3e23, 0x8089b10d, 0x1f533293,
+    0x644db070, 0xfb9733ee, 0x9270b5b6, 0x0daa3628, 0x76b4b4cb,
+    0xe96e3755, 0xee9faa97, 0x71452909, 0x0a5babea, 0x95812874,
+    0xfc66ae2c, 0x63bc2db2, 0x18a2af51, 0x87782ccf, 0xcb6da3e1,
+    0x54b7207f, 0x2fa9a29c, 0xb0732102, 0xd994a75a, 0x464e24c4,
+    0x3d50a627, 0xa28a25b9, 0xc652d052, 0x598853cc, 0x2296d12f,
+    0xbd4c52b1, 0xd4abd4e9, 0x4b715777, 0x306fd594, 0xafb5560a,
+    0xe3a0d924, 0x7c7a5aba, 0x0764d859, 0x98be5bc7, 0xf159dd9f,
+    0x6e835e01, 0x159ddce2, 0x8a475f7c, 0x8db6c2be, 0x126c4120,
+    0x6972c3c3, 0xf6a8405d, 0x9f4fc605, 0x0095459b, 0x7b8bc778,
+    0xe45144e6, 0xa844cbc8, 0x379e4856, 0x4c80cab5, 0xd35a492b,
+    0xbabdcf73, 0x25674ced, 0x5e79ce0e, 0xc1a34d90, 0x519af58a,
+    0xce407614, 0xb55ef4f7, 0x2a847769, 0x4363f131, 0xdcb972af,
+    0xa7a7f04c, 0x387d73d2, 0x7468fcfc, 0xebb27f62, 0x90acfd81,
+    0x0f767e1f, 0x6691f847, 0xf94b7bd9, 0x8255f93a, 0x1d8f7aa4,
+    0x1a7ee766, 0x85a464f8, 0xfebae61b, 0x61606585, 0x0887e3dd,
+    0x975d6043, 0xec43e2a0, 0x7399613e, 0x3f8cee10, 0xa0566d8e,
+    0xdb48ef6d, 0x44926cf3, 0x2d75eaab, 0xb2af6935, 0xc9b1ebd6,
+    0x566b6848},
+   {0x00000000, 0x65673b46, 0xcace768c, 0xafa94dca, 0x4eedeb59,
+    0x2b8ad01f, 0x84239dd5, 0xe144a693, 0x9ddbd6b2, 0xf8bcedf4,
+    0x5715a03e, 0x32729b78, 0xd3363deb, 0xb65106ad, 0x19f84b67,
+    0x7c9f7021, 0xe0c6ab25, 0x85a19063, 0x2a08dda9, 0x4f6fe6ef,
+    0xae2b407c, 0xcb4c7b3a, 0x64e536f0, 0x01820db6, 0x7d1d7d97,
+    0x187a46d1, 0xb7d30b1b, 0xd2b4305d, 0x33f096ce, 0x5697ad88,
+    0xf93ee042, 0x9c59db04, 0x1afc500b, 0x7f9b6b4d, 0xd0322687,
+    0xb5551dc1, 0x5411bb52, 0x31768014, 0x9edfcdde, 0xfbb8f698,
+    0x872786b9, 0xe240bdff, 0x4de9f035, 0x288ecb73, 0xc9ca6de0,
+    0xacad56a6, 0x03041b6c, 0x6663202a, 0xfa3afb2e, 0x9f5dc068,
+    0x30f48da2, 0x5593b6e4, 0xb4d71077, 0xd1b02b31, 0x7e1966fb,
+    0x1b7e5dbd, 0x67e12d9c, 0x028616da, 0xad2f5b10, 0xc8486056,
+    0x290cc6c5, 0x4c6bfd83, 0xe3c2b049, 0x86a58b0f, 0x35f8a016,
+    0x509f9b50, 0xff36d69a, 0x9a51eddc, 0x7b154b4f, 0x1e727009,
+    0xb1db3dc3, 0xd4bc0685, 0xa82376a4, 0xcd444de2, 0x62ed0028,
+    0x078a3b6e, 0xe6ce9dfd, 0x83a9a6bb, 0x2c00eb71, 0x4967d037,
+    0xd53e0b33, 0xb0593075, 0x1ff07dbf, 0x7a9746f9, 0x9bd3e06a,
+    0xfeb4db2c, 0x511d96e6, 0x347aada0, 0x48e5dd81, 0x2d82e6c7,
+    0x822bab0d, 0xe74c904b, 0x060836d8, 0x636f0d9e, 0xccc64054,
+    0xa9a17b12, 0x2f04f01d, 0x4a63cb5b, 0xe5ca8691, 0x80adbdd7,
+    0x61e91b44, 0x048e2002, 0xab276dc8, 0xce40568e, 0xb2df26af,
+    0xd7b81de9, 0x78115023, 0x1d766b65, 0xfc32cdf6, 0x9955f6b0,
+    0x36fcbb7a, 0x539b803c, 0xcfc25b38, 0xaaa5607e, 0x050c2db4,
+    0x606b16f2, 0x812fb061, 0xe4488b27, 0x4be1c6ed, 0x2e86fdab,
+    0x52198d8a, 0x377eb6cc, 0x98d7fb06, 0xfdb0c040, 0x1cf466d3,
+    0x79935d95, 0xd63a105f, 0xb35d2b19, 0x6bf1402c, 0x0e967b6a,
+    0xa13f36a0, 0xc4580de6, 0x251cab75, 0x407b9033, 0xefd2ddf9,
+    0x8ab5e6bf, 0xf62a969e, 0x934dadd8, 0x3ce4e012, 0x5983db54,
+    0xb8c77dc7, 0xdda04681, 0x72090b4b, 0x176e300d, 0x8b37eb09,
+    0xee50d04f, 0x41f99d85, 0x249ea6c3, 0xc5da0050, 0xa0bd3b16,
+    0x0f1476dc, 0x6a734d9a, 0x16ec3dbb, 0x738b06fd, 0xdc224b37,
+    0xb9457071, 0x5801d6e2, 0x3d66eda4, 0x92cfa06e, 0xf7a89b28,
+    0x710d1027, 0x146a2b61, 0xbbc366ab, 0xdea45ded, 0x3fe0fb7e,
+    0x5a87c038, 0xf52e8df2, 0x9049b6b4, 0xecd6c695, 0x89b1fdd3,
+    0x2618b019, 0x437f8b5f, 0xa23b2dcc, 0xc75c168a, 0x68f55b40,
+    0x0d926006, 0x91cbbb02, 0xf4ac8044, 0x5b05cd8e, 0x3e62f6c8,
+    0xdf26505b, 0xba416b1d, 0x15e826d7, 0x708f1d91, 0x0c106db0,
+    0x697756f6, 0xc6de1b3c, 0xa3b9207a, 0x42fd86e9, 0x279abdaf,
+    0x8833f065, 0xed54cb23, 0x5e09e03a, 0x3b6edb7c, 0x94c796b6,
+    0xf1a0adf0, 0x10e40b63, 0x75833025, 0xda2a7def, 0xbf4d46a9,
+    0xc3d23688, 0xa6b50dce, 0x091c4004, 0x6c7b7b42, 0x8d3fddd1,
+    0xe858e697, 0x47f1ab5d, 0x2296901b, 0xbecf4b1f, 0xdba87059,
+    0x74013d93, 0x116606d5, 0xf022a046, 0x95459b00, 0x3aecd6ca,
+    0x5f8bed8c, 0x23149dad, 0x4673a6eb, 0xe9daeb21, 0x8cbdd067,
+    0x6df976f4, 0x089e4db2, 0xa7370078, 0xc2503b3e, 0x44f5b031,
+    0x21928b77, 0x8e3bc6bd, 0xeb5cfdfb, 0x0a185b68, 0x6f7f602e,
+    0xc0d62de4, 0xa5b116a2, 0xd92e6683, 0xbc495dc5, 0x13e0100f,
+    0x76872b49, 0x97c38dda, 0xf2a4b69c, 0x5d0dfb56, 0x386ac010,
+    0xa4331b14, 0xc1542052, 0x6efd6d98, 0x0b9a56de, 0xeadef04d,
+    0x8fb9cb0b, 0x201086c1, 0x4577bd87, 0x39e8cda6, 0x5c8ff6e0,
+    0xf326bb2a, 0x9641806c, 0x770526ff, 0x12621db9, 0xbdcb5073,
+    0xd8ac6b35},
+   {0x00000000, 0xd7e28058, 0x74b406f1, 0xa35686a9, 0xe9680de2,
+    0x3e8a8dba, 0x9ddc0b13, 0x4a3e8b4b, 0x09a11d85, 0xde439ddd,
+    0x7d151b74, 0xaaf79b2c, 0xe0c91067, 0x372b903f, 0x947d1696,
+    0x439f96ce, 0x13423b0a, 0xc4a0bb52, 0x67f63dfb, 0xb014bda3,
+    0xfa2a36e8, 0x2dc8b6b0, 0x8e9e3019, 0x597cb041, 0x1ae3268f,
+    0xcd01a6d7, 0x6e57207e, 0xb9b5a026, 0xf38b2b6d, 0x2469ab35,
+    0x873f2d9c, 0x50ddadc4, 0x26847614, 0xf166f64c, 0x523070e5,
+    0x85d2f0bd, 0xcfec7bf6, 0x180efbae, 0xbb587d07, 0x6cbafd5f,
+    0x2f256b91, 0xf8c7ebc9, 0x5b916d60, 0x8c73ed38, 0xc64d6673,
+    0x11afe62b, 0xb2f96082, 0x651be0da, 0x35c64d1e, 0xe224cd46,
+    0x41724bef, 0x9690cbb7, 0xdcae40fc, 0x0b4cc0a4, 0xa81a460d,
+    0x7ff8c655, 0x3c67509b, 0xeb85d0c3, 0x48d3566a, 0x9f31d632,
+    0xd50f5d79, 0x02eddd21, 0xa1bb5b88, 0x7659dbd0, 0x4d08ec28,
+    0x9aea6c70, 0x39bcead9, 0xee5e6a81, 0xa460e1ca, 0x73826192,
+    0xd0d4e73b, 0x07366763, 0x44a9f1ad, 0x934b71f5, 0x301df75c,
+    0xe7ff7704, 0xadc1fc4f, 0x7a237c17, 0xd975fabe, 0x0e977ae6,
+    0x5e4ad722, 0x89a8577a, 0x2afed1d3, 0xfd1c518b, 0xb722dac0,
+    0x60c05a98, 0xc396dc31, 0x14745c69, 0x57ebcaa7, 0x80094aff,
+    0x235fcc56, 0xf4bd4c0e, 0xbe83c745, 0x6961471d, 0xca37c1b4,
+    0x1dd541ec, 0x6b8c9a3c, 0xbc6e1a64, 0x1f389ccd, 0xc8da1c95,
+    0x82e497de, 0x55061786, 0xf650912f, 0x21b21177, 0x622d87b9,
+    0xb5cf07e1, 0x16998148, 0xc17b0110, 0x8b458a5b, 0x5ca70a03,
+    0xfff18caa, 0x28130cf2, 0x78cea136, 0xaf2c216e, 0x0c7aa7c7,
+    0xdb98279f, 0x91a6acd4, 0x46442c8c, 0xe512aa25, 0x32f02a7d,
+    0x716fbcb3, 0xa68d3ceb, 0x05dbba42, 0xd2393a1a, 0x9807b151,
+    0x4fe53109, 0xecb3b7a0, 0x3b5137f8, 0x9a11d850, 0x4df35808,
+    0xeea5dea1, 0x39475ef9, 0x7379d5b2, 0xa49b55ea, 0x07cdd343,
+    0xd02f531b, 0x93b0c5d5, 0x4452458d, 0xe704c324, 0x30e6437c,
+    0x7ad8c837, 0xad3a486f, 0x0e6ccec6, 0xd98e4e9e, 0x8953e35a,
+    0x5eb16302, 0xfde7e5ab, 0x2a0565f3, 0x603beeb8, 0xb7d96ee0,
+    0x148fe849, 0xc36d6811, 0x80f2fedf, 0x57107e87, 0xf446f82e,
+    0x23a47876, 0x699af33d, 0xbe787365, 0x1d2ef5cc, 0xcacc7594,
+    0xbc95ae44, 0x6b772e1c, 0xc821a8b5, 0x1fc328ed, 0x55fda3a6,
+    0x821f23fe, 0x2149a557, 0xf6ab250f, 0xb534b3c1, 0x62d63399,
+    0xc180b530, 0x16623568, 0x5c5cbe23, 0x8bbe3e7b, 0x28e8b8d2,
+    0xff0a388a, 0xafd7954e, 0x78351516, 0xdb6393bf, 0x0c8113e7,
+    0x46bf98ac, 0x915d18f4, 0x320b9e5d, 0xe5e91e05, 0xa67688cb,
+    0x71940893, 0xd2c28e3a, 0x05200e62, 0x4f1e8529, 0x98fc0571,
+    0x3baa83d8, 0xec480380, 0xd7193478, 0x00fbb420, 0xa3ad3289,
+    0x744fb2d1, 0x3e71399a, 0xe993b9c2, 0x4ac53f6b, 0x9d27bf33,
+    0xdeb829fd, 0x095aa9a5, 0xaa0c2f0c, 0x7deeaf54, 0x37d0241f,
+    0xe032a447, 0x436422ee, 0x9486a2b6, 0xc45b0f72, 0x13b98f2a,
+    0xb0ef0983, 0x670d89db, 0x2d330290, 0xfad182c8, 0x59870461,
+    0x8e658439, 0xcdfa12f7, 0x1a1892af, 0xb94e1406, 0x6eac945e,
+    0x24921f15, 0xf3709f4d, 0x502619e4, 0x87c499bc, 0xf19d426c,
+    0x267fc234, 0x8529449d, 0x52cbc4c5, 0x18f54f8e, 0xcf17cfd6,
+    0x6c41497f, 0xbba3c927, 0xf83c5fe9, 0x2fdedfb1, 0x8c885918,
+    0x5b6ad940, 0x1154520b, 0xc6b6d253, 0x65e054fa, 0xb202d4a2,
+    0xe2df7966, 0x353df93e, 0x966b7f97, 0x4189ffcf, 0x0bb77484,
+    0xdc55f4dc, 0x7f037275, 0xa8e1f22d, 0xeb7e64e3, 0x3c9ce4bb,
+    0x9fca6212, 0x4828e24a, 0x02166901, 0xd5f4e959, 0x76a26ff0,
+    0xa140efa8},
+   {0x00000000, 0xef52b6e1, 0x05d46b83, 0xea86dd62, 0x0ba8d706,
+    0xe4fa61e7, 0x0e7cbc85, 0xe12e0a64, 0x1751ae0c, 0xf80318ed,
+    0x1285c58f, 0xfdd7736e, 0x1cf9790a, 0xf3abcfeb, 0x192d1289,
+    0xf67fa468, 0x2ea35c18, 0xc1f1eaf9, 0x2b77379b, 0xc425817a,
+    0x250b8b1e, 0xca593dff, 0x20dfe09d, 0xcf8d567c, 0x39f2f214,
+    0xd6a044f5, 0x3c269997, 0xd3742f76, 0x325a2512, 0xdd0893f3,
+    0x378e4e91, 0xd8dcf870, 0x5d46b830, 0xb2140ed1, 0x5892d3b3,
+    0xb7c06552, 0x56ee6f36, 0xb9bcd9d7, 0x533a04b5, 0xbc68b254,
+    0x4a17163c, 0xa545a0dd, 0x4fc37dbf, 0xa091cb5e, 0x41bfc13a,
+    0xaeed77db, 0x446baab9, 0xab391c58, 0x73e5e428, 0x9cb752c9,
+    0x76318fab, 0x9963394a, 0x784d332e, 0x971f85cf, 0x7d9958ad,
+    0x92cbee4c, 0x64b44a24, 0x8be6fcc5, 0x616021a7, 0x8e329746,
+    0x6f1c9d22, 0x804e2bc3, 0x6ac8f6a1, 0x859a4040, 0xba8d7060,
+    0x55dfc681, 0xbf591be3, 0x500bad02, 0xb125a766, 0x5e771187,
+    0xb4f1cce5, 0x5ba37a04, 0xaddcde6c, 0x428e688d, 0xa808b5ef,
+    0x475a030e, 0xa674096a, 0x4926bf8b, 0xa3a062e9, 0x4cf2d408,
+    0x942e2c78, 0x7b7c9a99, 0x91fa47fb, 0x7ea8f11a, 0x9f86fb7e,
+    0x70d44d9f, 0x9a5290fd, 0x7500261c, 0x837f8274, 0x6c2d3495,
+    0x86abe9f7, 0x69f95f16, 0x88d75572, 0x6785e393, 0x8d033ef1,
+    0x62518810, 0xe7cbc850, 0x08997eb1, 0xe21fa3d3, 0x0d4d1532,
+    0xec631f56, 0x0331a9b7, 0xe9b774d5, 0x06e5c234, 0xf09a665c,
+    0x1fc8d0bd, 0xf54e0ddf, 0x1a1cbb3e, 0xfb32b15a, 0x146007bb,
+    0xfee6dad9, 0x11b46c38, 0xc9689448, 0x263a22a9, 0xccbcffcb,
+    0x23ee492a, 0xc2c0434e, 0x2d92f5af, 0xc71428cd, 0x28469e2c,
+    0xde393a44, 0x316b8ca5, 0xdbed51c7, 0x34bfe726, 0xd591ed42,
+    0x3ac35ba3, 0xd04586c1, 0x3f173020, 0xae6be681, 0x41395060,
+    0xabbf8d02, 0x44ed3be3, 0xa5c33187, 0x4a918766, 0xa0175a04,
+    0x4f45ece5, 0xb93a488d, 0x5668fe6c, 0xbcee230e, 0x53bc95ef,
+    0xb2929f8b, 0x5dc0296a, 0xb746f408, 0x581442e9, 0x80c8ba99,
+    0x6f9a0c78, 0x851cd11a, 0x6a4e67fb, 0x8b606d9f, 0x6432db7e,
+    0x8eb4061c, 0x61e6b0fd, 0x97991495, 0x78cba274, 0x924d7f16,
+    0x7d1fc9f7, 0x9c31c393, 0x73637572, 0x99e5a810, 0x76b71ef1,
+    0xf32d5eb1, 0x1c7fe850, 0xf6f93532, 0x19ab83d3, 0xf88589b7,
+    0x17d73f56, 0xfd51e234, 0x120354d5, 0xe47cf0bd, 0x0b2e465c,
+    0xe1a89b3e, 0x0efa2ddf, 0xefd427bb, 0x0086915a, 0xea004c38,
+    0x0552fad9, 0xdd8e02a9, 0x32dcb448, 0xd85a692a, 0x3708dfcb,
+    0xd626d5af, 0x3974634e, 0xd3f2be2c, 0x3ca008cd, 0xcadfaca5,
+    0x258d1a44, 0xcf0bc726, 0x205971c7, 0xc1777ba3, 0x2e25cd42,
+    0xc4a31020, 0x2bf1a6c1, 0x14e696e1, 0xfbb42000, 0x1132fd62,
+    0xfe604b83, 0x1f4e41e7, 0xf01cf706, 0x1a9a2a64, 0xf5c89c85,
+    0x03b738ed, 0xece58e0c, 0x0663536e, 0xe931e58f, 0x081fefeb,
+    0xe74d590a, 0x0dcb8468, 0xe2993289, 0x3a45caf9, 0xd5177c18,
+    0x3f91a17a, 0xd0c3179b, 0x31ed1dff, 0xdebfab1e, 0x3439767c,
+    0xdb6bc09d, 0x2d1464f5, 0xc246d214, 0x28c00f76, 0xc792b997,
+    0x26bcb3f3, 0xc9ee0512, 0x2368d870, 0xcc3a6e91, 0x49a02ed1,
+    0xa6f29830, 0x4c744552, 0xa326f3b3, 0x4208f9d7, 0xad5a4f36,
+    0x47dc9254, 0xa88e24b5, 0x5ef180dd, 0xb1a3363c, 0x5b25eb5e,
+    0xb4775dbf, 0x555957db, 0xba0be13a, 0x508d3c58, 0xbfdf8ab9,
+    0x670372c9, 0x8851c428, 0x62d7194a, 0x8d85afab, 0x6caba5cf,
+    0x83f9132e, 0x697fce4c, 0x862d78ad, 0x7052dcc5, 0x9f006a24,
+    0x7586b746, 0x9ad401a7, 0x7bfa0bc3, 0x94a8bd22, 0x7e2e6040,
+    0x917cd6a1},
+   {0x00000000, 0x87a6cb43, 0xd43c90c7, 0x539a5b84, 0x730827cf,
+    0xf4aeec8c, 0xa734b708, 0x20927c4b, 0xe6104f9e, 0x61b684dd,
+    0x322cdf59, 0xb58a141a, 0x95186851, 0x12bea312, 0x4124f896,
+    0xc68233d5, 0x1751997d, 0x90f7523e, 0xc36d09ba, 0x44cbc2f9,
+    0x6459beb2, 0xe3ff75f1, 0xb0652e75, 0x37c3e536, 0xf141d6e3,
+    0x76e71da0, 0x257d4624, 0xa2db8d67, 0x8249f12c, 0x05ef3a6f,
+    0x567561eb, 0xd1d3aaa8, 0x2ea332fa, 0xa905f9b9, 0xfa9fa23d,
+    0x7d39697e, 0x5dab1535, 0xda0dde76, 0x899785f2, 0x0e314eb1,
+    0xc8b37d64, 0x4f15b627, 0x1c8feda3, 0x9b2926e0, 0xbbbb5aab,
+    0x3c1d91e8, 0x6f87ca6c, 0xe821012f, 0x39f2ab87, 0xbe5460c4,
+    0xedce3b40, 0x6a68f003, 0x4afa8c48, 0xcd5c470b, 0x9ec61c8f,
+    0x1960d7cc, 0xdfe2e419, 0x58442f5a, 0x0bde74de, 0x8c78bf9d,
+    0xaceac3d6, 0x2b4c0895, 0x78d65311, 0xff709852, 0x5d4665f4,
+    0xdae0aeb7, 0x897af533, 0x0edc3e70, 0x2e4e423b, 0xa9e88978,
+    0xfa72d2fc, 0x7dd419bf, 0xbb562a6a, 0x3cf0e129, 0x6f6abaad,
+    0xe8cc71ee, 0xc85e0da5, 0x4ff8c6e6, 0x1c629d62, 0x9bc45621,
+    0x4a17fc89, 0xcdb137ca, 0x9e2b6c4e, 0x198da70d, 0x391fdb46,
+    0xbeb91005, 0xed234b81, 0x6a8580c2, 0xac07b317, 0x2ba17854,
+    0x783b23d0, 0xff9de893, 0xdf0f94d8, 0x58a95f9b, 0x0b33041f,
+    0x8c95cf5c, 0x73e5570e, 0xf4439c4d, 0xa7d9c7c9, 0x207f0c8a,
+    0x00ed70c1, 0x874bbb82, 0xd4d1e006, 0x53772b45, 0x95f51890,
+    0x1253d3d3, 0x41c98857, 0xc66f4314, 0xe6fd3f5f, 0x615bf41c,
+    0x32c1af98, 0xb56764db, 0x64b4ce73, 0xe3120530, 0xb0885eb4,
+    0x372e95f7, 0x17bce9bc, 0x901a22ff, 0xc380797b, 0x4426b238,
+    0x82a481ed, 0x05024aae, 0x5698112a, 0xd13eda69, 0xf1aca622,
+    0x760a6d61, 0x259036e5, 0xa236fda6, 0xba8ccbe8, 0x3d2a00ab,
+    0x6eb05b2f, 0xe916906c, 0xc984ec27, 0x4e222764, 0x1db87ce0,
+    0x9a1eb7a3, 0x5c9c8476, 0xdb3a4f35, 0x88a014b1, 0x0f06dff2,
+    0x2f94a3b9, 0xa83268fa, 0xfba8337e, 0x7c0ef83d, 0xaddd5295,
+    0x2a7b99d6, 0x79e1c252, 0xfe470911, 0xded5755a, 0x5973be19,
+    0x0ae9e59d, 0x8d4f2ede, 0x4bcd1d0b, 0xcc6bd648, 0x9ff18dcc,
+    0x1857468f, 0x38c53ac4, 0xbf63f187, 0xecf9aa03, 0x6b5f6140,
+    0x942ff912, 0x13893251, 0x401369d5, 0xc7b5a296, 0xe727dedd,
+    0x6081159e, 0x331b4e1a, 0xb4bd8559, 0x723fb68c, 0xf5997dcf,
+    0xa603264b, 0x21a5ed08, 0x01379143, 0x86915a00, 0xd50b0184,
+    0x52adcac7, 0x837e606f, 0x04d8ab2c, 0x5742f0a8, 0xd0e43beb,
+    0xf07647a0, 0x77d08ce3, 0x244ad767, 0xa3ec1c24, 0x656e2ff1,
+    0xe2c8e4b2, 0xb152bf36, 0x36f47475, 0x1666083e, 0x91c0c37d,
+    0xc25a98f9, 0x45fc53ba, 0xe7caae1c, 0x606c655f, 0x33f63edb,
+    0xb450f598, 0x94c289d3, 0x13644290, 0x40fe1914, 0xc758d257,
+    0x01dae182, 0x867c2ac1, 0xd5e67145, 0x5240ba06, 0x72d2c64d,
+    0xf5740d0e, 0xa6ee568a, 0x21489dc9, 0xf09b3761, 0x773dfc22,
+    0x24a7a7a6, 0xa3016ce5, 0x839310ae, 0x0435dbed, 0x57af8069,
+    0xd0094b2a, 0x168b78ff, 0x912db3bc, 0xc2b7e838, 0x4511237b,
+    0x65835f30, 0xe2259473, 0xb1bfcff7, 0x361904b4, 0xc9699ce6,
+    0x4ecf57a5, 0x1d550c21, 0x9af3c762, 0xba61bb29, 0x3dc7706a,
+    0x6e5d2bee, 0xe9fbe0ad, 0x2f79d378, 0xa8df183b, 0xfb4543bf,
+    0x7ce388fc, 0x5c71f4b7, 0xdbd73ff4, 0x884d6470, 0x0febaf33,
+    0xde38059b, 0x599eced8, 0x0a04955c, 0x8da25e1f, 0xad302254,
+    0x2a96e917, 0x790cb293, 0xfeaa79d0, 0x38284a05, 0xbf8e8146,
+    0xec14dac2, 0x6bb21181, 0x4b206dca, 0xcc86a689, 0x9f1cfd0d,
+    0x18ba364e}};
+
+local const z_word_t FAR crc_braid_big_table[][256] = {
+   {0x0000000000000000, 0x43cba68700000000, 0xc7903cd400000000,
+    0x845b9a5300000000, 0xcf27087300000000, 0x8cecaef400000000,
+    0x08b734a700000000, 0x4b7c922000000000, 0x9e4f10e600000000,
+    0xdd84b66100000000, 0x59df2c3200000000, 0x1a148ab500000000,
+    0x5168189500000000, 0x12a3be1200000000, 0x96f8244100000000,
+    0xd53382c600000000, 0x7d99511700000000, 0x3e52f79000000000,
+    0xba096dc300000000, 0xf9c2cb4400000000, 0xb2be596400000000,
+    0xf175ffe300000000, 0x752e65b000000000, 0x36e5c33700000000,
+    0xe3d641f100000000, 0xa01de77600000000, 0x24467d2500000000,
+    0x678ddba200000000, 0x2cf1498200000000, 0x6f3aef0500000000,
+    0xeb61755600000000, 0xa8aad3d100000000, 0xfa32a32e00000000,
+    0xb9f905a900000000, 0x3da29ffa00000000, 0x7e69397d00000000,
+    0x3515ab5d00000000, 0x76de0dda00000000, 0xf285978900000000,
+    0xb14e310e00000000, 0x647db3c800000000, 0x27b6154f00000000,
+    0xa3ed8f1c00000000, 0xe026299b00000000, 0xab5abbbb00000000,
+    0xe8911d3c00000000, 0x6cca876f00000000, 0x2f0121e800000000,
+    0x87abf23900000000, 0xc46054be00000000, 0x403bceed00000000,
+    0x03f0686a00000000, 0x488cfa4a00000000, 0x0b475ccd00000000,
+    0x8f1cc69e00000000, 0xccd7601900000000, 0x19e4e2df00000000,
+    0x5a2f445800000000, 0xde74de0b00000000, 0x9dbf788c00000000,
+    0xd6c3eaac00000000, 0x95084c2b00000000, 0x1153d67800000000,
+    0x529870ff00000000, 0xf465465d00000000, 0xb7aee0da00000000,
+    0x33f57a8900000000, 0x703edc0e00000000, 0x3b424e2e00000000,
+    0x7889e8a900000000, 0xfcd272fa00000000, 0xbf19d47d00000000,
+    0x6a2a56bb00000000, 0x29e1f03c00000000, 0xadba6a6f00000000,
+    0xee71cce800000000, 0xa50d5ec800000000, 0xe6c6f84f00000000,
+    0x629d621c00000000, 0x2156c49b00000000, 0x89fc174a00000000,
+    0xca37b1cd00000000, 0x4e6c2b9e00000000, 0x0da78d1900000000,
+    0x46db1f3900000000, 0x0510b9be00000000, 0x814b23ed00000000,
+    0xc280856a00000000, 0x17b307ac00000000, 0x5478a12b00000000,
+    0xd0233b7800000000, 0x93e89dff00000000, 0xd8940fdf00000000,
+    0x9b5fa95800000000, 0x1f04330b00000000, 0x5ccf958c00000000,
+    0x0e57e57300000000, 0x4d9c43f400000000, 0xc9c7d9a700000000,
+    0x8a0c7f2000000000, 0xc170ed0000000000, 0x82bb4b8700000000,
+    0x06e0d1d400000000, 0x452b775300000000, 0x9018f59500000000,
+    0xd3d3531200000000, 0x5788c94100000000, 0x14436fc600000000,
+    0x5f3ffde600000000, 0x1cf45b6100000000, 0x98afc13200000000,
+    0xdb6467b500000000, 0x73ceb46400000000, 0x300512e300000000,
+    0xb45e88b000000000, 0xf7952e3700000000, 0xbce9bc1700000000,
+    0xff221a9000000000, 0x7b7980c300000000, 0x38b2264400000000,
+    0xed81a48200000000, 0xae4a020500000000, 0x2a11985600000000,
+    0x69da3ed100000000, 0x22a6acf100000000, 0x616d0a7600000000,
+    0xe536902500000000, 0xa6fd36a200000000, 0xe8cb8cba00000000,
+    0xab002a3d00000000, 0x2f5bb06e00000000, 0x6c9016e900000000,
+    0x27ec84c900000000, 0x6427224e00000000, 0xe07cb81d00000000,
+    0xa3b71e9a00000000, 0x76849c5c00000000, 0x354f3adb00000000,
+    0xb114a08800000000, 0xf2df060f00000000, 0xb9a3942f00000000,
+    0xfa6832a800000000, 0x7e33a8fb00000000, 0x3df80e7c00000000,
+    0x9552ddad00000000, 0xd6997b2a00000000, 0x52c2e17900000000,
+    0x110947fe00000000, 0x5a75d5de00000000, 0x19be735900000000,
+    0x9de5e90a00000000, 0xde2e4f8d00000000, 0x0b1dcd4b00000000,
+    0x48d66bcc00000000, 0xcc8df19f00000000, 0x8f46571800000000,
+    0xc43ac53800000000, 0x87f163bf00000000, 0x03aaf9ec00000000,
+    0x40615f6b00000000, 0x12f92f9400000000, 0x5132891300000000,
+    0xd569134000000000, 0x96a2b5c700000000, 0xddde27e700000000,
+    0x9e15816000000000, 0x1a4e1b3300000000, 0x5985bdb400000000,
+    0x8cb63f7200000000, 0xcf7d99f500000000, 0x4b2603a600000000,
+    0x08eda52100000000, 0x4391370100000000, 0x005a918600000000,
+    0x84010bd500000000, 0xc7caad5200000000, 0x6f607e8300000000,
+    0x2cabd80400000000, 0xa8f0425700000000, 0xeb3be4d000000000,
+    0xa04776f000000000, 0xe38cd07700000000, 0x67d74a2400000000,
+    0x241ceca300000000, 0xf12f6e6500000000, 0xb2e4c8e200000000,
+    0x36bf52b100000000, 0x7574f43600000000, 0x3e08661600000000,
+    0x7dc3c09100000000, 0xf9985ac200000000, 0xba53fc4500000000,
+    0x1caecae700000000, 0x5f656c6000000000, 0xdb3ef63300000000,
+    0x98f550b400000000, 0xd389c29400000000, 0x9042641300000000,
+    0x1419fe4000000000, 0x57d258c700000000, 0x82e1da0100000000,
+    0xc12a7c8600000000, 0x4571e6d500000000, 0x06ba405200000000,
+    0x4dc6d27200000000, 0x0e0d74f500000000, 0x8a56eea600000000,
+    0xc99d482100000000, 0x61379bf000000000, 0x22fc3d7700000000,
+    0xa6a7a72400000000, 0xe56c01a300000000, 0xae10938300000000,
+    0xeddb350400000000, 0x6980af5700000000, 0x2a4b09d000000000,
+    0xff788b1600000000, 0xbcb32d9100000000, 0x38e8b7c200000000,
+    0x7b23114500000000, 0x305f836500000000, 0x739425e200000000,
+    0xf7cfbfb100000000, 0xb404193600000000, 0xe69c69c900000000,
+    0xa557cf4e00000000, 0x210c551d00000000, 0x62c7f39a00000000,
+    0x29bb61ba00000000, 0x6a70c73d00000000, 0xee2b5d6e00000000,
+    0xade0fbe900000000, 0x78d3792f00000000, 0x3b18dfa800000000,
+    0xbf4345fb00000000, 0xfc88e37c00000000, 0xb7f4715c00000000,
+    0xf43fd7db00000000, 0x70644d8800000000, 0x33afeb0f00000000,
+    0x9b0538de00000000, 0xd8ce9e5900000000, 0x5c95040a00000000,
+    0x1f5ea28d00000000, 0x542230ad00000000, 0x17e9962a00000000,
+    0x93b20c7900000000, 0xd079aafe00000000, 0x054a283800000000,
+    0x46818ebf00000000, 0xc2da14ec00000000, 0x8111b26b00000000,
+    0xca6d204b00000000, 0x89a686cc00000000, 0x0dfd1c9f00000000,
+    0x4e36ba1800000000},
+   {0x0000000000000000, 0xe1b652ef00000000, 0x836bd40500000000,
+    0x62dd86ea00000000, 0x06d7a80b00000000, 0xe761fae400000000,
+    0x85bc7c0e00000000, 0x640a2ee100000000, 0x0cae511700000000,
+    0xed1803f800000000, 0x8fc5851200000000, 0x6e73d7fd00000000,
+    0x0a79f91c00000000, 0xebcfabf300000000, 0x89122d1900000000,
+    0x68a47ff600000000, 0x185ca32e00000000, 0xf9eaf1c100000000,
+    0x9b37772b00000000, 0x7a8125c400000000, 0x1e8b0b2500000000,
+    0xff3d59ca00000000, 0x9de0df2000000000, 0x7c568dcf00000000,
+    0x14f2f23900000000, 0xf544a0d600000000, 0x9799263c00000000,
+    0x762f74d300000000, 0x12255a3200000000, 0xf39308dd00000000,
+    0x914e8e3700000000, 0x70f8dcd800000000, 0x30b8465d00000000,
+    0xd10e14b200000000, 0xb3d3925800000000, 0x5265c0b700000000,
+    0x366fee5600000000, 0xd7d9bcb900000000, 0xb5043a5300000000,
+    0x54b268bc00000000, 0x3c16174a00000000, 0xdda045a500000000,
+    0xbf7dc34f00000000, 0x5ecb91a000000000, 0x3ac1bf4100000000,
+    0xdb77edae00000000, 0xb9aa6b4400000000, 0x581c39ab00000000,
+    0x28e4e57300000000, 0xc952b79c00000000, 0xab8f317600000000,
+    0x4a39639900000000, 0x2e334d7800000000, 0xcf851f9700000000,
+    0xad58997d00000000, 0x4ceecb9200000000, 0x244ab46400000000,
+    0xc5fce68b00000000, 0xa721606100000000, 0x4697328e00000000,
+    0x229d1c6f00000000, 0xc32b4e8000000000, 0xa1f6c86a00000000,
+    0x40409a8500000000, 0x60708dba00000000, 0x81c6df5500000000,
+    0xe31b59bf00000000, 0x02ad0b5000000000, 0x66a725b100000000,
+    0x8711775e00000000, 0xe5ccf1b400000000, 0x047aa35b00000000,
+    0x6cdedcad00000000, 0x8d688e4200000000, 0xefb508a800000000,
+    0x0e035a4700000000, 0x6a0974a600000000, 0x8bbf264900000000,
+    0xe962a0a300000000, 0x08d4f24c00000000, 0x782c2e9400000000,
+    0x999a7c7b00000000, 0xfb47fa9100000000, 0x1af1a87e00000000,
+    0x7efb869f00000000, 0x9f4dd47000000000, 0xfd90529a00000000,
+    0x1c26007500000000, 0x74827f8300000000, 0x95342d6c00000000,
+    0xf7e9ab8600000000, 0x165ff96900000000, 0x7255d78800000000,
+    0x93e3856700000000, 0xf13e038d00000000, 0x1088516200000000,
+    0x50c8cbe700000000, 0xb17e990800000000, 0xd3a31fe200000000,
+    0x32154d0d00000000, 0x561f63ec00000000, 0xb7a9310300000000,
+    0xd574b7e900000000, 0x34c2e50600000000, 0x5c669af000000000,
+    0xbdd0c81f00000000, 0xdf0d4ef500000000, 0x3ebb1c1a00000000,
+    0x5ab132fb00000000, 0xbb07601400000000, 0xd9dae6fe00000000,
+    0x386cb41100000000, 0x489468c900000000, 0xa9223a2600000000,
+    0xcbffbccc00000000, 0x2a49ee2300000000, 0x4e43c0c200000000,
+    0xaff5922d00000000, 0xcd2814c700000000, 0x2c9e462800000000,
+    0x443a39de00000000, 0xa58c6b3100000000, 0xc751eddb00000000,
+    0x26e7bf3400000000, 0x42ed91d500000000, 0xa35bc33a00000000,
+    0xc18645d000000000, 0x2030173f00000000, 0x81e66bae00000000,
+    0x6050394100000000, 0x028dbfab00000000, 0xe33bed4400000000,
+    0x8731c3a500000000, 0x6687914a00000000, 0x045a17a000000000,
+    0xe5ec454f00000000, 0x8d483ab900000000, 0x6cfe685600000000,
+    0x0e23eebc00000000, 0xef95bc5300000000, 0x8b9f92b200000000,
+    0x6a29c05d00000000, 0x08f446b700000000, 0xe942145800000000,
+    0x99bac88000000000, 0x780c9a6f00000000, 0x1ad11c8500000000,
+    0xfb674e6a00000000, 0x9f6d608b00000000, 0x7edb326400000000,
+    0x1c06b48e00000000, 0xfdb0e66100000000, 0x9514999700000000,
+    0x74a2cb7800000000, 0x167f4d9200000000, 0xf7c91f7d00000000,
+    0x93c3319c00000000, 0x7275637300000000, 0x10a8e59900000000,
+    0xf11eb77600000000, 0xb15e2df300000000, 0x50e87f1c00000000,
+    0x3235f9f600000000, 0xd383ab1900000000, 0xb78985f800000000,
+    0x563fd71700000000, 0x34e251fd00000000, 0xd554031200000000,
+    0xbdf07ce400000000, 0x5c462e0b00000000, 0x3e9ba8e100000000,
+    0xdf2dfa0e00000000, 0xbb27d4ef00000000, 0x5a91860000000000,
+    0x384c00ea00000000, 0xd9fa520500000000, 0xa9028edd00000000,
+    0x48b4dc3200000000, 0x2a695ad800000000, 0xcbdf083700000000,
+    0xafd526d600000000, 0x4e63743900000000, 0x2cbef2d300000000,
+    0xcd08a03c00000000, 0xa5acdfca00000000, 0x441a8d2500000000,
+    0x26c70bcf00000000, 0xc771592000000000, 0xa37b77c100000000,
+    0x42cd252e00000000, 0x2010a3c400000000, 0xc1a6f12b00000000,
+    0xe196e61400000000, 0x0020b4fb00000000, 0x62fd321100000000,
+    0x834b60fe00000000, 0xe7414e1f00000000, 0x06f71cf000000000,
+    0x642a9a1a00000000, 0x859cc8f500000000, 0xed38b70300000000,
+    0x0c8ee5ec00000000, 0x6e53630600000000, 0x8fe531e900000000,
+    0xebef1f0800000000, 0x0a594de700000000, 0x6884cb0d00000000,
+    0x893299e200000000, 0xf9ca453a00000000, 0x187c17d500000000,
+    0x7aa1913f00000000, 0x9b17c3d000000000, 0xff1ded3100000000,
+    0x1eabbfde00000000, 0x7c76393400000000, 0x9dc06bdb00000000,
+    0xf564142d00000000, 0x14d246c200000000, 0x760fc02800000000,
+    0x97b992c700000000, 0xf3b3bc2600000000, 0x1205eec900000000,
+    0x70d8682300000000, 0x916e3acc00000000, 0xd12ea04900000000,
+    0x3098f2a600000000, 0x5245744c00000000, 0xb3f326a300000000,
+    0xd7f9084200000000, 0x364f5aad00000000, 0x5492dc4700000000,
+    0xb5248ea800000000, 0xdd80f15e00000000, 0x3c36a3b100000000,
+    0x5eeb255b00000000, 0xbf5d77b400000000, 0xdb57595500000000,
+    0x3ae10bba00000000, 0x583c8d5000000000, 0xb98adfbf00000000,
+    0xc972036700000000, 0x28c4518800000000, 0x4a19d76200000000,
+    0xabaf858d00000000, 0xcfa5ab6c00000000, 0x2e13f98300000000,
+    0x4cce7f6900000000, 0xad782d8600000000, 0xc5dc527000000000,
+    0x246a009f00000000, 0x46b7867500000000, 0xa701d49a00000000,
+    0xc30bfa7b00000000, 0x22bda89400000000, 0x40602e7e00000000,
+    0xa1d67c9100000000},
+   {0x0000000000000000, 0x5880e2d700000000, 0xf106b47400000000,
+    0xa98656a300000000, 0xe20d68e900000000, 0xba8d8a3e00000000,
+    0x130bdc9d00000000, 0x4b8b3e4a00000000, 0x851da10900000000,
+    0xdd9d43de00000000, 0x741b157d00000000, 0x2c9bf7aa00000000,
+    0x6710c9e000000000, 0x3f902b3700000000, 0x96167d9400000000,
+    0xce969f4300000000, 0x0a3b421300000000, 0x52bba0c400000000,
+    0xfb3df66700000000, 0xa3bd14b000000000, 0xe8362afa00000000,
+    0xb0b6c82d00000000, 0x19309e8e00000000, 0x41b07c5900000000,
+    0x8f26e31a00000000, 0xd7a601cd00000000, 0x7e20576e00000000,
+    0x26a0b5b900000000, 0x6d2b8bf300000000, 0x35ab692400000000,
+    0x9c2d3f8700000000, 0xc4addd5000000000, 0x1476842600000000,
+    0x4cf666f100000000, 0xe570305200000000, 0xbdf0d28500000000,
+    0xf67beccf00000000, 0xaefb0e1800000000, 0x077d58bb00000000,
+    0x5ffdba6c00000000, 0x916b252f00000000, 0xc9ebc7f800000000,
+    0x606d915b00000000, 0x38ed738c00000000, 0x73664dc600000000,
+    0x2be6af1100000000, 0x8260f9b200000000, 0xdae01b6500000000,
+    0x1e4dc63500000000, 0x46cd24e200000000, 0xef4b724100000000,
+    0xb7cb909600000000, 0xfc40aedc00000000, 0xa4c04c0b00000000,
+    0x0d461aa800000000, 0x55c6f87f00000000, 0x9b50673c00000000,
+    0xc3d085eb00000000, 0x6a56d34800000000, 0x32d6319f00000000,
+    0x795d0fd500000000, 0x21dded0200000000, 0x885bbba100000000,
+    0xd0db597600000000, 0x28ec084d00000000, 0x706cea9a00000000,
+    0xd9eabc3900000000, 0x816a5eee00000000, 0xcae160a400000000,
+    0x9261827300000000, 0x3be7d4d000000000, 0x6367360700000000,
+    0xadf1a94400000000, 0xf5714b9300000000, 0x5cf71d3000000000,
+    0x0477ffe700000000, 0x4ffcc1ad00000000, 0x177c237a00000000,
+    0xbefa75d900000000, 0xe67a970e00000000, 0x22d74a5e00000000,
+    0x7a57a88900000000, 0xd3d1fe2a00000000, 0x8b511cfd00000000,
+    0xc0da22b700000000, 0x985ac06000000000, 0x31dc96c300000000,
+    0x695c741400000000, 0xa7caeb5700000000, 0xff4a098000000000,
+    0x56cc5f2300000000, 0x0e4cbdf400000000, 0x45c783be00000000,
+    0x1d47616900000000, 0xb4c137ca00000000, 0xec41d51d00000000,
+    0x3c9a8c6b00000000, 0x641a6ebc00000000, 0xcd9c381f00000000,
+    0x951cdac800000000, 0xde97e48200000000, 0x8617065500000000,
+    0x2f9150f600000000, 0x7711b22100000000, 0xb9872d6200000000,
+    0xe107cfb500000000, 0x4881991600000000, 0x10017bc100000000,
+    0x5b8a458b00000000, 0x030aa75c00000000, 0xaa8cf1ff00000000,
+    0xf20c132800000000, 0x36a1ce7800000000, 0x6e212caf00000000,
+    0xc7a77a0c00000000, 0x9f2798db00000000, 0xd4aca69100000000,
+    0x8c2c444600000000, 0x25aa12e500000000, 0x7d2af03200000000,
+    0xb3bc6f7100000000, 0xeb3c8da600000000, 0x42badb0500000000,
+    0x1a3a39d200000000, 0x51b1079800000000, 0x0931e54f00000000,
+    0xa0b7b3ec00000000, 0xf837513b00000000, 0x50d8119a00000000,
+    0x0858f34d00000000, 0xa1dea5ee00000000, 0xf95e473900000000,
+    0xb2d5797300000000, 0xea559ba400000000, 0x43d3cd0700000000,
+    0x1b532fd000000000, 0xd5c5b09300000000, 0x8d45524400000000,
+    0x24c304e700000000, 0x7c43e63000000000, 0x37c8d87a00000000,
+    0x6f483aad00000000, 0xc6ce6c0e00000000, 0x9e4e8ed900000000,
+    0x5ae3538900000000, 0x0263b15e00000000, 0xabe5e7fd00000000,
+    0xf365052a00000000, 0xb8ee3b6000000000, 0xe06ed9b700000000,
+    0x49e88f1400000000, 0x11686dc300000000, 0xdffef28000000000,
+    0x877e105700000000, 0x2ef846f400000000, 0x7678a42300000000,
+    0x3df39a6900000000, 0x657378be00000000, 0xccf52e1d00000000,
+    0x9475ccca00000000, 0x44ae95bc00000000, 0x1c2e776b00000000,
+    0xb5a821c800000000, 0xed28c31f00000000, 0xa6a3fd5500000000,
+    0xfe231f8200000000, 0x57a5492100000000, 0x0f25abf600000000,
+    0xc1b334b500000000, 0x9933d66200000000, 0x30b580c100000000,
+    0x6835621600000000, 0x23be5c5c00000000, 0x7b3ebe8b00000000,
+    0xd2b8e82800000000, 0x8a380aff00000000, 0x4e95d7af00000000,
+    0x1615357800000000, 0xbf9363db00000000, 0xe713810c00000000,
+    0xac98bf4600000000, 0xf4185d9100000000, 0x5d9e0b3200000000,
+    0x051ee9e500000000, 0xcb8876a600000000, 0x9308947100000000,
+    0x3a8ec2d200000000, 0x620e200500000000, 0x29851e4f00000000,
+    0x7105fc9800000000, 0xd883aa3b00000000, 0x800348ec00000000,
+    0x783419d700000000, 0x20b4fb0000000000, 0x8932ada300000000,
+    0xd1b24f7400000000, 0x9a39713e00000000, 0xc2b993e900000000,
+    0x6b3fc54a00000000, 0x33bf279d00000000, 0xfd29b8de00000000,
+    0xa5a95a0900000000, 0x0c2f0caa00000000, 0x54afee7d00000000,
+    0x1f24d03700000000, 0x47a432e000000000, 0xee22644300000000,
+    0xb6a2869400000000, 0x720f5bc400000000, 0x2a8fb91300000000,
+    0x8309efb000000000, 0xdb890d6700000000, 0x9002332d00000000,
+    0xc882d1fa00000000, 0x6104875900000000, 0x3984658e00000000,
+    0xf712facd00000000, 0xaf92181a00000000, 0x06144eb900000000,
+    0x5e94ac6e00000000, 0x151f922400000000, 0x4d9f70f300000000,
+    0xe419265000000000, 0xbc99c48700000000, 0x6c429df100000000,
+    0x34c27f2600000000, 0x9d44298500000000, 0xc5c4cb5200000000,
+    0x8e4ff51800000000, 0xd6cf17cf00000000, 0x7f49416c00000000,
+    0x27c9a3bb00000000, 0xe95f3cf800000000, 0xb1dfde2f00000000,
+    0x1859888c00000000, 0x40d96a5b00000000, 0x0b52541100000000,
+    0x53d2b6c600000000, 0xfa54e06500000000, 0xa2d402b200000000,
+    0x6679dfe200000000, 0x3ef93d3500000000, 0x977f6b9600000000,
+    0xcfff894100000000, 0x8474b70b00000000, 0xdcf455dc00000000,
+    0x7572037f00000000, 0x2df2e1a800000000, 0xe3647eeb00000000,
+    0xbbe49c3c00000000, 0x1262ca9f00000000, 0x4ae2284800000000,
+    0x0169160200000000, 0x59e9f4d500000000, 0xf06fa27600000000,
+    0xa8ef40a100000000},
+   {0x0000000000000000, 0x463b676500000000, 0x8c76ceca00000000,
+    0xca4da9af00000000, 0x59ebed4e00000000, 0x1fd08a2b00000000,
+    0xd59d238400000000, 0x93a644e100000000, 0xb2d6db9d00000000,
+    0xf4edbcf800000000, 0x3ea0155700000000, 0x789b723200000000,
+    0xeb3d36d300000000, 0xad0651b600000000, 0x674bf81900000000,
+    0x21709f7c00000000, 0x25abc6e000000000, 0x6390a18500000000,
+    0xa9dd082a00000000, 0xefe66f4f00000000, 0x7c402bae00000000,
+    0x3a7b4ccb00000000, 0xf036e56400000000, 0xb60d820100000000,
+    0x977d1d7d00000000, 0xd1467a1800000000, 0x1b0bd3b700000000,
+    0x5d30b4d200000000, 0xce96f03300000000, 0x88ad975600000000,
+    0x42e03ef900000000, 0x04db599c00000000, 0x0b50fc1a00000000,
+    0x4d6b9b7f00000000, 0x872632d000000000, 0xc11d55b500000000,
+    0x52bb115400000000, 0x1480763100000000, 0xdecddf9e00000000,
+    0x98f6b8fb00000000, 0xb986278700000000, 0xffbd40e200000000,
+    0x35f0e94d00000000, 0x73cb8e2800000000, 0xe06dcac900000000,
+    0xa656adac00000000, 0x6c1b040300000000, 0x2a20636600000000,
+    0x2efb3afa00000000, 0x68c05d9f00000000, 0xa28df43000000000,
+    0xe4b6935500000000, 0x7710d7b400000000, 0x312bb0d100000000,
+    0xfb66197e00000000, 0xbd5d7e1b00000000, 0x9c2de16700000000,
+    0xda16860200000000, 0x105b2fad00000000, 0x566048c800000000,
+    0xc5c60c2900000000, 0x83fd6b4c00000000, 0x49b0c2e300000000,
+    0x0f8ba58600000000, 0x16a0f83500000000, 0x509b9f5000000000,
+    0x9ad636ff00000000, 0xdced519a00000000, 0x4f4b157b00000000,
+    0x0970721e00000000, 0xc33ddbb100000000, 0x8506bcd400000000,
+    0xa47623a800000000, 0xe24d44cd00000000, 0x2800ed6200000000,
+    0x6e3b8a0700000000, 0xfd9dcee600000000, 0xbba6a98300000000,
+    0x71eb002c00000000, 0x37d0674900000000, 0x330b3ed500000000,
+    0x753059b000000000, 0xbf7df01f00000000, 0xf946977a00000000,
+    0x6ae0d39b00000000, 0x2cdbb4fe00000000, 0xe6961d5100000000,
+    0xa0ad7a3400000000, 0x81dde54800000000, 0xc7e6822d00000000,
+    0x0dab2b8200000000, 0x4b904ce700000000, 0xd836080600000000,
+    0x9e0d6f6300000000, 0x5440c6cc00000000, 0x127ba1a900000000,
+    0x1df0042f00000000, 0x5bcb634a00000000, 0x9186cae500000000,
+    0xd7bdad8000000000, 0x441be96100000000, 0x02208e0400000000,
+    0xc86d27ab00000000, 0x8e5640ce00000000, 0xaf26dfb200000000,
+    0xe91db8d700000000, 0x2350117800000000, 0x656b761d00000000,
+    0xf6cd32fc00000000, 0xb0f6559900000000, 0x7abbfc3600000000,
+    0x3c809b5300000000, 0x385bc2cf00000000, 0x7e60a5aa00000000,
+    0xb42d0c0500000000, 0xf2166b6000000000, 0x61b02f8100000000,
+    0x278b48e400000000, 0xedc6e14b00000000, 0xabfd862e00000000,
+    0x8a8d195200000000, 0xccb67e3700000000, 0x06fbd79800000000,
+    0x40c0b0fd00000000, 0xd366f41c00000000, 0x955d937900000000,
+    0x5f103ad600000000, 0x192b5db300000000, 0x2c40f16b00000000,
+    0x6a7b960e00000000, 0xa0363fa100000000, 0xe60d58c400000000,
+    0x75ab1c2500000000, 0x33907b4000000000, 0xf9ddd2ef00000000,
+    0xbfe6b58a00000000, 0x9e962af600000000, 0xd8ad4d9300000000,
+    0x12e0e43c00000000, 0x54db835900000000, 0xc77dc7b800000000,
+    0x8146a0dd00000000, 0x4b0b097200000000, 0x0d306e1700000000,
+    0x09eb378b00000000, 0x4fd050ee00000000, 0x859df94100000000,
+    0xc3a69e2400000000, 0x5000dac500000000, 0x163bbda000000000,
+    0xdc76140f00000000, 0x9a4d736a00000000, 0xbb3dec1600000000,
+    0xfd068b7300000000, 0x374b22dc00000000, 0x717045b900000000,
+    0xe2d6015800000000, 0xa4ed663d00000000, 0x6ea0cf9200000000,
+    0x289ba8f700000000, 0x27100d7100000000, 0x612b6a1400000000,
+    0xab66c3bb00000000, 0xed5da4de00000000, 0x7efbe03f00000000,
+    0x38c0875a00000000, 0xf28d2ef500000000, 0xb4b6499000000000,
+    0x95c6d6ec00000000, 0xd3fdb18900000000, 0x19b0182600000000,
+    0x5f8b7f4300000000, 0xcc2d3ba200000000, 0x8a165cc700000000,
+    0x405bf56800000000, 0x0660920d00000000, 0x02bbcb9100000000,
+    0x4480acf400000000, 0x8ecd055b00000000, 0xc8f6623e00000000,
+    0x5b5026df00000000, 0x1d6b41ba00000000, 0xd726e81500000000,
+    0x911d8f7000000000, 0xb06d100c00000000, 0xf656776900000000,
+    0x3c1bdec600000000, 0x7a20b9a300000000, 0xe986fd4200000000,
+    0xafbd9a2700000000, 0x65f0338800000000, 0x23cb54ed00000000,
+    0x3ae0095e00000000, 0x7cdb6e3b00000000, 0xb696c79400000000,
+    0xf0ada0f100000000, 0x630be41000000000, 0x2530837500000000,
+    0xef7d2ada00000000, 0xa9464dbf00000000, 0x8836d2c300000000,
+    0xce0db5a600000000, 0x04401c0900000000, 0x427b7b6c00000000,
+    0xd1dd3f8d00000000, 0x97e658e800000000, 0x5dabf14700000000,
+    0x1b90962200000000, 0x1f4bcfbe00000000, 0x5970a8db00000000,
+    0x933d017400000000, 0xd506661100000000, 0x46a022f000000000,
+    0x009b459500000000, 0xcad6ec3a00000000, 0x8ced8b5f00000000,
+    0xad9d142300000000, 0xeba6734600000000, 0x21ebdae900000000,
+    0x67d0bd8c00000000, 0xf476f96d00000000, 0xb24d9e0800000000,
+    0x780037a700000000, 0x3e3b50c200000000, 0x31b0f54400000000,
+    0x778b922100000000, 0xbdc63b8e00000000, 0xfbfd5ceb00000000,
+    0x685b180a00000000, 0x2e607f6f00000000, 0xe42dd6c000000000,
+    0xa216b1a500000000, 0x83662ed900000000, 0xc55d49bc00000000,
+    0x0f10e01300000000, 0x492b877600000000, 0xda8dc39700000000,
+    0x9cb6a4f200000000, 0x56fb0d5d00000000, 0x10c06a3800000000,
+    0x141b33a400000000, 0x522054c100000000, 0x986dfd6e00000000,
+    0xde569a0b00000000, 0x4df0deea00000000, 0x0bcbb98f00000000,
+    0xc186102000000000, 0x87bd774500000000, 0xa6cde83900000000,
+    0xe0f68f5c00000000, 0x2abb26f300000000, 0x6c80419600000000,
+    0xff26057700000000, 0xb91d621200000000, 0x7350cbbd00000000,
+    0x356bacd800000000},
+   {0x0000000000000000, 0x9e83da9f00000000, 0x7d01c4e400000000,
+    0xe3821e7b00000000, 0xbb04f91200000000, 0x2587238d00000000,
+    0xc6053df600000000, 0x5886e76900000000, 0x7609f22500000000,
+    0xe88a28ba00000000, 0x0b0836c100000000, 0x958bec5e00000000,
+    0xcd0d0b3700000000, 0x538ed1a800000000, 0xb00ccfd300000000,
+    0x2e8f154c00000000, 0xec12e44b00000000, 0x72913ed400000000,
+    0x911320af00000000, 0x0f90fa3000000000, 0x57161d5900000000,
+    0xc995c7c600000000, 0x2a17d9bd00000000, 0xb494032200000000,
+    0x9a1b166e00000000, 0x0498ccf100000000, 0xe71ad28a00000000,
+    0x7999081500000000, 0x211fef7c00000000, 0xbf9c35e300000000,
+    0x5c1e2b9800000000, 0xc29df10700000000, 0xd825c89700000000,
+    0x46a6120800000000, 0xa5240c7300000000, 0x3ba7d6ec00000000,
+    0x6321318500000000, 0xfda2eb1a00000000, 0x1e20f56100000000,
+    0x80a32ffe00000000, 0xae2c3ab200000000, 0x30afe02d00000000,
+    0xd32dfe5600000000, 0x4dae24c900000000, 0x1528c3a000000000,
+    0x8bab193f00000000, 0x6829074400000000, 0xf6aadddb00000000,
+    0x34372cdc00000000, 0xaab4f64300000000, 0x4936e83800000000,
+    0xd7b532a700000000, 0x8f33d5ce00000000, 0x11b00f5100000000,
+    0xf232112a00000000, 0x6cb1cbb500000000, 0x423edef900000000,
+    0xdcbd046600000000, 0x3f3f1a1d00000000, 0xa1bcc08200000000,
+    0xf93a27eb00000000, 0x67b9fd7400000000, 0x843be30f00000000,
+    0x1ab8399000000000, 0xf14de1f400000000, 0x6fce3b6b00000000,
+    0x8c4c251000000000, 0x12cfff8f00000000, 0x4a4918e600000000,
+    0xd4cac27900000000, 0x3748dc0200000000, 0xa9cb069d00000000,
+    0x874413d100000000, 0x19c7c94e00000000, 0xfa45d73500000000,
+    0x64c60daa00000000, 0x3c40eac300000000, 0xa2c3305c00000000,
+    0x41412e2700000000, 0xdfc2f4b800000000, 0x1d5f05bf00000000,
+    0x83dcdf2000000000, 0x605ec15b00000000, 0xfedd1bc400000000,
+    0xa65bfcad00000000, 0x38d8263200000000, 0xdb5a384900000000,
+    0x45d9e2d600000000, 0x6b56f79a00000000, 0xf5d52d0500000000,
+    0x1657337e00000000, 0x88d4e9e100000000, 0xd0520e8800000000,
+    0x4ed1d41700000000, 0xad53ca6c00000000, 0x33d010f300000000,
+    0x2968296300000000, 0xb7ebf3fc00000000, 0x5469ed8700000000,
+    0xcaea371800000000, 0x926cd07100000000, 0x0cef0aee00000000,
+    0xef6d149500000000, 0x71eece0a00000000, 0x5f61db4600000000,
+    0xc1e201d900000000, 0x22601fa200000000, 0xbce3c53d00000000,
+    0xe465225400000000, 0x7ae6f8cb00000000, 0x9964e6b000000000,
+    0x07e73c2f00000000, 0xc57acd2800000000, 0x5bf917b700000000,
+    0xb87b09cc00000000, 0x26f8d35300000000, 0x7e7e343a00000000,
+    0xe0fdeea500000000, 0x037ff0de00000000, 0x9dfc2a4100000000,
+    0xb3733f0d00000000, 0x2df0e59200000000, 0xce72fbe900000000,
+    0x50f1217600000000, 0x0877c61f00000000, 0x96f41c8000000000,
+    0x757602fb00000000, 0xebf5d86400000000, 0xa39db33200000000,
+    0x3d1e69ad00000000, 0xde9c77d600000000, 0x401fad4900000000,
+    0x18994a2000000000, 0x861a90bf00000000, 0x65988ec400000000,
+    0xfb1b545b00000000, 0xd594411700000000, 0x4b179b8800000000,
+    0xa89585f300000000, 0x36165f6c00000000, 0x6e90b80500000000,
+    0xf013629a00000000, 0x13917ce100000000, 0x8d12a67e00000000,
+    0x4f8f577900000000, 0xd10c8de600000000, 0x328e939d00000000,
+    0xac0d490200000000, 0xf48bae6b00000000, 0x6a0874f400000000,
+    0x898a6a8f00000000, 0x1709b01000000000, 0x3986a55c00000000,
+    0xa7057fc300000000, 0x448761b800000000, 0xda04bb2700000000,
+    0x82825c4e00000000, 0x1c0186d100000000, 0xff8398aa00000000,
+    0x6100423500000000, 0x7bb87ba500000000, 0xe53ba13a00000000,
+    0x06b9bf4100000000, 0x983a65de00000000, 0xc0bc82b700000000,
+    0x5e3f582800000000, 0xbdbd465300000000, 0x233e9ccc00000000,
+    0x0db1898000000000, 0x9332531f00000000, 0x70b04d6400000000,
+    0xee3397fb00000000, 0xb6b5709200000000, 0x2836aa0d00000000,
+    0xcbb4b47600000000, 0x55376ee900000000, 0x97aa9fee00000000,
+    0x0929457100000000, 0xeaab5b0a00000000, 0x7428819500000000,
+    0x2cae66fc00000000, 0xb22dbc6300000000, 0x51afa21800000000,
+    0xcf2c788700000000, 0xe1a36dcb00000000, 0x7f20b75400000000,
+    0x9ca2a92f00000000, 0x022173b000000000, 0x5aa794d900000000,
+    0xc4244e4600000000, 0x27a6503d00000000, 0xb9258aa200000000,
+    0x52d052c600000000, 0xcc53885900000000, 0x2fd1962200000000,
+    0xb1524cbd00000000, 0xe9d4abd400000000, 0x7757714b00000000,
+    0x94d56f3000000000, 0x0a56b5af00000000, 0x24d9a0e300000000,
+    0xba5a7a7c00000000, 0x59d8640700000000, 0xc75bbe9800000000,
+    0x9fdd59f100000000, 0x015e836e00000000, 0xe2dc9d1500000000,
+    0x7c5f478a00000000, 0xbec2b68d00000000, 0x20416c1200000000,
+    0xc3c3726900000000, 0x5d40a8f600000000, 0x05c64f9f00000000,
+    0x9b45950000000000, 0x78c78b7b00000000, 0xe64451e400000000,
+    0xc8cb44a800000000, 0x56489e3700000000, 0xb5ca804c00000000,
+    0x2b495ad300000000, 0x73cfbdba00000000, 0xed4c672500000000,
+    0x0ece795e00000000, 0x904da3c100000000, 0x8af59a5100000000,
+    0x147640ce00000000, 0xf7f45eb500000000, 0x6977842a00000000,
+    0x31f1634300000000, 0xaf72b9dc00000000, 0x4cf0a7a700000000,
+    0xd2737d3800000000, 0xfcfc687400000000, 0x627fb2eb00000000,
+    0x81fdac9000000000, 0x1f7e760f00000000, 0x47f8916600000000,
+    0xd97b4bf900000000, 0x3af9558200000000, 0xa47a8f1d00000000,
+    0x66e77e1a00000000, 0xf864a48500000000, 0x1be6bafe00000000,
+    0x8565606100000000, 0xdde3870800000000, 0x43605d9700000000,
+    0xa0e243ec00000000, 0x3e61997300000000, 0x10ee8c3f00000000,
+    0x8e6d56a000000000, 0x6def48db00000000, 0xf36c924400000000,
+    0xabea752d00000000, 0x3569afb200000000, 0xd6ebb1c900000000,
+    0x48686b5600000000},
+   {0x0000000000000000, 0xc064281700000000, 0x80c9502e00000000,
+    0x40ad783900000000, 0x0093a15c00000000, 0xc0f7894b00000000,
+    0x805af17200000000, 0x403ed96500000000, 0x002643b900000000,
+    0xc0426bae00000000, 0x80ef139700000000, 0x408b3b8000000000,
+    0x00b5e2e500000000, 0xc0d1caf200000000, 0x807cb2cb00000000,
+    0x40189adc00000000, 0x414af7a900000000, 0x812edfbe00000000,
+    0xc183a78700000000, 0x01e78f9000000000, 0x41d956f500000000,
+    0x81bd7ee200000000, 0xc11006db00000000, 0x01742ecc00000000,
+    0x416cb41000000000, 0x81089c0700000000, 0xc1a5e43e00000000,
+    0x01c1cc2900000000, 0x41ff154c00000000, 0x819b3d5b00000000,
+    0xc136456200000000, 0x01526d7500000000, 0xc3929f8800000000,
+    0x03f6b79f00000000, 0x435bcfa600000000, 0x833fe7b100000000,
+    0xc3013ed400000000, 0x036516c300000000, 0x43c86efa00000000,
+    0x83ac46ed00000000, 0xc3b4dc3100000000, 0x03d0f42600000000,
+    0x437d8c1f00000000, 0x8319a40800000000, 0xc3277d6d00000000,
+    0x0343557a00000000, 0x43ee2d4300000000, 0x838a055400000000,
+    0x82d8682100000000, 0x42bc403600000000, 0x0211380f00000000,
+    0xc275101800000000, 0x824bc97d00000000, 0x422fe16a00000000,
+    0x0282995300000000, 0xc2e6b14400000000, 0x82fe2b9800000000,
+    0x429a038f00000000, 0x02377bb600000000, 0xc25353a100000000,
+    0x826d8ac400000000, 0x4209a2d300000000, 0x02a4daea00000000,
+    0xc2c0f2fd00000000, 0xc7234eca00000000, 0x074766dd00000000,
+    0x47ea1ee400000000, 0x878e36f300000000, 0xc7b0ef9600000000,
+    0x07d4c78100000000, 0x4779bfb800000000, 0x871d97af00000000,
+    0xc7050d7300000000, 0x0761256400000000, 0x47cc5d5d00000000,
+    0x87a8754a00000000, 0xc796ac2f00000000, 0x07f2843800000000,
+    0x475ffc0100000000, 0x873bd41600000000, 0x8669b96300000000,
+    0x460d917400000000, 0x06a0e94d00000000, 0xc6c4c15a00000000,
+    0x86fa183f00000000, 0x469e302800000000, 0x0633481100000000,
+    0xc657600600000000, 0x864ffada00000000, 0x462bd2cd00000000,
+    0x0686aaf400000000, 0xc6e282e300000000, 0x86dc5b8600000000,
+    0x46b8739100000000, 0x06150ba800000000, 0xc67123bf00000000,
+    0x04b1d14200000000, 0xc4d5f95500000000, 0x8478816c00000000,
+    0x441ca97b00000000, 0x0422701e00000000, 0xc446580900000000,
+    0x84eb203000000000, 0x448f082700000000, 0x049792fb00000000,
+    0xc4f3baec00000000, 0x845ec2d500000000, 0x443aeac200000000,
+    0x040433a700000000, 0xc4601bb000000000, 0x84cd638900000000,
+    0x44a94b9e00000000, 0x45fb26eb00000000, 0x859f0efc00000000,
+    0xc53276c500000000, 0x05565ed200000000, 0x456887b700000000,
+    0x850cafa000000000, 0xc5a1d79900000000, 0x05c5ff8e00000000,
+    0x45dd655200000000, 0x85b94d4500000000, 0xc514357c00000000,
+    0x05701d6b00000000, 0x454ec40e00000000, 0x852aec1900000000,
+    0xc587942000000000, 0x05e3bc3700000000, 0xcf41ed4f00000000,
+    0x0f25c55800000000, 0x4f88bd6100000000, 0x8fec957600000000,
+    0xcfd24c1300000000, 0x0fb6640400000000, 0x4f1b1c3d00000000,
+    0x8f7f342a00000000, 0xcf67aef600000000, 0x0f0386e100000000,
+    0x4faefed800000000, 0x8fcad6cf00000000, 0xcff40faa00000000,
+    0x0f9027bd00000000, 0x4f3d5f8400000000, 0x8f59779300000000,
+    0x8e0b1ae600000000, 0x4e6f32f100000000, 0x0ec24ac800000000,
+    0xcea662df00000000, 0x8e98bbba00000000, 0x4efc93ad00000000,
+    0x0e51eb9400000000, 0xce35c38300000000, 0x8e2d595f00000000,
+    0x4e49714800000000, 0x0ee4097100000000, 0xce80216600000000,
+    0x8ebef80300000000, 0x4edad01400000000, 0x0e77a82d00000000,
+    0xce13803a00000000, 0x0cd372c700000000, 0xccb75ad000000000,
+    0x8c1a22e900000000, 0x4c7e0afe00000000, 0x0c40d39b00000000,
+    0xcc24fb8c00000000, 0x8c8983b500000000, 0x4cedaba200000000,
+    0x0cf5317e00000000, 0xcc91196900000000, 0x8c3c615000000000,
+    0x4c58494700000000, 0x0c66902200000000, 0xcc02b83500000000,
+    0x8cafc00c00000000, 0x4ccbe81b00000000, 0x4d99856e00000000,
+    0x8dfdad7900000000, 0xcd50d54000000000, 0x0d34fd5700000000,
+    0x4d0a243200000000, 0x8d6e0c2500000000, 0xcdc3741c00000000,
+    0x0da75c0b00000000, 0x4dbfc6d700000000, 0x8ddbeec000000000,
+    0xcd7696f900000000, 0x0d12beee00000000, 0x4d2c678b00000000,
+    0x8d484f9c00000000, 0xcde537a500000000, 0x0d811fb200000000,
+    0x0862a38500000000, 0xc8068b9200000000, 0x88abf3ab00000000,
+    0x48cfdbbc00000000, 0x08f102d900000000, 0xc8952ace00000000,
+    0x883852f700000000, 0x485c7ae000000000, 0x0844e03c00000000,
+    0xc820c82b00000000, 0x888db01200000000, 0x48e9980500000000,
+    0x08d7416000000000, 0xc8b3697700000000, 0x881e114e00000000,
+    0x487a395900000000, 0x4928542c00000000, 0x894c7c3b00000000,
+    0xc9e1040200000000, 0x09852c1500000000, 0x49bbf57000000000,
+    0x89dfdd6700000000, 0xc972a55e00000000, 0x09168d4900000000,
+    0x490e179500000000, 0x896a3f8200000000, 0xc9c747bb00000000,
+    0x09a36fac00000000, 0x499db6c900000000, 0x89f99ede00000000,
+    0xc954e6e700000000, 0x0930cef000000000, 0xcbf03c0d00000000,
+    0x0b94141a00000000, 0x4b396c2300000000, 0x8b5d443400000000,
+    0xcb639d5100000000, 0x0b07b54600000000, 0x4baacd7f00000000,
+    0x8bcee56800000000, 0xcbd67fb400000000, 0x0bb257a300000000,
+    0x4b1f2f9a00000000, 0x8b7b078d00000000, 0xcb45dee800000000,
+    0x0b21f6ff00000000, 0x4b8c8ec600000000, 0x8be8a6d100000000,
+    0x8abacba400000000, 0x4adee3b300000000, 0x0a739b8a00000000,
+    0xca17b39d00000000, 0x8a296af800000000, 0x4a4d42ef00000000,
+    0x0ae03ad600000000, 0xca8412c100000000, 0x8a9c881d00000000,
+    0x4af8a00a00000000, 0x0a55d83300000000, 0xca31f02400000000,
+    0x8a0f294100000000, 0x4a6b015600000000, 0x0ac6796f00000000,
+    0xcaa2517800000000},
+   {0x0000000000000000, 0xd4ea739b00000000, 0xe9d396ed00000000,
+    0x3d39e57600000000, 0x93a15c0000000000, 0x474b2f9b00000000,
+    0x7a72caed00000000, 0xae98b97600000000, 0x2643b90000000000,
+    0xf2a9ca9b00000000, 0xcf902fed00000000, 0x1b7a5c7600000000,
+    0xb5e2e50000000000, 0x6108969b00000000, 0x5c3173ed00000000,
+    0x88db007600000000, 0x4c86720100000000, 0x986c019a00000000,
+    0xa555e4ec00000000, 0x71bf977700000000, 0xdf272e0100000000,
+    0x0bcd5d9a00000000, 0x36f4b8ec00000000, 0xe21ecb7700000000,
+    0x6ac5cb0100000000, 0xbe2fb89a00000000, 0x83165dec00000000,
+    0x57fc2e7700000000, 0xf964970100000000, 0x2d8ee49a00000000,
+    0x10b701ec00000000, 0xc45d727700000000, 0x980ce50200000000,
+    0x4ce6969900000000, 0x71df73ef00000000, 0xa535007400000000,
+    0x0badb90200000000, 0xdf47ca9900000000, 0xe27e2fef00000000,
+    0x36945c7400000000, 0xbe4f5c0200000000, 0x6aa52f9900000000,
+    0x579ccaef00000000, 0x8376b97400000000, 0x2dee000200000000,
+    0xf904739900000000, 0xc43d96ef00000000, 0x10d7e57400000000,
+    0xd48a970300000000, 0x0060e49800000000, 0x3d5901ee00000000,
+    0xe9b3727500000000, 0x472bcb0300000000, 0x93c1b89800000000,
+    0xaef85dee00000000, 0x7a122e7500000000, 0xf2c92e0300000000,
+    0x26235d9800000000, 0x1b1ab8ee00000000, 0xcff0cb7500000000,
+    0x6168720300000000, 0xb582019800000000, 0x88bbe4ee00000000,
+    0x5c51977500000000, 0x3019ca0500000000, 0xe4f3b99e00000000,
+    0xd9ca5ce800000000, 0x0d202f7300000000, 0xa3b8960500000000,
+    0x7752e59e00000000, 0x4a6b00e800000000, 0x9e81737300000000,
+    0x165a730500000000, 0xc2b0009e00000000, 0xff89e5e800000000,
+    0x2b63967300000000, 0x85fb2f0500000000, 0x51115c9e00000000,
+    0x6c28b9e800000000, 0xb8c2ca7300000000, 0x7c9fb80400000000,
+    0xa875cb9f00000000, 0x954c2ee900000000, 0x41a65d7200000000,
+    0xef3ee40400000000, 0x3bd4979f00000000, 0x06ed72e900000000,
+    0xd207017200000000, 0x5adc010400000000, 0x8e36729f00000000,
+    0xb30f97e900000000, 0x67e5e47200000000, 0xc97d5d0400000000,
+    0x1d972e9f00000000, 0x20aecbe900000000, 0xf444b87200000000,
+    0xa8152f0700000000, 0x7cff5c9c00000000, 0x41c6b9ea00000000,
+    0x952cca7100000000, 0x3bb4730700000000, 0xef5e009c00000000,
+    0xd267e5ea00000000, 0x068d967100000000, 0x8e56960700000000,
+    0x5abce59c00000000, 0x678500ea00000000, 0xb36f737100000000,
+    0x1df7ca0700000000, 0xc91db99c00000000, 0xf4245cea00000000,
+    0x20ce2f7100000000, 0xe4935d0600000000, 0x30792e9d00000000,
+    0x0d40cbeb00000000, 0xd9aab87000000000, 0x7732010600000000,
+    0xa3d8729d00000000, 0x9ee197eb00000000, 0x4a0be47000000000,
+    0xc2d0e40600000000, 0x163a979d00000000, 0x2b0372eb00000000,
+    0xffe9017000000000, 0x5171b80600000000, 0x859bcb9d00000000,
+    0xb8a22eeb00000000, 0x6c485d7000000000, 0x6032940b00000000,
+    0xb4d8e79000000000, 0x89e102e600000000, 0x5d0b717d00000000,
+    0xf393c80b00000000, 0x2779bb9000000000, 0x1a405ee600000000,
+    0xceaa2d7d00000000, 0x46712d0b00000000, 0x929b5e9000000000,
+    0xafa2bbe600000000, 0x7b48c87d00000000, 0xd5d0710b00000000,
+    0x013a029000000000, 0x3c03e7e600000000, 0xe8e9947d00000000,
+    0x2cb4e60a00000000, 0xf85e959100000000, 0xc56770e700000000,
+    0x118d037c00000000, 0xbf15ba0a00000000, 0x6bffc99100000000,
+    0x56c62ce700000000, 0x822c5f7c00000000, 0x0af75f0a00000000,
+    0xde1d2c9100000000, 0xe324c9e700000000, 0x37ceba7c00000000,
+    0x9956030a00000000, 0x4dbc709100000000, 0x708595e700000000,
+    0xa46fe67c00000000, 0xf83e710900000000, 0x2cd4029200000000,
+    0x11ede7e400000000, 0xc507947f00000000, 0x6b9f2d0900000000,
+    0xbf755e9200000000, 0x824cbbe400000000, 0x56a6c87f00000000,
+    0xde7dc80900000000, 0x0a97bb9200000000, 0x37ae5ee400000000,
+    0xe3442d7f00000000, 0x4ddc940900000000, 0x9936e79200000000,
+    0xa40f02e400000000, 0x70e5717f00000000, 0xb4b8030800000000,
+    0x6052709300000000, 0x5d6b95e500000000, 0x8981e67e00000000,
+    0x27195f0800000000, 0xf3f32c9300000000, 0xcecac9e500000000,
+    0x1a20ba7e00000000, 0x92fbba0800000000, 0x4611c99300000000,
+    0x7b282ce500000000, 0xafc25f7e00000000, 0x015ae60800000000,
+    0xd5b0959300000000, 0xe88970e500000000, 0x3c63037e00000000,
+    0x502b5e0e00000000, 0x84c12d9500000000, 0xb9f8c8e300000000,
+    0x6d12bb7800000000, 0xc38a020e00000000, 0x1760719500000000,
+    0x2a5994e300000000, 0xfeb3e77800000000, 0x7668e70e00000000,
+    0xa282949500000000, 0x9fbb71e300000000, 0x4b51027800000000,
+    0xe5c9bb0e00000000, 0x3123c89500000000, 0x0c1a2de300000000,
+    0xd8f05e7800000000, 0x1cad2c0f00000000, 0xc8475f9400000000,
+    0xf57ebae200000000, 0x2194c97900000000, 0x8f0c700f00000000,
+    0x5be6039400000000, 0x66dfe6e200000000, 0xb235957900000000,
+    0x3aee950f00000000, 0xee04e69400000000, 0xd33d03e200000000,
+    0x07d7707900000000, 0xa94fc90f00000000, 0x7da5ba9400000000,
+    0x409c5fe200000000, 0x94762c7900000000, 0xc827bb0c00000000,
+    0x1ccdc89700000000, 0x21f42de100000000, 0xf51e5e7a00000000,
+    0x5b86e70c00000000, 0x8f6c949700000000, 0xb25571e100000000,
+    0x66bf027a00000000, 0xee64020c00000000, 0x3a8e719700000000,
+    0x07b794e100000000, 0xd35de77a00000000, 0x7dc55e0c00000000,
+    0xa92f2d9700000000, 0x9416c8e100000000, 0x40fcbb7a00000000,
+    0x84a1c90d00000000, 0x504bba9600000000, 0x6d725fe000000000,
+    0xb9982c7b00000000, 0x1700950d00000000, 0xc3eae69600000000,
+    0xfed303e000000000, 0x2a39707b00000000, 0xa2e2700d00000000,
+    0x7608039600000000, 0x4b31e6e000000000, 0x9fdb957b00000000,
+    0x31432c0d00000000, 0xe5a95f9600000000, 0xd890bae000000000,
+    0x0c7ac97b00000000},
+   {0x0000000000000000, 0x2765258100000000, 0x0fcc3bd900000000,
+    0x28a91e5800000000, 0x5f9e066900000000, 0x78fb23e800000000,
+    0x50523db000000000, 0x7737183100000000, 0xbe3c0dd200000000,
+    0x9959285300000000, 0xb1f0360b00000000, 0x9695138a00000000,
+    0xe1a20bbb00000000, 0xc6c72e3a00000000, 0xee6e306200000000,
+    0xc90b15e300000000, 0x3d7f6b7f00000000, 0x1a1a4efe00000000,
+    0x32b350a600000000, 0x15d6752700000000, 0x62e16d1600000000,
+    0x4584489700000000, 0x6d2d56cf00000000, 0x4a48734e00000000,
+    0x834366ad00000000, 0xa426432c00000000, 0x8c8f5d7400000000,
+    0xabea78f500000000, 0xdcdd60c400000000, 0xfbb8454500000000,
+    0xd3115b1d00000000, 0xf4747e9c00000000, 0x7afed6fe00000000,
+    0x5d9bf37f00000000, 0x7532ed2700000000, 0x5257c8a600000000,
+    0x2560d09700000000, 0x0205f51600000000, 0x2aaceb4e00000000,
+    0x0dc9cecf00000000, 0xc4c2db2c00000000, 0xe3a7fead00000000,
+    0xcb0ee0f500000000, 0xec6bc57400000000, 0x9b5cdd4500000000,
+    0xbc39f8c400000000, 0x9490e69c00000000, 0xb3f5c31d00000000,
+    0x4781bd8100000000, 0x60e4980000000000, 0x484d865800000000,
+    0x6f28a3d900000000, 0x181fbbe800000000, 0x3f7a9e6900000000,
+    0x17d3803100000000, 0x30b6a5b000000000, 0xf9bdb05300000000,
+    0xded895d200000000, 0xf6718b8a00000000, 0xd114ae0b00000000,
+    0xa623b63a00000000, 0x814693bb00000000, 0xa9ef8de300000000,
+    0x8e8aa86200000000, 0xb5fadc2600000000, 0x929ff9a700000000,
+    0xba36e7ff00000000, 0x9d53c27e00000000, 0xea64da4f00000000,
+    0xcd01ffce00000000, 0xe5a8e19600000000, 0xc2cdc41700000000,
+    0x0bc6d1f400000000, 0x2ca3f47500000000, 0x040aea2d00000000,
+    0x236fcfac00000000, 0x5458d79d00000000, 0x733df21c00000000,
+    0x5b94ec4400000000, 0x7cf1c9c500000000, 0x8885b75900000000,
+    0xafe092d800000000, 0x87498c8000000000, 0xa02ca90100000000,
+    0xd71bb13000000000, 0xf07e94b100000000, 0xd8d78ae900000000,
+    0xffb2af6800000000, 0x36b9ba8b00000000, 0x11dc9f0a00000000,
+    0x3975815200000000, 0x1e10a4d300000000, 0x6927bce200000000,
+    0x4e42996300000000, 0x66eb873b00000000, 0x418ea2ba00000000,
+    0xcf040ad800000000, 0xe8612f5900000000, 0xc0c8310100000000,
+    0xe7ad148000000000, 0x909a0cb100000000, 0xb7ff293000000000,
+    0x9f56376800000000, 0xb83312e900000000, 0x7138070a00000000,
+    0x565d228b00000000, 0x7ef43cd300000000, 0x5991195200000000,
+    0x2ea6016300000000, 0x09c324e200000000, 0x216a3aba00000000,
+    0x060f1f3b00000000, 0xf27b61a700000000, 0xd51e442600000000,
+    0xfdb75a7e00000000, 0xdad27fff00000000, 0xade567ce00000000,
+    0x8a80424f00000000, 0xa2295c1700000000, 0x854c799600000000,
+    0x4c476c7500000000, 0x6b2249f400000000, 0x438b57ac00000000,
+    0x64ee722d00000000, 0x13d96a1c00000000, 0x34bc4f9d00000000,
+    0x1c1551c500000000, 0x3b70744400000000, 0x6af5b94d00000000,
+    0x4d909ccc00000000, 0x6539829400000000, 0x425ca71500000000,
+    0x356bbf2400000000, 0x120e9aa500000000, 0x3aa784fd00000000,
+    0x1dc2a17c00000000, 0xd4c9b49f00000000, 0xf3ac911e00000000,
+    0xdb058f4600000000, 0xfc60aac700000000, 0x8b57b2f600000000,
+    0xac32977700000000, 0x849b892f00000000, 0xa3feacae00000000,
+    0x578ad23200000000, 0x70eff7b300000000, 0x5846e9eb00000000,
+    0x7f23cc6a00000000, 0x0814d45b00000000, 0x2f71f1da00000000,
+    0x07d8ef8200000000, 0x20bdca0300000000, 0xe9b6dfe000000000,
+    0xced3fa6100000000, 0xe67ae43900000000, 0xc11fc1b800000000,
+    0xb628d98900000000, 0x914dfc0800000000, 0xb9e4e25000000000,
+    0x9e81c7d100000000, 0x100b6fb300000000, 0x376e4a3200000000,
+    0x1fc7546a00000000, 0x38a271eb00000000, 0x4f9569da00000000,
+    0x68f04c5b00000000, 0x4059520300000000, 0x673c778200000000,
+    0xae37626100000000, 0x895247e000000000, 0xa1fb59b800000000,
+    0x869e7c3900000000, 0xf1a9640800000000, 0xd6cc418900000000,
+    0xfe655fd100000000, 0xd9007a5000000000, 0x2d7404cc00000000,
+    0x0a11214d00000000, 0x22b83f1500000000, 0x05dd1a9400000000,
+    0x72ea02a500000000, 0x558f272400000000, 0x7d26397c00000000,
+    0x5a431cfd00000000, 0x9348091e00000000, 0xb42d2c9f00000000,
+    0x9c8432c700000000, 0xbbe1174600000000, 0xccd60f7700000000,
+    0xebb32af600000000, 0xc31a34ae00000000, 0xe47f112f00000000,
+    0xdf0f656b00000000, 0xf86a40ea00000000, 0xd0c35eb200000000,
+    0xf7a67b3300000000, 0x8091630200000000, 0xa7f4468300000000,
+    0x8f5d58db00000000, 0xa8387d5a00000000, 0x613368b900000000,
+    0x46564d3800000000, 0x6eff536000000000, 0x499a76e100000000,
+    0x3ead6ed000000000, 0x19c84b5100000000, 0x3161550900000000,
+    0x1604708800000000, 0xe2700e1400000000, 0xc5152b9500000000,
+    0xedbc35cd00000000, 0xcad9104c00000000, 0xbdee087d00000000,
+    0x9a8b2dfc00000000, 0xb22233a400000000, 0x9547162500000000,
+    0x5c4c03c600000000, 0x7b29264700000000, 0x5380381f00000000,
+    0x74e51d9e00000000, 0x03d205af00000000, 0x24b7202e00000000,
+    0x0c1e3e7600000000, 0x2b7b1bf700000000, 0xa5f1b39500000000,
+    0x8294961400000000, 0xaa3d884c00000000, 0x8d58adcd00000000,
+    0xfa6fb5fc00000000, 0xdd0a907d00000000, 0xf5a38e2500000000,
+    0xd2c6aba400000000, 0x1bcdbe4700000000, 0x3ca89bc600000000,
+    0x1401859e00000000, 0x3364a01f00000000, 0x4453b82e00000000,
+    0x63369daf00000000, 0x4b9f83f700000000, 0x6cfaa67600000000,
+    0x988ed8ea00000000, 0xbfebfd6b00000000, 0x9742e33300000000,
+    0xb027c6b200000000, 0xc710de8300000000, 0xe075fb0200000000,
+    0xc8dce55a00000000, 0xefb9c0db00000000, 0x26b2d53800000000,
+    0x01d7f0b900000000, 0x297eeee100000000, 0x0e1bcb6000000000,
+    0x792cd35100000000, 0x5e49f6d000000000, 0x76e0e88800000000,
+    0x5185cd0900000000}};
+
+#else /* W == 4 */
+
+local const z_crc_t FAR crc_braid_table[][256] = {
+   {0x00000000, 0x9ba54c6f, 0xec3b9e9f, 0x779ed2f0, 0x03063b7f,
+    0x98a37710, 0xef3da5e0, 0x7498e98f, 0x060c76fe, 0x9da93a91,
+    0xea37e861, 0x7192a40e, 0x050a4d81, 0x9eaf01ee, 0xe931d31e,
+    0x72949f71, 0x0c18edfc, 0x97bda193, 0xe0237363, 0x7b863f0c,
+    0x0f1ed683, 0x94bb9aec, 0xe325481c, 0x78800473, 0x0a149b02,
+    0x91b1d76d, 0xe62f059d, 0x7d8a49f2, 0x0912a07d, 0x92b7ec12,
+    0xe5293ee2, 0x7e8c728d, 0x1831dbf8, 0x83949797, 0xf40a4567,
+    0x6faf0908, 0x1b37e087, 0x8092ace8, 0xf70c7e18, 0x6ca93277,
+    0x1e3dad06, 0x8598e169, 0xf2063399, 0x69a37ff6, 0x1d3b9679,
+    0x869eda16, 0xf10008e6, 0x6aa54489, 0x14293604, 0x8f8c7a6b,
+    0xf812a89b, 0x63b7e4f4, 0x172f0d7b, 0x8c8a4114, 0xfb1493e4,
+    0x60b1df8b, 0x122540fa, 0x89800c95, 0xfe1ede65, 0x65bb920a,
+    0x11237b85, 0x8a8637ea, 0xfd18e51a, 0x66bda975, 0x3063b7f0,
+    0xabc6fb9f, 0xdc58296f, 0x47fd6500, 0x33658c8f, 0xa8c0c0e0,
+    0xdf5e1210, 0x44fb5e7f, 0x366fc10e, 0xadca8d61, 0xda545f91,
+    0x41f113fe, 0x3569fa71, 0xaeccb61e, 0xd95264ee, 0x42f72881,
+    0x3c7b5a0c, 0xa7de1663, 0xd040c493, 0x4be588fc, 0x3f7d6173,
+    0xa4d82d1c, 0xd346ffec, 0x48e3b383, 0x3a772cf2, 0xa1d2609d,
+    0xd64cb26d, 0x4de9fe02, 0x3971178d, 0xa2d45be2, 0xd54a8912,
+    0x4eefc57d, 0x28526c08, 0xb3f72067, 0xc469f297, 0x5fccbef8,
+    0x2b545777, 0xb0f11b18, 0xc76fc9e8, 0x5cca8587, 0x2e5e1af6,
+    0xb5fb5699, 0xc2658469, 0x59c0c806, 0x2d582189, 0xb6fd6de6,
+    0xc163bf16, 0x5ac6f379, 0x244a81f4, 0xbfefcd9b, 0xc8711f6b,
+    0x53d45304, 0x274cba8b, 0xbce9f6e4, 0xcb772414, 0x50d2687b,
+    0x2246f70a, 0xb9e3bb65, 0xce7d6995, 0x55d825fa, 0x2140cc75,
+    0xbae5801a, 0xcd7b52ea, 0x56de1e85, 0x60c76fe0, 0xfb62238f,
+    0x8cfcf17f, 0x1759bd10, 0x63c1549f, 0xf86418f0, 0x8ffaca00,
+    0x145f866f, 0x66cb191e, 0xfd6e5571, 0x8af08781, 0x1155cbee,
+    0x65cd2261, 0xfe686e0e, 0x89f6bcfe, 0x1253f091, 0x6cdf821c,
+    0xf77ace73, 0x80e41c83, 0x1b4150ec, 0x6fd9b963, 0xf47cf50c,
+    0x83e227fc, 0x18476b93, 0x6ad3f4e2, 0xf176b88d, 0x86e86a7d,
+    0x1d4d2612, 0x69d5cf9d, 0xf27083f2, 0x85ee5102, 0x1e4b1d6d,
+    0x78f6b418, 0xe353f877, 0x94cd2a87, 0x0f6866e8, 0x7bf08f67,
+    0xe055c308, 0x97cb11f8, 0x0c6e5d97, 0x7efac2e6, 0xe55f8e89,
+    0x92c15c79, 0x09641016, 0x7dfcf999, 0xe659b5f6, 0x91c76706,
+    0x0a622b69, 0x74ee59e4, 0xef4b158b, 0x98d5c77b, 0x03708b14,
+    0x77e8629b, 0xec4d2ef4, 0x9bd3fc04, 0x0076b06b, 0x72e22f1a,
+    0xe9476375, 0x9ed9b185, 0x057cfdea, 0x71e41465, 0xea41580a,
+    0x9ddf8afa, 0x067ac695, 0x50a4d810, 0xcb01947f, 0xbc9f468f,
+    0x273a0ae0, 0x53a2e36f, 0xc807af00, 0xbf997df0, 0x243c319f,
+    0x56a8aeee, 0xcd0de281, 0xba933071, 0x21367c1e, 0x55ae9591,
+    0xce0bd9fe, 0xb9950b0e, 0x22304761, 0x5cbc35ec, 0xc7197983,
+    0xb087ab73, 0x2b22e71c, 0x5fba0e93, 0xc41f42fc, 0xb381900c,
+    0x2824dc63, 0x5ab04312, 0xc1150f7d, 0xb68bdd8d, 0x2d2e91e2,
+    0x59b6786d, 0xc2133402, 0xb58de6f2, 0x2e28aa9d, 0x489503e8,
+    0xd3304f87, 0xa4ae9d77, 0x3f0bd118, 0x4b933897, 0xd03674f8,
+    0xa7a8a608, 0x3c0dea67, 0x4e997516, 0xd53c3979, 0xa2a2eb89,
+    0x3907a7e6, 0x4d9f4e69, 0xd63a0206, 0xa1a4d0f6, 0x3a019c99,
+    0x448dee14, 0xdf28a27b, 0xa8b6708b, 0x33133ce4, 0x478bd56b,
+    0xdc2e9904, 0xabb04bf4, 0x3015079b, 0x428198ea, 0xd924d485,
+    0xaeba0675, 0x351f4a1a, 0x4187a395, 0xda22effa, 0xadbc3d0a,
+    0x36197165},
+   {0x00000000, 0xc18edfc0, 0x586cb9c1, 0x99e26601, 0xb0d97382,
+    0x7157ac42, 0xe8b5ca43, 0x293b1583, 0xbac3e145, 0x7b4d3e85,
+    0xe2af5884, 0x23218744, 0x0a1a92c7, 0xcb944d07, 0x52762b06,
+    0x93f8f4c6, 0xaef6c4cb, 0x6f781b0b, 0xf69a7d0a, 0x3714a2ca,
+    0x1e2fb749, 0xdfa16889, 0x46430e88, 0x87cdd148, 0x1435258e,
+    0xd5bbfa4e, 0x4c599c4f, 0x8dd7438f, 0xa4ec560c, 0x656289cc,
+    0xfc80efcd, 0x3d0e300d, 0x869c8fd7, 0x47125017, 0xdef03616,
+    0x1f7ee9d6, 0x3645fc55, 0xf7cb2395, 0x6e294594, 0xafa79a54,
+    0x3c5f6e92, 0xfdd1b152, 0x6433d753, 0xa5bd0893, 0x8c861d10,
+    0x4d08c2d0, 0xd4eaa4d1, 0x15647b11, 0x286a4b1c, 0xe9e494dc,
+    0x7006f2dd, 0xb1882d1d, 0x98b3389e, 0x593de75e, 0xc0df815f,
+    0x01515e9f, 0x92a9aa59, 0x53277599, 0xcac51398, 0x0b4bcc58,
+    0x2270d9db, 0xe3fe061b, 0x7a1c601a, 0xbb92bfda, 0xd64819ef,
+    0x17c6c62f, 0x8e24a02e, 0x4faa7fee, 0x66916a6d, 0xa71fb5ad,
+    0x3efdd3ac, 0xff730c6c, 0x6c8bf8aa, 0xad05276a, 0x34e7416b,
+    0xf5699eab, 0xdc528b28, 0x1ddc54e8, 0x843e32e9, 0x45b0ed29,
+    0x78bedd24, 0xb93002e4, 0x20d264e5, 0xe15cbb25, 0xc867aea6,
+    0x09e97166, 0x900b1767, 0x5185c8a7, 0xc27d3c61, 0x03f3e3a1,
+    0x9a1185a0, 0x5b9f5a60, 0x72a44fe3, 0xb32a9023, 0x2ac8f622,
+    0xeb4629e2, 0x50d49638, 0x915a49f8, 0x08b82ff9, 0xc936f039,
+    0xe00de5ba, 0x21833a7a, 0xb8615c7b, 0x79ef83bb, 0xea17777d,
+    0x2b99a8bd, 0xb27bcebc, 0x73f5117c, 0x5ace04ff, 0x9b40db3f,
+    0x02a2bd3e, 0xc32c62fe, 0xfe2252f3, 0x3fac8d33, 0xa64eeb32,
+    0x67c034f2, 0x4efb2171, 0x8f75feb1, 0x169798b0, 0xd7194770,
+    0x44e1b3b6, 0x856f6c76, 0x1c8d0a77, 0xdd03d5b7, 0xf438c034,
+    0x35b61ff4, 0xac5479f5, 0x6ddaa635, 0x77e1359f, 0xb66fea5f,
+    0x2f8d8c5e, 0xee03539e, 0xc738461d, 0x06b699dd, 0x9f54ffdc,
+    0x5eda201c, 0xcd22d4da, 0x0cac0b1a, 0x954e6d1b, 0x54c0b2db,
+    0x7dfba758, 0xbc757898, 0x25971e99, 0xe419c159, 0xd917f154,
+    0x18992e94, 0x817b4895, 0x40f59755, 0x69ce82d6, 0xa8405d16,
+    0x31a23b17, 0xf02ce4d7, 0x63d41011, 0xa25acfd1, 0x3bb8a9d0,
+    0xfa367610, 0xd30d6393, 0x1283bc53, 0x8b61da52, 0x4aef0592,
+    0xf17dba48, 0x30f36588, 0xa9110389, 0x689fdc49, 0x41a4c9ca,
+    0x802a160a, 0x19c8700b, 0xd846afcb, 0x4bbe5b0d, 0x8a3084cd,
+    0x13d2e2cc, 0xd25c3d0c, 0xfb67288f, 0x3ae9f74f, 0xa30b914e,
+    0x62854e8e, 0x5f8b7e83, 0x9e05a143, 0x07e7c742, 0xc6691882,
+    0xef520d01, 0x2edcd2c1, 0xb73eb4c0, 0x76b06b00, 0xe5489fc6,
+    0x24c64006, 0xbd242607, 0x7caaf9c7, 0x5591ec44, 0x941f3384,
+    0x0dfd5585, 0xcc738a45, 0xa1a92c70, 0x6027f3b0, 0xf9c595b1,
+    0x384b4a71, 0x11705ff2, 0xd0fe8032, 0x491ce633, 0x889239f3,
+    0x1b6acd35, 0xdae412f5, 0x430674f4, 0x8288ab34, 0xabb3beb7,
+    0x6a3d6177, 0xf3df0776, 0x3251d8b6, 0x0f5fe8bb, 0xced1377b,
+    0x5733517a, 0x96bd8eba, 0xbf869b39, 0x7e0844f9, 0xe7ea22f8,
+    0x2664fd38, 0xb59c09fe, 0x7412d63e, 0xedf0b03f, 0x2c7e6fff,
+    0x05457a7c, 0xc4cba5bc, 0x5d29c3bd, 0x9ca71c7d, 0x2735a3a7,
+    0xe6bb7c67, 0x7f591a66, 0xbed7c5a6, 0x97ecd025, 0x56620fe5,
+    0xcf8069e4, 0x0e0eb624, 0x9df642e2, 0x5c789d22, 0xc59afb23,
+    0x041424e3, 0x2d2f3160, 0xeca1eea0, 0x754388a1, 0xb4cd5761,
+    0x89c3676c, 0x484db8ac, 0xd1afdead, 0x1021016d, 0x391a14ee,
+    0xf894cb2e, 0x6176ad2f, 0xa0f872ef, 0x33008629, 0xf28e59e9,
+    0x6b6c3fe8, 0xaae2e028, 0x83d9f5ab, 0x42572a6b, 0xdbb54c6a,
+    0x1a3b93aa},
+   {0x00000000, 0xefc26b3e, 0x04f5d03d, 0xeb37bb03, 0x09eba07a,
+    0xe629cb44, 0x0d1e7047, 0xe2dc1b79, 0x13d740f4, 0xfc152bca,
+    0x172290c9, 0xf8e0fbf7, 0x1a3ce08e, 0xf5fe8bb0, 0x1ec930b3,
+    0xf10b5b8d, 0x27ae81e8, 0xc86cead6, 0x235b51d5, 0xcc993aeb,
+    0x2e452192, 0xc1874aac, 0x2ab0f1af, 0xc5729a91, 0x3479c11c,
+    0xdbbbaa22, 0x308c1121, 0xdf4e7a1f, 0x3d926166, 0xd2500a58,
+    0x3967b15b, 0xd6a5da65, 0x4f5d03d0, 0xa09f68ee, 0x4ba8d3ed,
+    0xa46ab8d3, 0x46b6a3aa, 0xa974c894, 0x42437397, 0xad8118a9,
+    0x5c8a4324, 0xb348281a, 0x587f9319, 0xb7bdf827, 0x5561e35e,
+    0xbaa38860, 0x51943363, 0xbe56585d, 0x68f38238, 0x8731e906,
+    0x6c065205, 0x83c4393b, 0x61182242, 0x8eda497c, 0x65edf27f,
+    0x8a2f9941, 0x7b24c2cc, 0x94e6a9f2, 0x7fd112f1, 0x901379cf,
+    0x72cf62b6, 0x9d0d0988, 0x763ab28b, 0x99f8d9b5, 0x9eba07a0,
+    0x71786c9e, 0x9a4fd79d, 0x758dbca3, 0x9751a7da, 0x7893cce4,
+    0x93a477e7, 0x7c661cd9, 0x8d6d4754, 0x62af2c6a, 0x89989769,
+    0x665afc57, 0x8486e72e, 0x6b448c10, 0x80733713, 0x6fb15c2d,
+    0xb9148648, 0x56d6ed76, 0xbde15675, 0x52233d4b, 0xb0ff2632,
+    0x5f3d4d0c, 0xb40af60f, 0x5bc89d31, 0xaac3c6bc, 0x4501ad82,
+    0xae361681, 0x41f47dbf, 0xa32866c6, 0x4cea0df8, 0xa7ddb6fb,
+    0x481fddc5, 0xd1e70470, 0x3e256f4e, 0xd512d44d, 0x3ad0bf73,
+    0xd80ca40a, 0x37cecf34, 0xdcf97437, 0x333b1f09, 0xc2304484,
+    0x2df22fba, 0xc6c594b9, 0x2907ff87, 0xcbdbe4fe, 0x24198fc0,
+    0xcf2e34c3, 0x20ec5ffd, 0xf6498598, 0x198beea6, 0xf2bc55a5,
+    0x1d7e3e9b, 0xffa225e2, 0x10604edc, 0xfb57f5df, 0x14959ee1,
+    0xe59ec56c, 0x0a5cae52, 0xe16b1551, 0x0ea97e6f, 0xec756516,
+    0x03b70e28, 0xe880b52b, 0x0742de15, 0xe6050901, 0x09c7623f,
+    0xe2f0d93c, 0x0d32b202, 0xefeea97b, 0x002cc245, 0xeb1b7946,
+    0x04d91278, 0xf5d249f5, 0x1a1022cb, 0xf12799c8, 0x1ee5f2f6,
+    0xfc39e98f, 0x13fb82b1, 0xf8cc39b2, 0x170e528c, 0xc1ab88e9,
+    0x2e69e3d7, 0xc55e58d4, 0x2a9c33ea, 0xc8402893, 0x278243ad,
+    0xccb5f8ae, 0x23779390, 0xd27cc81d, 0x3dbea323, 0xd6891820,
+    0x394b731e, 0xdb976867, 0x34550359, 0xdf62b85a, 0x30a0d364,
+    0xa9580ad1, 0x469a61ef, 0xadaddaec, 0x426fb1d2, 0xa0b3aaab,
+    0x4f71c195, 0xa4467a96, 0x4b8411a8, 0xba8f4a25, 0x554d211b,
+    0xbe7a9a18, 0x51b8f126, 0xb364ea5f, 0x5ca68161, 0xb7913a62,
+    0x5853515c, 0x8ef68b39, 0x6134e007, 0x8a035b04, 0x65c1303a,
+    0x871d2b43, 0x68df407d, 0x83e8fb7e, 0x6c2a9040, 0x9d21cbcd,
+    0x72e3a0f3, 0x99d41bf0, 0x761670ce, 0x94ca6bb7, 0x7b080089,
+    0x903fbb8a, 0x7ffdd0b4, 0x78bf0ea1, 0x977d659f, 0x7c4ade9c,
+    0x9388b5a2, 0x7154aedb, 0x9e96c5e5, 0x75a17ee6, 0x9a6315d8,
+    0x6b684e55, 0x84aa256b, 0x6f9d9e68, 0x805ff556, 0x6283ee2f,
+    0x8d418511, 0x66763e12, 0x89b4552c, 0x5f118f49, 0xb0d3e477,
+    0x5be45f74, 0xb426344a, 0x56fa2f33, 0xb938440d, 0x520fff0e,
+    0xbdcd9430, 0x4cc6cfbd, 0xa304a483, 0x48331f80, 0xa7f174be,
+    0x452d6fc7, 0xaaef04f9, 0x41d8bffa, 0xae1ad4c4, 0x37e20d71,
+    0xd820664f, 0x3317dd4c, 0xdcd5b672, 0x3e09ad0b, 0xd1cbc635,
+    0x3afc7d36, 0xd53e1608, 0x24354d85, 0xcbf726bb, 0x20c09db8,
+    0xcf02f686, 0x2ddeedff, 0xc21c86c1, 0x292b3dc2, 0xc6e956fc,
+    0x104c8c99, 0xff8ee7a7, 0x14b95ca4, 0xfb7b379a, 0x19a72ce3,
+    0xf66547dd, 0x1d52fcde, 0xf29097e0, 0x039bcc6d, 0xec59a753,
+    0x076e1c50, 0xe8ac776e, 0x0a706c17, 0xe5b20729, 0x0e85bc2a,
+    0xe147d714},
+   {0x00000000, 0x177b1443, 0x2ef62886, 0x398d3cc5, 0x5dec510c,
+    0x4a97454f, 0x731a798a, 0x64616dc9, 0xbbd8a218, 0xaca3b65b,
+    0x952e8a9e, 0x82559edd, 0xe634f314, 0xf14fe757, 0xc8c2db92,
+    0xdfb9cfd1, 0xacc04271, 0xbbbb5632, 0x82366af7, 0x954d7eb4,
+    0xf12c137d, 0xe657073e, 0xdfda3bfb, 0xc8a12fb8, 0x1718e069,
+    0x0063f42a, 0x39eec8ef, 0x2e95dcac, 0x4af4b165, 0x5d8fa526,
+    0x640299e3, 0x73798da0, 0x82f182a3, 0x958a96e0, 0xac07aa25,
+    0xbb7cbe66, 0xdf1dd3af, 0xc866c7ec, 0xf1ebfb29, 0xe690ef6a,
+    0x392920bb, 0x2e5234f8, 0x17df083d, 0x00a41c7e, 0x64c571b7,
+    0x73be65f4, 0x4a335931, 0x5d484d72, 0x2e31c0d2, 0x394ad491,
+    0x00c7e854, 0x17bcfc17, 0x73dd91de, 0x64a6859d, 0x5d2bb958,
+    0x4a50ad1b, 0x95e962ca, 0x82927689, 0xbb1f4a4c, 0xac645e0f,
+    0xc80533c6, 0xdf7e2785, 0xe6f31b40, 0xf1880f03, 0xde920307,
+    0xc9e91744, 0xf0642b81, 0xe71f3fc2, 0x837e520b, 0x94054648,
+    0xad887a8d, 0xbaf36ece, 0x654aa11f, 0x7231b55c, 0x4bbc8999,
+    0x5cc79dda, 0x38a6f013, 0x2fdde450, 0x1650d895, 0x012bccd6,
+    0x72524176, 0x65295535, 0x5ca469f0, 0x4bdf7db3, 0x2fbe107a,
+    0x38c50439, 0x014838fc, 0x16332cbf, 0xc98ae36e, 0xdef1f72d,
+    0xe77ccbe8, 0xf007dfab, 0x9466b262, 0x831da621, 0xba909ae4,
+    0xadeb8ea7, 0x5c6381a4, 0x4b1895e7, 0x7295a922, 0x65eebd61,
+    0x018fd0a8, 0x16f4c4eb, 0x2f79f82e, 0x3802ec6d, 0xe7bb23bc,
+    0xf0c037ff, 0xc94d0b3a, 0xde361f79, 0xba5772b0, 0xad2c66f3,
+    0x94a15a36, 0x83da4e75, 0xf0a3c3d5, 0xe7d8d796, 0xde55eb53,
+    0xc92eff10, 0xad4f92d9, 0xba34869a, 0x83b9ba5f, 0x94c2ae1c,
+    0x4b7b61cd, 0x5c00758e, 0x658d494b, 0x72f65d08, 0x169730c1,
+    0x01ec2482, 0x38611847, 0x2f1a0c04, 0x6655004f, 0x712e140c,
+    0x48a328c9, 0x5fd83c8a, 0x3bb95143, 0x2cc24500, 0x154f79c5,
+    0x02346d86, 0xdd8da257, 0xcaf6b614, 0xf37b8ad1, 0xe4009e92,
+    0x8061f35b, 0x971ae718, 0xae97dbdd, 0xb9eccf9e, 0xca95423e,
+    0xddee567d, 0xe4636ab8, 0xf3187efb, 0x97791332, 0x80020771,
+    0xb98f3bb4, 0xaef42ff7, 0x714de026, 0x6636f465, 0x5fbbc8a0,
+    0x48c0dce3, 0x2ca1b12a, 0x3bdaa569, 0x025799ac, 0x152c8def,
+    0xe4a482ec, 0xf3df96af, 0xca52aa6a, 0xdd29be29, 0xb948d3e0,
+    0xae33c7a3, 0x97befb66, 0x80c5ef25, 0x5f7c20f4, 0x480734b7,
+    0x718a0872, 0x66f11c31, 0x029071f8, 0x15eb65bb, 0x2c66597e,
+    0x3b1d4d3d, 0x4864c09d, 0x5f1fd4de, 0x6692e81b, 0x71e9fc58,
+    0x15889191, 0x02f385d2, 0x3b7eb917, 0x2c05ad54, 0xf3bc6285,
+    0xe4c776c6, 0xdd4a4a03, 0xca315e40, 0xae503389, 0xb92b27ca,
+    0x80a61b0f, 0x97dd0f4c, 0xb8c70348, 0xafbc170b, 0x96312bce,
+    0x814a3f8d, 0xe52b5244, 0xf2504607, 0xcbdd7ac2, 0xdca66e81,
+    0x031fa150, 0x1464b513, 0x2de989d6, 0x3a929d95, 0x5ef3f05c,
+    0x4988e41f, 0x7005d8da, 0x677ecc99, 0x14074139, 0x037c557a,
+    0x3af169bf, 0x2d8a7dfc, 0x49eb1035, 0x5e900476, 0x671d38b3,
+    0x70662cf0, 0xafdfe321, 0xb8a4f762, 0x8129cba7, 0x9652dfe4,
+    0xf233b22d, 0xe548a66e, 0xdcc59aab, 0xcbbe8ee8, 0x3a3681eb,
+    0x2d4d95a8, 0x14c0a96d, 0x03bbbd2e, 0x67dad0e7, 0x70a1c4a4,
+    0x492cf861, 0x5e57ec22, 0x81ee23f3, 0x969537b0, 0xaf180b75,
+    0xb8631f36, 0xdc0272ff, 0xcb7966bc, 0xf2f45a79, 0xe58f4e3a,
+    0x96f6c39a, 0x818dd7d9, 0xb800eb1c, 0xaf7bff5f, 0xcb1a9296,
+    0xdc6186d5, 0xe5ecba10, 0xf297ae53, 0x2d2e6182, 0x3a5575c1,
+    0x03d84904, 0x14a35d47, 0x70c2308e, 0x67b924cd, 0x5e341808,
+    0x494f0c4b}};
+
+local const z_word_t FAR crc_braid_big_table[][256] = {
+   {0x00000000, 0x43147b17, 0x8628f62e, 0xc53c8d39, 0x0c51ec5d,
+    0x4f45974a, 0x8a791a73, 0xc96d6164, 0x18a2d8bb, 0x5bb6a3ac,
+    0x9e8a2e95, 0xdd9e5582, 0x14f334e6, 0x57e74ff1, 0x92dbc2c8,
+    0xd1cfb9df, 0x7142c0ac, 0x3256bbbb, 0xf76a3682, 0xb47e4d95,
+    0x7d132cf1, 0x3e0757e6, 0xfb3bdadf, 0xb82fa1c8, 0x69e01817,
+    0x2af46300, 0xefc8ee39, 0xacdc952e, 0x65b1f44a, 0x26a58f5d,
+    0xe3990264, 0xa08d7973, 0xa382f182, 0xe0968a95, 0x25aa07ac,
+    0x66be7cbb, 0xafd31ddf, 0xecc766c8, 0x29fbebf1, 0x6aef90e6,
+    0xbb202939, 0xf834522e, 0x3d08df17, 0x7e1ca400, 0xb771c564,
+    0xf465be73, 0x3159334a, 0x724d485d, 0xd2c0312e, 0x91d44a39,
+    0x54e8c700, 0x17fcbc17, 0xde91dd73, 0x9d85a664, 0x58b92b5d,
+    0x1bad504a, 0xca62e995, 0x89769282, 0x4c4a1fbb, 0x0f5e64ac,
+    0xc63305c8, 0x85277edf, 0x401bf3e6, 0x030f88f1, 0x070392de,
+    0x4417e9c9, 0x812b64f0, 0xc23f1fe7, 0x0b527e83, 0x48460594,
+    0x8d7a88ad, 0xce6ef3ba, 0x1fa14a65, 0x5cb53172, 0x9989bc4b,
+    0xda9dc75c, 0x13f0a638, 0x50e4dd2f, 0x95d85016, 0xd6cc2b01,
+    0x76415272, 0x35552965, 0xf069a45c, 0xb37ddf4b, 0x7a10be2f,
+    0x3904c538, 0xfc384801, 0xbf2c3316, 0x6ee38ac9, 0x2df7f1de,
+    0xe8cb7ce7, 0xabdf07f0, 0x62b26694, 0x21a61d83, 0xe49a90ba,
+    0xa78eebad, 0xa481635c, 0xe795184b, 0x22a99572, 0x61bdee65,
+    0xa8d08f01, 0xebc4f416, 0x2ef8792f, 0x6dec0238, 0xbc23bbe7,
+    0xff37c0f0, 0x3a0b4dc9, 0x791f36de, 0xb07257ba, 0xf3662cad,
+    0x365aa194, 0x754eda83, 0xd5c3a3f0, 0x96d7d8e7, 0x53eb55de,
+    0x10ff2ec9, 0xd9924fad, 0x9a8634ba, 0x5fbab983, 0x1caec294,
+    0xcd617b4b, 0x8e75005c, 0x4b498d65, 0x085df672, 0xc1309716,
+    0x8224ec01, 0x47186138, 0x040c1a2f, 0x4f005566, 0x0c142e71,
+    0xc928a348, 0x8a3cd85f, 0x4351b93b, 0x0045c22c, 0xc5794f15,
+    0x866d3402, 0x57a28ddd, 0x14b6f6ca, 0xd18a7bf3, 0x929e00e4,
+    0x5bf36180, 0x18e71a97, 0xdddb97ae, 0x9ecfecb9, 0x3e4295ca,
+    0x7d56eedd, 0xb86a63e4, 0xfb7e18f3, 0x32137997, 0x71070280,
+    0xb43b8fb9, 0xf72ff4ae, 0x26e04d71, 0x65f43666, 0xa0c8bb5f,
+    0xe3dcc048, 0x2ab1a12c, 0x69a5da3b, 0xac995702, 0xef8d2c15,
+    0xec82a4e4, 0xaf96dff3, 0x6aaa52ca, 0x29be29dd, 0xe0d348b9,
+    0xa3c733ae, 0x66fbbe97, 0x25efc580, 0xf4207c5f, 0xb7340748,
+    0x72088a71, 0x311cf166, 0xf8719002, 0xbb65eb15, 0x7e59662c,
+    0x3d4d1d3b, 0x9dc06448, 0xded41f5f, 0x1be89266, 0x58fce971,
+    0x91918815, 0xd285f302, 0x17b97e3b, 0x54ad052c, 0x8562bcf3,
+    0xc676c7e4, 0x034a4add, 0x405e31ca, 0x893350ae, 0xca272bb9,
+    0x0f1ba680, 0x4c0fdd97, 0x4803c7b8, 0x0b17bcaf, 0xce2b3196,
+    0x8d3f4a81, 0x44522be5, 0x074650f2, 0xc27addcb, 0x816ea6dc,
+    0x50a11f03, 0x13b56414, 0xd689e92d, 0x959d923a, 0x5cf0f35e,
+    0x1fe48849, 0xdad80570, 0x99cc7e67, 0x39410714, 0x7a557c03,
+    0xbf69f13a, 0xfc7d8a2d, 0x3510eb49, 0x7604905e, 0xb3381d67,
+    0xf02c6670, 0x21e3dfaf, 0x62f7a4b8, 0xa7cb2981, 0xe4df5296,
+    0x2db233f2, 0x6ea648e5, 0xab9ac5dc, 0xe88ebecb, 0xeb81363a,
+    0xa8954d2d, 0x6da9c014, 0x2ebdbb03, 0xe7d0da67, 0xa4c4a170,
+    0x61f82c49, 0x22ec575e, 0xf323ee81, 0xb0379596, 0x750b18af,
+    0x361f63b8, 0xff7202dc, 0xbc6679cb, 0x795af4f2, 0x3a4e8fe5,
+    0x9ac3f696, 0xd9d78d81, 0x1ceb00b8, 0x5fff7baf, 0x96921acb,
+    0xd58661dc, 0x10baece5, 0x53ae97f2, 0x82612e2d, 0xc175553a,
+    0x0449d803, 0x475da314, 0x8e30c270, 0xcd24b967, 0x0818345e,
+    0x4b0c4f49},
+   {0x00000000, 0x3e6bc2ef, 0x3dd0f504, 0x03bb37eb, 0x7aa0eb09,
+    0x44cb29e6, 0x47701e0d, 0x791bdce2, 0xf440d713, 0xca2b15fc,
+    0xc9902217, 0xf7fbe0f8, 0x8ee03c1a, 0xb08bfef5, 0xb330c91e,
+    0x8d5b0bf1, 0xe881ae27, 0xd6ea6cc8, 0xd5515b23, 0xeb3a99cc,
+    0x9221452e, 0xac4a87c1, 0xaff1b02a, 0x919a72c5, 0x1cc17934,
+    0x22aabbdb, 0x21118c30, 0x1f7a4edf, 0x6661923d, 0x580a50d2,
+    0x5bb16739, 0x65daa5d6, 0xd0035d4f, 0xee689fa0, 0xedd3a84b,
+    0xd3b86aa4, 0xaaa3b646, 0x94c874a9, 0x97734342, 0xa91881ad,
+    0x24438a5c, 0x1a2848b3, 0x19937f58, 0x27f8bdb7, 0x5ee36155,
+    0x6088a3ba, 0x63339451, 0x5d5856be, 0x3882f368, 0x06e93187,
+    0x0552066c, 0x3b39c483, 0x42221861, 0x7c49da8e, 0x7ff2ed65,
+    0x41992f8a, 0xccc2247b, 0xf2a9e694, 0xf112d17f, 0xcf791390,
+    0xb662cf72, 0x88090d9d, 0x8bb23a76, 0xb5d9f899, 0xa007ba9e,
+    0x9e6c7871, 0x9dd74f9a, 0xa3bc8d75, 0xdaa75197, 0xe4cc9378,
+    0xe777a493, 0xd91c667c, 0x54476d8d, 0x6a2caf62, 0x69979889,
+    0x57fc5a66, 0x2ee78684, 0x108c446b, 0x13377380, 0x2d5cb16f,
+    0x488614b9, 0x76edd656, 0x7556e1bd, 0x4b3d2352, 0x3226ffb0,
+    0x0c4d3d5f, 0x0ff60ab4, 0x319dc85b, 0xbcc6c3aa, 0x82ad0145,
+    0x811636ae, 0xbf7df441, 0xc66628a3, 0xf80dea4c, 0xfbb6dda7,
+    0xc5dd1f48, 0x7004e7d1, 0x4e6f253e, 0x4dd412d5, 0x73bfd03a,
+    0x0aa40cd8, 0x34cfce37, 0x3774f9dc, 0x091f3b33, 0x844430c2,
+    0xba2ff22d, 0xb994c5c6, 0x87ff0729, 0xfee4dbcb, 0xc08f1924,
+    0xc3342ecf, 0xfd5fec20, 0x988549f6, 0xa6ee8b19, 0xa555bcf2,
+    0x9b3e7e1d, 0xe225a2ff, 0xdc4e6010, 0xdff557fb, 0xe19e9514,
+    0x6cc59ee5, 0x52ae5c0a, 0x51156be1, 0x6f7ea90e, 0x166575ec,
+    0x280eb703, 0x2bb580e8, 0x15de4207, 0x010905e6, 0x3f62c709,
+    0x3cd9f0e2, 0x02b2320d, 0x7ba9eeef, 0x45c22c00, 0x46791beb,
+    0x7812d904, 0xf549d2f5, 0xcb22101a, 0xc89927f1, 0xf6f2e51e,
+    0x8fe939fc, 0xb182fb13, 0xb239ccf8, 0x8c520e17, 0xe988abc1,
+    0xd7e3692e, 0xd4585ec5, 0xea339c2a, 0x932840c8, 0xad438227,
+    0xaef8b5cc, 0x90937723, 0x1dc87cd2, 0x23a3be3d, 0x201889d6,
+    0x1e734b39, 0x676897db, 0x59035534, 0x5ab862df, 0x64d3a030,
+    0xd10a58a9, 0xef619a46, 0xecdaadad, 0xd2b16f42, 0xabaab3a0,
+    0x95c1714f, 0x967a46a4, 0xa811844b, 0x254a8fba, 0x1b214d55,
+    0x189a7abe, 0x26f1b851, 0x5fea64b3, 0x6181a65c, 0x623a91b7,
+    0x5c515358, 0x398bf68e, 0x07e03461, 0x045b038a, 0x3a30c165,
+    0x432b1d87, 0x7d40df68, 0x7efbe883, 0x40902a6c, 0xcdcb219d,
+    0xf3a0e372, 0xf01bd499, 0xce701676, 0xb76bca94, 0x8900087b,
+    0x8abb3f90, 0xb4d0fd7f, 0xa10ebf78, 0x9f657d97, 0x9cde4a7c,
+    0xa2b58893, 0xdbae5471, 0xe5c5969e, 0xe67ea175, 0xd815639a,
+    0x554e686b, 0x6b25aa84, 0x689e9d6f, 0x56f55f80, 0x2fee8362,
+    0x1185418d, 0x123e7666, 0x2c55b489, 0x498f115f, 0x77e4d3b0,
+    0x745fe45b, 0x4a3426b4, 0x332ffa56, 0x0d4438b9, 0x0eff0f52,
+    0x3094cdbd, 0xbdcfc64c, 0x83a404a3, 0x801f3348, 0xbe74f1a7,
+    0xc76f2d45, 0xf904efaa, 0xfabfd841, 0xc4d41aae, 0x710de237,
+    0x4f6620d8, 0x4cdd1733, 0x72b6d5dc, 0x0bad093e, 0x35c6cbd1,
+    0x367dfc3a, 0x08163ed5, 0x854d3524, 0xbb26f7cb, 0xb89dc020,
+    0x86f602cf, 0xffedde2d, 0xc1861cc2, 0xc23d2b29, 0xfc56e9c6,
+    0x998c4c10, 0xa7e78eff, 0xa45cb914, 0x9a377bfb, 0xe32ca719,
+    0xdd4765f6, 0xdefc521d, 0xe09790f2, 0x6dcc9b03, 0x53a759ec,
+    0x501c6e07, 0x6e77ace8, 0x176c700a, 0x2907b2e5, 0x2abc850e,
+    0x14d747e1},
+   {0x00000000, 0xc0df8ec1, 0xc1b96c58, 0x0166e299, 0x8273d9b0,
+    0x42ac5771, 0x43cab5e8, 0x83153b29, 0x45e1c3ba, 0x853e4d7b,
+    0x8458afe2, 0x44872123, 0xc7921a0a, 0x074d94cb, 0x062b7652,
+    0xc6f4f893, 0xcbc4f6ae, 0x0b1b786f, 0x0a7d9af6, 0xcaa21437,
+    0x49b72f1e, 0x8968a1df, 0x880e4346, 0x48d1cd87, 0x8e253514,
+    0x4efabbd5, 0x4f9c594c, 0x8f43d78d, 0x0c56eca4, 0xcc896265,
+    0xcdef80fc, 0x0d300e3d, 0xd78f9c86, 0x17501247, 0x1636f0de,
+    0xd6e97e1f, 0x55fc4536, 0x9523cbf7, 0x9445296e, 0x549aa7af,
+    0x926e5f3c, 0x52b1d1fd, 0x53d73364, 0x9308bda5, 0x101d868c,
+    0xd0c2084d, 0xd1a4ead4, 0x117b6415, 0x1c4b6a28, 0xdc94e4e9,
+    0xddf20670, 0x1d2d88b1, 0x9e38b398, 0x5ee73d59, 0x5f81dfc0,
+    0x9f5e5101, 0x59aaa992, 0x99752753, 0x9813c5ca, 0x58cc4b0b,
+    0xdbd97022, 0x1b06fee3, 0x1a601c7a, 0xdabf92bb, 0xef1948d6,
+    0x2fc6c617, 0x2ea0248e, 0xee7faa4f, 0x6d6a9166, 0xadb51fa7,
+    0xacd3fd3e, 0x6c0c73ff, 0xaaf88b6c, 0x6a2705ad, 0x6b41e734,
+    0xab9e69f5, 0x288b52dc, 0xe854dc1d, 0xe9323e84, 0x29edb045,
+    0x24ddbe78, 0xe40230b9, 0xe564d220, 0x25bb5ce1, 0xa6ae67c8,
+    0x6671e909, 0x67170b90, 0xa7c88551, 0x613c7dc2, 0xa1e3f303,
+    0xa085119a, 0x605a9f5b, 0xe34fa472, 0x23902ab3, 0x22f6c82a,
+    0xe22946eb, 0x3896d450, 0xf8495a91, 0xf92fb808, 0x39f036c9,
+    0xbae50de0, 0x7a3a8321, 0x7b5c61b8, 0xbb83ef79, 0x7d7717ea,
+    0xbda8992b, 0xbcce7bb2, 0x7c11f573, 0xff04ce5a, 0x3fdb409b,
+    0x3ebda202, 0xfe622cc3, 0xf35222fe, 0x338dac3f, 0x32eb4ea6,
+    0xf234c067, 0x7121fb4e, 0xb1fe758f, 0xb0989716, 0x704719d7,
+    0xb6b3e144, 0x766c6f85, 0x770a8d1c, 0xb7d503dd, 0x34c038f4,
+    0xf41fb635, 0xf57954ac, 0x35a6da6d, 0x9f35e177, 0x5fea6fb6,
+    0x5e8c8d2f, 0x9e5303ee, 0x1d4638c7, 0xdd99b606, 0xdcff549f,
+    0x1c20da5e, 0xdad422cd, 0x1a0bac0c, 0x1b6d4e95, 0xdbb2c054,
+    0x58a7fb7d, 0x987875bc, 0x991e9725, 0x59c119e4, 0x54f117d9,
+    0x942e9918, 0x95487b81, 0x5597f540, 0xd682ce69, 0x165d40a8,
+    0x173ba231, 0xd7e42cf0, 0x1110d463, 0xd1cf5aa2, 0xd0a9b83b,
+    0x107636fa, 0x93630dd3, 0x53bc8312, 0x52da618b, 0x9205ef4a,
+    0x48ba7df1, 0x8865f330, 0x890311a9, 0x49dc9f68, 0xcac9a441,
+    0x0a162a80, 0x0b70c819, 0xcbaf46d8, 0x0d5bbe4b, 0xcd84308a,
+    0xcce2d213, 0x0c3d5cd2, 0x8f2867fb, 0x4ff7e93a, 0x4e910ba3,
+    0x8e4e8562, 0x837e8b5f, 0x43a1059e, 0x42c7e707, 0x821869c6,
+    0x010d52ef, 0xc1d2dc2e, 0xc0b43eb7, 0x006bb076, 0xc69f48e5,
+    0x0640c624, 0x072624bd, 0xc7f9aa7c, 0x44ec9155, 0x84331f94,
+    0x8555fd0d, 0x458a73cc, 0x702ca9a1, 0xb0f32760, 0xb195c5f9,
+    0x714a4b38, 0xf25f7011, 0x3280fed0, 0x33e61c49, 0xf3399288,
+    0x35cd6a1b, 0xf512e4da, 0xf4740643, 0x34ab8882, 0xb7beb3ab,
+    0x77613d6a, 0x7607dff3, 0xb6d85132, 0xbbe85f0f, 0x7b37d1ce,
+    0x7a513357, 0xba8ebd96, 0x399b86bf, 0xf944087e, 0xf822eae7,
+    0x38fd6426, 0xfe099cb5, 0x3ed61274, 0x3fb0f0ed, 0xff6f7e2c,
+    0x7c7a4505, 0xbca5cbc4, 0xbdc3295d, 0x7d1ca79c, 0xa7a33527,
+    0x677cbbe6, 0x661a597f, 0xa6c5d7be, 0x25d0ec97, 0xe50f6256,
+    0xe46980cf, 0x24b60e0e, 0xe242f69d, 0x229d785c, 0x23fb9ac5,
+    0xe3241404, 0x60312f2d, 0xa0eea1ec, 0xa1884375, 0x6157cdb4,
+    0x6c67c389, 0xacb84d48, 0xaddeafd1, 0x6d012110, 0xee141a39,
+    0x2ecb94f8, 0x2fad7661, 0xef72f8a0, 0x29860033, 0xe9598ef2,
+    0xe83f6c6b, 0x28e0e2aa, 0xabf5d983, 0x6b2a5742, 0x6a4cb5db,
+    0xaa933b1a},
+   {0x00000000, 0x6f4ca59b, 0x9f9e3bec, 0xf0d29e77, 0x7f3b0603,
+    0x1077a398, 0xe0a53def, 0x8fe99874, 0xfe760c06, 0x913aa99d,
+    0x61e837ea, 0x0ea49271, 0x814d0a05, 0xee01af9e, 0x1ed331e9,
+    0x719f9472, 0xfced180c, 0x93a1bd97, 0x637323e0, 0x0c3f867b,
+    0x83d61e0f, 0xec9abb94, 0x1c4825e3, 0x73048078, 0x029b140a,
+    0x6dd7b191, 0x9d052fe6, 0xf2498a7d, 0x7da01209, 0x12ecb792,
+    0xe23e29e5, 0x8d728c7e, 0xf8db3118, 0x97979483, 0x67450af4,
+    0x0809af6f, 0x87e0371b, 0xe8ac9280, 0x187e0cf7, 0x7732a96c,
+    0x06ad3d1e, 0x69e19885, 0x993306f2, 0xf67fa369, 0x79963b1d,
+    0x16da9e86, 0xe60800f1, 0x8944a56a, 0x04362914, 0x6b7a8c8f,
+    0x9ba812f8, 0xf4e4b763, 0x7b0d2f17, 0x14418a8c, 0xe49314fb,
+    0x8bdfb160, 0xfa402512, 0x950c8089, 0x65de1efe, 0x0a92bb65,
+    0x857b2311, 0xea37868a, 0x1ae518fd, 0x75a9bd66, 0xf0b76330,
+    0x9ffbc6ab, 0x6f2958dc, 0x0065fd47, 0x8f8c6533, 0xe0c0c0a8,
+    0x10125edf, 0x7f5efb44, 0x0ec16f36, 0x618dcaad, 0x915f54da,
+    0xfe13f141, 0x71fa6935, 0x1eb6ccae, 0xee6452d9, 0x8128f742,
+    0x0c5a7b3c, 0x6316dea7, 0x93c440d0, 0xfc88e54b, 0x73617d3f,
+    0x1c2dd8a4, 0xecff46d3, 0x83b3e348, 0xf22c773a, 0x9d60d2a1,
+    0x6db24cd6, 0x02fee94d, 0x8d177139, 0xe25bd4a2, 0x12894ad5,
+    0x7dc5ef4e, 0x086c5228, 0x6720f7b3, 0x97f269c4, 0xf8becc5f,
+    0x7757542b, 0x181bf1b0, 0xe8c96fc7, 0x8785ca5c, 0xf61a5e2e,
+    0x9956fbb5, 0x698465c2, 0x06c8c059, 0x8921582d, 0xe66dfdb6,
+    0x16bf63c1, 0x79f3c65a, 0xf4814a24, 0x9bcdefbf, 0x6b1f71c8,
+    0x0453d453, 0x8bba4c27, 0xe4f6e9bc, 0x142477cb, 0x7b68d250,
+    0x0af74622, 0x65bbe3b9, 0x95697dce, 0xfa25d855, 0x75cc4021,
+    0x1a80e5ba, 0xea527bcd, 0x851ede56, 0xe06fc760, 0x8f2362fb,
+    0x7ff1fc8c, 0x10bd5917, 0x9f54c163, 0xf01864f8, 0x00cafa8f,
+    0x6f865f14, 0x1e19cb66, 0x71556efd, 0x8187f08a, 0xeecb5511,
+    0x6122cd65, 0x0e6e68fe, 0xfebcf689, 0x91f05312, 0x1c82df6c,
+    0x73ce7af7, 0x831ce480, 0xec50411b, 0x63b9d96f, 0x0cf57cf4,
+    0xfc27e283, 0x936b4718, 0xe2f4d36a, 0x8db876f1, 0x7d6ae886,
+    0x12264d1d, 0x9dcfd569, 0xf28370f2, 0x0251ee85, 0x6d1d4b1e,
+    0x18b4f678, 0x77f853e3, 0x872acd94, 0xe866680f, 0x678ff07b,
+    0x08c355e0, 0xf811cb97, 0x975d6e0c, 0xe6c2fa7e, 0x898e5fe5,
+    0x795cc192, 0x16106409, 0x99f9fc7d, 0xf6b559e6, 0x0667c791,
+    0x692b620a, 0xe459ee74, 0x8b154bef, 0x7bc7d598, 0x148b7003,
+    0x9b62e877, 0xf42e4dec, 0x04fcd39b, 0x6bb07600, 0x1a2fe272,
+    0x756347e9, 0x85b1d99e, 0xeafd7c05, 0x6514e471, 0x0a5841ea,
+    0xfa8adf9d, 0x95c67a06, 0x10d8a450, 0x7f9401cb, 0x8f469fbc,
+    0xe00a3a27, 0x6fe3a253, 0x00af07c8, 0xf07d99bf, 0x9f313c24,
+    0xeeaea856, 0x81e20dcd, 0x713093ba, 0x1e7c3621, 0x9195ae55,
+    0xfed90bce, 0x0e0b95b9, 0x61473022, 0xec35bc5c, 0x837919c7,
+    0x73ab87b0, 0x1ce7222b, 0x930eba5f, 0xfc421fc4, 0x0c9081b3,
+    0x63dc2428, 0x1243b05a, 0x7d0f15c1, 0x8ddd8bb6, 0xe2912e2d,
+    0x6d78b659, 0x023413c2, 0xf2e68db5, 0x9daa282e, 0xe8039548,
+    0x874f30d3, 0x779daea4, 0x18d10b3f, 0x9738934b, 0xf87436d0,
+    0x08a6a8a7, 0x67ea0d3c, 0x1675994e, 0x79393cd5, 0x89eba2a2,
+    0xe6a70739, 0x694e9f4d, 0x06023ad6, 0xf6d0a4a1, 0x999c013a,
+    0x14ee8d44, 0x7ba228df, 0x8b70b6a8, 0xe43c1333, 0x6bd58b47,
+    0x04992edc, 0xf44bb0ab, 0x9b071530, 0xea988142, 0x85d424d9,
+    0x7506baae, 0x1a4a1f35, 0x95a38741, 0xfaef22da, 0x0a3dbcad,
+    0x65711936}};
+
+#endif
+
+#endif
+
+#if N == 4
+
+#if W == 8
+
+local const z_crc_t FAR crc_braid_table[][256] = {
+   {0x00000000, 0xf1da05aa, 0x38c50d15, 0xc91f08bf, 0x718a1a2a,
+    0x80501f80, 0x494f173f, 0xb8951295, 0xe3143454, 0x12ce31fe,
+    0xdbd13941, 0x2a0b3ceb, 0x929e2e7e, 0x63442bd4, 0xaa5b236b,
+    0x5b8126c1, 0x1d596ee9, 0xec836b43, 0x259c63fc, 0xd4466656,
+    0x6cd374c3, 0x9d097169, 0x541679d6, 0xa5cc7c7c, 0xfe4d5abd,
+    0x0f975f17, 0xc68857a8, 0x37525202, 0x8fc74097, 0x7e1d453d,
+    0xb7024d82, 0x46d84828, 0x3ab2ddd2, 0xcb68d878, 0x0277d0c7,
+    0xf3add56d, 0x4b38c7f8, 0xbae2c252, 0x73fdcaed, 0x8227cf47,
+    0xd9a6e986, 0x287cec2c, 0xe163e493, 0x10b9e139, 0xa82cf3ac,
+    0x59f6f606, 0x90e9feb9, 0x6133fb13, 0x27ebb33b, 0xd631b691,
+    0x1f2ebe2e, 0xeef4bb84, 0x5661a911, 0xa7bbacbb, 0x6ea4a404,
+    0x9f7ea1ae, 0xc4ff876f, 0x352582c5, 0xfc3a8a7a, 0x0de08fd0,
+    0xb5759d45, 0x44af98ef, 0x8db09050, 0x7c6a95fa, 0x7565bba4,
+    0x84bfbe0e, 0x4da0b6b1, 0xbc7ab31b, 0x04efa18e, 0xf535a424,
+    0x3c2aac9b, 0xcdf0a931, 0x96718ff0, 0x67ab8a5a, 0xaeb482e5,
+    0x5f6e874f, 0xe7fb95da, 0x16219070, 0xdf3e98cf, 0x2ee49d65,
+    0x683cd54d, 0x99e6d0e7, 0x50f9d858, 0xa123ddf2, 0x19b6cf67,
+    0xe86ccacd, 0x2173c272, 0xd0a9c7d8, 0x8b28e119, 0x7af2e4b3,
+    0xb3edec0c, 0x4237e9a6, 0xfaa2fb33, 0x0b78fe99, 0xc267f626,
+    0x33bdf38c, 0x4fd76676, 0xbe0d63dc, 0x77126b63, 0x86c86ec9,
+    0x3e5d7c5c, 0xcf8779f6, 0x06987149, 0xf74274e3, 0xacc35222,
+    0x5d195788, 0x94065f37, 0x65dc5a9d, 0xdd494808, 0x2c934da2,
+    0xe58c451d, 0x145640b7, 0x528e089f, 0xa3540d35, 0x6a4b058a,
+    0x9b910020, 0x230412b5, 0xd2de171f, 0x1bc11fa0, 0xea1b1a0a,
+    0xb19a3ccb, 0x40403961, 0x895f31de, 0x78853474, 0xc01026e1,
+    0x31ca234b, 0xf8d52bf4, 0x090f2e5e, 0xeacb7748, 0x1b1172e2,
+    0xd20e7a5d, 0x23d47ff7, 0x9b416d62, 0x6a9b68c8, 0xa3846077,
+    0x525e65dd, 0x09df431c, 0xf80546b6, 0x311a4e09, 0xc0c04ba3,
+    0x78555936, 0x898f5c9c, 0x40905423, 0xb14a5189, 0xf79219a1,
+    0x06481c0b, 0xcf5714b4, 0x3e8d111e, 0x8618038b, 0x77c20621,
+    0xbedd0e9e, 0x4f070b34, 0x14862df5, 0xe55c285f, 0x2c4320e0,
+    0xdd99254a, 0x650c37df, 0x94d63275, 0x5dc93aca, 0xac133f60,
+    0xd079aa9a, 0x21a3af30, 0xe8bca78f, 0x1966a225, 0xa1f3b0b0,
+    0x5029b51a, 0x9936bda5, 0x68ecb80f, 0x336d9ece, 0xc2b79b64,
+    0x0ba893db, 0xfa729671, 0x42e784e4, 0xb33d814e, 0x7a2289f1,
+    0x8bf88c5b, 0xcd20c473, 0x3cfac1d9, 0xf5e5c966, 0x043fcccc,
+    0xbcaade59, 0x4d70dbf3, 0x846fd34c, 0x75b5d6e6, 0x2e34f027,
+    0xdfeef58d, 0x16f1fd32, 0xe72bf898, 0x5fbeea0d, 0xae64efa7,
+    0x677be718, 0x96a1e2b2, 0x9faeccec, 0x6e74c946, 0xa76bc1f9,
+    0x56b1c453, 0xee24d6c6, 0x1ffed36c, 0xd6e1dbd3, 0x273bde79,
+    0x7cbaf8b8, 0x8d60fd12, 0x447ff5ad, 0xb5a5f007, 0x0d30e292,
+    0xfceae738, 0x35f5ef87, 0xc42fea2d, 0x82f7a205, 0x732da7af,
+    0xba32af10, 0x4be8aaba, 0xf37db82f, 0x02a7bd85, 0xcbb8b53a,
+    0x3a62b090, 0x61e39651, 0x903993fb, 0x59269b44, 0xa8fc9eee,
+    0x10698c7b, 0xe1b389d1, 0x28ac816e, 0xd97684c4, 0xa51c113e,
+    0x54c61494, 0x9dd91c2b, 0x6c031981, 0xd4960b14, 0x254c0ebe,
+    0xec530601, 0x1d8903ab, 0x4608256a, 0xb7d220c0, 0x7ecd287f,
+    0x8f172dd5, 0x37823f40, 0xc6583aea, 0x0f473255, 0xfe9d37ff,
+    0xb8457fd7, 0x499f7a7d, 0x808072c2, 0x715a7768, 0xc9cf65fd,
+    0x38156057, 0xf10a68e8, 0x00d06d42, 0x5b514b83, 0xaa8b4e29,
+    0x63944696, 0x924e433c, 0x2adb51a9, 0xdb015403, 0x121e5cbc,
+    0xe3c45916},
+   {0x00000000, 0x0ee7e8d1, 0x1dcfd1a2, 0x13283973, 0x3b9fa344,
+    0x35784b95, 0x265072e6, 0x28b79a37, 0x773f4688, 0x79d8ae59,
+    0x6af0972a, 0x64177ffb, 0x4ca0e5cc, 0x42470d1d, 0x516f346e,
+    0x5f88dcbf, 0xee7e8d10, 0xe09965c1, 0xf3b15cb2, 0xfd56b463,
+    0xd5e12e54, 0xdb06c685, 0xc82efff6, 0xc6c91727, 0x9941cb98,
+    0x97a62349, 0x848e1a3a, 0x8a69f2eb, 0xa2de68dc, 0xac39800d,
+    0xbf11b97e, 0xb1f651af, 0x078c1c61, 0x096bf4b0, 0x1a43cdc3,
+    0x14a42512, 0x3c13bf25, 0x32f457f4, 0x21dc6e87, 0x2f3b8656,
+    0x70b35ae9, 0x7e54b238, 0x6d7c8b4b, 0x639b639a, 0x4b2cf9ad,
+    0x45cb117c, 0x56e3280f, 0x5804c0de, 0xe9f29171, 0xe71579a0,
+    0xf43d40d3, 0xfadaa802, 0xd26d3235, 0xdc8adae4, 0xcfa2e397,
+    0xc1450b46, 0x9ecdd7f9, 0x902a3f28, 0x8302065b, 0x8de5ee8a,
+    0xa55274bd, 0xabb59c6c, 0xb89da51f, 0xb67a4dce, 0x0f1838c2,
+    0x01ffd013, 0x12d7e960, 0x1c3001b1, 0x34879b86, 0x3a607357,
+    0x29484a24, 0x27afa2f5, 0x78277e4a, 0x76c0969b, 0x65e8afe8,
+    0x6b0f4739, 0x43b8dd0e, 0x4d5f35df, 0x5e770cac, 0x5090e47d,
+    0xe166b5d2, 0xef815d03, 0xfca96470, 0xf24e8ca1, 0xdaf91696,
+    0xd41efe47, 0xc736c734, 0xc9d12fe5, 0x9659f35a, 0x98be1b8b,
+    0x8b9622f8, 0x8571ca29, 0xadc6501e, 0xa321b8cf, 0xb00981bc,
+    0xbeee696d, 0x089424a3, 0x0673cc72, 0x155bf501, 0x1bbc1dd0,
+    0x330b87e7, 0x3dec6f36, 0x2ec45645, 0x2023be94, 0x7fab622b,
+    0x714c8afa, 0x6264b389, 0x6c835b58, 0x4434c16f, 0x4ad329be,
+    0x59fb10cd, 0x571cf81c, 0xe6eaa9b3, 0xe80d4162, 0xfb257811,
+    0xf5c290c0, 0xdd750af7, 0xd392e226, 0xc0badb55, 0xce5d3384,
+    0x91d5ef3b, 0x9f3207ea, 0x8c1a3e99, 0x82fdd648, 0xaa4a4c7f,
+    0xa4ada4ae, 0xb7859ddd, 0xb962750c, 0x1e307184, 0x10d79955,
+    0x03ffa026, 0x0d1848f7, 0x25afd2c0, 0x2b483a11, 0x38600362,
+    0x3687ebb3, 0x690f370c, 0x67e8dfdd, 0x74c0e6ae, 0x7a270e7f,
+    0x52909448, 0x5c777c99, 0x4f5f45ea, 0x41b8ad3b, 0xf04efc94,
+    0xfea91445, 0xed812d36, 0xe366c5e7, 0xcbd15fd0, 0xc536b701,
+    0xd61e8e72, 0xd8f966a3, 0x8771ba1c, 0x899652cd, 0x9abe6bbe,
+    0x9459836f, 0xbcee1958, 0xb209f189, 0xa121c8fa, 0xafc6202b,
+    0x19bc6de5, 0x175b8534, 0x0473bc47, 0x0a945496, 0x2223cea1,
+    0x2cc42670, 0x3fec1f03, 0x310bf7d2, 0x6e832b6d, 0x6064c3bc,
+    0x734cfacf, 0x7dab121e, 0x551c8829, 0x5bfb60f8, 0x48d3598b,
+    0x4634b15a, 0xf7c2e0f5, 0xf9250824, 0xea0d3157, 0xe4ead986,
+    0xcc5d43b1, 0xc2baab60, 0xd1929213, 0xdf757ac2, 0x80fda67d,
+    0x8e1a4eac, 0x9d3277df, 0x93d59f0e, 0xbb620539, 0xb585ede8,
+    0xa6add49b, 0xa84a3c4a, 0x11284946, 0x1fcfa197, 0x0ce798e4,
+    0x02007035, 0x2ab7ea02, 0x245002d3, 0x37783ba0, 0x399fd371,
+    0x66170fce, 0x68f0e71f, 0x7bd8de6c, 0x753f36bd, 0x5d88ac8a,
+    0x536f445b, 0x40477d28, 0x4ea095f9, 0xff56c456, 0xf1b12c87,
+    0xe29915f4, 0xec7efd25, 0xc4c96712, 0xca2e8fc3, 0xd906b6b0,
+    0xd7e15e61, 0x886982de, 0x868e6a0f, 0x95a6537c, 0x9b41bbad,
+    0xb3f6219a, 0xbd11c94b, 0xae39f038, 0xa0de18e9, 0x16a45527,
+    0x1843bdf6, 0x0b6b8485, 0x058c6c54, 0x2d3bf663, 0x23dc1eb2,
+    0x30f427c1, 0x3e13cf10, 0x619b13af, 0x6f7cfb7e, 0x7c54c20d,
+    0x72b32adc, 0x5a04b0eb, 0x54e3583a, 0x47cb6149, 0x492c8998,
+    0xf8dad837, 0xf63d30e6, 0xe5150995, 0xebf2e144, 0xc3457b73,
+    0xcda293a2, 0xde8aaad1, 0xd06d4200, 0x8fe59ebf, 0x8102766e,
+    0x922a4f1d, 0x9ccda7cc, 0xb47a3dfb, 0xba9dd52a, 0xa9b5ec59,
+    0xa7520488},
+   {0x00000000, 0x3c60e308, 0x78c1c610, 0x44a12518, 0xf1838c20,
+    0xcde36f28, 0x89424a30, 0xb522a938, 0x38761e01, 0x0416fd09,
+    0x40b7d811, 0x7cd73b19, 0xc9f59221, 0xf5957129, 0xb1345431,
+    0x8d54b739, 0x70ec3c02, 0x4c8cdf0a, 0x082dfa12, 0x344d191a,
+    0x816fb022, 0xbd0f532a, 0xf9ae7632, 0xc5ce953a, 0x489a2203,
+    0x74fac10b, 0x305be413, 0x0c3b071b, 0xb919ae23, 0x85794d2b,
+    0xc1d86833, 0xfdb88b3b, 0xe1d87804, 0xddb89b0c, 0x9919be14,
+    0xa5795d1c, 0x105bf424, 0x2c3b172c, 0x689a3234, 0x54fad13c,
+    0xd9ae6605, 0xe5ce850d, 0xa16fa015, 0x9d0f431d, 0x282dea25,
+    0x144d092d, 0x50ec2c35, 0x6c8ccf3d, 0x91344406, 0xad54a70e,
+    0xe9f58216, 0xd595611e, 0x60b7c826, 0x5cd72b2e, 0x18760e36,
+    0x2416ed3e, 0xa9425a07, 0x9522b90f, 0xd1839c17, 0xede37f1f,
+    0x58c1d627, 0x64a1352f, 0x20001037, 0x1c60f33f, 0x18c1f649,
+    0x24a11541, 0x60003059, 0x5c60d351, 0xe9427a69, 0xd5229961,
+    0x9183bc79, 0xade35f71, 0x20b7e848, 0x1cd70b40, 0x58762e58,
+    0x6416cd50, 0xd1346468, 0xed548760, 0xa9f5a278, 0x95954170,
+    0x682dca4b, 0x544d2943, 0x10ec0c5b, 0x2c8cef53, 0x99ae466b,
+    0xa5cea563, 0xe16f807b, 0xdd0f6373, 0x505bd44a, 0x6c3b3742,
+    0x289a125a, 0x14faf152, 0xa1d8586a, 0x9db8bb62, 0xd9199e7a,
+    0xe5797d72, 0xf9198e4d, 0xc5796d45, 0x81d8485d, 0xbdb8ab55,
+    0x089a026d, 0x34fae165, 0x705bc47d, 0x4c3b2775, 0xc16f904c,
+    0xfd0f7344, 0xb9ae565c, 0x85ceb554, 0x30ec1c6c, 0x0c8cff64,
+    0x482dda7c, 0x744d3974, 0x89f5b24f, 0xb5955147, 0xf134745f,
+    0xcd549757, 0x78763e6f, 0x4416dd67, 0x00b7f87f, 0x3cd71b77,
+    0xb183ac4e, 0x8de34f46, 0xc9426a5e, 0xf5228956, 0x4000206e,
+    0x7c60c366, 0x38c1e67e, 0x04a10576, 0x3183ec92, 0x0de30f9a,
+    0x49422a82, 0x7522c98a, 0xc00060b2, 0xfc6083ba, 0xb8c1a6a2,
+    0x84a145aa, 0x09f5f293, 0x3595119b, 0x71343483, 0x4d54d78b,
+    0xf8767eb3, 0xc4169dbb, 0x80b7b8a3, 0xbcd75bab, 0x416fd090,
+    0x7d0f3398, 0x39ae1680, 0x05cef588, 0xb0ec5cb0, 0x8c8cbfb8,
+    0xc82d9aa0, 0xf44d79a8, 0x7919ce91, 0x45792d99, 0x01d80881,
+    0x3db8eb89, 0x889a42b1, 0xb4faa1b9, 0xf05b84a1, 0xcc3b67a9,
+    0xd05b9496, 0xec3b779e, 0xa89a5286, 0x94fab18e, 0x21d818b6,
+    0x1db8fbbe, 0x5919dea6, 0x65793dae, 0xe82d8a97, 0xd44d699f,
+    0x90ec4c87, 0xac8caf8f, 0x19ae06b7, 0x25cee5bf, 0x616fc0a7,
+    0x5d0f23af, 0xa0b7a894, 0x9cd74b9c, 0xd8766e84, 0xe4168d8c,
+    0x513424b4, 0x6d54c7bc, 0x29f5e2a4, 0x159501ac, 0x98c1b695,
+    0xa4a1559d, 0xe0007085, 0xdc60938d, 0x69423ab5, 0x5522d9bd,
+    0x1183fca5, 0x2de31fad, 0x29421adb, 0x1522f9d3, 0x5183dccb,
+    0x6de33fc3, 0xd8c196fb, 0xe4a175f3, 0xa00050eb, 0x9c60b3e3,
+    0x113404da, 0x2d54e7d2, 0x69f5c2ca, 0x559521c2, 0xe0b788fa,
+    0xdcd76bf2, 0x98764eea, 0xa416ade2, 0x59ae26d9, 0x65cec5d1,
+    0x216fe0c9, 0x1d0f03c1, 0xa82daaf9, 0x944d49f1, 0xd0ec6ce9,
+    0xec8c8fe1, 0x61d838d8, 0x5db8dbd0, 0x1919fec8, 0x25791dc0,
+    0x905bb4f8, 0xac3b57f0, 0xe89a72e8, 0xd4fa91e0, 0xc89a62df,
+    0xf4fa81d7, 0xb05ba4cf, 0x8c3b47c7, 0x3919eeff, 0x05790df7,
+    0x41d828ef, 0x7db8cbe7, 0xf0ec7cde, 0xcc8c9fd6, 0x882dbace,
+    0xb44d59c6, 0x016ff0fe, 0x3d0f13f6, 0x79ae36ee, 0x45ced5e6,
+    0xb8765edd, 0x8416bdd5, 0xc0b798cd, 0xfcd77bc5, 0x49f5d2fd,
+    0x759531f5, 0x313414ed, 0x0d54f7e5, 0x800040dc, 0xbc60a3d4,
+    0xf8c186cc, 0xc4a165c4, 0x7183ccfc, 0x4de32ff4, 0x09420aec,
+    0x3522e9e4},
+   {0x00000000, 0x6307d924, 0xc60fb248, 0xa5086b6c, 0x576e62d1,
+    0x3469bbf5, 0x9161d099, 0xf26609bd, 0xaedcc5a2, 0xcddb1c86,
+    0x68d377ea, 0x0bd4aece, 0xf9b2a773, 0x9ab57e57, 0x3fbd153b,
+    0x5cbacc1f, 0x86c88d05, 0xe5cf5421, 0x40c73f4d, 0x23c0e669,
+    0xd1a6efd4, 0xb2a136f0, 0x17a95d9c, 0x74ae84b8, 0x281448a7,
+    0x4b139183, 0xee1bfaef, 0x8d1c23cb, 0x7f7a2a76, 0x1c7df352,
+    0xb975983e, 0xda72411a, 0xd6e01c4b, 0xb5e7c56f, 0x10efae03,
+    0x73e87727, 0x818e7e9a, 0xe289a7be, 0x4781ccd2, 0x248615f6,
+    0x783cd9e9, 0x1b3b00cd, 0xbe336ba1, 0xdd34b285, 0x2f52bb38,
+    0x4c55621c, 0xe95d0970, 0x8a5ad054, 0x5028914e, 0x332f486a,
+    0x96272306, 0xf520fa22, 0x0746f39f, 0x64412abb, 0xc14941d7,
+    0xa24e98f3, 0xfef454ec, 0x9df38dc8, 0x38fbe6a4, 0x5bfc3f80,
+    0xa99a363d, 0xca9def19, 0x6f958475, 0x0c925d51, 0x76b13ed7,
+    0x15b6e7f3, 0xb0be8c9f, 0xd3b955bb, 0x21df5c06, 0x42d88522,
+    0xe7d0ee4e, 0x84d7376a, 0xd86dfb75, 0xbb6a2251, 0x1e62493d,
+    0x7d659019, 0x8f0399a4, 0xec044080, 0x490c2bec, 0x2a0bf2c8,
+    0xf079b3d2, 0x937e6af6, 0x3676019a, 0x5571d8be, 0xa717d103,
+    0xc4100827, 0x6118634b, 0x021fba6f, 0x5ea57670, 0x3da2af54,
+    0x98aac438, 0xfbad1d1c, 0x09cb14a1, 0x6acccd85, 0xcfc4a6e9,
+    0xacc37fcd, 0xa051229c, 0xc356fbb8, 0x665e90d4, 0x055949f0,
+    0xf73f404d, 0x94389969, 0x3130f205, 0x52372b21, 0x0e8de73e,
+    0x6d8a3e1a, 0xc8825576, 0xab858c52, 0x59e385ef, 0x3ae45ccb,
+    0x9fec37a7, 0xfcebee83, 0x2699af99, 0x459e76bd, 0xe0961dd1,
+    0x8391c4f5, 0x71f7cd48, 0x12f0146c, 0xb7f87f00, 0xd4ffa624,
+    0x88456a3b, 0xeb42b31f, 0x4e4ad873, 0x2d4d0157, 0xdf2b08ea,
+    0xbc2cd1ce, 0x1924baa2, 0x7a236386, 0xed627dae, 0x8e65a48a,
+    0x2b6dcfe6, 0x486a16c2, 0xba0c1f7f, 0xd90bc65b, 0x7c03ad37,
+    0x1f047413, 0x43beb80c, 0x20b96128, 0x85b10a44, 0xe6b6d360,
+    0x14d0dadd, 0x77d703f9, 0xd2df6895, 0xb1d8b1b1, 0x6baaf0ab,
+    0x08ad298f, 0xada542e3, 0xcea29bc7, 0x3cc4927a, 0x5fc34b5e,
+    0xfacb2032, 0x99ccf916, 0xc5763509, 0xa671ec2d, 0x03798741,
+    0x607e5e65, 0x921857d8, 0xf11f8efc, 0x5417e590, 0x37103cb4,
+    0x3b8261e5, 0x5885b8c1, 0xfd8dd3ad, 0x9e8a0a89, 0x6cec0334,
+    0x0febda10, 0xaae3b17c, 0xc9e46858, 0x955ea447, 0xf6597d63,
+    0x5351160f, 0x3056cf2b, 0xc230c696, 0xa1371fb2, 0x043f74de,
+    0x6738adfa, 0xbd4aece0, 0xde4d35c4, 0x7b455ea8, 0x1842878c,
+    0xea248e31, 0x89235715, 0x2c2b3c79, 0x4f2ce55d, 0x13962942,
+    0x7091f066, 0xd5999b0a, 0xb69e422e, 0x44f84b93, 0x27ff92b7,
+    0x82f7f9db, 0xe1f020ff, 0x9bd34379, 0xf8d49a5d, 0x5ddcf131,
+    0x3edb2815, 0xccbd21a8, 0xafbaf88c, 0x0ab293e0, 0x69b54ac4,
+    0x350f86db, 0x56085fff, 0xf3003493, 0x9007edb7, 0x6261e40a,
+    0x01663d2e, 0xa46e5642, 0xc7698f66, 0x1d1bce7c, 0x7e1c1758,
+    0xdb147c34, 0xb813a510, 0x4a75acad, 0x29727589, 0x8c7a1ee5,
+    0xef7dc7c1, 0xb3c70bde, 0xd0c0d2fa, 0x75c8b996, 0x16cf60b2,
+    0xe4a9690f, 0x87aeb02b, 0x22a6db47, 0x41a10263, 0x4d335f32,
+    0x2e348616, 0x8b3ced7a, 0xe83b345e, 0x1a5d3de3, 0x795ae4c7,
+    0xdc528fab, 0xbf55568f, 0xe3ef9a90, 0x80e843b4, 0x25e028d8,
+    0x46e7f1fc, 0xb481f841, 0xd7862165, 0x728e4a09, 0x1189932d,
+    0xcbfbd237, 0xa8fc0b13, 0x0df4607f, 0x6ef3b95b, 0x9c95b0e6,
+    0xff9269c2, 0x5a9a02ae, 0x399ddb8a, 0x65271795, 0x0620ceb1,
+    0xa328a5dd, 0xc02f7cf9, 0x32497544, 0x514eac60, 0xf446c70c,
+    0x97411e28},
+   {0x00000000, 0x01b5fd1d, 0x036bfa3a, 0x02de0727, 0x06d7f474,
+    0x07620969, 0x05bc0e4e, 0x0409f353, 0x0dafe8e8, 0x0c1a15f5,
+    0x0ec412d2, 0x0f71efcf, 0x0b781c9c, 0x0acde181, 0x0813e6a6,
+    0x09a61bbb, 0x1b5fd1d0, 0x1aea2ccd, 0x18342bea, 0x1981d6f7,
+    0x1d8825a4, 0x1c3dd8b9, 0x1ee3df9e, 0x1f562283, 0x16f03938,
+    0x1745c425, 0x159bc302, 0x142e3e1f, 0x1027cd4c, 0x11923051,
+    0x134c3776, 0x12f9ca6b, 0x36bfa3a0, 0x370a5ebd, 0x35d4599a,
+    0x3461a487, 0x306857d4, 0x31ddaac9, 0x3303adee, 0x32b650f3,
+    0x3b104b48, 0x3aa5b655, 0x387bb172, 0x39ce4c6f, 0x3dc7bf3c,
+    0x3c724221, 0x3eac4506, 0x3f19b81b, 0x2de07270, 0x2c558f6d,
+    0x2e8b884a, 0x2f3e7557, 0x2b378604, 0x2a827b19, 0x285c7c3e,
+    0x29e98123, 0x204f9a98, 0x21fa6785, 0x232460a2, 0x22919dbf,
+    0x26986eec, 0x272d93f1, 0x25f394d6, 0x244669cb, 0x6d7f4740,
+    0x6ccaba5d, 0x6e14bd7a, 0x6fa14067, 0x6ba8b334, 0x6a1d4e29,
+    0x68c3490e, 0x6976b413, 0x60d0afa8, 0x616552b5, 0x63bb5592,
+    0x620ea88f, 0x66075bdc, 0x67b2a6c1, 0x656ca1e6, 0x64d95cfb,
+    0x76209690, 0x77956b8d, 0x754b6caa, 0x74fe91b7, 0x70f762e4,
+    0x71429ff9, 0x739c98de, 0x722965c3, 0x7b8f7e78, 0x7a3a8365,
+    0x78e48442, 0x7951795f, 0x7d588a0c, 0x7ced7711, 0x7e337036,
+    0x7f868d2b, 0x5bc0e4e0, 0x5a7519fd, 0x58ab1eda, 0x591ee3c7,
+    0x5d171094, 0x5ca2ed89, 0x5e7ceaae, 0x5fc917b3, 0x566f0c08,
+    0x57daf115, 0x5504f632, 0x54b10b2f, 0x50b8f87c, 0x510d0561,
+    0x53d30246, 0x5266ff5b, 0x409f3530, 0x412ac82d, 0x43f4cf0a,
+    0x42413217, 0x4648c144, 0x47fd3c59, 0x45233b7e, 0x4496c663,
+    0x4d30ddd8, 0x4c8520c5, 0x4e5b27e2, 0x4feedaff, 0x4be729ac,
+    0x4a52d4b1, 0x488cd396, 0x49392e8b, 0xdafe8e80, 0xdb4b739d,
+    0xd99574ba, 0xd82089a7, 0xdc297af4, 0xdd9c87e9, 0xdf4280ce,
+    0xdef77dd3, 0xd7516668, 0xd6e49b75, 0xd43a9c52, 0xd58f614f,
+    0xd186921c, 0xd0336f01, 0xd2ed6826, 0xd358953b, 0xc1a15f50,
+    0xc014a24d, 0xc2caa56a, 0xc37f5877, 0xc776ab24, 0xc6c35639,
+    0xc41d511e, 0xc5a8ac03, 0xcc0eb7b8, 0xcdbb4aa5, 0xcf654d82,
+    0xced0b09f, 0xcad943cc, 0xcb6cbed1, 0xc9b2b9f6, 0xc80744eb,
+    0xec412d20, 0xedf4d03d, 0xef2ad71a, 0xee9f2a07, 0xea96d954,
+    0xeb232449, 0xe9fd236e, 0xe848de73, 0xe1eec5c8, 0xe05b38d5,
+    0xe2853ff2, 0xe330c2ef, 0xe73931bc, 0xe68ccca1, 0xe452cb86,
+    0xe5e7369b, 0xf71efcf0, 0xf6ab01ed, 0xf47506ca, 0xf5c0fbd7,
+    0xf1c90884, 0xf07cf599, 0xf2a2f2be, 0xf3170fa3, 0xfab11418,
+    0xfb04e905, 0xf9daee22, 0xf86f133f, 0xfc66e06c, 0xfdd31d71,
+    0xff0d1a56, 0xfeb8e74b, 0xb781c9c0, 0xb63434dd, 0xb4ea33fa,
+    0xb55fcee7, 0xb1563db4, 0xb0e3c0a9, 0xb23dc78e, 0xb3883a93,
+    0xba2e2128, 0xbb9bdc35, 0xb945db12, 0xb8f0260f, 0xbcf9d55c,
+    0xbd4c2841, 0xbf922f66, 0xbe27d27b, 0xacde1810, 0xad6be50d,
+    0xafb5e22a, 0xae001f37, 0xaa09ec64, 0xabbc1179, 0xa962165e,
+    0xa8d7eb43, 0xa171f0f8, 0xa0c40de5, 0xa21a0ac2, 0xa3aff7df,
+    0xa7a6048c, 0xa613f991, 0xa4cdfeb6, 0xa57803ab, 0x813e6a60,
+    0x808b977d, 0x8255905a, 0x83e06d47, 0x87e99e14, 0x865c6309,
+    0x8482642e, 0x85379933, 0x8c918288, 0x8d247f95, 0x8ffa78b2,
+    0x8e4f85af, 0x8a4676fc, 0x8bf38be1, 0x892d8cc6, 0x889871db,
+    0x9a61bbb0, 0x9bd446ad, 0x990a418a, 0x98bfbc97, 0x9cb64fc4,
+    0x9d03b2d9, 0x9fddb5fe, 0x9e6848e3, 0x97ce5358, 0x967bae45,
+    0x94a5a962, 0x9510547f, 0x9119a72c, 0x90ac5a31, 0x92725d16,
+    0x93c7a00b},
+   {0x00000000, 0x6e8c1b41, 0xdd183682, 0xb3942dc3, 0x61416b45,
+    0x0fcd7004, 0xbc595dc7, 0xd2d54686, 0xc282d68a, 0xac0ecdcb,
+    0x1f9ae008, 0x7116fb49, 0xa3c3bdcf, 0xcd4fa68e, 0x7edb8b4d,
+    0x1057900c, 0x5e74ab55, 0x30f8b014, 0x836c9dd7, 0xede08696,
+    0x3f35c010, 0x51b9db51, 0xe22df692, 0x8ca1edd3, 0x9cf67ddf,
+    0xf27a669e, 0x41ee4b5d, 0x2f62501c, 0xfdb7169a, 0x933b0ddb,
+    0x20af2018, 0x4e233b59, 0xbce956aa, 0xd2654deb, 0x61f16028,
+    0x0f7d7b69, 0xdda83def, 0xb32426ae, 0x00b00b6d, 0x6e3c102c,
+    0x7e6b8020, 0x10e79b61, 0xa373b6a2, 0xcdffade3, 0x1f2aeb65,
+    0x71a6f024, 0xc232dde7, 0xacbec6a6, 0xe29dfdff, 0x8c11e6be,
+    0x3f85cb7d, 0x5109d03c, 0x83dc96ba, 0xed508dfb, 0x5ec4a038,
+    0x3048bb79, 0x201f2b75, 0x4e933034, 0xfd071df7, 0x938b06b6,
+    0x415e4030, 0x2fd25b71, 0x9c4676b2, 0xf2ca6df3, 0xa2a3ab15,
+    0xcc2fb054, 0x7fbb9d97, 0x113786d6, 0xc3e2c050, 0xad6edb11,
+    0x1efaf6d2, 0x7076ed93, 0x60217d9f, 0x0ead66de, 0xbd394b1d,
+    0xd3b5505c, 0x016016da, 0x6fec0d9b, 0xdc782058, 0xb2f43b19,
+    0xfcd70040, 0x925b1b01, 0x21cf36c2, 0x4f432d83, 0x9d966b05,
+    0xf31a7044, 0x408e5d87, 0x2e0246c6, 0x3e55d6ca, 0x50d9cd8b,
+    0xe34de048, 0x8dc1fb09, 0x5f14bd8f, 0x3198a6ce, 0x820c8b0d,
+    0xec80904c, 0x1e4afdbf, 0x70c6e6fe, 0xc352cb3d, 0xadded07c,
+    0x7f0b96fa, 0x11878dbb, 0xa213a078, 0xcc9fbb39, 0xdcc82b35,
+    0xb2443074, 0x01d01db7, 0x6f5c06f6, 0xbd894070, 0xd3055b31,
+    0x609176f2, 0x0e1d6db3, 0x403e56ea, 0x2eb24dab, 0x9d266068,
+    0xf3aa7b29, 0x217f3daf, 0x4ff326ee, 0xfc670b2d, 0x92eb106c,
+    0x82bc8060, 0xec309b21, 0x5fa4b6e2, 0x3128ada3, 0xe3fdeb25,
+    0x8d71f064, 0x3ee5dda7, 0x5069c6e6, 0x9e36506b, 0xf0ba4b2a,
+    0x432e66e9, 0x2da27da8, 0xff773b2e, 0x91fb206f, 0x226f0dac,
+    0x4ce316ed, 0x5cb486e1, 0x32389da0, 0x81acb063, 0xef20ab22,
+    0x3df5eda4, 0x5379f6e5, 0xe0eddb26, 0x8e61c067, 0xc042fb3e,
+    0xaecee07f, 0x1d5acdbc, 0x73d6d6fd, 0xa103907b, 0xcf8f8b3a,
+    0x7c1ba6f9, 0x1297bdb8, 0x02c02db4, 0x6c4c36f5, 0xdfd81b36,
+    0xb1540077, 0x638146f1, 0x0d0d5db0, 0xbe997073, 0xd0156b32,
+    0x22df06c1, 0x4c531d80, 0xffc73043, 0x914b2b02, 0x439e6d84,
+    0x2d1276c5, 0x9e865b06, 0xf00a4047, 0xe05dd04b, 0x8ed1cb0a,
+    0x3d45e6c9, 0x53c9fd88, 0x811cbb0e, 0xef90a04f, 0x5c048d8c,
+    0x328896cd, 0x7cabad94, 0x1227b6d5, 0xa1b39b16, 0xcf3f8057,
+    0x1deac6d1, 0x7366dd90, 0xc0f2f053, 0xae7eeb12, 0xbe297b1e,
+    0xd0a5605f, 0x63314d9c, 0x0dbd56dd, 0xdf68105b, 0xb1e40b1a,
+    0x027026d9, 0x6cfc3d98, 0x3c95fb7e, 0x5219e03f, 0xe18dcdfc,
+    0x8f01d6bd, 0x5dd4903b, 0x33588b7a, 0x80cca6b9, 0xee40bdf8,
+    0xfe172df4, 0x909b36b5, 0x230f1b76, 0x4d830037, 0x9f5646b1,
+    0xf1da5df0, 0x424e7033, 0x2cc26b72, 0x62e1502b, 0x0c6d4b6a,
+    0xbff966a9, 0xd1757de8, 0x03a03b6e, 0x6d2c202f, 0xdeb80dec,
+    0xb03416ad, 0xa06386a1, 0xceef9de0, 0x7d7bb023, 0x13f7ab62,
+    0xc122ede4, 0xafaef6a5, 0x1c3adb66, 0x72b6c027, 0x807cadd4,
+    0xeef0b695, 0x5d649b56, 0x33e88017, 0xe13dc691, 0x8fb1ddd0,
+    0x3c25f013, 0x52a9eb52, 0x42fe7b5e, 0x2c72601f, 0x9fe64ddc,
+    0xf16a569d, 0x23bf101b, 0x4d330b5a, 0xfea72699, 0x902b3dd8,
+    0xde080681, 0xb0841dc0, 0x03103003, 0x6d9c2b42, 0xbf496dc4,
+    0xd1c57685, 0x62515b46, 0x0cdd4007, 0x1c8ad00b, 0x7206cb4a,
+    0xc192e689, 0xaf1efdc8, 0x7dcbbb4e, 0x1347a00f, 0xa0d38dcc,
+    0xce5f968d},
+   {0x00000000, 0xe71da697, 0x154a4b6f, 0xf257edf8, 0x2a9496de,
+    0xcd893049, 0x3fdeddb1, 0xd8c37b26, 0x55292dbc, 0xb2348b2b,
+    0x406366d3, 0xa77ec044, 0x7fbdbb62, 0x98a01df5, 0x6af7f00d,
+    0x8dea569a, 0xaa525b78, 0x4d4ffdef, 0xbf181017, 0x5805b680,
+    0x80c6cda6, 0x67db6b31, 0x958c86c9, 0x7291205e, 0xff7b76c4,
+    0x1866d053, 0xea313dab, 0x0d2c9b3c, 0xd5efe01a, 0x32f2468d,
+    0xc0a5ab75, 0x27b80de2, 0x8fd5b0b1, 0x68c81626, 0x9a9ffbde,
+    0x7d825d49, 0xa541266f, 0x425c80f8, 0xb00b6d00, 0x5716cb97,
+    0xdafc9d0d, 0x3de13b9a, 0xcfb6d662, 0x28ab70f5, 0xf0680bd3,
+    0x1775ad44, 0xe52240bc, 0x023fe62b, 0x2587ebc9, 0xc29a4d5e,
+    0x30cda0a6, 0xd7d00631, 0x0f137d17, 0xe80edb80, 0x1a593678,
+    0xfd4490ef, 0x70aec675, 0x97b360e2, 0x65e48d1a, 0x82f92b8d,
+    0x5a3a50ab, 0xbd27f63c, 0x4f701bc4, 0xa86dbd53, 0xc4da6723,
+    0x23c7c1b4, 0xd1902c4c, 0x368d8adb, 0xee4ef1fd, 0x0953576a,
+    0xfb04ba92, 0x1c191c05, 0x91f34a9f, 0x76eeec08, 0x84b901f0,
+    0x63a4a767, 0xbb67dc41, 0x5c7a7ad6, 0xae2d972e, 0x493031b9,
+    0x6e883c5b, 0x89959acc, 0x7bc27734, 0x9cdfd1a3, 0x441caa85,
+    0xa3010c12, 0x5156e1ea, 0xb64b477d, 0x3ba111e7, 0xdcbcb770,
+    0x2eeb5a88, 0xc9f6fc1f, 0x11358739, 0xf62821ae, 0x047fcc56,
+    0xe3626ac1, 0x4b0fd792, 0xac127105, 0x5e459cfd, 0xb9583a6a,
+    0x619b414c, 0x8686e7db, 0x74d10a23, 0x93ccacb4, 0x1e26fa2e,
+    0xf93b5cb9, 0x0b6cb141, 0xec7117d6, 0x34b26cf0, 0xd3afca67,
+    0x21f8279f, 0xc6e58108, 0xe15d8cea, 0x06402a7d, 0xf417c785,
+    0x130a6112, 0xcbc91a34, 0x2cd4bca3, 0xde83515b, 0x399ef7cc,
+    0xb474a156, 0x536907c1, 0xa13eea39, 0x46234cae, 0x9ee03788,
+    0x79fd911f, 0x8baa7ce7, 0x6cb7da70, 0x52c5c807, 0xb5d86e90,
+    0x478f8368, 0xa09225ff, 0x78515ed9, 0x9f4cf84e, 0x6d1b15b6,
+    0x8a06b321, 0x07ece5bb, 0xe0f1432c, 0x12a6aed4, 0xf5bb0843,
+    0x2d787365, 0xca65d5f2, 0x3832380a, 0xdf2f9e9d, 0xf897937f,
+    0x1f8a35e8, 0xedddd810, 0x0ac07e87, 0xd20305a1, 0x351ea336,
+    0xc7494ece, 0x2054e859, 0xadbebec3, 0x4aa31854, 0xb8f4f5ac,
+    0x5fe9533b, 0x872a281d, 0x60378e8a, 0x92606372, 0x757dc5e5,
+    0xdd1078b6, 0x3a0dde21, 0xc85a33d9, 0x2f47954e, 0xf784ee68,
+    0x109948ff, 0xe2cea507, 0x05d30390, 0x8839550a, 0x6f24f39d,
+    0x9d731e65, 0x7a6eb8f2, 0xa2adc3d4, 0x45b06543, 0xb7e788bb,
+    0x50fa2e2c, 0x774223ce, 0x905f8559, 0x620868a1, 0x8515ce36,
+    0x5dd6b510, 0xbacb1387, 0x489cfe7f, 0xaf8158e8, 0x226b0e72,
+    0xc576a8e5, 0x3721451d, 0xd03ce38a, 0x08ff98ac, 0xefe23e3b,
+    0x1db5d3c3, 0xfaa87554, 0x961faf24, 0x710209b3, 0x8355e44b,
+    0x644842dc, 0xbc8b39fa, 0x5b969f6d, 0xa9c17295, 0x4edcd402,
+    0xc3368298, 0x242b240f, 0xd67cc9f7, 0x31616f60, 0xe9a21446,
+    0x0ebfb2d1, 0xfce85f29, 0x1bf5f9be, 0x3c4df45c, 0xdb5052cb,
+    0x2907bf33, 0xce1a19a4, 0x16d96282, 0xf1c4c415, 0x039329ed,
+    0xe48e8f7a, 0x6964d9e0, 0x8e797f77, 0x7c2e928f, 0x9b333418,
+    0x43f04f3e, 0xa4ede9a9, 0x56ba0451, 0xb1a7a2c6, 0x19ca1f95,
+    0xfed7b902, 0x0c8054fa, 0xeb9df26d, 0x335e894b, 0xd4432fdc,
+    0x2614c224, 0xc10964b3, 0x4ce33229, 0xabfe94be, 0x59a97946,
+    0xbeb4dfd1, 0x6677a4f7, 0x816a0260, 0x733def98, 0x9420490f,
+    0xb39844ed, 0x5485e27a, 0xa6d20f82, 0x41cfa915, 0x990cd233,
+    0x7e1174a4, 0x8c46995c, 0x6b5b3fcb, 0xe6b16951, 0x01accfc6,
+    0xf3fb223e, 0x14e684a9, 0xcc25ff8f, 0x2b385918, 0xd96fb4e0,
+    0x3e721277},
+   {0x00000000, 0xa58b900e, 0x9066265d, 0x35edb653, 0xfbbd4afb,
+    0x5e36daf5, 0x6bdb6ca6, 0xce50fca8, 0x2c0b93b7, 0x898003b9,
+    0xbc6db5ea, 0x19e625e4, 0xd7b6d94c, 0x723d4942, 0x47d0ff11,
+    0xe25b6f1f, 0x5817276e, 0xfd9cb760, 0xc8710133, 0x6dfa913d,
+    0xa3aa6d95, 0x0621fd9b, 0x33cc4bc8, 0x9647dbc6, 0x741cb4d9,
+    0xd19724d7, 0xe47a9284, 0x41f1028a, 0x8fa1fe22, 0x2a2a6e2c,
+    0x1fc7d87f, 0xba4c4871, 0xb02e4edc, 0x15a5ded2, 0x20486881,
+    0x85c3f88f, 0x4b930427, 0xee189429, 0xdbf5227a, 0x7e7eb274,
+    0x9c25dd6b, 0x39ae4d65, 0x0c43fb36, 0xa9c86b38, 0x67989790,
+    0xc213079e, 0xf7feb1cd, 0x527521c3, 0xe83969b2, 0x4db2f9bc,
+    0x785f4fef, 0xddd4dfe1, 0x13842349, 0xb60fb347, 0x83e20514,
+    0x2669951a, 0xc432fa05, 0x61b96a0b, 0x5454dc58, 0xf1df4c56,
+    0x3f8fb0fe, 0x9a0420f0, 0xafe996a3, 0x0a6206ad, 0xbb2d9bf9,
+    0x1ea60bf7, 0x2b4bbda4, 0x8ec02daa, 0x4090d102, 0xe51b410c,
+    0xd0f6f75f, 0x757d6751, 0x9726084e, 0x32ad9840, 0x07402e13,
+    0xa2cbbe1d, 0x6c9b42b5, 0xc910d2bb, 0xfcfd64e8, 0x5976f4e6,
+    0xe33abc97, 0x46b12c99, 0x735c9aca, 0xd6d70ac4, 0x1887f66c,
+    0xbd0c6662, 0x88e1d031, 0x2d6a403f, 0xcf312f20, 0x6ababf2e,
+    0x5f57097d, 0xfadc9973, 0x348c65db, 0x9107f5d5, 0xa4ea4386,
+    0x0161d388, 0x0b03d525, 0xae88452b, 0x9b65f378, 0x3eee6376,
+    0xf0be9fde, 0x55350fd0, 0x60d8b983, 0xc553298d, 0x27084692,
+    0x8283d69c, 0xb76e60cf, 0x12e5f0c1, 0xdcb50c69, 0x793e9c67,
+    0x4cd32a34, 0xe958ba3a, 0x5314f24b, 0xf69f6245, 0xc372d416,
+    0x66f94418, 0xa8a9b8b0, 0x0d2228be, 0x38cf9eed, 0x9d440ee3,
+    0x7f1f61fc, 0xda94f1f2, 0xef7947a1, 0x4af2d7af, 0x84a22b07,
+    0x2129bb09, 0x14c40d5a, 0xb14f9d54, 0xad2a31b3, 0x08a1a1bd,
+    0x3d4c17ee, 0x98c787e0, 0x56977b48, 0xf31ceb46, 0xc6f15d15,
+    0x637acd1b, 0x8121a204, 0x24aa320a, 0x11478459, 0xb4cc1457,
+    0x7a9ce8ff, 0xdf1778f1, 0xeafacea2, 0x4f715eac, 0xf53d16dd,
+    0x50b686d3, 0x655b3080, 0xc0d0a08e, 0x0e805c26, 0xab0bcc28,
+    0x9ee67a7b, 0x3b6dea75, 0xd936856a, 0x7cbd1564, 0x4950a337,
+    0xecdb3339, 0x228bcf91, 0x87005f9f, 0xb2ede9cc, 0x176679c2,
+    0x1d047f6f, 0xb88fef61, 0x8d625932, 0x28e9c93c, 0xe6b93594,
+    0x4332a59a, 0x76df13c9, 0xd35483c7, 0x310fecd8, 0x94847cd6,
+    0xa169ca85, 0x04e25a8b, 0xcab2a623, 0x6f39362d, 0x5ad4807e,
+    0xff5f1070, 0x45135801, 0xe098c80f, 0xd5757e5c, 0x70feee52,
+    0xbeae12fa, 0x1b2582f4, 0x2ec834a7, 0x8b43a4a9, 0x6918cbb6,
+    0xcc935bb8, 0xf97eedeb, 0x5cf57de5, 0x92a5814d, 0x372e1143,
+    0x02c3a710, 0xa748371e, 0x1607aa4a, 0xb38c3a44, 0x86618c17,
+    0x23ea1c19, 0xedbae0b1, 0x483170bf, 0x7ddcc6ec, 0xd85756e2,
+    0x3a0c39fd, 0x9f87a9f3, 0xaa6a1fa0, 0x0fe18fae, 0xc1b17306,
+    0x643ae308, 0x51d7555b, 0xf45cc555, 0x4e108d24, 0xeb9b1d2a,
+    0xde76ab79, 0x7bfd3b77, 0xb5adc7df, 0x102657d1, 0x25cbe182,
+    0x8040718c, 0x621b1e93, 0xc7908e9d, 0xf27d38ce, 0x57f6a8c0,
+    0x99a65468, 0x3c2dc466, 0x09c07235, 0xac4be23b, 0xa629e496,
+    0x03a27498, 0x364fc2cb, 0x93c452c5, 0x5d94ae6d, 0xf81f3e63,
+    0xcdf28830, 0x6879183e, 0x8a227721, 0x2fa9e72f, 0x1a44517c,
+    0xbfcfc172, 0x719f3dda, 0xd414add4, 0xe1f91b87, 0x44728b89,
+    0xfe3ec3f8, 0x5bb553f6, 0x6e58e5a5, 0xcbd375ab, 0x05838903,
+    0xa008190d, 0x95e5af5e, 0x306e3f50, 0xd235504f, 0x77bec041,
+    0x42537612, 0xe7d8e61c, 0x29881ab4, 0x8c038aba, 0xb9ee3ce9,
+    0x1c65ace7}};
+
+local const z_word_t FAR crc_braid_big_table[][256] = {
+   {0x0000000000000000, 0x0e908ba500000000, 0x5d26669000000000,
+    0x53b6ed3500000000, 0xfb4abdfb00000000, 0xf5da365e00000000,
+    0xa66cdb6b00000000, 0xa8fc50ce00000000, 0xb7930b2c00000000,
+    0xb903808900000000, 0xeab56dbc00000000, 0xe425e61900000000,
+    0x4cd9b6d700000000, 0x42493d7200000000, 0x11ffd04700000000,
+    0x1f6f5be200000000, 0x6e27175800000000, 0x60b79cfd00000000,
+    0x330171c800000000, 0x3d91fa6d00000000, 0x956daaa300000000,
+    0x9bfd210600000000, 0xc84bcc3300000000, 0xc6db479600000000,
+    0xd9b41c7400000000, 0xd72497d100000000, 0x84927ae400000000,
+    0x8a02f14100000000, 0x22fea18f00000000, 0x2c6e2a2a00000000,
+    0x7fd8c71f00000000, 0x71484cba00000000, 0xdc4e2eb000000000,
+    0xd2dea51500000000, 0x8168482000000000, 0x8ff8c38500000000,
+    0x2704934b00000000, 0x299418ee00000000, 0x7a22f5db00000000,
+    0x74b27e7e00000000, 0x6bdd259c00000000, 0x654dae3900000000,
+    0x36fb430c00000000, 0x386bc8a900000000, 0x9097986700000000,
+    0x9e0713c200000000, 0xcdb1fef700000000, 0xc321755200000000,
+    0xb26939e800000000, 0xbcf9b24d00000000, 0xef4f5f7800000000,
+    0xe1dfd4dd00000000, 0x4923841300000000, 0x47b30fb600000000,
+    0x1405e28300000000, 0x1a95692600000000, 0x05fa32c400000000,
+    0x0b6ab96100000000, 0x58dc545400000000, 0x564cdff100000000,
+    0xfeb08f3f00000000, 0xf020049a00000000, 0xa396e9af00000000,
+    0xad06620a00000000, 0xf99b2dbb00000000, 0xf70ba61e00000000,
+    0xa4bd4b2b00000000, 0xaa2dc08e00000000, 0x02d1904000000000,
+    0x0c411be500000000, 0x5ff7f6d000000000, 0x51677d7500000000,
+    0x4e08269700000000, 0x4098ad3200000000, 0x132e400700000000,
+    0x1dbecba200000000, 0xb5429b6c00000000, 0xbbd210c900000000,
+    0xe864fdfc00000000, 0xe6f4765900000000, 0x97bc3ae300000000,
+    0x992cb14600000000, 0xca9a5c7300000000, 0xc40ad7d600000000,
+    0x6cf6871800000000, 0x62660cbd00000000, 0x31d0e18800000000,
+    0x3f406a2d00000000, 0x202f31cf00000000, 0x2ebfba6a00000000,
+    0x7d09575f00000000, 0x7399dcfa00000000, 0xdb658c3400000000,
+    0xd5f5079100000000, 0x8643eaa400000000, 0x88d3610100000000,
+    0x25d5030b00000000, 0x2b4588ae00000000, 0x78f3659b00000000,
+    0x7663ee3e00000000, 0xde9fbef000000000, 0xd00f355500000000,
+    0x83b9d86000000000, 0x8d2953c500000000, 0x9246082700000000,
+    0x9cd6838200000000, 0xcf606eb700000000, 0xc1f0e51200000000,
+    0x690cb5dc00000000, 0x679c3e7900000000, 0x342ad34c00000000,
+    0x3aba58e900000000, 0x4bf2145300000000, 0x45629ff600000000,
+    0x16d472c300000000, 0x1844f96600000000, 0xb0b8a9a800000000,
+    0xbe28220d00000000, 0xed9ecf3800000000, 0xe30e449d00000000,
+    0xfc611f7f00000000, 0xf2f194da00000000, 0xa14779ef00000000,
+    0xafd7f24a00000000, 0x072ba28400000000, 0x09bb292100000000,
+    0x5a0dc41400000000, 0x549d4fb100000000, 0xb3312aad00000000,
+    0xbda1a10800000000, 0xee174c3d00000000, 0xe087c79800000000,
+    0x487b975600000000, 0x46eb1cf300000000, 0x155df1c600000000,
+    0x1bcd7a6300000000, 0x04a2218100000000, 0x0a32aa2400000000,
+    0x5984471100000000, 0x5714ccb400000000, 0xffe89c7a00000000,
+    0xf17817df00000000, 0xa2cefaea00000000, 0xac5e714f00000000,
+    0xdd163df500000000, 0xd386b65000000000, 0x80305b6500000000,
+    0x8ea0d0c000000000, 0x265c800e00000000, 0x28cc0bab00000000,
+    0x7b7ae69e00000000, 0x75ea6d3b00000000, 0x6a8536d900000000,
+    0x6415bd7c00000000, 0x37a3504900000000, 0x3933dbec00000000,
+    0x91cf8b2200000000, 0x9f5f008700000000, 0xcce9edb200000000,
+    0xc279661700000000, 0x6f7f041d00000000, 0x61ef8fb800000000,
+    0x3259628d00000000, 0x3cc9e92800000000, 0x9435b9e600000000,
+    0x9aa5324300000000, 0xc913df7600000000, 0xc78354d300000000,
+    0xd8ec0f3100000000, 0xd67c849400000000, 0x85ca69a100000000,
+    0x8b5ae20400000000, 0x23a6b2ca00000000, 0x2d36396f00000000,
+    0x7e80d45a00000000, 0x70105fff00000000, 0x0158134500000000,
+    0x0fc898e000000000, 0x5c7e75d500000000, 0x52eefe7000000000,
+    0xfa12aebe00000000, 0xf482251b00000000, 0xa734c82e00000000,
+    0xa9a4438b00000000, 0xb6cb186900000000, 0xb85b93cc00000000,
+    0xebed7ef900000000, 0xe57df55c00000000, 0x4d81a59200000000,
+    0x43112e3700000000, 0x10a7c30200000000, 0x1e3748a700000000,
+    0x4aaa071600000000, 0x443a8cb300000000, 0x178c618600000000,
+    0x191cea2300000000, 0xb1e0baed00000000, 0xbf70314800000000,
+    0xecc6dc7d00000000, 0xe25657d800000000, 0xfd390c3a00000000,
+    0xf3a9879f00000000, 0xa01f6aaa00000000, 0xae8fe10f00000000,
+    0x0673b1c100000000, 0x08e33a6400000000, 0x5b55d75100000000,
+    0x55c55cf400000000, 0x248d104e00000000, 0x2a1d9beb00000000,
+    0x79ab76de00000000, 0x773bfd7b00000000, 0xdfc7adb500000000,
+    0xd157261000000000, 0x82e1cb2500000000, 0x8c71408000000000,
+    0x931e1b6200000000, 0x9d8e90c700000000, 0xce387df200000000,
+    0xc0a8f65700000000, 0x6854a69900000000, 0x66c42d3c00000000,
+    0x3572c00900000000, 0x3be24bac00000000, 0x96e429a600000000,
+    0x9874a20300000000, 0xcbc24f3600000000, 0xc552c49300000000,
+    0x6dae945d00000000, 0x633e1ff800000000, 0x3088f2cd00000000,
+    0x3e18796800000000, 0x2177228a00000000, 0x2fe7a92f00000000,
+    0x7c51441a00000000, 0x72c1cfbf00000000, 0xda3d9f7100000000,
+    0xd4ad14d400000000, 0x871bf9e100000000, 0x898b724400000000,
+    0xf8c33efe00000000, 0xf653b55b00000000, 0xa5e5586e00000000,
+    0xab75d3cb00000000, 0x0389830500000000, 0x0d1908a000000000,
+    0x5eafe59500000000, 0x503f6e3000000000, 0x4f5035d200000000,
+    0x41c0be7700000000, 0x1276534200000000, 0x1ce6d8e700000000,
+    0xb41a882900000000, 0xba8a038c00000000, 0xe93ceeb900000000,
+    0xe7ac651c00000000},
+   {0x0000000000000000, 0x97a61de700000000, 0x6f4b4a1500000000,
+    0xf8ed57f200000000, 0xde96942a00000000, 0x493089cd00000000,
+    0xb1ddde3f00000000, 0x267bc3d800000000, 0xbc2d295500000000,
+    0x2b8b34b200000000, 0xd366634000000000, 0x44c07ea700000000,
+    0x62bbbd7f00000000, 0xf51da09800000000, 0x0df0f76a00000000,
+    0x9a56ea8d00000000, 0x785b52aa00000000, 0xeffd4f4d00000000,
+    0x171018bf00000000, 0x80b6055800000000, 0xa6cdc68000000000,
+    0x316bdb6700000000, 0xc9868c9500000000, 0x5e20917200000000,
+    0xc4767bff00000000, 0x53d0661800000000, 0xab3d31ea00000000,
+    0x3c9b2c0d00000000, 0x1ae0efd500000000, 0x8d46f23200000000,
+    0x75aba5c000000000, 0xe20db82700000000, 0xb1b0d58f00000000,
+    0x2616c86800000000, 0xdefb9f9a00000000, 0x495d827d00000000,
+    0x6f2641a500000000, 0xf8805c4200000000, 0x006d0bb000000000,
+    0x97cb165700000000, 0x0d9dfcda00000000, 0x9a3be13d00000000,
+    0x62d6b6cf00000000, 0xf570ab2800000000, 0xd30b68f000000000,
+    0x44ad751700000000, 0xbc4022e500000000, 0x2be63f0200000000,
+    0xc9eb872500000000, 0x5e4d9ac200000000, 0xa6a0cd3000000000,
+    0x3106d0d700000000, 0x177d130f00000000, 0x80db0ee800000000,
+    0x7836591a00000000, 0xef9044fd00000000, 0x75c6ae7000000000,
+    0xe260b39700000000, 0x1a8de46500000000, 0x8d2bf98200000000,
+    0xab503a5a00000000, 0x3cf627bd00000000, 0xc41b704f00000000,
+    0x53bd6da800000000, 0x2367dac400000000, 0xb4c1c72300000000,
+    0x4c2c90d100000000, 0xdb8a8d3600000000, 0xfdf14eee00000000,
+    0x6a57530900000000, 0x92ba04fb00000000, 0x051c191c00000000,
+    0x9f4af39100000000, 0x08ecee7600000000, 0xf001b98400000000,
+    0x67a7a46300000000, 0x41dc67bb00000000, 0xd67a7a5c00000000,
+    0x2e972dae00000000, 0xb931304900000000, 0x5b3c886e00000000,
+    0xcc9a958900000000, 0x3477c27b00000000, 0xa3d1df9c00000000,
+    0x85aa1c4400000000, 0x120c01a300000000, 0xeae1565100000000,
+    0x7d474bb600000000, 0xe711a13b00000000, 0x70b7bcdc00000000,
+    0x885aeb2e00000000, 0x1ffcf6c900000000, 0x3987351100000000,
+    0xae2128f600000000, 0x56cc7f0400000000, 0xc16a62e300000000,
+    0x92d70f4b00000000, 0x057112ac00000000, 0xfd9c455e00000000,
+    0x6a3a58b900000000, 0x4c419b6100000000, 0xdbe7868600000000,
+    0x230ad17400000000, 0xb4accc9300000000, 0x2efa261e00000000,
+    0xb95c3bf900000000, 0x41b16c0b00000000, 0xd61771ec00000000,
+    0xf06cb23400000000, 0x67caafd300000000, 0x9f27f82100000000,
+    0x0881e5c600000000, 0xea8c5de100000000, 0x7d2a400600000000,
+    0x85c717f400000000, 0x12610a1300000000, 0x341ac9cb00000000,
+    0xa3bcd42c00000000, 0x5b5183de00000000, 0xccf79e3900000000,
+    0x56a174b400000000, 0xc107695300000000, 0x39ea3ea100000000,
+    0xae4c234600000000, 0x8837e09e00000000, 0x1f91fd7900000000,
+    0xe77caa8b00000000, 0x70dab76c00000000, 0x07c8c55200000000,
+    0x906ed8b500000000, 0x68838f4700000000, 0xff2592a000000000,
+    0xd95e517800000000, 0x4ef84c9f00000000, 0xb6151b6d00000000,
+    0x21b3068a00000000, 0xbbe5ec0700000000, 0x2c43f1e000000000,
+    0xd4aea61200000000, 0x4308bbf500000000, 0x6573782d00000000,
+    0xf2d565ca00000000, 0x0a38323800000000, 0x9d9e2fdf00000000,
+    0x7f9397f800000000, 0xe8358a1f00000000, 0x10d8dded00000000,
+    0x877ec00a00000000, 0xa10503d200000000, 0x36a31e3500000000,
+    0xce4e49c700000000, 0x59e8542000000000, 0xc3bebead00000000,
+    0x5418a34a00000000, 0xacf5f4b800000000, 0x3b53e95f00000000,
+    0x1d282a8700000000, 0x8a8e376000000000, 0x7263609200000000,
+    0xe5c57d7500000000, 0xb67810dd00000000, 0x21de0d3a00000000,
+    0xd9335ac800000000, 0x4e95472f00000000, 0x68ee84f700000000,
+    0xff48991000000000, 0x07a5cee200000000, 0x9003d30500000000,
+    0x0a55398800000000, 0x9df3246f00000000, 0x651e739d00000000,
+    0xf2b86e7a00000000, 0xd4c3ada200000000, 0x4365b04500000000,
+    0xbb88e7b700000000, 0x2c2efa5000000000, 0xce23427700000000,
+    0x59855f9000000000, 0xa168086200000000, 0x36ce158500000000,
+    0x10b5d65d00000000, 0x8713cbba00000000, 0x7ffe9c4800000000,
+    0xe85881af00000000, 0x720e6b2200000000, 0xe5a876c500000000,
+    0x1d45213700000000, 0x8ae33cd000000000, 0xac98ff0800000000,
+    0x3b3ee2ef00000000, 0xc3d3b51d00000000, 0x5475a8fa00000000,
+    0x24af1f9600000000, 0xb309027100000000, 0x4be4558300000000,
+    0xdc42486400000000, 0xfa398bbc00000000, 0x6d9f965b00000000,
+    0x9572c1a900000000, 0x02d4dc4e00000000, 0x988236c300000000,
+    0x0f242b2400000000, 0xf7c97cd600000000, 0x606f613100000000,
+    0x4614a2e900000000, 0xd1b2bf0e00000000, 0x295fe8fc00000000,
+    0xbef9f51b00000000, 0x5cf44d3c00000000, 0xcb5250db00000000,
+    0x33bf072900000000, 0xa4191ace00000000, 0x8262d91600000000,
+    0x15c4c4f100000000, 0xed29930300000000, 0x7a8f8ee400000000,
+    0xe0d9646900000000, 0x777f798e00000000, 0x8f922e7c00000000,
+    0x1834339b00000000, 0x3e4ff04300000000, 0xa9e9eda400000000,
+    0x5104ba5600000000, 0xc6a2a7b100000000, 0x951fca1900000000,
+    0x02b9d7fe00000000, 0xfa54800c00000000, 0x6df29deb00000000,
+    0x4b895e3300000000, 0xdc2f43d400000000, 0x24c2142600000000,
+    0xb36409c100000000, 0x2932e34c00000000, 0xbe94feab00000000,
+    0x4679a95900000000, 0xd1dfb4be00000000, 0xf7a4776600000000,
+    0x60026a8100000000, 0x98ef3d7300000000, 0x0f49209400000000,
+    0xed4498b300000000, 0x7ae2855400000000, 0x820fd2a600000000,
+    0x15a9cf4100000000, 0x33d20c9900000000, 0xa474117e00000000,
+    0x5c99468c00000000, 0xcb3f5b6b00000000, 0x5169b1e600000000,
+    0xc6cfac0100000000, 0x3e22fbf300000000, 0xa984e61400000000,
+    0x8fff25cc00000000, 0x1859382b00000000, 0xe0b46fd900000000,
+    0x7712723e00000000},
+   {0x0000000000000000, 0x411b8c6e00000000, 0x823618dd00000000,
+    0xc32d94b300000000, 0x456b416100000000, 0x0470cd0f00000000,
+    0xc75d59bc00000000, 0x8646d5d200000000, 0x8ad682c200000000,
+    0xcbcd0eac00000000, 0x08e09a1f00000000, 0x49fb167100000000,
+    0xcfbdc3a300000000, 0x8ea64fcd00000000, 0x4d8bdb7e00000000,
+    0x0c90571000000000, 0x55ab745e00000000, 0x14b0f83000000000,
+    0xd79d6c8300000000, 0x9686e0ed00000000, 0x10c0353f00000000,
+    0x51dbb95100000000, 0x92f62de200000000, 0xd3eda18c00000000,
+    0xdf7df69c00000000, 0x9e667af200000000, 0x5d4bee4100000000,
+    0x1c50622f00000000, 0x9a16b7fd00000000, 0xdb0d3b9300000000,
+    0x1820af2000000000, 0x593b234e00000000, 0xaa56e9bc00000000,
+    0xeb4d65d200000000, 0x2860f16100000000, 0x697b7d0f00000000,
+    0xef3da8dd00000000, 0xae2624b300000000, 0x6d0bb00000000000,
+    0x2c103c6e00000000, 0x20806b7e00000000, 0x619be71000000000,
+    0xa2b673a300000000, 0xe3adffcd00000000, 0x65eb2a1f00000000,
+    0x24f0a67100000000, 0xe7dd32c200000000, 0xa6c6beac00000000,
+    0xfffd9de200000000, 0xbee6118c00000000, 0x7dcb853f00000000,
+    0x3cd0095100000000, 0xba96dc8300000000, 0xfb8d50ed00000000,
+    0x38a0c45e00000000, 0x79bb483000000000, 0x752b1f2000000000,
+    0x3430934e00000000, 0xf71d07fd00000000, 0xb6068b9300000000,
+    0x30405e4100000000, 0x715bd22f00000000, 0xb276469c00000000,
+    0xf36dcaf200000000, 0x15aba3a200000000, 0x54b02fcc00000000,
+    0x979dbb7f00000000, 0xd686371100000000, 0x50c0e2c300000000,
+    0x11db6ead00000000, 0xd2f6fa1e00000000, 0x93ed767000000000,
+    0x9f7d216000000000, 0xde66ad0e00000000, 0x1d4b39bd00000000,
+    0x5c50b5d300000000, 0xda16600100000000, 0x9b0dec6f00000000,
+    0x582078dc00000000, 0x193bf4b200000000, 0x4000d7fc00000000,
+    0x011b5b9200000000, 0xc236cf2100000000, 0x832d434f00000000,
+    0x056b969d00000000, 0x44701af300000000, 0x875d8e4000000000,
+    0xc646022e00000000, 0xcad6553e00000000, 0x8bcdd95000000000,
+    0x48e04de300000000, 0x09fbc18d00000000, 0x8fbd145f00000000,
+    0xcea6983100000000, 0x0d8b0c8200000000, 0x4c9080ec00000000,
+    0xbffd4a1e00000000, 0xfee6c67000000000, 0x3dcb52c300000000,
+    0x7cd0dead00000000, 0xfa960b7f00000000, 0xbb8d871100000000,
+    0x78a013a200000000, 0x39bb9fcc00000000, 0x352bc8dc00000000,
+    0x743044b200000000, 0xb71dd00100000000, 0xf6065c6f00000000,
+    0x704089bd00000000, 0x315b05d300000000, 0xf276916000000000,
+    0xb36d1d0e00000000, 0xea563e4000000000, 0xab4db22e00000000,
+    0x6860269d00000000, 0x297baaf300000000, 0xaf3d7f2100000000,
+    0xee26f34f00000000, 0x2d0b67fc00000000, 0x6c10eb9200000000,
+    0x6080bc8200000000, 0x219b30ec00000000, 0xe2b6a45f00000000,
+    0xa3ad283100000000, 0x25ebfde300000000, 0x64f0718d00000000,
+    0xa7dde53e00000000, 0xe6c6695000000000, 0x6b50369e00000000,
+    0x2a4bbaf000000000, 0xe9662e4300000000, 0xa87da22d00000000,
+    0x2e3b77ff00000000, 0x6f20fb9100000000, 0xac0d6f2200000000,
+    0xed16e34c00000000, 0xe186b45c00000000, 0xa09d383200000000,
+    0x63b0ac8100000000, 0x22ab20ef00000000, 0xa4edf53d00000000,
+    0xe5f6795300000000, 0x26dbede000000000, 0x67c0618e00000000,
+    0x3efb42c000000000, 0x7fe0ceae00000000, 0xbccd5a1d00000000,
+    0xfdd6d67300000000, 0x7b9003a100000000, 0x3a8b8fcf00000000,
+    0xf9a61b7c00000000, 0xb8bd971200000000, 0xb42dc00200000000,
+    0xf5364c6c00000000, 0x361bd8df00000000, 0x770054b100000000,
+    0xf146816300000000, 0xb05d0d0d00000000, 0x737099be00000000,
+    0x326b15d000000000, 0xc106df2200000000, 0x801d534c00000000,
+    0x4330c7ff00000000, 0x022b4b9100000000, 0x846d9e4300000000,
+    0xc576122d00000000, 0x065b869e00000000, 0x47400af000000000,
+    0x4bd05de000000000, 0x0acbd18e00000000, 0xc9e6453d00000000,
+    0x88fdc95300000000, 0x0ebb1c8100000000, 0x4fa090ef00000000,
+    0x8c8d045c00000000, 0xcd96883200000000, 0x94adab7c00000000,
+    0xd5b6271200000000, 0x169bb3a100000000, 0x57803fcf00000000,
+    0xd1c6ea1d00000000, 0x90dd667300000000, 0x53f0f2c000000000,
+    0x12eb7eae00000000, 0x1e7b29be00000000, 0x5f60a5d000000000,
+    0x9c4d316300000000, 0xdd56bd0d00000000, 0x5b1068df00000000,
+    0x1a0be4b100000000, 0xd926700200000000, 0x983dfc6c00000000,
+    0x7efb953c00000000, 0x3fe0195200000000, 0xfccd8de100000000,
+    0xbdd6018f00000000, 0x3b90d45d00000000, 0x7a8b583300000000,
+    0xb9a6cc8000000000, 0xf8bd40ee00000000, 0xf42d17fe00000000,
+    0xb5369b9000000000, 0x761b0f2300000000, 0x3700834d00000000,
+    0xb146569f00000000, 0xf05ddaf100000000, 0x33704e4200000000,
+    0x726bc22c00000000, 0x2b50e16200000000, 0x6a4b6d0c00000000,
+    0xa966f9bf00000000, 0xe87d75d100000000, 0x6e3ba00300000000,
+    0x2f202c6d00000000, 0xec0db8de00000000, 0xad1634b000000000,
+    0xa18663a000000000, 0xe09defce00000000, 0x23b07b7d00000000,
+    0x62abf71300000000, 0xe4ed22c100000000, 0xa5f6aeaf00000000,
+    0x66db3a1c00000000, 0x27c0b67200000000, 0xd4ad7c8000000000,
+    0x95b6f0ee00000000, 0x569b645d00000000, 0x1780e83300000000,
+    0x91c63de100000000, 0xd0ddb18f00000000, 0x13f0253c00000000,
+    0x52eba95200000000, 0x5e7bfe4200000000, 0x1f60722c00000000,
+    0xdc4de69f00000000, 0x9d566af100000000, 0x1b10bf2300000000,
+    0x5a0b334d00000000, 0x9926a7fe00000000, 0xd83d2b9000000000,
+    0x810608de00000000, 0xc01d84b000000000, 0x0330100300000000,
+    0x422b9c6d00000000, 0xc46d49bf00000000, 0x8576c5d100000000,
+    0x465b516200000000, 0x0740dd0c00000000, 0x0bd08a1c00000000,
+    0x4acb067200000000, 0x89e692c100000000, 0xc8fd1eaf00000000,
+    0x4ebbcb7d00000000, 0x0fa0471300000000, 0xcc8dd3a000000000,
+    0x8d965fce00000000},
+   {0x0000000000000000, 0x1dfdb50100000000, 0x3afa6b0300000000,
+    0x2707de0200000000, 0x74f4d70600000000, 0x6909620700000000,
+    0x4e0ebc0500000000, 0x53f3090400000000, 0xe8e8af0d00000000,
+    0xf5151a0c00000000, 0xd212c40e00000000, 0xcfef710f00000000,
+    0x9c1c780b00000000, 0x81e1cd0a00000000, 0xa6e6130800000000,
+    0xbb1ba60900000000, 0xd0d15f1b00000000, 0xcd2cea1a00000000,
+    0xea2b341800000000, 0xf7d6811900000000, 0xa425881d00000000,
+    0xb9d83d1c00000000, 0x9edfe31e00000000, 0x8322561f00000000,
+    0x3839f01600000000, 0x25c4451700000000, 0x02c39b1500000000,
+    0x1f3e2e1400000000, 0x4ccd271000000000, 0x5130921100000000,
+    0x76374c1300000000, 0x6bcaf91200000000, 0xa0a3bf3600000000,
+    0xbd5e0a3700000000, 0x9a59d43500000000, 0x87a4613400000000,
+    0xd457683000000000, 0xc9aadd3100000000, 0xeead033300000000,
+    0xf350b63200000000, 0x484b103b00000000, 0x55b6a53a00000000,
+    0x72b17b3800000000, 0x6f4cce3900000000, 0x3cbfc73d00000000,
+    0x2142723c00000000, 0x0645ac3e00000000, 0x1bb8193f00000000,
+    0x7072e02d00000000, 0x6d8f552c00000000, 0x4a888b2e00000000,
+    0x57753e2f00000000, 0x0486372b00000000, 0x197b822a00000000,
+    0x3e7c5c2800000000, 0x2381e92900000000, 0x989a4f2000000000,
+    0x8567fa2100000000, 0xa260242300000000, 0xbf9d912200000000,
+    0xec6e982600000000, 0xf1932d2700000000, 0xd694f32500000000,
+    0xcb69462400000000, 0x40477f6d00000000, 0x5dbaca6c00000000,
+    0x7abd146e00000000, 0x6740a16f00000000, 0x34b3a86b00000000,
+    0x294e1d6a00000000, 0x0e49c36800000000, 0x13b4766900000000,
+    0xa8afd06000000000, 0xb552656100000000, 0x9255bb6300000000,
+    0x8fa80e6200000000, 0xdc5b076600000000, 0xc1a6b26700000000,
+    0xe6a16c6500000000, 0xfb5cd96400000000, 0x9096207600000000,
+    0x8d6b957700000000, 0xaa6c4b7500000000, 0xb791fe7400000000,
+    0xe462f77000000000, 0xf99f427100000000, 0xde989c7300000000,
+    0xc365297200000000, 0x787e8f7b00000000, 0x65833a7a00000000,
+    0x4284e47800000000, 0x5f79517900000000, 0x0c8a587d00000000,
+    0x1177ed7c00000000, 0x3670337e00000000, 0x2b8d867f00000000,
+    0xe0e4c05b00000000, 0xfd19755a00000000, 0xda1eab5800000000,
+    0xc7e31e5900000000, 0x9410175d00000000, 0x89eda25c00000000,
+    0xaeea7c5e00000000, 0xb317c95f00000000, 0x080c6f5600000000,
+    0x15f1da5700000000, 0x32f6045500000000, 0x2f0bb15400000000,
+    0x7cf8b85000000000, 0x61050d5100000000, 0x4602d35300000000,
+    0x5bff665200000000, 0x30359f4000000000, 0x2dc82a4100000000,
+    0x0acff44300000000, 0x1732414200000000, 0x44c1484600000000,
+    0x593cfd4700000000, 0x7e3b234500000000, 0x63c6964400000000,
+    0xd8dd304d00000000, 0xc520854c00000000, 0xe2275b4e00000000,
+    0xffdaee4f00000000, 0xac29e74b00000000, 0xb1d4524a00000000,
+    0x96d38c4800000000, 0x8b2e394900000000, 0x808efeda00000000,
+    0x9d734bdb00000000, 0xba7495d900000000, 0xa78920d800000000,
+    0xf47a29dc00000000, 0xe9879cdd00000000, 0xce8042df00000000,
+    0xd37df7de00000000, 0x686651d700000000, 0x759be4d600000000,
+    0x529c3ad400000000, 0x4f618fd500000000, 0x1c9286d100000000,
+    0x016f33d000000000, 0x2668edd200000000, 0x3b9558d300000000,
+    0x505fa1c100000000, 0x4da214c000000000, 0x6aa5cac200000000,
+    0x77587fc300000000, 0x24ab76c700000000, 0x3956c3c600000000,
+    0x1e511dc400000000, 0x03aca8c500000000, 0xb8b70ecc00000000,
+    0xa54abbcd00000000, 0x824d65cf00000000, 0x9fb0d0ce00000000,
+    0xcc43d9ca00000000, 0xd1be6ccb00000000, 0xf6b9b2c900000000,
+    0xeb4407c800000000, 0x202d41ec00000000, 0x3dd0f4ed00000000,
+    0x1ad72aef00000000, 0x072a9fee00000000, 0x54d996ea00000000,
+    0x492423eb00000000, 0x6e23fde900000000, 0x73de48e800000000,
+    0xc8c5eee100000000, 0xd5385be000000000, 0xf23f85e200000000,
+    0xefc230e300000000, 0xbc3139e700000000, 0xa1cc8ce600000000,
+    0x86cb52e400000000, 0x9b36e7e500000000, 0xf0fc1ef700000000,
+    0xed01abf600000000, 0xca0675f400000000, 0xd7fbc0f500000000,
+    0x8408c9f100000000, 0x99f57cf000000000, 0xbef2a2f200000000,
+    0xa30f17f300000000, 0x1814b1fa00000000, 0x05e904fb00000000,
+    0x22eedaf900000000, 0x3f136ff800000000, 0x6ce066fc00000000,
+    0x711dd3fd00000000, 0x561a0dff00000000, 0x4be7b8fe00000000,
+    0xc0c981b700000000, 0xdd3434b600000000, 0xfa33eab400000000,
+    0xe7ce5fb500000000, 0xb43d56b100000000, 0xa9c0e3b000000000,
+    0x8ec73db200000000, 0x933a88b300000000, 0x28212eba00000000,
+    0x35dc9bbb00000000, 0x12db45b900000000, 0x0f26f0b800000000,
+    0x5cd5f9bc00000000, 0x41284cbd00000000, 0x662f92bf00000000,
+    0x7bd227be00000000, 0x1018deac00000000, 0x0de56bad00000000,
+    0x2ae2b5af00000000, 0x371f00ae00000000, 0x64ec09aa00000000,
+    0x7911bcab00000000, 0x5e1662a900000000, 0x43ebd7a800000000,
+    0xf8f071a100000000, 0xe50dc4a000000000, 0xc20a1aa200000000,
+    0xdff7afa300000000, 0x8c04a6a700000000, 0x91f913a600000000,
+    0xb6fecda400000000, 0xab0378a500000000, 0x606a3e8100000000,
+    0x7d978b8000000000, 0x5a90558200000000, 0x476de08300000000,
+    0x149ee98700000000, 0x09635c8600000000, 0x2e64828400000000,
+    0x3399378500000000, 0x8882918c00000000, 0x957f248d00000000,
+    0xb278fa8f00000000, 0xaf854f8e00000000, 0xfc76468a00000000,
+    0xe18bf38b00000000, 0xc68c2d8900000000, 0xdb71988800000000,
+    0xb0bb619a00000000, 0xad46d49b00000000, 0x8a410a9900000000,
+    0x97bcbf9800000000, 0xc44fb69c00000000, 0xd9b2039d00000000,
+    0xfeb5dd9f00000000, 0xe348689e00000000, 0x5853ce9700000000,
+    0x45ae7b9600000000, 0x62a9a59400000000, 0x7f54109500000000,
+    0x2ca7199100000000, 0x315aac9000000000, 0x165d729200000000,
+    0x0ba0c79300000000},
+   {0x0000000000000000, 0x24d9076300000000, 0x48b20fc600000000,
+    0x6c6b08a500000000, 0xd1626e5700000000, 0xf5bb693400000000,
+    0x99d0619100000000, 0xbd0966f200000000, 0xa2c5dcae00000000,
+    0x861cdbcd00000000, 0xea77d36800000000, 0xceaed40b00000000,
+    0x73a7b2f900000000, 0x577eb59a00000000, 0x3b15bd3f00000000,
+    0x1fccba5c00000000, 0x058dc88600000000, 0x2154cfe500000000,
+    0x4d3fc74000000000, 0x69e6c02300000000, 0xd4efa6d100000000,
+    0xf036a1b200000000, 0x9c5da91700000000, 0xb884ae7400000000,
+    0xa748142800000000, 0x8391134b00000000, 0xeffa1bee00000000,
+    0xcb231c8d00000000, 0x762a7a7f00000000, 0x52f37d1c00000000,
+    0x3e9875b900000000, 0x1a4172da00000000, 0x4b1ce0d600000000,
+    0x6fc5e7b500000000, 0x03aeef1000000000, 0x2777e87300000000,
+    0x9a7e8e8100000000, 0xbea789e200000000, 0xd2cc814700000000,
+    0xf615862400000000, 0xe9d93c7800000000, 0xcd003b1b00000000,
+    0xa16b33be00000000, 0x85b234dd00000000, 0x38bb522f00000000,
+    0x1c62554c00000000, 0x70095de900000000, 0x54d05a8a00000000,
+    0x4e91285000000000, 0x6a482f3300000000, 0x0623279600000000,
+    0x22fa20f500000000, 0x9ff3460700000000, 0xbb2a416400000000,
+    0xd74149c100000000, 0xf3984ea200000000, 0xec54f4fe00000000,
+    0xc88df39d00000000, 0xa4e6fb3800000000, 0x803ffc5b00000000,
+    0x3d369aa900000000, 0x19ef9dca00000000, 0x7584956f00000000,
+    0x515d920c00000000, 0xd73eb17600000000, 0xf3e7b61500000000,
+    0x9f8cbeb000000000, 0xbb55b9d300000000, 0x065cdf2100000000,
+    0x2285d84200000000, 0x4eeed0e700000000, 0x6a37d78400000000,
+    0x75fb6dd800000000, 0x51226abb00000000, 0x3d49621e00000000,
+    0x1990657d00000000, 0xa499038f00000000, 0x804004ec00000000,
+    0xec2b0c4900000000, 0xc8f20b2a00000000, 0xd2b379f000000000,
+    0xf66a7e9300000000, 0x9a01763600000000, 0xbed8715500000000,
+    0x03d117a700000000, 0x270810c400000000, 0x4b63186100000000,
+    0x6fba1f0200000000, 0x7076a55e00000000, 0x54afa23d00000000,
+    0x38c4aa9800000000, 0x1c1dadfb00000000, 0xa114cb0900000000,
+    0x85cdcc6a00000000, 0xe9a6c4cf00000000, 0xcd7fc3ac00000000,
+    0x9c2251a000000000, 0xb8fb56c300000000, 0xd4905e6600000000,
+    0xf049590500000000, 0x4d403ff700000000, 0x6999389400000000,
+    0x05f2303100000000, 0x212b375200000000, 0x3ee78d0e00000000,
+    0x1a3e8a6d00000000, 0x765582c800000000, 0x528c85ab00000000,
+    0xef85e35900000000, 0xcb5ce43a00000000, 0xa737ec9f00000000,
+    0x83eeebfc00000000, 0x99af992600000000, 0xbd769e4500000000,
+    0xd11d96e000000000, 0xf5c4918300000000, 0x48cdf77100000000,
+    0x6c14f01200000000, 0x007ff8b700000000, 0x24a6ffd400000000,
+    0x3b6a458800000000, 0x1fb342eb00000000, 0x73d84a4e00000000,
+    0x57014d2d00000000, 0xea082bdf00000000, 0xced12cbc00000000,
+    0xa2ba241900000000, 0x8663237a00000000, 0xae7d62ed00000000,
+    0x8aa4658e00000000, 0xe6cf6d2b00000000, 0xc2166a4800000000,
+    0x7f1f0cba00000000, 0x5bc60bd900000000, 0x37ad037c00000000,
+    0x1374041f00000000, 0x0cb8be4300000000, 0x2861b92000000000,
+    0x440ab18500000000, 0x60d3b6e600000000, 0xdddad01400000000,
+    0xf903d77700000000, 0x9568dfd200000000, 0xb1b1d8b100000000,
+    0xabf0aa6b00000000, 0x8f29ad0800000000, 0xe342a5ad00000000,
+    0xc79ba2ce00000000, 0x7a92c43c00000000, 0x5e4bc35f00000000,
+    0x3220cbfa00000000, 0x16f9cc9900000000, 0x093576c500000000,
+    0x2dec71a600000000, 0x4187790300000000, 0x655e7e6000000000,
+    0xd857189200000000, 0xfc8e1ff100000000, 0x90e5175400000000,
+    0xb43c103700000000, 0xe561823b00000000, 0xc1b8855800000000,
+    0xadd38dfd00000000, 0x890a8a9e00000000, 0x3403ec6c00000000,
+    0x10daeb0f00000000, 0x7cb1e3aa00000000, 0x5868e4c900000000,
+    0x47a45e9500000000, 0x637d59f600000000, 0x0f16515300000000,
+    0x2bcf563000000000, 0x96c630c200000000, 0xb21f37a100000000,
+    0xde743f0400000000, 0xfaad386700000000, 0xe0ec4abd00000000,
+    0xc4354dde00000000, 0xa85e457b00000000, 0x8c87421800000000,
+    0x318e24ea00000000, 0x1557238900000000, 0x793c2b2c00000000,
+    0x5de52c4f00000000, 0x4229961300000000, 0x66f0917000000000,
+    0x0a9b99d500000000, 0x2e429eb600000000, 0x934bf84400000000,
+    0xb792ff2700000000, 0xdbf9f78200000000, 0xff20f0e100000000,
+    0x7943d39b00000000, 0x5d9ad4f800000000, 0x31f1dc5d00000000,
+    0x1528db3e00000000, 0xa821bdcc00000000, 0x8cf8baaf00000000,
+    0xe093b20a00000000, 0xc44ab56900000000, 0xdb860f3500000000,
+    0xff5f085600000000, 0x933400f300000000, 0xb7ed079000000000,
+    0x0ae4616200000000, 0x2e3d660100000000, 0x42566ea400000000,
+    0x668f69c700000000, 0x7cce1b1d00000000, 0x58171c7e00000000,
+    0x347c14db00000000, 0x10a513b800000000, 0xadac754a00000000,
+    0x8975722900000000, 0xe51e7a8c00000000, 0xc1c77def00000000,
+    0xde0bc7b300000000, 0xfad2c0d000000000, 0x96b9c87500000000,
+    0xb260cf1600000000, 0x0f69a9e400000000, 0x2bb0ae8700000000,
+    0x47dba62200000000, 0x6302a14100000000, 0x325f334d00000000,
+    0x1686342e00000000, 0x7aed3c8b00000000, 0x5e343be800000000,
+    0xe33d5d1a00000000, 0xc7e45a7900000000, 0xab8f52dc00000000,
+    0x8f5655bf00000000, 0x909aefe300000000, 0xb443e88000000000,
+    0xd828e02500000000, 0xfcf1e74600000000, 0x41f881b400000000,
+    0x652186d700000000, 0x094a8e7200000000, 0x2d93891100000000,
+    0x37d2fbcb00000000, 0x130bfca800000000, 0x7f60f40d00000000,
+    0x5bb9f36e00000000, 0xe6b0959c00000000, 0xc26992ff00000000,
+    0xae029a5a00000000, 0x8adb9d3900000000, 0x9517276500000000,
+    0xb1ce200600000000, 0xdda528a300000000, 0xf97c2fc000000000,
+    0x4475493200000000, 0x60ac4e5100000000, 0x0cc746f400000000,
+    0x281e419700000000},
+   {0x0000000000000000, 0x08e3603c00000000, 0x10c6c17800000000,
+    0x1825a14400000000, 0x208c83f100000000, 0x286fe3cd00000000,
+    0x304a428900000000, 0x38a922b500000000, 0x011e763800000000,
+    0x09fd160400000000, 0x11d8b74000000000, 0x193bd77c00000000,
+    0x2192f5c900000000, 0x297195f500000000, 0x315434b100000000,
+    0x39b7548d00000000, 0x023cec7000000000, 0x0adf8c4c00000000,
+    0x12fa2d0800000000, 0x1a194d3400000000, 0x22b06f8100000000,
+    0x2a530fbd00000000, 0x3276aef900000000, 0x3a95cec500000000,
+    0x03229a4800000000, 0x0bc1fa7400000000, 0x13e45b3000000000,
+    0x1b073b0c00000000, 0x23ae19b900000000, 0x2b4d798500000000,
+    0x3368d8c100000000, 0x3b8bb8fd00000000, 0x0478d8e100000000,
+    0x0c9bb8dd00000000, 0x14be199900000000, 0x1c5d79a500000000,
+    0x24f45b1000000000, 0x2c173b2c00000000, 0x34329a6800000000,
+    0x3cd1fa5400000000, 0x0566aed900000000, 0x0d85cee500000000,
+    0x15a06fa100000000, 0x1d430f9d00000000, 0x25ea2d2800000000,
+    0x2d094d1400000000, 0x352cec5000000000, 0x3dcf8c6c00000000,
+    0x0644349100000000, 0x0ea754ad00000000, 0x1682f5e900000000,
+    0x1e6195d500000000, 0x26c8b76000000000, 0x2e2bd75c00000000,
+    0x360e761800000000, 0x3eed162400000000, 0x075a42a900000000,
+    0x0fb9229500000000, 0x179c83d100000000, 0x1f7fe3ed00000000,
+    0x27d6c15800000000, 0x2f35a16400000000, 0x3710002000000000,
+    0x3ff3601c00000000, 0x49f6c11800000000, 0x4115a12400000000,
+    0x5930006000000000, 0x51d3605c00000000, 0x697a42e900000000,
+    0x619922d500000000, 0x79bc839100000000, 0x715fe3ad00000000,
+    0x48e8b72000000000, 0x400bd71c00000000, 0x582e765800000000,
+    0x50cd166400000000, 0x686434d100000000, 0x608754ed00000000,
+    0x78a2f5a900000000, 0x7041959500000000, 0x4bca2d6800000000,
+    0x43294d5400000000, 0x5b0cec1000000000, 0x53ef8c2c00000000,
+    0x6b46ae9900000000, 0x63a5cea500000000, 0x7b806fe100000000,
+    0x73630fdd00000000, 0x4ad45b5000000000, 0x42373b6c00000000,
+    0x5a129a2800000000, 0x52f1fa1400000000, 0x6a58d8a100000000,
+    0x62bbb89d00000000, 0x7a9e19d900000000, 0x727d79e500000000,
+    0x4d8e19f900000000, 0x456d79c500000000, 0x5d48d88100000000,
+    0x55abb8bd00000000, 0x6d029a0800000000, 0x65e1fa3400000000,
+    0x7dc45b7000000000, 0x75273b4c00000000, 0x4c906fc100000000,
+    0x44730ffd00000000, 0x5c56aeb900000000, 0x54b5ce8500000000,
+    0x6c1cec3000000000, 0x64ff8c0c00000000, 0x7cda2d4800000000,
+    0x74394d7400000000, 0x4fb2f58900000000, 0x475195b500000000,
+    0x5f7434f100000000, 0x579754cd00000000, 0x6f3e767800000000,
+    0x67dd164400000000, 0x7ff8b70000000000, 0x771bd73c00000000,
+    0x4eac83b100000000, 0x464fe38d00000000, 0x5e6a42c900000000,
+    0x568922f500000000, 0x6e20004000000000, 0x66c3607c00000000,
+    0x7ee6c13800000000, 0x7605a10400000000, 0x92ec833100000000,
+    0x9a0fe30d00000000, 0x822a424900000000, 0x8ac9227500000000,
+    0xb26000c000000000, 0xba8360fc00000000, 0xa2a6c1b800000000,
+    0xaa45a18400000000, 0x93f2f50900000000, 0x9b11953500000000,
+    0x8334347100000000, 0x8bd7544d00000000, 0xb37e76f800000000,
+    0xbb9d16c400000000, 0xa3b8b78000000000, 0xab5bd7bc00000000,
+    0x90d06f4100000000, 0x98330f7d00000000, 0x8016ae3900000000,
+    0x88f5ce0500000000, 0xb05cecb000000000, 0xb8bf8c8c00000000,
+    0xa09a2dc800000000, 0xa8794df400000000, 0x91ce197900000000,
+    0x992d794500000000, 0x8108d80100000000, 0x89ebb83d00000000,
+    0xb1429a8800000000, 0xb9a1fab400000000, 0xa1845bf000000000,
+    0xa9673bcc00000000, 0x96945bd000000000, 0x9e773bec00000000,
+    0x86529aa800000000, 0x8eb1fa9400000000, 0xb618d82100000000,
+    0xbefbb81d00000000, 0xa6de195900000000, 0xae3d796500000000,
+    0x978a2de800000000, 0x9f694dd400000000, 0x874cec9000000000,
+    0x8faf8cac00000000, 0xb706ae1900000000, 0xbfe5ce2500000000,
+    0xa7c06f6100000000, 0xaf230f5d00000000, 0x94a8b7a000000000,
+    0x9c4bd79c00000000, 0x846e76d800000000, 0x8c8d16e400000000,
+    0xb424345100000000, 0xbcc7546d00000000, 0xa4e2f52900000000,
+    0xac01951500000000, 0x95b6c19800000000, 0x9d55a1a400000000,
+    0x857000e000000000, 0x8d9360dc00000000, 0xb53a426900000000,
+    0xbdd9225500000000, 0xa5fc831100000000, 0xad1fe32d00000000,
+    0xdb1a422900000000, 0xd3f9221500000000, 0xcbdc835100000000,
+    0xc33fe36d00000000, 0xfb96c1d800000000, 0xf375a1e400000000,
+    0xeb5000a000000000, 0xe3b3609c00000000, 0xda04341100000000,
+    0xd2e7542d00000000, 0xcac2f56900000000, 0xc221955500000000,
+    0xfa88b7e000000000, 0xf26bd7dc00000000, 0xea4e769800000000,
+    0xe2ad16a400000000, 0xd926ae5900000000, 0xd1c5ce6500000000,
+    0xc9e06f2100000000, 0xc1030f1d00000000, 0xf9aa2da800000000,
+    0xf1494d9400000000, 0xe96cecd000000000, 0xe18f8cec00000000,
+    0xd838d86100000000, 0xd0dbb85d00000000, 0xc8fe191900000000,
+    0xc01d792500000000, 0xf8b45b9000000000, 0xf0573bac00000000,
+    0xe8729ae800000000, 0xe091fad400000000, 0xdf629ac800000000,
+    0xd781faf400000000, 0xcfa45bb000000000, 0xc7473b8c00000000,
+    0xffee193900000000, 0xf70d790500000000, 0xef28d84100000000,
+    0xe7cbb87d00000000, 0xde7cecf000000000, 0xd69f8ccc00000000,
+    0xceba2d8800000000, 0xc6594db400000000, 0xfef06f0100000000,
+    0xf6130f3d00000000, 0xee36ae7900000000, 0xe6d5ce4500000000,
+    0xdd5e76b800000000, 0xd5bd168400000000, 0xcd98b7c000000000,
+    0xc57bd7fc00000000, 0xfdd2f54900000000, 0xf531957500000000,
+    0xed14343100000000, 0xe5f7540d00000000, 0xdc40008000000000,
+    0xd4a360bc00000000, 0xcc86c1f800000000, 0xc465a1c400000000,
+    0xfccc837100000000, 0xf42fe34d00000000, 0xec0a420900000000,
+    0xe4e9223500000000},
+   {0x0000000000000000, 0xd1e8e70e00000000, 0xa2d1cf1d00000000,
+    0x7339281300000000, 0x44a39f3b00000000, 0x954b783500000000,
+    0xe672502600000000, 0x379ab72800000000, 0x88463f7700000000,
+    0x59aed87900000000, 0x2a97f06a00000000, 0xfb7f176400000000,
+    0xcce5a04c00000000, 0x1d0d474200000000, 0x6e346f5100000000,
+    0xbfdc885f00000000, 0x108d7eee00000000, 0xc16599e000000000,
+    0xb25cb1f300000000, 0x63b456fd00000000, 0x542ee1d500000000,
+    0x85c606db00000000, 0xf6ff2ec800000000, 0x2717c9c600000000,
+    0x98cb419900000000, 0x4923a69700000000, 0x3a1a8e8400000000,
+    0xebf2698a00000000, 0xdc68dea200000000, 0x0d8039ac00000000,
+    0x7eb911bf00000000, 0xaf51f6b100000000, 0x611c8c0700000000,
+    0xb0f46b0900000000, 0xc3cd431a00000000, 0x1225a41400000000,
+    0x25bf133c00000000, 0xf457f43200000000, 0x876edc2100000000,
+    0x56863b2f00000000, 0xe95ab37000000000, 0x38b2547e00000000,
+    0x4b8b7c6d00000000, 0x9a639b6300000000, 0xadf92c4b00000000,
+    0x7c11cb4500000000, 0x0f28e35600000000, 0xdec0045800000000,
+    0x7191f2e900000000, 0xa07915e700000000, 0xd3403df400000000,
+    0x02a8dafa00000000, 0x35326dd200000000, 0xe4da8adc00000000,
+    0x97e3a2cf00000000, 0x460b45c100000000, 0xf9d7cd9e00000000,
+    0x283f2a9000000000, 0x5b06028300000000, 0x8aeee58d00000000,
+    0xbd7452a500000000, 0x6c9cb5ab00000000, 0x1fa59db800000000,
+    0xce4d7ab600000000, 0xc238180f00000000, 0x13d0ff0100000000,
+    0x60e9d71200000000, 0xb101301c00000000, 0x869b873400000000,
+    0x5773603a00000000, 0x244a482900000000, 0xf5a2af2700000000,
+    0x4a7e277800000000, 0x9b96c07600000000, 0xe8afe86500000000,
+    0x39470f6b00000000, 0x0eddb84300000000, 0xdf355f4d00000000,
+    0xac0c775e00000000, 0x7de4905000000000, 0xd2b566e100000000,
+    0x035d81ef00000000, 0x7064a9fc00000000, 0xa18c4ef200000000,
+    0x9616f9da00000000, 0x47fe1ed400000000, 0x34c736c700000000,
+    0xe52fd1c900000000, 0x5af3599600000000, 0x8b1bbe9800000000,
+    0xf822968b00000000, 0x29ca718500000000, 0x1e50c6ad00000000,
+    0xcfb821a300000000, 0xbc8109b000000000, 0x6d69eebe00000000,
+    0xa324940800000000, 0x72cc730600000000, 0x01f55b1500000000,
+    0xd01dbc1b00000000, 0xe7870b3300000000, 0x366fec3d00000000,
+    0x4556c42e00000000, 0x94be232000000000, 0x2b62ab7f00000000,
+    0xfa8a4c7100000000, 0x89b3646200000000, 0x585b836c00000000,
+    0x6fc1344400000000, 0xbe29d34a00000000, 0xcd10fb5900000000,
+    0x1cf81c5700000000, 0xb3a9eae600000000, 0x62410de800000000,
+    0x117825fb00000000, 0xc090c2f500000000, 0xf70a75dd00000000,
+    0x26e292d300000000, 0x55dbbac000000000, 0x84335dce00000000,
+    0x3befd59100000000, 0xea07329f00000000, 0x993e1a8c00000000,
+    0x48d6fd8200000000, 0x7f4c4aaa00000000, 0xaea4ada400000000,
+    0xdd9d85b700000000, 0x0c7562b900000000, 0x8471301e00000000,
+    0x5599d71000000000, 0x26a0ff0300000000, 0xf748180d00000000,
+    0xc0d2af2500000000, 0x113a482b00000000, 0x6203603800000000,
+    0xb3eb873600000000, 0x0c370f6900000000, 0xdddfe86700000000,
+    0xaee6c07400000000, 0x7f0e277a00000000, 0x4894905200000000,
+    0x997c775c00000000, 0xea455f4f00000000, 0x3badb84100000000,
+    0x94fc4ef000000000, 0x4514a9fe00000000, 0x362d81ed00000000,
+    0xe7c566e300000000, 0xd05fd1cb00000000, 0x01b736c500000000,
+    0x728e1ed600000000, 0xa366f9d800000000, 0x1cba718700000000,
+    0xcd52968900000000, 0xbe6bbe9a00000000, 0x6f83599400000000,
+    0x5819eebc00000000, 0x89f109b200000000, 0xfac821a100000000,
+    0x2b20c6af00000000, 0xe56dbc1900000000, 0x34855b1700000000,
+    0x47bc730400000000, 0x9654940a00000000, 0xa1ce232200000000,
+    0x7026c42c00000000, 0x031fec3f00000000, 0xd2f70b3100000000,
+    0x6d2b836e00000000, 0xbcc3646000000000, 0xcffa4c7300000000,
+    0x1e12ab7d00000000, 0x29881c5500000000, 0xf860fb5b00000000,
+    0x8b59d34800000000, 0x5ab1344600000000, 0xf5e0c2f700000000,
+    0x240825f900000000, 0x57310dea00000000, 0x86d9eae400000000,
+    0xb1435dcc00000000, 0x60abbac200000000, 0x139292d100000000,
+    0xc27a75df00000000, 0x7da6fd8000000000, 0xac4e1a8e00000000,
+    0xdf77329d00000000, 0x0e9fd59300000000, 0x390562bb00000000,
+    0xe8ed85b500000000, 0x9bd4ada600000000, 0x4a3c4aa800000000,
+    0x4649281100000000, 0x97a1cf1f00000000, 0xe498e70c00000000,
+    0x3570000200000000, 0x02eab72a00000000, 0xd302502400000000,
+    0xa03b783700000000, 0x71d39f3900000000, 0xce0f176600000000,
+    0x1fe7f06800000000, 0x6cded87b00000000, 0xbd363f7500000000,
+    0x8aac885d00000000, 0x5b446f5300000000, 0x287d474000000000,
+    0xf995a04e00000000, 0x56c456ff00000000, 0x872cb1f100000000,
+    0xf41599e200000000, 0x25fd7eec00000000, 0x1267c9c400000000,
+    0xc38f2eca00000000, 0xb0b606d900000000, 0x615ee1d700000000,
+    0xde82698800000000, 0x0f6a8e8600000000, 0x7c53a69500000000,
+    0xadbb419b00000000, 0x9a21f6b300000000, 0x4bc911bd00000000,
+    0x38f039ae00000000, 0xe918dea000000000, 0x2755a41600000000,
+    0xf6bd431800000000, 0x85846b0b00000000, 0x546c8c0500000000,
+    0x63f63b2d00000000, 0xb21edc2300000000, 0xc127f43000000000,
+    0x10cf133e00000000, 0xaf139b6100000000, 0x7efb7c6f00000000,
+    0x0dc2547c00000000, 0xdc2ab37200000000, 0xebb0045a00000000,
+    0x3a58e35400000000, 0x4961cb4700000000, 0x98892c4900000000,
+    0x37d8daf800000000, 0xe6303df600000000, 0x950915e500000000,
+    0x44e1f2eb00000000, 0x737b45c300000000, 0xa293a2cd00000000,
+    0xd1aa8ade00000000, 0x00426dd000000000, 0xbf9ee58f00000000,
+    0x6e76028100000000, 0x1d4f2a9200000000, 0xcca7cd9c00000000,
+    0xfb3d7ab400000000, 0x2ad59dba00000000, 0x59ecb5a900000000,
+    0x880452a700000000},
+   {0x0000000000000000, 0xaa05daf100000000, 0x150dc53800000000,
+    0xbf081fc900000000, 0x2a1a8a7100000000, 0x801f508000000000,
+    0x3f174f4900000000, 0x951295b800000000, 0x543414e300000000,
+    0xfe31ce1200000000, 0x4139d1db00000000, 0xeb3c0b2a00000000,
+    0x7e2e9e9200000000, 0xd42b446300000000, 0x6b235baa00000000,
+    0xc126815b00000000, 0xe96e591d00000000, 0x436b83ec00000000,
+    0xfc639c2500000000, 0x566646d400000000, 0xc374d36c00000000,
+    0x6971099d00000000, 0xd679165400000000, 0x7c7ccca500000000,
+    0xbd5a4dfe00000000, 0x175f970f00000000, 0xa85788c600000000,
+    0x0252523700000000, 0x9740c78f00000000, 0x3d451d7e00000000,
+    0x824d02b700000000, 0x2848d84600000000, 0xd2ddb23a00000000,
+    0x78d868cb00000000, 0xc7d0770200000000, 0x6dd5adf300000000,
+    0xf8c7384b00000000, 0x52c2e2ba00000000, 0xedcafd7300000000,
+    0x47cf278200000000, 0x86e9a6d900000000, 0x2cec7c2800000000,
+    0x93e463e100000000, 0x39e1b91000000000, 0xacf32ca800000000,
+    0x06f6f65900000000, 0xb9fee99000000000, 0x13fb336100000000,
+    0x3bb3eb2700000000, 0x91b631d600000000, 0x2ebe2e1f00000000,
+    0x84bbf4ee00000000, 0x11a9615600000000, 0xbbacbba700000000,
+    0x04a4a46e00000000, 0xaea17e9f00000000, 0x6f87ffc400000000,
+    0xc582253500000000, 0x7a8a3afc00000000, 0xd08fe00d00000000,
+    0x459d75b500000000, 0xef98af4400000000, 0x5090b08d00000000,
+    0xfa956a7c00000000, 0xa4bb657500000000, 0x0ebebf8400000000,
+    0xb1b6a04d00000000, 0x1bb37abc00000000, 0x8ea1ef0400000000,
+    0x24a435f500000000, 0x9bac2a3c00000000, 0x31a9f0cd00000000,
+    0xf08f719600000000, 0x5a8aab6700000000, 0xe582b4ae00000000,
+    0x4f876e5f00000000, 0xda95fbe700000000, 0x7090211600000000,
+    0xcf983edf00000000, 0x659de42e00000000, 0x4dd53c6800000000,
+    0xe7d0e69900000000, 0x58d8f95000000000, 0xf2dd23a100000000,
+    0x67cfb61900000000, 0xcdca6ce800000000, 0x72c2732100000000,
+    0xd8c7a9d000000000, 0x19e1288b00000000, 0xb3e4f27a00000000,
+    0x0cecedb300000000, 0xa6e9374200000000, 0x33fba2fa00000000,
+    0x99fe780b00000000, 0x26f667c200000000, 0x8cf3bd3300000000,
+    0x7666d74f00000000, 0xdc630dbe00000000, 0x636b127700000000,
+    0xc96ec88600000000, 0x5c7c5d3e00000000, 0xf67987cf00000000,
+    0x4971980600000000, 0xe37442f700000000, 0x2252c3ac00000000,
+    0x8857195d00000000, 0x375f069400000000, 0x9d5adc6500000000,
+    0x084849dd00000000, 0xa24d932c00000000, 0x1d458ce500000000,
+    0xb740561400000000, 0x9f088e5200000000, 0x350d54a300000000,
+    0x8a054b6a00000000, 0x2000919b00000000, 0xb512042300000000,
+    0x1f17ded200000000, 0xa01fc11b00000000, 0x0a1a1bea00000000,
+    0xcb3c9ab100000000, 0x6139404000000000, 0xde315f8900000000,
+    0x7434857800000000, 0xe12610c000000000, 0x4b23ca3100000000,
+    0xf42bd5f800000000, 0x5e2e0f0900000000, 0x4877cbea00000000,
+    0xe272111b00000000, 0x5d7a0ed200000000, 0xf77fd42300000000,
+    0x626d419b00000000, 0xc8689b6a00000000, 0x776084a300000000,
+    0xdd655e5200000000, 0x1c43df0900000000, 0xb64605f800000000,
+    0x094e1a3100000000, 0xa34bc0c000000000, 0x3659557800000000,
+    0x9c5c8f8900000000, 0x2354904000000000, 0x89514ab100000000,
+    0xa11992f700000000, 0x0b1c480600000000, 0xb41457cf00000000,
+    0x1e118d3e00000000, 0x8b03188600000000, 0x2106c27700000000,
+    0x9e0eddbe00000000, 0x340b074f00000000, 0xf52d861400000000,
+    0x5f285ce500000000, 0xe020432c00000000, 0x4a2599dd00000000,
+    0xdf370c6500000000, 0x7532d69400000000, 0xca3ac95d00000000,
+    0x603f13ac00000000, 0x9aaa79d000000000, 0x30afa32100000000,
+    0x8fa7bce800000000, 0x25a2661900000000, 0xb0b0f3a100000000,
+    0x1ab5295000000000, 0xa5bd369900000000, 0x0fb8ec6800000000,
+    0xce9e6d3300000000, 0x649bb7c200000000, 0xdb93a80b00000000,
+    0x719672fa00000000, 0xe484e74200000000, 0x4e813db300000000,
+    0xf189227a00000000, 0x5b8cf88b00000000, 0x73c420cd00000000,
+    0xd9c1fa3c00000000, 0x66c9e5f500000000, 0xcccc3f0400000000,
+    0x59deaabc00000000, 0xf3db704d00000000, 0x4cd36f8400000000,
+    0xe6d6b57500000000, 0x27f0342e00000000, 0x8df5eedf00000000,
+    0x32fdf11600000000, 0x98f82be700000000, 0x0deabe5f00000000,
+    0xa7ef64ae00000000, 0x18e77b6700000000, 0xb2e2a19600000000,
+    0xecccae9f00000000, 0x46c9746e00000000, 0xf9c16ba700000000,
+    0x53c4b15600000000, 0xc6d624ee00000000, 0x6cd3fe1f00000000,
+    0xd3dbe1d600000000, 0x79de3b2700000000, 0xb8f8ba7c00000000,
+    0x12fd608d00000000, 0xadf57f4400000000, 0x07f0a5b500000000,
+    0x92e2300d00000000, 0x38e7eafc00000000, 0x87eff53500000000,
+    0x2dea2fc400000000, 0x05a2f78200000000, 0xafa72d7300000000,
+    0x10af32ba00000000, 0xbaaae84b00000000, 0x2fb87df300000000,
+    0x85bda70200000000, 0x3ab5b8cb00000000, 0x90b0623a00000000,
+    0x5196e36100000000, 0xfb93399000000000, 0x449b265900000000,
+    0xee9efca800000000, 0x7b8c691000000000, 0xd189b3e100000000,
+    0x6e81ac2800000000, 0xc48476d900000000, 0x3e111ca500000000,
+    0x9414c65400000000, 0x2b1cd99d00000000, 0x8119036c00000000,
+    0x140b96d400000000, 0xbe0e4c2500000000, 0x010653ec00000000,
+    0xab03891d00000000, 0x6a25084600000000, 0xc020d2b700000000,
+    0x7f28cd7e00000000, 0xd52d178f00000000, 0x403f823700000000,
+    0xea3a58c600000000, 0x5532470f00000000, 0xff379dfe00000000,
+    0xd77f45b800000000, 0x7d7a9f4900000000, 0xc272808000000000,
+    0x68775a7100000000, 0xfd65cfc900000000, 0x5760153800000000,
+    0xe8680af100000000, 0x426dd00000000000, 0x834b515b00000000,
+    0x294e8baa00000000, 0x9646946300000000, 0x3c434e9200000000,
+    0xa951db2a00000000, 0x035401db00000000, 0xbc5c1e1200000000,
+    0x1659c4e300000000}};
+
+#else /* W == 4 */
+
+local const z_crc_t FAR crc_braid_table[][256] = {
+   {0x00000000, 0xae689191, 0x87a02563, 0x29c8b4f2, 0xd4314c87,
+    0x7a59dd16, 0x539169e4, 0xfdf9f875, 0x73139f4f, 0xdd7b0ede,
+    0xf4b3ba2c, 0x5adb2bbd, 0xa722d3c8, 0x094a4259, 0x2082f6ab,
+    0x8eea673a, 0xe6273e9e, 0x484faf0f, 0x61871bfd, 0xcfef8a6c,
+    0x32167219, 0x9c7ee388, 0xb5b6577a, 0x1bdec6eb, 0x9534a1d1,
+    0x3b5c3040, 0x129484b2, 0xbcfc1523, 0x4105ed56, 0xef6d7cc7,
+    0xc6a5c835, 0x68cd59a4, 0x173f7b7d, 0xb957eaec, 0x909f5e1e,
+    0x3ef7cf8f, 0xc30e37fa, 0x6d66a66b, 0x44ae1299, 0xeac68308,
+    0x642ce432, 0xca4475a3, 0xe38cc151, 0x4de450c0, 0xb01da8b5,
+    0x1e753924, 0x37bd8dd6, 0x99d51c47, 0xf11845e3, 0x5f70d472,
+    0x76b86080, 0xd8d0f111, 0x25290964, 0x8b4198f5, 0xa2892c07,
+    0x0ce1bd96, 0x820bdaac, 0x2c634b3d, 0x05abffcf, 0xabc36e5e,
+    0x563a962b, 0xf85207ba, 0xd19ab348, 0x7ff222d9, 0x2e7ef6fa,
+    0x8016676b, 0xa9ded399, 0x07b64208, 0xfa4fba7d, 0x54272bec,
+    0x7def9f1e, 0xd3870e8f, 0x5d6d69b5, 0xf305f824, 0xdacd4cd6,
+    0x74a5dd47, 0x895c2532, 0x2734b4a3, 0x0efc0051, 0xa09491c0,
+    0xc859c864, 0x663159f5, 0x4ff9ed07, 0xe1917c96, 0x1c6884e3,
+    0xb2001572, 0x9bc8a180, 0x35a03011, 0xbb4a572b, 0x1522c6ba,
+    0x3cea7248, 0x9282e3d9, 0x6f7b1bac, 0xc1138a3d, 0xe8db3ecf,
+    0x46b3af5e, 0x39418d87, 0x97291c16, 0xbee1a8e4, 0x10893975,
+    0xed70c100, 0x43185091, 0x6ad0e463, 0xc4b875f2, 0x4a5212c8,
+    0xe43a8359, 0xcdf237ab, 0x639aa63a, 0x9e635e4f, 0x300bcfde,
+    0x19c37b2c, 0xb7abeabd, 0xdf66b319, 0x710e2288, 0x58c6967a,
+    0xf6ae07eb, 0x0b57ff9e, 0xa53f6e0f, 0x8cf7dafd, 0x229f4b6c,
+    0xac752c56, 0x021dbdc7, 0x2bd50935, 0x85bd98a4, 0x784460d1,
+    0xd62cf140, 0xffe445b2, 0x518cd423, 0x5cfdedf4, 0xf2957c65,
+    0xdb5dc897, 0x75355906, 0x88cca173, 0x26a430e2, 0x0f6c8410,
+    0xa1041581, 0x2fee72bb, 0x8186e32a, 0xa84e57d8, 0x0626c649,
+    0xfbdf3e3c, 0x55b7afad, 0x7c7f1b5f, 0xd2178ace, 0xbadad36a,
+    0x14b242fb, 0x3d7af609, 0x93126798, 0x6eeb9fed, 0xc0830e7c,
+    0xe94bba8e, 0x47232b1f, 0xc9c94c25, 0x67a1ddb4, 0x4e696946,
+    0xe001f8d7, 0x1df800a2, 0xb3909133, 0x9a5825c1, 0x3430b450,
+    0x4bc29689, 0xe5aa0718, 0xcc62b3ea, 0x620a227b, 0x9ff3da0e,
+    0x319b4b9f, 0x1853ff6d, 0xb63b6efc, 0x38d109c6, 0x96b99857,
+    0xbf712ca5, 0x1119bd34, 0xece04541, 0x4288d4d0, 0x6b406022,
+    0xc528f1b3, 0xade5a817, 0x038d3986, 0x2a458d74, 0x842d1ce5,
+    0x79d4e490, 0xd7bc7501, 0xfe74c1f3, 0x501c5062, 0xdef63758,
+    0x709ea6c9, 0x5956123b, 0xf73e83aa, 0x0ac77bdf, 0xa4afea4e,
+    0x8d675ebc, 0x230fcf2d, 0x72831b0e, 0xdceb8a9f, 0xf5233e6d,
+    0x5b4baffc, 0xa6b25789, 0x08dac618, 0x211272ea, 0x8f7ae37b,
+    0x01908441, 0xaff815d0, 0x8630a122, 0x285830b3, 0xd5a1c8c6,
+    0x7bc95957, 0x5201eda5, 0xfc697c34, 0x94a42590, 0x3accb401,
+    0x130400f3, 0xbd6c9162, 0x40956917, 0xeefdf886, 0xc7354c74,
+    0x695ddde5, 0xe7b7badf, 0x49df2b4e, 0x60179fbc, 0xce7f0e2d,
+    0x3386f658, 0x9dee67c9, 0xb426d33b, 0x1a4e42aa, 0x65bc6073,
+    0xcbd4f1e2, 0xe21c4510, 0x4c74d481, 0xb18d2cf4, 0x1fe5bd65,
+    0x362d0997, 0x98459806, 0x16afff3c, 0xb8c76ead, 0x910fda5f,
+    0x3f674bce, 0xc29eb3bb, 0x6cf6222a, 0x453e96d8, 0xeb560749,
+    0x839b5eed, 0x2df3cf7c, 0x043b7b8e, 0xaa53ea1f, 0x57aa126a,
+    0xf9c283fb, 0xd00a3709, 0x7e62a698, 0xf088c1a2, 0x5ee05033,
+    0x7728e4c1, 0xd9407550, 0x24b98d25, 0x8ad11cb4, 0xa319a846,
+    0x0d7139d7},
+   {0x00000000, 0xb9fbdbe8, 0xa886b191, 0x117d6a79, 0x8a7c6563,
+    0x3387be8b, 0x22fad4f2, 0x9b010f1a, 0xcf89cc87, 0x7672176f,
+    0x670f7d16, 0xdef4a6fe, 0x45f5a9e4, 0xfc0e720c, 0xed731875,
+    0x5488c39d, 0x44629f4f, 0xfd9944a7, 0xece42ede, 0x551ff536,
+    0xce1efa2c, 0x77e521c4, 0x66984bbd, 0xdf639055, 0x8beb53c8,
+    0x32108820, 0x236de259, 0x9a9639b1, 0x019736ab, 0xb86ced43,
+    0xa911873a, 0x10ea5cd2, 0x88c53e9e, 0x313ee576, 0x20438f0f,
+    0x99b854e7, 0x02b95bfd, 0xbb428015, 0xaa3fea6c, 0x13c43184,
+    0x474cf219, 0xfeb729f1, 0xefca4388, 0x56319860, 0xcd30977a,
+    0x74cb4c92, 0x65b626eb, 0xdc4dfd03, 0xcca7a1d1, 0x755c7a39,
+    0x64211040, 0xdddacba8, 0x46dbc4b2, 0xff201f5a, 0xee5d7523,
+    0x57a6aecb, 0x032e6d56, 0xbad5b6be, 0xaba8dcc7, 0x1253072f,
+    0x89520835, 0x30a9d3dd, 0x21d4b9a4, 0x982f624c, 0xcafb7b7d,
+    0x7300a095, 0x627dcaec, 0xdb861104, 0x40871e1e, 0xf97cc5f6,
+    0xe801af8f, 0x51fa7467, 0x0572b7fa, 0xbc896c12, 0xadf4066b,
+    0x140fdd83, 0x8f0ed299, 0x36f50971, 0x27886308, 0x9e73b8e0,
+    0x8e99e432, 0x37623fda, 0x261f55a3, 0x9fe48e4b, 0x04e58151,
+    0xbd1e5ab9, 0xac6330c0, 0x1598eb28, 0x411028b5, 0xf8ebf35d,
+    0xe9969924, 0x506d42cc, 0xcb6c4dd6, 0x7297963e, 0x63eafc47,
+    0xda1127af, 0x423e45e3, 0xfbc59e0b, 0xeab8f472, 0x53432f9a,
+    0xc8422080, 0x71b9fb68, 0x60c49111, 0xd93f4af9, 0x8db78964,
+    0x344c528c, 0x253138f5, 0x9ccae31d, 0x07cbec07, 0xbe3037ef,
+    0xaf4d5d96, 0x16b6867e, 0x065cdaac, 0xbfa70144, 0xaeda6b3d,
+    0x1721b0d5, 0x8c20bfcf, 0x35db6427, 0x24a60e5e, 0x9d5dd5b6,
+    0xc9d5162b, 0x702ecdc3, 0x6153a7ba, 0xd8a87c52, 0x43a97348,
+    0xfa52a8a0, 0xeb2fc2d9, 0x52d41931, 0x4e87f0bb, 0xf77c2b53,
+    0xe601412a, 0x5ffa9ac2, 0xc4fb95d8, 0x7d004e30, 0x6c7d2449,
+    0xd586ffa1, 0x810e3c3c, 0x38f5e7d4, 0x29888dad, 0x90735645,
+    0x0b72595f, 0xb28982b7, 0xa3f4e8ce, 0x1a0f3326, 0x0ae56ff4,
+    0xb31eb41c, 0xa263de65, 0x1b98058d, 0x80990a97, 0x3962d17f,
+    0x281fbb06, 0x91e460ee, 0xc56ca373, 0x7c97789b, 0x6dea12e2,
+    0xd411c90a, 0x4f10c610, 0xf6eb1df8, 0xe7967781, 0x5e6dac69,
+    0xc642ce25, 0x7fb915cd, 0x6ec47fb4, 0xd73fa45c, 0x4c3eab46,
+    0xf5c570ae, 0xe4b81ad7, 0x5d43c13f, 0x09cb02a2, 0xb030d94a,
+    0xa14db333, 0x18b668db, 0x83b767c1, 0x3a4cbc29, 0x2b31d650,
+    0x92ca0db8, 0x8220516a, 0x3bdb8a82, 0x2aa6e0fb, 0x935d3b13,
+    0x085c3409, 0xb1a7efe1, 0xa0da8598, 0x19215e70, 0x4da99ded,
+    0xf4524605, 0xe52f2c7c, 0x5cd4f794, 0xc7d5f88e, 0x7e2e2366,
+    0x6f53491f, 0xd6a892f7, 0x847c8bc6, 0x3d87502e, 0x2cfa3a57,
+    0x9501e1bf, 0x0e00eea5, 0xb7fb354d, 0xa6865f34, 0x1f7d84dc,
+    0x4bf54741, 0xf20e9ca9, 0xe373f6d0, 0x5a882d38, 0xc1892222,
+    0x7872f9ca, 0x690f93b3, 0xd0f4485b, 0xc01e1489, 0x79e5cf61,
+    0x6898a518, 0xd1637ef0, 0x4a6271ea, 0xf399aa02, 0xe2e4c07b,
+    0x5b1f1b93, 0x0f97d80e, 0xb66c03e6, 0xa711699f, 0x1eeab277,
+    0x85ebbd6d, 0x3c106685, 0x2d6d0cfc, 0x9496d714, 0x0cb9b558,
+    0xb5426eb0, 0xa43f04c9, 0x1dc4df21, 0x86c5d03b, 0x3f3e0bd3,
+    0x2e4361aa, 0x97b8ba42, 0xc33079df, 0x7acba237, 0x6bb6c84e,
+    0xd24d13a6, 0x494c1cbc, 0xf0b7c754, 0xe1caad2d, 0x583176c5,
+    0x48db2a17, 0xf120f1ff, 0xe05d9b86, 0x59a6406e, 0xc2a74f74,
+    0x7b5c949c, 0x6a21fee5, 0xd3da250d, 0x8752e690, 0x3ea93d78,
+    0x2fd45701, 0x962f8ce9, 0x0d2e83f3, 0xb4d5581b, 0xa5a83262,
+    0x1c53e98a},
+   {0x00000000, 0x9d0fe176, 0xe16ec4ad, 0x7c6125db, 0x19ac8f1b,
+    0x84a36e6d, 0xf8c24bb6, 0x65cdaac0, 0x33591e36, 0xae56ff40,
+    0xd237da9b, 0x4f383bed, 0x2af5912d, 0xb7fa705b, 0xcb9b5580,
+    0x5694b4f6, 0x66b23c6c, 0xfbbddd1a, 0x87dcf8c1, 0x1ad319b7,
+    0x7f1eb377, 0xe2115201, 0x9e7077da, 0x037f96ac, 0x55eb225a,
+    0xc8e4c32c, 0xb485e6f7, 0x298a0781, 0x4c47ad41, 0xd1484c37,
+    0xad2969ec, 0x3026889a, 0xcd6478d8, 0x506b99ae, 0x2c0abc75,
+    0xb1055d03, 0xd4c8f7c3, 0x49c716b5, 0x35a6336e, 0xa8a9d218,
+    0xfe3d66ee, 0x63328798, 0x1f53a243, 0x825c4335, 0xe791e9f5,
+    0x7a9e0883, 0x06ff2d58, 0x9bf0cc2e, 0xabd644b4, 0x36d9a5c2,
+    0x4ab88019, 0xd7b7616f, 0xb27acbaf, 0x2f752ad9, 0x53140f02,
+    0xce1bee74, 0x988f5a82, 0x0580bbf4, 0x79e19e2f, 0xe4ee7f59,
+    0x8123d599, 0x1c2c34ef, 0x604d1134, 0xfd42f042, 0x41b9f7f1,
+    0xdcb61687, 0xa0d7335c, 0x3dd8d22a, 0x581578ea, 0xc51a999c,
+    0xb97bbc47, 0x24745d31, 0x72e0e9c7, 0xefef08b1, 0x938e2d6a,
+    0x0e81cc1c, 0x6b4c66dc, 0xf64387aa, 0x8a22a271, 0x172d4307,
+    0x270bcb9d, 0xba042aeb, 0xc6650f30, 0x5b6aee46, 0x3ea74486,
+    0xa3a8a5f0, 0xdfc9802b, 0x42c6615d, 0x1452d5ab, 0x895d34dd,
+    0xf53c1106, 0x6833f070, 0x0dfe5ab0, 0x90f1bbc6, 0xec909e1d,
+    0x719f7f6b, 0x8cdd8f29, 0x11d26e5f, 0x6db34b84, 0xf0bcaaf2,
+    0x95710032, 0x087ee144, 0x741fc49f, 0xe91025e9, 0xbf84911f,
+    0x228b7069, 0x5eea55b2, 0xc3e5b4c4, 0xa6281e04, 0x3b27ff72,
+    0x4746daa9, 0xda493bdf, 0xea6fb345, 0x77605233, 0x0b0177e8,
+    0x960e969e, 0xf3c33c5e, 0x6eccdd28, 0x12adf8f3, 0x8fa21985,
+    0xd936ad73, 0x44394c05, 0x385869de, 0xa55788a8, 0xc09a2268,
+    0x5d95c31e, 0x21f4e6c5, 0xbcfb07b3, 0x8373efe2, 0x1e7c0e94,
+    0x621d2b4f, 0xff12ca39, 0x9adf60f9, 0x07d0818f, 0x7bb1a454,
+    0xe6be4522, 0xb02af1d4, 0x2d2510a2, 0x51443579, 0xcc4bd40f,
+    0xa9867ecf, 0x34899fb9, 0x48e8ba62, 0xd5e75b14, 0xe5c1d38e,
+    0x78ce32f8, 0x04af1723, 0x99a0f655, 0xfc6d5c95, 0x6162bde3,
+    0x1d039838, 0x800c794e, 0xd698cdb8, 0x4b972cce, 0x37f60915,
+    0xaaf9e863, 0xcf3442a3, 0x523ba3d5, 0x2e5a860e, 0xb3556778,
+    0x4e17973a, 0xd318764c, 0xaf795397, 0x3276b2e1, 0x57bb1821,
+    0xcab4f957, 0xb6d5dc8c, 0x2bda3dfa, 0x7d4e890c, 0xe041687a,
+    0x9c204da1, 0x012facd7, 0x64e20617, 0xf9ede761, 0x858cc2ba,
+    0x188323cc, 0x28a5ab56, 0xb5aa4a20, 0xc9cb6ffb, 0x54c48e8d,
+    0x3109244d, 0xac06c53b, 0xd067e0e0, 0x4d680196, 0x1bfcb560,
+    0x86f35416, 0xfa9271cd, 0x679d90bb, 0x02503a7b, 0x9f5fdb0d,
+    0xe33efed6, 0x7e311fa0, 0xc2ca1813, 0x5fc5f965, 0x23a4dcbe,
+    0xbeab3dc8, 0xdb669708, 0x4669767e, 0x3a0853a5, 0xa707b2d3,
+    0xf1930625, 0x6c9ce753, 0x10fdc288, 0x8df223fe, 0xe83f893e,
+    0x75306848, 0x09514d93, 0x945eace5, 0xa478247f, 0x3977c509,
+    0x4516e0d2, 0xd81901a4, 0xbdd4ab64, 0x20db4a12, 0x5cba6fc9,
+    0xc1b58ebf, 0x97213a49, 0x0a2edb3f, 0x764ffee4, 0xeb401f92,
+    0x8e8db552, 0x13825424, 0x6fe371ff, 0xf2ec9089, 0x0fae60cb,
+    0x92a181bd, 0xeec0a466, 0x73cf4510, 0x1602efd0, 0x8b0d0ea6,
+    0xf76c2b7d, 0x6a63ca0b, 0x3cf77efd, 0xa1f89f8b, 0xdd99ba50,
+    0x40965b26, 0x255bf1e6, 0xb8541090, 0xc435354b, 0x593ad43d,
+    0x691c5ca7, 0xf413bdd1, 0x8872980a, 0x157d797c, 0x70b0d3bc,
+    0xedbf32ca, 0x91de1711, 0x0cd1f667, 0x5a454291, 0xc74aa3e7,
+    0xbb2b863c, 0x2624674a, 0x43e9cd8a, 0xdee62cfc, 0xa2870927,
+    0x3f88e851},
+   {0x00000000, 0xdd96d985, 0x605cb54b, 0xbdca6cce, 0xc0b96a96,
+    0x1d2fb313, 0xa0e5dfdd, 0x7d730658, 0x5a03d36d, 0x87950ae8,
+    0x3a5f6626, 0xe7c9bfa3, 0x9abab9fb, 0x472c607e, 0xfae60cb0,
+    0x2770d535, 0xb407a6da, 0x69917f5f, 0xd45b1391, 0x09cdca14,
+    0x74becc4c, 0xa92815c9, 0x14e27907, 0xc974a082, 0xee0475b7,
+    0x3392ac32, 0x8e58c0fc, 0x53ce1979, 0x2ebd1f21, 0xf32bc6a4,
+    0x4ee1aa6a, 0x937773ef, 0xb37e4bf5, 0x6ee89270, 0xd322febe,
+    0x0eb4273b, 0x73c72163, 0xae51f8e6, 0x139b9428, 0xce0d4dad,
+    0xe97d9898, 0x34eb411d, 0x89212dd3, 0x54b7f456, 0x29c4f20e,
+    0xf4522b8b, 0x49984745, 0x940e9ec0, 0x0779ed2f, 0xdaef34aa,
+    0x67255864, 0xbab381e1, 0xc7c087b9, 0x1a565e3c, 0xa79c32f2,
+    0x7a0aeb77, 0x5d7a3e42, 0x80ece7c7, 0x3d268b09, 0xe0b0528c,
+    0x9dc354d4, 0x40558d51, 0xfd9fe19f, 0x2009381a, 0xbd8d91ab,
+    0x601b482e, 0xddd124e0, 0x0047fd65, 0x7d34fb3d, 0xa0a222b8,
+    0x1d684e76, 0xc0fe97f3, 0xe78e42c6, 0x3a189b43, 0x87d2f78d,
+    0x5a442e08, 0x27372850, 0xfaa1f1d5, 0x476b9d1b, 0x9afd449e,
+    0x098a3771, 0xd41ceef4, 0x69d6823a, 0xb4405bbf, 0xc9335de7,
+    0x14a58462, 0xa96fe8ac, 0x74f93129, 0x5389e41c, 0x8e1f3d99,
+    0x33d55157, 0xee4388d2, 0x93308e8a, 0x4ea6570f, 0xf36c3bc1,
+    0x2efae244, 0x0ef3da5e, 0xd36503db, 0x6eaf6f15, 0xb339b690,
+    0xce4ab0c8, 0x13dc694d, 0xae160583, 0x7380dc06, 0x54f00933,
+    0x8966d0b6, 0x34acbc78, 0xe93a65fd, 0x944963a5, 0x49dfba20,
+    0xf415d6ee, 0x29830f6b, 0xbaf47c84, 0x6762a501, 0xdaa8c9cf,
+    0x073e104a, 0x7a4d1612, 0xa7dbcf97, 0x1a11a359, 0xc7877adc,
+    0xe0f7afe9, 0x3d61766c, 0x80ab1aa2, 0x5d3dc327, 0x204ec57f,
+    0xfdd81cfa, 0x40127034, 0x9d84a9b1, 0xa06a2517, 0x7dfcfc92,
+    0xc036905c, 0x1da049d9, 0x60d34f81, 0xbd459604, 0x008ffaca,
+    0xdd19234f, 0xfa69f67a, 0x27ff2fff, 0x9a354331, 0x47a39ab4,
+    0x3ad09cec, 0xe7464569, 0x5a8c29a7, 0x871af022, 0x146d83cd,
+    0xc9fb5a48, 0x74313686, 0xa9a7ef03, 0xd4d4e95b, 0x094230de,
+    0xb4885c10, 0x691e8595, 0x4e6e50a0, 0x93f88925, 0x2e32e5eb,
+    0xf3a43c6e, 0x8ed73a36, 0x5341e3b3, 0xee8b8f7d, 0x331d56f8,
+    0x13146ee2, 0xce82b767, 0x7348dba9, 0xaede022c, 0xd3ad0474,
+    0x0e3bddf1, 0xb3f1b13f, 0x6e6768ba, 0x4917bd8f, 0x9481640a,
+    0x294b08c4, 0xf4ddd141, 0x89aed719, 0x54380e9c, 0xe9f26252,
+    0x3464bbd7, 0xa713c838, 0x7a8511bd, 0xc74f7d73, 0x1ad9a4f6,
+    0x67aaa2ae, 0xba3c7b2b, 0x07f617e5, 0xda60ce60, 0xfd101b55,
+    0x2086c2d0, 0x9d4cae1e, 0x40da779b, 0x3da971c3, 0xe03fa846,
+    0x5df5c488, 0x80631d0d, 0x1de7b4bc, 0xc0716d39, 0x7dbb01f7,
+    0xa02dd872, 0xdd5ede2a, 0x00c807af, 0xbd026b61, 0x6094b2e4,
+    0x47e467d1, 0x9a72be54, 0x27b8d29a, 0xfa2e0b1f, 0x875d0d47,
+    0x5acbd4c2, 0xe701b80c, 0x3a976189, 0xa9e01266, 0x7476cbe3,
+    0xc9bca72d, 0x142a7ea8, 0x695978f0, 0xb4cfa175, 0x0905cdbb,
+    0xd493143e, 0xf3e3c10b, 0x2e75188e, 0x93bf7440, 0x4e29adc5,
+    0x335aab9d, 0xeecc7218, 0x53061ed6, 0x8e90c753, 0xae99ff49,
+    0x730f26cc, 0xcec54a02, 0x13539387, 0x6e2095df, 0xb3b64c5a,
+    0x0e7c2094, 0xd3eaf911, 0xf49a2c24, 0x290cf5a1, 0x94c6996f,
+    0x495040ea, 0x342346b2, 0xe9b59f37, 0x547ff3f9, 0x89e92a7c,
+    0x1a9e5993, 0xc7088016, 0x7ac2ecd8, 0xa754355d, 0xda273305,
+    0x07b1ea80, 0xba7b864e, 0x67ed5fcb, 0x409d8afe, 0x9d0b537b,
+    0x20c13fb5, 0xfd57e630, 0x8024e068, 0x5db239ed, 0xe0785523,
+    0x3dee8ca6}};
+
+local const z_word_t FAR crc_braid_big_table[][256] = {
+   {0x00000000, 0x85d996dd, 0x4bb55c60, 0xce6ccabd, 0x966ab9c0,
+    0x13b32f1d, 0xdddfe5a0, 0x5806737d, 0x6dd3035a, 0xe80a9587,
+    0x26665f3a, 0xa3bfc9e7, 0xfbb9ba9a, 0x7e602c47, 0xb00ce6fa,
+    0x35d57027, 0xdaa607b4, 0x5f7f9169, 0x91135bd4, 0x14cacd09,
+    0x4cccbe74, 0xc91528a9, 0x0779e214, 0x82a074c9, 0xb77504ee,
+    0x32ac9233, 0xfcc0588e, 0x7919ce53, 0x211fbd2e, 0xa4c62bf3,
+    0x6aaae14e, 0xef737793, 0xf54b7eb3, 0x7092e86e, 0xbefe22d3,
+    0x3b27b40e, 0x6321c773, 0xe6f851ae, 0x28949b13, 0xad4d0dce,
+    0x98987de9, 0x1d41eb34, 0xd32d2189, 0x56f4b754, 0x0ef2c429,
+    0x8b2b52f4, 0x45479849, 0xc09e0e94, 0x2fed7907, 0xaa34efda,
+    0x64582567, 0xe181b3ba, 0xb987c0c7, 0x3c5e561a, 0xf2329ca7,
+    0x77eb0a7a, 0x423e7a5d, 0xc7e7ec80, 0x098b263d, 0x8c52b0e0,
+    0xd454c39d, 0x518d5540, 0x9fe19ffd, 0x1a380920, 0xab918dbd,
+    0x2e481b60, 0xe024d1dd, 0x65fd4700, 0x3dfb347d, 0xb822a2a0,
+    0x764e681d, 0xf397fec0, 0xc6428ee7, 0x439b183a, 0x8df7d287,
+    0x082e445a, 0x50283727, 0xd5f1a1fa, 0x1b9d6b47, 0x9e44fd9a,
+    0x71378a09, 0xf4ee1cd4, 0x3a82d669, 0xbf5b40b4, 0xe75d33c9,
+    0x6284a514, 0xace86fa9, 0x2931f974, 0x1ce48953, 0x993d1f8e,
+    0x5751d533, 0xd28843ee, 0x8a8e3093, 0x0f57a64e, 0xc13b6cf3,
+    0x44e2fa2e, 0x5edaf30e, 0xdb0365d3, 0x156faf6e, 0x90b639b3,
+    0xc8b04ace, 0x4d69dc13, 0x830516ae, 0x06dc8073, 0x3309f054,
+    0xb6d06689, 0x78bcac34, 0xfd653ae9, 0xa5634994, 0x20badf49,
+    0xeed615f4, 0x6b0f8329, 0x847cf4ba, 0x01a56267, 0xcfc9a8da,
+    0x4a103e07, 0x12164d7a, 0x97cfdba7, 0x59a3111a, 0xdc7a87c7,
+    0xe9aff7e0, 0x6c76613d, 0xa21aab80, 0x27c33d5d, 0x7fc54e20,
+    0xfa1cd8fd, 0x34701240, 0xb1a9849d, 0x17256aa0, 0x92fcfc7d,
+    0x5c9036c0, 0xd949a01d, 0x814fd360, 0x049645bd, 0xcafa8f00,
+    0x4f2319dd, 0x7af669fa, 0xff2fff27, 0x3143359a, 0xb49aa347,
+    0xec9cd03a, 0x694546e7, 0xa7298c5a, 0x22f01a87, 0xcd836d14,
+    0x485afbc9, 0x86363174, 0x03efa7a9, 0x5be9d4d4, 0xde304209,
+    0x105c88b4, 0x95851e69, 0xa0506e4e, 0x2589f893, 0xebe5322e,
+    0x6e3ca4f3, 0x363ad78e, 0xb3e34153, 0x7d8f8bee, 0xf8561d33,
+    0xe26e1413, 0x67b782ce, 0xa9db4873, 0x2c02deae, 0x7404add3,
+    0xf1dd3b0e, 0x3fb1f1b3, 0xba68676e, 0x8fbd1749, 0x0a648194,
+    0xc4084b29, 0x41d1ddf4, 0x19d7ae89, 0x9c0e3854, 0x5262f2e9,
+    0xd7bb6434, 0x38c813a7, 0xbd11857a, 0x737d4fc7, 0xf6a4d91a,
+    0xaea2aa67, 0x2b7b3cba, 0xe517f607, 0x60ce60da, 0x551b10fd,
+    0xd0c28620, 0x1eae4c9d, 0x9b77da40, 0xc371a93d, 0x46a83fe0,
+    0x88c4f55d, 0x0d1d6380, 0xbcb4e71d, 0x396d71c0, 0xf701bb7d,
+    0x72d82da0, 0x2ade5edd, 0xaf07c800, 0x616b02bd, 0xe4b29460,
+    0xd167e447, 0x54be729a, 0x9ad2b827, 0x1f0b2efa, 0x470d5d87,
+    0xc2d4cb5a, 0x0cb801e7, 0x8961973a, 0x6612e0a9, 0xe3cb7674,
+    0x2da7bcc9, 0xa87e2a14, 0xf0785969, 0x75a1cfb4, 0xbbcd0509,
+    0x3e1493d4, 0x0bc1e3f3, 0x8e18752e, 0x4074bf93, 0xc5ad294e,
+    0x9dab5a33, 0x1872ccee, 0xd61e0653, 0x53c7908e, 0x49ff99ae,
+    0xcc260f73, 0x024ac5ce, 0x87935313, 0xdf95206e, 0x5a4cb6b3,
+    0x94207c0e, 0x11f9ead3, 0x242c9af4, 0xa1f50c29, 0x6f99c694,
+    0xea405049, 0xb2462334, 0x379fb5e9, 0xf9f37f54, 0x7c2ae989,
+    0x93599e1a, 0x168008c7, 0xd8ecc27a, 0x5d3554a7, 0x053327da,
+    0x80eab107, 0x4e867bba, 0xcb5fed67, 0xfe8a9d40, 0x7b530b9d,
+    0xb53fc120, 0x30e657fd, 0x68e02480, 0xed39b25d, 0x235578e0,
+    0xa68cee3d},
+   {0x00000000, 0x76e10f9d, 0xadc46ee1, 0xdb25617c, 0x1b8fac19,
+    0x6d6ea384, 0xb64bc2f8, 0xc0aacd65, 0x361e5933, 0x40ff56ae,
+    0x9bda37d2, 0xed3b384f, 0x2d91f52a, 0x5b70fab7, 0x80559bcb,
+    0xf6b49456, 0x6c3cb266, 0x1addbdfb, 0xc1f8dc87, 0xb719d31a,
+    0x77b31e7f, 0x015211e2, 0xda77709e, 0xac967f03, 0x5a22eb55,
+    0x2cc3e4c8, 0xf7e685b4, 0x81078a29, 0x41ad474c, 0x374c48d1,
+    0xec6929ad, 0x9a882630, 0xd87864cd, 0xae996b50, 0x75bc0a2c,
+    0x035d05b1, 0xc3f7c8d4, 0xb516c749, 0x6e33a635, 0x18d2a9a8,
+    0xee663dfe, 0x98873263, 0x43a2531f, 0x35435c82, 0xf5e991e7,
+    0x83089e7a, 0x582dff06, 0x2eccf09b, 0xb444d6ab, 0xc2a5d936,
+    0x1980b84a, 0x6f61b7d7, 0xafcb7ab2, 0xd92a752f, 0x020f1453,
+    0x74ee1bce, 0x825a8f98, 0xf4bb8005, 0x2f9ee179, 0x597feee4,
+    0x99d52381, 0xef342c1c, 0x34114d60, 0x42f042fd, 0xf1f7b941,
+    0x8716b6dc, 0x5c33d7a0, 0x2ad2d83d, 0xea781558, 0x9c991ac5,
+    0x47bc7bb9, 0x315d7424, 0xc7e9e072, 0xb108efef, 0x6a2d8e93,
+    0x1ccc810e, 0xdc664c6b, 0xaa8743f6, 0x71a2228a, 0x07432d17,
+    0x9dcb0b27, 0xeb2a04ba, 0x300f65c6, 0x46ee6a5b, 0x8644a73e,
+    0xf0a5a8a3, 0x2b80c9df, 0x5d61c642, 0xabd55214, 0xdd345d89,
+    0x06113cf5, 0x70f03368, 0xb05afe0d, 0xc6bbf190, 0x1d9e90ec,
+    0x6b7f9f71, 0x298fdd8c, 0x5f6ed211, 0x844bb36d, 0xf2aabcf0,
+    0x32007195, 0x44e17e08, 0x9fc41f74, 0xe92510e9, 0x1f9184bf,
+    0x69708b22, 0xb255ea5e, 0xc4b4e5c3, 0x041e28a6, 0x72ff273b,
+    0xa9da4647, 0xdf3b49da, 0x45b36fea, 0x33526077, 0xe877010b,
+    0x9e960e96, 0x5e3cc3f3, 0x28ddcc6e, 0xf3f8ad12, 0x8519a28f,
+    0x73ad36d9, 0x054c3944, 0xde695838, 0xa88857a5, 0x68229ac0,
+    0x1ec3955d, 0xc5e6f421, 0xb307fbbc, 0xe2ef7383, 0x940e7c1e,
+    0x4f2b1d62, 0x39ca12ff, 0xf960df9a, 0x8f81d007, 0x54a4b17b,
+    0x2245bee6, 0xd4f12ab0, 0xa210252d, 0x79354451, 0x0fd44bcc,
+    0xcf7e86a9, 0xb99f8934, 0x62bae848, 0x145be7d5, 0x8ed3c1e5,
+    0xf832ce78, 0x2317af04, 0x55f6a099, 0x955c6dfc, 0xe3bd6261,
+    0x3898031d, 0x4e790c80, 0xb8cd98d6, 0xce2c974b, 0x1509f637,
+    0x63e8f9aa, 0xa34234cf, 0xd5a33b52, 0x0e865a2e, 0x786755b3,
+    0x3a97174e, 0x4c7618d3, 0x975379af, 0xe1b27632, 0x2118bb57,
+    0x57f9b4ca, 0x8cdcd5b6, 0xfa3dda2b, 0x0c894e7d, 0x7a6841e0,
+    0xa14d209c, 0xd7ac2f01, 0x1706e264, 0x61e7edf9, 0xbac28c85,
+    0xcc238318, 0x56aba528, 0x204aaab5, 0xfb6fcbc9, 0x8d8ec454,
+    0x4d240931, 0x3bc506ac, 0xe0e067d0, 0x9601684d, 0x60b5fc1b,
+    0x1654f386, 0xcd7192fa, 0xbb909d67, 0x7b3a5002, 0x0ddb5f9f,
+    0xd6fe3ee3, 0xa01f317e, 0x1318cac2, 0x65f9c55f, 0xbedca423,
+    0xc83dabbe, 0x089766db, 0x7e766946, 0xa553083a, 0xd3b207a7,
+    0x250693f1, 0x53e79c6c, 0x88c2fd10, 0xfe23f28d, 0x3e893fe8,
+    0x48683075, 0x934d5109, 0xe5ac5e94, 0x7f2478a4, 0x09c57739,
+    0xd2e01645, 0xa40119d8, 0x64abd4bd, 0x124adb20, 0xc96fba5c,
+    0xbf8eb5c1, 0x493a2197, 0x3fdb2e0a, 0xe4fe4f76, 0x921f40eb,
+    0x52b58d8e, 0x24548213, 0xff71e36f, 0x8990ecf2, 0xcb60ae0f,
+    0xbd81a192, 0x66a4c0ee, 0x1045cf73, 0xd0ef0216, 0xa60e0d8b,
+    0x7d2b6cf7, 0x0bca636a, 0xfd7ef73c, 0x8b9ff8a1, 0x50ba99dd,
+    0x265b9640, 0xe6f15b25, 0x901054b8, 0x4b3535c4, 0x3dd43a59,
+    0xa75c1c69, 0xd1bd13f4, 0x0a987288, 0x7c797d15, 0xbcd3b070,
+    0xca32bfed, 0x1117de91, 0x67f6d10c, 0x9142455a, 0xe7a34ac7,
+    0x3c862bbb, 0x4a672426, 0x8acde943, 0xfc2ce6de, 0x270987a2,
+    0x51e8883f},
+   {0x00000000, 0xe8dbfbb9, 0x91b186a8, 0x796a7d11, 0x63657c8a,
+    0x8bbe8733, 0xf2d4fa22, 0x1a0f019b, 0x87cc89cf, 0x6f177276,
+    0x167d0f67, 0xfea6f4de, 0xe4a9f545, 0x0c720efc, 0x751873ed,
+    0x9dc38854, 0x4f9f6244, 0xa74499fd, 0xde2ee4ec, 0x36f51f55,
+    0x2cfa1ece, 0xc421e577, 0xbd4b9866, 0x559063df, 0xc853eb8b,
+    0x20881032, 0x59e26d23, 0xb139969a, 0xab369701, 0x43ed6cb8,
+    0x3a8711a9, 0xd25cea10, 0x9e3ec588, 0x76e53e31, 0x0f8f4320,
+    0xe754b899, 0xfd5bb902, 0x158042bb, 0x6cea3faa, 0x8431c413,
+    0x19f24c47, 0xf129b7fe, 0x8843caef, 0x60983156, 0x7a9730cd,
+    0x924ccb74, 0xeb26b665, 0x03fd4ddc, 0xd1a1a7cc, 0x397a5c75,
+    0x40102164, 0xa8cbdadd, 0xb2c4db46, 0x5a1f20ff, 0x23755dee,
+    0xcbaea657, 0x566d2e03, 0xbeb6d5ba, 0xc7dca8ab, 0x2f075312,
+    0x35085289, 0xddd3a930, 0xa4b9d421, 0x4c622f98, 0x7d7bfbca,
+    0x95a00073, 0xecca7d62, 0x041186db, 0x1e1e8740, 0xf6c57cf9,
+    0x8faf01e8, 0x6774fa51, 0xfab77205, 0x126c89bc, 0x6b06f4ad,
+    0x83dd0f14, 0x99d20e8f, 0x7109f536, 0x08638827, 0xe0b8739e,
+    0x32e4998e, 0xda3f6237, 0xa3551f26, 0x4b8ee49f, 0x5181e504,
+    0xb95a1ebd, 0xc03063ac, 0x28eb9815, 0xb5281041, 0x5df3ebf8,
+    0x249996e9, 0xcc426d50, 0xd64d6ccb, 0x3e969772, 0x47fcea63,
+    0xaf2711da, 0xe3453e42, 0x0b9ec5fb, 0x72f4b8ea, 0x9a2f4353,
+    0x802042c8, 0x68fbb971, 0x1191c460, 0xf94a3fd9, 0x6489b78d,
+    0x8c524c34, 0xf5383125, 0x1de3ca9c, 0x07eccb07, 0xef3730be,
+    0x965d4daf, 0x7e86b616, 0xacda5c06, 0x4401a7bf, 0x3d6bdaae,
+    0xd5b02117, 0xcfbf208c, 0x2764db35, 0x5e0ea624, 0xb6d55d9d,
+    0x2b16d5c9, 0xc3cd2e70, 0xbaa75361, 0x527ca8d8, 0x4873a943,
+    0xa0a852fa, 0xd9c22feb, 0x3119d452, 0xbbf0874e, 0x532b7cf7,
+    0x2a4101e6, 0xc29afa5f, 0xd895fbc4, 0x304e007d, 0x49247d6c,
+    0xa1ff86d5, 0x3c3c0e81, 0xd4e7f538, 0xad8d8829, 0x45567390,
+    0x5f59720b, 0xb78289b2, 0xcee8f4a3, 0x26330f1a, 0xf46fe50a,
+    0x1cb41eb3, 0x65de63a2, 0x8d05981b, 0x970a9980, 0x7fd16239,
+    0x06bb1f28, 0xee60e491, 0x73a36cc5, 0x9b78977c, 0xe212ea6d,
+    0x0ac911d4, 0x10c6104f, 0xf81debf6, 0x817796e7, 0x69ac6d5e,
+    0x25ce42c6, 0xcd15b97f, 0xb47fc46e, 0x5ca43fd7, 0x46ab3e4c,
+    0xae70c5f5, 0xd71ab8e4, 0x3fc1435d, 0xa202cb09, 0x4ad930b0,
+    0x33b34da1, 0xdb68b618, 0xc167b783, 0x29bc4c3a, 0x50d6312b,
+    0xb80dca92, 0x6a512082, 0x828adb3b, 0xfbe0a62a, 0x133b5d93,
+    0x09345c08, 0xe1efa7b1, 0x9885daa0, 0x705e2119, 0xed9da94d,
+    0x054652f4, 0x7c2c2fe5, 0x94f7d45c, 0x8ef8d5c7, 0x66232e7e,
+    0x1f49536f, 0xf792a8d6, 0xc68b7c84, 0x2e50873d, 0x573afa2c,
+    0xbfe10195, 0xa5ee000e, 0x4d35fbb7, 0x345f86a6, 0xdc847d1f,
+    0x4147f54b, 0xa99c0ef2, 0xd0f673e3, 0x382d885a, 0x222289c1,
+    0xcaf97278, 0xb3930f69, 0x5b48f4d0, 0x89141ec0, 0x61cfe579,
+    0x18a59868, 0xf07e63d1, 0xea71624a, 0x02aa99f3, 0x7bc0e4e2,
+    0x931b1f5b, 0x0ed8970f, 0xe6036cb6, 0x9f6911a7, 0x77b2ea1e,
+    0x6dbdeb85, 0x8566103c, 0xfc0c6d2d, 0x14d79694, 0x58b5b90c,
+    0xb06e42b5, 0xc9043fa4, 0x21dfc41d, 0x3bd0c586, 0xd30b3e3f,
+    0xaa61432e, 0x42bab897, 0xdf7930c3, 0x37a2cb7a, 0x4ec8b66b,
+    0xa6134dd2, 0xbc1c4c49, 0x54c7b7f0, 0x2dadcae1, 0xc5763158,
+    0x172adb48, 0xfff120f1, 0x869b5de0, 0x6e40a659, 0x744fa7c2,
+    0x9c945c7b, 0xe5fe216a, 0x0d25dad3, 0x90e65287, 0x783da93e,
+    0x0157d42f, 0xe98c2f96, 0xf3832e0d, 0x1b58d5b4, 0x6232a8a5,
+    0x8ae9531c},
+   {0x00000000, 0x919168ae, 0x6325a087, 0xf2b4c829, 0x874c31d4,
+    0x16dd597a, 0xe4699153, 0x75f8f9fd, 0x4f9f1373, 0xde0e7bdd,
+    0x2cbab3f4, 0xbd2bdb5a, 0xc8d322a7, 0x59424a09, 0xabf68220,
+    0x3a67ea8e, 0x9e3e27e6, 0x0faf4f48, 0xfd1b8761, 0x6c8aefcf,
+    0x19721632, 0x88e37e9c, 0x7a57b6b5, 0xebc6de1b, 0xd1a13495,
+    0x40305c3b, 0xb2849412, 0x2315fcbc, 0x56ed0541, 0xc77c6def,
+    0x35c8a5c6, 0xa459cd68, 0x7d7b3f17, 0xecea57b9, 0x1e5e9f90,
+    0x8fcff73e, 0xfa370ec3, 0x6ba6666d, 0x9912ae44, 0x0883c6ea,
+    0x32e42c64, 0xa37544ca, 0x51c18ce3, 0xc050e44d, 0xb5a81db0,
+    0x2439751e, 0xd68dbd37, 0x471cd599, 0xe34518f1, 0x72d4705f,
+    0x8060b876, 0x11f1d0d8, 0x64092925, 0xf598418b, 0x072c89a2,
+    0x96bde10c, 0xacda0b82, 0x3d4b632c, 0xcfffab05, 0x5e6ec3ab,
+    0x2b963a56, 0xba0752f8, 0x48b39ad1, 0xd922f27f, 0xfaf67e2e,
+    0x6b671680, 0x99d3dea9, 0x0842b607, 0x7dba4ffa, 0xec2b2754,
+    0x1e9fef7d, 0x8f0e87d3, 0xb5696d5d, 0x24f805f3, 0xd64ccdda,
+    0x47dda574, 0x32255c89, 0xa3b43427, 0x5100fc0e, 0xc09194a0,
+    0x64c859c8, 0xf5593166, 0x07edf94f, 0x967c91e1, 0xe384681c,
+    0x721500b2, 0x80a1c89b, 0x1130a035, 0x2b574abb, 0xbac62215,
+    0x4872ea3c, 0xd9e38292, 0xac1b7b6f, 0x3d8a13c1, 0xcf3edbe8,
+    0x5eafb346, 0x878d4139, 0x161c2997, 0xe4a8e1be, 0x75398910,
+    0x00c170ed, 0x91501843, 0x63e4d06a, 0xf275b8c4, 0xc812524a,
+    0x59833ae4, 0xab37f2cd, 0x3aa69a63, 0x4f5e639e, 0xdecf0b30,
+    0x2c7bc319, 0xbdeaabb7, 0x19b366df, 0x88220e71, 0x7a96c658,
+    0xeb07aef6, 0x9eff570b, 0x0f6e3fa5, 0xfddaf78c, 0x6c4b9f22,
+    0x562c75ac, 0xc7bd1d02, 0x3509d52b, 0xa498bd85, 0xd1604478,
+    0x40f12cd6, 0xb245e4ff, 0x23d48c51, 0xf4edfd5c, 0x657c95f2,
+    0x97c85ddb, 0x06593575, 0x73a1cc88, 0xe230a426, 0x10846c0f,
+    0x811504a1, 0xbb72ee2f, 0x2ae38681, 0xd8574ea8, 0x49c62606,
+    0x3c3edffb, 0xadafb755, 0x5f1b7f7c, 0xce8a17d2, 0x6ad3daba,
+    0xfb42b214, 0x09f67a3d, 0x98671293, 0xed9feb6e, 0x7c0e83c0,
+    0x8eba4be9, 0x1f2b2347, 0x254cc9c9, 0xb4dda167, 0x4669694e,
+    0xd7f801e0, 0xa200f81d, 0x339190b3, 0xc125589a, 0x50b43034,
+    0x8996c24b, 0x1807aae5, 0xeab362cc, 0x7b220a62, 0x0edaf39f,
+    0x9f4b9b31, 0x6dff5318, 0xfc6e3bb6, 0xc609d138, 0x5798b996,
+    0xa52c71bf, 0x34bd1911, 0x4145e0ec, 0xd0d48842, 0x2260406b,
+    0xb3f128c5, 0x17a8e5ad, 0x86398d03, 0x748d452a, 0xe51c2d84,
+    0x90e4d479, 0x0175bcd7, 0xf3c174fe, 0x62501c50, 0x5837f6de,
+    0xc9a69e70, 0x3b125659, 0xaa833ef7, 0xdf7bc70a, 0x4eeaafa4,
+    0xbc5e678d, 0x2dcf0f23, 0x0e1b8372, 0x9f8aebdc, 0x6d3e23f5,
+    0xfcaf4b5b, 0x8957b2a6, 0x18c6da08, 0xea721221, 0x7be37a8f,
+    0x41849001, 0xd015f8af, 0x22a13086, 0xb3305828, 0xc6c8a1d5,
+    0x5759c97b, 0xa5ed0152, 0x347c69fc, 0x9025a494, 0x01b4cc3a,
+    0xf3000413, 0x62916cbd, 0x17699540, 0x86f8fdee, 0x744c35c7,
+    0xe5dd5d69, 0xdfbab7e7, 0x4e2bdf49, 0xbc9f1760, 0x2d0e7fce,
+    0x58f68633, 0xc967ee9d, 0x3bd326b4, 0xaa424e1a, 0x7360bc65,
+    0xe2f1d4cb, 0x10451ce2, 0x81d4744c, 0xf42c8db1, 0x65bde51f,
+    0x97092d36, 0x06984598, 0x3cffaf16, 0xad6ec7b8, 0x5fda0f91,
+    0xce4b673f, 0xbbb39ec2, 0x2a22f66c, 0xd8963e45, 0x490756eb,
+    0xed5e9b83, 0x7ccff32d, 0x8e7b3b04, 0x1fea53aa, 0x6a12aa57,
+    0xfb83c2f9, 0x09370ad0, 0x98a6627e, 0xa2c188f0, 0x3350e05e,
+    0xc1e42877, 0x507540d9, 0x258db924, 0xb41cd18a, 0x46a819a3,
+    0xd739710d}};
+
+#endif
+
+#endif
+
+#if N == 5
+
+#if W == 8
+
+local const z_crc_t FAR crc_braid_table[][256] = {
+   {0x00000000, 0xaf449247, 0x85f822cf, 0x2abcb088, 0xd08143df,
+    0x7fc5d198, 0x55796110, 0xfa3df357, 0x7a7381ff, 0xd53713b8,
+    0xff8ba330, 0x50cf3177, 0xaaf2c220, 0x05b65067, 0x2f0ae0ef,
+    0x804e72a8, 0xf4e703fe, 0x5ba391b9, 0x711f2131, 0xde5bb376,
+    0x24664021, 0x8b22d266, 0xa19e62ee, 0x0edaf0a9, 0x8e948201,
+    0x21d01046, 0x0b6ca0ce, 0xa4283289, 0x5e15c1de, 0xf1515399,
+    0xdbede311, 0x74a97156, 0x32bf01bd, 0x9dfb93fa, 0xb7472372,
+    0x1803b135, 0xe23e4262, 0x4d7ad025, 0x67c660ad, 0xc882f2ea,
+    0x48cc8042, 0xe7881205, 0xcd34a28d, 0x627030ca, 0x984dc39d,
+    0x370951da, 0x1db5e152, 0xb2f17315, 0xc6580243, 0x691c9004,
+    0x43a0208c, 0xece4b2cb, 0x16d9419c, 0xb99dd3db, 0x93216353,
+    0x3c65f114, 0xbc2b83bc, 0x136f11fb, 0x39d3a173, 0x96973334,
+    0x6caac063, 0xc3ee5224, 0xe952e2ac, 0x461670eb, 0x657e037a,
+    0xca3a913d, 0xe08621b5, 0x4fc2b3f2, 0xb5ff40a5, 0x1abbd2e2,
+    0x3007626a, 0x9f43f02d, 0x1f0d8285, 0xb04910c2, 0x9af5a04a,
+    0x35b1320d, 0xcf8cc15a, 0x60c8531d, 0x4a74e395, 0xe53071d2,
+    0x91990084, 0x3edd92c3, 0x1461224b, 0xbb25b00c, 0x4118435b,
+    0xee5cd11c, 0xc4e06194, 0x6ba4f3d3, 0xebea817b, 0x44ae133c,
+    0x6e12a3b4, 0xc15631f3, 0x3b6bc2a4, 0x942f50e3, 0xbe93e06b,
+    0x11d7722c, 0x57c102c7, 0xf8859080, 0xd2392008, 0x7d7db24f,
+    0x87404118, 0x2804d35f, 0x02b863d7, 0xadfcf190, 0x2db28338,
+    0x82f6117f, 0xa84aa1f7, 0x070e33b0, 0xfd33c0e7, 0x527752a0,
+    0x78cbe228, 0xd78f706f, 0xa3260139, 0x0c62937e, 0x26de23f6,
+    0x899ab1b1, 0x73a742e6, 0xdce3d0a1, 0xf65f6029, 0x591bf26e,
+    0xd95580c6, 0x76111281, 0x5cada209, 0xf3e9304e, 0x09d4c319,
+    0xa690515e, 0x8c2ce1d6, 0x23687391, 0xcafc06f4, 0x65b894b3,
+    0x4f04243b, 0xe040b67c, 0x1a7d452b, 0xb539d76c, 0x9f8567e4,
+    0x30c1f5a3, 0xb08f870b, 0x1fcb154c, 0x3577a5c4, 0x9a333783,
+    0x600ec4d4, 0xcf4a5693, 0xe5f6e61b, 0x4ab2745c, 0x3e1b050a,
+    0x915f974d, 0xbbe327c5, 0x14a7b582, 0xee9a46d5, 0x41ded492,
+    0x6b62641a, 0xc426f65d, 0x446884f5, 0xeb2c16b2, 0xc190a63a,
+    0x6ed4347d, 0x94e9c72a, 0x3bad556d, 0x1111e5e5, 0xbe5577a2,
+    0xf8430749, 0x5707950e, 0x7dbb2586, 0xd2ffb7c1, 0x28c24496,
+    0x8786d6d1, 0xad3a6659, 0x027ef41e, 0x823086b6, 0x2d7414f1,
+    0x07c8a479, 0xa88c363e, 0x52b1c569, 0xfdf5572e, 0xd749e7a6,
+    0x780d75e1, 0x0ca404b7, 0xa3e096f0, 0x895c2678, 0x2618b43f,
+    0xdc254768, 0x7361d52f, 0x59dd65a7, 0xf699f7e0, 0x76d78548,
+    0xd993170f, 0xf32fa787, 0x5c6b35c0, 0xa656c697, 0x091254d0,
+    0x23aee458, 0x8cea761f, 0xaf82058e, 0x00c697c9, 0x2a7a2741,
+    0x853eb506, 0x7f034651, 0xd047d416, 0xfafb649e, 0x55bff6d9,
+    0xd5f18471, 0x7ab51636, 0x5009a6be, 0xff4d34f9, 0x0570c7ae,
+    0xaa3455e9, 0x8088e561, 0x2fcc7726, 0x5b650670, 0xf4219437,
+    0xde9d24bf, 0x71d9b6f8, 0x8be445af, 0x24a0d7e8, 0x0e1c6760,
+    0xa158f527, 0x2116878f, 0x8e5215c8, 0xa4eea540, 0x0baa3707,
+    0xf197c450, 0x5ed35617, 0x746fe69f, 0xdb2b74d8, 0x9d3d0433,
+    0x32799674, 0x18c526fc, 0xb781b4bb, 0x4dbc47ec, 0xe2f8d5ab,
+    0xc8446523, 0x6700f764, 0xe74e85cc, 0x480a178b, 0x62b6a703,
+    0xcdf23544, 0x37cfc613, 0x988b5454, 0xb237e4dc, 0x1d73769b,
+    0x69da07cd, 0xc69e958a, 0xec222502, 0x4366b745, 0xb95b4412,
+    0x161fd655, 0x3ca366dd, 0x93e7f49a, 0x13a98632, 0xbced1475,
+    0x9651a4fd, 0x391536ba, 0xc328c5ed, 0x6c6c57aa, 0x46d0e722,
+    0xe9947565},
+   {0x00000000, 0x4e890ba9, 0x9d121752, 0xd39b1cfb, 0xe15528e5,
+    0xafdc234c, 0x7c473fb7, 0x32ce341e, 0x19db578b, 0x57525c22,
+    0x84c940d9, 0xca404b70, 0xf88e7f6e, 0xb60774c7, 0x659c683c,
+    0x2b156395, 0x33b6af16, 0x7d3fa4bf, 0xaea4b844, 0xe02db3ed,
+    0xd2e387f3, 0x9c6a8c5a, 0x4ff190a1, 0x01789b08, 0x2a6df89d,
+    0x64e4f334, 0xb77fefcf, 0xf9f6e466, 0xcb38d078, 0x85b1dbd1,
+    0x562ac72a, 0x18a3cc83, 0x676d5e2c, 0x29e45585, 0xfa7f497e,
+    0xb4f642d7, 0x863876c9, 0xc8b17d60, 0x1b2a619b, 0x55a36a32,
+    0x7eb609a7, 0x303f020e, 0xe3a41ef5, 0xad2d155c, 0x9fe32142,
+    0xd16a2aeb, 0x02f13610, 0x4c783db9, 0x54dbf13a, 0x1a52fa93,
+    0xc9c9e668, 0x8740edc1, 0xb58ed9df, 0xfb07d276, 0x289cce8d,
+    0x6615c524, 0x4d00a6b1, 0x0389ad18, 0xd012b1e3, 0x9e9bba4a,
+    0xac558e54, 0xe2dc85fd, 0x31479906, 0x7fce92af, 0xcedabc58,
+    0x8053b7f1, 0x53c8ab0a, 0x1d41a0a3, 0x2f8f94bd, 0x61069f14,
+    0xb29d83ef, 0xfc148846, 0xd701ebd3, 0x9988e07a, 0x4a13fc81,
+    0x049af728, 0x3654c336, 0x78ddc89f, 0xab46d464, 0xe5cfdfcd,
+    0xfd6c134e, 0xb3e518e7, 0x607e041c, 0x2ef70fb5, 0x1c393bab,
+    0x52b03002, 0x812b2cf9, 0xcfa22750, 0xe4b744c5, 0xaa3e4f6c,
+    0x79a55397, 0x372c583e, 0x05e26c20, 0x4b6b6789, 0x98f07b72,
+    0xd67970db, 0xa9b7e274, 0xe73ee9dd, 0x34a5f526, 0x7a2cfe8f,
+    0x48e2ca91, 0x066bc138, 0xd5f0ddc3, 0x9b79d66a, 0xb06cb5ff,
+    0xfee5be56, 0x2d7ea2ad, 0x63f7a904, 0x51399d1a, 0x1fb096b3,
+    0xcc2b8a48, 0x82a281e1, 0x9a014d62, 0xd48846cb, 0x07135a30,
+    0x499a5199, 0x7b546587, 0x35dd6e2e, 0xe64672d5, 0xa8cf797c,
+    0x83da1ae9, 0xcd531140, 0x1ec80dbb, 0x50410612, 0x628f320c,
+    0x2c0639a5, 0xff9d255e, 0xb1142ef7, 0x46c47ef1, 0x084d7558,
+    0xdbd669a3, 0x955f620a, 0xa7915614, 0xe9185dbd, 0x3a834146,
+    0x740a4aef, 0x5f1f297a, 0x119622d3, 0xc20d3e28, 0x8c843581,
+    0xbe4a019f, 0xf0c30a36, 0x235816cd, 0x6dd11d64, 0x7572d1e7,
+    0x3bfbda4e, 0xe860c6b5, 0xa6e9cd1c, 0x9427f902, 0xdaaef2ab,
+    0x0935ee50, 0x47bce5f9, 0x6ca9866c, 0x22208dc5, 0xf1bb913e,
+    0xbf329a97, 0x8dfcae89, 0xc375a520, 0x10eeb9db, 0x5e67b272,
+    0x21a920dd, 0x6f202b74, 0xbcbb378f, 0xf2323c26, 0xc0fc0838,
+    0x8e750391, 0x5dee1f6a, 0x136714c3, 0x38727756, 0x76fb7cff,
+    0xa5606004, 0xebe96bad, 0xd9275fb3, 0x97ae541a, 0x443548e1,
+    0x0abc4348, 0x121f8fcb, 0x5c968462, 0x8f0d9899, 0xc1849330,
+    0xf34aa72e, 0xbdc3ac87, 0x6e58b07c, 0x20d1bbd5, 0x0bc4d840,
+    0x454dd3e9, 0x96d6cf12, 0xd85fc4bb, 0xea91f0a5, 0xa418fb0c,
+    0x7783e7f7, 0x390aec5e, 0x881ec2a9, 0xc697c900, 0x150cd5fb,
+    0x5b85de52, 0x694bea4c, 0x27c2e1e5, 0xf459fd1e, 0xbad0f6b7,
+    0x91c59522, 0xdf4c9e8b, 0x0cd78270, 0x425e89d9, 0x7090bdc7,
+    0x3e19b66e, 0xed82aa95, 0xa30ba13c, 0xbba86dbf, 0xf5216616,
+    0x26ba7aed, 0x68337144, 0x5afd455a, 0x14744ef3, 0xc7ef5208,
+    0x896659a1, 0xa2733a34, 0xecfa319d, 0x3f612d66, 0x71e826cf,
+    0x432612d1, 0x0daf1978, 0xde340583, 0x90bd0e2a, 0xef739c85,
+    0xa1fa972c, 0x72618bd7, 0x3ce8807e, 0x0e26b460, 0x40afbfc9,
+    0x9334a332, 0xddbda89b, 0xf6a8cb0e, 0xb821c0a7, 0x6bbadc5c,
+    0x2533d7f5, 0x17fde3eb, 0x5974e842, 0x8aeff4b9, 0xc466ff10,
+    0xdcc53393, 0x924c383a, 0x41d724c1, 0x0f5e2f68, 0x3d901b76,
+    0x731910df, 0xa0820c24, 0xee0b078d, 0xc51e6418, 0x8b976fb1,
+    0x580c734a, 0x168578e3, 0x244b4cfd, 0x6ac24754, 0xb9595baf,
+    0xf7d05006},
+   {0x00000000, 0x8d88fde2, 0xc060fd85, 0x4de80067, 0x5bb0fd4b,
+    0xd63800a9, 0x9bd000ce, 0x1658fd2c, 0xb761fa96, 0x3ae90774,
+    0x77010713, 0xfa89faf1, 0xecd107dd, 0x6159fa3f, 0x2cb1fa58,
+    0xa13907ba, 0xb5b2f36d, 0x383a0e8f, 0x75d20ee8, 0xf85af30a,
+    0xee020e26, 0x638af3c4, 0x2e62f3a3, 0xa3ea0e41, 0x02d309fb,
+    0x8f5bf419, 0xc2b3f47e, 0x4f3b099c, 0x5963f4b0, 0xd4eb0952,
+    0x99030935, 0x148bf4d7, 0xb014e09b, 0x3d9c1d79, 0x70741d1e,
+    0xfdfce0fc, 0xeba41dd0, 0x662ce032, 0x2bc4e055, 0xa64c1db7,
+    0x07751a0d, 0x8afde7ef, 0xc715e788, 0x4a9d1a6a, 0x5cc5e746,
+    0xd14d1aa4, 0x9ca51ac3, 0x112de721, 0x05a613f6, 0x882eee14,
+    0xc5c6ee73, 0x484e1391, 0x5e16eebd, 0xd39e135f, 0x9e761338,
+    0x13feeeda, 0xb2c7e960, 0x3f4f1482, 0x72a714e5, 0xff2fe907,
+    0xe977142b, 0x64ffe9c9, 0x2917e9ae, 0xa49f144c, 0xbb58c777,
+    0x36d03a95, 0x7b383af2, 0xf6b0c710, 0xe0e83a3c, 0x6d60c7de,
+    0x2088c7b9, 0xad003a5b, 0x0c393de1, 0x81b1c003, 0xcc59c064,
+    0x41d13d86, 0x5789c0aa, 0xda013d48, 0x97e93d2f, 0x1a61c0cd,
+    0x0eea341a, 0x8362c9f8, 0xce8ac99f, 0x4302347d, 0x555ac951,
+    0xd8d234b3, 0x953a34d4, 0x18b2c936, 0xb98bce8c, 0x3403336e,
+    0x79eb3309, 0xf463ceeb, 0xe23b33c7, 0x6fb3ce25, 0x225bce42,
+    0xafd333a0, 0x0b4c27ec, 0x86c4da0e, 0xcb2cda69, 0x46a4278b,
+    0x50fcdaa7, 0xdd742745, 0x909c2722, 0x1d14dac0, 0xbc2ddd7a,
+    0x31a52098, 0x7c4d20ff, 0xf1c5dd1d, 0xe79d2031, 0x6a15ddd3,
+    0x27fdddb4, 0xaa752056, 0xbefed481, 0x33762963, 0x7e9e2904,
+    0xf316d4e6, 0xe54e29ca, 0x68c6d428, 0x252ed44f, 0xa8a629ad,
+    0x099f2e17, 0x8417d3f5, 0xc9ffd392, 0x44772e70, 0x522fd35c,
+    0xdfa72ebe, 0x924f2ed9, 0x1fc7d33b, 0xadc088af, 0x2048754d,
+    0x6da0752a, 0xe02888c8, 0xf67075e4, 0x7bf88806, 0x36108861,
+    0xbb987583, 0x1aa17239, 0x97298fdb, 0xdac18fbc, 0x5749725e,
+    0x41118f72, 0xcc997290, 0x817172f7, 0x0cf98f15, 0x18727bc2,
+    0x95fa8620, 0xd8128647, 0x559a7ba5, 0x43c28689, 0xce4a7b6b,
+    0x83a27b0c, 0x0e2a86ee, 0xaf138154, 0x229b7cb6, 0x6f737cd1,
+    0xe2fb8133, 0xf4a37c1f, 0x792b81fd, 0x34c3819a, 0xb94b7c78,
+    0x1dd46834, 0x905c95d6, 0xddb495b1, 0x503c6853, 0x4664957f,
+    0xcbec689d, 0x860468fa, 0x0b8c9518, 0xaab592a2, 0x273d6f40,
+    0x6ad56f27, 0xe75d92c5, 0xf1056fe9, 0x7c8d920b, 0x3165926c,
+    0xbced6f8e, 0xa8669b59, 0x25ee66bb, 0x680666dc, 0xe58e9b3e,
+    0xf3d66612, 0x7e5e9bf0, 0x33b69b97, 0xbe3e6675, 0x1f0761cf,
+    0x928f9c2d, 0xdf679c4a, 0x52ef61a8, 0x44b79c84, 0xc93f6166,
+    0x84d76101, 0x095f9ce3, 0x16984fd8, 0x9b10b23a, 0xd6f8b25d,
+    0x5b704fbf, 0x4d28b293, 0xc0a04f71, 0x8d484f16, 0x00c0b2f4,
+    0xa1f9b54e, 0x2c7148ac, 0x619948cb, 0xec11b529, 0xfa494805,
+    0x77c1b5e7, 0x3a29b580, 0xb7a14862, 0xa32abcb5, 0x2ea24157,
+    0x634a4130, 0xeec2bcd2, 0xf89a41fe, 0x7512bc1c, 0x38fabc7b,
+    0xb5724199, 0x144b4623, 0x99c3bbc1, 0xd42bbba6, 0x59a34644,
+    0x4ffbbb68, 0xc273468a, 0x8f9b46ed, 0x0213bb0f, 0xa68caf43,
+    0x2b0452a1, 0x66ec52c6, 0xeb64af24, 0xfd3c5208, 0x70b4afea,
+    0x3d5caf8d, 0xb0d4526f, 0x11ed55d5, 0x9c65a837, 0xd18da850,
+    0x5c0555b2, 0x4a5da89e, 0xc7d5557c, 0x8a3d551b, 0x07b5a8f9,
+    0x133e5c2e, 0x9eb6a1cc, 0xd35ea1ab, 0x5ed65c49, 0x488ea165,
+    0xc5065c87, 0x88ee5ce0, 0x0566a102, 0xa45fa6b8, 0x29d75b5a,
+    0x643f5b3d, 0xe9b7a6df, 0xffef5bf3, 0x7267a611, 0x3f8fa676,
+    0xb2075b94},
+   {0x00000000, 0x80f0171f, 0xda91287f, 0x5a613f60, 0x6e5356bf,
+    0xeea341a0, 0xb4c27ec0, 0x343269df, 0xdca6ad7e, 0x5c56ba61,
+    0x06378501, 0x86c7921e, 0xb2f5fbc1, 0x3205ecde, 0x6864d3be,
+    0xe894c4a1, 0x623c5cbd, 0xe2cc4ba2, 0xb8ad74c2, 0x385d63dd,
+    0x0c6f0a02, 0x8c9f1d1d, 0xd6fe227d, 0x560e3562, 0xbe9af1c3,
+    0x3e6ae6dc, 0x640bd9bc, 0xe4fbcea3, 0xd0c9a77c, 0x5039b063,
+    0x0a588f03, 0x8aa8981c, 0xc478b97a, 0x4488ae65, 0x1ee99105,
+    0x9e19861a, 0xaa2befc5, 0x2adbf8da, 0x70bac7ba, 0xf04ad0a5,
+    0x18de1404, 0x982e031b, 0xc24f3c7b, 0x42bf2b64, 0x768d42bb,
+    0xf67d55a4, 0xac1c6ac4, 0x2cec7ddb, 0xa644e5c7, 0x26b4f2d8,
+    0x7cd5cdb8, 0xfc25daa7, 0xc817b378, 0x48e7a467, 0x12869b07,
+    0x92768c18, 0x7ae248b9, 0xfa125fa6, 0xa07360c6, 0x208377d9,
+    0x14b11e06, 0x94410919, 0xce203679, 0x4ed02166, 0x538074b5,
+    0xd37063aa, 0x89115cca, 0x09e14bd5, 0x3dd3220a, 0xbd233515,
+    0xe7420a75, 0x67b21d6a, 0x8f26d9cb, 0x0fd6ced4, 0x55b7f1b4,
+    0xd547e6ab, 0xe1758f74, 0x6185986b, 0x3be4a70b, 0xbb14b014,
+    0x31bc2808, 0xb14c3f17, 0xeb2d0077, 0x6bdd1768, 0x5fef7eb7,
+    0xdf1f69a8, 0x857e56c8, 0x058e41d7, 0xed1a8576, 0x6dea9269,
+    0x378bad09, 0xb77bba16, 0x8349d3c9, 0x03b9c4d6, 0x59d8fbb6,
+    0xd928eca9, 0x97f8cdcf, 0x1708dad0, 0x4d69e5b0, 0xcd99f2af,
+    0xf9ab9b70, 0x795b8c6f, 0x233ab30f, 0xa3caa410, 0x4b5e60b1,
+    0xcbae77ae, 0x91cf48ce, 0x113f5fd1, 0x250d360e, 0xa5fd2111,
+    0xff9c1e71, 0x7f6c096e, 0xf5c49172, 0x7534866d, 0x2f55b90d,
+    0xafa5ae12, 0x9b97c7cd, 0x1b67d0d2, 0x4106efb2, 0xc1f6f8ad,
+    0x29623c0c, 0xa9922b13, 0xf3f31473, 0x7303036c, 0x47316ab3,
+    0xc7c17dac, 0x9da042cc, 0x1d5055d3, 0xa700e96a, 0x27f0fe75,
+    0x7d91c115, 0xfd61d60a, 0xc953bfd5, 0x49a3a8ca, 0x13c297aa,
+    0x933280b5, 0x7ba64414, 0xfb56530b, 0xa1376c6b, 0x21c77b74,
+    0x15f512ab, 0x950505b4, 0xcf643ad4, 0x4f942dcb, 0xc53cb5d7,
+    0x45cca2c8, 0x1fad9da8, 0x9f5d8ab7, 0xab6fe368, 0x2b9ff477,
+    0x71fecb17, 0xf10edc08, 0x199a18a9, 0x996a0fb6, 0xc30b30d6,
+    0x43fb27c9, 0x77c94e16, 0xf7395909, 0xad586669, 0x2da87176,
+    0x63785010, 0xe388470f, 0xb9e9786f, 0x39196f70, 0x0d2b06af,
+    0x8ddb11b0, 0xd7ba2ed0, 0x574a39cf, 0xbfdefd6e, 0x3f2eea71,
+    0x654fd511, 0xe5bfc20e, 0xd18dabd1, 0x517dbcce, 0x0b1c83ae,
+    0x8bec94b1, 0x01440cad, 0x81b41bb2, 0xdbd524d2, 0x5b2533cd,
+    0x6f175a12, 0xefe74d0d, 0xb586726d, 0x35766572, 0xdde2a1d3,
+    0x5d12b6cc, 0x077389ac, 0x87839eb3, 0xb3b1f76c, 0x3341e073,
+    0x6920df13, 0xe9d0c80c, 0xf4809ddf, 0x74708ac0, 0x2e11b5a0,
+    0xaee1a2bf, 0x9ad3cb60, 0x1a23dc7f, 0x4042e31f, 0xc0b2f400,
+    0x282630a1, 0xa8d627be, 0xf2b718de, 0x72470fc1, 0x4675661e,
+    0xc6857101, 0x9ce44e61, 0x1c14597e, 0x96bcc162, 0x164cd67d,
+    0x4c2de91d, 0xccddfe02, 0xf8ef97dd, 0x781f80c2, 0x227ebfa2,
+    0xa28ea8bd, 0x4a1a6c1c, 0xcaea7b03, 0x908b4463, 0x107b537c,
+    0x24493aa3, 0xa4b92dbc, 0xfed812dc, 0x7e2805c3, 0x30f824a5,
+    0xb00833ba, 0xea690cda, 0x6a991bc5, 0x5eab721a, 0xde5b6505,
+    0x843a5a65, 0x04ca4d7a, 0xec5e89db, 0x6cae9ec4, 0x36cfa1a4,
+    0xb63fb6bb, 0x820ddf64, 0x02fdc87b, 0x589cf71b, 0xd86ce004,
+    0x52c47818, 0xd2346f07, 0x88555067, 0x08a54778, 0x3c972ea7,
+    0xbc6739b8, 0xe60606d8, 0x66f611c7, 0x8e62d566, 0x0e92c279,
+    0x54f3fd19, 0xd403ea06, 0xe03183d9, 0x60c194c6, 0x3aa0aba6,
+    0xba50bcb9},
+   {0x00000000, 0x9570d495, 0xf190af6b, 0x64e07bfe, 0x38505897,
+    0xad208c02, 0xc9c0f7fc, 0x5cb02369, 0x70a0b12e, 0xe5d065bb,
+    0x81301e45, 0x1440cad0, 0x48f0e9b9, 0xdd803d2c, 0xb96046d2,
+    0x2c109247, 0xe141625c, 0x7431b6c9, 0x10d1cd37, 0x85a119a2,
+    0xd9113acb, 0x4c61ee5e, 0x288195a0, 0xbdf14135, 0x91e1d372,
+    0x049107e7, 0x60717c19, 0xf501a88c, 0xa9b18be5, 0x3cc15f70,
+    0x5821248e, 0xcd51f01b, 0x19f3c2f9, 0x8c83166c, 0xe8636d92,
+    0x7d13b907, 0x21a39a6e, 0xb4d34efb, 0xd0333505, 0x4543e190,
+    0x695373d7, 0xfc23a742, 0x98c3dcbc, 0x0db30829, 0x51032b40,
+    0xc473ffd5, 0xa093842b, 0x35e350be, 0xf8b2a0a5, 0x6dc27430,
+    0x09220fce, 0x9c52db5b, 0xc0e2f832, 0x55922ca7, 0x31725759,
+    0xa40283cc, 0x8812118b, 0x1d62c51e, 0x7982bee0, 0xecf26a75,
+    0xb042491c, 0x25329d89, 0x41d2e677, 0xd4a232e2, 0x33e785f2,
+    0xa6975167, 0xc2772a99, 0x5707fe0c, 0x0bb7dd65, 0x9ec709f0,
+    0xfa27720e, 0x6f57a69b, 0x434734dc, 0xd637e049, 0xb2d79bb7,
+    0x27a74f22, 0x7b176c4b, 0xee67b8de, 0x8a87c320, 0x1ff717b5,
+    0xd2a6e7ae, 0x47d6333b, 0x233648c5, 0xb6469c50, 0xeaf6bf39,
+    0x7f866bac, 0x1b661052, 0x8e16c4c7, 0xa2065680, 0x37768215,
+    0x5396f9eb, 0xc6e62d7e, 0x9a560e17, 0x0f26da82, 0x6bc6a17c,
+    0xfeb675e9, 0x2a14470b, 0xbf64939e, 0xdb84e860, 0x4ef43cf5,
+    0x12441f9c, 0x8734cb09, 0xe3d4b0f7, 0x76a46462, 0x5ab4f625,
+    0xcfc422b0, 0xab24594e, 0x3e548ddb, 0x62e4aeb2, 0xf7947a27,
+    0x937401d9, 0x0604d54c, 0xcb552557, 0x5e25f1c2, 0x3ac58a3c,
+    0xafb55ea9, 0xf3057dc0, 0x6675a955, 0x0295d2ab, 0x97e5063e,
+    0xbbf59479, 0x2e8540ec, 0x4a653b12, 0xdf15ef87, 0x83a5ccee,
+    0x16d5187b, 0x72356385, 0xe745b710, 0x67cf0be4, 0xf2bfdf71,
+    0x965fa48f, 0x032f701a, 0x5f9f5373, 0xcaef87e6, 0xae0ffc18,
+    0x3b7f288d, 0x176fbaca, 0x821f6e5f, 0xe6ff15a1, 0x738fc134,
+    0x2f3fe25d, 0xba4f36c8, 0xdeaf4d36, 0x4bdf99a3, 0x868e69b8,
+    0x13febd2d, 0x771ec6d3, 0xe26e1246, 0xbede312f, 0x2baee5ba,
+    0x4f4e9e44, 0xda3e4ad1, 0xf62ed896, 0x635e0c03, 0x07be77fd,
+    0x92cea368, 0xce7e8001, 0x5b0e5494, 0x3fee2f6a, 0xaa9efbff,
+    0x7e3cc91d, 0xeb4c1d88, 0x8fac6676, 0x1adcb2e3, 0x466c918a,
+    0xd31c451f, 0xb7fc3ee1, 0x228cea74, 0x0e9c7833, 0x9becaca6,
+    0xff0cd758, 0x6a7c03cd, 0x36cc20a4, 0xa3bcf431, 0xc75c8fcf,
+    0x522c5b5a, 0x9f7dab41, 0x0a0d7fd4, 0x6eed042a, 0xfb9dd0bf,
+    0xa72df3d6, 0x325d2743, 0x56bd5cbd, 0xc3cd8828, 0xefdd1a6f,
+    0x7aadcefa, 0x1e4db504, 0x8b3d6191, 0xd78d42f8, 0x42fd966d,
+    0x261ded93, 0xb36d3906, 0x54288e16, 0xc1585a83, 0xa5b8217d,
+    0x30c8f5e8, 0x6c78d681, 0xf9080214, 0x9de879ea, 0x0898ad7f,
+    0x24883f38, 0xb1f8ebad, 0xd5189053, 0x406844c6, 0x1cd867af,
+    0x89a8b33a, 0xed48c8c4, 0x78381c51, 0xb569ec4a, 0x201938df,
+    0x44f94321, 0xd18997b4, 0x8d39b4dd, 0x18496048, 0x7ca91bb6,
+    0xe9d9cf23, 0xc5c95d64, 0x50b989f1, 0x3459f20f, 0xa129269a,
+    0xfd9905f3, 0x68e9d166, 0x0c09aa98, 0x99797e0d, 0x4ddb4cef,
+    0xd8ab987a, 0xbc4be384, 0x293b3711, 0x758b1478, 0xe0fbc0ed,
+    0x841bbb13, 0x116b6f86, 0x3d7bfdc1, 0xa80b2954, 0xcceb52aa,
+    0x599b863f, 0x052ba556, 0x905b71c3, 0xf4bb0a3d, 0x61cbdea8,
+    0xac9a2eb3, 0x39eafa26, 0x5d0a81d8, 0xc87a554d, 0x94ca7624,
+    0x01baa2b1, 0x655ad94f, 0xf02a0dda, 0xdc3a9f9d, 0x494a4b08,
+    0x2daa30f6, 0xb8dae463, 0xe46ac70a, 0x711a139f, 0x15fa6861,
+    0x808abcf4},
+   {0x00000000, 0xcf9e17c8, 0x444d29d1, 0x8bd33e19, 0x889a53a2,
+    0x4704446a, 0xccd77a73, 0x03496dbb, 0xca45a105, 0x05dbb6cd,
+    0x8e0888d4, 0x41969f1c, 0x42dff2a7, 0x8d41e56f, 0x0692db76,
+    0xc90cccbe, 0x4ffa444b, 0x80645383, 0x0bb76d9a, 0xc4297a52,
+    0xc76017e9, 0x08fe0021, 0x832d3e38, 0x4cb329f0, 0x85bfe54e,
+    0x4a21f286, 0xc1f2cc9f, 0x0e6cdb57, 0x0d25b6ec, 0xc2bba124,
+    0x49689f3d, 0x86f688f5, 0x9ff48896, 0x506a9f5e, 0xdbb9a147,
+    0x1427b68f, 0x176edb34, 0xd8f0ccfc, 0x5323f2e5, 0x9cbde52d,
+    0x55b12993, 0x9a2f3e5b, 0x11fc0042, 0xde62178a, 0xdd2b7a31,
+    0x12b56df9, 0x996653e0, 0x56f84428, 0xd00eccdd, 0x1f90db15,
+    0x9443e50c, 0x5bddf2c4, 0x58949f7f, 0x970a88b7, 0x1cd9b6ae,
+    0xd347a166, 0x1a4b6dd8, 0xd5d57a10, 0x5e064409, 0x919853c1,
+    0x92d13e7a, 0x5d4f29b2, 0xd69c17ab, 0x19020063, 0xe498176d,
+    0x2b0600a5, 0xa0d53ebc, 0x6f4b2974, 0x6c0244cf, 0xa39c5307,
+    0x284f6d1e, 0xe7d17ad6, 0x2eddb668, 0xe143a1a0, 0x6a909fb9,
+    0xa50e8871, 0xa647e5ca, 0x69d9f202, 0xe20acc1b, 0x2d94dbd3,
+    0xab625326, 0x64fc44ee, 0xef2f7af7, 0x20b16d3f, 0x23f80084,
+    0xec66174c, 0x67b52955, 0xa82b3e9d, 0x6127f223, 0xaeb9e5eb,
+    0x256adbf2, 0xeaf4cc3a, 0xe9bda181, 0x2623b649, 0xadf08850,
+    0x626e9f98, 0x7b6c9ffb, 0xb4f28833, 0x3f21b62a, 0xf0bfa1e2,
+    0xf3f6cc59, 0x3c68db91, 0xb7bbe588, 0x7825f240, 0xb1293efe,
+    0x7eb72936, 0xf564172f, 0x3afa00e7, 0x39b36d5c, 0xf62d7a94,
+    0x7dfe448d, 0xb2605345, 0x3496dbb0, 0xfb08cc78, 0x70dbf261,
+    0xbf45e5a9, 0xbc0c8812, 0x73929fda, 0xf841a1c3, 0x37dfb60b,
+    0xfed37ab5, 0x314d6d7d, 0xba9e5364, 0x750044ac, 0x76492917,
+    0xb9d73edf, 0x320400c6, 0xfd9a170e, 0x1241289b, 0xdddf3f53,
+    0x560c014a, 0x99921682, 0x9adb7b39, 0x55456cf1, 0xde9652e8,
+    0x11084520, 0xd804899e, 0x179a9e56, 0x9c49a04f, 0x53d7b787,
+    0x509eda3c, 0x9f00cdf4, 0x14d3f3ed, 0xdb4de425, 0x5dbb6cd0,
+    0x92257b18, 0x19f64501, 0xd66852c9, 0xd5213f72, 0x1abf28ba,
+    0x916c16a3, 0x5ef2016b, 0x97fecdd5, 0x5860da1d, 0xd3b3e404,
+    0x1c2df3cc, 0x1f649e77, 0xd0fa89bf, 0x5b29b7a6, 0x94b7a06e,
+    0x8db5a00d, 0x422bb7c5, 0xc9f889dc, 0x06669e14, 0x052ff3af,
+    0xcab1e467, 0x4162da7e, 0x8efccdb6, 0x47f00108, 0x886e16c0,
+    0x03bd28d9, 0xcc233f11, 0xcf6a52aa, 0x00f44562, 0x8b277b7b,
+    0x44b96cb3, 0xc24fe446, 0x0dd1f38e, 0x8602cd97, 0x499cda5f,
+    0x4ad5b7e4, 0x854ba02c, 0x0e989e35, 0xc10689fd, 0x080a4543,
+    0xc794528b, 0x4c476c92, 0x83d97b5a, 0x809016e1, 0x4f0e0129,
+    0xc4dd3f30, 0x0b4328f8, 0xf6d93ff6, 0x3947283e, 0xb2941627,
+    0x7d0a01ef, 0x7e436c54, 0xb1dd7b9c, 0x3a0e4585, 0xf590524d,
+    0x3c9c9ef3, 0xf302893b, 0x78d1b722, 0xb74fa0ea, 0xb406cd51,
+    0x7b98da99, 0xf04be480, 0x3fd5f348, 0xb9237bbd, 0x76bd6c75,
+    0xfd6e526c, 0x32f045a4, 0x31b9281f, 0xfe273fd7, 0x75f401ce,
+    0xba6a1606, 0x7366dab8, 0xbcf8cd70, 0x372bf369, 0xf8b5e4a1,
+    0xfbfc891a, 0x34629ed2, 0xbfb1a0cb, 0x702fb703, 0x692db760,
+    0xa6b3a0a8, 0x2d609eb1, 0xe2fe8979, 0xe1b7e4c2, 0x2e29f30a,
+    0xa5facd13, 0x6a64dadb, 0xa3681665, 0x6cf601ad, 0xe7253fb4,
+    0x28bb287c, 0x2bf245c7, 0xe46c520f, 0x6fbf6c16, 0xa0217bde,
+    0x26d7f32b, 0xe949e4e3, 0x629adafa, 0xad04cd32, 0xae4da089,
+    0x61d3b741, 0xea008958, 0x259e9e90, 0xec92522e, 0x230c45e6,
+    0xa8df7bff, 0x67416c37, 0x6408018c, 0xab961644, 0x2045285d,
+    0xefdb3f95},
+   {0x00000000, 0x24825136, 0x4904a26c, 0x6d86f35a, 0x920944d8,
+    0xb68b15ee, 0xdb0de6b4, 0xff8fb782, 0xff638ff1, 0xdbe1dec7,
+    0xb6672d9d, 0x92e57cab, 0x6d6acb29, 0x49e89a1f, 0x246e6945,
+    0x00ec3873, 0x25b619a3, 0x01344895, 0x6cb2bbcf, 0x4830eaf9,
+    0xb7bf5d7b, 0x933d0c4d, 0xfebbff17, 0xda39ae21, 0xdad59652,
+    0xfe57c764, 0x93d1343e, 0xb7536508, 0x48dcd28a, 0x6c5e83bc,
+    0x01d870e6, 0x255a21d0, 0x4b6c3346, 0x6fee6270, 0x0268912a,
+    0x26eac01c, 0xd965779e, 0xfde726a8, 0x9061d5f2, 0xb4e384c4,
+    0xb40fbcb7, 0x908ded81, 0xfd0b1edb, 0xd9894fed, 0x2606f86f,
+    0x0284a959, 0x6f025a03, 0x4b800b35, 0x6eda2ae5, 0x4a587bd3,
+    0x27de8889, 0x035cd9bf, 0xfcd36e3d, 0xd8513f0b, 0xb5d7cc51,
+    0x91559d67, 0x91b9a514, 0xb53bf422, 0xd8bd0778, 0xfc3f564e,
+    0x03b0e1cc, 0x2732b0fa, 0x4ab443a0, 0x6e361296, 0x96d8668c,
+    0xb25a37ba, 0xdfdcc4e0, 0xfb5e95d6, 0x04d12254, 0x20537362,
+    0x4dd58038, 0x6957d10e, 0x69bbe97d, 0x4d39b84b, 0x20bf4b11,
+    0x043d1a27, 0xfbb2ada5, 0xdf30fc93, 0xb2b60fc9, 0x96345eff,
+    0xb36e7f2f, 0x97ec2e19, 0xfa6add43, 0xdee88c75, 0x21673bf7,
+    0x05e56ac1, 0x6863999b, 0x4ce1c8ad, 0x4c0df0de, 0x688fa1e8,
+    0x050952b2, 0x218b0384, 0xde04b406, 0xfa86e530, 0x9700166a,
+    0xb382475c, 0xddb455ca, 0xf93604fc, 0x94b0f7a6, 0xb032a690,
+    0x4fbd1112, 0x6b3f4024, 0x06b9b37e, 0x223be248, 0x22d7da3b,
+    0x06558b0d, 0x6bd37857, 0x4f512961, 0xb0de9ee3, 0x945ccfd5,
+    0xf9da3c8f, 0xdd586db9, 0xf8024c69, 0xdc801d5f, 0xb106ee05,
+    0x9584bf33, 0x6a0b08b1, 0x4e895987, 0x230faadd, 0x078dfbeb,
+    0x0761c398, 0x23e392ae, 0x4e6561f4, 0x6ae730c2, 0x95688740,
+    0xb1ead676, 0xdc6c252c, 0xf8ee741a, 0xf6c1cb59, 0xd2439a6f,
+    0xbfc56935, 0x9b473803, 0x64c88f81, 0x404adeb7, 0x2dcc2ded,
+    0x094e7cdb, 0x09a244a8, 0x2d20159e, 0x40a6e6c4, 0x6424b7f2,
+    0x9bab0070, 0xbf295146, 0xd2afa21c, 0xf62df32a, 0xd377d2fa,
+    0xf7f583cc, 0x9a737096, 0xbef121a0, 0x417e9622, 0x65fcc714,
+    0x087a344e, 0x2cf86578, 0x2c145d0b, 0x08960c3d, 0x6510ff67,
+    0x4192ae51, 0xbe1d19d3, 0x9a9f48e5, 0xf719bbbf, 0xd39bea89,
+    0xbdadf81f, 0x992fa929, 0xf4a95a73, 0xd02b0b45, 0x2fa4bcc7,
+    0x0b26edf1, 0x66a01eab, 0x42224f9d, 0x42ce77ee, 0x664c26d8,
+    0x0bcad582, 0x2f4884b4, 0xd0c73336, 0xf4456200, 0x99c3915a,
+    0xbd41c06c, 0x981be1bc, 0xbc99b08a, 0xd11f43d0, 0xf59d12e6,
+    0x0a12a564, 0x2e90f452, 0x43160708, 0x6794563e, 0x67786e4d,
+    0x43fa3f7b, 0x2e7ccc21, 0x0afe9d17, 0xf5712a95, 0xd1f37ba3,
+    0xbc7588f9, 0x98f7d9cf, 0x6019add5, 0x449bfce3, 0x291d0fb9,
+    0x0d9f5e8f, 0xf210e90d, 0xd692b83b, 0xbb144b61, 0x9f961a57,
+    0x9f7a2224, 0xbbf87312, 0xd67e8048, 0xf2fcd17e, 0x0d7366fc,
+    0x29f137ca, 0x4477c490, 0x60f595a6, 0x45afb476, 0x612de540,
+    0x0cab161a, 0x2829472c, 0xd7a6f0ae, 0xf324a198, 0x9ea252c2,
+    0xba2003f4, 0xbacc3b87, 0x9e4e6ab1, 0xf3c899eb, 0xd74ac8dd,
+    0x28c57f5f, 0x0c472e69, 0x61c1dd33, 0x45438c05, 0x2b759e93,
+    0x0ff7cfa5, 0x62713cff, 0x46f36dc9, 0xb97cda4b, 0x9dfe8b7d,
+    0xf0787827, 0xd4fa2911, 0xd4161162, 0xf0944054, 0x9d12b30e,
+    0xb990e238, 0x461f55ba, 0x629d048c, 0x0f1bf7d6, 0x2b99a6e0,
+    0x0ec38730, 0x2a41d606, 0x47c7255c, 0x6345746a, 0x9ccac3e8,
+    0xb84892de, 0xd5ce6184, 0xf14c30b2, 0xf1a008c1, 0xd52259f7,
+    0xb8a4aaad, 0x9c26fb9b, 0x63a94c19, 0x472b1d2f, 0x2aadee75,
+    0x0e2fbf43},
+   {0x00000000, 0x36f290f3, 0x6de521e6, 0x5b17b115, 0xdbca43cc,
+    0xed38d33f, 0xb62f622a, 0x80ddf2d9, 0x6ce581d9, 0x5a17112a,
+    0x0100a03f, 0x37f230cc, 0xb72fc215, 0x81dd52e6, 0xdacae3f3,
+    0xec387300, 0xd9cb03b2, 0xef399341, 0xb42e2254, 0x82dcb2a7,
+    0x0201407e, 0x34f3d08d, 0x6fe46198, 0x5916f16b, 0xb52e826b,
+    0x83dc1298, 0xd8cba38d, 0xee39337e, 0x6ee4c1a7, 0x58165154,
+    0x0301e041, 0x35f370b2, 0x68e70125, 0x5e1591d6, 0x050220c3,
+    0x33f0b030, 0xb32d42e9, 0x85dfd21a, 0xdec8630f, 0xe83af3fc,
+    0x040280fc, 0x32f0100f, 0x69e7a11a, 0x5f1531e9, 0xdfc8c330,
+    0xe93a53c3, 0xb22de2d6, 0x84df7225, 0xb12c0297, 0x87de9264,
+    0xdcc92371, 0xea3bb382, 0x6ae6415b, 0x5c14d1a8, 0x070360bd,
+    0x31f1f04e, 0xddc9834e, 0xeb3b13bd, 0xb02ca2a8, 0x86de325b,
+    0x0603c082, 0x30f15071, 0x6be6e164, 0x5d147197, 0xd1ce024a,
+    0xe73c92b9, 0xbc2b23ac, 0x8ad9b35f, 0x0a044186, 0x3cf6d175,
+    0x67e16060, 0x5113f093, 0xbd2b8393, 0x8bd91360, 0xd0cea275,
+    0xe63c3286, 0x66e1c05f, 0x501350ac, 0x0b04e1b9, 0x3df6714a,
+    0x080501f8, 0x3ef7910b, 0x65e0201e, 0x5312b0ed, 0xd3cf4234,
+    0xe53dd2c7, 0xbe2a63d2, 0x88d8f321, 0x64e08021, 0x521210d2,
+    0x0905a1c7, 0x3ff73134, 0xbf2ac3ed, 0x89d8531e, 0xd2cfe20b,
+    0xe43d72f8, 0xb929036f, 0x8fdb939c, 0xd4cc2289, 0xe23eb27a,
+    0x62e340a3, 0x5411d050, 0x0f066145, 0x39f4f1b6, 0xd5cc82b6,
+    0xe33e1245, 0xb829a350, 0x8edb33a3, 0x0e06c17a, 0x38f45189,
+    0x63e3e09c, 0x5511706f, 0x60e200dd, 0x5610902e, 0x0d07213b,
+    0x3bf5b1c8, 0xbb284311, 0x8ddad3e2, 0xd6cd62f7, 0xe03ff204,
+    0x0c078104, 0x3af511f7, 0x61e2a0e2, 0x57103011, 0xd7cdc2c8,
+    0xe13f523b, 0xba28e32e, 0x8cda73dd, 0x78ed02d5, 0x4e1f9226,
+    0x15082333, 0x23fab3c0, 0xa3274119, 0x95d5d1ea, 0xcec260ff,
+    0xf830f00c, 0x1408830c, 0x22fa13ff, 0x79eda2ea, 0x4f1f3219,
+    0xcfc2c0c0, 0xf9305033, 0xa227e126, 0x94d571d5, 0xa1260167,
+    0x97d49194, 0xccc32081, 0xfa31b072, 0x7aec42ab, 0x4c1ed258,
+    0x1709634d, 0x21fbf3be, 0xcdc380be, 0xfb31104d, 0xa026a158,
+    0x96d431ab, 0x1609c372, 0x20fb5381, 0x7bece294, 0x4d1e7267,
+    0x100a03f0, 0x26f89303, 0x7def2216, 0x4b1db2e5, 0xcbc0403c,
+    0xfd32d0cf, 0xa62561da, 0x90d7f129, 0x7cef8229, 0x4a1d12da,
+    0x110aa3cf, 0x27f8333c, 0xa725c1e5, 0x91d75116, 0xcac0e003,
+    0xfc3270f0, 0xc9c10042, 0xff3390b1, 0xa42421a4, 0x92d6b157,
+    0x120b438e, 0x24f9d37d, 0x7fee6268, 0x491cf29b, 0xa524819b,
+    0x93d61168, 0xc8c1a07d, 0xfe33308e, 0x7eeec257, 0x481c52a4,
+    0x130be3b1, 0x25f97342, 0xa923009f, 0x9fd1906c, 0xc4c62179,
+    0xf234b18a, 0x72e94353, 0x441bd3a0, 0x1f0c62b5, 0x29fef246,
+    0xc5c68146, 0xf33411b5, 0xa823a0a0, 0x9ed13053, 0x1e0cc28a,
+    0x28fe5279, 0x73e9e36c, 0x451b739f, 0x70e8032d, 0x461a93de,
+    0x1d0d22cb, 0x2bffb238, 0xab2240e1, 0x9dd0d012, 0xc6c76107,
+    0xf035f1f4, 0x1c0d82f4, 0x2aff1207, 0x71e8a312, 0x471a33e1,
+    0xc7c7c138, 0xf13551cb, 0xaa22e0de, 0x9cd0702d, 0xc1c401ba,
+    0xf7369149, 0xac21205c, 0x9ad3b0af, 0x1a0e4276, 0x2cfcd285,
+    0x77eb6390, 0x4119f363, 0xad218063, 0x9bd31090, 0xc0c4a185,
+    0xf6363176, 0x76ebc3af, 0x4019535c, 0x1b0ee249, 0x2dfc72ba,
+    0x180f0208, 0x2efd92fb, 0x75ea23ee, 0x4318b31d, 0xc3c541c4,
+    0xf537d137, 0xae206022, 0x98d2f0d1, 0x74ea83d1, 0x42181322,
+    0x190fa237, 0x2ffd32c4, 0xaf20c01d, 0x99d250ee, 0xc2c5e1fb,
+    0xf4377108}};
+
+local const z_word_t FAR crc_braid_big_table[][256] = {
+   {0x0000000000000000, 0xf390f23600000000, 0xe621e56d00000000,
+    0x15b1175b00000000, 0xcc43cadb00000000, 0x3fd338ed00000000,
+    0x2a622fb600000000, 0xd9f2dd8000000000, 0xd981e56c00000000,
+    0x2a11175a00000000, 0x3fa0000100000000, 0xcc30f23700000000,
+    0x15c22fb700000000, 0xe652dd8100000000, 0xf3e3cada00000000,
+    0x007338ec00000000, 0xb203cbd900000000, 0x419339ef00000000,
+    0x54222eb400000000, 0xa7b2dc8200000000, 0x7e40010200000000,
+    0x8dd0f33400000000, 0x9861e46f00000000, 0x6bf1165900000000,
+    0x6b822eb500000000, 0x9812dc8300000000, 0x8da3cbd800000000,
+    0x7e3339ee00000000, 0xa7c1e46e00000000, 0x5451165800000000,
+    0x41e0010300000000, 0xb270f33500000000, 0x2501e76800000000,
+    0xd691155e00000000, 0xc320020500000000, 0x30b0f03300000000,
+    0xe9422db300000000, 0x1ad2df8500000000, 0x0f63c8de00000000,
+    0xfcf33ae800000000, 0xfc80020400000000, 0x0f10f03200000000,
+    0x1aa1e76900000000, 0xe931155f00000000, 0x30c3c8df00000000,
+    0xc3533ae900000000, 0xd6e22db200000000, 0x2572df8400000000,
+    0x97022cb100000000, 0x6492de8700000000, 0x7123c9dc00000000,
+    0x82b33bea00000000, 0x5b41e66a00000000, 0xa8d1145c00000000,
+    0xbd60030700000000, 0x4ef0f13100000000, 0x4e83c9dd00000000,
+    0xbd133beb00000000, 0xa8a22cb000000000, 0x5b32de8600000000,
+    0x82c0030600000000, 0x7150f13000000000, 0x64e1e66b00000000,
+    0x9771145d00000000, 0x4a02ced100000000, 0xb9923ce700000000,
+    0xac232bbc00000000, 0x5fb3d98a00000000, 0x8641040a00000000,
+    0x75d1f63c00000000, 0x6060e16700000000, 0x93f0135100000000,
+    0x93832bbd00000000, 0x6013d98b00000000, 0x75a2ced000000000,
+    0x86323ce600000000, 0x5fc0e16600000000, 0xac50135000000000,
+    0xb9e1040b00000000, 0x4a71f63d00000000, 0xf801050800000000,
+    0x0b91f73e00000000, 0x1e20e06500000000, 0xedb0125300000000,
+    0x3442cfd300000000, 0xc7d23de500000000, 0xd2632abe00000000,
+    0x21f3d88800000000, 0x2180e06400000000, 0xd210125200000000,
+    0xc7a1050900000000, 0x3431f73f00000000, 0xedc32abf00000000,
+    0x1e53d88900000000, 0x0be2cfd200000000, 0xf8723de400000000,
+    0x6f0329b900000000, 0x9c93db8f00000000, 0x8922ccd400000000,
+    0x7ab23ee200000000, 0xa340e36200000000, 0x50d0115400000000,
+    0x4561060f00000000, 0xb6f1f43900000000, 0xb682ccd500000000,
+    0x45123ee300000000, 0x50a329b800000000, 0xa333db8e00000000,
+    0x7ac1060e00000000, 0x8951f43800000000, 0x9ce0e36300000000,
+    0x6f70115500000000, 0xdd00e26000000000, 0x2e90105600000000,
+    0x3b21070d00000000, 0xc8b1f53b00000000, 0x114328bb00000000,
+    0xe2d3da8d00000000, 0xf762cdd600000000, 0x04f23fe000000000,
+    0x0481070c00000000, 0xf711f53a00000000, 0xe2a0e26100000000,
+    0x1130105700000000, 0xc8c2cdd700000000, 0x3b523fe100000000,
+    0x2ee328ba00000000, 0xdd73da8c00000000, 0xd502ed7800000000,
+    0x26921f4e00000000, 0x3323081500000000, 0xc0b3fa2300000000,
+    0x194127a300000000, 0xead1d59500000000, 0xff60c2ce00000000,
+    0x0cf030f800000000, 0x0c83081400000000, 0xff13fa2200000000,
+    0xeaa2ed7900000000, 0x19321f4f00000000, 0xc0c0c2cf00000000,
+    0x335030f900000000, 0x26e127a200000000, 0xd571d59400000000,
+    0x670126a100000000, 0x9491d49700000000, 0x8120c3cc00000000,
+    0x72b031fa00000000, 0xab42ec7a00000000, 0x58d21e4c00000000,
+    0x4d63091700000000, 0xbef3fb2100000000, 0xbe80c3cd00000000,
+    0x4d1031fb00000000, 0x58a126a000000000, 0xab31d49600000000,
+    0x72c3091600000000, 0x8153fb2000000000, 0x94e2ec7b00000000,
+    0x67721e4d00000000, 0xf0030a1000000000, 0x0393f82600000000,
+    0x1622ef7d00000000, 0xe5b21d4b00000000, 0x3c40c0cb00000000,
+    0xcfd032fd00000000, 0xda6125a600000000, 0x29f1d79000000000,
+    0x2982ef7c00000000, 0xda121d4a00000000, 0xcfa30a1100000000,
+    0x3c33f82700000000, 0xe5c125a700000000, 0x1651d79100000000,
+    0x03e0c0ca00000000, 0xf07032fc00000000, 0x4200c1c900000000,
+    0xb19033ff00000000, 0xa42124a400000000, 0x57b1d69200000000,
+    0x8e430b1200000000, 0x7dd3f92400000000, 0x6862ee7f00000000,
+    0x9bf21c4900000000, 0x9b8124a500000000, 0x6811d69300000000,
+    0x7da0c1c800000000, 0x8e3033fe00000000, 0x57c2ee7e00000000,
+    0xa4521c4800000000, 0xb1e30b1300000000, 0x4273f92500000000,
+    0x9f0023a900000000, 0x6c90d19f00000000, 0x7921c6c400000000,
+    0x8ab134f200000000, 0x5343e97200000000, 0xa0d31b4400000000,
+    0xb5620c1f00000000, 0x46f2fe2900000000, 0x4681c6c500000000,
+    0xb51134f300000000, 0xa0a023a800000000, 0x5330d19e00000000,
+    0x8ac20c1e00000000, 0x7952fe2800000000, 0x6ce3e97300000000,
+    0x9f731b4500000000, 0x2d03e87000000000, 0xde931a4600000000,
+    0xcb220d1d00000000, 0x38b2ff2b00000000, 0xe14022ab00000000,
+    0x12d0d09d00000000, 0x0761c7c600000000, 0xf4f135f000000000,
+    0xf4820d1c00000000, 0x0712ff2a00000000, 0x12a3e87100000000,
+    0xe1331a4700000000, 0x38c1c7c700000000, 0xcb5135f100000000,
+    0xdee022aa00000000, 0x2d70d09c00000000, 0xba01c4c100000000,
+    0x499136f700000000, 0x5c2021ac00000000, 0xafb0d39a00000000,
+    0x76420e1a00000000, 0x85d2fc2c00000000, 0x9063eb7700000000,
+    0x63f3194100000000, 0x638021ad00000000, 0x9010d39b00000000,
+    0x85a1c4c000000000, 0x763136f600000000, 0xafc3eb7600000000,
+    0x5c53194000000000, 0x49e20e1b00000000, 0xba72fc2d00000000,
+    0x08020f1800000000, 0xfb92fd2e00000000, 0xee23ea7500000000,
+    0x1db3184300000000, 0xc441c5c300000000, 0x37d137f500000000,
+    0x226020ae00000000, 0xd1f0d29800000000, 0xd183ea7400000000,
+    0x2213184200000000, 0x37a20f1900000000, 0xc432fd2f00000000,
+    0x1dc020af00000000, 0xee50d29900000000, 0xfbe1c5c200000000,
+    0x087137f400000000},
+   {0x0000000000000000, 0x3651822400000000, 0x6ca2044900000000,
+    0x5af3866d00000000, 0xd844099200000000, 0xee158bb600000000,
+    0xb4e60ddb00000000, 0x82b78fff00000000, 0xf18f63ff00000000,
+    0xc7dee1db00000000, 0x9d2d67b600000000, 0xab7ce59200000000,
+    0x29cb6a6d00000000, 0x1f9ae84900000000, 0x45696e2400000000,
+    0x7338ec0000000000, 0xa319b62500000000, 0x9548340100000000,
+    0xcfbbb26c00000000, 0xf9ea304800000000, 0x7b5dbfb700000000,
+    0x4d0c3d9300000000, 0x17ffbbfe00000000, 0x21ae39da00000000,
+    0x5296d5da00000000, 0x64c757fe00000000, 0x3e34d19300000000,
+    0x086553b700000000, 0x8ad2dc4800000000, 0xbc835e6c00000000,
+    0xe670d80100000000, 0xd0215a2500000000, 0x46336c4b00000000,
+    0x7062ee6f00000000, 0x2a91680200000000, 0x1cc0ea2600000000,
+    0x9e7765d900000000, 0xa826e7fd00000000, 0xf2d5619000000000,
+    0xc484e3b400000000, 0xb7bc0fb400000000, 0x81ed8d9000000000,
+    0xdb1e0bfd00000000, 0xed4f89d900000000, 0x6ff8062600000000,
+    0x59a9840200000000, 0x035a026f00000000, 0x350b804b00000000,
+    0xe52ada6e00000000, 0xd37b584a00000000, 0x8988de2700000000,
+    0xbfd95c0300000000, 0x3d6ed3fc00000000, 0x0b3f51d800000000,
+    0x51ccd7b500000000, 0x679d559100000000, 0x14a5b99100000000,
+    0x22f43bb500000000, 0x7807bdd800000000, 0x4e563ffc00000000,
+    0xcce1b00300000000, 0xfab0322700000000, 0xa043b44a00000000,
+    0x9612366e00000000, 0x8c66d89600000000, 0xba375ab200000000,
+    0xe0c4dcdf00000000, 0xd6955efb00000000, 0x5422d10400000000,
+    0x6273532000000000, 0x3880d54d00000000, 0x0ed1576900000000,
+    0x7de9bb6900000000, 0x4bb8394d00000000, 0x114bbf2000000000,
+    0x271a3d0400000000, 0xa5adb2fb00000000, 0x93fc30df00000000,
+    0xc90fb6b200000000, 0xff5e349600000000, 0x2f7f6eb300000000,
+    0x192eec9700000000, 0x43dd6afa00000000, 0x758ce8de00000000,
+    0xf73b672100000000, 0xc16ae50500000000, 0x9b99636800000000,
+    0xadc8e14c00000000, 0xdef00d4c00000000, 0xe8a18f6800000000,
+    0xb252090500000000, 0x84038b2100000000, 0x06b404de00000000,
+    0x30e586fa00000000, 0x6a16009700000000, 0x5c4782b300000000,
+    0xca55b4dd00000000, 0xfc0436f900000000, 0xa6f7b09400000000,
+    0x90a632b000000000, 0x1211bd4f00000000, 0x24403f6b00000000,
+    0x7eb3b90600000000, 0x48e23b2200000000, 0x3bdad72200000000,
+    0x0d8b550600000000, 0x5778d36b00000000, 0x6129514f00000000,
+    0xe39edeb000000000, 0xd5cf5c9400000000, 0x8f3cdaf900000000,
+    0xb96d58dd00000000, 0x694c02f800000000, 0x5f1d80dc00000000,
+    0x05ee06b100000000, 0x33bf849500000000, 0xb1080b6a00000000,
+    0x8759894e00000000, 0xddaa0f2300000000, 0xebfb8d0700000000,
+    0x98c3610700000000, 0xae92e32300000000, 0xf461654e00000000,
+    0xc230e76a00000000, 0x4087689500000000, 0x76d6eab100000000,
+    0x2c256cdc00000000, 0x1a74eef800000000, 0x59cbc1f600000000,
+    0x6f9a43d200000000, 0x3569c5bf00000000, 0x0338479b00000000,
+    0x818fc86400000000, 0xb7de4a4000000000, 0xed2dcc2d00000000,
+    0xdb7c4e0900000000, 0xa844a20900000000, 0x9e15202d00000000,
+    0xc4e6a64000000000, 0xf2b7246400000000, 0x7000ab9b00000000,
+    0x465129bf00000000, 0x1ca2afd200000000, 0x2af32df600000000,
+    0xfad277d300000000, 0xcc83f5f700000000, 0x9670739a00000000,
+    0xa021f1be00000000, 0x22967e4100000000, 0x14c7fc6500000000,
+    0x4e347a0800000000, 0x7865f82c00000000, 0x0b5d142c00000000,
+    0x3d0c960800000000, 0x67ff106500000000, 0x51ae924100000000,
+    0xd3191dbe00000000, 0xe5489f9a00000000, 0xbfbb19f700000000,
+    0x89ea9bd300000000, 0x1ff8adbd00000000, 0x29a92f9900000000,
+    0x735aa9f400000000, 0x450b2bd000000000, 0xc7bca42f00000000,
+    0xf1ed260b00000000, 0xab1ea06600000000, 0x9d4f224200000000,
+    0xee77ce4200000000, 0xd8264c6600000000, 0x82d5ca0b00000000,
+    0xb484482f00000000, 0x3633c7d000000000, 0x006245f400000000,
+    0x5a91c39900000000, 0x6cc041bd00000000, 0xbce11b9800000000,
+    0x8ab099bc00000000, 0xd0431fd100000000, 0xe6129df500000000,
+    0x64a5120a00000000, 0x52f4902e00000000, 0x0807164300000000,
+    0x3e56946700000000, 0x4d6e786700000000, 0x7b3ffa4300000000,
+    0x21cc7c2e00000000, 0x179dfe0a00000000, 0x952a71f500000000,
+    0xa37bf3d100000000, 0xf98875bc00000000, 0xcfd9f79800000000,
+    0xd5ad196000000000, 0xe3fc9b4400000000, 0xb90f1d2900000000,
+    0x8f5e9f0d00000000, 0x0de910f200000000, 0x3bb892d600000000,
+    0x614b14bb00000000, 0x571a969f00000000, 0x24227a9f00000000,
+    0x1273f8bb00000000, 0x48807ed600000000, 0x7ed1fcf200000000,
+    0xfc66730d00000000, 0xca37f12900000000, 0x90c4774400000000,
+    0xa695f56000000000, 0x76b4af4500000000, 0x40e52d6100000000,
+    0x1a16ab0c00000000, 0x2c47292800000000, 0xaef0a6d700000000,
+    0x98a124f300000000, 0xc252a29e00000000, 0xf40320ba00000000,
+    0x873bccba00000000, 0xb16a4e9e00000000, 0xeb99c8f300000000,
+    0xddc84ad700000000, 0x5f7fc52800000000, 0x692e470c00000000,
+    0x33ddc16100000000, 0x058c434500000000, 0x939e752b00000000,
+    0xa5cff70f00000000, 0xff3c716200000000, 0xc96df34600000000,
+    0x4bda7cb900000000, 0x7d8bfe9d00000000, 0x277878f000000000,
+    0x1129fad400000000, 0x621116d400000000, 0x544094f000000000,
+    0x0eb3129d00000000, 0x38e290b900000000, 0xba551f4600000000,
+    0x8c049d6200000000, 0xd6f71b0f00000000, 0xe0a6992b00000000,
+    0x3087c30e00000000, 0x06d6412a00000000, 0x5c25c74700000000,
+    0x6a74456300000000, 0xe8c3ca9c00000000, 0xde9248b800000000,
+    0x8461ced500000000, 0xb2304cf100000000, 0xc108a0f100000000,
+    0xf75922d500000000, 0xadaaa4b800000000, 0x9bfb269c00000000,
+    0x194ca96300000000, 0x2f1d2b4700000000, 0x75eead2a00000000,
+    0x43bf2f0e00000000},
+   {0x0000000000000000, 0xc8179ecf00000000, 0xd1294d4400000000,
+    0x193ed38b00000000, 0xa2539a8800000000, 0x6a44044700000000,
+    0x737ad7cc00000000, 0xbb6d490300000000, 0x05a145ca00000000,
+    0xcdb6db0500000000, 0xd488088e00000000, 0x1c9f964100000000,
+    0xa7f2df4200000000, 0x6fe5418d00000000, 0x76db920600000000,
+    0xbecc0cc900000000, 0x4b44fa4f00000000, 0x8353648000000000,
+    0x9a6db70b00000000, 0x527a29c400000000, 0xe91760c700000000,
+    0x2100fe0800000000, 0x383e2d8300000000, 0xf029b34c00000000,
+    0x4ee5bf8500000000, 0x86f2214a00000000, 0x9fccf2c100000000,
+    0x57db6c0e00000000, 0xecb6250d00000000, 0x24a1bbc200000000,
+    0x3d9f684900000000, 0xf588f68600000000, 0x9688f49f00000000,
+    0x5e9f6a5000000000, 0x47a1b9db00000000, 0x8fb6271400000000,
+    0x34db6e1700000000, 0xfcccf0d800000000, 0xe5f2235300000000,
+    0x2de5bd9c00000000, 0x9329b15500000000, 0x5b3e2f9a00000000,
+    0x4200fc1100000000, 0x8a1762de00000000, 0x317a2bdd00000000,
+    0xf96db51200000000, 0xe053669900000000, 0x2844f85600000000,
+    0xddcc0ed000000000, 0x15db901f00000000, 0x0ce5439400000000,
+    0xc4f2dd5b00000000, 0x7f9f945800000000, 0xb7880a9700000000,
+    0xaeb6d91c00000000, 0x66a147d300000000, 0xd86d4b1a00000000,
+    0x107ad5d500000000, 0x0944065e00000000, 0xc153989100000000,
+    0x7a3ed19200000000, 0xb2294f5d00000000, 0xab179cd600000000,
+    0x6300021900000000, 0x6d1798e400000000, 0xa500062b00000000,
+    0xbc3ed5a000000000, 0x74294b6f00000000, 0xcf44026c00000000,
+    0x07539ca300000000, 0x1e6d4f2800000000, 0xd67ad1e700000000,
+    0x68b6dd2e00000000, 0xa0a143e100000000, 0xb99f906a00000000,
+    0x71880ea500000000, 0xcae547a600000000, 0x02f2d96900000000,
+    0x1bcc0ae200000000, 0xd3db942d00000000, 0x265362ab00000000,
+    0xee44fc6400000000, 0xf77a2fef00000000, 0x3f6db12000000000,
+    0x8400f82300000000, 0x4c1766ec00000000, 0x5529b56700000000,
+    0x9d3e2ba800000000, 0x23f2276100000000, 0xebe5b9ae00000000,
+    0xf2db6a2500000000, 0x3accf4ea00000000, 0x81a1bde900000000,
+    0x49b6232600000000, 0x5088f0ad00000000, 0x989f6e6200000000,
+    0xfb9f6c7b00000000, 0x3388f2b400000000, 0x2ab6213f00000000,
+    0xe2a1bff000000000, 0x59ccf6f300000000, 0x91db683c00000000,
+    0x88e5bbb700000000, 0x40f2257800000000, 0xfe3e29b100000000,
+    0x3629b77e00000000, 0x2f1764f500000000, 0xe700fa3a00000000,
+    0x5c6db33900000000, 0x947a2df600000000, 0x8d44fe7d00000000,
+    0x455360b200000000, 0xb0db963400000000, 0x78cc08fb00000000,
+    0x61f2db7000000000, 0xa9e545bf00000000, 0x12880cbc00000000,
+    0xda9f927300000000, 0xc3a141f800000000, 0x0bb6df3700000000,
+    0xb57ad3fe00000000, 0x7d6d4d3100000000, 0x64539eba00000000,
+    0xac44007500000000, 0x1729497600000000, 0xdf3ed7b900000000,
+    0xc600043200000000, 0x0e179afd00000000, 0x9b28411200000000,
+    0x533fdfdd00000000, 0x4a010c5600000000, 0x8216929900000000,
+    0x397bdb9a00000000, 0xf16c455500000000, 0xe85296de00000000,
+    0x2045081100000000, 0x9e8904d800000000, 0x569e9a1700000000,
+    0x4fa0499c00000000, 0x87b7d75300000000, 0x3cda9e5000000000,
+    0xf4cd009f00000000, 0xedf3d31400000000, 0x25e44ddb00000000,
+    0xd06cbb5d00000000, 0x187b259200000000, 0x0145f61900000000,
+    0xc95268d600000000, 0x723f21d500000000, 0xba28bf1a00000000,
+    0xa3166c9100000000, 0x6b01f25e00000000, 0xd5cdfe9700000000,
+    0x1dda605800000000, 0x04e4b3d300000000, 0xccf32d1c00000000,
+    0x779e641f00000000, 0xbf89fad000000000, 0xa6b7295b00000000,
+    0x6ea0b79400000000, 0x0da0b58d00000000, 0xc5b72b4200000000,
+    0xdc89f8c900000000, 0x149e660600000000, 0xaff32f0500000000,
+    0x67e4b1ca00000000, 0x7eda624100000000, 0xb6cdfc8e00000000,
+    0x0801f04700000000, 0xc0166e8800000000, 0xd928bd0300000000,
+    0x113f23cc00000000, 0xaa526acf00000000, 0x6245f40000000000,
+    0x7b7b278b00000000, 0xb36cb94400000000, 0x46e44fc200000000,
+    0x8ef3d10d00000000, 0x97cd028600000000, 0x5fda9c4900000000,
+    0xe4b7d54a00000000, 0x2ca04b8500000000, 0x359e980e00000000,
+    0xfd8906c100000000, 0x43450a0800000000, 0x8b5294c700000000,
+    0x926c474c00000000, 0x5a7bd98300000000, 0xe116908000000000,
+    0x29010e4f00000000, 0x303fddc400000000, 0xf828430b00000000,
+    0xf63fd9f600000000, 0x3e28473900000000, 0x271694b200000000,
+    0xef010a7d00000000, 0x546c437e00000000, 0x9c7bddb100000000,
+    0x85450e3a00000000, 0x4d5290f500000000, 0xf39e9c3c00000000,
+    0x3b8902f300000000, 0x22b7d17800000000, 0xeaa04fb700000000,
+    0x51cd06b400000000, 0x99da987b00000000, 0x80e44bf000000000,
+    0x48f3d53f00000000, 0xbd7b23b900000000, 0x756cbd7600000000,
+    0x6c526efd00000000, 0xa445f03200000000, 0x1f28b93100000000,
+    0xd73f27fe00000000, 0xce01f47500000000, 0x06166aba00000000,
+    0xb8da667300000000, 0x70cdf8bc00000000, 0x69f32b3700000000,
+    0xa1e4b5f800000000, 0x1a89fcfb00000000, 0xd29e623400000000,
+    0xcba0b1bf00000000, 0x03b72f7000000000, 0x60b72d6900000000,
+    0xa8a0b3a600000000, 0xb19e602d00000000, 0x7989fee200000000,
+    0xc2e4b7e100000000, 0x0af3292e00000000, 0x13cdfaa500000000,
+    0xdbda646a00000000, 0x651668a300000000, 0xad01f66c00000000,
+    0xb43f25e700000000, 0x7c28bb2800000000, 0xc745f22b00000000,
+    0x0f526ce400000000, 0x166cbf6f00000000, 0xde7b21a000000000,
+    0x2bf3d72600000000, 0xe3e449e900000000, 0xfada9a6200000000,
+    0x32cd04ad00000000, 0x89a04dae00000000, 0x41b7d36100000000,
+    0x588900ea00000000, 0x909e9e2500000000, 0x2e5292ec00000000,
+    0xe6450c2300000000, 0xff7bdfa800000000, 0x376c416700000000,
+    0x8c01086400000000, 0x441696ab00000000, 0x5d28452000000000,
+    0x953fdbef00000000},
+   {0x0000000000000000, 0x95d4709500000000, 0x6baf90f100000000,
+    0xfe7be06400000000, 0x9758503800000000, 0x028c20ad00000000,
+    0xfcf7c0c900000000, 0x6923b05c00000000, 0x2eb1a07000000000,
+    0xbb65d0e500000000, 0x451e308100000000, 0xd0ca401400000000,
+    0xb9e9f04800000000, 0x2c3d80dd00000000, 0xd24660b900000000,
+    0x4792102c00000000, 0x5c6241e100000000, 0xc9b6317400000000,
+    0x37cdd11000000000, 0xa219a18500000000, 0xcb3a11d900000000,
+    0x5eee614c00000000, 0xa095812800000000, 0x3541f1bd00000000,
+    0x72d3e19100000000, 0xe707910400000000, 0x197c716000000000,
+    0x8ca801f500000000, 0xe58bb1a900000000, 0x705fc13c00000000,
+    0x8e24215800000000, 0x1bf051cd00000000, 0xf9c2f31900000000,
+    0x6c16838c00000000, 0x926d63e800000000, 0x07b9137d00000000,
+    0x6e9aa32100000000, 0xfb4ed3b400000000, 0x053533d000000000,
+    0x90e1434500000000, 0xd773536900000000, 0x42a723fc00000000,
+    0xbcdcc39800000000, 0x2908b30d00000000, 0x402b035100000000,
+    0xd5ff73c400000000, 0x2b8493a000000000, 0xbe50e33500000000,
+    0xa5a0b2f800000000, 0x3074c26d00000000, 0xce0f220900000000,
+    0x5bdb529c00000000, 0x32f8e2c000000000, 0xa72c925500000000,
+    0x5957723100000000, 0xcc8302a400000000, 0x8b11128800000000,
+    0x1ec5621d00000000, 0xe0be827900000000, 0x756af2ec00000000,
+    0x1c4942b000000000, 0x899d322500000000, 0x77e6d24100000000,
+    0xe232a2d400000000, 0xf285e73300000000, 0x675197a600000000,
+    0x992a77c200000000, 0x0cfe075700000000, 0x65ddb70b00000000,
+    0xf009c79e00000000, 0x0e7227fa00000000, 0x9ba6576f00000000,
+    0xdc34474300000000, 0x49e037d600000000, 0xb79bd7b200000000,
+    0x224fa72700000000, 0x4b6c177b00000000, 0xdeb867ee00000000,
+    0x20c3878a00000000, 0xb517f71f00000000, 0xaee7a6d200000000,
+    0x3b33d64700000000, 0xc548362300000000, 0x509c46b600000000,
+    0x39bff6ea00000000, 0xac6b867f00000000, 0x5210661b00000000,
+    0xc7c4168e00000000, 0x805606a200000000, 0x1582763700000000,
+    0xebf9965300000000, 0x7e2de6c600000000, 0x170e569a00000000,
+    0x82da260f00000000, 0x7ca1c66b00000000, 0xe975b6fe00000000,
+    0x0b47142a00000000, 0x9e9364bf00000000, 0x60e884db00000000,
+    0xf53cf44e00000000, 0x9c1f441200000000, 0x09cb348700000000,
+    0xf7b0d4e300000000, 0x6264a47600000000, 0x25f6b45a00000000,
+    0xb022c4cf00000000, 0x4e5924ab00000000, 0xdb8d543e00000000,
+    0xb2aee46200000000, 0x277a94f700000000, 0xd901749300000000,
+    0x4cd5040600000000, 0x572555cb00000000, 0xc2f1255e00000000,
+    0x3c8ac53a00000000, 0xa95eb5af00000000, 0xc07d05f300000000,
+    0x55a9756600000000, 0xabd2950200000000, 0x3e06e59700000000,
+    0x7994f5bb00000000, 0xec40852e00000000, 0x123b654a00000000,
+    0x87ef15df00000000, 0xeecca58300000000, 0x7b18d51600000000,
+    0x8563357200000000, 0x10b745e700000000, 0xe40bcf6700000000,
+    0x71dfbff200000000, 0x8fa45f9600000000, 0x1a702f0300000000,
+    0x73539f5f00000000, 0xe687efca00000000, 0x18fc0fae00000000,
+    0x8d287f3b00000000, 0xcaba6f1700000000, 0x5f6e1f8200000000,
+    0xa115ffe600000000, 0x34c18f7300000000, 0x5de23f2f00000000,
+    0xc8364fba00000000, 0x364dafde00000000, 0xa399df4b00000000,
+    0xb8698e8600000000, 0x2dbdfe1300000000, 0xd3c61e7700000000,
+    0x46126ee200000000, 0x2f31debe00000000, 0xbae5ae2b00000000,
+    0x449e4e4f00000000, 0xd14a3eda00000000, 0x96d82ef600000000,
+    0x030c5e6300000000, 0xfd77be0700000000, 0x68a3ce9200000000,
+    0x01807ece00000000, 0x94540e5b00000000, 0x6a2fee3f00000000,
+    0xfffb9eaa00000000, 0x1dc93c7e00000000, 0x881d4ceb00000000,
+    0x7666ac8f00000000, 0xe3b2dc1a00000000, 0x8a916c4600000000,
+    0x1f451cd300000000, 0xe13efcb700000000, 0x74ea8c2200000000,
+    0x33789c0e00000000, 0xa6acec9b00000000, 0x58d70cff00000000,
+    0xcd037c6a00000000, 0xa420cc3600000000, 0x31f4bca300000000,
+    0xcf8f5cc700000000, 0x5a5b2c5200000000, 0x41ab7d9f00000000,
+    0xd47f0d0a00000000, 0x2a04ed6e00000000, 0xbfd09dfb00000000,
+    0xd6f32da700000000, 0x43275d3200000000, 0xbd5cbd5600000000,
+    0x2888cdc300000000, 0x6f1addef00000000, 0xfacead7a00000000,
+    0x04b54d1e00000000, 0x91613d8b00000000, 0xf8428dd700000000,
+    0x6d96fd4200000000, 0x93ed1d2600000000, 0x06396db300000000,
+    0x168e285400000000, 0x835a58c100000000, 0x7d21b8a500000000,
+    0xe8f5c83000000000, 0x81d6786c00000000, 0x140208f900000000,
+    0xea79e89d00000000, 0x7fad980800000000, 0x383f882400000000,
+    0xadebf8b100000000, 0x539018d500000000, 0xc644684000000000,
+    0xaf67d81c00000000, 0x3ab3a88900000000, 0xc4c848ed00000000,
+    0x511c387800000000, 0x4aec69b500000000, 0xdf38192000000000,
+    0x2143f94400000000, 0xb49789d100000000, 0xddb4398d00000000,
+    0x4860491800000000, 0xb61ba97c00000000, 0x23cfd9e900000000,
+    0x645dc9c500000000, 0xf189b95000000000, 0x0ff2593400000000,
+    0x9a2629a100000000, 0xf30599fd00000000, 0x66d1e96800000000,
+    0x98aa090c00000000, 0x0d7e799900000000, 0xef4cdb4d00000000,
+    0x7a98abd800000000, 0x84e34bbc00000000, 0x11373b2900000000,
+    0x78148b7500000000, 0xedc0fbe000000000, 0x13bb1b8400000000,
+    0x866f6b1100000000, 0xc1fd7b3d00000000, 0x54290ba800000000,
+    0xaa52ebcc00000000, 0x3f869b5900000000, 0x56a52b0500000000,
+    0xc3715b9000000000, 0x3d0abbf400000000, 0xa8decb6100000000,
+    0xb32e9aac00000000, 0x26faea3900000000, 0xd8810a5d00000000,
+    0x4d557ac800000000, 0x2476ca9400000000, 0xb1a2ba0100000000,
+    0x4fd95a6500000000, 0xda0d2af000000000, 0x9d9f3adc00000000,
+    0x084b4a4900000000, 0xf630aa2d00000000, 0x63e4dab800000000,
+    0x0ac76ae400000000, 0x9f131a7100000000, 0x6168fa1500000000,
+    0xf4bc8a8000000000},
+   {0x0000000000000000, 0x1f17f08000000000, 0x7f2891da00000000,
+    0x603f615a00000000, 0xbf56536e00000000, 0xa041a3ee00000000,
+    0xc07ec2b400000000, 0xdf69323400000000, 0x7eada6dc00000000,
+    0x61ba565c00000000, 0x0185370600000000, 0x1e92c78600000000,
+    0xc1fbf5b200000000, 0xdeec053200000000, 0xbed3646800000000,
+    0xa1c494e800000000, 0xbd5c3c6200000000, 0xa24bcce200000000,
+    0xc274adb800000000, 0xdd635d3800000000, 0x020a6f0c00000000,
+    0x1d1d9f8c00000000, 0x7d22fed600000000, 0x62350e5600000000,
+    0xc3f19abe00000000, 0xdce66a3e00000000, 0xbcd90b6400000000,
+    0xa3cefbe400000000, 0x7ca7c9d000000000, 0x63b0395000000000,
+    0x038f580a00000000, 0x1c98a88a00000000, 0x7ab978c400000000,
+    0x65ae884400000000, 0x0591e91e00000000, 0x1a86199e00000000,
+    0xc5ef2baa00000000, 0xdaf8db2a00000000, 0xbac7ba7000000000,
+    0xa5d04af000000000, 0x0414de1800000000, 0x1b032e9800000000,
+    0x7b3c4fc200000000, 0x642bbf4200000000, 0xbb428d7600000000,
+    0xa4557df600000000, 0xc46a1cac00000000, 0xdb7dec2c00000000,
+    0xc7e544a600000000, 0xd8f2b42600000000, 0xb8cdd57c00000000,
+    0xa7da25fc00000000, 0x78b317c800000000, 0x67a4e74800000000,
+    0x079b861200000000, 0x188c769200000000, 0xb948e27a00000000,
+    0xa65f12fa00000000, 0xc66073a000000000, 0xd977832000000000,
+    0x061eb11400000000, 0x1909419400000000, 0x793620ce00000000,
+    0x6621d04e00000000, 0xb574805300000000, 0xaa6370d300000000,
+    0xca5c118900000000, 0xd54be10900000000, 0x0a22d33d00000000,
+    0x153523bd00000000, 0x750a42e700000000, 0x6a1db26700000000,
+    0xcbd9268f00000000, 0xd4ced60f00000000, 0xb4f1b75500000000,
+    0xabe647d500000000, 0x748f75e100000000, 0x6b98856100000000,
+    0x0ba7e43b00000000, 0x14b014bb00000000, 0x0828bc3100000000,
+    0x173f4cb100000000, 0x77002deb00000000, 0x6817dd6b00000000,
+    0xb77eef5f00000000, 0xa8691fdf00000000, 0xc8567e8500000000,
+    0xd7418e0500000000, 0x76851aed00000000, 0x6992ea6d00000000,
+    0x09ad8b3700000000, 0x16ba7bb700000000, 0xc9d3498300000000,
+    0xd6c4b90300000000, 0xb6fbd85900000000, 0xa9ec28d900000000,
+    0xcfcdf89700000000, 0xd0da081700000000, 0xb0e5694d00000000,
+    0xaff299cd00000000, 0x709babf900000000, 0x6f8c5b7900000000,
+    0x0fb33a2300000000, 0x10a4caa300000000, 0xb1605e4b00000000,
+    0xae77aecb00000000, 0xce48cf9100000000, 0xd15f3f1100000000,
+    0x0e360d2500000000, 0x1121fda500000000, 0x711e9cff00000000,
+    0x6e096c7f00000000, 0x7291c4f500000000, 0x6d86347500000000,
+    0x0db9552f00000000, 0x12aea5af00000000, 0xcdc7979b00000000,
+    0xd2d0671b00000000, 0xb2ef064100000000, 0xadf8f6c100000000,
+    0x0c3c622900000000, 0x132b92a900000000, 0x7314f3f300000000,
+    0x6c03037300000000, 0xb36a314700000000, 0xac7dc1c700000000,
+    0xcc42a09d00000000, 0xd355501d00000000, 0x6ae900a700000000,
+    0x75fef02700000000, 0x15c1917d00000000, 0x0ad661fd00000000,
+    0xd5bf53c900000000, 0xcaa8a34900000000, 0xaa97c21300000000,
+    0xb580329300000000, 0x1444a67b00000000, 0x0b5356fb00000000,
+    0x6b6c37a100000000, 0x747bc72100000000, 0xab12f51500000000,
+    0xb405059500000000, 0xd43a64cf00000000, 0xcb2d944f00000000,
+    0xd7b53cc500000000, 0xc8a2cc4500000000, 0xa89dad1f00000000,
+    0xb78a5d9f00000000, 0x68e36fab00000000, 0x77f49f2b00000000,
+    0x17cbfe7100000000, 0x08dc0ef100000000, 0xa9189a1900000000,
+    0xb60f6a9900000000, 0xd6300bc300000000, 0xc927fb4300000000,
+    0x164ec97700000000, 0x095939f700000000, 0x696658ad00000000,
+    0x7671a82d00000000, 0x1050786300000000, 0x0f4788e300000000,
+    0x6f78e9b900000000, 0x706f193900000000, 0xaf062b0d00000000,
+    0xb011db8d00000000, 0xd02ebad700000000, 0xcf394a5700000000,
+    0x6efddebf00000000, 0x71ea2e3f00000000, 0x11d54f6500000000,
+    0x0ec2bfe500000000, 0xd1ab8dd100000000, 0xcebc7d5100000000,
+    0xae831c0b00000000, 0xb194ec8b00000000, 0xad0c440100000000,
+    0xb21bb48100000000, 0xd224d5db00000000, 0xcd33255b00000000,
+    0x125a176f00000000, 0x0d4de7ef00000000, 0x6d7286b500000000,
+    0x7265763500000000, 0xd3a1e2dd00000000, 0xccb6125d00000000,
+    0xac89730700000000, 0xb39e838700000000, 0x6cf7b1b300000000,
+    0x73e0413300000000, 0x13df206900000000, 0x0cc8d0e900000000,
+    0xdf9d80f400000000, 0xc08a707400000000, 0xa0b5112e00000000,
+    0xbfa2e1ae00000000, 0x60cbd39a00000000, 0x7fdc231a00000000,
+    0x1fe3424000000000, 0x00f4b2c000000000, 0xa130262800000000,
+    0xbe27d6a800000000, 0xde18b7f200000000, 0xc10f477200000000,
+    0x1e66754600000000, 0x017185c600000000, 0x614ee49c00000000,
+    0x7e59141c00000000, 0x62c1bc9600000000, 0x7dd64c1600000000,
+    0x1de92d4c00000000, 0x02feddcc00000000, 0xdd97eff800000000,
+    0xc2801f7800000000, 0xa2bf7e2200000000, 0xbda88ea200000000,
+    0x1c6c1a4a00000000, 0x037beaca00000000, 0x63448b9000000000,
+    0x7c537b1000000000, 0xa33a492400000000, 0xbc2db9a400000000,
+    0xdc12d8fe00000000, 0xc305287e00000000, 0xa524f83000000000,
+    0xba3308b000000000, 0xda0c69ea00000000, 0xc51b996a00000000,
+    0x1a72ab5e00000000, 0x05655bde00000000, 0x655a3a8400000000,
+    0x7a4dca0400000000, 0xdb895eec00000000, 0xc49eae6c00000000,
+    0xa4a1cf3600000000, 0xbbb63fb600000000, 0x64df0d8200000000,
+    0x7bc8fd0200000000, 0x1bf79c5800000000, 0x04e06cd800000000,
+    0x1878c45200000000, 0x076f34d200000000, 0x6750558800000000,
+    0x7847a50800000000, 0xa72e973c00000000, 0xb83967bc00000000,
+    0xd80606e600000000, 0xc711f66600000000, 0x66d5628e00000000,
+    0x79c2920e00000000, 0x19fdf35400000000, 0x06ea03d400000000,
+    0xd98331e000000000, 0xc694c16000000000, 0xa6aba03a00000000,
+    0xb9bc50ba00000000},
+   {0x0000000000000000, 0xe2fd888d00000000, 0x85fd60c000000000,
+    0x6700e84d00000000, 0x4bfdb05b00000000, 0xa90038d600000000,
+    0xce00d09b00000000, 0x2cfd581600000000, 0x96fa61b700000000,
+    0x7407e93a00000000, 0x1307017700000000, 0xf1fa89fa00000000,
+    0xdd07d1ec00000000, 0x3ffa596100000000, 0x58fab12c00000000,
+    0xba0739a100000000, 0x6df3b2b500000000, 0x8f0e3a3800000000,
+    0xe80ed27500000000, 0x0af35af800000000, 0x260e02ee00000000,
+    0xc4f38a6300000000, 0xa3f3622e00000000, 0x410eeaa300000000,
+    0xfb09d30200000000, 0x19f45b8f00000000, 0x7ef4b3c200000000,
+    0x9c093b4f00000000, 0xb0f4635900000000, 0x5209ebd400000000,
+    0x3509039900000000, 0xd7f48b1400000000, 0x9be014b000000000,
+    0x791d9c3d00000000, 0x1e1d747000000000, 0xfce0fcfd00000000,
+    0xd01da4eb00000000, 0x32e02c6600000000, 0x55e0c42b00000000,
+    0xb71d4ca600000000, 0x0d1a750700000000, 0xefe7fd8a00000000,
+    0x88e715c700000000, 0x6a1a9d4a00000000, 0x46e7c55c00000000,
+    0xa41a4dd100000000, 0xc31aa59c00000000, 0x21e72d1100000000,
+    0xf613a60500000000, 0x14ee2e8800000000, 0x73eec6c500000000,
+    0x91134e4800000000, 0xbdee165e00000000, 0x5f139ed300000000,
+    0x3813769e00000000, 0xdaeefe1300000000, 0x60e9c7b200000000,
+    0x82144f3f00000000, 0xe514a77200000000, 0x07e92fff00000000,
+    0x2b1477e900000000, 0xc9e9ff6400000000, 0xaee9172900000000,
+    0x4c149fa400000000, 0x77c758bb00000000, 0x953ad03600000000,
+    0xf23a387b00000000, 0x10c7b0f600000000, 0x3c3ae8e000000000,
+    0xdec7606d00000000, 0xb9c7882000000000, 0x5b3a00ad00000000,
+    0xe13d390c00000000, 0x03c0b18100000000, 0x64c059cc00000000,
+    0x863dd14100000000, 0xaac0895700000000, 0x483d01da00000000,
+    0x2f3de99700000000, 0xcdc0611a00000000, 0x1a34ea0e00000000,
+    0xf8c9628300000000, 0x9fc98ace00000000, 0x7d34024300000000,
+    0x51c95a5500000000, 0xb334d2d800000000, 0xd4343a9500000000,
+    0x36c9b21800000000, 0x8cce8bb900000000, 0x6e33033400000000,
+    0x0933eb7900000000, 0xebce63f400000000, 0xc7333be200000000,
+    0x25ceb36f00000000, 0x42ce5b2200000000, 0xa033d3af00000000,
+    0xec274c0b00000000, 0x0edac48600000000, 0x69da2ccb00000000,
+    0x8b27a44600000000, 0xa7dafc5000000000, 0x452774dd00000000,
+    0x22279c9000000000, 0xc0da141d00000000, 0x7add2dbc00000000,
+    0x9820a53100000000, 0xff204d7c00000000, 0x1dddc5f100000000,
+    0x31209de700000000, 0xd3dd156a00000000, 0xb4ddfd2700000000,
+    0x562075aa00000000, 0x81d4febe00000000, 0x6329763300000000,
+    0x04299e7e00000000, 0xe6d416f300000000, 0xca294ee500000000,
+    0x28d4c66800000000, 0x4fd42e2500000000, 0xad29a6a800000000,
+    0x172e9f0900000000, 0xf5d3178400000000, 0x92d3ffc900000000,
+    0x702e774400000000, 0x5cd32f5200000000, 0xbe2ea7df00000000,
+    0xd92e4f9200000000, 0x3bd3c71f00000000, 0xaf88c0ad00000000,
+    0x4d75482000000000, 0x2a75a06d00000000, 0xc88828e000000000,
+    0xe47570f600000000, 0x0688f87b00000000, 0x6188103600000000,
+    0x837598bb00000000, 0x3972a11a00000000, 0xdb8f299700000000,
+    0xbc8fc1da00000000, 0x5e72495700000000, 0x728f114100000000,
+    0x907299cc00000000, 0xf772718100000000, 0x158ff90c00000000,
+    0xc27b721800000000, 0x2086fa9500000000, 0x478612d800000000,
+    0xa57b9a5500000000, 0x8986c24300000000, 0x6b7b4ace00000000,
+    0x0c7ba28300000000, 0xee862a0e00000000, 0x548113af00000000,
+    0xb67c9b2200000000, 0xd17c736f00000000, 0x3381fbe200000000,
+    0x1f7ca3f400000000, 0xfd812b7900000000, 0x9a81c33400000000,
+    0x787c4bb900000000, 0x3468d41d00000000, 0xd6955c9000000000,
+    0xb195b4dd00000000, 0x53683c5000000000, 0x7f95644600000000,
+    0x9d68eccb00000000, 0xfa68048600000000, 0x18958c0b00000000,
+    0xa292b5aa00000000, 0x406f3d2700000000, 0x276fd56a00000000,
+    0xc5925de700000000, 0xe96f05f100000000, 0x0b928d7c00000000,
+    0x6c92653100000000, 0x8e6fedbc00000000, 0x599b66a800000000,
+    0xbb66ee2500000000, 0xdc66066800000000, 0x3e9b8ee500000000,
+    0x1266d6f300000000, 0xf09b5e7e00000000, 0x979bb63300000000,
+    0x75663ebe00000000, 0xcf61071f00000000, 0x2d9c8f9200000000,
+    0x4a9c67df00000000, 0xa861ef5200000000, 0x849cb74400000000,
+    0x66613fc900000000, 0x0161d78400000000, 0xe39c5f0900000000,
+    0xd84f981600000000, 0x3ab2109b00000000, 0x5db2f8d600000000,
+    0xbf4f705b00000000, 0x93b2284d00000000, 0x714fa0c000000000,
+    0x164f488d00000000, 0xf4b2c00000000000, 0x4eb5f9a100000000,
+    0xac48712c00000000, 0xcb48996100000000, 0x29b511ec00000000,
+    0x054849fa00000000, 0xe7b5c17700000000, 0x80b5293a00000000,
+    0x6248a1b700000000, 0xb5bc2aa300000000, 0x5741a22e00000000,
+    0x30414a6300000000, 0xd2bcc2ee00000000, 0xfe419af800000000,
+    0x1cbc127500000000, 0x7bbcfa3800000000, 0x994172b500000000,
+    0x23464b1400000000, 0xc1bbc39900000000, 0xa6bb2bd400000000,
+    0x4446a35900000000, 0x68bbfb4f00000000, 0x8a4673c200000000,
+    0xed469b8f00000000, 0x0fbb130200000000, 0x43af8ca600000000,
+    0xa152042b00000000, 0xc652ec6600000000, 0x24af64eb00000000,
+    0x08523cfd00000000, 0xeaafb47000000000, 0x8daf5c3d00000000,
+    0x6f52d4b000000000, 0xd555ed1100000000, 0x37a8659c00000000,
+    0x50a88dd100000000, 0xb255055c00000000, 0x9ea85d4a00000000,
+    0x7c55d5c700000000, 0x1b553d8a00000000, 0xf9a8b50700000000,
+    0x2e5c3e1300000000, 0xcca1b69e00000000, 0xaba15ed300000000,
+    0x495cd65e00000000, 0x65a18e4800000000, 0x875c06c500000000,
+    0xe05cee8800000000, 0x02a1660500000000, 0xb8a65fa400000000,
+    0x5a5bd72900000000, 0x3d5b3f6400000000, 0xdfa6b7e900000000,
+    0xf35befff00000000, 0x11a6677200000000, 0x76a68f3f00000000,
+    0x945b07b200000000},
+   {0x0000000000000000, 0xa90b894e00000000, 0x5217129d00000000,
+    0xfb1c9bd300000000, 0xe52855e100000000, 0x4c23dcaf00000000,
+    0xb73f477c00000000, 0x1e34ce3200000000, 0x8b57db1900000000,
+    0x225c525700000000, 0xd940c98400000000, 0x704b40ca00000000,
+    0x6e7f8ef800000000, 0xc77407b600000000, 0x3c689c6500000000,
+    0x9563152b00000000, 0x16afb63300000000, 0xbfa43f7d00000000,
+    0x44b8a4ae00000000, 0xedb32de000000000, 0xf387e3d200000000,
+    0x5a8c6a9c00000000, 0xa190f14f00000000, 0x089b780100000000,
+    0x9df86d2a00000000, 0x34f3e46400000000, 0xcfef7fb700000000,
+    0x66e4f6f900000000, 0x78d038cb00000000, 0xd1dbb18500000000,
+    0x2ac72a5600000000, 0x83cca31800000000, 0x2c5e6d6700000000,
+    0x8555e42900000000, 0x7e497ffa00000000, 0xd742f6b400000000,
+    0xc976388600000000, 0x607db1c800000000, 0x9b612a1b00000000,
+    0x326aa35500000000, 0xa709b67e00000000, 0x0e023f3000000000,
+    0xf51ea4e300000000, 0x5c152dad00000000, 0x4221e39f00000000,
+    0xeb2a6ad100000000, 0x1036f10200000000, 0xb93d784c00000000,
+    0x3af1db5400000000, 0x93fa521a00000000, 0x68e6c9c900000000,
+    0xc1ed408700000000, 0xdfd98eb500000000, 0x76d207fb00000000,
+    0x8dce9c2800000000, 0x24c5156600000000, 0xb1a6004d00000000,
+    0x18ad890300000000, 0xe3b112d000000000, 0x4aba9b9e00000000,
+    0x548e55ac00000000, 0xfd85dce200000000, 0x0699473100000000,
+    0xaf92ce7f00000000, 0x58bcdace00000000, 0xf1b7538000000000,
+    0x0aabc85300000000, 0xa3a0411d00000000, 0xbd948f2f00000000,
+    0x149f066100000000, 0xef839db200000000, 0x468814fc00000000,
+    0xd3eb01d700000000, 0x7ae0889900000000, 0x81fc134a00000000,
+    0x28f79a0400000000, 0x36c3543600000000, 0x9fc8dd7800000000,
+    0x64d446ab00000000, 0xcddfcfe500000000, 0x4e136cfd00000000,
+    0xe718e5b300000000, 0x1c047e6000000000, 0xb50ff72e00000000,
+    0xab3b391c00000000, 0x0230b05200000000, 0xf92c2b8100000000,
+    0x5027a2cf00000000, 0xc544b7e400000000, 0x6c4f3eaa00000000,
+    0x9753a57900000000, 0x3e582c3700000000, 0x206ce20500000000,
+    0x89676b4b00000000, 0x727bf09800000000, 0xdb7079d600000000,
+    0x74e2b7a900000000, 0xdde93ee700000000, 0x26f5a53400000000,
+    0x8ffe2c7a00000000, 0x91cae24800000000, 0x38c16b0600000000,
+    0xc3ddf0d500000000, 0x6ad6799b00000000, 0xffb56cb000000000,
+    0x56bee5fe00000000, 0xada27e2d00000000, 0x04a9f76300000000,
+    0x1a9d395100000000, 0xb396b01f00000000, 0x488a2bcc00000000,
+    0xe181a28200000000, 0x624d019a00000000, 0xcb4688d400000000,
+    0x305a130700000000, 0x99519a4900000000, 0x8765547b00000000,
+    0x2e6edd3500000000, 0xd57246e600000000, 0x7c79cfa800000000,
+    0xe91ada8300000000, 0x401153cd00000000, 0xbb0dc81e00000000,
+    0x1206415000000000, 0x0c328f6200000000, 0xa539062c00000000,
+    0x5e259dff00000000, 0xf72e14b100000000, 0xf17ec44600000000,
+    0x58754d0800000000, 0xa369d6db00000000, 0x0a625f9500000000,
+    0x145691a700000000, 0xbd5d18e900000000, 0x4641833a00000000,
+    0xef4a0a7400000000, 0x7a291f5f00000000, 0xd322961100000000,
+    0x283e0dc200000000, 0x8135848c00000000, 0x9f014abe00000000,
+    0x360ac3f000000000, 0xcd16582300000000, 0x641dd16d00000000,
+    0xe7d1727500000000, 0x4edafb3b00000000, 0xb5c660e800000000,
+    0x1ccde9a600000000, 0x02f9279400000000, 0xabf2aeda00000000,
+    0x50ee350900000000, 0xf9e5bc4700000000, 0x6c86a96c00000000,
+    0xc58d202200000000, 0x3e91bbf100000000, 0x979a32bf00000000,
+    0x89aefc8d00000000, 0x20a575c300000000, 0xdbb9ee1000000000,
+    0x72b2675e00000000, 0xdd20a92100000000, 0x742b206f00000000,
+    0x8f37bbbc00000000, 0x263c32f200000000, 0x3808fcc000000000,
+    0x9103758e00000000, 0x6a1fee5d00000000, 0xc314671300000000,
+    0x5677723800000000, 0xff7cfb7600000000, 0x046060a500000000,
+    0xad6be9eb00000000, 0xb35f27d900000000, 0x1a54ae9700000000,
+    0xe148354400000000, 0x4843bc0a00000000, 0xcb8f1f1200000000,
+    0x6284965c00000000, 0x99980d8f00000000, 0x309384c100000000,
+    0x2ea74af300000000, 0x87acc3bd00000000, 0x7cb0586e00000000,
+    0xd5bbd12000000000, 0x40d8c40b00000000, 0xe9d34d4500000000,
+    0x12cfd69600000000, 0xbbc45fd800000000, 0xa5f091ea00000000,
+    0x0cfb18a400000000, 0xf7e7837700000000, 0x5eec0a3900000000,
+    0xa9c21e8800000000, 0x00c997c600000000, 0xfbd50c1500000000,
+    0x52de855b00000000, 0x4cea4b6900000000, 0xe5e1c22700000000,
+    0x1efd59f400000000, 0xb7f6d0ba00000000, 0x2295c59100000000,
+    0x8b9e4cdf00000000, 0x7082d70c00000000, 0xd9895e4200000000,
+    0xc7bd907000000000, 0x6eb6193e00000000, 0x95aa82ed00000000,
+    0x3ca10ba300000000, 0xbf6da8bb00000000, 0x166621f500000000,
+    0xed7aba2600000000, 0x4471336800000000, 0x5a45fd5a00000000,
+    0xf34e741400000000, 0x0852efc700000000, 0xa159668900000000,
+    0x343a73a200000000, 0x9d31faec00000000, 0x662d613f00000000,
+    0xcf26e87100000000, 0xd112264300000000, 0x7819af0d00000000,
+    0x830534de00000000, 0x2a0ebd9000000000, 0x859c73ef00000000,
+    0x2c97faa100000000, 0xd78b617200000000, 0x7e80e83c00000000,
+    0x60b4260e00000000, 0xc9bfaf4000000000, 0x32a3349300000000,
+    0x9ba8bddd00000000, 0x0ecba8f600000000, 0xa7c021b800000000,
+    0x5cdcba6b00000000, 0xf5d7332500000000, 0xebe3fd1700000000,
+    0x42e8745900000000, 0xb9f4ef8a00000000, 0x10ff66c400000000,
+    0x9333c5dc00000000, 0x3a384c9200000000, 0xc124d74100000000,
+    0x682f5e0f00000000, 0x761b903d00000000, 0xdf10197300000000,
+    0x240c82a000000000, 0x8d070bee00000000, 0x18641ec500000000,
+    0xb16f978b00000000, 0x4a730c5800000000, 0xe378851600000000,
+    0xfd4c4b2400000000, 0x5447c26a00000000, 0xaf5b59b900000000,
+    0x0650d0f700000000},
+   {0x0000000000000000, 0x479244af00000000, 0xcf22f88500000000,
+    0x88b0bc2a00000000, 0xdf4381d000000000, 0x98d1c57f00000000,
+    0x1061795500000000, 0x57f33dfa00000000, 0xff81737a00000000,
+    0xb81337d500000000, 0x30a38bff00000000, 0x7731cf5000000000,
+    0x20c2f2aa00000000, 0x6750b60500000000, 0xefe00a2f00000000,
+    0xa8724e8000000000, 0xfe03e7f400000000, 0xb991a35b00000000,
+    0x31211f7100000000, 0x76b35bde00000000, 0x2140662400000000,
+    0x66d2228b00000000, 0xee629ea100000000, 0xa9f0da0e00000000,
+    0x0182948e00000000, 0x4610d02100000000, 0xcea06c0b00000000,
+    0x893228a400000000, 0xdec1155e00000000, 0x995351f100000000,
+    0x11e3eddb00000000, 0x5671a97400000000, 0xbd01bf3200000000,
+    0xfa93fb9d00000000, 0x722347b700000000, 0x35b1031800000000,
+    0x62423ee200000000, 0x25d07a4d00000000, 0xad60c66700000000,
+    0xeaf282c800000000, 0x4280cc4800000000, 0x051288e700000000,
+    0x8da234cd00000000, 0xca30706200000000, 0x9dc34d9800000000,
+    0xda51093700000000, 0x52e1b51d00000000, 0x1573f1b200000000,
+    0x430258c600000000, 0x04901c6900000000, 0x8c20a04300000000,
+    0xcbb2e4ec00000000, 0x9c41d91600000000, 0xdbd39db900000000,
+    0x5363219300000000, 0x14f1653c00000000, 0xbc832bbc00000000,
+    0xfb116f1300000000, 0x73a1d33900000000, 0x3433979600000000,
+    0x63c0aa6c00000000, 0x2452eec300000000, 0xace252e900000000,
+    0xeb70164600000000, 0x7a037e6500000000, 0x3d913aca00000000,
+    0xb52186e000000000, 0xf2b3c24f00000000, 0xa540ffb500000000,
+    0xe2d2bb1a00000000, 0x6a62073000000000, 0x2df0439f00000000,
+    0x85820d1f00000000, 0xc21049b000000000, 0x4aa0f59a00000000,
+    0x0d32b13500000000, 0x5ac18ccf00000000, 0x1d53c86000000000,
+    0x95e3744a00000000, 0xd27130e500000000, 0x8400999100000000,
+    0xc392dd3e00000000, 0x4b22611400000000, 0x0cb025bb00000000,
+    0x5b43184100000000, 0x1cd15cee00000000, 0x9461e0c400000000,
+    0xd3f3a46b00000000, 0x7b81eaeb00000000, 0x3c13ae4400000000,
+    0xb4a3126e00000000, 0xf33156c100000000, 0xa4c26b3b00000000,
+    0xe3502f9400000000, 0x6be093be00000000, 0x2c72d71100000000,
+    0xc702c15700000000, 0x809085f800000000, 0x082039d200000000,
+    0x4fb27d7d00000000, 0x1841408700000000, 0x5fd3042800000000,
+    0xd763b80200000000, 0x90f1fcad00000000, 0x3883b22d00000000,
+    0x7f11f68200000000, 0xf7a14aa800000000, 0xb0330e0700000000,
+    0xe7c033fd00000000, 0xa052775200000000, 0x28e2cb7800000000,
+    0x6f708fd700000000, 0x390126a300000000, 0x7e93620c00000000,
+    0xf623de2600000000, 0xb1b19a8900000000, 0xe642a77300000000,
+    0xa1d0e3dc00000000, 0x29605ff600000000, 0x6ef21b5900000000,
+    0xc68055d900000000, 0x8112117600000000, 0x09a2ad5c00000000,
+    0x4e30e9f300000000, 0x19c3d40900000000, 0x5e5190a600000000,
+    0xd6e12c8c00000000, 0x9173682300000000, 0xf406fcca00000000,
+    0xb394b86500000000, 0x3b24044f00000000, 0x7cb640e000000000,
+    0x2b457d1a00000000, 0x6cd739b500000000, 0xe467859f00000000,
+    0xa3f5c13000000000, 0x0b878fb000000000, 0x4c15cb1f00000000,
+    0xc4a5773500000000, 0x8337339a00000000, 0xd4c40e6000000000,
+    0x93564acf00000000, 0x1be6f6e500000000, 0x5c74b24a00000000,
+    0x0a051b3e00000000, 0x4d975f9100000000, 0xc527e3bb00000000,
+    0x82b5a71400000000, 0xd5469aee00000000, 0x92d4de4100000000,
+    0x1a64626b00000000, 0x5df626c400000000, 0xf584684400000000,
+    0xb2162ceb00000000, 0x3aa690c100000000, 0x7d34d46e00000000,
+    0x2ac7e99400000000, 0x6d55ad3b00000000, 0xe5e5111100000000,
+    0xa27755be00000000, 0x490743f800000000, 0x0e95075700000000,
+    0x8625bb7d00000000, 0xc1b7ffd200000000, 0x9644c22800000000,
+    0xd1d6868700000000, 0x59663aad00000000, 0x1ef47e0200000000,
+    0xb686308200000000, 0xf114742d00000000, 0x79a4c80700000000,
+    0x3e368ca800000000, 0x69c5b15200000000, 0x2e57f5fd00000000,
+    0xa6e749d700000000, 0xe1750d7800000000, 0xb704a40c00000000,
+    0xf096e0a300000000, 0x78265c8900000000, 0x3fb4182600000000,
+    0x684725dc00000000, 0x2fd5617300000000, 0xa765dd5900000000,
+    0xe0f799f600000000, 0x4885d77600000000, 0x0f1793d900000000,
+    0x87a72ff300000000, 0xc0356b5c00000000, 0x97c656a600000000,
+    0xd054120900000000, 0x58e4ae2300000000, 0x1f76ea8c00000000,
+    0x8e0582af00000000, 0xc997c60000000000, 0x41277a2a00000000,
+    0x06b53e8500000000, 0x5146037f00000000, 0x16d447d000000000,
+    0x9e64fbfa00000000, 0xd9f6bf5500000000, 0x7184f1d500000000,
+    0x3616b57a00000000, 0xbea6095000000000, 0xf9344dff00000000,
+    0xaec7700500000000, 0xe95534aa00000000, 0x61e5888000000000,
+    0x2677cc2f00000000, 0x7006655b00000000, 0x379421f400000000,
+    0xbf249dde00000000, 0xf8b6d97100000000, 0xaf45e48b00000000,
+    0xe8d7a02400000000, 0x60671c0e00000000, 0x27f558a100000000,
+    0x8f87162100000000, 0xc815528e00000000, 0x40a5eea400000000,
+    0x0737aa0b00000000, 0x50c497f100000000, 0x1756d35e00000000,
+    0x9fe66f7400000000, 0xd8742bdb00000000, 0x33043d9d00000000,
+    0x7496793200000000, 0xfc26c51800000000, 0xbbb481b700000000,
+    0xec47bc4d00000000, 0xabd5f8e200000000, 0x236544c800000000,
+    0x64f7006700000000, 0xcc854ee700000000, 0x8b170a4800000000,
+    0x03a7b66200000000, 0x4435f2cd00000000, 0x13c6cf3700000000,
+    0x54548b9800000000, 0xdce437b200000000, 0x9b76731d00000000,
+    0xcd07da6900000000, 0x8a959ec600000000, 0x022522ec00000000,
+    0x45b7664300000000, 0x12445bb900000000, 0x55d61f1600000000,
+    0xdd66a33c00000000, 0x9af4e79300000000, 0x3286a91300000000,
+    0x7514edbc00000000, 0xfda4519600000000, 0xba36153900000000,
+    0xedc528c300000000, 0xaa576c6c00000000, 0x22e7d04600000000,
+    0x657594e900000000}};
+
+#else /* W == 4 */
+
+local const z_crc_t FAR crc_braid_table[][256] = {
+   {0x00000000, 0x65673b46, 0xcace768c, 0xafa94dca, 0x4eedeb59,
+    0x2b8ad01f, 0x84239dd5, 0xe144a693, 0x9ddbd6b2, 0xf8bcedf4,
+    0x5715a03e, 0x32729b78, 0xd3363deb, 0xb65106ad, 0x19f84b67,
+    0x7c9f7021, 0xe0c6ab25, 0x85a19063, 0x2a08dda9, 0x4f6fe6ef,
+    0xae2b407c, 0xcb4c7b3a, 0x64e536f0, 0x01820db6, 0x7d1d7d97,
+    0x187a46d1, 0xb7d30b1b, 0xd2b4305d, 0x33f096ce, 0x5697ad88,
+    0xf93ee042, 0x9c59db04, 0x1afc500b, 0x7f9b6b4d, 0xd0322687,
+    0xb5551dc1, 0x5411bb52, 0x31768014, 0x9edfcdde, 0xfbb8f698,
+    0x872786b9, 0xe240bdff, 0x4de9f035, 0x288ecb73, 0xc9ca6de0,
+    0xacad56a6, 0x03041b6c, 0x6663202a, 0xfa3afb2e, 0x9f5dc068,
+    0x30f48da2, 0x5593b6e4, 0xb4d71077, 0xd1b02b31, 0x7e1966fb,
+    0x1b7e5dbd, 0x67e12d9c, 0x028616da, 0xad2f5b10, 0xc8486056,
+    0x290cc6c5, 0x4c6bfd83, 0xe3c2b049, 0x86a58b0f, 0x35f8a016,
+    0x509f9b50, 0xff36d69a, 0x9a51eddc, 0x7b154b4f, 0x1e727009,
+    0xb1db3dc3, 0xd4bc0685, 0xa82376a4, 0xcd444de2, 0x62ed0028,
+    0x078a3b6e, 0xe6ce9dfd, 0x83a9a6bb, 0x2c00eb71, 0x4967d037,
+    0xd53e0b33, 0xb0593075, 0x1ff07dbf, 0x7a9746f9, 0x9bd3e06a,
+    0xfeb4db2c, 0x511d96e6, 0x347aada0, 0x48e5dd81, 0x2d82e6c7,
+    0x822bab0d, 0xe74c904b, 0x060836d8, 0x636f0d9e, 0xccc64054,
+    0xa9a17b12, 0x2f04f01d, 0x4a63cb5b, 0xe5ca8691, 0x80adbdd7,
+    0x61e91b44, 0x048e2002, 0xab276dc8, 0xce40568e, 0xb2df26af,
+    0xd7b81de9, 0x78115023, 0x1d766b65, 0xfc32cdf6, 0x9955f6b0,
+    0x36fcbb7a, 0x539b803c, 0xcfc25b38, 0xaaa5607e, 0x050c2db4,
+    0x606b16f2, 0x812fb061, 0xe4488b27, 0x4be1c6ed, 0x2e86fdab,
+    0x52198d8a, 0x377eb6cc, 0x98d7fb06, 0xfdb0c040, 0x1cf466d3,
+    0x79935d95, 0xd63a105f, 0xb35d2b19, 0x6bf1402c, 0x0e967b6a,
+    0xa13f36a0, 0xc4580de6, 0x251cab75, 0x407b9033, 0xefd2ddf9,
+    0x8ab5e6bf, 0xf62a969e, 0x934dadd8, 0x3ce4e012, 0x5983db54,
+    0xb8c77dc7, 0xdda04681, 0x72090b4b, 0x176e300d, 0x8b37eb09,
+    0xee50d04f, 0x41f99d85, 0x249ea6c3, 0xc5da0050, 0xa0bd3b16,
+    0x0f1476dc, 0x6a734d9a, 0x16ec3dbb, 0x738b06fd, 0xdc224b37,
+    0xb9457071, 0x5801d6e2, 0x3d66eda4, 0x92cfa06e, 0xf7a89b28,
+    0x710d1027, 0x146a2b61, 0xbbc366ab, 0xdea45ded, 0x3fe0fb7e,
+    0x5a87c038, 0xf52e8df2, 0x9049b6b4, 0xecd6c695, 0x89b1fdd3,
+    0x2618b019, 0x437f8b5f, 0xa23b2dcc, 0xc75c168a, 0x68f55b40,
+    0x0d926006, 0x91cbbb02, 0xf4ac8044, 0x5b05cd8e, 0x3e62f6c8,
+    0xdf26505b, 0xba416b1d, 0x15e826d7, 0x708f1d91, 0x0c106db0,
+    0x697756f6, 0xc6de1b3c, 0xa3b9207a, 0x42fd86e9, 0x279abdaf,
+    0x8833f065, 0xed54cb23, 0x5e09e03a, 0x3b6edb7c, 0x94c796b6,
+    0xf1a0adf0, 0x10e40b63, 0x75833025, 0xda2a7def, 0xbf4d46a9,
+    0xc3d23688, 0xa6b50dce, 0x091c4004, 0x6c7b7b42, 0x8d3fddd1,
+    0xe858e697, 0x47f1ab5d, 0x2296901b, 0xbecf4b1f, 0xdba87059,
+    0x74013d93, 0x116606d5, 0xf022a046, 0x95459b00, 0x3aecd6ca,
+    0x5f8bed8c, 0x23149dad, 0x4673a6eb, 0xe9daeb21, 0x8cbdd067,
+    0x6df976f4, 0x089e4db2, 0xa7370078, 0xc2503b3e, 0x44f5b031,
+    0x21928b77, 0x8e3bc6bd, 0xeb5cfdfb, 0x0a185b68, 0x6f7f602e,
+    0xc0d62de4, 0xa5b116a2, 0xd92e6683, 0xbc495dc5, 0x13e0100f,
+    0x76872b49, 0x97c38dda, 0xf2a4b69c, 0x5d0dfb56, 0x386ac010,
+    0xa4331b14, 0xc1542052, 0x6efd6d98, 0x0b9a56de, 0xeadef04d,
+    0x8fb9cb0b, 0x201086c1, 0x4577bd87, 0x39e8cda6, 0x5c8ff6e0,
+    0xf326bb2a, 0x9641806c, 0x770526ff, 0x12621db9, 0xbdcb5073,
+    0xd8ac6b35},
+   {0x00000000, 0xd7e28058, 0x74b406f1, 0xa35686a9, 0xe9680de2,
+    0x3e8a8dba, 0x9ddc0b13, 0x4a3e8b4b, 0x09a11d85, 0xde439ddd,
+    0x7d151b74, 0xaaf79b2c, 0xe0c91067, 0x372b903f, 0x947d1696,
+    0x439f96ce, 0x13423b0a, 0xc4a0bb52, 0x67f63dfb, 0xb014bda3,
+    0xfa2a36e8, 0x2dc8b6b0, 0x8e9e3019, 0x597cb041, 0x1ae3268f,
+    0xcd01a6d7, 0x6e57207e, 0xb9b5a026, 0xf38b2b6d, 0x2469ab35,
+    0x873f2d9c, 0x50ddadc4, 0x26847614, 0xf166f64c, 0x523070e5,
+    0x85d2f0bd, 0xcfec7bf6, 0x180efbae, 0xbb587d07, 0x6cbafd5f,
+    0x2f256b91, 0xf8c7ebc9, 0x5b916d60, 0x8c73ed38, 0xc64d6673,
+    0x11afe62b, 0xb2f96082, 0x651be0da, 0x35c64d1e, 0xe224cd46,
+    0x41724bef, 0x9690cbb7, 0xdcae40fc, 0x0b4cc0a4, 0xa81a460d,
+    0x7ff8c655, 0x3c67509b, 0xeb85d0c3, 0x48d3566a, 0x9f31d632,
+    0xd50f5d79, 0x02eddd21, 0xa1bb5b88, 0x7659dbd0, 0x4d08ec28,
+    0x9aea6c70, 0x39bcead9, 0xee5e6a81, 0xa460e1ca, 0x73826192,
+    0xd0d4e73b, 0x07366763, 0x44a9f1ad, 0x934b71f5, 0x301df75c,
+    0xe7ff7704, 0xadc1fc4f, 0x7a237c17, 0xd975fabe, 0x0e977ae6,
+    0x5e4ad722, 0x89a8577a, 0x2afed1d3, 0xfd1c518b, 0xb722dac0,
+    0x60c05a98, 0xc396dc31, 0x14745c69, 0x57ebcaa7, 0x80094aff,
+    0x235fcc56, 0xf4bd4c0e, 0xbe83c745, 0x6961471d, 0xca37c1b4,
+    0x1dd541ec, 0x6b8c9a3c, 0xbc6e1a64, 0x1f389ccd, 0xc8da1c95,
+    0x82e497de, 0x55061786, 0xf650912f, 0x21b21177, 0x622d87b9,
+    0xb5cf07e1, 0x16998148, 0xc17b0110, 0x8b458a5b, 0x5ca70a03,
+    0xfff18caa, 0x28130cf2, 0x78cea136, 0xaf2c216e, 0x0c7aa7c7,
+    0xdb98279f, 0x91a6acd4, 0x46442c8c, 0xe512aa25, 0x32f02a7d,
+    0x716fbcb3, 0xa68d3ceb, 0x05dbba42, 0xd2393a1a, 0x9807b151,
+    0x4fe53109, 0xecb3b7a0, 0x3b5137f8, 0x9a11d850, 0x4df35808,
+    0xeea5dea1, 0x39475ef9, 0x7379d5b2, 0xa49b55ea, 0x07cdd343,
+    0xd02f531b, 0x93b0c5d5, 0x4452458d, 0xe704c324, 0x30e6437c,
+    0x7ad8c837, 0xad3a486f, 0x0e6ccec6, 0xd98e4e9e, 0x8953e35a,
+    0x5eb16302, 0xfde7e5ab, 0x2a0565f3, 0x603beeb8, 0xb7d96ee0,
+    0x148fe849, 0xc36d6811, 0x80f2fedf, 0x57107e87, 0xf446f82e,
+    0x23a47876, 0x699af33d, 0xbe787365, 0x1d2ef5cc, 0xcacc7594,
+    0xbc95ae44, 0x6b772e1c, 0xc821a8b5, 0x1fc328ed, 0x55fda3a6,
+    0x821f23fe, 0x2149a557, 0xf6ab250f, 0xb534b3c1, 0x62d63399,
+    0xc180b530, 0x16623568, 0x5c5cbe23, 0x8bbe3e7b, 0x28e8b8d2,
+    0xff0a388a, 0xafd7954e, 0x78351516, 0xdb6393bf, 0x0c8113e7,
+    0x46bf98ac, 0x915d18f4, 0x320b9e5d, 0xe5e91e05, 0xa67688cb,
+    0x71940893, 0xd2c28e3a, 0x05200e62, 0x4f1e8529, 0x98fc0571,
+    0x3baa83d8, 0xec480380, 0xd7193478, 0x00fbb420, 0xa3ad3289,
+    0x744fb2d1, 0x3e71399a, 0xe993b9c2, 0x4ac53f6b, 0x9d27bf33,
+    0xdeb829fd, 0x095aa9a5, 0xaa0c2f0c, 0x7deeaf54, 0x37d0241f,
+    0xe032a447, 0x436422ee, 0x9486a2b6, 0xc45b0f72, 0x13b98f2a,
+    0xb0ef0983, 0x670d89db, 0x2d330290, 0xfad182c8, 0x59870461,
+    0x8e658439, 0xcdfa12f7, 0x1a1892af, 0xb94e1406, 0x6eac945e,
+    0x24921f15, 0xf3709f4d, 0x502619e4, 0x87c499bc, 0xf19d426c,
+    0x267fc234, 0x8529449d, 0x52cbc4c5, 0x18f54f8e, 0xcf17cfd6,
+    0x6c41497f, 0xbba3c927, 0xf83c5fe9, 0x2fdedfb1, 0x8c885918,
+    0x5b6ad940, 0x1154520b, 0xc6b6d253, 0x65e054fa, 0xb202d4a2,
+    0xe2df7966, 0x353df93e, 0x966b7f97, 0x4189ffcf, 0x0bb77484,
+    0xdc55f4dc, 0x7f037275, 0xa8e1f22d, 0xeb7e64e3, 0x3c9ce4bb,
+    0x9fca6212, 0x4828e24a, 0x02166901, 0xd5f4e959, 0x76a26ff0,
+    0xa140efa8},
+   {0x00000000, 0xef52b6e1, 0x05d46b83, 0xea86dd62, 0x0ba8d706,
+    0xe4fa61e7, 0x0e7cbc85, 0xe12e0a64, 0x1751ae0c, 0xf80318ed,
+    0x1285c58f, 0xfdd7736e, 0x1cf9790a, 0xf3abcfeb, 0x192d1289,
+    0xf67fa468, 0x2ea35c18, 0xc1f1eaf9, 0x2b77379b, 0xc425817a,
+    0x250b8b1e, 0xca593dff, 0x20dfe09d, 0xcf8d567c, 0x39f2f214,
+    0xd6a044f5, 0x3c269997, 0xd3742f76, 0x325a2512, 0xdd0893f3,
+    0x378e4e91, 0xd8dcf870, 0x5d46b830, 0xb2140ed1, 0x5892d3b3,
+    0xb7c06552, 0x56ee6f36, 0xb9bcd9d7, 0x533a04b5, 0xbc68b254,
+    0x4a17163c, 0xa545a0dd, 0x4fc37dbf, 0xa091cb5e, 0x41bfc13a,
+    0xaeed77db, 0x446baab9, 0xab391c58, 0x73e5e428, 0x9cb752c9,
+    0x76318fab, 0x9963394a, 0x784d332e, 0x971f85cf, 0x7d9958ad,
+    0x92cbee4c, 0x64b44a24, 0x8be6fcc5, 0x616021a7, 0x8e329746,
+    0x6f1c9d22, 0x804e2bc3, 0x6ac8f6a1, 0x859a4040, 0xba8d7060,
+    0x55dfc681, 0xbf591be3, 0x500bad02, 0xb125a766, 0x5e771187,
+    0xb4f1cce5, 0x5ba37a04, 0xaddcde6c, 0x428e688d, 0xa808b5ef,
+    0x475a030e, 0xa674096a, 0x4926bf8b, 0xa3a062e9, 0x4cf2d408,
+    0x942e2c78, 0x7b7c9a99, 0x91fa47fb, 0x7ea8f11a, 0x9f86fb7e,
+    0x70d44d9f, 0x9a5290fd, 0x7500261c, 0x837f8274, 0x6c2d3495,
+    0x86abe9f7, 0x69f95f16, 0x88d75572, 0x6785e393, 0x8d033ef1,
+    0x62518810, 0xe7cbc850, 0x08997eb1, 0xe21fa3d3, 0x0d4d1532,
+    0xec631f56, 0x0331a9b7, 0xe9b774d5, 0x06e5c234, 0xf09a665c,
+    0x1fc8d0bd, 0xf54e0ddf, 0x1a1cbb3e, 0xfb32b15a, 0x146007bb,
+    0xfee6dad9, 0x11b46c38, 0xc9689448, 0x263a22a9, 0xccbcffcb,
+    0x23ee492a, 0xc2c0434e, 0x2d92f5af, 0xc71428cd, 0x28469e2c,
+    0xde393a44, 0x316b8ca5, 0xdbed51c7, 0x34bfe726, 0xd591ed42,
+    0x3ac35ba3, 0xd04586c1, 0x3f173020, 0xae6be681, 0x41395060,
+    0xabbf8d02, 0x44ed3be3, 0xa5c33187, 0x4a918766, 0xa0175a04,
+    0x4f45ece5, 0xb93a488d, 0x5668fe6c, 0xbcee230e, 0x53bc95ef,
+    0xb2929f8b, 0x5dc0296a, 0xb746f408, 0x581442e9, 0x80c8ba99,
+    0x6f9a0c78, 0x851cd11a, 0x6a4e67fb, 0x8b606d9f, 0x6432db7e,
+    0x8eb4061c, 0x61e6b0fd, 0x97991495, 0x78cba274, 0x924d7f16,
+    0x7d1fc9f7, 0x9c31c393, 0x73637572, 0x99e5a810, 0x76b71ef1,
+    0xf32d5eb1, 0x1c7fe850, 0xf6f93532, 0x19ab83d3, 0xf88589b7,
+    0x17d73f56, 0xfd51e234, 0x120354d5, 0xe47cf0bd, 0x0b2e465c,
+    0xe1a89b3e, 0x0efa2ddf, 0xefd427bb, 0x0086915a, 0xea004c38,
+    0x0552fad9, 0xdd8e02a9, 0x32dcb448, 0xd85a692a, 0x3708dfcb,
+    0xd626d5af, 0x3974634e, 0xd3f2be2c, 0x3ca008cd, 0xcadfaca5,
+    0x258d1a44, 0xcf0bc726, 0x205971c7, 0xc1777ba3, 0x2e25cd42,
+    0xc4a31020, 0x2bf1a6c1, 0x14e696e1, 0xfbb42000, 0x1132fd62,
+    0xfe604b83, 0x1f4e41e7, 0xf01cf706, 0x1a9a2a64, 0xf5c89c85,
+    0x03b738ed, 0xece58e0c, 0x0663536e, 0xe931e58f, 0x081fefeb,
+    0xe74d590a, 0x0dcb8468, 0xe2993289, 0x3a45caf9, 0xd5177c18,
+    0x3f91a17a, 0xd0c3179b, 0x31ed1dff, 0xdebfab1e, 0x3439767c,
+    0xdb6bc09d, 0x2d1464f5, 0xc246d214, 0x28c00f76, 0xc792b997,
+    0x26bcb3f3, 0xc9ee0512, 0x2368d870, 0xcc3a6e91, 0x49a02ed1,
+    0xa6f29830, 0x4c744552, 0xa326f3b3, 0x4208f9d7, 0xad5a4f36,
+    0x47dc9254, 0xa88e24b5, 0x5ef180dd, 0xb1a3363c, 0x5b25eb5e,
+    0xb4775dbf, 0x555957db, 0xba0be13a, 0x508d3c58, 0xbfdf8ab9,
+    0x670372c9, 0x8851c428, 0x62d7194a, 0x8d85afab, 0x6caba5cf,
+    0x83f9132e, 0x697fce4c, 0x862d78ad, 0x7052dcc5, 0x9f006a24,
+    0x7586b746, 0x9ad401a7, 0x7bfa0bc3, 0x94a8bd22, 0x7e2e6040,
+    0x917cd6a1},
+   {0x00000000, 0x87a6cb43, 0xd43c90c7, 0x539a5b84, 0x730827cf,
+    0xf4aeec8c, 0xa734b708, 0x20927c4b, 0xe6104f9e, 0x61b684dd,
+    0x322cdf59, 0xb58a141a, 0x95186851, 0x12bea312, 0x4124f896,
+    0xc68233d5, 0x1751997d, 0x90f7523e, 0xc36d09ba, 0x44cbc2f9,
+    0x6459beb2, 0xe3ff75f1, 0xb0652e75, 0x37c3e536, 0xf141d6e3,
+    0x76e71da0, 0x257d4624, 0xa2db8d67, 0x8249f12c, 0x05ef3a6f,
+    0x567561eb, 0xd1d3aaa8, 0x2ea332fa, 0xa905f9b9, 0xfa9fa23d,
+    0x7d39697e, 0x5dab1535, 0xda0dde76, 0x899785f2, 0x0e314eb1,
+    0xc8b37d64, 0x4f15b627, 0x1c8feda3, 0x9b2926e0, 0xbbbb5aab,
+    0x3c1d91e8, 0x6f87ca6c, 0xe821012f, 0x39f2ab87, 0xbe5460c4,
+    0xedce3b40, 0x6a68f003, 0x4afa8c48, 0xcd5c470b, 0x9ec61c8f,
+    0x1960d7cc, 0xdfe2e419, 0x58442f5a, 0x0bde74de, 0x8c78bf9d,
+    0xaceac3d6, 0x2b4c0895, 0x78d65311, 0xff709852, 0x5d4665f4,
+    0xdae0aeb7, 0x897af533, 0x0edc3e70, 0x2e4e423b, 0xa9e88978,
+    0xfa72d2fc, 0x7dd419bf, 0xbb562a6a, 0x3cf0e129, 0x6f6abaad,
+    0xe8cc71ee, 0xc85e0da5, 0x4ff8c6e6, 0x1c629d62, 0x9bc45621,
+    0x4a17fc89, 0xcdb137ca, 0x9e2b6c4e, 0x198da70d, 0x391fdb46,
+    0xbeb91005, 0xed234b81, 0x6a8580c2, 0xac07b317, 0x2ba17854,
+    0x783b23d0, 0xff9de893, 0xdf0f94d8, 0x58a95f9b, 0x0b33041f,
+    0x8c95cf5c, 0x73e5570e, 0xf4439c4d, 0xa7d9c7c9, 0x207f0c8a,
+    0x00ed70c1, 0x874bbb82, 0xd4d1e006, 0x53772b45, 0x95f51890,
+    0x1253d3d3, 0x41c98857, 0xc66f4314, 0xe6fd3f5f, 0x615bf41c,
+    0x32c1af98, 0xb56764db, 0x64b4ce73, 0xe3120530, 0xb0885eb4,
+    0x372e95f7, 0x17bce9bc, 0x901a22ff, 0xc380797b, 0x4426b238,
+    0x82a481ed, 0x05024aae, 0x5698112a, 0xd13eda69, 0xf1aca622,
+    0x760a6d61, 0x259036e5, 0xa236fda6, 0xba8ccbe8, 0x3d2a00ab,
+    0x6eb05b2f, 0xe916906c, 0xc984ec27, 0x4e222764, 0x1db87ce0,
+    0x9a1eb7a3, 0x5c9c8476, 0xdb3a4f35, 0x88a014b1, 0x0f06dff2,
+    0x2f94a3b9, 0xa83268fa, 0xfba8337e, 0x7c0ef83d, 0xaddd5295,
+    0x2a7b99d6, 0x79e1c252, 0xfe470911, 0xded5755a, 0x5973be19,
+    0x0ae9e59d, 0x8d4f2ede, 0x4bcd1d0b, 0xcc6bd648, 0x9ff18dcc,
+    0x1857468f, 0x38c53ac4, 0xbf63f187, 0xecf9aa03, 0x6b5f6140,
+    0x942ff912, 0x13893251, 0x401369d5, 0xc7b5a296, 0xe727dedd,
+    0x6081159e, 0x331b4e1a, 0xb4bd8559, 0x723fb68c, 0xf5997dcf,
+    0xa603264b, 0x21a5ed08, 0x01379143, 0x86915a00, 0xd50b0184,
+    0x52adcac7, 0x837e606f, 0x04d8ab2c, 0x5742f0a8, 0xd0e43beb,
+    0xf07647a0, 0x77d08ce3, 0x244ad767, 0xa3ec1c24, 0x656e2ff1,
+    0xe2c8e4b2, 0xb152bf36, 0x36f47475, 0x1666083e, 0x91c0c37d,
+    0xc25a98f9, 0x45fc53ba, 0xe7caae1c, 0x606c655f, 0x33f63edb,
+    0xb450f598, 0x94c289d3, 0x13644290, 0x40fe1914, 0xc758d257,
+    0x01dae182, 0x867c2ac1, 0xd5e67145, 0x5240ba06, 0x72d2c64d,
+    0xf5740d0e, 0xa6ee568a, 0x21489dc9, 0xf09b3761, 0x773dfc22,
+    0x24a7a7a6, 0xa3016ce5, 0x839310ae, 0x0435dbed, 0x57af8069,
+    0xd0094b2a, 0x168b78ff, 0x912db3bc, 0xc2b7e838, 0x4511237b,
+    0x65835f30, 0xe2259473, 0xb1bfcff7, 0x361904b4, 0xc9699ce6,
+    0x4ecf57a5, 0x1d550c21, 0x9af3c762, 0xba61bb29, 0x3dc7706a,
+    0x6e5d2bee, 0xe9fbe0ad, 0x2f79d378, 0xa8df183b, 0xfb4543bf,
+    0x7ce388fc, 0x5c71f4b7, 0xdbd73ff4, 0x884d6470, 0x0febaf33,
+    0xde38059b, 0x599eced8, 0x0a04955c, 0x8da25e1f, 0xad302254,
+    0x2a96e917, 0x790cb293, 0xfeaa79d0, 0x38284a05, 0xbf8e8146,
+    0xec14dac2, 0x6bb21181, 0x4b206dca, 0xcc86a689, 0x9f1cfd0d,
+    0x18ba364e}};
+
+local const z_word_t FAR crc_braid_big_table[][256] = {
+   {0x00000000, 0x43cba687, 0xc7903cd4, 0x845b9a53, 0xcf270873,
+    0x8cecaef4, 0x08b734a7, 0x4b7c9220, 0x9e4f10e6, 0xdd84b661,
+    0x59df2c32, 0x1a148ab5, 0x51681895, 0x12a3be12, 0x96f82441,
+    0xd53382c6, 0x7d995117, 0x3e52f790, 0xba096dc3, 0xf9c2cb44,
+    0xb2be5964, 0xf175ffe3, 0x752e65b0, 0x36e5c337, 0xe3d641f1,
+    0xa01de776, 0x24467d25, 0x678ddba2, 0x2cf14982, 0x6f3aef05,
+    0xeb617556, 0xa8aad3d1, 0xfa32a32e, 0xb9f905a9, 0x3da29ffa,
+    0x7e69397d, 0x3515ab5d, 0x76de0dda, 0xf2859789, 0xb14e310e,
+    0x647db3c8, 0x27b6154f, 0xa3ed8f1c, 0xe026299b, 0xab5abbbb,
+    0xe8911d3c, 0x6cca876f, 0x2f0121e8, 0x87abf239, 0xc46054be,
+    0x403bceed, 0x03f0686a, 0x488cfa4a, 0x0b475ccd, 0x8f1cc69e,
+    0xccd76019, 0x19e4e2df, 0x5a2f4458, 0xde74de0b, 0x9dbf788c,
+    0xd6c3eaac, 0x95084c2b, 0x1153d678, 0x529870ff, 0xf465465d,
+    0xb7aee0da, 0x33f57a89, 0x703edc0e, 0x3b424e2e, 0x7889e8a9,
+    0xfcd272fa, 0xbf19d47d, 0x6a2a56bb, 0x29e1f03c, 0xadba6a6f,
+    0xee71cce8, 0xa50d5ec8, 0xe6c6f84f, 0x629d621c, 0x2156c49b,
+    0x89fc174a, 0xca37b1cd, 0x4e6c2b9e, 0x0da78d19, 0x46db1f39,
+    0x0510b9be, 0x814b23ed, 0xc280856a, 0x17b307ac, 0x5478a12b,
+    0xd0233b78, 0x93e89dff, 0xd8940fdf, 0x9b5fa958, 0x1f04330b,
+    0x5ccf958c, 0x0e57e573, 0x4d9c43f4, 0xc9c7d9a7, 0x8a0c7f20,
+    0xc170ed00, 0x82bb4b87, 0x06e0d1d4, 0x452b7753, 0x9018f595,
+    0xd3d35312, 0x5788c941, 0x14436fc6, 0x5f3ffde6, 0x1cf45b61,
+    0x98afc132, 0xdb6467b5, 0x73ceb464, 0x300512e3, 0xb45e88b0,
+    0xf7952e37, 0xbce9bc17, 0xff221a90, 0x7b7980c3, 0x38b22644,
+    0xed81a482, 0xae4a0205, 0x2a119856, 0x69da3ed1, 0x22a6acf1,
+    0x616d0a76, 0xe5369025, 0xa6fd36a2, 0xe8cb8cba, 0xab002a3d,
+    0x2f5bb06e, 0x6c9016e9, 0x27ec84c9, 0x6427224e, 0xe07cb81d,
+    0xa3b71e9a, 0x76849c5c, 0x354f3adb, 0xb114a088, 0xf2df060f,
+    0xb9a3942f, 0xfa6832a8, 0x7e33a8fb, 0x3df80e7c, 0x9552ddad,
+    0xd6997b2a, 0x52c2e179, 0x110947fe, 0x5a75d5de, 0x19be7359,
+    0x9de5e90a, 0xde2e4f8d, 0x0b1dcd4b, 0x48d66bcc, 0xcc8df19f,
+    0x8f465718, 0xc43ac538, 0x87f163bf, 0x03aaf9ec, 0x40615f6b,
+    0x12f92f94, 0x51328913, 0xd5691340, 0x96a2b5c7, 0xddde27e7,
+    0x9e158160, 0x1a4e1b33, 0x5985bdb4, 0x8cb63f72, 0xcf7d99f5,
+    0x4b2603a6, 0x08eda521, 0x43913701, 0x005a9186, 0x84010bd5,
+    0xc7caad52, 0x6f607e83, 0x2cabd804, 0xa8f04257, 0xeb3be4d0,
+    0xa04776f0, 0xe38cd077, 0x67d74a24, 0x241ceca3, 0xf12f6e65,
+    0xb2e4c8e2, 0x36bf52b1, 0x7574f436, 0x3e086616, 0x7dc3c091,
+    0xf9985ac2, 0xba53fc45, 0x1caecae7, 0x5f656c60, 0xdb3ef633,
+    0x98f550b4, 0xd389c294, 0x90426413, 0x1419fe40, 0x57d258c7,
+    0x82e1da01, 0xc12a7c86, 0x4571e6d5, 0x06ba4052, 0x4dc6d272,
+    0x0e0d74f5, 0x8a56eea6, 0xc99d4821, 0x61379bf0, 0x22fc3d77,
+    0xa6a7a724, 0xe56c01a3, 0xae109383, 0xeddb3504, 0x6980af57,
+    0x2a4b09d0, 0xff788b16, 0xbcb32d91, 0x38e8b7c2, 0x7b231145,
+    0x305f8365, 0x739425e2, 0xf7cfbfb1, 0xb4041936, 0xe69c69c9,
+    0xa557cf4e, 0x210c551d, 0x62c7f39a, 0x29bb61ba, 0x6a70c73d,
+    0xee2b5d6e, 0xade0fbe9, 0x78d3792f, 0x3b18dfa8, 0xbf4345fb,
+    0xfc88e37c, 0xb7f4715c, 0xf43fd7db, 0x70644d88, 0x33afeb0f,
+    0x9b0538de, 0xd8ce9e59, 0x5c95040a, 0x1f5ea28d, 0x542230ad,
+    0x17e9962a, 0x93b20c79, 0xd079aafe, 0x054a2838, 0x46818ebf,
+    0xc2da14ec, 0x8111b26b, 0xca6d204b, 0x89a686cc, 0x0dfd1c9f,
+    0x4e36ba18},
+   {0x00000000, 0xe1b652ef, 0x836bd405, 0x62dd86ea, 0x06d7a80b,
+    0xe761fae4, 0x85bc7c0e, 0x640a2ee1, 0x0cae5117, 0xed1803f8,
+    0x8fc58512, 0x6e73d7fd, 0x0a79f91c, 0xebcfabf3, 0x89122d19,
+    0x68a47ff6, 0x185ca32e, 0xf9eaf1c1, 0x9b37772b, 0x7a8125c4,
+    0x1e8b0b25, 0xff3d59ca, 0x9de0df20, 0x7c568dcf, 0x14f2f239,
+    0xf544a0d6, 0x9799263c, 0x762f74d3, 0x12255a32, 0xf39308dd,
+    0x914e8e37, 0x70f8dcd8, 0x30b8465d, 0xd10e14b2, 0xb3d39258,
+    0x5265c0b7, 0x366fee56, 0xd7d9bcb9, 0xb5043a53, 0x54b268bc,
+    0x3c16174a, 0xdda045a5, 0xbf7dc34f, 0x5ecb91a0, 0x3ac1bf41,
+    0xdb77edae, 0xb9aa6b44, 0x581c39ab, 0x28e4e573, 0xc952b79c,
+    0xab8f3176, 0x4a396399, 0x2e334d78, 0xcf851f97, 0xad58997d,
+    0x4ceecb92, 0x244ab464, 0xc5fce68b, 0xa7216061, 0x4697328e,
+    0x229d1c6f, 0xc32b4e80, 0xa1f6c86a, 0x40409a85, 0x60708dba,
+    0x81c6df55, 0xe31b59bf, 0x02ad0b50, 0x66a725b1, 0x8711775e,
+    0xe5ccf1b4, 0x047aa35b, 0x6cdedcad, 0x8d688e42, 0xefb508a8,
+    0x0e035a47, 0x6a0974a6, 0x8bbf2649, 0xe962a0a3, 0x08d4f24c,
+    0x782c2e94, 0x999a7c7b, 0xfb47fa91, 0x1af1a87e, 0x7efb869f,
+    0x9f4dd470, 0xfd90529a, 0x1c260075, 0x74827f83, 0x95342d6c,
+    0xf7e9ab86, 0x165ff969, 0x7255d788, 0x93e38567, 0xf13e038d,
+    0x10885162, 0x50c8cbe7, 0xb17e9908, 0xd3a31fe2, 0x32154d0d,
+    0x561f63ec, 0xb7a93103, 0xd574b7e9, 0x34c2e506, 0x5c669af0,
+    0xbdd0c81f, 0xdf0d4ef5, 0x3ebb1c1a, 0x5ab132fb, 0xbb076014,
+    0xd9dae6fe, 0x386cb411, 0x489468c9, 0xa9223a26, 0xcbffbccc,
+    0x2a49ee23, 0x4e43c0c2, 0xaff5922d, 0xcd2814c7, 0x2c9e4628,
+    0x443a39de, 0xa58c6b31, 0xc751eddb, 0x26e7bf34, 0x42ed91d5,
+    0xa35bc33a, 0xc18645d0, 0x2030173f, 0x81e66bae, 0x60503941,
+    0x028dbfab, 0xe33bed44, 0x8731c3a5, 0x6687914a, 0x045a17a0,
+    0xe5ec454f, 0x8d483ab9, 0x6cfe6856, 0x0e23eebc, 0xef95bc53,
+    0x8b9f92b2, 0x6a29c05d, 0x08f446b7, 0xe9421458, 0x99bac880,
+    0x780c9a6f, 0x1ad11c85, 0xfb674e6a, 0x9f6d608b, 0x7edb3264,
+    0x1c06b48e, 0xfdb0e661, 0x95149997, 0x74a2cb78, 0x167f4d92,
+    0xf7c91f7d, 0x93c3319c, 0x72756373, 0x10a8e599, 0xf11eb776,
+    0xb15e2df3, 0x50e87f1c, 0x3235f9f6, 0xd383ab19, 0xb78985f8,
+    0x563fd717, 0x34e251fd, 0xd5540312, 0xbdf07ce4, 0x5c462e0b,
+    0x3e9ba8e1, 0xdf2dfa0e, 0xbb27d4ef, 0x5a918600, 0x384c00ea,
+    0xd9fa5205, 0xa9028edd, 0x48b4dc32, 0x2a695ad8, 0xcbdf0837,
+    0xafd526d6, 0x4e637439, 0x2cbef2d3, 0xcd08a03c, 0xa5acdfca,
+    0x441a8d25, 0x26c70bcf, 0xc7715920, 0xa37b77c1, 0x42cd252e,
+    0x2010a3c4, 0xc1a6f12b, 0xe196e614, 0x0020b4fb, 0x62fd3211,
+    0x834b60fe, 0xe7414e1f, 0x06f71cf0, 0x642a9a1a, 0x859cc8f5,
+    0xed38b703, 0x0c8ee5ec, 0x6e536306, 0x8fe531e9, 0xebef1f08,
+    0x0a594de7, 0x6884cb0d, 0x893299e2, 0xf9ca453a, 0x187c17d5,
+    0x7aa1913f, 0x9b17c3d0, 0xff1ded31, 0x1eabbfde, 0x7c763934,
+    0x9dc06bdb, 0xf564142d, 0x14d246c2, 0x760fc028, 0x97b992c7,
+    0xf3b3bc26, 0x1205eec9, 0x70d86823, 0x916e3acc, 0xd12ea049,
+    0x3098f2a6, 0x5245744c, 0xb3f326a3, 0xd7f90842, 0x364f5aad,
+    0x5492dc47, 0xb5248ea8, 0xdd80f15e, 0x3c36a3b1, 0x5eeb255b,
+    0xbf5d77b4, 0xdb575955, 0x3ae10bba, 0x583c8d50, 0xb98adfbf,
+    0xc9720367, 0x28c45188, 0x4a19d762, 0xabaf858d, 0xcfa5ab6c,
+    0x2e13f983, 0x4cce7f69, 0xad782d86, 0xc5dc5270, 0x246a009f,
+    0x46b78675, 0xa701d49a, 0xc30bfa7b, 0x22bda894, 0x40602e7e,
+    0xa1d67c91},
+   {0x00000000, 0x5880e2d7, 0xf106b474, 0xa98656a3, 0xe20d68e9,
+    0xba8d8a3e, 0x130bdc9d, 0x4b8b3e4a, 0x851da109, 0xdd9d43de,
+    0x741b157d, 0x2c9bf7aa, 0x6710c9e0, 0x3f902b37, 0x96167d94,
+    0xce969f43, 0x0a3b4213, 0x52bba0c4, 0xfb3df667, 0xa3bd14b0,
+    0xe8362afa, 0xb0b6c82d, 0x19309e8e, 0x41b07c59, 0x8f26e31a,
+    0xd7a601cd, 0x7e20576e, 0x26a0b5b9, 0x6d2b8bf3, 0x35ab6924,
+    0x9c2d3f87, 0xc4addd50, 0x14768426, 0x4cf666f1, 0xe5703052,
+    0xbdf0d285, 0xf67beccf, 0xaefb0e18, 0x077d58bb, 0x5ffdba6c,
+    0x916b252f, 0xc9ebc7f8, 0x606d915b, 0x38ed738c, 0x73664dc6,
+    0x2be6af11, 0x8260f9b2, 0xdae01b65, 0x1e4dc635, 0x46cd24e2,
+    0xef4b7241, 0xb7cb9096, 0xfc40aedc, 0xa4c04c0b, 0x0d461aa8,
+    0x55c6f87f, 0x9b50673c, 0xc3d085eb, 0x6a56d348, 0x32d6319f,
+    0x795d0fd5, 0x21dded02, 0x885bbba1, 0xd0db5976, 0x28ec084d,
+    0x706cea9a, 0xd9eabc39, 0x816a5eee, 0xcae160a4, 0x92618273,
+    0x3be7d4d0, 0x63673607, 0xadf1a944, 0xf5714b93, 0x5cf71d30,
+    0x0477ffe7, 0x4ffcc1ad, 0x177c237a, 0xbefa75d9, 0xe67a970e,
+    0x22d74a5e, 0x7a57a889, 0xd3d1fe2a, 0x8b511cfd, 0xc0da22b7,
+    0x985ac060, 0x31dc96c3, 0x695c7414, 0xa7caeb57, 0xff4a0980,
+    0x56cc5f23, 0x0e4cbdf4, 0x45c783be, 0x1d476169, 0xb4c137ca,
+    0xec41d51d, 0x3c9a8c6b, 0x641a6ebc, 0xcd9c381f, 0x951cdac8,
+    0xde97e482, 0x86170655, 0x2f9150f6, 0x7711b221, 0xb9872d62,
+    0xe107cfb5, 0x48819916, 0x10017bc1, 0x5b8a458b, 0x030aa75c,
+    0xaa8cf1ff, 0xf20c1328, 0x36a1ce78, 0x6e212caf, 0xc7a77a0c,
+    0x9f2798db, 0xd4aca691, 0x8c2c4446, 0x25aa12e5, 0x7d2af032,
+    0xb3bc6f71, 0xeb3c8da6, 0x42badb05, 0x1a3a39d2, 0x51b10798,
+    0x0931e54f, 0xa0b7b3ec, 0xf837513b, 0x50d8119a, 0x0858f34d,
+    0xa1dea5ee, 0xf95e4739, 0xb2d57973, 0xea559ba4, 0x43d3cd07,
+    0x1b532fd0, 0xd5c5b093, 0x8d455244, 0x24c304e7, 0x7c43e630,
+    0x37c8d87a, 0x6f483aad, 0xc6ce6c0e, 0x9e4e8ed9, 0x5ae35389,
+    0x0263b15e, 0xabe5e7fd, 0xf365052a, 0xb8ee3b60, 0xe06ed9b7,
+    0x49e88f14, 0x11686dc3, 0xdffef280, 0x877e1057, 0x2ef846f4,
+    0x7678a423, 0x3df39a69, 0x657378be, 0xccf52e1d, 0x9475ccca,
+    0x44ae95bc, 0x1c2e776b, 0xb5a821c8, 0xed28c31f, 0xa6a3fd55,
+    0xfe231f82, 0x57a54921, 0x0f25abf6, 0xc1b334b5, 0x9933d662,
+    0x30b580c1, 0x68356216, 0x23be5c5c, 0x7b3ebe8b, 0xd2b8e828,
+    0x8a380aff, 0x4e95d7af, 0x16153578, 0xbf9363db, 0xe713810c,
+    0xac98bf46, 0xf4185d91, 0x5d9e0b32, 0x051ee9e5, 0xcb8876a6,
+    0x93089471, 0x3a8ec2d2, 0x620e2005, 0x29851e4f, 0x7105fc98,
+    0xd883aa3b, 0x800348ec, 0x783419d7, 0x20b4fb00, 0x8932ada3,
+    0xd1b24f74, 0x9a39713e, 0xc2b993e9, 0x6b3fc54a, 0x33bf279d,
+    0xfd29b8de, 0xa5a95a09, 0x0c2f0caa, 0x54afee7d, 0x1f24d037,
+    0x47a432e0, 0xee226443, 0xb6a28694, 0x720f5bc4, 0x2a8fb913,
+    0x8309efb0, 0xdb890d67, 0x9002332d, 0xc882d1fa, 0x61048759,
+    0x3984658e, 0xf712facd, 0xaf92181a, 0x06144eb9, 0x5e94ac6e,
+    0x151f9224, 0x4d9f70f3, 0xe4192650, 0xbc99c487, 0x6c429df1,
+    0x34c27f26, 0x9d442985, 0xc5c4cb52, 0x8e4ff518, 0xd6cf17cf,
+    0x7f49416c, 0x27c9a3bb, 0xe95f3cf8, 0xb1dfde2f, 0x1859888c,
+    0x40d96a5b, 0x0b525411, 0x53d2b6c6, 0xfa54e065, 0xa2d402b2,
+    0x6679dfe2, 0x3ef93d35, 0x977f6b96, 0xcfff8941, 0x8474b70b,
+    0xdcf455dc, 0x7572037f, 0x2df2e1a8, 0xe3647eeb, 0xbbe49c3c,
+    0x1262ca9f, 0x4ae22848, 0x01691602, 0x59e9f4d5, 0xf06fa276,
+    0xa8ef40a1},
+   {0x00000000, 0x463b6765, 0x8c76ceca, 0xca4da9af, 0x59ebed4e,
+    0x1fd08a2b, 0xd59d2384, 0x93a644e1, 0xb2d6db9d, 0xf4edbcf8,
+    0x3ea01557, 0x789b7232, 0xeb3d36d3, 0xad0651b6, 0x674bf819,
+    0x21709f7c, 0x25abc6e0, 0x6390a185, 0xa9dd082a, 0xefe66f4f,
+    0x7c402bae, 0x3a7b4ccb, 0xf036e564, 0xb60d8201, 0x977d1d7d,
+    0xd1467a18, 0x1b0bd3b7, 0x5d30b4d2, 0xce96f033, 0x88ad9756,
+    0x42e03ef9, 0x04db599c, 0x0b50fc1a, 0x4d6b9b7f, 0x872632d0,
+    0xc11d55b5, 0x52bb1154, 0x14807631, 0xdecddf9e, 0x98f6b8fb,
+    0xb9862787, 0xffbd40e2, 0x35f0e94d, 0x73cb8e28, 0xe06dcac9,
+    0xa656adac, 0x6c1b0403, 0x2a206366, 0x2efb3afa, 0x68c05d9f,
+    0xa28df430, 0xe4b69355, 0x7710d7b4, 0x312bb0d1, 0xfb66197e,
+    0xbd5d7e1b, 0x9c2de167, 0xda168602, 0x105b2fad, 0x566048c8,
+    0xc5c60c29, 0x83fd6b4c, 0x49b0c2e3, 0x0f8ba586, 0x16a0f835,
+    0x509b9f50, 0x9ad636ff, 0xdced519a, 0x4f4b157b, 0x0970721e,
+    0xc33ddbb1, 0x8506bcd4, 0xa47623a8, 0xe24d44cd, 0x2800ed62,
+    0x6e3b8a07, 0xfd9dcee6, 0xbba6a983, 0x71eb002c, 0x37d06749,
+    0x330b3ed5, 0x753059b0, 0xbf7df01f, 0xf946977a, 0x6ae0d39b,
+    0x2cdbb4fe, 0xe6961d51, 0xa0ad7a34, 0x81dde548, 0xc7e6822d,
+    0x0dab2b82, 0x4b904ce7, 0xd8360806, 0x9e0d6f63, 0x5440c6cc,
+    0x127ba1a9, 0x1df0042f, 0x5bcb634a, 0x9186cae5, 0xd7bdad80,
+    0x441be961, 0x02208e04, 0xc86d27ab, 0x8e5640ce, 0xaf26dfb2,
+    0xe91db8d7, 0x23501178, 0x656b761d, 0xf6cd32fc, 0xb0f65599,
+    0x7abbfc36, 0x3c809b53, 0x385bc2cf, 0x7e60a5aa, 0xb42d0c05,
+    0xf2166b60, 0x61b02f81, 0x278b48e4, 0xedc6e14b, 0xabfd862e,
+    0x8a8d1952, 0xccb67e37, 0x06fbd798, 0x40c0b0fd, 0xd366f41c,
+    0x955d9379, 0x5f103ad6, 0x192b5db3, 0x2c40f16b, 0x6a7b960e,
+    0xa0363fa1, 0xe60d58c4, 0x75ab1c25, 0x33907b40, 0xf9ddd2ef,
+    0xbfe6b58a, 0x9e962af6, 0xd8ad4d93, 0x12e0e43c, 0x54db8359,
+    0xc77dc7b8, 0x8146a0dd, 0x4b0b0972, 0x0d306e17, 0x09eb378b,
+    0x4fd050ee, 0x859df941, 0xc3a69e24, 0x5000dac5, 0x163bbda0,
+    0xdc76140f, 0x9a4d736a, 0xbb3dec16, 0xfd068b73, 0x374b22dc,
+    0x717045b9, 0xe2d60158, 0xa4ed663d, 0x6ea0cf92, 0x289ba8f7,
+    0x27100d71, 0x612b6a14, 0xab66c3bb, 0xed5da4de, 0x7efbe03f,
+    0x38c0875a, 0xf28d2ef5, 0xb4b64990, 0x95c6d6ec, 0xd3fdb189,
+    0x19b01826, 0x5f8b7f43, 0xcc2d3ba2, 0x8a165cc7, 0x405bf568,
+    0x0660920d, 0x02bbcb91, 0x4480acf4, 0x8ecd055b, 0xc8f6623e,
+    0x5b5026df, 0x1d6b41ba, 0xd726e815, 0x911d8f70, 0xb06d100c,
+    0xf6567769, 0x3c1bdec6, 0x7a20b9a3, 0xe986fd42, 0xafbd9a27,
+    0x65f03388, 0x23cb54ed, 0x3ae0095e, 0x7cdb6e3b, 0xb696c794,
+    0xf0ada0f1, 0x630be410, 0x25308375, 0xef7d2ada, 0xa9464dbf,
+    0x8836d2c3, 0xce0db5a6, 0x04401c09, 0x427b7b6c, 0xd1dd3f8d,
+    0x97e658e8, 0x5dabf147, 0x1b909622, 0x1f4bcfbe, 0x5970a8db,
+    0x933d0174, 0xd5066611, 0x46a022f0, 0x009b4595, 0xcad6ec3a,
+    0x8ced8b5f, 0xad9d1423, 0xeba67346, 0x21ebdae9, 0x67d0bd8c,
+    0xf476f96d, 0xb24d9e08, 0x780037a7, 0x3e3b50c2, 0x31b0f544,
+    0x778b9221, 0xbdc63b8e, 0xfbfd5ceb, 0x685b180a, 0x2e607f6f,
+    0xe42dd6c0, 0xa216b1a5, 0x83662ed9, 0xc55d49bc, 0x0f10e013,
+    0x492b8776, 0xda8dc397, 0x9cb6a4f2, 0x56fb0d5d, 0x10c06a38,
+    0x141b33a4, 0x522054c1, 0x986dfd6e, 0xde569a0b, 0x4df0deea,
+    0x0bcbb98f, 0xc1861020, 0x87bd7745, 0xa6cde839, 0xe0f68f5c,
+    0x2abb26f3, 0x6c804196, 0xff260577, 0xb91d6212, 0x7350cbbd,
+    0x356bacd8}};
+
+#endif
+
+#endif
+
+#if N == 6
+
+#if W == 8
+
+local const z_crc_t FAR crc_braid_table[][256] = {
+   {0x00000000, 0x3db1ecdc, 0x7b63d9b8, 0x46d23564, 0xf6c7b370,
+    0xcb765fac, 0x8da46ac8, 0xb0158614, 0x36fe60a1, 0x0b4f8c7d,
+    0x4d9db919, 0x702c55c5, 0xc039d3d1, 0xfd883f0d, 0xbb5a0a69,
+    0x86ebe6b5, 0x6dfcc142, 0x504d2d9e, 0x169f18fa, 0x2b2ef426,
+    0x9b3b7232, 0xa68a9eee, 0xe058ab8a, 0xdde94756, 0x5b02a1e3,
+    0x66b34d3f, 0x2061785b, 0x1dd09487, 0xadc51293, 0x9074fe4f,
+    0xd6a6cb2b, 0xeb1727f7, 0xdbf98284, 0xe6486e58, 0xa09a5b3c,
+    0x9d2bb7e0, 0x2d3e31f4, 0x108fdd28, 0x565de84c, 0x6bec0490,
+    0xed07e225, 0xd0b60ef9, 0x96643b9d, 0xabd5d741, 0x1bc05155,
+    0x2671bd89, 0x60a388ed, 0x5d126431, 0xb60543c6, 0x8bb4af1a,
+    0xcd669a7e, 0xf0d776a2, 0x40c2f0b6, 0x7d731c6a, 0x3ba1290e,
+    0x0610c5d2, 0x80fb2367, 0xbd4acfbb, 0xfb98fadf, 0xc6291603,
+    0x763c9017, 0x4b8d7ccb, 0x0d5f49af, 0x30eea573, 0x6c820349,
+    0x5133ef95, 0x17e1daf1, 0x2a50362d, 0x9a45b039, 0xa7f45ce5,
+    0xe1266981, 0xdc97855d, 0x5a7c63e8, 0x67cd8f34, 0x211fba50,
+    0x1cae568c, 0xacbbd098, 0x910a3c44, 0xd7d80920, 0xea69e5fc,
+    0x017ec20b, 0x3ccf2ed7, 0x7a1d1bb3, 0x47acf76f, 0xf7b9717b,
+    0xca089da7, 0x8cdaa8c3, 0xb16b441f, 0x3780a2aa, 0x0a314e76,
+    0x4ce37b12, 0x715297ce, 0xc14711da, 0xfcf6fd06, 0xba24c862,
+    0x879524be, 0xb77b81cd, 0x8aca6d11, 0xcc185875, 0xf1a9b4a9,
+    0x41bc32bd, 0x7c0dde61, 0x3adfeb05, 0x076e07d9, 0x8185e16c,
+    0xbc340db0, 0xfae638d4, 0xc757d408, 0x7742521c, 0x4af3bec0,
+    0x0c218ba4, 0x31906778, 0xda87408f, 0xe736ac53, 0xa1e49937,
+    0x9c5575eb, 0x2c40f3ff, 0x11f11f23, 0x57232a47, 0x6a92c69b,
+    0xec79202e, 0xd1c8ccf2, 0x971af996, 0xaaab154a, 0x1abe935e,
+    0x270f7f82, 0x61dd4ae6, 0x5c6ca63a, 0xd9040692, 0xe4b5ea4e,
+    0xa267df2a, 0x9fd633f6, 0x2fc3b5e2, 0x1272593e, 0x54a06c5a,
+    0x69118086, 0xeffa6633, 0xd24b8aef, 0x9499bf8b, 0xa9285357,
+    0x193dd543, 0x248c399f, 0x625e0cfb, 0x5fefe027, 0xb4f8c7d0,
+    0x89492b0c, 0xcf9b1e68, 0xf22af2b4, 0x423f74a0, 0x7f8e987c,
+    0x395cad18, 0x04ed41c4, 0x8206a771, 0xbfb74bad, 0xf9657ec9,
+    0xc4d49215, 0x74c11401, 0x4970f8dd, 0x0fa2cdb9, 0x32132165,
+    0x02fd8416, 0x3f4c68ca, 0x799e5dae, 0x442fb172, 0xf43a3766,
+    0xc98bdbba, 0x8f59eede, 0xb2e80202, 0x3403e4b7, 0x09b2086b,
+    0x4f603d0f, 0x72d1d1d3, 0xc2c457c7, 0xff75bb1b, 0xb9a78e7f,
+    0x841662a3, 0x6f014554, 0x52b0a988, 0x14629cec, 0x29d37030,
+    0x99c6f624, 0xa4771af8, 0xe2a52f9c, 0xdf14c340, 0x59ff25f5,
+    0x644ec929, 0x229cfc4d, 0x1f2d1091, 0xaf389685, 0x92897a59,
+    0xd45b4f3d, 0xe9eaa3e1, 0xb58605db, 0x8837e907, 0xcee5dc63,
+    0xf35430bf, 0x4341b6ab, 0x7ef05a77, 0x38226f13, 0x059383cf,
+    0x8378657a, 0xbec989a6, 0xf81bbcc2, 0xc5aa501e, 0x75bfd60a,
+    0x480e3ad6, 0x0edc0fb2, 0x336de36e, 0xd87ac499, 0xe5cb2845,
+    0xa3191d21, 0x9ea8f1fd, 0x2ebd77e9, 0x130c9b35, 0x55deae51,
+    0x686f428d, 0xee84a438, 0xd33548e4, 0x95e77d80, 0xa856915c,
+    0x18431748, 0x25f2fb94, 0x6320cef0, 0x5e91222c, 0x6e7f875f,
+    0x53ce6b83, 0x151c5ee7, 0x28adb23b, 0x98b8342f, 0xa509d8f3,
+    0xe3dbed97, 0xde6a014b, 0x5881e7fe, 0x65300b22, 0x23e23e46,
+    0x1e53d29a, 0xae46548e, 0x93f7b852, 0xd5258d36, 0xe89461ea,
+    0x0383461d, 0x3e32aac1, 0x78e09fa5, 0x45517379, 0xf544f56d,
+    0xc8f519b1, 0x8e272cd5, 0xb396c009, 0x357d26bc, 0x08ccca60,
+    0x4e1eff04, 0x73af13d8, 0xc3ba95cc, 0xfe0b7910, 0xb8d94c74,
+    0x8568a0a8},
+   {0x00000000, 0x69790b65, 0xd2f216ca, 0xbb8b1daf, 0x7e952bd5,
+    0x17ec20b0, 0xac673d1f, 0xc51e367a, 0xfd2a57aa, 0x94535ccf,
+    0x2fd84160, 0x46a14a05, 0x83bf7c7f, 0xeac6771a, 0x514d6ab5,
+    0x383461d0, 0x2125a915, 0x485ca270, 0xf3d7bfdf, 0x9aaeb4ba,
+    0x5fb082c0, 0x36c989a5, 0x8d42940a, 0xe43b9f6f, 0xdc0ffebf,
+    0xb576f5da, 0x0efde875, 0x6784e310, 0xa29ad56a, 0xcbe3de0f,
+    0x7068c3a0, 0x1911c8c5, 0x424b522a, 0x2b32594f, 0x90b944e0,
+    0xf9c04f85, 0x3cde79ff, 0x55a7729a, 0xee2c6f35, 0x87556450,
+    0xbf610580, 0xd6180ee5, 0x6d93134a, 0x04ea182f, 0xc1f42e55,
+    0xa88d2530, 0x1306389f, 0x7a7f33fa, 0x636efb3f, 0x0a17f05a,
+    0xb19cedf5, 0xd8e5e690, 0x1dfbd0ea, 0x7482db8f, 0xcf09c620,
+    0xa670cd45, 0x9e44ac95, 0xf73da7f0, 0x4cb6ba5f, 0x25cfb13a,
+    0xe0d18740, 0x89a88c25, 0x3223918a, 0x5b5a9aef, 0x8496a454,
+    0xedefaf31, 0x5664b29e, 0x3f1db9fb, 0xfa038f81, 0x937a84e4,
+    0x28f1994b, 0x4188922e, 0x79bcf3fe, 0x10c5f89b, 0xab4ee534,
+    0xc237ee51, 0x0729d82b, 0x6e50d34e, 0xd5dbcee1, 0xbca2c584,
+    0xa5b30d41, 0xccca0624, 0x77411b8b, 0x1e3810ee, 0xdb262694,
+    0xb25f2df1, 0x09d4305e, 0x60ad3b3b, 0x58995aeb, 0x31e0518e,
+    0x8a6b4c21, 0xe3124744, 0x260c713e, 0x4f757a5b, 0xf4fe67f4,
+    0x9d876c91, 0xc6ddf67e, 0xafa4fd1b, 0x142fe0b4, 0x7d56ebd1,
+    0xb848ddab, 0xd131d6ce, 0x6abacb61, 0x03c3c004, 0x3bf7a1d4,
+    0x528eaab1, 0xe905b71e, 0x807cbc7b, 0x45628a01, 0x2c1b8164,
+    0x97909ccb, 0xfee997ae, 0xe7f85f6b, 0x8e81540e, 0x350a49a1,
+    0x5c7342c4, 0x996d74be, 0xf0147fdb, 0x4b9f6274, 0x22e66911,
+    0x1ad208c1, 0x73ab03a4, 0xc8201e0b, 0xa159156e, 0x64472314,
+    0x0d3e2871, 0xb6b535de, 0xdfcc3ebb, 0xd25c4ee9, 0xbb25458c,
+    0x00ae5823, 0x69d75346, 0xacc9653c, 0xc5b06e59, 0x7e3b73f6,
+    0x17427893, 0x2f761943, 0x460f1226, 0xfd840f89, 0x94fd04ec,
+    0x51e33296, 0x389a39f3, 0x8311245c, 0xea682f39, 0xf379e7fc,
+    0x9a00ec99, 0x218bf136, 0x48f2fa53, 0x8deccc29, 0xe495c74c,
+    0x5f1edae3, 0x3667d186, 0x0e53b056, 0x672abb33, 0xdca1a69c,
+    0xb5d8adf9, 0x70c69b83, 0x19bf90e6, 0xa2348d49, 0xcb4d862c,
+    0x90171cc3, 0xf96e17a6, 0x42e50a09, 0x2b9c016c, 0xee823716,
+    0x87fb3c73, 0x3c7021dc, 0x55092ab9, 0x6d3d4b69, 0x0444400c,
+    0xbfcf5da3, 0xd6b656c6, 0x13a860bc, 0x7ad16bd9, 0xc15a7676,
+    0xa8237d13, 0xb132b5d6, 0xd84bbeb3, 0x63c0a31c, 0x0ab9a879,
+    0xcfa79e03, 0xa6de9566, 0x1d5588c9, 0x742c83ac, 0x4c18e27c,
+    0x2561e919, 0x9eeaf4b6, 0xf793ffd3, 0x328dc9a9, 0x5bf4c2cc,
+    0xe07fdf63, 0x8906d406, 0x56caeabd, 0x3fb3e1d8, 0x8438fc77,
+    0xed41f712, 0x285fc168, 0x4126ca0d, 0xfaadd7a2, 0x93d4dcc7,
+    0xabe0bd17, 0xc299b672, 0x7912abdd, 0x106ba0b8, 0xd57596c2,
+    0xbc0c9da7, 0x07878008, 0x6efe8b6d, 0x77ef43a8, 0x1e9648cd,
+    0xa51d5562, 0xcc645e07, 0x097a687d, 0x60036318, 0xdb887eb7,
+    0xb2f175d2, 0x8ac51402, 0xe3bc1f67, 0x583702c8, 0x314e09ad,
+    0xf4503fd7, 0x9d2934b2, 0x26a2291d, 0x4fdb2278, 0x1481b897,
+    0x7df8b3f2, 0xc673ae5d, 0xaf0aa538, 0x6a149342, 0x036d9827,
+    0xb8e68588, 0xd19f8eed, 0xe9abef3d, 0x80d2e458, 0x3b59f9f7,
+    0x5220f292, 0x973ec4e8, 0xfe47cf8d, 0x45ccd222, 0x2cb5d947,
+    0x35a41182, 0x5cdd1ae7, 0xe7560748, 0x8e2f0c2d, 0x4b313a57,
+    0x22483132, 0x99c32c9d, 0xf0ba27f8, 0xc88e4628, 0xa1f74d4d,
+    0x1a7c50e2, 0x73055b87, 0xb61b6dfd, 0xdf626698, 0x64e97b37,
+    0x0d907052},
+   {0x00000000, 0x7fc99b93, 0xff933726, 0x805aacb5, 0x2457680d,
+    0x5b9ef39e, 0xdbc45f2b, 0xa40dc4b8, 0x48aed01a, 0x37674b89,
+    0xb73de73c, 0xc8f47caf, 0x6cf9b817, 0x13302384, 0x936a8f31,
+    0xeca314a2, 0x915da034, 0xee943ba7, 0x6ece9712, 0x11070c81,
+    0xb50ac839, 0xcac353aa, 0x4a99ff1f, 0x3550648c, 0xd9f3702e,
+    0xa63aebbd, 0x26604708, 0x59a9dc9b, 0xfda41823, 0x826d83b0,
+    0x02372f05, 0x7dfeb496, 0xf9ca4629, 0x8603ddba, 0x0659710f,
+    0x7990ea9c, 0xdd9d2e24, 0xa254b5b7, 0x220e1902, 0x5dc78291,
+    0xb1649633, 0xcead0da0, 0x4ef7a115, 0x313e3a86, 0x9533fe3e,
+    0xeafa65ad, 0x6aa0c918, 0x1569528b, 0x6897e61d, 0x175e7d8e,
+    0x9704d13b, 0xe8cd4aa8, 0x4cc08e10, 0x33091583, 0xb353b936,
+    0xcc9a22a5, 0x20393607, 0x5ff0ad94, 0xdfaa0121, 0xa0639ab2,
+    0x046e5e0a, 0x7ba7c599, 0xfbfd692c, 0x8434f2bf, 0x28e58a13,
+    0x572c1180, 0xd776bd35, 0xa8bf26a6, 0x0cb2e21e, 0x737b798d,
+    0xf321d538, 0x8ce84eab, 0x604b5a09, 0x1f82c19a, 0x9fd86d2f,
+    0xe011f6bc, 0x441c3204, 0x3bd5a997, 0xbb8f0522, 0xc4469eb1,
+    0xb9b82a27, 0xc671b1b4, 0x462b1d01, 0x39e28692, 0x9def422a,
+    0xe226d9b9, 0x627c750c, 0x1db5ee9f, 0xf116fa3d, 0x8edf61ae,
+    0x0e85cd1b, 0x714c5688, 0xd5419230, 0xaa8809a3, 0x2ad2a516,
+    0x551b3e85, 0xd12fcc3a, 0xaee657a9, 0x2ebcfb1c, 0x5175608f,
+    0xf578a437, 0x8ab13fa4, 0x0aeb9311, 0x75220882, 0x99811c20,
+    0xe64887b3, 0x66122b06, 0x19dbb095, 0xbdd6742d, 0xc21fefbe,
+    0x4245430b, 0x3d8cd898, 0x40726c0e, 0x3fbbf79d, 0xbfe15b28,
+    0xc028c0bb, 0x64250403, 0x1bec9f90, 0x9bb63325, 0xe47fa8b6,
+    0x08dcbc14, 0x77152787, 0xf74f8b32, 0x888610a1, 0x2c8bd419,
+    0x53424f8a, 0xd318e33f, 0xacd178ac, 0x51cb1426, 0x2e028fb5,
+    0xae582300, 0xd191b893, 0x759c7c2b, 0x0a55e7b8, 0x8a0f4b0d,
+    0xf5c6d09e, 0x1965c43c, 0x66ac5faf, 0xe6f6f31a, 0x993f6889,
+    0x3d32ac31, 0x42fb37a2, 0xc2a19b17, 0xbd680084, 0xc096b412,
+    0xbf5f2f81, 0x3f058334, 0x40cc18a7, 0xe4c1dc1f, 0x9b08478c,
+    0x1b52eb39, 0x649b70aa, 0x88386408, 0xf7f1ff9b, 0x77ab532e,
+    0x0862c8bd, 0xac6f0c05, 0xd3a69796, 0x53fc3b23, 0x2c35a0b0,
+    0xa801520f, 0xd7c8c99c, 0x57926529, 0x285bfeba, 0x8c563a02,
+    0xf39fa191, 0x73c50d24, 0x0c0c96b7, 0xe0af8215, 0x9f661986,
+    0x1f3cb533, 0x60f52ea0, 0xc4f8ea18, 0xbb31718b, 0x3b6bdd3e,
+    0x44a246ad, 0x395cf23b, 0x469569a8, 0xc6cfc51d, 0xb9065e8e,
+    0x1d0b9a36, 0x62c201a5, 0xe298ad10, 0x9d513683, 0x71f22221,
+    0x0e3bb9b2, 0x8e611507, 0xf1a88e94, 0x55a54a2c, 0x2a6cd1bf,
+    0xaa367d0a, 0xd5ffe699, 0x792e9e35, 0x06e705a6, 0x86bda913,
+    0xf9743280, 0x5d79f638, 0x22b06dab, 0xa2eac11e, 0xdd235a8d,
+    0x31804e2f, 0x4e49d5bc, 0xce137909, 0xb1dae29a, 0x15d72622,
+    0x6a1ebdb1, 0xea441104, 0x958d8a97, 0xe8733e01, 0x97baa592,
+    0x17e00927, 0x682992b4, 0xcc24560c, 0xb3edcd9f, 0x33b7612a,
+    0x4c7efab9, 0xa0ddee1b, 0xdf147588, 0x5f4ed93d, 0x208742ae,
+    0x848a8616, 0xfb431d85, 0x7b19b130, 0x04d02aa3, 0x80e4d81c,
+    0xff2d438f, 0x7f77ef3a, 0x00be74a9, 0xa4b3b011, 0xdb7a2b82,
+    0x5b208737, 0x24e91ca4, 0xc84a0806, 0xb7839395, 0x37d93f20,
+    0x4810a4b3, 0xec1d600b, 0x93d4fb98, 0x138e572d, 0x6c47ccbe,
+    0x11b97828, 0x6e70e3bb, 0xee2a4f0e, 0x91e3d49d, 0x35ee1025,
+    0x4a278bb6, 0xca7d2703, 0xb5b4bc90, 0x5917a832, 0x26de33a1,
+    0xa6849f14, 0xd94d0487, 0x7d40c03f, 0x02895bac, 0x82d3f719,
+    0xfd1a6c8a},
+   {0x00000000, 0xa396284c, 0x9c5d56d9, 0x3fcb7e95, 0xe3cbabf3,
+    0x405d83bf, 0x7f96fd2a, 0xdc00d566, 0x1ce651a7, 0xbf7079eb,
+    0x80bb077e, 0x232d2f32, 0xff2dfa54, 0x5cbbd218, 0x6370ac8d,
+    0xc0e684c1, 0x39cca34e, 0x9a5a8b02, 0xa591f597, 0x0607dddb,
+    0xda0708bd, 0x799120f1, 0x465a5e64, 0xe5cc7628, 0x252af2e9,
+    0x86bcdaa5, 0xb977a430, 0x1ae18c7c, 0xc6e1591a, 0x65777156,
+    0x5abc0fc3, 0xf92a278f, 0x7399469c, 0xd00f6ed0, 0xefc41045,
+    0x4c523809, 0x9052ed6f, 0x33c4c523, 0x0c0fbbb6, 0xaf9993fa,
+    0x6f7f173b, 0xcce93f77, 0xf32241e2, 0x50b469ae, 0x8cb4bcc8,
+    0x2f229484, 0x10e9ea11, 0xb37fc25d, 0x4a55e5d2, 0xe9c3cd9e,
+    0xd608b30b, 0x759e9b47, 0xa99e4e21, 0x0a08666d, 0x35c318f8,
+    0x965530b4, 0x56b3b475, 0xf5259c39, 0xcaeee2ac, 0x6978cae0,
+    0xb5781f86, 0x16ee37ca, 0x2925495f, 0x8ab36113, 0xe7328d38,
+    0x44a4a574, 0x7b6fdbe1, 0xd8f9f3ad, 0x04f926cb, 0xa76f0e87,
+    0x98a47012, 0x3b32585e, 0xfbd4dc9f, 0x5842f4d3, 0x67898a46,
+    0xc41fa20a, 0x181f776c, 0xbb895f20, 0x844221b5, 0x27d409f9,
+    0xdefe2e76, 0x7d68063a, 0x42a378af, 0xe13550e3, 0x3d358585,
+    0x9ea3adc9, 0xa168d35c, 0x02fefb10, 0xc2187fd1, 0x618e579d,
+    0x5e452908, 0xfdd30144, 0x21d3d422, 0x8245fc6e, 0xbd8e82fb,
+    0x1e18aab7, 0x94abcba4, 0x373de3e8, 0x08f69d7d, 0xab60b531,
+    0x77606057, 0xd4f6481b, 0xeb3d368e, 0x48ab1ec2, 0x884d9a03,
+    0x2bdbb24f, 0x1410ccda, 0xb786e496, 0x6b8631f0, 0xc81019bc,
+    0xf7db6729, 0x544d4f65, 0xad6768ea, 0x0ef140a6, 0x313a3e33,
+    0x92ac167f, 0x4eacc319, 0xed3aeb55, 0xd2f195c0, 0x7167bd8c,
+    0xb181394d, 0x12171101, 0x2ddc6f94, 0x8e4a47d8, 0x524a92be,
+    0xf1dcbaf2, 0xce17c467, 0x6d81ec2b, 0x15141c31, 0xb682347d,
+    0x89494ae8, 0x2adf62a4, 0xf6dfb7c2, 0x55499f8e, 0x6a82e11b,
+    0xc914c957, 0x09f24d96, 0xaa6465da, 0x95af1b4f, 0x36393303,
+    0xea39e665, 0x49afce29, 0x7664b0bc, 0xd5f298f0, 0x2cd8bf7f,
+    0x8f4e9733, 0xb085e9a6, 0x1313c1ea, 0xcf13148c, 0x6c853cc0,
+    0x534e4255, 0xf0d86a19, 0x303eeed8, 0x93a8c694, 0xac63b801,
+    0x0ff5904d, 0xd3f5452b, 0x70636d67, 0x4fa813f2, 0xec3e3bbe,
+    0x668d5aad, 0xc51b72e1, 0xfad00c74, 0x59462438, 0x8546f15e,
+    0x26d0d912, 0x191ba787, 0xba8d8fcb, 0x7a6b0b0a, 0xd9fd2346,
+    0xe6365dd3, 0x45a0759f, 0x99a0a0f9, 0x3a3688b5, 0x05fdf620,
+    0xa66bde6c, 0x5f41f9e3, 0xfcd7d1af, 0xc31caf3a, 0x608a8776,
+    0xbc8a5210, 0x1f1c7a5c, 0x20d704c9, 0x83412c85, 0x43a7a844,
+    0xe0318008, 0xdffafe9d, 0x7c6cd6d1, 0xa06c03b7, 0x03fa2bfb,
+    0x3c31556e, 0x9fa77d22, 0xf2269109, 0x51b0b945, 0x6e7bc7d0,
+    0xcdedef9c, 0x11ed3afa, 0xb27b12b6, 0x8db06c23, 0x2e26446f,
+    0xeec0c0ae, 0x4d56e8e2, 0x729d9677, 0xd10bbe3b, 0x0d0b6b5d,
+    0xae9d4311, 0x91563d84, 0x32c015c8, 0xcbea3247, 0x687c1a0b,
+    0x57b7649e, 0xf4214cd2, 0x282199b4, 0x8bb7b1f8, 0xb47ccf6d,
+    0x17eae721, 0xd70c63e0, 0x749a4bac, 0x4b513539, 0xe8c71d75,
+    0x34c7c813, 0x9751e05f, 0xa89a9eca, 0x0b0cb686, 0x81bfd795,
+    0x2229ffd9, 0x1de2814c, 0xbe74a900, 0x62747c66, 0xc1e2542a,
+    0xfe292abf, 0x5dbf02f3, 0x9d598632, 0x3ecfae7e, 0x0104d0eb,
+    0xa292f8a7, 0x7e922dc1, 0xdd04058d, 0xe2cf7b18, 0x41595354,
+    0xb87374db, 0x1be55c97, 0x242e2202, 0x87b80a4e, 0x5bb8df28,
+    0xf82ef764, 0xc7e589f1, 0x6473a1bd, 0xa495257c, 0x07030d30,
+    0x38c873a5, 0x9b5e5be9, 0x475e8e8f, 0xe4c8a6c3, 0xdb03d856,
+    0x7895f01a},
+   {0x00000000, 0x2a283862, 0x545070c4, 0x7e7848a6, 0xa8a0e188,
+    0x8288d9ea, 0xfcf0914c, 0xd6d8a92e, 0x8a30c551, 0xa018fd33,
+    0xde60b595, 0xf4488df7, 0x229024d9, 0x08b81cbb, 0x76c0541d,
+    0x5ce86c7f, 0xcf108ce3, 0xe538b481, 0x9b40fc27, 0xb168c445,
+    0x67b06d6b, 0x4d985509, 0x33e01daf, 0x19c825cd, 0x452049b2,
+    0x6f0871d0, 0x11703976, 0x3b580114, 0xed80a83a, 0xc7a89058,
+    0xb9d0d8fe, 0x93f8e09c, 0x45501f87, 0x6f7827e5, 0x11006f43,
+    0x3b285721, 0xedf0fe0f, 0xc7d8c66d, 0xb9a08ecb, 0x9388b6a9,
+    0xcf60dad6, 0xe548e2b4, 0x9b30aa12, 0xb1189270, 0x67c03b5e,
+    0x4de8033c, 0x33904b9a, 0x19b873f8, 0x8a409364, 0xa068ab06,
+    0xde10e3a0, 0xf438dbc2, 0x22e072ec, 0x08c84a8e, 0x76b00228,
+    0x5c983a4a, 0x00705635, 0x2a586e57, 0x542026f1, 0x7e081e93,
+    0xa8d0b7bd, 0x82f88fdf, 0xfc80c779, 0xd6a8ff1b, 0x8aa03f0e,
+    0xa088076c, 0xdef04fca, 0xf4d877a8, 0x2200de86, 0x0828e6e4,
+    0x7650ae42, 0x5c789620, 0x0090fa5f, 0x2ab8c23d, 0x54c08a9b,
+    0x7ee8b2f9, 0xa8301bd7, 0x821823b5, 0xfc606b13, 0xd6485371,
+    0x45b0b3ed, 0x6f988b8f, 0x11e0c329, 0x3bc8fb4b, 0xed105265,
+    0xc7386a07, 0xb94022a1, 0x93681ac3, 0xcf8076bc, 0xe5a84ede,
+    0x9bd00678, 0xb1f83e1a, 0x67209734, 0x4d08af56, 0x3370e7f0,
+    0x1958df92, 0xcff02089, 0xe5d818eb, 0x9ba0504d, 0xb188682f,
+    0x6750c101, 0x4d78f963, 0x3300b1c5, 0x192889a7, 0x45c0e5d8,
+    0x6fe8ddba, 0x1190951c, 0x3bb8ad7e, 0xed600450, 0xc7483c32,
+    0xb9307494, 0x93184cf6, 0x00e0ac6a, 0x2ac89408, 0x54b0dcae,
+    0x7e98e4cc, 0xa8404de2, 0x82687580, 0xfc103d26, 0xd6380544,
+    0x8ad0693b, 0xa0f85159, 0xde8019ff, 0xf4a8219d, 0x227088b3,
+    0x0858b0d1, 0x7620f877, 0x5c08c015, 0xce31785d, 0xe419403f,
+    0x9a610899, 0xb04930fb, 0x669199d5, 0x4cb9a1b7, 0x32c1e911,
+    0x18e9d173, 0x4401bd0c, 0x6e29856e, 0x1051cdc8, 0x3a79f5aa,
+    0xeca15c84, 0xc68964e6, 0xb8f12c40, 0x92d91422, 0x0121f4be,
+    0x2b09ccdc, 0x5571847a, 0x7f59bc18, 0xa9811536, 0x83a92d54,
+    0xfdd165f2, 0xd7f95d90, 0x8b1131ef, 0xa139098d, 0xdf41412b,
+    0xf5697949, 0x23b1d067, 0x0999e805, 0x77e1a0a3, 0x5dc998c1,
+    0x8b6167da, 0xa1495fb8, 0xdf31171e, 0xf5192f7c, 0x23c18652,
+    0x09e9be30, 0x7791f696, 0x5db9cef4, 0x0151a28b, 0x2b799ae9,
+    0x5501d24f, 0x7f29ea2d, 0xa9f14303, 0x83d97b61, 0xfda133c7,
+    0xd7890ba5, 0x4471eb39, 0x6e59d35b, 0x10219bfd, 0x3a09a39f,
+    0xecd10ab1, 0xc6f932d3, 0xb8817a75, 0x92a94217, 0xce412e68,
+    0xe469160a, 0x9a115eac, 0xb03966ce, 0x66e1cfe0, 0x4cc9f782,
+    0x32b1bf24, 0x18998746, 0x44914753, 0x6eb97f31, 0x10c13797,
+    0x3ae90ff5, 0xec31a6db, 0xc6199eb9, 0xb861d61f, 0x9249ee7d,
+    0xcea18202, 0xe489ba60, 0x9af1f2c6, 0xb0d9caa4, 0x6601638a,
+    0x4c295be8, 0x3251134e, 0x18792b2c, 0x8b81cbb0, 0xa1a9f3d2,
+    0xdfd1bb74, 0xf5f98316, 0x23212a38, 0x0909125a, 0x77715afc,
+    0x5d59629e, 0x01b10ee1, 0x2b993683, 0x55e17e25, 0x7fc94647,
+    0xa911ef69, 0x8339d70b, 0xfd419fad, 0xd769a7cf, 0x01c158d4,
+    0x2be960b6, 0x55912810, 0x7fb91072, 0xa961b95c, 0x8349813e,
+    0xfd31c998, 0xd719f1fa, 0x8bf19d85, 0xa1d9a5e7, 0xdfa1ed41,
+    0xf589d523, 0x23517c0d, 0x0979446f, 0x77010cc9, 0x5d2934ab,
+    0xced1d437, 0xe4f9ec55, 0x9a81a4f3, 0xb0a99c91, 0x667135bf,
+    0x4c590ddd, 0x3221457b, 0x18097d19, 0x44e11166, 0x6ec92904,
+    0x10b161a2, 0x3a9959c0, 0xec41f0ee, 0xc669c88c, 0xb811802a,
+    0x9239b848},
+   {0x00000000, 0x4713f6fb, 0x8e27edf6, 0xc9341b0d, 0xc73eddad,
+    0x802d2b56, 0x4919305b, 0x0e0ac6a0, 0x550cbd1b, 0x121f4be0,
+    0xdb2b50ed, 0x9c38a616, 0x923260b6, 0xd521964d, 0x1c158d40,
+    0x5b067bbb, 0xaa197a36, 0xed0a8ccd, 0x243e97c0, 0x632d613b,
+    0x6d27a79b, 0x2a345160, 0xe3004a6d, 0xa413bc96, 0xff15c72d,
+    0xb80631d6, 0x71322adb, 0x3621dc20, 0x382b1a80, 0x7f38ec7b,
+    0xb60cf776, 0xf11f018d, 0x8f43f22d, 0xc85004d6, 0x01641fdb,
+    0x4677e920, 0x487d2f80, 0x0f6ed97b, 0xc65ac276, 0x8149348d,
+    0xda4f4f36, 0x9d5cb9cd, 0x5468a2c0, 0x137b543b, 0x1d71929b,
+    0x5a626460, 0x93567f6d, 0xd4458996, 0x255a881b, 0x62497ee0,
+    0xab7d65ed, 0xec6e9316, 0xe26455b6, 0xa577a34d, 0x6c43b840,
+    0x2b504ebb, 0x70563500, 0x3745c3fb, 0xfe71d8f6, 0xb9622e0d,
+    0xb768e8ad, 0xf07b1e56, 0x394f055b, 0x7e5cf3a0, 0xc5f6e21b,
+    0x82e514e0, 0x4bd10fed, 0x0cc2f916, 0x02c83fb6, 0x45dbc94d,
+    0x8cefd240, 0xcbfc24bb, 0x90fa5f00, 0xd7e9a9fb, 0x1eddb2f6,
+    0x59ce440d, 0x57c482ad, 0x10d77456, 0xd9e36f5b, 0x9ef099a0,
+    0x6fef982d, 0x28fc6ed6, 0xe1c875db, 0xa6db8320, 0xa8d14580,
+    0xefc2b37b, 0x26f6a876, 0x61e55e8d, 0x3ae32536, 0x7df0d3cd,
+    0xb4c4c8c0, 0xf3d73e3b, 0xfdddf89b, 0xbace0e60, 0x73fa156d,
+    0x34e9e396, 0x4ab51036, 0x0da6e6cd, 0xc492fdc0, 0x83810b3b,
+    0x8d8bcd9b, 0xca983b60, 0x03ac206d, 0x44bfd696, 0x1fb9ad2d,
+    0x58aa5bd6, 0x919e40db, 0xd68db620, 0xd8877080, 0x9f94867b,
+    0x56a09d76, 0x11b36b8d, 0xe0ac6a00, 0xa7bf9cfb, 0x6e8b87f6,
+    0x2998710d, 0x2792b7ad, 0x60814156, 0xa9b55a5b, 0xeea6aca0,
+    0xb5a0d71b, 0xf2b321e0, 0x3b873aed, 0x7c94cc16, 0x729e0ab6,
+    0x358dfc4d, 0xfcb9e740, 0xbbaa11bb, 0x509cc277, 0x178f348c,
+    0xdebb2f81, 0x99a8d97a, 0x97a21fda, 0xd0b1e921, 0x1985f22c,
+    0x5e9604d7, 0x05907f6c, 0x42838997, 0x8bb7929a, 0xcca46461,
+    0xc2aea2c1, 0x85bd543a, 0x4c894f37, 0x0b9ab9cc, 0xfa85b841,
+    0xbd964eba, 0x74a255b7, 0x33b1a34c, 0x3dbb65ec, 0x7aa89317,
+    0xb39c881a, 0xf48f7ee1, 0xaf89055a, 0xe89af3a1, 0x21aee8ac,
+    0x66bd1e57, 0x68b7d8f7, 0x2fa42e0c, 0xe6903501, 0xa183c3fa,
+    0xdfdf305a, 0x98ccc6a1, 0x51f8ddac, 0x16eb2b57, 0x18e1edf7,
+    0x5ff21b0c, 0x96c60001, 0xd1d5f6fa, 0x8ad38d41, 0xcdc07bba,
+    0x04f460b7, 0x43e7964c, 0x4ded50ec, 0x0afea617, 0xc3cabd1a,
+    0x84d94be1, 0x75c64a6c, 0x32d5bc97, 0xfbe1a79a, 0xbcf25161,
+    0xb2f897c1, 0xf5eb613a, 0x3cdf7a37, 0x7bcc8ccc, 0x20caf777,
+    0x67d9018c, 0xaeed1a81, 0xe9feec7a, 0xe7f42ada, 0xa0e7dc21,
+    0x69d3c72c, 0x2ec031d7, 0x956a206c, 0xd279d697, 0x1b4dcd9a,
+    0x5c5e3b61, 0x5254fdc1, 0x15470b3a, 0xdc731037, 0x9b60e6cc,
+    0xc0669d77, 0x87756b8c, 0x4e417081, 0x0952867a, 0x075840da,
+    0x404bb621, 0x897fad2c, 0xce6c5bd7, 0x3f735a5a, 0x7860aca1,
+    0xb154b7ac, 0xf6474157, 0xf84d87f7, 0xbf5e710c, 0x766a6a01,
+    0x31799cfa, 0x6a7fe741, 0x2d6c11ba, 0xe4580ab7, 0xa34bfc4c,
+    0xad413aec, 0xea52cc17, 0x2366d71a, 0x647521e1, 0x1a29d241,
+    0x5d3a24ba, 0x940e3fb7, 0xd31dc94c, 0xdd170fec, 0x9a04f917,
+    0x5330e21a, 0x142314e1, 0x4f256f5a, 0x083699a1, 0xc10282ac,
+    0x86117457, 0x881bb2f7, 0xcf08440c, 0x063c5f01, 0x412fa9fa,
+    0xb030a877, 0xf7235e8c, 0x3e174581, 0x7904b37a, 0x770e75da,
+    0x301d8321, 0xf929982c, 0xbe3a6ed7, 0xe53c156c, 0xa22fe397,
+    0x6b1bf89a, 0x2c080e61, 0x2202c8c1, 0x65113e3a, 0xac252537,
+    0xeb36d3cc},
+   {0x00000000, 0xa13984ee, 0x99020f9d, 0x383b8b73, 0xe975197b,
+    0x484c9d95, 0x707716e6, 0xd14e9208, 0x099b34b7, 0xa8a2b059,
+    0x90993b2a, 0x31a0bfc4, 0xe0ee2dcc, 0x41d7a922, 0x79ec2251,
+    0xd8d5a6bf, 0x1336696e, 0xb20fed80, 0x8a3466f3, 0x2b0de21d,
+    0xfa437015, 0x5b7af4fb, 0x63417f88, 0xc278fb66, 0x1aad5dd9,
+    0xbb94d937, 0x83af5244, 0x2296d6aa, 0xf3d844a2, 0x52e1c04c,
+    0x6ada4b3f, 0xcbe3cfd1, 0x266cd2dc, 0x87555632, 0xbf6edd41,
+    0x1e5759af, 0xcf19cba7, 0x6e204f49, 0x561bc43a, 0xf72240d4,
+    0x2ff7e66b, 0x8ece6285, 0xb6f5e9f6, 0x17cc6d18, 0xc682ff10,
+    0x67bb7bfe, 0x5f80f08d, 0xfeb97463, 0x355abbb2, 0x94633f5c,
+    0xac58b42f, 0x0d6130c1, 0xdc2fa2c9, 0x7d162627, 0x452dad54,
+    0xe41429ba, 0x3cc18f05, 0x9df80beb, 0xa5c38098, 0x04fa0476,
+    0xd5b4967e, 0x748d1290, 0x4cb699e3, 0xed8f1d0d, 0x4cd9a5b8,
+    0xede02156, 0xd5dbaa25, 0x74e22ecb, 0xa5acbcc3, 0x0495382d,
+    0x3caeb35e, 0x9d9737b0, 0x4542910f, 0xe47b15e1, 0xdc409e92,
+    0x7d791a7c, 0xac378874, 0x0d0e0c9a, 0x353587e9, 0x940c0307,
+    0x5fefccd6, 0xfed64838, 0xc6edc34b, 0x67d447a5, 0xb69ad5ad,
+    0x17a35143, 0x2f98da30, 0x8ea15ede, 0x5674f861, 0xf74d7c8f,
+    0xcf76f7fc, 0x6e4f7312, 0xbf01e11a, 0x1e3865f4, 0x2603ee87,
+    0x873a6a69, 0x6ab57764, 0xcb8cf38a, 0xf3b778f9, 0x528efc17,
+    0x83c06e1f, 0x22f9eaf1, 0x1ac26182, 0xbbfbe56c, 0x632e43d3,
+    0xc217c73d, 0xfa2c4c4e, 0x5b15c8a0, 0x8a5b5aa8, 0x2b62de46,
+    0x13595535, 0xb260d1db, 0x79831e0a, 0xd8ba9ae4, 0xe0811197,
+    0x41b89579, 0x90f60771, 0x31cf839f, 0x09f408ec, 0xa8cd8c02,
+    0x70182abd, 0xd121ae53, 0xe91a2520, 0x4823a1ce, 0x996d33c6,
+    0x3854b728, 0x006f3c5b, 0xa156b8b5, 0x99b34b70, 0x388acf9e,
+    0x00b144ed, 0xa188c003, 0x70c6520b, 0xd1ffd6e5, 0xe9c45d96,
+    0x48fdd978, 0x90287fc7, 0x3111fb29, 0x092a705a, 0xa813f4b4,
+    0x795d66bc, 0xd864e252, 0xe05f6921, 0x4166edcf, 0x8a85221e,
+    0x2bbca6f0, 0x13872d83, 0xb2bea96d, 0x63f03b65, 0xc2c9bf8b,
+    0xfaf234f8, 0x5bcbb016, 0x831e16a9, 0x22279247, 0x1a1c1934,
+    0xbb259dda, 0x6a6b0fd2, 0xcb528b3c, 0xf369004f, 0x525084a1,
+    0xbfdf99ac, 0x1ee61d42, 0x26dd9631, 0x87e412df, 0x56aa80d7,
+    0xf7930439, 0xcfa88f4a, 0x6e910ba4, 0xb644ad1b, 0x177d29f5,
+    0x2f46a286, 0x8e7f2668, 0x5f31b460, 0xfe08308e, 0xc633bbfd,
+    0x670a3f13, 0xace9f0c2, 0x0dd0742c, 0x35ebff5f, 0x94d27bb1,
+    0x459ce9b9, 0xe4a56d57, 0xdc9ee624, 0x7da762ca, 0xa572c475,
+    0x044b409b, 0x3c70cbe8, 0x9d494f06, 0x4c07dd0e, 0xed3e59e0,
+    0xd505d293, 0x743c567d, 0xd56aeec8, 0x74536a26, 0x4c68e155,
+    0xed5165bb, 0x3c1ff7b3, 0x9d26735d, 0xa51df82e, 0x04247cc0,
+    0xdcf1da7f, 0x7dc85e91, 0x45f3d5e2, 0xe4ca510c, 0x3584c304,
+    0x94bd47ea, 0xac86cc99, 0x0dbf4877, 0xc65c87a6, 0x67650348,
+    0x5f5e883b, 0xfe670cd5, 0x2f299edd, 0x8e101a33, 0xb62b9140,
+    0x171215ae, 0xcfc7b311, 0x6efe37ff, 0x56c5bc8c, 0xf7fc3862,
+    0x26b2aa6a, 0x878b2e84, 0xbfb0a5f7, 0x1e892119, 0xf3063c14,
+    0x523fb8fa, 0x6a043389, 0xcb3db767, 0x1a73256f, 0xbb4aa181,
+    0x83712af2, 0x2248ae1c, 0xfa9d08a3, 0x5ba48c4d, 0x639f073e,
+    0xc2a683d0, 0x13e811d8, 0xb2d19536, 0x8aea1e45, 0x2bd39aab,
+    0xe030557a, 0x4109d194, 0x79325ae7, 0xd80bde09, 0x09454c01,
+    0xa87cc8ef, 0x9047439c, 0x317ec772, 0xe9ab61cd, 0x4892e523,
+    0x70a96e50, 0xd190eabe, 0x00de78b6, 0xa1e7fc58, 0x99dc772b,
+    0x38e5f3c5},
+   {0x00000000, 0xe81790a1, 0x0b5e2703, 0xe349b7a2, 0x16bc4e06,
+    0xfeabdea7, 0x1de26905, 0xf5f5f9a4, 0x2d789c0c, 0xc56f0cad,
+    0x2626bb0f, 0xce312bae, 0x3bc4d20a, 0xd3d342ab, 0x309af509,
+    0xd88d65a8, 0x5af13818, 0xb2e6a8b9, 0x51af1f1b, 0xb9b88fba,
+    0x4c4d761e, 0xa45ae6bf, 0x4713511d, 0xaf04c1bc, 0x7789a414,
+    0x9f9e34b5, 0x7cd78317, 0x94c013b6, 0x6135ea12, 0x89227ab3,
+    0x6a6bcd11, 0x827c5db0, 0xb5e27030, 0x5df5e091, 0xbebc5733,
+    0x56abc792, 0xa35e3e36, 0x4b49ae97, 0xa8001935, 0x40178994,
+    0x989aec3c, 0x708d7c9d, 0x93c4cb3f, 0x7bd35b9e, 0x8e26a23a,
+    0x6631329b, 0x85788539, 0x6d6f1598, 0xef134828, 0x0704d889,
+    0xe44d6f2b, 0x0c5aff8a, 0xf9af062e, 0x11b8968f, 0xf2f1212d,
+    0x1ae6b18c, 0xc26bd424, 0x2a7c4485, 0xc935f327, 0x21226386,
+    0xd4d79a22, 0x3cc00a83, 0xdf89bd21, 0x379e2d80, 0xb0b5e621,
+    0x58a27680, 0xbbebc122, 0x53fc5183, 0xa609a827, 0x4e1e3886,
+    0xad578f24, 0x45401f85, 0x9dcd7a2d, 0x75daea8c, 0x96935d2e,
+    0x7e84cd8f, 0x8b71342b, 0x6366a48a, 0x802f1328, 0x68388389,
+    0xea44de39, 0x02534e98, 0xe11af93a, 0x090d699b, 0xfcf8903f,
+    0x14ef009e, 0xf7a6b73c, 0x1fb1279d, 0xc73c4235, 0x2f2bd294,
+    0xcc626536, 0x2475f597, 0xd1800c33, 0x39979c92, 0xdade2b30,
+    0x32c9bb91, 0x05579611, 0xed4006b0, 0x0e09b112, 0xe61e21b3,
+    0x13ebd817, 0xfbfc48b6, 0x18b5ff14, 0xf0a26fb5, 0x282f0a1d,
+    0xc0389abc, 0x23712d1e, 0xcb66bdbf, 0x3e93441b, 0xd684d4ba,
+    0x35cd6318, 0xdddaf3b9, 0x5fa6ae09, 0xb7b13ea8, 0x54f8890a,
+    0xbcef19ab, 0x491ae00f, 0xa10d70ae, 0x4244c70c, 0xaa5357ad,
+    0x72de3205, 0x9ac9a2a4, 0x79801506, 0x919785a7, 0x64627c03,
+    0x8c75eca2, 0x6f3c5b00, 0x872bcba1, 0xba1aca03, 0x520d5aa2,
+    0xb144ed00, 0x59537da1, 0xaca68405, 0x44b114a4, 0xa7f8a306,
+    0x4fef33a7, 0x9762560f, 0x7f75c6ae, 0x9c3c710c, 0x742be1ad,
+    0x81de1809, 0x69c988a8, 0x8a803f0a, 0x6297afab, 0xe0ebf21b,
+    0x08fc62ba, 0xebb5d518, 0x03a245b9, 0xf657bc1d, 0x1e402cbc,
+    0xfd099b1e, 0x151e0bbf, 0xcd936e17, 0x2584feb6, 0xc6cd4914,
+    0x2edad9b5, 0xdb2f2011, 0x3338b0b0, 0xd0710712, 0x386697b3,
+    0x0ff8ba33, 0xe7ef2a92, 0x04a69d30, 0xecb10d91, 0x1944f435,
+    0xf1536494, 0x121ad336, 0xfa0d4397, 0x2280263f, 0xca97b69e,
+    0x29de013c, 0xc1c9919d, 0x343c6839, 0xdc2bf898, 0x3f624f3a,
+    0xd775df9b, 0x5509822b, 0xbd1e128a, 0x5e57a528, 0xb6403589,
+    0x43b5cc2d, 0xaba25c8c, 0x48ebeb2e, 0xa0fc7b8f, 0x78711e27,
+    0x90668e86, 0x732f3924, 0x9b38a985, 0x6ecd5021, 0x86dac080,
+    0x65937722, 0x8d84e783, 0x0aaf2c22, 0xe2b8bc83, 0x01f10b21,
+    0xe9e69b80, 0x1c136224, 0xf404f285, 0x174d4527, 0xff5ad586,
+    0x27d7b02e, 0xcfc0208f, 0x2c89972d, 0xc49e078c, 0x316bfe28,
+    0xd97c6e89, 0x3a35d92b, 0xd222498a, 0x505e143a, 0xb849849b,
+    0x5b003339, 0xb317a398, 0x46e25a3c, 0xaef5ca9d, 0x4dbc7d3f,
+    0xa5abed9e, 0x7d268836, 0x95311897, 0x7678af35, 0x9e6f3f94,
+    0x6b9ac630, 0x838d5691, 0x60c4e133, 0x88d37192, 0xbf4d5c12,
+    0x575accb3, 0xb4137b11, 0x5c04ebb0, 0xa9f11214, 0x41e682b5,
+    0xa2af3517, 0x4ab8a5b6, 0x9235c01e, 0x7a2250bf, 0x996be71d,
+    0x717c77bc, 0x84898e18, 0x6c9e1eb9, 0x8fd7a91b, 0x67c039ba,
+    0xe5bc640a, 0x0dabf4ab, 0xeee24309, 0x06f5d3a8, 0xf3002a0c,
+    0x1b17baad, 0xf85e0d0f, 0x10499dae, 0xc8c4f806, 0x20d368a7,
+    0xc39adf05, 0x2b8d4fa4, 0xde78b600, 0x366f26a1, 0xd5269103,
+    0x3d3101a2}};
+
+local const z_word_t FAR crc_braid_big_table[][256] = {
+   {0x0000000000000000, 0xa19017e800000000, 0x03275e0b00000000,
+    0xa2b749e300000000, 0x064ebc1600000000, 0xa7deabfe00000000,
+    0x0569e21d00000000, 0xa4f9f5f500000000, 0x0c9c782d00000000,
+    0xad0c6fc500000000, 0x0fbb262600000000, 0xae2b31ce00000000,
+    0x0ad2c43b00000000, 0xab42d3d300000000, 0x09f59a3000000000,
+    0xa8658dd800000000, 0x1838f15a00000000, 0xb9a8e6b200000000,
+    0x1b1faf5100000000, 0xba8fb8b900000000, 0x1e764d4c00000000,
+    0xbfe65aa400000000, 0x1d51134700000000, 0xbcc104af00000000,
+    0x14a4897700000000, 0xb5349e9f00000000, 0x1783d77c00000000,
+    0xb613c09400000000, 0x12ea356100000000, 0xb37a228900000000,
+    0x11cd6b6a00000000, 0xb05d7c8200000000, 0x3070e2b500000000,
+    0x91e0f55d00000000, 0x3357bcbe00000000, 0x92c7ab5600000000,
+    0x363e5ea300000000, 0x97ae494b00000000, 0x351900a800000000,
+    0x9489174000000000, 0x3cec9a9800000000, 0x9d7c8d7000000000,
+    0x3fcbc49300000000, 0x9e5bd37b00000000, 0x3aa2268e00000000,
+    0x9b32316600000000, 0x3985788500000000, 0x98156f6d00000000,
+    0x284813ef00000000, 0x89d8040700000000, 0x2b6f4de400000000,
+    0x8aff5a0c00000000, 0x2e06aff900000000, 0x8f96b81100000000,
+    0x2d21f1f200000000, 0x8cb1e61a00000000, 0x24d46bc200000000,
+    0x85447c2a00000000, 0x27f335c900000000, 0x8663222100000000,
+    0x229ad7d400000000, 0x830ac03c00000000, 0x21bd89df00000000,
+    0x802d9e3700000000, 0x21e6b5b000000000, 0x8076a25800000000,
+    0x22c1ebbb00000000, 0x8351fc5300000000, 0x27a809a600000000,
+    0x86381e4e00000000, 0x248f57ad00000000, 0x851f404500000000,
+    0x2d7acd9d00000000, 0x8ceada7500000000, 0x2e5d939600000000,
+    0x8fcd847e00000000, 0x2b34718b00000000, 0x8aa4666300000000,
+    0x28132f8000000000, 0x8983386800000000, 0x39de44ea00000000,
+    0x984e530200000000, 0x3af91ae100000000, 0x9b690d0900000000,
+    0x3f90f8fc00000000, 0x9e00ef1400000000, 0x3cb7a6f700000000,
+    0x9d27b11f00000000, 0x35423cc700000000, 0x94d22b2f00000000,
+    0x366562cc00000000, 0x97f5752400000000, 0x330c80d100000000,
+    0x929c973900000000, 0x302bdeda00000000, 0x91bbc93200000000,
+    0x1196570500000000, 0xb00640ed00000000, 0x12b1090e00000000,
+    0xb3211ee600000000, 0x17d8eb1300000000, 0xb648fcfb00000000,
+    0x14ffb51800000000, 0xb56fa2f000000000, 0x1d0a2f2800000000,
+    0xbc9a38c000000000, 0x1e2d712300000000, 0xbfbd66cb00000000,
+    0x1b44933e00000000, 0xbad484d600000000, 0x1863cd3500000000,
+    0xb9f3dadd00000000, 0x09aea65f00000000, 0xa83eb1b700000000,
+    0x0a89f85400000000, 0xab19efbc00000000, 0x0fe01a4900000000,
+    0xae700da100000000, 0x0cc7444200000000, 0xad5753aa00000000,
+    0x0532de7200000000, 0xa4a2c99a00000000, 0x0615807900000000,
+    0xa785979100000000, 0x037c626400000000, 0xa2ec758c00000000,
+    0x005b3c6f00000000, 0xa1cb2b8700000000, 0x03ca1aba00000000,
+    0xa25a0d5200000000, 0x00ed44b100000000, 0xa17d535900000000,
+    0x0584a6ac00000000, 0xa414b14400000000, 0x06a3f8a700000000,
+    0xa733ef4f00000000, 0x0f56629700000000, 0xaec6757f00000000,
+    0x0c713c9c00000000, 0xade12b7400000000, 0x0918de8100000000,
+    0xa888c96900000000, 0x0a3f808a00000000, 0xabaf976200000000,
+    0x1bf2ebe000000000, 0xba62fc0800000000, 0x18d5b5eb00000000,
+    0xb945a20300000000, 0x1dbc57f600000000, 0xbc2c401e00000000,
+    0x1e9b09fd00000000, 0xbf0b1e1500000000, 0x176e93cd00000000,
+    0xb6fe842500000000, 0x1449cdc600000000, 0xb5d9da2e00000000,
+    0x11202fdb00000000, 0xb0b0383300000000, 0x120771d000000000,
+    0xb397663800000000, 0x33baf80f00000000, 0x922aefe700000000,
+    0x309da60400000000, 0x910db1ec00000000, 0x35f4441900000000,
+    0x946453f100000000, 0x36d31a1200000000, 0x97430dfa00000000,
+    0x3f26802200000000, 0x9eb697ca00000000, 0x3c01de2900000000,
+    0x9d91c9c100000000, 0x39683c3400000000, 0x98f82bdc00000000,
+    0x3a4f623f00000000, 0x9bdf75d700000000, 0x2b82095500000000,
+    0x8a121ebd00000000, 0x28a5575e00000000, 0x893540b600000000,
+    0x2dccb54300000000, 0x8c5ca2ab00000000, 0x2eebeb4800000000,
+    0x8f7bfca000000000, 0x271e717800000000, 0x868e669000000000,
+    0x24392f7300000000, 0x85a9389b00000000, 0x2150cd6e00000000,
+    0x80c0da8600000000, 0x2277936500000000, 0x83e7848d00000000,
+    0x222caf0a00000000, 0x83bcb8e200000000, 0x210bf10100000000,
+    0x809be6e900000000, 0x2462131c00000000, 0x85f204f400000000,
+    0x27454d1700000000, 0x86d55aff00000000, 0x2eb0d72700000000,
+    0x8f20c0cf00000000, 0x2d97892c00000000, 0x8c079ec400000000,
+    0x28fe6b3100000000, 0x896e7cd900000000, 0x2bd9353a00000000,
+    0x8a4922d200000000, 0x3a145e5000000000, 0x9b8449b800000000,
+    0x3933005b00000000, 0x98a317b300000000, 0x3c5ae24600000000,
+    0x9dcaf5ae00000000, 0x3f7dbc4d00000000, 0x9eedaba500000000,
+    0x3688267d00000000, 0x9718319500000000, 0x35af787600000000,
+    0x943f6f9e00000000, 0x30c69a6b00000000, 0x91568d8300000000,
+    0x33e1c46000000000, 0x9271d38800000000, 0x125c4dbf00000000,
+    0xb3cc5a5700000000, 0x117b13b400000000, 0xb0eb045c00000000,
+    0x1412f1a900000000, 0xb582e64100000000, 0x1735afa200000000,
+    0xb6a5b84a00000000, 0x1ec0359200000000, 0xbf50227a00000000,
+    0x1de76b9900000000, 0xbc777c7100000000, 0x188e898400000000,
+    0xb91e9e6c00000000, 0x1ba9d78f00000000, 0xba39c06700000000,
+    0x0a64bce500000000, 0xabf4ab0d00000000, 0x0943e2ee00000000,
+    0xa8d3f50600000000, 0x0c2a00f300000000, 0xadba171b00000000,
+    0x0f0d5ef800000000, 0xae9d491000000000, 0x06f8c4c800000000,
+    0xa768d32000000000, 0x05df9ac300000000, 0xa44f8d2b00000000,
+    0x00b678de00000000, 0xa1266f3600000000, 0x039126d500000000,
+    0xa201313d00000000},
+   {0x0000000000000000, 0xee8439a100000000, 0x9d0f029900000000,
+    0x738b3b3800000000, 0x7b1975e900000000, 0x959d4c4800000000,
+    0xe616777000000000, 0x08924ed100000000, 0xb7349b0900000000,
+    0x59b0a2a800000000, 0x2a3b999000000000, 0xc4bfa03100000000,
+    0xcc2deee000000000, 0x22a9d74100000000, 0x5122ec7900000000,
+    0xbfa6d5d800000000, 0x6e69361300000000, 0x80ed0fb200000000,
+    0xf366348a00000000, 0x1de20d2b00000000, 0x157043fa00000000,
+    0xfbf47a5b00000000, 0x887f416300000000, 0x66fb78c200000000,
+    0xd95dad1a00000000, 0x37d994bb00000000, 0x4452af8300000000,
+    0xaad6962200000000, 0xa244d8f300000000, 0x4cc0e15200000000,
+    0x3f4bda6a00000000, 0xd1cfe3cb00000000, 0xdcd26c2600000000,
+    0x3256558700000000, 0x41dd6ebf00000000, 0xaf59571e00000000,
+    0xa7cb19cf00000000, 0x494f206e00000000, 0x3ac41b5600000000,
+    0xd44022f700000000, 0x6be6f72f00000000, 0x8562ce8e00000000,
+    0xf6e9f5b600000000, 0x186dcc1700000000, 0x10ff82c600000000,
+    0xfe7bbb6700000000, 0x8df0805f00000000, 0x6374b9fe00000000,
+    0xb2bb5a3500000000, 0x5c3f639400000000, 0x2fb458ac00000000,
+    0xc130610d00000000, 0xc9a22fdc00000000, 0x2726167d00000000,
+    0x54ad2d4500000000, 0xba2914e400000000, 0x058fc13c00000000,
+    0xeb0bf89d00000000, 0x9880c3a500000000, 0x7604fa0400000000,
+    0x7e96b4d500000000, 0x90128d7400000000, 0xe399b64c00000000,
+    0x0d1d8fed00000000, 0xb8a5d94c00000000, 0x5621e0ed00000000,
+    0x25aadbd500000000, 0xcb2ee27400000000, 0xc3bcaca500000000,
+    0x2d38950400000000, 0x5eb3ae3c00000000, 0xb037979d00000000,
+    0x0f91424500000000, 0xe1157be400000000, 0x929e40dc00000000,
+    0x7c1a797d00000000, 0x748837ac00000000, 0x9a0c0e0d00000000,
+    0xe987353500000000, 0x07030c9400000000, 0xd6ccef5f00000000,
+    0x3848d6fe00000000, 0x4bc3edc600000000, 0xa547d46700000000,
+    0xadd59ab600000000, 0x4351a31700000000, 0x30da982f00000000,
+    0xde5ea18e00000000, 0x61f8745600000000, 0x8f7c4df700000000,
+    0xfcf776cf00000000, 0x12734f6e00000000, 0x1ae101bf00000000,
+    0xf465381e00000000, 0x87ee032600000000, 0x696a3a8700000000,
+    0x6477b56a00000000, 0x8af38ccb00000000, 0xf978b7f300000000,
+    0x17fc8e5200000000, 0x1f6ec08300000000, 0xf1eaf92200000000,
+    0x8261c21a00000000, 0x6ce5fbbb00000000, 0xd3432e6300000000,
+    0x3dc717c200000000, 0x4e4c2cfa00000000, 0xa0c8155b00000000,
+    0xa85a5b8a00000000, 0x46de622b00000000, 0x3555591300000000,
+    0xdbd160b200000000, 0x0a1e837900000000, 0xe49abad800000000,
+    0x971181e000000000, 0x7995b84100000000, 0x7107f69000000000,
+    0x9f83cf3100000000, 0xec08f40900000000, 0x028ccda800000000,
+    0xbd2a187000000000, 0x53ae21d100000000, 0x20251ae900000000,
+    0xcea1234800000000, 0xc6336d9900000000, 0x28b7543800000000,
+    0x5b3c6f0000000000, 0xb5b856a100000000, 0x704bb39900000000,
+    0x9ecf8a3800000000, 0xed44b10000000000, 0x03c088a100000000,
+    0x0b52c67000000000, 0xe5d6ffd100000000, 0x965dc4e900000000,
+    0x78d9fd4800000000, 0xc77f289000000000, 0x29fb113100000000,
+    0x5a702a0900000000, 0xb4f413a800000000, 0xbc665d7900000000,
+    0x52e264d800000000, 0x21695fe000000000, 0xcfed664100000000,
+    0x1e22858a00000000, 0xf0a6bc2b00000000, 0x832d871300000000,
+    0x6da9beb200000000, 0x653bf06300000000, 0x8bbfc9c200000000,
+    0xf834f2fa00000000, 0x16b0cb5b00000000, 0xa9161e8300000000,
+    0x4792272200000000, 0x34191c1a00000000, 0xda9d25bb00000000,
+    0xd20f6b6a00000000, 0x3c8b52cb00000000, 0x4f0069f300000000,
+    0xa184505200000000, 0xac99dfbf00000000, 0x421de61e00000000,
+    0x3196dd2600000000, 0xdf12e48700000000, 0xd780aa5600000000,
+    0x390493f700000000, 0x4a8fa8cf00000000, 0xa40b916e00000000,
+    0x1bad44b600000000, 0xf5297d1700000000, 0x86a2462f00000000,
+    0x68267f8e00000000, 0x60b4315f00000000, 0x8e3008fe00000000,
+    0xfdbb33c600000000, 0x133f0a6700000000, 0xc2f0e9ac00000000,
+    0x2c74d00d00000000, 0x5fffeb3500000000, 0xb17bd29400000000,
+    0xb9e99c4500000000, 0x576da5e400000000, 0x24e69edc00000000,
+    0xca62a77d00000000, 0x75c472a500000000, 0x9b404b0400000000,
+    0xe8cb703c00000000, 0x064f499d00000000, 0x0edd074c00000000,
+    0xe0593eed00000000, 0x93d205d500000000, 0x7d563c7400000000,
+    0xc8ee6ad500000000, 0x266a537400000000, 0x55e1684c00000000,
+    0xbb6551ed00000000, 0xb3f71f3c00000000, 0x5d73269d00000000,
+    0x2ef81da500000000, 0xc07c240400000000, 0x7fdaf1dc00000000,
+    0x915ec87d00000000, 0xe2d5f34500000000, 0x0c51cae400000000,
+    0x04c3843500000000, 0xea47bd9400000000, 0x99cc86ac00000000,
+    0x7748bf0d00000000, 0xa6875cc600000000, 0x4803656700000000,
+    0x3b885e5f00000000, 0xd50c67fe00000000, 0xdd9e292f00000000,
+    0x331a108e00000000, 0x40912bb600000000, 0xae15121700000000,
+    0x11b3c7cf00000000, 0xff37fe6e00000000, 0x8cbcc55600000000,
+    0x6238fcf700000000, 0x6aaab22600000000, 0x842e8b8700000000,
+    0xf7a5b0bf00000000, 0x1921891e00000000, 0x143c06f300000000,
+    0xfab83f5200000000, 0x8933046a00000000, 0x67b73dcb00000000,
+    0x6f25731a00000000, 0x81a14abb00000000, 0xf22a718300000000,
+    0x1cae482200000000, 0xa3089dfa00000000, 0x4d8ca45b00000000,
+    0x3e079f6300000000, 0xd083a6c200000000, 0xd811e81300000000,
+    0x3695d1b200000000, 0x451eea8a00000000, 0xab9ad32b00000000,
+    0x7a5530e000000000, 0x94d1094100000000, 0xe75a327900000000,
+    0x09de0bd800000000, 0x014c450900000000, 0xefc87ca800000000,
+    0x9c43479000000000, 0x72c77e3100000000, 0xcd61abe900000000,
+    0x23e5924800000000, 0x506ea97000000000, 0xbeea90d100000000,
+    0xb678de0000000000, 0x58fce7a100000000, 0x2b77dc9900000000,
+    0xc5f3e53800000000},
+   {0x0000000000000000, 0xfbf6134700000000, 0xf6ed278e00000000,
+    0x0d1b34c900000000, 0xaddd3ec700000000, 0x562b2d8000000000,
+    0x5b30194900000000, 0xa0c60a0e00000000, 0x1bbd0c5500000000,
+    0xe04b1f1200000000, 0xed502bdb00000000, 0x16a6389c00000000,
+    0xb660329200000000, 0x4d9621d500000000, 0x408d151c00000000,
+    0xbb7b065b00000000, 0x367a19aa00000000, 0xcd8c0aed00000000,
+    0xc0973e2400000000, 0x3b612d6300000000, 0x9ba7276d00000000,
+    0x6051342a00000000, 0x6d4a00e300000000, 0x96bc13a400000000,
+    0x2dc715ff00000000, 0xd63106b800000000, 0xdb2a327100000000,
+    0x20dc213600000000, 0x801a2b3800000000, 0x7bec387f00000000,
+    0x76f70cb600000000, 0x8d011ff100000000, 0x2df2438f00000000,
+    0xd60450c800000000, 0xdb1f640100000000, 0x20e9774600000000,
+    0x802f7d4800000000, 0x7bd96e0f00000000, 0x76c25ac600000000,
+    0x8d34498100000000, 0x364f4fda00000000, 0xcdb95c9d00000000,
+    0xc0a2685400000000, 0x3b547b1300000000, 0x9b92711d00000000,
+    0x6064625a00000000, 0x6d7f569300000000, 0x968945d400000000,
+    0x1b885a2500000000, 0xe07e496200000000, 0xed657dab00000000,
+    0x16936eec00000000, 0xb65564e200000000, 0x4da377a500000000,
+    0x40b8436c00000000, 0xbb4e502b00000000, 0x0035567000000000,
+    0xfbc3453700000000, 0xf6d871fe00000000, 0x0d2e62b900000000,
+    0xade868b700000000, 0x561e7bf000000000, 0x5b054f3900000000,
+    0xa0f35c7e00000000, 0x1be2f6c500000000, 0xe014e58200000000,
+    0xed0fd14b00000000, 0x16f9c20c00000000, 0xb63fc80200000000,
+    0x4dc9db4500000000, 0x40d2ef8c00000000, 0xbb24fccb00000000,
+    0x005ffa9000000000, 0xfba9e9d700000000, 0xf6b2dd1e00000000,
+    0x0d44ce5900000000, 0xad82c45700000000, 0x5674d71000000000,
+    0x5b6fe3d900000000, 0xa099f09e00000000, 0x2d98ef6f00000000,
+    0xd66efc2800000000, 0xdb75c8e100000000, 0x2083dba600000000,
+    0x8045d1a800000000, 0x7bb3c2ef00000000, 0x76a8f62600000000,
+    0x8d5ee56100000000, 0x3625e33a00000000, 0xcdd3f07d00000000,
+    0xc0c8c4b400000000, 0x3b3ed7f300000000, 0x9bf8ddfd00000000,
+    0x600eceba00000000, 0x6d15fa7300000000, 0x96e3e93400000000,
+    0x3610b54a00000000, 0xcde6a60d00000000, 0xc0fd92c400000000,
+    0x3b0b818300000000, 0x9bcd8b8d00000000, 0x603b98ca00000000,
+    0x6d20ac0300000000, 0x96d6bf4400000000, 0x2dadb91f00000000,
+    0xd65baa5800000000, 0xdb409e9100000000, 0x20b68dd600000000,
+    0x807087d800000000, 0x7b86949f00000000, 0x769da05600000000,
+    0x8d6bb31100000000, 0x006aace000000000, 0xfb9cbfa700000000,
+    0xf6878b6e00000000, 0x0d71982900000000, 0xadb7922700000000,
+    0x5641816000000000, 0x5b5ab5a900000000, 0xa0aca6ee00000000,
+    0x1bd7a0b500000000, 0xe021b3f200000000, 0xed3a873b00000000,
+    0x16cc947c00000000, 0xb60a9e7200000000, 0x4dfc8d3500000000,
+    0x40e7b9fc00000000, 0xbb11aabb00000000, 0x77c29c5000000000,
+    0x8c348f1700000000, 0x812fbbde00000000, 0x7ad9a89900000000,
+    0xda1fa29700000000, 0x21e9b1d000000000, 0x2cf2851900000000,
+    0xd704965e00000000, 0x6c7f900500000000, 0x9789834200000000,
+    0x9a92b78b00000000, 0x6164a4cc00000000, 0xc1a2aec200000000,
+    0x3a54bd8500000000, 0x374f894c00000000, 0xccb99a0b00000000,
+    0x41b885fa00000000, 0xba4e96bd00000000, 0xb755a27400000000,
+    0x4ca3b13300000000, 0xec65bb3d00000000, 0x1793a87a00000000,
+    0x1a889cb300000000, 0xe17e8ff400000000, 0x5a0589af00000000,
+    0xa1f39ae800000000, 0xace8ae2100000000, 0x571ebd6600000000,
+    0xf7d8b76800000000, 0x0c2ea42f00000000, 0x013590e600000000,
+    0xfac383a100000000, 0x5a30dfdf00000000, 0xa1c6cc9800000000,
+    0xacddf85100000000, 0x572beb1600000000, 0xf7ede11800000000,
+    0x0c1bf25f00000000, 0x0100c69600000000, 0xfaf6d5d100000000,
+    0x418dd38a00000000, 0xba7bc0cd00000000, 0xb760f40400000000,
+    0x4c96e74300000000, 0xec50ed4d00000000, 0x17a6fe0a00000000,
+    0x1abdcac300000000, 0xe14bd98400000000, 0x6c4ac67500000000,
+    0x97bcd53200000000, 0x9aa7e1fb00000000, 0x6151f2bc00000000,
+    0xc197f8b200000000, 0x3a61ebf500000000, 0x377adf3c00000000,
+    0xcc8ccc7b00000000, 0x77f7ca2000000000, 0x8c01d96700000000,
+    0x811aedae00000000, 0x7aecfee900000000, 0xda2af4e700000000,
+    0x21dce7a000000000, 0x2cc7d36900000000, 0xd731c02e00000000,
+    0x6c206a9500000000, 0x97d679d200000000, 0x9acd4d1b00000000,
+    0x613b5e5c00000000, 0xc1fd545200000000, 0x3a0b471500000000,
+    0x371073dc00000000, 0xcce6609b00000000, 0x779d66c000000000,
+    0x8c6b758700000000, 0x8170414e00000000, 0x7a86520900000000,
+    0xda40580700000000, 0x21b64b4000000000, 0x2cad7f8900000000,
+    0xd75b6cce00000000, 0x5a5a733f00000000, 0xa1ac607800000000,
+    0xacb754b100000000, 0x574147f600000000, 0xf7874df800000000,
+    0x0c715ebf00000000, 0x016a6a7600000000, 0xfa9c793100000000,
+    0x41e77f6a00000000, 0xba116c2d00000000, 0xb70a58e400000000,
+    0x4cfc4ba300000000, 0xec3a41ad00000000, 0x17cc52ea00000000,
+    0x1ad7662300000000, 0xe121756400000000, 0x41d2291a00000000,
+    0xba243a5d00000000, 0xb73f0e9400000000, 0x4cc91dd300000000,
+    0xec0f17dd00000000, 0x17f9049a00000000, 0x1ae2305300000000,
+    0xe114231400000000, 0x5a6f254f00000000, 0xa199360800000000,
+    0xac8202c100000000, 0x5774118600000000, 0xf7b21b8800000000,
+    0x0c4408cf00000000, 0x015f3c0600000000, 0xfaa92f4100000000,
+    0x77a830b000000000, 0x8c5e23f700000000, 0x8145173e00000000,
+    0x7ab3047900000000, 0xda750e7700000000, 0x21831d3000000000,
+    0x2c9829f900000000, 0xd76e3abe00000000, 0x6c153ce500000000,
+    0x97e32fa200000000, 0x9af81b6b00000000, 0x610e082c00000000,
+    0xc1c8022200000000, 0x3a3e116500000000, 0x372525ac00000000,
+    0xccd336eb00000000},
+   {0x0000000000000000, 0x6238282a00000000, 0xc470505400000000,
+    0xa648787e00000000, 0x88e1a0a800000000, 0xead9888200000000,
+    0x4c91f0fc00000000, 0x2ea9d8d600000000, 0x51c5308a00000000,
+    0x33fd18a000000000, 0x95b560de00000000, 0xf78d48f400000000,
+    0xd924902200000000, 0xbb1cb80800000000, 0x1d54c07600000000,
+    0x7f6ce85c00000000, 0xe38c10cf00000000, 0x81b438e500000000,
+    0x27fc409b00000000, 0x45c468b100000000, 0x6b6db06700000000,
+    0x0955984d00000000, 0xaf1de03300000000, 0xcd25c81900000000,
+    0xb249204500000000, 0xd071086f00000000, 0x7639701100000000,
+    0x1401583b00000000, 0x3aa880ed00000000, 0x5890a8c700000000,
+    0xfed8d0b900000000, 0x9ce0f89300000000, 0x871f504500000000,
+    0xe527786f00000000, 0x436f001100000000, 0x2157283b00000000,
+    0x0ffef0ed00000000, 0x6dc6d8c700000000, 0xcb8ea0b900000000,
+    0xa9b6889300000000, 0xd6da60cf00000000, 0xb4e248e500000000,
+    0x12aa309b00000000, 0x709218b100000000, 0x5e3bc06700000000,
+    0x3c03e84d00000000, 0x9a4b903300000000, 0xf873b81900000000,
+    0x6493408a00000000, 0x06ab68a000000000, 0xa0e310de00000000,
+    0xc2db38f400000000, 0xec72e02200000000, 0x8e4ac80800000000,
+    0x2802b07600000000, 0x4a3a985c00000000, 0x3556700000000000,
+    0x576e582a00000000, 0xf126205400000000, 0x931e087e00000000,
+    0xbdb7d0a800000000, 0xdf8ff88200000000, 0x79c780fc00000000,
+    0x1bffa8d600000000, 0x0e3fa08a00000000, 0x6c0788a000000000,
+    0xca4ff0de00000000, 0xa877d8f400000000, 0x86de002200000000,
+    0xe4e6280800000000, 0x42ae507600000000, 0x2096785c00000000,
+    0x5ffa900000000000, 0x3dc2b82a00000000, 0x9b8ac05400000000,
+    0xf9b2e87e00000000, 0xd71b30a800000000, 0xb523188200000000,
+    0x136b60fc00000000, 0x715348d600000000, 0xedb3b04500000000,
+    0x8f8b986f00000000, 0x29c3e01100000000, 0x4bfbc83b00000000,
+    0x655210ed00000000, 0x076a38c700000000, 0xa12240b900000000,
+    0xc31a689300000000, 0xbc7680cf00000000, 0xde4ea8e500000000,
+    0x7806d09b00000000, 0x1a3ef8b100000000, 0x3497206700000000,
+    0x56af084d00000000, 0xf0e7703300000000, 0x92df581900000000,
+    0x8920f0cf00000000, 0xeb18d8e500000000, 0x4d50a09b00000000,
+    0x2f6888b100000000, 0x01c1506700000000, 0x63f9784d00000000,
+    0xc5b1003300000000, 0xa789281900000000, 0xd8e5c04500000000,
+    0xbadde86f00000000, 0x1c95901100000000, 0x7eadb83b00000000,
+    0x500460ed00000000, 0x323c48c700000000, 0x947430b900000000,
+    0xf64c189300000000, 0x6aace00000000000, 0x0894c82a00000000,
+    0xaedcb05400000000, 0xcce4987e00000000, 0xe24d40a800000000,
+    0x8075688200000000, 0x263d10fc00000000, 0x440538d600000000,
+    0x3b69d08a00000000, 0x5951f8a000000000, 0xff1980de00000000,
+    0x9d21a8f400000000, 0xb388702200000000, 0xd1b0580800000000,
+    0x77f8207600000000, 0x15c0085c00000000, 0x5d7831ce00000000,
+    0x3f4019e400000000, 0x9908619a00000000, 0xfb3049b000000000,
+    0xd599916600000000, 0xb7a1b94c00000000, 0x11e9c13200000000,
+    0x73d1e91800000000, 0x0cbd014400000000, 0x6e85296e00000000,
+    0xc8cd511000000000, 0xaaf5793a00000000, 0x845ca1ec00000000,
+    0xe66489c600000000, 0x402cf1b800000000, 0x2214d99200000000,
+    0xbef4210100000000, 0xdccc092b00000000, 0x7a84715500000000,
+    0x18bc597f00000000, 0x361581a900000000, 0x542da98300000000,
+    0xf265d1fd00000000, 0x905df9d700000000, 0xef31118b00000000,
+    0x8d0939a100000000, 0x2b4141df00000000, 0x497969f500000000,
+    0x67d0b12300000000, 0x05e8990900000000, 0xa3a0e17700000000,
+    0xc198c95d00000000, 0xda67618b00000000, 0xb85f49a100000000,
+    0x1e1731df00000000, 0x7c2f19f500000000, 0x5286c12300000000,
+    0x30bee90900000000, 0x96f6917700000000, 0xf4ceb95d00000000,
+    0x8ba2510100000000, 0xe99a792b00000000, 0x4fd2015500000000,
+    0x2dea297f00000000, 0x0343f1a900000000, 0x617bd98300000000,
+    0xc733a1fd00000000, 0xa50b89d700000000, 0x39eb714400000000,
+    0x5bd3596e00000000, 0xfd9b211000000000, 0x9fa3093a00000000,
+    0xb10ad1ec00000000, 0xd332f9c600000000, 0x757a81b800000000,
+    0x1742a99200000000, 0x682e41ce00000000, 0x0a1669e400000000,
+    0xac5e119a00000000, 0xce6639b000000000, 0xe0cfe16600000000,
+    0x82f7c94c00000000, 0x24bfb13200000000, 0x4687991800000000,
+    0x5347914400000000, 0x317fb96e00000000, 0x9737c11000000000,
+    0xf50fe93a00000000, 0xdba631ec00000000, 0xb99e19c600000000,
+    0x1fd661b800000000, 0x7dee499200000000, 0x0282a1ce00000000,
+    0x60ba89e400000000, 0xc6f2f19a00000000, 0xa4cad9b000000000,
+    0x8a63016600000000, 0xe85b294c00000000, 0x4e13513200000000,
+    0x2c2b791800000000, 0xb0cb818b00000000, 0xd2f3a9a100000000,
+    0x74bbd1df00000000, 0x1683f9f500000000, 0x382a212300000000,
+    0x5a12090900000000, 0xfc5a717700000000, 0x9e62595d00000000,
+    0xe10eb10100000000, 0x8336992b00000000, 0x257ee15500000000,
+    0x4746c97f00000000, 0x69ef11a900000000, 0x0bd7398300000000,
+    0xad9f41fd00000000, 0xcfa769d700000000, 0xd458c10100000000,
+    0xb660e92b00000000, 0x1028915500000000, 0x7210b97f00000000,
+    0x5cb961a900000000, 0x3e81498300000000, 0x98c931fd00000000,
+    0xfaf119d700000000, 0x859df18b00000000, 0xe7a5d9a100000000,
+    0x41eda1df00000000, 0x23d589f500000000, 0x0d7c512300000000,
+    0x6f44790900000000, 0xc90c017700000000, 0xab34295d00000000,
+    0x37d4d1ce00000000, 0x55ecf9e400000000, 0xf3a4819a00000000,
+    0x919ca9b000000000, 0xbf35716600000000, 0xdd0d594c00000000,
+    0x7b45213200000000, 0x197d091800000000, 0x6611e14400000000,
+    0x0429c96e00000000, 0xa261b11000000000, 0xc059993a00000000,
+    0xeef041ec00000000, 0x8cc869c600000000, 0x2a8011b800000000,
+    0x48b8399200000000},
+   {0x0000000000000000, 0x4c2896a300000000, 0xd9565d9c00000000,
+    0x957ecb3f00000000, 0xf3abcbe300000000, 0xbf835d4000000000,
+    0x2afd967f00000000, 0x66d500dc00000000, 0xa751e61c00000000,
+    0xeb7970bf00000000, 0x7e07bb8000000000, 0x322f2d2300000000,
+    0x54fa2dff00000000, 0x18d2bb5c00000000, 0x8dac706300000000,
+    0xc184e6c000000000, 0x4ea3cc3900000000, 0x028b5a9a00000000,
+    0x97f591a500000000, 0xdbdd070600000000, 0xbd0807da00000000,
+    0xf120917900000000, 0x645e5a4600000000, 0x2876cce500000000,
+    0xe9f22a2500000000, 0xa5dabc8600000000, 0x30a477b900000000,
+    0x7c8ce11a00000000, 0x1a59e1c600000000, 0x5671776500000000,
+    0xc30fbc5a00000000, 0x8f272af900000000, 0x9c46997300000000,
+    0xd06e0fd000000000, 0x4510c4ef00000000, 0x0938524c00000000,
+    0x6fed529000000000, 0x23c5c43300000000, 0xb6bb0f0c00000000,
+    0xfa9399af00000000, 0x3b177f6f00000000, 0x773fe9cc00000000,
+    0xe24122f300000000, 0xae69b45000000000, 0xc8bcb48c00000000,
+    0x8494222f00000000, 0x11eae91000000000, 0x5dc27fb300000000,
+    0xd2e5554a00000000, 0x9ecdc3e900000000, 0x0bb308d600000000,
+    0x479b9e7500000000, 0x214e9ea900000000, 0x6d66080a00000000,
+    0xf818c33500000000, 0xb430559600000000, 0x75b4b35600000000,
+    0x399c25f500000000, 0xace2eeca00000000, 0xe0ca786900000000,
+    0x861f78b500000000, 0xca37ee1600000000, 0x5f49252900000000,
+    0x1361b38a00000000, 0x388d32e700000000, 0x74a5a44400000000,
+    0xe1db6f7b00000000, 0xadf3f9d800000000, 0xcb26f90400000000,
+    0x870e6fa700000000, 0x1270a49800000000, 0x5e58323b00000000,
+    0x9fdcd4fb00000000, 0xd3f4425800000000, 0x468a896700000000,
+    0x0aa21fc400000000, 0x6c771f1800000000, 0x205f89bb00000000,
+    0xb521428400000000, 0xf909d42700000000, 0x762efede00000000,
+    0x3a06687d00000000, 0xaf78a34200000000, 0xe35035e100000000,
+    0x8585353d00000000, 0xc9ada39e00000000, 0x5cd368a100000000,
+    0x10fbfe0200000000, 0xd17f18c200000000, 0x9d578e6100000000,
+    0x0829455e00000000, 0x4401d3fd00000000, 0x22d4d32100000000,
+    0x6efc458200000000, 0xfb828ebd00000000, 0xb7aa181e00000000,
+    0xa4cbab9400000000, 0xe8e33d3700000000, 0x7d9df60800000000,
+    0x31b560ab00000000, 0x5760607700000000, 0x1b48f6d400000000,
+    0x8e363deb00000000, 0xc21eab4800000000, 0x039a4d8800000000,
+    0x4fb2db2b00000000, 0xdacc101400000000, 0x96e486b700000000,
+    0xf031866b00000000, 0xbc1910c800000000, 0x2967dbf700000000,
+    0x654f4d5400000000, 0xea6867ad00000000, 0xa640f10e00000000,
+    0x333e3a3100000000, 0x7f16ac9200000000, 0x19c3ac4e00000000,
+    0x55eb3aed00000000, 0xc095f1d200000000, 0x8cbd677100000000,
+    0x4d3981b100000000, 0x0111171200000000, 0x946fdc2d00000000,
+    0xd8474a8e00000000, 0xbe924a5200000000, 0xf2badcf100000000,
+    0x67c417ce00000000, 0x2bec816d00000000, 0x311c141500000000,
+    0x7d3482b600000000, 0xe84a498900000000, 0xa462df2a00000000,
+    0xc2b7dff600000000, 0x8e9f495500000000, 0x1be1826a00000000,
+    0x57c914c900000000, 0x964df20900000000, 0xda6564aa00000000,
+    0x4f1baf9500000000, 0x0333393600000000, 0x65e639ea00000000,
+    0x29ceaf4900000000, 0xbcb0647600000000, 0xf098f2d500000000,
+    0x7fbfd82c00000000, 0x33974e8f00000000, 0xa6e985b000000000,
+    0xeac1131300000000, 0x8c1413cf00000000, 0xc03c856c00000000,
+    0x55424e5300000000, 0x196ad8f000000000, 0xd8ee3e3000000000,
+    0x94c6a89300000000, 0x01b863ac00000000, 0x4d90f50f00000000,
+    0x2b45f5d300000000, 0x676d637000000000, 0xf213a84f00000000,
+    0xbe3b3eec00000000, 0xad5a8d6600000000, 0xe1721bc500000000,
+    0x740cd0fa00000000, 0x3824465900000000, 0x5ef1468500000000,
+    0x12d9d02600000000, 0x87a71b1900000000, 0xcb8f8dba00000000,
+    0x0a0b6b7a00000000, 0x4623fdd900000000, 0xd35d36e600000000,
+    0x9f75a04500000000, 0xf9a0a09900000000, 0xb588363a00000000,
+    0x20f6fd0500000000, 0x6cde6ba600000000, 0xe3f9415f00000000,
+    0xafd1d7fc00000000, 0x3aaf1cc300000000, 0x76878a6000000000,
+    0x10528abc00000000, 0x5c7a1c1f00000000, 0xc904d72000000000,
+    0x852c418300000000, 0x44a8a74300000000, 0x088031e000000000,
+    0x9dfefadf00000000, 0xd1d66c7c00000000, 0xb7036ca000000000,
+    0xfb2bfa0300000000, 0x6e55313c00000000, 0x227da79f00000000,
+    0x099126f200000000, 0x45b9b05100000000, 0xd0c77b6e00000000,
+    0x9cefedcd00000000, 0xfa3aed1100000000, 0xb6127bb200000000,
+    0x236cb08d00000000, 0x6f44262e00000000, 0xaec0c0ee00000000,
+    0xe2e8564d00000000, 0x77969d7200000000, 0x3bbe0bd100000000,
+    0x5d6b0b0d00000000, 0x11439dae00000000, 0x843d569100000000,
+    0xc815c03200000000, 0x4732eacb00000000, 0x0b1a7c6800000000,
+    0x9e64b75700000000, 0xd24c21f400000000, 0xb499212800000000,
+    0xf8b1b78b00000000, 0x6dcf7cb400000000, 0x21e7ea1700000000,
+    0xe0630cd700000000, 0xac4b9a7400000000, 0x3935514b00000000,
+    0x751dc7e800000000, 0x13c8c73400000000, 0x5fe0519700000000,
+    0xca9e9aa800000000, 0x86b60c0b00000000, 0x95d7bf8100000000,
+    0xd9ff292200000000, 0x4c81e21d00000000, 0x00a974be00000000,
+    0x667c746200000000, 0x2a54e2c100000000, 0xbf2a29fe00000000,
+    0xf302bf5d00000000, 0x3286599d00000000, 0x7eaecf3e00000000,
+    0xebd0040100000000, 0xa7f892a200000000, 0xc12d927e00000000,
+    0x8d0504dd00000000, 0x187bcfe200000000, 0x5453594100000000,
+    0xdb7473b800000000, 0x975ce51b00000000, 0x02222e2400000000,
+    0x4e0ab88700000000, 0x28dfb85b00000000, 0x64f72ef800000000,
+    0xf189e5c700000000, 0xbda1736400000000, 0x7c2595a400000000,
+    0x300d030700000000, 0xa573c83800000000, 0xe95b5e9b00000000,
+    0x8f8e5e4700000000, 0xc3a6c8e400000000, 0x56d803db00000000,
+    0x1af0957800000000},
+   {0x0000000000000000, 0x939bc97f00000000, 0x263793ff00000000,
+    0xb5ac5a8000000000, 0x0d68572400000000, 0x9ef39e5b00000000,
+    0x2b5fc4db00000000, 0xb8c40da400000000, 0x1ad0ae4800000000,
+    0x894b673700000000, 0x3ce73db700000000, 0xaf7cf4c800000000,
+    0x17b8f96c00000000, 0x8423301300000000, 0x318f6a9300000000,
+    0xa214a3ec00000000, 0x34a05d9100000000, 0xa73b94ee00000000,
+    0x1297ce6e00000000, 0x810c071100000000, 0x39c80ab500000000,
+    0xaa53c3ca00000000, 0x1fff994a00000000, 0x8c64503500000000,
+    0x2e70f3d900000000, 0xbdeb3aa600000000, 0x0847602600000000,
+    0x9bdca95900000000, 0x2318a4fd00000000, 0xb0836d8200000000,
+    0x052f370200000000, 0x96b4fe7d00000000, 0x2946caf900000000,
+    0xbadd038600000000, 0x0f71590600000000, 0x9cea907900000000,
+    0x242e9ddd00000000, 0xb7b554a200000000, 0x02190e2200000000,
+    0x9182c75d00000000, 0x339664b100000000, 0xa00dadce00000000,
+    0x15a1f74e00000000, 0x863a3e3100000000, 0x3efe339500000000,
+    0xad65faea00000000, 0x18c9a06a00000000, 0x8b52691500000000,
+    0x1de6976800000000, 0x8e7d5e1700000000, 0x3bd1049700000000,
+    0xa84acde800000000, 0x108ec04c00000000, 0x8315093300000000,
+    0x36b953b300000000, 0xa5229acc00000000, 0x0736392000000000,
+    0x94adf05f00000000, 0x2101aadf00000000, 0xb29a63a000000000,
+    0x0a5e6e0400000000, 0x99c5a77b00000000, 0x2c69fdfb00000000,
+    0xbff2348400000000, 0x138ae52800000000, 0x80112c5700000000,
+    0x35bd76d700000000, 0xa626bfa800000000, 0x1ee2b20c00000000,
+    0x8d797b7300000000, 0x38d521f300000000, 0xab4ee88c00000000,
+    0x095a4b6000000000, 0x9ac1821f00000000, 0x2f6dd89f00000000,
+    0xbcf611e000000000, 0x04321c4400000000, 0x97a9d53b00000000,
+    0x22058fbb00000000, 0xb19e46c400000000, 0x272ab8b900000000,
+    0xb4b171c600000000, 0x011d2b4600000000, 0x9286e23900000000,
+    0x2a42ef9d00000000, 0xb9d926e200000000, 0x0c757c6200000000,
+    0x9feeb51d00000000, 0x3dfa16f100000000, 0xae61df8e00000000,
+    0x1bcd850e00000000, 0x88564c7100000000, 0x309241d500000000,
+    0xa30988aa00000000, 0x16a5d22a00000000, 0x853e1b5500000000,
+    0x3acc2fd100000000, 0xa957e6ae00000000, 0x1cfbbc2e00000000,
+    0x8f60755100000000, 0x37a478f500000000, 0xa43fb18a00000000,
+    0x1193eb0a00000000, 0x8208227500000000, 0x201c819900000000,
+    0xb38748e600000000, 0x062b126600000000, 0x95b0db1900000000,
+    0x2d74d6bd00000000, 0xbeef1fc200000000, 0x0b43454200000000,
+    0x98d88c3d00000000, 0x0e6c724000000000, 0x9df7bb3f00000000,
+    0x285be1bf00000000, 0xbbc028c000000000, 0x0304256400000000,
+    0x909fec1b00000000, 0x2533b69b00000000, 0xb6a87fe400000000,
+    0x14bcdc0800000000, 0x8727157700000000, 0x328b4ff700000000,
+    0xa110868800000000, 0x19d48b2c00000000, 0x8a4f425300000000,
+    0x3fe318d300000000, 0xac78d1ac00000000, 0x2614cb5100000000,
+    0xb58f022e00000000, 0x002358ae00000000, 0x93b891d100000000,
+    0x2b7c9c7500000000, 0xb8e7550a00000000, 0x0d4b0f8a00000000,
+    0x9ed0c6f500000000, 0x3cc4651900000000, 0xaf5fac6600000000,
+    0x1af3f6e600000000, 0x89683f9900000000, 0x31ac323d00000000,
+    0xa237fb4200000000, 0x179ba1c200000000, 0x840068bd00000000,
+    0x12b496c000000000, 0x812f5fbf00000000, 0x3483053f00000000,
+    0xa718cc4000000000, 0x1fdcc1e400000000, 0x8c47089b00000000,
+    0x39eb521b00000000, 0xaa709b6400000000, 0x0864388800000000,
+    0x9bfff1f700000000, 0x2e53ab7700000000, 0xbdc8620800000000,
+    0x050c6fac00000000, 0x9697a6d300000000, 0x233bfc5300000000,
+    0xb0a0352c00000000, 0x0f5201a800000000, 0x9cc9c8d700000000,
+    0x2965925700000000, 0xbafe5b2800000000, 0x023a568c00000000,
+    0x91a19ff300000000, 0x240dc57300000000, 0xb7960c0c00000000,
+    0x1582afe000000000, 0x8619669f00000000, 0x33b53c1f00000000,
+    0xa02ef56000000000, 0x18eaf8c400000000, 0x8b7131bb00000000,
+    0x3edd6b3b00000000, 0xad46a24400000000, 0x3bf25c3900000000,
+    0xa869954600000000, 0x1dc5cfc600000000, 0x8e5e06b900000000,
+    0x369a0b1d00000000, 0xa501c26200000000, 0x10ad98e200000000,
+    0x8336519d00000000, 0x2122f27100000000, 0xb2b93b0e00000000,
+    0x0715618e00000000, 0x948ea8f100000000, 0x2c4aa55500000000,
+    0xbfd16c2a00000000, 0x0a7d36aa00000000, 0x99e6ffd500000000,
+    0x359e2e7900000000, 0xa605e70600000000, 0x13a9bd8600000000,
+    0x803274f900000000, 0x38f6795d00000000, 0xab6db02200000000,
+    0x1ec1eaa200000000, 0x8d5a23dd00000000, 0x2f4e803100000000,
+    0xbcd5494e00000000, 0x097913ce00000000, 0x9ae2dab100000000,
+    0x2226d71500000000, 0xb1bd1e6a00000000, 0x041144ea00000000,
+    0x978a8d9500000000, 0x013e73e800000000, 0x92a5ba9700000000,
+    0x2709e01700000000, 0xb492296800000000, 0x0c5624cc00000000,
+    0x9fcdedb300000000, 0x2a61b73300000000, 0xb9fa7e4c00000000,
+    0x1beedda000000000, 0x887514df00000000, 0x3dd94e5f00000000,
+    0xae42872000000000, 0x16868a8400000000, 0x851d43fb00000000,
+    0x30b1197b00000000, 0xa32ad00400000000, 0x1cd8e48000000000,
+    0x8f432dff00000000, 0x3aef777f00000000, 0xa974be0000000000,
+    0x11b0b3a400000000, 0x822b7adb00000000, 0x3787205b00000000,
+    0xa41ce92400000000, 0x06084ac800000000, 0x959383b700000000,
+    0x203fd93700000000, 0xb3a4104800000000, 0x0b601dec00000000,
+    0x98fbd49300000000, 0x2d578e1300000000, 0xbecc476c00000000,
+    0x2878b91100000000, 0xbbe3706e00000000, 0x0e4f2aee00000000,
+    0x9dd4e39100000000, 0x2510ee3500000000, 0xb68b274a00000000,
+    0x03277dca00000000, 0x90bcb4b500000000, 0x32a8175900000000,
+    0xa133de2600000000, 0x149f84a600000000, 0x87044dd900000000,
+    0x3fc0407d00000000, 0xac5b890200000000, 0x19f7d38200000000,
+    0x8a6c1afd00000000},
+   {0x0000000000000000, 0x650b796900000000, 0xca16f2d200000000,
+    0xaf1d8bbb00000000, 0xd52b957e00000000, 0xb020ec1700000000,
+    0x1f3d67ac00000000, 0x7a361ec500000000, 0xaa572afd00000000,
+    0xcf5c539400000000, 0x6041d82f00000000, 0x054aa14600000000,
+    0x7f7cbf8300000000, 0x1a77c6ea00000000, 0xb56a4d5100000000,
+    0xd061343800000000, 0x15a9252100000000, 0x70a25c4800000000,
+    0xdfbfd7f300000000, 0xbab4ae9a00000000, 0xc082b05f00000000,
+    0xa589c93600000000, 0x0a94428d00000000, 0x6f9f3be400000000,
+    0xbffe0fdc00000000, 0xdaf576b500000000, 0x75e8fd0e00000000,
+    0x10e3846700000000, 0x6ad59aa200000000, 0x0fdee3cb00000000,
+    0xa0c3687000000000, 0xc5c8111900000000, 0x2a524b4200000000,
+    0x4f59322b00000000, 0xe044b99000000000, 0x854fc0f900000000,
+    0xff79de3c00000000, 0x9a72a75500000000, 0x356f2cee00000000,
+    0x5064558700000000, 0x800561bf00000000, 0xe50e18d600000000,
+    0x4a13936d00000000, 0x2f18ea0400000000, 0x552ef4c100000000,
+    0x30258da800000000, 0x9f38061300000000, 0xfa337f7a00000000,
+    0x3ffb6e6300000000, 0x5af0170a00000000, 0xf5ed9cb100000000,
+    0x90e6e5d800000000, 0xead0fb1d00000000, 0x8fdb827400000000,
+    0x20c609cf00000000, 0x45cd70a600000000, 0x95ac449e00000000,
+    0xf0a73df700000000, 0x5fbab64c00000000, 0x3ab1cf2500000000,
+    0x4087d1e000000000, 0x258ca88900000000, 0x8a91233200000000,
+    0xef9a5a5b00000000, 0x54a4968400000000, 0x31afefed00000000,
+    0x9eb2645600000000, 0xfbb91d3f00000000, 0x818f03fa00000000,
+    0xe4847a9300000000, 0x4b99f12800000000, 0x2e92884100000000,
+    0xfef3bc7900000000, 0x9bf8c51000000000, 0x34e54eab00000000,
+    0x51ee37c200000000, 0x2bd8290700000000, 0x4ed3506e00000000,
+    0xe1cedbd500000000, 0x84c5a2bc00000000, 0x410db3a500000000,
+    0x2406cacc00000000, 0x8b1b417700000000, 0xee10381e00000000,
+    0x942626db00000000, 0xf12d5fb200000000, 0x5e30d40900000000,
+    0x3b3bad6000000000, 0xeb5a995800000000, 0x8e51e03100000000,
+    0x214c6b8a00000000, 0x444712e300000000, 0x3e710c2600000000,
+    0x5b7a754f00000000, 0xf467fef400000000, 0x916c879d00000000,
+    0x7ef6ddc600000000, 0x1bfda4af00000000, 0xb4e02f1400000000,
+    0xd1eb567d00000000, 0xabdd48b800000000, 0xced631d100000000,
+    0x61cbba6a00000000, 0x04c0c30300000000, 0xd4a1f73b00000000,
+    0xb1aa8e5200000000, 0x1eb705e900000000, 0x7bbc7c8000000000,
+    0x018a624500000000, 0x64811b2c00000000, 0xcb9c909700000000,
+    0xae97e9fe00000000, 0x6b5ff8e700000000, 0x0e54818e00000000,
+    0xa1490a3500000000, 0xc442735c00000000, 0xbe746d9900000000,
+    0xdb7f14f000000000, 0x74629f4b00000000, 0x1169e62200000000,
+    0xc108d21a00000000, 0xa403ab7300000000, 0x0b1e20c800000000,
+    0x6e1559a100000000, 0x1423476400000000, 0x71283e0d00000000,
+    0xde35b5b600000000, 0xbb3eccdf00000000, 0xe94e5cd200000000,
+    0x8c4525bb00000000, 0x2358ae0000000000, 0x4653d76900000000,
+    0x3c65c9ac00000000, 0x596eb0c500000000, 0xf6733b7e00000000,
+    0x9378421700000000, 0x4319762f00000000, 0x26120f4600000000,
+    0x890f84fd00000000, 0xec04fd9400000000, 0x9632e35100000000,
+    0xf3399a3800000000, 0x5c24118300000000, 0x392f68ea00000000,
+    0xfce779f300000000, 0x99ec009a00000000, 0x36f18b2100000000,
+    0x53faf24800000000, 0x29ccec8d00000000, 0x4cc795e400000000,
+    0xe3da1e5f00000000, 0x86d1673600000000, 0x56b0530e00000000,
+    0x33bb2a6700000000, 0x9ca6a1dc00000000, 0xf9add8b500000000,
+    0x839bc67000000000, 0xe690bf1900000000, 0x498d34a200000000,
+    0x2c864dcb00000000, 0xc31c179000000000, 0xa6176ef900000000,
+    0x090ae54200000000, 0x6c019c2b00000000, 0x163782ee00000000,
+    0x733cfb8700000000, 0xdc21703c00000000, 0xb92a095500000000,
+    0x694b3d6d00000000, 0x0c40440400000000, 0xa35dcfbf00000000,
+    0xc656b6d600000000, 0xbc60a81300000000, 0xd96bd17a00000000,
+    0x76765ac100000000, 0x137d23a800000000, 0xd6b532b100000000,
+    0xb3be4bd800000000, 0x1ca3c06300000000, 0x79a8b90a00000000,
+    0x039ea7cf00000000, 0x6695dea600000000, 0xc988551d00000000,
+    0xac832c7400000000, 0x7ce2184c00000000, 0x19e9612500000000,
+    0xb6f4ea9e00000000, 0xd3ff93f700000000, 0xa9c98d3200000000,
+    0xccc2f45b00000000, 0x63df7fe000000000, 0x06d4068900000000,
+    0xbdeaca5600000000, 0xd8e1b33f00000000, 0x77fc388400000000,
+    0x12f741ed00000000, 0x68c15f2800000000, 0x0dca264100000000,
+    0xa2d7adfa00000000, 0xc7dcd49300000000, 0x17bde0ab00000000,
+    0x72b699c200000000, 0xddab127900000000, 0xb8a06b1000000000,
+    0xc29675d500000000, 0xa79d0cbc00000000, 0x0880870700000000,
+    0x6d8bfe6e00000000, 0xa843ef7700000000, 0xcd48961e00000000,
+    0x62551da500000000, 0x075e64cc00000000, 0x7d687a0900000000,
+    0x1863036000000000, 0xb77e88db00000000, 0xd275f1b200000000,
+    0x0214c58a00000000, 0x671fbce300000000, 0xc802375800000000,
+    0xad094e3100000000, 0xd73f50f400000000, 0xb234299d00000000,
+    0x1d29a22600000000, 0x7822db4f00000000, 0x97b8811400000000,
+    0xf2b3f87d00000000, 0x5dae73c600000000, 0x38a50aaf00000000,
+    0x4293146a00000000, 0x27986d0300000000, 0x8885e6b800000000,
+    0xed8e9fd100000000, 0x3defabe900000000, 0x58e4d28000000000,
+    0xf7f9593b00000000, 0x92f2205200000000, 0xe8c43e9700000000,
+    0x8dcf47fe00000000, 0x22d2cc4500000000, 0x47d9b52c00000000,
+    0x8211a43500000000, 0xe71add5c00000000, 0x480756e700000000,
+    0x2d0c2f8e00000000, 0x573a314b00000000, 0x3231482200000000,
+    0x9d2cc39900000000, 0xf827baf000000000, 0x28468ec800000000,
+    0x4d4df7a100000000, 0xe2507c1a00000000, 0x875b057300000000,
+    0xfd6d1bb600000000, 0x986662df00000000, 0x377be96400000000,
+    0x5270900d00000000},
+   {0x0000000000000000, 0xdcecb13d00000000, 0xb8d9637b00000000,
+    0x6435d24600000000, 0x70b3c7f600000000, 0xac5f76cb00000000,
+    0xc86aa48d00000000, 0x148615b000000000, 0xa160fe3600000000,
+    0x7d8c4f0b00000000, 0x19b99d4d00000000, 0xc5552c7000000000,
+    0xd1d339c000000000, 0x0d3f88fd00000000, 0x690a5abb00000000,
+    0xb5e6eb8600000000, 0x42c1fc6d00000000, 0x9e2d4d5000000000,
+    0xfa189f1600000000, 0x26f42e2b00000000, 0x32723b9b00000000,
+    0xee9e8aa600000000, 0x8aab58e000000000, 0x5647e9dd00000000,
+    0xe3a1025b00000000, 0x3f4db36600000000, 0x5b78612000000000,
+    0x8794d01d00000000, 0x9312c5ad00000000, 0x4ffe749000000000,
+    0x2bcba6d600000000, 0xf72717eb00000000, 0x8482f9db00000000,
+    0x586e48e600000000, 0x3c5b9aa000000000, 0xe0b72b9d00000000,
+    0xf4313e2d00000000, 0x28dd8f1000000000, 0x4ce85d5600000000,
+    0x9004ec6b00000000, 0x25e207ed00000000, 0xf90eb6d000000000,
+    0x9d3b649600000000, 0x41d7d5ab00000000, 0x5551c01b00000000,
+    0x89bd712600000000, 0xed88a36000000000, 0x3164125d00000000,
+    0xc64305b600000000, 0x1aafb48b00000000, 0x7e9a66cd00000000,
+    0xa276d7f000000000, 0xb6f0c24000000000, 0x6a1c737d00000000,
+    0x0e29a13b00000000, 0xd2c5100600000000, 0x6723fb8000000000,
+    0xbbcf4abd00000000, 0xdffa98fb00000000, 0x031629c600000000,
+    0x17903c7600000000, 0xcb7c8d4b00000000, 0xaf495f0d00000000,
+    0x73a5ee3000000000, 0x4903826c00000000, 0x95ef335100000000,
+    0xf1dae11700000000, 0x2d36502a00000000, 0x39b0459a00000000,
+    0xe55cf4a700000000, 0x816926e100000000, 0x5d8597dc00000000,
+    0xe8637c5a00000000, 0x348fcd6700000000, 0x50ba1f2100000000,
+    0x8c56ae1c00000000, 0x98d0bbac00000000, 0x443c0a9100000000,
+    0x2009d8d700000000, 0xfce569ea00000000, 0x0bc27e0100000000,
+    0xd72ecf3c00000000, 0xb31b1d7a00000000, 0x6ff7ac4700000000,
+    0x7b71b9f700000000, 0xa79d08ca00000000, 0xc3a8da8c00000000,
+    0x1f446bb100000000, 0xaaa2803700000000, 0x764e310a00000000,
+    0x127be34c00000000, 0xce97527100000000, 0xda1147c100000000,
+    0x06fdf6fc00000000, 0x62c824ba00000000, 0xbe24958700000000,
+    0xcd817bb700000000, 0x116dca8a00000000, 0x755818cc00000000,
+    0xa9b4a9f100000000, 0xbd32bc4100000000, 0x61de0d7c00000000,
+    0x05ebdf3a00000000, 0xd9076e0700000000, 0x6ce1858100000000,
+    0xb00d34bc00000000, 0xd438e6fa00000000, 0x08d457c700000000,
+    0x1c52427700000000, 0xc0bef34a00000000, 0xa48b210c00000000,
+    0x7867903100000000, 0x8f4087da00000000, 0x53ac36e700000000,
+    0x3799e4a100000000, 0xeb75559c00000000, 0xfff3402c00000000,
+    0x231ff11100000000, 0x472a235700000000, 0x9bc6926a00000000,
+    0x2e2079ec00000000, 0xf2ccc8d100000000, 0x96f91a9700000000,
+    0x4a15abaa00000000, 0x5e93be1a00000000, 0x827f0f2700000000,
+    0xe64add6100000000, 0x3aa66c5c00000000, 0x920604d900000000,
+    0x4eeab5e400000000, 0x2adf67a200000000, 0xf633d69f00000000,
+    0xe2b5c32f00000000, 0x3e59721200000000, 0x5a6ca05400000000,
+    0x8680116900000000, 0x3366faef00000000, 0xef8a4bd200000000,
+    0x8bbf999400000000, 0x575328a900000000, 0x43d53d1900000000,
+    0x9f398c2400000000, 0xfb0c5e6200000000, 0x27e0ef5f00000000,
+    0xd0c7f8b400000000, 0x0c2b498900000000, 0x681e9bcf00000000,
+    0xb4f22af200000000, 0xa0743f4200000000, 0x7c988e7f00000000,
+    0x18ad5c3900000000, 0xc441ed0400000000, 0x71a7068200000000,
+    0xad4bb7bf00000000, 0xc97e65f900000000, 0x1592d4c400000000,
+    0x0114c17400000000, 0xddf8704900000000, 0xb9cda20f00000000,
+    0x6521133200000000, 0x1684fd0200000000, 0xca684c3f00000000,
+    0xae5d9e7900000000, 0x72b12f4400000000, 0x66373af400000000,
+    0xbadb8bc900000000, 0xdeee598f00000000, 0x0202e8b200000000,
+    0xb7e4033400000000, 0x6b08b20900000000, 0x0f3d604f00000000,
+    0xd3d1d17200000000, 0xc757c4c200000000, 0x1bbb75ff00000000,
+    0x7f8ea7b900000000, 0xa362168400000000, 0x5445016f00000000,
+    0x88a9b05200000000, 0xec9c621400000000, 0x3070d32900000000,
+    0x24f6c69900000000, 0xf81a77a400000000, 0x9c2fa5e200000000,
+    0x40c314df00000000, 0xf525ff5900000000, 0x29c94e6400000000,
+    0x4dfc9c2200000000, 0x91102d1f00000000, 0x859638af00000000,
+    0x597a899200000000, 0x3d4f5bd400000000, 0xe1a3eae900000000,
+    0xdb0586b500000000, 0x07e9378800000000, 0x63dce5ce00000000,
+    0xbf3054f300000000, 0xabb6414300000000, 0x775af07e00000000,
+    0x136f223800000000, 0xcf83930500000000, 0x7a65788300000000,
+    0xa689c9be00000000, 0xc2bc1bf800000000, 0x1e50aac500000000,
+    0x0ad6bf7500000000, 0xd63a0e4800000000, 0xb20fdc0e00000000,
+    0x6ee36d3300000000, 0x99c47ad800000000, 0x4528cbe500000000,
+    0x211d19a300000000, 0xfdf1a89e00000000, 0xe977bd2e00000000,
+    0x359b0c1300000000, 0x51aede5500000000, 0x8d426f6800000000,
+    0x38a484ee00000000, 0xe44835d300000000, 0x807de79500000000,
+    0x5c9156a800000000, 0x4817431800000000, 0x94fbf22500000000,
+    0xf0ce206300000000, 0x2c22915e00000000, 0x5f877f6e00000000,
+    0x836bce5300000000, 0xe75e1c1500000000, 0x3bb2ad2800000000,
+    0x2f34b89800000000, 0xf3d809a500000000, 0x97eddbe300000000,
+    0x4b016ade00000000, 0xfee7815800000000, 0x220b306500000000,
+    0x463ee22300000000, 0x9ad2531e00000000, 0x8e5446ae00000000,
+    0x52b8f79300000000, 0x368d25d500000000, 0xea6194e800000000,
+    0x1d46830300000000, 0xc1aa323e00000000, 0xa59fe07800000000,
+    0x7973514500000000, 0x6df544f500000000, 0xb119f5c800000000,
+    0xd52c278e00000000, 0x09c096b300000000, 0xbc267d3500000000,
+    0x60cacc0800000000, 0x04ff1e4e00000000, 0xd813af7300000000,
+    0xcc95bac300000000, 0x10790bfe00000000, 0x744cd9b800000000,
+    0xa8a0688500000000}};
+
+#else /* W == 4 */
+
+local const z_crc_t FAR crc_braid_table[][256] = {
+   {0x00000000, 0x81256527, 0xd93bcc0f, 0x581ea928, 0x69069e5f,
+    0xe823fb78, 0xb03d5250, 0x31183777, 0xd20d3cbe, 0x53285999,
+    0x0b36f0b1, 0x8a139596, 0xbb0ba2e1, 0x3a2ec7c6, 0x62306eee,
+    0xe3150bc9, 0x7f6b7f3d, 0xfe4e1a1a, 0xa650b332, 0x2775d615,
+    0x166de162, 0x97488445, 0xcf562d6d, 0x4e73484a, 0xad664383,
+    0x2c4326a4, 0x745d8f8c, 0xf578eaab, 0xc460dddc, 0x4545b8fb,
+    0x1d5b11d3, 0x9c7e74f4, 0xfed6fe7a, 0x7ff39b5d, 0x27ed3275,
+    0xa6c85752, 0x97d06025, 0x16f50502, 0x4eebac2a, 0xcfcec90d,
+    0x2cdbc2c4, 0xadfea7e3, 0xf5e00ecb, 0x74c56bec, 0x45dd5c9b,
+    0xc4f839bc, 0x9ce69094, 0x1dc3f5b3, 0x81bd8147, 0x0098e460,
+    0x58864d48, 0xd9a3286f, 0xe8bb1f18, 0x699e7a3f, 0x3180d317,
+    0xb0a5b630, 0x53b0bdf9, 0xd295d8de, 0x8a8b71f6, 0x0bae14d1,
+    0x3ab623a6, 0xbb934681, 0xe38defa9, 0x62a88a8e, 0x26dcfab5,
+    0xa7f99f92, 0xffe736ba, 0x7ec2539d, 0x4fda64ea, 0xceff01cd,
+    0x96e1a8e5, 0x17c4cdc2, 0xf4d1c60b, 0x75f4a32c, 0x2dea0a04,
+    0xaccf6f23, 0x9dd75854, 0x1cf23d73, 0x44ec945b, 0xc5c9f17c,
+    0x59b78588, 0xd892e0af, 0x808c4987, 0x01a92ca0, 0x30b11bd7,
+    0xb1947ef0, 0xe98ad7d8, 0x68afb2ff, 0x8bbab936, 0x0a9fdc11,
+    0x52817539, 0xd3a4101e, 0xe2bc2769, 0x6399424e, 0x3b87eb66,
+    0xbaa28e41, 0xd80a04cf, 0x592f61e8, 0x0131c8c0, 0x8014ade7,
+    0xb10c9a90, 0x3029ffb7, 0x6837569f, 0xe91233b8, 0x0a073871,
+    0x8b225d56, 0xd33cf47e, 0x52199159, 0x6301a62e, 0xe224c309,
+    0xba3a6a21, 0x3b1f0f06, 0xa7617bf2, 0x26441ed5, 0x7e5ab7fd,
+    0xff7fd2da, 0xce67e5ad, 0x4f42808a, 0x175c29a2, 0x96794c85,
+    0x756c474c, 0xf449226b, 0xac578b43, 0x2d72ee64, 0x1c6ad913,
+    0x9d4fbc34, 0xc551151c, 0x4474703b, 0x4db9f56a, 0xcc9c904d,
+    0x94823965, 0x15a75c42, 0x24bf6b35, 0xa59a0e12, 0xfd84a73a,
+    0x7ca1c21d, 0x9fb4c9d4, 0x1e91acf3, 0x468f05db, 0xc7aa60fc,
+    0xf6b2578b, 0x779732ac, 0x2f899b84, 0xaeacfea3, 0x32d28a57,
+    0xb3f7ef70, 0xebe94658, 0x6acc237f, 0x5bd41408, 0xdaf1712f,
+    0x82efd807, 0x03cabd20, 0xe0dfb6e9, 0x61fad3ce, 0x39e47ae6,
+    0xb8c11fc1, 0x89d928b6, 0x08fc4d91, 0x50e2e4b9, 0xd1c7819e,
+    0xb36f0b10, 0x324a6e37, 0x6a54c71f, 0xeb71a238, 0xda69954f,
+    0x5b4cf068, 0x03525940, 0x82773c67, 0x616237ae, 0xe0475289,
+    0xb859fba1, 0x397c9e86, 0x0864a9f1, 0x8941ccd6, 0xd15f65fe,
+    0x507a00d9, 0xcc04742d, 0x4d21110a, 0x153fb822, 0x941add05,
+    0xa502ea72, 0x24278f55, 0x7c39267d, 0xfd1c435a, 0x1e094893,
+    0x9f2c2db4, 0xc732849c, 0x4617e1bb, 0x770fd6cc, 0xf62ab3eb,
+    0xae341ac3, 0x2f117fe4, 0x6b650fdf, 0xea406af8, 0xb25ec3d0,
+    0x337ba6f7, 0x02639180, 0x8346f4a7, 0xdb585d8f, 0x5a7d38a8,
+    0xb9683361, 0x384d5646, 0x6053ff6e, 0xe1769a49, 0xd06ead3e,
+    0x514bc819, 0x09556131, 0x88700416, 0x140e70e2, 0x952b15c5,
+    0xcd35bced, 0x4c10d9ca, 0x7d08eebd, 0xfc2d8b9a, 0xa43322b2,
+    0x25164795, 0xc6034c5c, 0x4726297b, 0x1f388053, 0x9e1de574,
+    0xaf05d203, 0x2e20b724, 0x763e1e0c, 0xf71b7b2b, 0x95b3f1a5,
+    0x14969482, 0x4c883daa, 0xcdad588d, 0xfcb56ffa, 0x7d900add,
+    0x258ea3f5, 0xa4abc6d2, 0x47becd1b, 0xc69ba83c, 0x9e850114,
+    0x1fa06433, 0x2eb85344, 0xaf9d3663, 0xf7839f4b, 0x76a6fa6c,
+    0xead88e98, 0x6bfdebbf, 0x33e34297, 0xb2c627b0, 0x83de10c7,
+    0x02fb75e0, 0x5ae5dcc8, 0xdbc0b9ef, 0x38d5b226, 0xb9f0d701,
+    0xe1ee7e29, 0x60cb1b0e, 0x51d32c79, 0xd0f6495e, 0x88e8e076,
+    0x09cd8551},
+   {0x00000000, 0x9b73ead4, 0xed96d3e9, 0x76e5393d, 0x005ca193,
+    0x9b2f4b47, 0xedca727a, 0x76b998ae, 0x00b94326, 0x9bcaa9f2,
+    0xed2f90cf, 0x765c7a1b, 0x00e5e2b5, 0x9b960861, 0xed73315c,
+    0x7600db88, 0x0172864c, 0x9a016c98, 0xece455a5, 0x7797bf71,
+    0x012e27df, 0x9a5dcd0b, 0xecb8f436, 0x77cb1ee2, 0x01cbc56a,
+    0x9ab82fbe, 0xec5d1683, 0x772efc57, 0x019764f9, 0x9ae48e2d,
+    0xec01b710, 0x77725dc4, 0x02e50c98, 0x9996e64c, 0xef73df71,
+    0x740035a5, 0x02b9ad0b, 0x99ca47df, 0xef2f7ee2, 0x745c9436,
+    0x025c4fbe, 0x992fa56a, 0xefca9c57, 0x74b97683, 0x0200ee2d,
+    0x997304f9, 0xef963dc4, 0x74e5d710, 0x03978ad4, 0x98e46000,
+    0xee01593d, 0x7572b3e9, 0x03cb2b47, 0x98b8c193, 0xee5df8ae,
+    0x752e127a, 0x032ec9f2, 0x985d2326, 0xeeb81a1b, 0x75cbf0cf,
+    0x03726861, 0x980182b5, 0xeee4bb88, 0x7597515c, 0x05ca1930,
+    0x9eb9f3e4, 0xe85ccad9, 0x732f200d, 0x0596b8a3, 0x9ee55277,
+    0xe8006b4a, 0x7373819e, 0x05735a16, 0x9e00b0c2, 0xe8e589ff,
+    0x7396632b, 0x052ffb85, 0x9e5c1151, 0xe8b9286c, 0x73cac2b8,
+    0x04b89f7c, 0x9fcb75a8, 0xe92e4c95, 0x725da641, 0x04e43eef,
+    0x9f97d43b, 0xe972ed06, 0x720107d2, 0x0401dc5a, 0x9f72368e,
+    0xe9970fb3, 0x72e4e567, 0x045d7dc9, 0x9f2e971d, 0xe9cbae20,
+    0x72b844f4, 0x072f15a8, 0x9c5cff7c, 0xeab9c641, 0x71ca2c95,
+    0x0773b43b, 0x9c005eef, 0xeae567d2, 0x71968d06, 0x0796568e,
+    0x9ce5bc5a, 0xea008567, 0x71736fb3, 0x07caf71d, 0x9cb91dc9,
+    0xea5c24f4, 0x712fce20, 0x065d93e4, 0x9d2e7930, 0xebcb400d,
+    0x70b8aad9, 0x06013277, 0x9d72d8a3, 0xeb97e19e, 0x70e40b4a,
+    0x06e4d0c2, 0x9d973a16, 0xeb72032b, 0x7001e9ff, 0x06b87151,
+    0x9dcb9b85, 0xeb2ea2b8, 0x705d486c, 0x0b943260, 0x90e7d8b4,
+    0xe602e189, 0x7d710b5d, 0x0bc893f3, 0x90bb7927, 0xe65e401a,
+    0x7d2daace, 0x0b2d7146, 0x905e9b92, 0xe6bba2af, 0x7dc8487b,
+    0x0b71d0d5, 0x90023a01, 0xe6e7033c, 0x7d94e9e8, 0x0ae6b42c,
+    0x91955ef8, 0xe77067c5, 0x7c038d11, 0x0aba15bf, 0x91c9ff6b,
+    0xe72cc656, 0x7c5f2c82, 0x0a5ff70a, 0x912c1dde, 0xe7c924e3,
+    0x7cbace37, 0x0a035699, 0x9170bc4d, 0xe7958570, 0x7ce66fa4,
+    0x09713ef8, 0x9202d42c, 0xe4e7ed11, 0x7f9407c5, 0x092d9f6b,
+    0x925e75bf, 0xe4bb4c82, 0x7fc8a656, 0x09c87dde, 0x92bb970a,
+    0xe45eae37, 0x7f2d44e3, 0x0994dc4d, 0x92e73699, 0xe4020fa4,
+    0x7f71e570, 0x0803b8b4, 0x93705260, 0xe5956b5d, 0x7ee68189,
+    0x085f1927, 0x932cf3f3, 0xe5c9cace, 0x7eba201a, 0x08bafb92,
+    0x93c91146, 0xe52c287b, 0x7e5fc2af, 0x08e65a01, 0x9395b0d5,
+    0xe57089e8, 0x7e03633c, 0x0e5e2b50, 0x952dc184, 0xe3c8f8b9,
+    0x78bb126d, 0x0e028ac3, 0x95716017, 0xe394592a, 0x78e7b3fe,
+    0x0ee76876, 0x959482a2, 0xe371bb9f, 0x7802514b, 0x0ebbc9e5,
+    0x95c82331, 0xe32d1a0c, 0x785ef0d8, 0x0f2cad1c, 0x945f47c8,
+    0xe2ba7ef5, 0x79c99421, 0x0f700c8f, 0x9403e65b, 0xe2e6df66,
+    0x799535b2, 0x0f95ee3a, 0x94e604ee, 0xe2033dd3, 0x7970d707,
+    0x0fc94fa9, 0x94baa57d, 0xe25f9c40, 0x792c7694, 0x0cbb27c8,
+    0x97c8cd1c, 0xe12df421, 0x7a5e1ef5, 0x0ce7865b, 0x97946c8f,
+    0xe17155b2, 0x7a02bf66, 0x0c0264ee, 0x97718e3a, 0xe194b707,
+    0x7ae75dd3, 0x0c5ec57d, 0x972d2fa9, 0xe1c81694, 0x7abbfc40,
+    0x0dc9a184, 0x96ba4b50, 0xe05f726d, 0x7b2c98b9, 0x0d950017,
+    0x96e6eac3, 0xe003d3fe, 0x7b70392a, 0x0d70e2a2, 0x96030876,
+    0xe0e6314b, 0x7b95db9f, 0x0d2c4331, 0x965fa9e5, 0xe0ba90d8,
+    0x7bc97a0c},
+   {0x00000000, 0x172864c0, 0x2e50c980, 0x3978ad40, 0x5ca19300,
+    0x4b89f7c0, 0x72f15a80, 0x65d93e40, 0xb9432600, 0xae6b42c0,
+    0x9713ef80, 0x803b8b40, 0xe5e2b500, 0xf2cad1c0, 0xcbb27c80,
+    0xdc9a1840, 0xa9f74a41, 0xbedf2e81, 0x87a783c1, 0x908fe701,
+    0xf556d941, 0xe27ebd81, 0xdb0610c1, 0xcc2e7401, 0x10b46c41,
+    0x079c0881, 0x3ee4a5c1, 0x29ccc101, 0x4c15ff41, 0x5b3d9b81,
+    0x624536c1, 0x756d5201, 0x889f92c3, 0x9fb7f603, 0xa6cf5b43,
+    0xb1e73f83, 0xd43e01c3, 0xc3166503, 0xfa6ec843, 0xed46ac83,
+    0x31dcb4c3, 0x26f4d003, 0x1f8c7d43, 0x08a41983, 0x6d7d27c3,
+    0x7a554303, 0x432dee43, 0x54058a83, 0x2168d882, 0x3640bc42,
+    0x0f381102, 0x181075c2, 0x7dc94b82, 0x6ae12f42, 0x53998202,
+    0x44b1e6c2, 0x982bfe82, 0x8f039a42, 0xb67b3702, 0xa15353c2,
+    0xc48a6d82, 0xd3a20942, 0xeadaa402, 0xfdf2c0c2, 0xca4e23c7,
+    0xdd664707, 0xe41eea47, 0xf3368e87, 0x96efb0c7, 0x81c7d407,
+    0xb8bf7947, 0xaf971d87, 0x730d05c7, 0x64256107, 0x5d5dcc47,
+    0x4a75a887, 0x2fac96c7, 0x3884f207, 0x01fc5f47, 0x16d43b87,
+    0x63b96986, 0x74910d46, 0x4de9a006, 0x5ac1c4c6, 0x3f18fa86,
+    0x28309e46, 0x11483306, 0x066057c6, 0xdafa4f86, 0xcdd22b46,
+    0xf4aa8606, 0xe382e2c6, 0x865bdc86, 0x9173b846, 0xa80b1506,
+    0xbf2371c6, 0x42d1b104, 0x55f9d5c4, 0x6c817884, 0x7ba91c44,
+    0x1e702204, 0x095846c4, 0x3020eb84, 0x27088f44, 0xfb929704,
+    0xecbaf3c4, 0xd5c25e84, 0xc2ea3a44, 0xa7330404, 0xb01b60c4,
+    0x8963cd84, 0x9e4ba944, 0xeb26fb45, 0xfc0e9f85, 0xc57632c5,
+    0xd25e5605, 0xb7876845, 0xa0af0c85, 0x99d7a1c5, 0x8effc505,
+    0x5265dd45, 0x454db985, 0x7c3514c5, 0x6b1d7005, 0x0ec44e45,
+    0x19ec2a85, 0x209487c5, 0x37bce305, 0x4fed41cf, 0x58c5250f,
+    0x61bd884f, 0x7695ec8f, 0x134cd2cf, 0x0464b60f, 0x3d1c1b4f,
+    0x2a347f8f, 0xf6ae67cf, 0xe186030f, 0xd8feae4f, 0xcfd6ca8f,
+    0xaa0ff4cf, 0xbd27900f, 0x845f3d4f, 0x9377598f, 0xe61a0b8e,
+    0xf1326f4e, 0xc84ac20e, 0xdf62a6ce, 0xbabb988e, 0xad93fc4e,
+    0x94eb510e, 0x83c335ce, 0x5f592d8e, 0x4871494e, 0x7109e40e,
+    0x662180ce, 0x03f8be8e, 0x14d0da4e, 0x2da8770e, 0x3a8013ce,
+    0xc772d30c, 0xd05ab7cc, 0xe9221a8c, 0xfe0a7e4c, 0x9bd3400c,
+    0x8cfb24cc, 0xb583898c, 0xa2abed4c, 0x7e31f50c, 0x691991cc,
+    0x50613c8c, 0x4749584c, 0x2290660c, 0x35b802cc, 0x0cc0af8c,
+    0x1be8cb4c, 0x6e85994d, 0x79adfd8d, 0x40d550cd, 0x57fd340d,
+    0x32240a4d, 0x250c6e8d, 0x1c74c3cd, 0x0b5ca70d, 0xd7c6bf4d,
+    0xc0eedb8d, 0xf99676cd, 0xeebe120d, 0x8b672c4d, 0x9c4f488d,
+    0xa537e5cd, 0xb21f810d, 0x85a36208, 0x928b06c8, 0xabf3ab88,
+    0xbcdbcf48, 0xd902f108, 0xce2a95c8, 0xf7523888, 0xe07a5c48,
+    0x3ce04408, 0x2bc820c8, 0x12b08d88, 0x0598e948, 0x6041d708,
+    0x7769b3c8, 0x4e111e88, 0x59397a48, 0x2c542849, 0x3b7c4c89,
+    0x0204e1c9, 0x152c8509, 0x70f5bb49, 0x67dddf89, 0x5ea572c9,
+    0x498d1609, 0x95170e49, 0x823f6a89, 0xbb47c7c9, 0xac6fa309,
+    0xc9b69d49, 0xde9ef989, 0xe7e654c9, 0xf0ce3009, 0x0d3cf0cb,
+    0x1a14940b, 0x236c394b, 0x34445d8b, 0x519d63cb, 0x46b5070b,
+    0x7fcdaa4b, 0x68e5ce8b, 0xb47fd6cb, 0xa357b20b, 0x9a2f1f4b,
+    0x8d077b8b, 0xe8de45cb, 0xfff6210b, 0xc68e8c4b, 0xd1a6e88b,
+    0xa4cbba8a, 0xb3e3de4a, 0x8a9b730a, 0x9db317ca, 0xf86a298a,
+    0xef424d4a, 0xd63ae00a, 0xc11284ca, 0x1d889c8a, 0x0aa0f84a,
+    0x33d8550a, 0x24f031ca, 0x41290f8a, 0x56016b4a, 0x6f79c60a,
+    0x7851a2ca},
+   {0x00000000, 0x9fda839e, 0xe4c4017d, 0x7b1e82e3, 0x12f904bb,
+    0x8d238725, 0xf63d05c6, 0x69e78658, 0x25f20976, 0xba288ae8,
+    0xc136080b, 0x5eec8b95, 0x370b0dcd, 0xa8d18e53, 0xd3cf0cb0,
+    0x4c158f2e, 0x4be412ec, 0xd43e9172, 0xaf201391, 0x30fa900f,
+    0x591d1657, 0xc6c795c9, 0xbdd9172a, 0x220394b4, 0x6e161b9a,
+    0xf1cc9804, 0x8ad21ae7, 0x15089979, 0x7cef1f21, 0xe3359cbf,
+    0x982b1e5c, 0x07f19dc2, 0x97c825d8, 0x0812a646, 0x730c24a5,
+    0xecd6a73b, 0x85312163, 0x1aeba2fd, 0x61f5201e, 0xfe2fa380,
+    0xb23a2cae, 0x2de0af30, 0x56fe2dd3, 0xc924ae4d, 0xa0c32815,
+    0x3f19ab8b, 0x44072968, 0xdbddaaf6, 0xdc2c3734, 0x43f6b4aa,
+    0x38e83649, 0xa732b5d7, 0xced5338f, 0x510fb011, 0x2a1132f2,
+    0xb5cbb16c, 0xf9de3e42, 0x6604bddc, 0x1d1a3f3f, 0x82c0bca1,
+    0xeb273af9, 0x74fdb967, 0x0fe33b84, 0x9039b81a, 0xf4e14df1,
+    0x6b3bce6f, 0x10254c8c, 0x8fffcf12, 0xe618494a, 0x79c2cad4,
+    0x02dc4837, 0x9d06cba9, 0xd1134487, 0x4ec9c719, 0x35d745fa,
+    0xaa0dc664, 0xc3ea403c, 0x5c30c3a2, 0x272e4141, 0xb8f4c2df,
+    0xbf055f1d, 0x20dfdc83, 0x5bc15e60, 0xc41bddfe, 0xadfc5ba6,
+    0x3226d838, 0x49385adb, 0xd6e2d945, 0x9af7566b, 0x052dd5f5,
+    0x7e335716, 0xe1e9d488, 0x880e52d0, 0x17d4d14e, 0x6cca53ad,
+    0xf310d033, 0x63296829, 0xfcf3ebb7, 0x87ed6954, 0x1837eaca,
+    0x71d06c92, 0xee0aef0c, 0x95146def, 0x0aceee71, 0x46db615f,
+    0xd901e2c1, 0xa21f6022, 0x3dc5e3bc, 0x542265e4, 0xcbf8e67a,
+    0xb0e66499, 0x2f3ce707, 0x28cd7ac5, 0xb717f95b, 0xcc097bb8,
+    0x53d3f826, 0x3a347e7e, 0xa5eefde0, 0xdef07f03, 0x412afc9d,
+    0x0d3f73b3, 0x92e5f02d, 0xe9fb72ce, 0x7621f150, 0x1fc67708,
+    0x801cf496, 0xfb027675, 0x64d8f5eb, 0x32b39da3, 0xad691e3d,
+    0xd6779cde, 0x49ad1f40, 0x204a9918, 0xbf901a86, 0xc48e9865,
+    0x5b541bfb, 0x174194d5, 0x889b174b, 0xf38595a8, 0x6c5f1636,
+    0x05b8906e, 0x9a6213f0, 0xe17c9113, 0x7ea6128d, 0x79578f4f,
+    0xe68d0cd1, 0x9d938e32, 0x02490dac, 0x6bae8bf4, 0xf474086a,
+    0x8f6a8a89, 0x10b00917, 0x5ca58639, 0xc37f05a7, 0xb8618744,
+    0x27bb04da, 0x4e5c8282, 0xd186011c, 0xaa9883ff, 0x35420061,
+    0xa57bb87b, 0x3aa13be5, 0x41bfb906, 0xde653a98, 0xb782bcc0,
+    0x28583f5e, 0x5346bdbd, 0xcc9c3e23, 0x8089b10d, 0x1f533293,
+    0x644db070, 0xfb9733ee, 0x9270b5b6, 0x0daa3628, 0x76b4b4cb,
+    0xe96e3755, 0xee9faa97, 0x71452909, 0x0a5babea, 0x95812874,
+    0xfc66ae2c, 0x63bc2db2, 0x18a2af51, 0x87782ccf, 0xcb6da3e1,
+    0x54b7207f, 0x2fa9a29c, 0xb0732102, 0xd994a75a, 0x464e24c4,
+    0x3d50a627, 0xa28a25b9, 0xc652d052, 0x598853cc, 0x2296d12f,
+    0xbd4c52b1, 0xd4abd4e9, 0x4b715777, 0x306fd594, 0xafb5560a,
+    0xe3a0d924, 0x7c7a5aba, 0x0764d859, 0x98be5bc7, 0xf159dd9f,
+    0x6e835e01, 0x159ddce2, 0x8a475f7c, 0x8db6c2be, 0x126c4120,
+    0x6972c3c3, 0xf6a8405d, 0x9f4fc605, 0x0095459b, 0x7b8bc778,
+    0xe45144e6, 0xa844cbc8, 0x379e4856, 0x4c80cab5, 0xd35a492b,
+    0xbabdcf73, 0x25674ced, 0x5e79ce0e, 0xc1a34d90, 0x519af58a,
+    0xce407614, 0xb55ef4f7, 0x2a847769, 0x4363f131, 0xdcb972af,
+    0xa7a7f04c, 0x387d73d2, 0x7468fcfc, 0xebb27f62, 0x90acfd81,
+    0x0f767e1f, 0x6691f847, 0xf94b7bd9, 0x8255f93a, 0x1d8f7aa4,
+    0x1a7ee766, 0x85a464f8, 0xfebae61b, 0x61606585, 0x0887e3dd,
+    0x975d6043, 0xec43e2a0, 0x7399613e, 0x3f8cee10, 0xa0566d8e,
+    0xdb48ef6d, 0x44926cf3, 0x2d75eaab, 0xb2af6935, 0xc9b1ebd6,
+    0x566b6848}};
+
+local const z_word_t FAR crc_braid_big_table[][256] = {
+   {0x00000000, 0x9e83da9f, 0x7d01c4e4, 0xe3821e7b, 0xbb04f912,
+    0x2587238d, 0xc6053df6, 0x5886e769, 0x7609f225, 0xe88a28ba,
+    0x0b0836c1, 0x958bec5e, 0xcd0d0b37, 0x538ed1a8, 0xb00ccfd3,
+    0x2e8f154c, 0xec12e44b, 0x72913ed4, 0x911320af, 0x0f90fa30,
+    0x57161d59, 0xc995c7c6, 0x2a17d9bd, 0xb4940322, 0x9a1b166e,
+    0x0498ccf1, 0xe71ad28a, 0x79990815, 0x211fef7c, 0xbf9c35e3,
+    0x5c1e2b98, 0xc29df107, 0xd825c897, 0x46a61208, 0xa5240c73,
+    0x3ba7d6ec, 0x63213185, 0xfda2eb1a, 0x1e20f561, 0x80a32ffe,
+    0xae2c3ab2, 0x30afe02d, 0xd32dfe56, 0x4dae24c9, 0x1528c3a0,
+    0x8bab193f, 0x68290744, 0xf6aadddb, 0x34372cdc, 0xaab4f643,
+    0x4936e838, 0xd7b532a7, 0x8f33d5ce, 0x11b00f51, 0xf232112a,
+    0x6cb1cbb5, 0x423edef9, 0xdcbd0466, 0x3f3f1a1d, 0xa1bcc082,
+    0xf93a27eb, 0x67b9fd74, 0x843be30f, 0x1ab83990, 0xf14de1f4,
+    0x6fce3b6b, 0x8c4c2510, 0x12cfff8f, 0x4a4918e6, 0xd4cac279,
+    0x3748dc02, 0xa9cb069d, 0x874413d1, 0x19c7c94e, 0xfa45d735,
+    0x64c60daa, 0x3c40eac3, 0xa2c3305c, 0x41412e27, 0xdfc2f4b8,
+    0x1d5f05bf, 0x83dcdf20, 0x605ec15b, 0xfedd1bc4, 0xa65bfcad,
+    0x38d82632, 0xdb5a3849, 0x45d9e2d6, 0x6b56f79a, 0xf5d52d05,
+    0x1657337e, 0x88d4e9e1, 0xd0520e88, 0x4ed1d417, 0xad53ca6c,
+    0x33d010f3, 0x29682963, 0xb7ebf3fc, 0x5469ed87, 0xcaea3718,
+    0x926cd071, 0x0cef0aee, 0xef6d1495, 0x71eece0a, 0x5f61db46,
+    0xc1e201d9, 0x22601fa2, 0xbce3c53d, 0xe4652254, 0x7ae6f8cb,
+    0x9964e6b0, 0x07e73c2f, 0xc57acd28, 0x5bf917b7, 0xb87b09cc,
+    0x26f8d353, 0x7e7e343a, 0xe0fdeea5, 0x037ff0de, 0x9dfc2a41,
+    0xb3733f0d, 0x2df0e592, 0xce72fbe9, 0x50f12176, 0x0877c61f,
+    0x96f41c80, 0x757602fb, 0xebf5d864, 0xa39db332, 0x3d1e69ad,
+    0xde9c77d6, 0x401fad49, 0x18994a20, 0x861a90bf, 0x65988ec4,
+    0xfb1b545b, 0xd5944117, 0x4b179b88, 0xa89585f3, 0x36165f6c,
+    0x6e90b805, 0xf013629a, 0x13917ce1, 0x8d12a67e, 0x4f8f5779,
+    0xd10c8de6, 0x328e939d, 0xac0d4902, 0xf48bae6b, 0x6a0874f4,
+    0x898a6a8f, 0x1709b010, 0x3986a55c, 0xa7057fc3, 0x448761b8,
+    0xda04bb27, 0x82825c4e, 0x1c0186d1, 0xff8398aa, 0x61004235,
+    0x7bb87ba5, 0xe53ba13a, 0x06b9bf41, 0x983a65de, 0xc0bc82b7,
+    0x5e3f5828, 0xbdbd4653, 0x233e9ccc, 0x0db18980, 0x9332531f,
+    0x70b04d64, 0xee3397fb, 0xb6b57092, 0x2836aa0d, 0xcbb4b476,
+    0x55376ee9, 0x97aa9fee, 0x09294571, 0xeaab5b0a, 0x74288195,
+    0x2cae66fc, 0xb22dbc63, 0x51afa218, 0xcf2c7887, 0xe1a36dcb,
+    0x7f20b754, 0x9ca2a92f, 0x022173b0, 0x5aa794d9, 0xc4244e46,
+    0x27a6503d, 0xb9258aa2, 0x52d052c6, 0xcc538859, 0x2fd19622,
+    0xb1524cbd, 0xe9d4abd4, 0x7757714b, 0x94d56f30, 0x0a56b5af,
+    0x24d9a0e3, 0xba5a7a7c, 0x59d86407, 0xc75bbe98, 0x9fdd59f1,
+    0x015e836e, 0xe2dc9d15, 0x7c5f478a, 0xbec2b68d, 0x20416c12,
+    0xc3c37269, 0x5d40a8f6, 0x05c64f9f, 0x9b459500, 0x78c78b7b,
+    0xe64451e4, 0xc8cb44a8, 0x56489e37, 0xb5ca804c, 0x2b495ad3,
+    0x73cfbdba, 0xed4c6725, 0x0ece795e, 0x904da3c1, 0x8af59a51,
+    0x147640ce, 0xf7f45eb5, 0x6977842a, 0x31f16343, 0xaf72b9dc,
+    0x4cf0a7a7, 0xd2737d38, 0xfcfc6874, 0x627fb2eb, 0x81fdac90,
+    0x1f7e760f, 0x47f89166, 0xd97b4bf9, 0x3af95582, 0xa47a8f1d,
+    0x66e77e1a, 0xf864a485, 0x1be6bafe, 0x85656061, 0xdde38708,
+    0x43605d97, 0xa0e243ec, 0x3e619973, 0x10ee8c3f, 0x8e6d56a0,
+    0x6def48db, 0xf36c9244, 0xabea752d, 0x3569afb2, 0xd6ebb1c9,
+    0x48686b56},
+   {0x00000000, 0xc0642817, 0x80c9502e, 0x40ad7839, 0x0093a15c,
+    0xc0f7894b, 0x805af172, 0x403ed965, 0x002643b9, 0xc0426bae,
+    0x80ef1397, 0x408b3b80, 0x00b5e2e5, 0xc0d1caf2, 0x807cb2cb,
+    0x40189adc, 0x414af7a9, 0x812edfbe, 0xc183a787, 0x01e78f90,
+    0x41d956f5, 0x81bd7ee2, 0xc11006db, 0x01742ecc, 0x416cb410,
+    0x81089c07, 0xc1a5e43e, 0x01c1cc29, 0x41ff154c, 0x819b3d5b,
+    0xc1364562, 0x01526d75, 0xc3929f88, 0x03f6b79f, 0x435bcfa6,
+    0x833fe7b1, 0xc3013ed4, 0x036516c3, 0x43c86efa, 0x83ac46ed,
+    0xc3b4dc31, 0x03d0f426, 0x437d8c1f, 0x8319a408, 0xc3277d6d,
+    0x0343557a, 0x43ee2d43, 0x838a0554, 0x82d86821, 0x42bc4036,
+    0x0211380f, 0xc2751018, 0x824bc97d, 0x422fe16a, 0x02829953,
+    0xc2e6b144, 0x82fe2b98, 0x429a038f, 0x02377bb6, 0xc25353a1,
+    0x826d8ac4, 0x4209a2d3, 0x02a4daea, 0xc2c0f2fd, 0xc7234eca,
+    0x074766dd, 0x47ea1ee4, 0x878e36f3, 0xc7b0ef96, 0x07d4c781,
+    0x4779bfb8, 0x871d97af, 0xc7050d73, 0x07612564, 0x47cc5d5d,
+    0x87a8754a, 0xc796ac2f, 0x07f28438, 0x475ffc01, 0x873bd416,
+    0x8669b963, 0x460d9174, 0x06a0e94d, 0xc6c4c15a, 0x86fa183f,
+    0x469e3028, 0x06334811, 0xc6576006, 0x864ffada, 0x462bd2cd,
+    0x0686aaf4, 0xc6e282e3, 0x86dc5b86, 0x46b87391, 0x06150ba8,
+    0xc67123bf, 0x04b1d142, 0xc4d5f955, 0x8478816c, 0x441ca97b,
+    0x0422701e, 0xc4465809, 0x84eb2030, 0x448f0827, 0x049792fb,
+    0xc4f3baec, 0x845ec2d5, 0x443aeac2, 0x040433a7, 0xc4601bb0,
+    0x84cd6389, 0x44a94b9e, 0x45fb26eb, 0x859f0efc, 0xc53276c5,
+    0x05565ed2, 0x456887b7, 0x850cafa0, 0xc5a1d799, 0x05c5ff8e,
+    0x45dd6552, 0x85b94d45, 0xc514357c, 0x05701d6b, 0x454ec40e,
+    0x852aec19, 0xc5879420, 0x05e3bc37, 0xcf41ed4f, 0x0f25c558,
+    0x4f88bd61, 0x8fec9576, 0xcfd24c13, 0x0fb66404, 0x4f1b1c3d,
+    0x8f7f342a, 0xcf67aef6, 0x0f0386e1, 0x4faefed8, 0x8fcad6cf,
+    0xcff40faa, 0x0f9027bd, 0x4f3d5f84, 0x8f597793, 0x8e0b1ae6,
+    0x4e6f32f1, 0x0ec24ac8, 0xcea662df, 0x8e98bbba, 0x4efc93ad,
+    0x0e51eb94, 0xce35c383, 0x8e2d595f, 0x4e497148, 0x0ee40971,
+    0xce802166, 0x8ebef803, 0x4edad014, 0x0e77a82d, 0xce13803a,
+    0x0cd372c7, 0xccb75ad0, 0x8c1a22e9, 0x4c7e0afe, 0x0c40d39b,
+    0xcc24fb8c, 0x8c8983b5, 0x4cedaba2, 0x0cf5317e, 0xcc911969,
+    0x8c3c6150, 0x4c584947, 0x0c669022, 0xcc02b835, 0x8cafc00c,
+    0x4ccbe81b, 0x4d99856e, 0x8dfdad79, 0xcd50d540, 0x0d34fd57,
+    0x4d0a2432, 0x8d6e0c25, 0xcdc3741c, 0x0da75c0b, 0x4dbfc6d7,
+    0x8ddbeec0, 0xcd7696f9, 0x0d12beee, 0x4d2c678b, 0x8d484f9c,
+    0xcde537a5, 0x0d811fb2, 0x0862a385, 0xc8068b92, 0x88abf3ab,
+    0x48cfdbbc, 0x08f102d9, 0xc8952ace, 0x883852f7, 0x485c7ae0,
+    0x0844e03c, 0xc820c82b, 0x888db012, 0x48e99805, 0x08d74160,
+    0xc8b36977, 0x881e114e, 0x487a3959, 0x4928542c, 0x894c7c3b,
+    0xc9e10402, 0x09852c15, 0x49bbf570, 0x89dfdd67, 0xc972a55e,
+    0x09168d49, 0x490e1795, 0x896a3f82, 0xc9c747bb, 0x09a36fac,
+    0x499db6c9, 0x89f99ede, 0xc954e6e7, 0x0930cef0, 0xcbf03c0d,
+    0x0b94141a, 0x4b396c23, 0x8b5d4434, 0xcb639d51, 0x0b07b546,
+    0x4baacd7f, 0x8bcee568, 0xcbd67fb4, 0x0bb257a3, 0x4b1f2f9a,
+    0x8b7b078d, 0xcb45dee8, 0x0b21f6ff, 0x4b8c8ec6, 0x8be8a6d1,
+    0x8abacba4, 0x4adee3b3, 0x0a739b8a, 0xca17b39d, 0x8a296af8,
+    0x4a4d42ef, 0x0ae03ad6, 0xca8412c1, 0x8a9c881d, 0x4af8a00a,
+    0x0a55d833, 0xca31f024, 0x8a0f2941, 0x4a6b0156, 0x0ac6796f,
+    0xcaa25178},
+   {0x00000000, 0xd4ea739b, 0xe9d396ed, 0x3d39e576, 0x93a15c00,
+    0x474b2f9b, 0x7a72caed, 0xae98b976, 0x2643b900, 0xf2a9ca9b,
+    0xcf902fed, 0x1b7a5c76, 0xb5e2e500, 0x6108969b, 0x5c3173ed,
+    0x88db0076, 0x4c867201, 0x986c019a, 0xa555e4ec, 0x71bf9777,
+    0xdf272e01, 0x0bcd5d9a, 0x36f4b8ec, 0xe21ecb77, 0x6ac5cb01,
+    0xbe2fb89a, 0x83165dec, 0x57fc2e77, 0xf9649701, 0x2d8ee49a,
+    0x10b701ec, 0xc45d7277, 0x980ce502, 0x4ce69699, 0x71df73ef,
+    0xa5350074, 0x0badb902, 0xdf47ca99, 0xe27e2fef, 0x36945c74,
+    0xbe4f5c02, 0x6aa52f99, 0x579ccaef, 0x8376b974, 0x2dee0002,
+    0xf9047399, 0xc43d96ef, 0x10d7e574, 0xd48a9703, 0x0060e498,
+    0x3d5901ee, 0xe9b37275, 0x472bcb03, 0x93c1b898, 0xaef85dee,
+    0x7a122e75, 0xf2c92e03, 0x26235d98, 0x1b1ab8ee, 0xcff0cb75,
+    0x61687203, 0xb5820198, 0x88bbe4ee, 0x5c519775, 0x3019ca05,
+    0xe4f3b99e, 0xd9ca5ce8, 0x0d202f73, 0xa3b89605, 0x7752e59e,
+    0x4a6b00e8, 0x9e817373, 0x165a7305, 0xc2b0009e, 0xff89e5e8,
+    0x2b639673, 0x85fb2f05, 0x51115c9e, 0x6c28b9e8, 0xb8c2ca73,
+    0x7c9fb804, 0xa875cb9f, 0x954c2ee9, 0x41a65d72, 0xef3ee404,
+    0x3bd4979f, 0x06ed72e9, 0xd2070172, 0x5adc0104, 0x8e36729f,
+    0xb30f97e9, 0x67e5e472, 0xc97d5d04, 0x1d972e9f, 0x20aecbe9,
+    0xf444b872, 0xa8152f07, 0x7cff5c9c, 0x41c6b9ea, 0x952cca71,
+    0x3bb47307, 0xef5e009c, 0xd267e5ea, 0x068d9671, 0x8e569607,
+    0x5abce59c, 0x678500ea, 0xb36f7371, 0x1df7ca07, 0xc91db99c,
+    0xf4245cea, 0x20ce2f71, 0xe4935d06, 0x30792e9d, 0x0d40cbeb,
+    0xd9aab870, 0x77320106, 0xa3d8729d, 0x9ee197eb, 0x4a0be470,
+    0xc2d0e406, 0x163a979d, 0x2b0372eb, 0xffe90170, 0x5171b806,
+    0x859bcb9d, 0xb8a22eeb, 0x6c485d70, 0x6032940b, 0xb4d8e790,
+    0x89e102e6, 0x5d0b717d, 0xf393c80b, 0x2779bb90, 0x1a405ee6,
+    0xceaa2d7d, 0x46712d0b, 0x929b5e90, 0xafa2bbe6, 0x7b48c87d,
+    0xd5d0710b, 0x013a0290, 0x3c03e7e6, 0xe8e9947d, 0x2cb4e60a,
+    0xf85e9591, 0xc56770e7, 0x118d037c, 0xbf15ba0a, 0x6bffc991,
+    0x56c62ce7, 0x822c5f7c, 0x0af75f0a, 0xde1d2c91, 0xe324c9e7,
+    0x37ceba7c, 0x9956030a, 0x4dbc7091, 0x708595e7, 0xa46fe67c,
+    0xf83e7109, 0x2cd40292, 0x11ede7e4, 0xc507947f, 0x6b9f2d09,
+    0xbf755e92, 0x824cbbe4, 0x56a6c87f, 0xde7dc809, 0x0a97bb92,
+    0x37ae5ee4, 0xe3442d7f, 0x4ddc9409, 0x9936e792, 0xa40f02e4,
+    0x70e5717f, 0xb4b80308, 0x60527093, 0x5d6b95e5, 0x8981e67e,
+    0x27195f08, 0xf3f32c93, 0xcecac9e5, 0x1a20ba7e, 0x92fbba08,
+    0x4611c993, 0x7b282ce5, 0xafc25f7e, 0x015ae608, 0xd5b09593,
+    0xe88970e5, 0x3c63037e, 0x502b5e0e, 0x84c12d95, 0xb9f8c8e3,
+    0x6d12bb78, 0xc38a020e, 0x17607195, 0x2a5994e3, 0xfeb3e778,
+    0x7668e70e, 0xa2829495, 0x9fbb71e3, 0x4b510278, 0xe5c9bb0e,
+    0x3123c895, 0x0c1a2de3, 0xd8f05e78, 0x1cad2c0f, 0xc8475f94,
+    0xf57ebae2, 0x2194c979, 0x8f0c700f, 0x5be60394, 0x66dfe6e2,
+    0xb2359579, 0x3aee950f, 0xee04e694, 0xd33d03e2, 0x07d77079,
+    0xa94fc90f, 0x7da5ba94, 0x409c5fe2, 0x94762c79, 0xc827bb0c,
+    0x1ccdc897, 0x21f42de1, 0xf51e5e7a, 0x5b86e70c, 0x8f6c9497,
+    0xb25571e1, 0x66bf027a, 0xee64020c, 0x3a8e7197, 0x07b794e1,
+    0xd35de77a, 0x7dc55e0c, 0xa92f2d97, 0x9416c8e1, 0x40fcbb7a,
+    0x84a1c90d, 0x504bba96, 0x6d725fe0, 0xb9982c7b, 0x1700950d,
+    0xc3eae696, 0xfed303e0, 0x2a39707b, 0xa2e2700d, 0x76080396,
+    0x4b31e6e0, 0x9fdb957b, 0x31432c0d, 0xe5a95f96, 0xd890bae0,
+    0x0c7ac97b},
+   {0x00000000, 0x27652581, 0x0fcc3bd9, 0x28a91e58, 0x5f9e0669,
+    0x78fb23e8, 0x50523db0, 0x77371831, 0xbe3c0dd2, 0x99592853,
+    0xb1f0360b, 0x9695138a, 0xe1a20bbb, 0xc6c72e3a, 0xee6e3062,
+    0xc90b15e3, 0x3d7f6b7f, 0x1a1a4efe, 0x32b350a6, 0x15d67527,
+    0x62e16d16, 0x45844897, 0x6d2d56cf, 0x4a48734e, 0x834366ad,
+    0xa426432c, 0x8c8f5d74, 0xabea78f5, 0xdcdd60c4, 0xfbb84545,
+    0xd3115b1d, 0xf4747e9c, 0x7afed6fe, 0x5d9bf37f, 0x7532ed27,
+    0x5257c8a6, 0x2560d097, 0x0205f516, 0x2aaceb4e, 0x0dc9cecf,
+    0xc4c2db2c, 0xe3a7fead, 0xcb0ee0f5, 0xec6bc574, 0x9b5cdd45,
+    0xbc39f8c4, 0x9490e69c, 0xb3f5c31d, 0x4781bd81, 0x60e49800,
+    0x484d8658, 0x6f28a3d9, 0x181fbbe8, 0x3f7a9e69, 0x17d38031,
+    0x30b6a5b0, 0xf9bdb053, 0xded895d2, 0xf6718b8a, 0xd114ae0b,
+    0xa623b63a, 0x814693bb, 0xa9ef8de3, 0x8e8aa862, 0xb5fadc26,
+    0x929ff9a7, 0xba36e7ff, 0x9d53c27e, 0xea64da4f, 0xcd01ffce,
+    0xe5a8e196, 0xc2cdc417, 0x0bc6d1f4, 0x2ca3f475, 0x040aea2d,
+    0x236fcfac, 0x5458d79d, 0x733df21c, 0x5b94ec44, 0x7cf1c9c5,
+    0x8885b759, 0xafe092d8, 0x87498c80, 0xa02ca901, 0xd71bb130,
+    0xf07e94b1, 0xd8d78ae9, 0xffb2af68, 0x36b9ba8b, 0x11dc9f0a,
+    0x39758152, 0x1e10a4d3, 0x6927bce2, 0x4e429963, 0x66eb873b,
+    0x418ea2ba, 0xcf040ad8, 0xe8612f59, 0xc0c83101, 0xe7ad1480,
+    0x909a0cb1, 0xb7ff2930, 0x9f563768, 0xb83312e9, 0x7138070a,
+    0x565d228b, 0x7ef43cd3, 0x59911952, 0x2ea60163, 0x09c324e2,
+    0x216a3aba, 0x060f1f3b, 0xf27b61a7, 0xd51e4426, 0xfdb75a7e,
+    0xdad27fff, 0xade567ce, 0x8a80424f, 0xa2295c17, 0x854c7996,
+    0x4c476c75, 0x6b2249f4, 0x438b57ac, 0x64ee722d, 0x13d96a1c,
+    0x34bc4f9d, 0x1c1551c5, 0x3b707444, 0x6af5b94d, 0x4d909ccc,
+    0x65398294, 0x425ca715, 0x356bbf24, 0x120e9aa5, 0x3aa784fd,
+    0x1dc2a17c, 0xd4c9b49f, 0xf3ac911e, 0xdb058f46, 0xfc60aac7,
+    0x8b57b2f6, 0xac329777, 0x849b892f, 0xa3feacae, 0x578ad232,
+    0x70eff7b3, 0x5846e9eb, 0x7f23cc6a, 0x0814d45b, 0x2f71f1da,
+    0x07d8ef82, 0x20bdca03, 0xe9b6dfe0, 0xced3fa61, 0xe67ae439,
+    0xc11fc1b8, 0xb628d989, 0x914dfc08, 0xb9e4e250, 0x9e81c7d1,
+    0x100b6fb3, 0x376e4a32, 0x1fc7546a, 0x38a271eb, 0x4f9569da,
+    0x68f04c5b, 0x40595203, 0x673c7782, 0xae376261, 0x895247e0,
+    0xa1fb59b8, 0x869e7c39, 0xf1a96408, 0xd6cc4189, 0xfe655fd1,
+    0xd9007a50, 0x2d7404cc, 0x0a11214d, 0x22b83f15, 0x05dd1a94,
+    0x72ea02a5, 0x558f2724, 0x7d26397c, 0x5a431cfd, 0x9348091e,
+    0xb42d2c9f, 0x9c8432c7, 0xbbe11746, 0xccd60f77, 0xebb32af6,
+    0xc31a34ae, 0xe47f112f, 0xdf0f656b, 0xf86a40ea, 0xd0c35eb2,
+    0xf7a67b33, 0x80916302, 0xa7f44683, 0x8f5d58db, 0xa8387d5a,
+    0x613368b9, 0x46564d38, 0x6eff5360, 0x499a76e1, 0x3ead6ed0,
+    0x19c84b51, 0x31615509, 0x16047088, 0xe2700e14, 0xc5152b95,
+    0xedbc35cd, 0xcad9104c, 0xbdee087d, 0x9a8b2dfc, 0xb22233a4,
+    0x95471625, 0x5c4c03c6, 0x7b292647, 0x5380381f, 0x74e51d9e,
+    0x03d205af, 0x24b7202e, 0x0c1e3e76, 0x2b7b1bf7, 0xa5f1b395,
+    0x82949614, 0xaa3d884c, 0x8d58adcd, 0xfa6fb5fc, 0xdd0a907d,
+    0xf5a38e25, 0xd2c6aba4, 0x1bcdbe47, 0x3ca89bc6, 0x1401859e,
+    0x3364a01f, 0x4453b82e, 0x63369daf, 0x4b9f83f7, 0x6cfaa676,
+    0x988ed8ea, 0xbfebfd6b, 0x9742e333, 0xb027c6b2, 0xc710de83,
+    0xe075fb02, 0xc8dce55a, 0xefb9c0db, 0x26b2d538, 0x01d7f0b9,
+    0x297eeee1, 0x0e1bcb60, 0x792cd351, 0x5e49f6d0, 0x76e0e888,
+    0x5185cd09}};
+
+#endif
+
+#endif
+
+#endif
+
+local const z_crc_t FAR x2n_table[] = {
+    0x40000000, 0x20000000, 0x08000000, 0x00800000, 0x00008000,
+    0xedb88320, 0xb1e6b092, 0xa06a2517, 0xed627dae, 0x88d14467,
+    0xd7bbfe6a, 0xec447f11, 0x8e7ea170, 0x6427800e, 0x4d47bae0,
+    0x09fe548f, 0x83852d0f, 0x30362f1a, 0x7b5a9cc3, 0x31fec169,
+    0x9fec022a, 0x6c8dedc4, 0x15d6874d, 0x5fde7a4e, 0xbad90e37,
+    0x2e4e5eef, 0x4eaba214, 0xa8a472c0, 0x429a969e, 0x148d302a,
+    0xc40ba6d0, 0xc4e22c3c};
diff --git a/zlib/deflate.c b/zlib/deflate.c
index 586cd15..012ea81 100644
--- a/zlib/deflate.c
+++ b/zlib/deflate.c
@@ -1,5 +1,5 @@
 /* deflate.c -- compress data using the deflation algorithm
- * Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler
+ * Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -47,12 +47,12 @@
  *
  */
 
-/* @(#) $Id: deflate.c,v 1.1.1.2 2002/03/11 21:53:23 tromey Exp $ */
+/* @(#) $Id$ */
 
 #include "deflate.h"
 
 const char deflate_copyright[] =
-   " deflate 1.2.11 Copyright 1995-2017 Jean-loup Gailly and Mark Adler ";
+   " deflate 1.3.1 Copyright 1995-2024 Jean-loup Gailly and Mark Adler ";
 /*
   If you use the zlib library in a product, an acknowledgment is welcome
   in the documentation of your product. If for some reason you cannot
@@ -60,9 +60,6 @@ const char deflate_copyright[] =
   copyright string in the executable of your product.
  */
 
-/* ===========================================================================
- *  Function prototypes.
- */
 typedef enum {
     need_more,      /* block not completed, need more input or more output */
     block_done,     /* block flush performed */
@@ -70,35 +67,16 @@ typedef enum {
     finish_done     /* finish done, accept no more input or output */
 } block_state;
 
-typedef block_state (*compress_func) OF((deflate_state *s, int flush));
+typedef block_state (*compress_func)(deflate_state *s, int flush);
 /* Compression function. Returns the block state after the call. */
 
-local int deflateStateCheck      OF((z_streamp strm));
-local void slide_hash     OF((deflate_state *s));
-local void fill_window    OF((deflate_state *s));
-local block_state deflate_stored OF((deflate_state *s, int flush));
-local block_state deflate_fast   OF((deflate_state *s, int flush));
+local block_state deflate_stored(deflate_state *s, int flush);
+local block_state deflate_fast(deflate_state *s, int flush);
 #ifndef FASTEST
-local block_state deflate_slow   OF((deflate_state *s, int flush));
-#endif
-local block_state deflate_rle    OF((deflate_state *s, int flush));
-local block_state deflate_huff   OF((deflate_state *s, int flush));
-local void lm_init        OF((deflate_state *s));
-local void putShortMSB    OF((deflate_state *s, uInt b));
-local void flush_pending  OF((z_streamp strm));
-local unsigned read_buf   OF((z_streamp strm, Bytef *buf, unsigned size));
-#ifdef ASMV
-#  pragma message("Assembler code may have bugs -- use at your own risk")
-      void match_init OF((void)); /* asm code initialization */
-      uInt longest_match  OF((deflate_state *s, IPos cur_match));
-#else
-local uInt longest_match  OF((deflate_state *s, IPos cur_match));
-#endif
-
-#ifdef ZLIB_DEBUG
-local  void check_match OF((deflate_state *s, IPos start, IPos match,
-                            int length));
+local block_state deflate_slow(deflate_state *s, int flush);
 #endif
+local block_state deflate_rle(deflate_state *s, int flush);
+local block_state deflate_huff(deflate_state *s, int flush);
 
 /* ===========================================================================
  * Local data
@@ -160,7 +138,7 @@ local const config configuration_table[10] = {
  *    characters, so that a running hash key can be computed from the previous
  *    key instead of complete recalculation each time.
  */
-#define UPDATE_HASH(s,h,c) (h = (((h)<<s->hash_shift) ^ (c)) & s->hash_mask)
+#define UPDATE_HASH(s,h,c) (h = (((h) << s->hash_shift) ^ (c)) & s->hash_mask)
 
 
 /* ===========================================================================
@@ -190,17 +168,23 @@ local const config configuration_table[10] = {
  * prev[] will be initialized on the fly.
  */
 #define CLEAR_HASH(s) \
-    s->head[s->hash_size-1] = NIL; \
-    zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head));
+    do { \
+        s->head[s->hash_size - 1] = NIL; \
+        zmemzero((Bytef *)s->head, \
+                 (unsigned)(s->hash_size - 1)*sizeof(*s->head)); \
+    } while (0)
 
 /* ===========================================================================
  * Slide the hash table when sliding the window down (could be avoided with 32
  * bit values at the expense of memory usage). We slide even when level == 0 to
  * keep the hash table consistent if we switch back to level > 0 later.
  */
-local void slide_hash(s)
-    deflate_state *s;
-{
+#if defined(__has_feature)
+#  if __has_feature(memory_sanitizer)
+     __attribute__((no_sanitize("memory")))
+#  endif
+#endif
+local void slide_hash(deflate_state *s) {
     unsigned n, m;
     Posf *p;
     uInt wsize = s->w_size;
@@ -224,39 +208,181 @@ local void slide_hash(s)
 #endif
 }
 
+/* ===========================================================================
+ * Read a new buffer from the current input stream, update the adler32
+ * and total number of bytes read.  All deflate() input goes through
+ * this function so some applications may wish to modify it to avoid
+ * allocating a large strm->next_in buffer and copying from it.
+ * (See also flush_pending()).
+ */
+local unsigned read_buf(z_streamp strm, Bytef *buf, unsigned size) {
+    unsigned len = strm->avail_in;
+
+    if (len > size) len = size;
+    if (len == 0) return 0;
+
+    strm->avail_in  -= len;
+
+    zmemcpy(buf, strm->next_in, len);
+    if (strm->state->wrap == 1) {
+        strm->adler = adler32(strm->adler, buf, len);
+    }
+#ifdef GZIP
+    else if (strm->state->wrap == 2) {
+        strm->adler = crc32(strm->adler, buf, len);
+    }
+#endif
+    strm->next_in  += len;
+    strm->total_in += len;
+
+    return len;
+}
+
+/* ===========================================================================
+ * Fill the window when the lookahead becomes insufficient.
+ * Updates strstart and lookahead.
+ *
+ * IN assertion: lookahead < MIN_LOOKAHEAD
+ * OUT assertions: strstart <= window_size-MIN_LOOKAHEAD
+ *    At least one byte has been read, or avail_in == 0; reads are
+ *    performed for at least two bytes (required for the zip translate_eol
+ *    option -- not supported here).
+ */
+local void fill_window(deflate_state *s) {
+    unsigned n;
+    unsigned more;    /* Amount of free space at the end of the window. */
+    uInt wsize = s->w_size;
+
+    Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead");
+
+    do {
+        more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart);
+
+        /* Deal with !@#$% 64K limit: */
+        if (sizeof(int) <= 2) {
+            if (more == 0 && s->strstart == 0 && s->lookahead == 0) {
+                more = wsize;
+
+            } else if (more == (unsigned)(-1)) {
+                /* Very unlikely, but possible on 16 bit machine if
+                 * strstart == 0 && lookahead == 1 (input done a byte at time)
+                 */
+                more--;
+            }
+        }
+
+        /* If the window is almost full and there is insufficient lookahead,
+         * move the upper half to the lower one to make room in the upper half.
+         */
+        if (s->strstart >= wsize + MAX_DIST(s)) {
+
+            zmemcpy(s->window, s->window + wsize, (unsigned)wsize - more);
+            s->match_start -= wsize;
+            s->strstart    -= wsize; /* we now have strstart >= MAX_DIST */
+            s->block_start -= (long) wsize;
+            if (s->insert > s->strstart)
+                s->insert = s->strstart;
+            slide_hash(s);
+            more += wsize;
+        }
+        if (s->strm->avail_in == 0) break;
+
+        /* If there was no sliding:
+         *    strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 &&
+         *    more == window_size - lookahead - strstart
+         * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1)
+         * => more >= window_size - 2*WSIZE + 2
+         * In the BIG_MEM or MMAP case (not yet supported),
+         *   window_size == input_size + MIN_LOOKAHEAD  &&
+         *   strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD.
+         * Otherwise, window_size == 2*WSIZE so more >= 2.
+         * If there was sliding, more >= WSIZE. So in all cases, more >= 2.
+         */
+        Assert(more >= 2, "more < 2");
+
+        n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more);
+        s->lookahead += n;
+
+        /* Initialize the hash value now that we have some input: */
+        if (s->lookahead + s->insert >= MIN_MATCH) {
+            uInt str = s->strstart - s->insert;
+            s->ins_h = s->window[str];
+            UPDATE_HASH(s, s->ins_h, s->window[str + 1]);
+#if MIN_MATCH != 3
+            Call UPDATE_HASH() MIN_MATCH-3 more times
+#endif
+            while (s->insert) {
+                UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]);
+#ifndef FASTEST
+                s->prev[str & s->w_mask] = s->head[s->ins_h];
+#endif
+                s->head[s->ins_h] = (Pos)str;
+                str++;
+                s->insert--;
+                if (s->lookahead + s->insert < MIN_MATCH)
+                    break;
+            }
+        }
+        /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage,
+         * but this is not important since only literal bytes will be emitted.
+         */
+
+    } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0);
+
+    /* If the WIN_INIT bytes after the end of the current data have never been
+     * written, then zero those bytes in order to avoid memory check reports of
+     * the use of uninitialized (or uninitialised as Julian writes) bytes by
+     * the longest match routines.  Update the high water mark for the next
+     * time through here.  WIN_INIT is set to MAX_MATCH since the longest match
+     * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead.
+     */
+    if (s->high_water < s->window_size) {
+        ulg curr = s->strstart + (ulg)(s->lookahead);
+        ulg init;
+
+        if (s->high_water < curr) {
+            /* Previous high water mark below current data -- zero WIN_INIT
+             * bytes or up to end of window, whichever is less.
+             */
+            init = s->window_size - curr;
+            if (init > WIN_INIT)
+                init = WIN_INIT;
+            zmemzero(s->window + curr, (unsigned)init);
+            s->high_water = curr + init;
+        }
+        else if (s->high_water < (ulg)curr + WIN_INIT) {
+            /* High water mark at or above current data, but below current data
+             * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up
+             * to end of window, whichever is less.
+             */
+            init = (ulg)curr + WIN_INIT - s->high_water;
+            if (init > s->window_size - s->high_water)
+                init = s->window_size - s->high_water;
+            zmemzero(s->window + s->high_water, (unsigned)init);
+            s->high_water += init;
+        }
+    }
+
+    Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD,
+           "not enough room for search");
+}
+
 /* ========================================================================= */
-int ZEXPORT deflateInit_(strm, level, version, stream_size)
-    z_streamp strm;
-    int level;
-    const char *version;
-    int stream_size;
-{
+int ZEXPORT deflateInit_(z_streamp strm, int level, const char *version,
+                         int stream_size) {
     return deflateInit2_(strm, level, Z_DEFLATED, MAX_WBITS, DEF_MEM_LEVEL,
                          Z_DEFAULT_STRATEGY, version, stream_size);
     /* To do: ignore strm->next_in if we use it as window */
 }
 
 /* ========================================================================= */
-int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
-                  version, stream_size)
-    z_streamp strm;
-    int  level;
-    int  method;
-    int  windowBits;
-    int  memLevel;
-    int  strategy;
-    const char *version;
-    int stream_size;
-{
+int ZEXPORT deflateInit2_(z_streamp strm, int level, int method,
+                          int windowBits, int memLevel, int strategy,
+                          const char *version, int stream_size) {
     deflate_state *s;
     int wrap = 1;
     static const char my_version[] = ZLIB_VERSION;
 
-    ushf *overlay;
-    /* We overlay pending_buf and d_buf+l_buf. This works since the average
-     * output size for (length,distance) codes is <= 24 bits.
-     */
-
     if (version == Z_NULL || version[0] != my_version[0] ||
         stream_size != sizeof(z_stream)) {
         return Z_VERSION_ERROR;
@@ -287,6 +413,8 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
 
     if (windowBits < 0) { /* suppress zlib wrapper */
         wrap = 0;
+        if (windowBits < -15)
+            return Z_STREAM_ERROR;
         windowBits = -windowBits;
     }
 #ifdef GZIP
@@ -316,7 +444,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
     s->hash_bits = (uInt)memLevel + 7;
     s->hash_size = 1 << s->hash_bits;
     s->hash_mask = s->hash_size - 1;
-    s->hash_shift =  ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH);
+    s->hash_shift =  ((s->hash_bits + MIN_MATCH-1) / MIN_MATCH);
 
     s->window = (Bytef *) ZALLOC(strm, s->w_size, 2*sizeof(Byte));
     s->prev   = (Posf *)  ZALLOC(strm, s->w_size, sizeof(Pos));
@@ -326,9 +454,47 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
 
     s->lit_bufsize = 1 << (memLevel + 6); /* 16K elements by default */
 
-    overlay = (ushf *) ZALLOC(strm, s->lit_bufsize, sizeof(ush)+2);
-    s->pending_buf = (uchf *) overlay;
-    s->pending_buf_size = (ulg)s->lit_bufsize * (sizeof(ush)+2L);
+    /* We overlay pending_buf and sym_buf. This works since the average size
+     * for length/distance pairs over any compressed block is assured to be 31
+     * bits or less.
+     *
+     * Analysis: The longest fixed codes are a length code of 8 bits plus 5
+     * extra bits, for lengths 131 to 257. The longest fixed distance codes are
+     * 5 bits plus 13 extra bits, for distances 16385 to 32768. The longest
+     * possible fixed-codes length/distance pair is then 31 bits total.
+     *
+     * sym_buf starts one-fourth of the way into pending_buf. So there are
+     * three bytes in sym_buf for every four bytes in pending_buf. Each symbol
+     * in sym_buf is three bytes -- two for the distance and one for the
+     * literal/length. As each symbol is consumed, the pointer to the next
+     * sym_buf value to read moves forward three bytes. From that symbol, up to
+     * 31 bits are written to pending_buf. The closest the written pending_buf
+     * bits gets to the next sym_buf symbol to read is just before the last
+     * code is written. At that time, 31*(n - 2) bits have been written, just
+     * after 24*(n - 2) bits have been consumed from sym_buf. sym_buf starts at
+     * 8*n bits into pending_buf. (Note that the symbol buffer fills when n - 1
+     * symbols are written.) The closest the writing gets to what is unread is
+     * then n + 14 bits. Here n is lit_bufsize, which is 16384 by default, and
+     * can range from 128 to 32768.
+     *
+     * Therefore, at a minimum, there are 142 bits of space between what is
+     * written and what is read in the overlain buffers, so the symbols cannot
+     * be overwritten by the compressed data. That space is actually 139 bits,
+     * due to the three-bit fixed-code block header.
+     *
+     * That covers the case where either Z_FIXED is specified, forcing fixed
+     * codes, or when the use of fixed codes is chosen, because that choice
+     * results in a smaller compressed block than dynamic codes. That latter
+     * condition then assures that the above analysis also covers all dynamic
+     * blocks. A dynamic-code block will only be chosen to be emitted if it has
+     * fewer bits than a fixed-code block would for the same set of symbols.
+     * Therefore its average symbol length is assured to be less than 31. So
+     * the compressed data for a dynamic block also cannot overwrite the
+     * symbols from which it is being constructed.
+     */
+
+    s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, LIT_BUFS);
+    s->pending_buf_size = (ulg)s->lit_bufsize * 4;
 
     if (s->window == Z_NULL || s->prev == Z_NULL || s->head == Z_NULL ||
         s->pending_buf == Z_NULL) {
@@ -337,8 +503,18 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
         deflateEnd (strm);
         return Z_MEM_ERROR;
     }
-    s->d_buf = overlay + s->lit_bufsize/sizeof(ush);
-    s->l_buf = s->pending_buf + (1+sizeof(ush))*s->lit_bufsize;
+#ifdef LIT_MEM
+    s->d_buf = (ushf *)(s->pending_buf + (s->lit_bufsize << 1));
+    s->l_buf = s->pending_buf + (s->lit_bufsize << 2);
+    s->sym_end = s->lit_bufsize - 1;
+#else
+    s->sym_buf = s->pending_buf + s->lit_bufsize;
+    s->sym_end = (s->lit_bufsize - 1) * 3;
+#endif
+    /* We avoid equality with lit_bufsize*3 because of wraparound at 64K
+     * on 16 bit machines and because stored blocks are restricted to
+     * 64K-1 bytes.
+     */
 
     s->level = level;
     s->strategy = strategy;
@@ -350,9 +526,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
 /* =========================================================================
  * Check for a valid deflate stream state. Return 0 if ok, 1 if not.
  */
-local int deflateStateCheck (strm)
-    z_streamp strm;
-{
+local int deflateStateCheck(z_streamp strm) {
     deflate_state *s;
     if (strm == Z_NULL ||
         strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0)
@@ -373,11 +547,8 @@ local int deflateStateCheck (strm)
 }
 
 /* ========================================================================= */
-int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength)
-    z_streamp strm;
-    const Bytef *dictionary;
-    uInt  dictLength;
-{
+int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef *dictionary,
+                                 uInt  dictLength) {
     deflate_state *s;
     uInt str, n;
     int wrap;
@@ -442,11 +613,8 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength)
 }
 
 /* ========================================================================= */
-int ZEXPORT deflateGetDictionary (strm, dictionary, dictLength)
-    z_streamp strm;
-    Bytef *dictionary;
-    uInt  *dictLength;
-{
+int ZEXPORT deflateGetDictionary(z_streamp strm, Bytef *dictionary,
+                                 uInt *dictLength) {
     deflate_state *s;
     uInt len;
 
@@ -464,9 +632,7 @@ int ZEXPORT deflateGetDictionary (strm, dictionary, dictLength)
 }
 
 /* ========================================================================= */
-int ZEXPORT deflateResetKeep (strm)
-    z_streamp strm;
-{
+int ZEXPORT deflateResetKeep(z_streamp strm) {
     deflate_state *s;
 
     if (deflateStateCheck(strm)) {
@@ -488,23 +654,45 @@ int ZEXPORT deflateResetKeep (strm)
 #ifdef GZIP
         s->wrap == 2 ? GZIP_STATE :
 #endif
-        s->wrap ? INIT_STATE : BUSY_STATE;
+        INIT_STATE;
     strm->adler =
 #ifdef GZIP
         s->wrap == 2 ? crc32(0L, Z_NULL, 0) :
 #endif
         adler32(0L, Z_NULL, 0);
-    s->last_flush = Z_NO_FLUSH;
+    s->last_flush = -2;
 
     _tr_init(s);
 
     return Z_OK;
 }
 
+/* ===========================================================================
+ * Initialize the "longest match" routines for a new zlib stream
+ */
+local void lm_init(deflate_state *s) {
+    s->window_size = (ulg)2L*s->w_size;
+
+    CLEAR_HASH(s);
+
+    /* Set the default configuration parameters:
+     */
+    s->max_lazy_match   = configuration_table[s->level].max_lazy;
+    s->good_match       = configuration_table[s->level].good_length;
+    s->nice_match       = configuration_table[s->level].nice_length;
+    s->max_chain_length = configuration_table[s->level].max_chain;
+
+    s->strstart = 0;
+    s->block_start = 0L;
+    s->lookahead = 0;
+    s->insert = 0;
+    s->match_length = s->prev_length = MIN_MATCH-1;
+    s->match_available = 0;
+    s->ins_h = 0;
+}
+
 /* ========================================================================= */
-int ZEXPORT deflateReset (strm)
-    z_streamp strm;
-{
+int ZEXPORT deflateReset(z_streamp strm) {
     int ret;
 
     ret = deflateResetKeep(strm);
@@ -514,10 +702,7 @@ int ZEXPORT deflateReset (strm)
 }
 
 /* ========================================================================= */
-int ZEXPORT deflateSetHeader (strm, head)
-    z_streamp strm;
-    gz_headerp head;
-{
+int ZEXPORT deflateSetHeader(z_streamp strm, gz_headerp head) {
     if (deflateStateCheck(strm) || strm->state->wrap != 2)
         return Z_STREAM_ERROR;
     strm->state->gzhead = head;
@@ -525,11 +710,7 @@ int ZEXPORT deflateSetHeader (strm, head)
 }
 
 /* ========================================================================= */
-int ZEXPORT deflatePending (strm, pending, bits)
-    unsigned *pending;
-    int *bits;
-    z_streamp strm;
-{
+int ZEXPORT deflatePending(z_streamp strm, unsigned *pending, int *bits) {
     if (deflateStateCheck(strm)) return Z_STREAM_ERROR;
     if (pending != Z_NULL)
         *pending = strm->state->pending;
@@ -539,18 +720,21 @@ int ZEXPORT deflatePending (strm, pending, bits)
 }
 
 /* ========================================================================= */
-int ZEXPORT deflatePrime (strm, bits, value)
-    z_streamp strm;
-    int bits;
-    int value;
-{
+int ZEXPORT deflatePrime(z_streamp strm, int bits, int value) {
     deflate_state *s;
     int put;
 
     if (deflateStateCheck(strm)) return Z_STREAM_ERROR;
     s = strm->state;
-    if ((Bytef *)(s->d_buf) < s->pending_out + ((Buf_size + 7) >> 3))
+#ifdef LIT_MEM
+    if (bits < 0 || bits > 16 ||
+        (uchf *)s->d_buf < s->pending_out + ((Buf_size + 7) >> 3))
+        return Z_BUF_ERROR;
+#else
+    if (bits < 0 || bits > 16 ||
+        s->sym_buf < s->pending_out + ((Buf_size + 7) >> 3))
         return Z_BUF_ERROR;
+#endif
     do {
         put = Buf_size - s->bi_valid;
         if (put > bits)
@@ -565,11 +749,7 @@ int ZEXPORT deflatePrime (strm, bits, value)
 }
 
 /* ========================================================================= */
-int ZEXPORT deflateParams(strm, level, strategy)
-    z_streamp strm;
-    int level;
-    int strategy;
-{
+int ZEXPORT deflateParams(z_streamp strm, int level, int strategy) {
     deflate_state *s;
     compress_func func;
 
@@ -587,12 +767,12 @@ int ZEXPORT deflateParams(strm, level, strategy)
     func = configuration_table[s->level].func;
 
     if ((strategy != s->strategy || func != configuration_table[level].func) &&
-        s->high_water) {
+        s->last_flush != -2) {
         /* Flush the last buffer: */
         int err = deflate(strm, Z_BLOCK);
         if (err == Z_STREAM_ERROR)
             return err;
-        if (strm->avail_out == 0)
+        if (strm->avail_in || (s->strstart - s->block_start) + s->lookahead)
             return Z_BUF_ERROR;
     }
     if (s->level != level) {
@@ -614,13 +794,8 @@ int ZEXPORT deflateParams(strm, level, strategy)
 }
 
 /* ========================================================================= */
-int ZEXPORT deflateTune(strm, good_length, max_lazy, nice_length, max_chain)
-    z_streamp strm;
-    int good_length;
-    int max_lazy;
-    int nice_length;
-    int max_chain;
-{
+int ZEXPORT deflateTune(z_streamp strm, int good_length, int max_lazy,
+                        int nice_length, int max_chain) {
     deflate_state *s;
 
     if (deflateStateCheck(strm)) return Z_STREAM_ERROR;
@@ -633,36 +808,47 @@ int ZEXPORT deflateTune(strm, good_length, max_lazy, nice_length, max_chain)
 }
 
 /* =========================================================================
- * For the default windowBits of 15 and memLevel of 8, this function returns
- * a close to exact, as well as small, upper bound on the compressed size.
- * They are coded as constants here for a reason--if the #define's are
- * changed, then this function needs to be changed as well.  The return
- * value for 15 and 8 only works for those exact settings.
+ * For the default windowBits of 15 and memLevel of 8, this function returns a
+ * close to exact, as well as small, upper bound on the compressed size. This
+ * is an expansion of ~0.03%, plus a small constant.
+ *
+ * For any setting other than those defaults for windowBits and memLevel, one
+ * of two worst case bounds is returned. This is at most an expansion of ~4% or
+ * ~13%, plus a small constant.
+ *
+ * Both the 0.03% and 4% derive from the overhead of stored blocks. The first
+ * one is for stored blocks of 16383 bytes (memLevel == 8), whereas the second
+ * is for stored blocks of 127 bytes (the worst case memLevel == 1). The
+ * expansion results from five bytes of header for each stored block.
  *
- * For any setting other than those defaults for windowBits and memLevel,
- * the value returned is a conservative worst case for the maximum expansion
- * resulting from using fixed blocks instead of stored blocks, which deflate
- * can emit on compressed data for some combinations of the parameters.
+ * The larger expansion of 13% results from a window size less than or equal to
+ * the symbols buffer size (windowBits <= memLevel + 7). In that case some of
+ * the data being compressed may have slid out of the sliding window, impeding
+ * a stored block from being emitted. Then the only choice is a fixed or
+ * dynamic block, where a fixed block limits the maximum expansion to 9 bits
+ * per 8-bit byte, plus 10 bits for every block. The smallest block size for
+ * which this can occur is 255 (memLevel == 2).
  *
- * This function could be more sophisticated to provide closer upper bounds for
- * every combination of windowBits and memLevel.  But even the conservative
- * upper bound of about 14% expansion does not seem onerous for output buffer
- * allocation.
+ * Shifts are used to approximate divisions, for speed.
  */
-uLong ZEXPORT deflateBound(strm, sourceLen)
-    z_streamp strm;
-    uLong sourceLen;
-{
+uLong ZEXPORT deflateBound(z_streamp strm, uLong sourceLen) {
     deflate_state *s;
-    uLong complen, wraplen;
+    uLong fixedlen, storelen, wraplen;
 
-    /* conservative upper bound for compressed data */
-    complen = sourceLen +
-              ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 5;
+    /* upper bound for fixed blocks with 9-bit literals and length 255
+       (memLevel == 2, which is the lowest that may not use stored blocks) --
+       ~13% overhead plus a small constant */
+    fixedlen = sourceLen + (sourceLen >> 3) + (sourceLen >> 8) +
+               (sourceLen >> 9) + 4;
 
-    /* if can't get parameters, return conservative bound plus zlib wrapper */
+    /* upper bound for stored blocks with length 127 (memLevel == 1) --
+       ~4% overhead plus a small constant */
+    storelen = sourceLen + (sourceLen >> 5) + (sourceLen >> 7) +
+               (sourceLen >> 11) + 7;
+
+    /* if can't get parameters, return larger bound plus a zlib wrapper */
     if (deflateStateCheck(strm))
-        return complen + 6;
+        return (fixedlen > storelen ? fixedlen : storelen) + 6;
 
     /* compute wrapper length */
     s = strm->state;
@@ -699,11 +885,13 @@ uLong ZEXPORT deflateBound(strm, sourceLen)
         wraplen = 6;
     }
 
-    /* if not default parameters, return conservative bound */
+    /* if not default parameters, return one of the conservative bounds */
     if (s->w_bits != 15 || s->hash_bits != 8 + 7)
-        return complen + wraplen;
+        return (s->w_bits <= s->hash_bits && s->level ? fixedlen : storelen) +
+               wraplen;
 
-    /* default settings: return tight bound for that case */
+    /* default settings: return tight bound for that case -- ~0.03% overhead
+       plus a small constant */
     return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) +
            (sourceLen >> 25) + 13 - 6 + wraplen;
 }
@@ -713,10 +901,7 @@ uLong ZEXPORT deflateBound(strm, sourceLen)
  * IN assertion: the stream state is correct and there is enough room in
  * pending_buf.
  */
-local void putShortMSB (s, b)
-    deflate_state *s;
-    uInt b;
-{
+local void putShortMSB(deflate_state *s, uInt b) {
     put_byte(s, (Byte)(b >> 8));
     put_byte(s, (Byte)(b & 0xff));
 }
@@ -727,9 +912,7 @@ local void putShortMSB (s, b)
  * applications may wish to modify it to avoid allocating a large
  * strm->next_out buffer and copying into it. (See also read_buf()).
  */
-local void flush_pending(strm)
-    z_streamp strm;
-{
+local void flush_pending(z_streamp strm) {
     unsigned len;
     deflate_state *s = strm->state;
 
@@ -760,10 +943,7 @@ local void flush_pending(strm)
     } while (0)
 
 /* ========================================================================= */
-int ZEXPORT deflate (strm, flush)
-    z_streamp strm;
-    int flush;
-{
+int ZEXPORT deflate(z_streamp strm, int flush) {
     int old_flush; /* value of flush param for previous deflate call */
     deflate_state *s;
 
@@ -811,9 +991,11 @@ int ZEXPORT deflate (strm, flush)
     }
 
     /* Write the header */
+    if (s->status == INIT_STATE && s->wrap == 0)
+        s->status = BUSY_STATE;
     if (s->status == INIT_STATE) {
         /* zlib header */
-        uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8;
+        uInt header = (Z_DEFLATED + ((s->w_bits - 8) << 4)) << 8;
         uInt level_flags;
 
         if (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2)
@@ -1073,9 +1255,7 @@ int ZEXPORT deflate (strm, flush)
 }
 
 /* ========================================================================= */
-int ZEXPORT deflateEnd (strm)
-    z_streamp strm;
-{
+int ZEXPORT deflateEnd(z_streamp strm) {
     int status;
 
     if (deflateStateCheck(strm)) return Z_STREAM_ERROR;
@@ -1099,16 +1279,14 @@ int ZEXPORT deflateEnd (strm)
  * To simplify the source, this is not supported for 16-bit MSDOS (which
  * doesn't have enough memory anyway to duplicate compression states).
  */
-int ZEXPORT deflateCopy (dest, source)
-    z_streamp dest;
-    z_streamp source;
-{
+int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) {
 #ifdef MAXSEG_64K
+    (void)dest;
+    (void)source;
     return Z_STREAM_ERROR;
 #else
     deflate_state *ds;
     deflate_state *ss;
-    ushf *overlay;
 
 
     if (deflateStateCheck(source) || dest == Z_NULL) {
@@ -1128,8 +1306,7 @@ int ZEXPORT deflateCopy (dest, source)
     ds->window = (Bytef *) ZALLOC(dest, ds->w_size, 2*sizeof(Byte));
     ds->prev   = (Posf *)  ZALLOC(dest, ds->w_size, sizeof(Pos));
     ds->head   = (Posf *)  ZALLOC(dest, ds->hash_size, sizeof(Pos));
-    overlay = (ushf *) ZALLOC(dest, ds->lit_bufsize, sizeof(ush)+2);
-    ds->pending_buf = (uchf *) overlay;
+    ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, LIT_BUFS);
 
     if (ds->window == Z_NULL || ds->prev == Z_NULL || ds->head == Z_NULL ||
         ds->pending_buf == Z_NULL) {
@@ -1140,11 +1317,15 @@ int ZEXPORT deflateCopy (dest, source)
     zmemcpy(ds->window, ss->window, ds->w_size * 2 * sizeof(Byte));
     zmemcpy((voidpf)ds->prev, (voidpf)ss->prev, ds->w_size * sizeof(Pos));
     zmemcpy((voidpf)ds->head, (voidpf)ss->head, ds->hash_size * sizeof(Pos));
-    zmemcpy(ds->pending_buf, ss->pending_buf, (uInt)ds->pending_buf_size);
+    zmemcpy(ds->pending_buf, ss->pending_buf, ds->lit_bufsize * LIT_BUFS);
 
     ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf);
-    ds->d_buf = overlay + ds->lit_bufsize/sizeof(ush);
-    ds->l_buf = ds->pending_buf + (1+sizeof(ush))*ds->lit_bufsize;
+#ifdef LIT_MEM
+    ds->d_buf = (ushf *)(ds->pending_buf + (ds->lit_bufsize << 1));
+    ds->l_buf = ds->pending_buf + (ds->lit_bufsize << 2);
+#else
+    ds->sym_buf = ds->pending_buf + ds->lit_bufsize;
+#endif
 
     ds->l_desc.dyn_tree = ds->dyn_ltree;
     ds->d_desc.dyn_tree = ds->dyn_dtree;
@@ -1154,71 +1335,6 @@ int ZEXPORT deflateCopy (dest, source)
 #endif /* MAXSEG_64K */
 }
 
-/* ===========================================================================
- * Read a new buffer from the current input stream, update the adler32
- * and total number of bytes read.  All deflate() input goes through
- * this function so some applications may wish to modify it to avoid
- * allocating a large strm->next_in buffer and copying from it.
- * (See also flush_pending()).
- */
-local unsigned read_buf(strm, buf, size)
-    z_streamp strm;
-    Bytef *buf;
-    unsigned size;
-{
-    unsigned len = strm->avail_in;
-
-    if (len > size) len = size;
-    if (len == 0) return 0;
-
-    strm->avail_in  -= len;
-
-    zmemcpy(buf, strm->next_in, len);
-    if (strm->state->wrap == 1) {
-        strm->adler = adler32(strm->adler, buf, len);
-    }
-#ifdef GZIP
-    else if (strm->state->wrap == 2) {
-        strm->adler = crc32(strm->adler, buf, len);
-    }
-#endif
-    strm->next_in  += len;
-    strm->total_in += len;
-
-    return len;
-}
-
-/* ===========================================================================
- * Initialize the "longest match" routines for a new zlib stream
- */
-local void lm_init (s)
-    deflate_state *s;
-{
-    s->window_size = (ulg)2L*s->w_size;
-
-    CLEAR_HASH(s);
-
-    /* Set the default configuration parameters:
-     */
-    s->max_lazy_match   = configuration_table[s->level].max_lazy;
-    s->good_match       = configuration_table[s->level].good_length;
-    s->nice_match       = configuration_table[s->level].nice_length;
-    s->max_chain_length = configuration_table[s->level].max_chain;
-
-    s->strstart = 0;
-    s->block_start = 0L;
-    s->lookahead = 0;
-    s->insert = 0;
-    s->match_length = s->prev_length = MIN_MATCH-1;
-    s->match_available = 0;
-    s->ins_h = 0;
-#ifndef FASTEST
-#ifdef ASMV
-    match_init(); /* initialize the asm code */
-#endif
-#endif
-}
-
 #ifndef FASTEST
 /* ===========================================================================
  * Set match_start to the longest match starting at the given string and
@@ -1229,14 +1345,7 @@ local void lm_init (s)
  *   string (strstart) and its distance is <= MAX_DIST, and prev_length >= 1
  * OUT assertion: the match length is not greater than s->lookahead.
  */
-#ifndef ASMV
-/* For 80x86 and 680x0, an optimized version will be provided in match.asm or
- * match.S. The code will be functionally equivalent.
- */
-local uInt longest_match(s, cur_match)
-    deflate_state *s;
-    IPos cur_match;                             /* current match */
-{
+local uInt longest_match(deflate_state *s, IPos cur_match) {
     unsigned chain_length = s->max_chain_length;/* max hash chain length */
     register Bytef *scan = s->window + s->strstart; /* current string */
     register Bytef *match;                      /* matched string */
@@ -1257,10 +1366,10 @@ local uInt longest_match(s, cur_match)
      */
     register Bytef *strend = s->window + s->strstart + MAX_MATCH - 1;
     register ush scan_start = *(ushf*)scan;
-    register ush scan_end   = *(ushf*)(scan+best_len-1);
+    register ush scan_end   = *(ushf*)(scan + best_len - 1);
 #else
     register Bytef *strend = s->window + s->strstart + MAX_MATCH;
-    register Byte scan_end1  = scan[best_len-1];
+    register Byte scan_end1  = scan[best_len - 1];
     register Byte scan_end   = scan[best_len];
 #endif
 
@@ -1278,7 +1387,8 @@ local uInt longest_match(s, cur_match)
      */
     if ((uInt)nice_match > s->lookahead) nice_match = (int)s->lookahead;
 
-    Assert((ulg)s->strstart <= s->window_size-MIN_LOOKAHEAD, "need lookahead");
+    Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD,
+           "need lookahead");
 
     do {
         Assert(cur_match < s->strstart, "no future");
@@ -1296,43 +1406,44 @@ local uInt longest_match(s, cur_match)
         /* This code assumes sizeof(unsigned short) == 2. Do not use
          * UNALIGNED_OK if your compiler uses a different size.
          */
-        if (*(ushf*)(match+best_len-1) != scan_end ||
+        if (*(ushf*)(match + best_len - 1) != scan_end ||
             *(ushf*)match != scan_start) continue;
 
         /* It is not necessary to compare scan[2] and match[2] since they are
          * always equal when the other bytes match, given that the hash keys
          * are equal and that HASH_BITS >= 8. Compare 2 bytes at a time at
-         * strstart+3, +5, ... up to strstart+257. We check for insufficient
+         * strstart + 3, + 5, up to strstart + 257. We check for insufficient
          * lookahead only every 4th comparison; the 128th check will be made
-         * at strstart+257. If MAX_MATCH-2 is not a multiple of 8, it is
+         * at strstart + 257. If MAX_MATCH-2 is not a multiple of 8, it is
          * necessary to put more guard bytes at the end of the window, or
          * to check more often for insufficient lookahead.
          */
         Assert(scan[2] == match[2], "scan[2]?");
         scan++, match++;
         do {
-        } while (*(ushf*)(scan+=2) == *(ushf*)(match+=2) &&
-                 *(ushf*)(scan+=2) == *(ushf*)(match+=2) &&
-                 *(ushf*)(scan+=2) == *(ushf*)(match+=2) &&
-                 *(ushf*)(scan+=2) == *(ushf*)(match+=2) &&
+        } while (*(ushf*)(scan += 2) == *(ushf*)(match += 2) &&
+                 *(ushf*)(scan += 2) == *(ushf*)(match += 2) &&
+                 *(ushf*)(scan += 2) == *(ushf*)(match += 2) &&
+                 *(ushf*)(scan += 2) == *(ushf*)(match += 2) &&
                  scan < strend);
         /* The funny "do {}" generates better code on most compilers */
 
-        /* Here, scan <= window+strstart+257 */
-        Assert(scan <= s->window+(unsigned)(s->window_size-1), "wild scan");
+        /* Here, scan <= window + strstart + 257 */
+        Assert(scan <= s->window + (unsigned)(s->window_size - 1),
+               "wild scan");
         if (*scan == *match) scan++;
 
-        len = (MAX_MATCH - 1) - (int)(strend-scan);
+        len = (MAX_MATCH - 1) - (int)(strend - scan);
         scan = strend - (MAX_MATCH-1);
 
 #else /* UNALIGNED_OK */
 
-        if (match[best_len]   != scan_end  ||
-            match[best_len-1] != scan_end1 ||
-            *match            != *scan     ||
-            *++match          != scan[1])      continue;
+        if (match[best_len]     != scan_end  ||
+            match[best_len - 1] != scan_end1 ||
+            *match              != *scan     ||
+            *++match            != scan[1])      continue;
 
-        /* The check at best_len-1 can be removed because it will be made
+        /* The check at best_len - 1 can be removed because it will be made
          * again later. (This heuristic is not always a win.)
          * It is not necessary to compare scan[2] and match[2] since they
          * are always equal when the other bytes match, given that
@@ -1342,7 +1453,7 @@ local uInt longest_match(s, cur_match)
         Assert(*scan == *match, "match[2]?");
 
         /* We check for insufficient lookahead only every 8th comparison;
-         * the 256th check will be made at strstart+258.
+         * the 256th check will be made at strstart + 258.
          */
         do {
         } while (*++scan == *++match && *++scan == *++match &&
@@ -1351,7 +1462,8 @@ local uInt longest_match(s, cur_match)
                  *++scan == *++match && *++scan == *++match &&
                  scan < strend);
 
-        Assert(scan <= s->window+(unsigned)(s->window_size-1), "wild scan");
+        Assert(scan <= s->window + (unsigned)(s->window_size - 1),
+               "wild scan");
 
         len = MAX_MATCH - (int)(strend - scan);
         scan = strend - MAX_MATCH;
@@ -1363,9 +1475,9 @@ local uInt longest_match(s, cur_match)
             best_len = len;
             if (len >= nice_match) break;
 #ifdef UNALIGNED_OK
-            scan_end = *(ushf*)(scan+best_len-1);
+            scan_end = *(ushf*)(scan + best_len - 1);
 #else
-            scan_end1  = scan[best_len-1];
+            scan_end1  = scan[best_len - 1];
             scan_end   = scan[best_len];
 #endif
         }
@@ -1375,17 +1487,13 @@ local uInt longest_match(s, cur_match)
     if ((uInt)best_len <= s->lookahead) return (uInt)best_len;
     return s->lookahead;
 }
-#endif /* ASMV */
 
 #else /* FASTEST */
 
 /* ---------------------------------------------------------------------------
  * Optimized version for FASTEST only
  */
-local uInt longest_match(s, cur_match)
-    deflate_state *s;
-    IPos cur_match;                             /* current match */
-{
+local uInt longest_match(deflate_state *s, IPos cur_match) {
     register Bytef *scan = s->window + s->strstart; /* current string */
     register Bytef *match;                       /* matched string */
     register int len;                           /* length of current match */
@@ -1396,7 +1504,8 @@ local uInt longest_match(s, cur_match)
      */
     Assert(s->hash_bits >= 8 && MAX_MATCH == 258, "Code too clever");
 
-    Assert((ulg)s->strstart <= s->window_size-MIN_LOOKAHEAD, "need lookahead");
+    Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD,
+           "need lookahead");
 
     Assert(cur_match < s->strstart, "no future");
 
@@ -1406,7 +1515,7 @@ local uInt longest_match(s, cur_match)
      */
     if (match[0] != scan[0] || match[1] != scan[1]) return MIN_MATCH-1;
 
-    /* The check at best_len-1 can be removed because it will be made
+    /* The check at best_len - 1 can be removed because it will be made
      * again later. (This heuristic is not always a win.)
      * It is not necessary to compare scan[2] and match[2] since they
      * are always equal when the other bytes match, given that
@@ -1416,7 +1525,7 @@ local uInt longest_match(s, cur_match)
     Assert(*scan == *match, "match[2]?");
 
     /* We check for insufficient lookahead only every 8th comparison;
-     * the 256th check will be made at strstart+258.
+     * the 256th check will be made at strstart + 258.
      */
     do {
     } while (*++scan == *++match && *++scan == *++match &&
@@ -1425,7 +1534,7 @@ local uInt longest_match(s, cur_match)
              *++scan == *++match && *++scan == *++match &&
              scan < strend);
 
-    Assert(scan <= s->window+(unsigned)(s->window_size-1), "wild scan");
+    Assert(scan <= s->window + (unsigned)(s->window_size - 1), "wild scan");
 
     len = MAX_MATCH - (int)(strend - scan);
 
@@ -1445,23 +1554,27 @@ local uInt longest_match(s, cur_match)
 /* ===========================================================================
  * Check that the match at match_start is indeed a match.
  */
-local void check_match(s, start, match, length)
-    deflate_state *s;
-    IPos start, match;
-    int length;
-{
+local void check_match(deflate_state *s, IPos start, IPos match, int length) {
     /* check that the match is indeed a match */
-    if (zmemcmp(s->window + match,
-                s->window + start, length) != EQUAL) {
-        fprintf(stderr, " start %u, match %u, length %d\n",
-                start, match, length);
+    Bytef *back = s->window + (int)match, *here = s->window + start;
+    IPos len = length;
+    if (match == (IPos)-1) {
+        /* match starts one byte before the current window -- just compare the
+           subsequent length-1 bytes */
+        back++;
+        here++;
+        len--;
+    }
+    if (zmemcmp(back, here, len) != EQUAL) {
+        fprintf(stderr, " start %u, match %d, length %d\n",
+                start, (int)match, length);
         do {
-            fprintf(stderr, "%c%c", s->window[match++], s->window[start++]);
-        } while (--length != 0);
+            fprintf(stderr, "(%02x %02x)", *back++, *here++);
+        } while (--len != 0);
         z_error("invalid match");
     }
     if (z_verbose > 1) {
-        fprintf(stderr,"\\[%d,%d]", start-match, length);
+        fprintf(stderr,"\\[%d,%d]", start - match, length);
         do { putc(s->window[start++], stderr); } while (--length != 0);
     }
 }
@@ -1470,135 +1583,6 @@ local void check_match(s, start, match, length)
 #endif /* ZLIB_DEBUG */
 
 /* ===========================================================================
- * Fill the window when the lookahead becomes insufficient.
- * Updates strstart and lookahead.
- *
- * IN assertion: lookahead < MIN_LOOKAHEAD
- * OUT assertions: strstart <= window_size-MIN_LOOKAHEAD
- *    At least one byte has been read, or avail_in == 0; reads are
- *    performed for at least two bytes (required for the zip translate_eol
- *    option -- not supported here).
- */
-local void fill_window(s)
-    deflate_state *s;
-{
-    unsigned n;
-    unsigned more;    /* Amount of free space at the end of the window. */
-    uInt wsize = s->w_size;
-
-    Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead");
-
-    do {
-        more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart);
-
-        /* Deal with !@#$% 64K limit: */
-        if (sizeof(int) <= 2) {
-            if (more == 0 && s->strstart == 0 && s->lookahead == 0) {
-                more = wsize;
-
-            } else if (more == (unsigned)(-1)) {
-                /* Very unlikely, but possible on 16 bit machine if
-                 * strstart == 0 && lookahead == 1 (input done a byte at time)
-                 */
-                more--;
-            }
-        }
-
-        /* If the window is almost full and there is insufficient lookahead,
-         * move the upper half to the lower one to make room in the upper half.
-         */
-        if (s->strstart >= wsize+MAX_DIST(s)) {
-
-            zmemcpy(s->window, s->window+wsize, (unsigned)wsize - more);
-            s->match_start -= wsize;
-            s->strstart    -= wsize; /* we now have strstart >= MAX_DIST */
-            s->block_start -= (long) wsize;
-            slide_hash(s);
-            more += wsize;
-        }
-        if (s->strm->avail_in == 0) break;
-
-        /* If there was no sliding:
-         *    strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 &&
-         *    more == window_size - lookahead - strstart
-         * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1)
-         * => more >= window_size - 2*WSIZE + 2
-         * In the BIG_MEM or MMAP case (not yet supported),
-         *   window_size == input_size + MIN_LOOKAHEAD  &&
-         *   strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD.
-         * Otherwise, window_size == 2*WSIZE so more >= 2.
-         * If there was sliding, more >= WSIZE. So in all cases, more >= 2.
-         */
-        Assert(more >= 2, "more < 2");
-
-        n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more);
-        s->lookahead += n;
-
-        /* Initialize the hash value now that we have some input: */
-        if (s->lookahead + s->insert >= MIN_MATCH) {
-            uInt str = s->strstart - s->insert;
-            s->ins_h = s->window[str];
-            UPDATE_HASH(s, s->ins_h, s->window[str + 1]);
-#if MIN_MATCH != 3
-            Call UPDATE_HASH() MIN_MATCH-3 more times
-#endif
-            while (s->insert) {
-                UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]);
-#ifndef FASTEST
-                s->prev[str & s->w_mask] = s->head[s->ins_h];
-#endif
-                s->head[s->ins_h] = (Pos)str;
-                str++;
-                s->insert--;
-                if (s->lookahead + s->insert < MIN_MATCH)
-                    break;
-            }
-        }
-        /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage,
-         * but this is not important since only literal bytes will be emitted.
-         */
-
-    } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0);
-
-    /* If the WIN_INIT bytes after the end of the current data have never been
-     * written, then zero those bytes in order to avoid memory check reports of
-     * the use of uninitialized (or uninitialised as Julian writes) bytes by
-     * the longest match routines.  Update the high water mark for the next
-     * time through here.  WIN_INIT is set to MAX_MATCH since the longest match
-     * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead.
-     */
-    if (s->high_water < s->window_size) {
-        ulg curr = s->strstart + (ulg)(s->lookahead);
-        ulg init;
-
-        if (s->high_water < curr) {
-            /* Previous high water mark below current data -- zero WIN_INIT
-             * bytes or up to end of window, whichever is less.
-             */
-            init = s->window_size - curr;
-            if (init > WIN_INIT)
-                init = WIN_INIT;
-            zmemzero(s->window + curr, (unsigned)init);
-            s->high_water = curr + init;
-        }
-        else if (s->high_water < (ulg)curr + WIN_INIT) {
-            /* High water mark at or above current data, but below current data
-             * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up
-             * to end of window, whichever is less.
-             */
-            init = (ulg)curr + WIN_INIT - s->high_water;
-            if (init > s->window_size - s->high_water)
-                init = s->window_size - s->high_water;
-            zmemzero(s->window + s->high_water, (unsigned)init);
-            s->high_water += init;
-        }
-    }
-
-    Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD,
-           "not enough room for search");
-}
-
-/* ===========================================================================
  * Flush the current block, with given end-of-file flag.
  * IN assertion: strstart is set to the end of the current match.
  */
@@ -1638,12 +1622,9 @@ local void fill_window(s)
  *
  * deflate_stored() is written to minimize the number of times an input byte is
  * copied. It is most efficient with large input and output buffers, which
- * maximizes the opportunites to have a single copy from next_in to next_out.
+ * maximizes the opportunities to have a single copy from next_in to next_out.
  */
-local block_state deflate_stored(s, flush)
-    deflate_state *s;
-    int flush;
-{
+local block_state deflate_stored(deflate_state *s, int flush) {
     /* Smallest worthy block size when not flushing or finishing. By default
      * this is 32K. This can be as small as 507 bytes for memLevel == 1. For
      * large input and output buffers, the stored block size will be larger.
@@ -1742,6 +1723,7 @@ local block_state deflate_stored(s, flush)
             s->matches = 2;         /* clear hash */
             zmemcpy(s->window, s->strm->next_in - s->w_size, s->w_size);
             s->strstart = s->w_size;
+            s->insert = s->strstart;
         }
         else {
             if (s->window_size - s->strstart <= used) {
@@ -1750,12 +1732,14 @@ local block_state deflate_stored(s, flush)
                 zmemcpy(s->window, s->window + s->w_size, s->strstart);
                 if (s->matches < 2)
                     s->matches++;   /* add a pending slide_hash() */
+                if (s->insert > s->strstart)
+                    s->insert = s->strstart;
             }
             zmemcpy(s->window + s->strstart, s->strm->next_in - used, used);
             s->strstart += used;
+            s->insert += MIN(used, s->w_size - s->insert);
         }
         s->block_start = s->strstart;
-        s->insert += MIN(used, s->w_size - s->insert);
     }
     if (s->high_water < s->strstart)
         s->high_water = s->strstart;
@@ -1770,7 +1754,7 @@ local block_state deflate_stored(s, flush)
         return block_done;
 
     /* Fill the window with any remaining input. */
-    have = s->window_size - s->strstart - 1;
+    have = s->window_size - s->strstart;
     if (s->strm->avail_in > have && s->block_start >= (long)s->w_size) {
         /* Slide the window down. */
         s->block_start -= s->w_size;
@@ -1779,12 +1763,15 @@ local block_state deflate_stored(s, flush)
         if (s->matches < 2)
             s->matches++;           /* add a pending slide_hash() */
         have += s->w_size;          /* more space now */
+        if (s->insert > s->strstart)
+            s->insert = s->strstart;
     }
     if (have > s->strm->avail_in)
         have = s->strm->avail_in;
     if (have) {
         read_buf(s->strm, s->window + s->strstart, have);
         s->strstart += have;
+        s->insert += MIN(have, s->w_size - s->insert);
     }
     if (s->high_water < s->strstart)
         s->high_water = s->strstart;
@@ -1821,10 +1808,7 @@ local block_state deflate_stored(s, flush)
  * new strings in the dictionary only for unmatched strings or for short
  * matches. It is used only for the fast compression options.
  */
-local block_state deflate_fast(s, flush)
-    deflate_state *s;
-    int flush;
-{
+local block_state deflate_fast(deflate_state *s, int flush) {
     IPos hash_head;       /* head of the hash chain */
     int bflush;           /* set if current block must be flushed */
 
@@ -1842,7 +1826,7 @@ local block_state deflate_fast(s, flush)
             if (s->lookahead == 0) break; /* flush the current block */
         }
 
-        /* Insert the string window[strstart .. strstart+2] in the
+        /* Insert the string window[strstart .. strstart + 2] in the
          * dictionary, and set hash_head to the head of the hash chain:
          */
         hash_head = NIL;
@@ -1890,7 +1874,7 @@ local block_state deflate_fast(s, flush)
                 s->strstart += s->match_length;
                 s->match_length = 0;
                 s->ins_h = s->window[s->strstart];
-                UPDATE_HASH(s, s->ins_h, s->window[s->strstart+1]);
+                UPDATE_HASH(s, s->ins_h, s->window[s->strstart + 1]);
 #if MIN_MATCH != 3
                 Call UPDATE_HASH() MIN_MATCH-3 more times
 #endif
@@ -1901,7 +1885,7 @@ local block_state deflate_fast(s, flush)
         } else {
             /* No match, output a literal byte */
             Tracevv((stderr,"%c", s->window[s->strstart]));
-            _tr_tally_lit (s, s->window[s->strstart], bflush);
+            _tr_tally_lit(s, s->window[s->strstart], bflush);
             s->lookahead--;
             s->strstart++;
         }
@@ -1912,7 +1896,7 @@ local block_state deflate_fast(s, flush)
         FLUSH_BLOCK(s, 1);
         return finish_done;
     }
-    if (s->last_lit)
+    if (s->sym_next)
         FLUSH_BLOCK(s, 0);
     return block_done;
 }
@@ -1923,10 +1907,7 @@ local block_state deflate_fast(s, flush)
  * evaluation for matches: a match is finally adopted only if there is
  * no better match at the next window position.
  */
-local block_state deflate_slow(s, flush)
-    deflate_state *s;
-    int flush;
-{
+local block_state deflate_slow(deflate_state *s, int flush) {
     IPos hash_head;          /* head of hash chain */
     int bflush;              /* set if current block must be flushed */
 
@@ -1945,7 +1926,7 @@ local block_state deflate_slow(s, flush)
             if (s->lookahead == 0) break; /* flush the current block */
         }
 
-        /* Insert the string window[strstart .. strstart+2] in the
+        /* Insert the string window[strstart .. strstart + 2] in the
          * dictionary, and set hash_head to the head of the hash chain:
          */
         hash_head = NIL;
@@ -1987,17 +1968,17 @@ local block_state deflate_slow(s, flush)
             uInt max_insert = s->strstart + s->lookahead - MIN_MATCH;
             /* Do not insert strings in hash table beyond this. */
 
-            check_match(s, s->strstart-1, s->prev_match, s->prev_length);
+            check_match(s, s->strstart - 1, s->prev_match, s->prev_length);
 
-            _tr_tally_dist(s, s->strstart -1 - s->prev_match,
+            _tr_tally_dist(s, s->strstart - 1 - s->prev_match,
                            s->prev_length - MIN_MATCH, bflush);
 
             /* Insert in hash table all strings up to the end of the match.
-             * strstart-1 and strstart are already inserted. If there is not
+             * strstart - 1 and strstart are already inserted. If there is not
              * enough lookahead, the last two strings are not inserted in
              * the hash table.
              */
-            s->lookahead -= s->prev_length-1;
+            s->lookahead -= s->prev_length - 1;
             s->prev_length -= 2;
             do {
                 if (++s->strstart <= max_insert) {
@@ -2015,8 +1996,8 @@ local block_state deflate_slow(s, flush)
              * single literal. If there was a match but the current match
              * is longer, truncate the previous match to a single literal.
              */
-            Tracevv((stderr,"%c", s->window[s->strstart-1]));
-            _tr_tally_lit(s, s->window[s->strstart-1], bflush);
+            Tracevv((stderr,"%c", s->window[s->strstart - 1]));
+            _tr_tally_lit(s, s->window[s->strstart - 1], bflush);
             if (bflush) {
                 FLUSH_BLOCK_ONLY(s, 0);
             }
@@ -2034,8 +2015,8 @@ local block_state deflate_slow(s, flush)
     }
     Assert (flush != Z_NO_FLUSH, "no flush?");
     if (s->match_available) {
-        Tracevv((stderr,"%c", s->window[s->strstart-1]));
-        _tr_tally_lit(s, s->window[s->strstart-1], bflush);
+        Tracevv((stderr,"%c", s->window[s->strstart - 1]));
+        _tr_tally_lit(s, s->window[s->strstart - 1], bflush);
         s->match_available = 0;
     }
     s->insert = s->strstart < MIN_MATCH-1 ? s->strstart : MIN_MATCH-1;
@@ -2043,7 +2024,7 @@ local block_state deflate_slow(s, flush)
         FLUSH_BLOCK(s, 1);
         return finish_done;
     }
-    if (s->last_lit)
+    if (s->sym_next)
         FLUSH_BLOCK(s, 0);
     return block_done;
 }
@@ -2054,10 +2035,7 @@ local block_state deflate_slow(s, flush)
  * one.  Do not maintain a hash table.  (It will be regenerated if this run of
  * deflate switches away from Z_RLE.)
  */
-local block_state deflate_rle(s, flush)
-    deflate_state *s;
-    int flush;
-{
+local block_state deflate_rle(deflate_state *s, int flush) {
     int bflush;             /* set if current block must be flushed */
     uInt prev;              /* byte at distance one to match */
     Bytef *scan, *strend;   /* scan goes up to strend for length of run */
@@ -2092,7 +2070,8 @@ local block_state deflate_rle(s, flush)
                 if (s->match_length > s->lookahead)
                     s->match_length = s->lookahead;
             }
-            Assert(scan <= s->window+(uInt)(s->window_size-1), "wild scan");
+            Assert(scan <= s->window + (uInt)(s->window_size - 1),
+                   "wild scan");
         }
 
         /* Emit match if have run of MIN_MATCH or longer, else emit literal */
@@ -2107,7 +2086,7 @@ local block_state deflate_rle(s, flush)
         } else {
             /* No match, output a literal byte */
             Tracevv((stderr,"%c", s->window[s->strstart]));
-            _tr_tally_lit (s, s->window[s->strstart], bflush);
+            _tr_tally_lit(s, s->window[s->strstart], bflush);
             s->lookahead--;
             s->strstart++;
         }
@@ -2118,7 +2097,7 @@ local block_state deflate_rle(s, flush)
         FLUSH_BLOCK(s, 1);
         return finish_done;
     }
-    if (s->last_lit)
+    if (s->sym_next)
         FLUSH_BLOCK(s, 0);
     return block_done;
 }
@@ -2127,10 +2106,7 @@ local block_state deflate_rle(s, flush)
  * For Z_HUFFMAN_ONLY, do not look for matches.  Do not maintain a hash table.
  * (It will be regenerated if this run of deflate switches away from Huffman.)
  */
-local block_state deflate_huff(s, flush)
-    deflate_state *s;
-    int flush;
-{
+local block_state deflate_huff(deflate_state *s, int flush) {
     int bflush;             /* set if current block must be flushed */
 
     for (;;) {
@@ -2147,7 +2123,7 @@ local block_state deflate_huff(s, flush)
         /* Output a literal byte */
         s->match_length = 0;
         Tracevv((stderr,"%c", s->window[s->strstart]));
-        _tr_tally_lit (s, s->window[s->strstart], bflush);
+        _tr_tally_lit(s, s->window[s->strstart], bflush);
         s->lookahead--;
         s->strstart++;
         if (bflush) FLUSH_BLOCK(s, 0);
@@ -2157,7 +2133,7 @@ local block_state deflate_huff(s, flush)
         FLUSH_BLOCK(s, 1);
         return finish_done;
     }
-    if (s->last_lit)
+    if (s->sym_next)
         FLUSH_BLOCK(s, 0);
     return block_done;
 }
diff --git a/zlib/deflate.h b/zlib/deflate.h
index 360ca23..300c6ad 100644
--- a/zlib/deflate.h
+++ b/zlib/deflate.h
@@ -1,5 +1,5 @@
 /* deflate.h -- internal compression state
- * Copyright (C) 1995-2016 Jean-loup Gailly
+ * Copyright (C) 1995-2024 Jean-loup Gailly
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -8,7 +8,7 @@
    subject to change. Applications should only use zlib.h.
  */
 
-/* @(#) $Id: deflate.h,v 1.1.1.2 2002/03/11 21:53:23 tromey Exp $ */
+/* @(#) $Id$ */
 
 #ifndef DEFLATE_H
 #define DEFLATE_H
@@ -23,6 +23,10 @@
 #  define GZIP
 #endif
 
+/* define LIT_MEM to slightly increase the speed of deflate (order 1% to 2%) at
+   the cost of a larger memory footprint */
+/* #define LIT_MEM */
+
 /* ===========================================================================
  * Internal compression state.
  */
@@ -217,7 +221,14 @@ typedef struct internal_state {
     /* Depth of each subtree used as tie breaker for trees of equal frequency
      */
 
-    uchf *l_buf;          /* buffer for literals or lengths */
+#ifdef LIT_MEM
+#   define LIT_BUFS 5
+    ushf *d_buf;          /* buffer for distances */
+    uchf *l_buf;          /* buffer for literals/lengths */
+#else
+#   define LIT_BUFS 4
+    uchf *sym_buf;        /* buffer for distances and literals/lengths */
+#endif
 
     uInt  lit_bufsize;
     /* Size of match buffer for literals/lengths.  There are 4 reasons for
@@ -239,13 +250,8 @@ typedef struct internal_state {
      *   - I can't count above 4
      */
 
-    uInt last_lit;      /* running index in l_buf */
-
-    ushf *d_buf;
-    /* Buffer for distances. To simplify the code, d_buf and l_buf have
-     * the same number of elements. To use different lengths, an extra flag
-     * array would be necessary.
-     */
+    uInt sym_next;      /* running index in symbol buffer */
+    uInt sym_end;       /* symbol table full when sym_next reaches this */
 
     ulg opt_len;        /* bit length of current block with optimal trees */
     ulg static_len;     /* bit length of current block with static trees */
@@ -296,14 +302,14 @@ typedef struct internal_state {
    memory checker errors from longest match routines */
 
         /* in trees.c */
-void ZLIB_INTERNAL _tr_init OF((deflate_state *s));
-int ZLIB_INTERNAL _tr_tally OF((deflate_state *s, unsigned dist, unsigned lc));
-void ZLIB_INTERNAL _tr_flush_block OF((deflate_state *s, charf *buf,
-                        ulg stored_len, int last));
-void ZLIB_INTERNAL _tr_flush_bits OF((deflate_state *s));
-void ZLIB_INTERNAL _tr_align OF((deflate_state *s));
-void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf,
-                        ulg stored_len, int last));
+void ZLIB_INTERNAL _tr_init(deflate_state *s);
+int ZLIB_INTERNAL _tr_tally(deflate_state *s, unsigned dist, unsigned lc);
+void ZLIB_INTERNAL _tr_flush_block(deflate_state *s, charf *buf,
+                                   ulg stored_len, int last);
+void ZLIB_INTERNAL _tr_flush_bits(deflate_state *s);
+void ZLIB_INTERNAL _tr_align(deflate_state *s);
+void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf,
+                                    ulg stored_len, int last);
 
 #define d_code(dist) \
    ((dist) < 256 ? _dist_code[dist] : _dist_code[256+((dist)>>7)])
@@ -323,23 +329,45 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf,
   extern const uch ZLIB_INTERNAL _dist_code[];
 #endif
 
+#ifdef LIT_MEM
+# define _tr_tally_lit(s, c, flush) \
+  { uch cc = (c); \
+    s->d_buf[s->sym_next] = 0; \
+    s->l_buf[s->sym_next++] = cc; \
+    s->dyn_ltree[cc].Freq++; \
+    flush = (s->sym_next == s->sym_end); \
+   }
+# define _tr_tally_dist(s, distance, length, flush) \
+  { uch len = (uch)(length); \
+    ush dist = (ush)(distance); \
+    s->d_buf[s->sym_next] = dist; \
+    s->l_buf[s->sym_next++] = len; \
+    dist--; \
+    s->dyn_ltree[_length_code[len]+LITERALS+1].Freq++; \
+    s->dyn_dtree[d_code(dist)].Freq++; \
+    flush = (s->sym_next == s->sym_end); \
+  }
+#else
 # define _tr_tally_lit(s, c, flush) \
   { uch cc = (c); \
-    s->d_buf[s->last_lit] = 0; \
-    s->l_buf[s->last_lit++] = cc; \
+    s->sym_buf[s->sym_next++] = 0; \
+    s->sym_buf[s->sym_next++] = 0; \
+    s->sym_buf[s->sym_next++] = cc; \
     s->dyn_ltree[cc].Freq++; \
-    flush = (s->last_lit == s->lit_bufsize-1); \
+    flush = (s->sym_next == s->sym_end); \
    }
 # define _tr_tally_dist(s, distance, length, flush) \
   { uch len = (uch)(length); \
     ush dist = (ush)(distance); \
-    s->d_buf[s->last_lit] = dist; \
-    s->l_buf[s->last_lit++] = len; \
+    s->sym_buf[s->sym_next++] = (uch)dist; \
+    s->sym_buf[s->sym_next++] = (uch)(dist >> 8); \
+    s->sym_buf[s->sym_next++] = len; \
     dist--; \
     s->dyn_ltree[_length_code[len]+LITERALS+1].Freq++; \
     s->dyn_dtree[d_code(dist)].Freq++; \
-    flush = (s->last_lit == s->lit_bufsize-1); \
+    flush = (s->sym_next == s->sym_end); \
   }
+#endif
 #else
 # define _tr_tally_lit(s, c, flush) flush = _tr_tally(s, 0, c)
 # define _tr_tally_dist(s, distance, length, flush) \
diff --git a/zlib/doc/algorithm.txt b/zlib/doc/algorithm.txt
index c97f495..029e5a3 100644
--- a/zlib/doc/algorithm.txt
+++ b/zlib/doc/algorithm.txt
@@ -77,7 +77,7 @@ table took no time (and if you had infinite memory), then there would only
 be a first level table to cover all the way to the longest code.  However,
 building the table ends up taking a lot longer for more bits since short
 codes are replicated many times in such a table.  What inflate() does is
-simply to make the number of bits in the first table a variable, and  then
+simply to make the number of bits in the first table a variable, and then
 to set that variable for the maximum speed.
 
 For inflate, which has 286 possible codes for the literal/length tree, the size
diff --git a/zlib/doc/crc-doc.1.0.pdf b/zlib/doc/crc-doc.1.0.pdf
new file mode 100644
index 0000000..d6942ec
--- /dev/null
+++ b/zlib/doc/crc-doc.1.0.pdf
diff --git a/zlib/doc/txtvsbin.txt b/zlib/doc/txtvsbin.txt
index 3d0f063..2a901ea 100644
--- a/zlib/doc/txtvsbin.txt
+++ b/zlib/doc/txtvsbin.txt
@@ -38,15 +38,15 @@ The Algorithm
 
 The algorithm works by dividing the set of bytecodes [0..255] into three
 categories:
-- The white list of textual bytecodes:
+- The allow list of textual bytecodes:
   9 (TAB), 10 (LF), 13 (CR), 32 (SPACE) to 255.
 - The gray list of tolerated bytecodes:
   7 (BEL), 8 (BS), 11 (VT), 12 (FF), 26 (SUB), 27 (ESC).
-- The black list of undesired, non-textual bytecodes:
+- The block list of undesired, non-textual bytecodes:
   0 (NUL) to 6, 14 to 31.
 
-If a file contains at least one byte that belongs to the white list and
-no byte that belongs to the black list, then the file is categorized as
+If a file contains at least one byte that belongs to the allow list and
+no byte that belongs to the block list, then the file is categorized as
 plain text; otherwise, it is categorized as binary.  (The boundary case,
 when the file is empty, automatically falls into the latter category.)
 
@@ -84,9 +84,9 @@ consistent results, regardless what alphabet encoding is being used.
 results on a text encoded, say, using ISO-8859-16 versus UTF-8.)
 
 There is an extra category of plain text files that are "polluted" with
-one or more black-listed codes, either by mistake or by peculiar design
+one or more block-listed codes, either by mistake or by peculiar design
 considerations.  In such cases, a scheme that tolerates a small fraction
-of black-listed codes would provide an increased recall (i.e. more true
+of block-listed codes would provide an increased recall (i.e. more true
 positives).  This, however, incurs a reduced precision overall, since
 false positives are more likely to appear in binary files that contain
 large chunks of textual data.  Furthermore, "polluted" plain text should
diff --git a/zlib/examples/README.examples b/zlib/examples/README.examples
index 56a3171..e3a4b88 100644
--- a/zlib/examples/README.examples
+++ b/zlib/examples/README.examples
@@ -34,6 +34,10 @@ gzlog.h
       and deflateSetDictionary()
     - illustrates use of a gzip header extra field
 
+gznorm.c
+    normalize a gzip file by combining members into a single member
+    - demonstrates how to concatenate deflate streams using Z_BLOCK
+
 zlib_how.html
     painfully comprehensive description of zpipe.c (see below)
     - describes in excruciating detail the use of deflate() and inflate()
@@ -44,6 +48,7 @@ zpipe.c
     - deeply commented in zlib_how.html (see above)
 
 zran.c
+zran.h
     index a zlib or gzip stream and randomly access it
     - illustrates the use of Z_BLOCK, inflatePrime(), and
       inflateSetDictionary() to provide random access
diff --git a/zlib/examples/enough.c b/zlib/examples/enough.c
index b991144..8a3cade 100644
--- a/zlib/examples/enough.c
+++ b/zlib/examples/enough.c
@@ -1,7 +1,7 @@
 /* enough.c -- determine the maximum size of inflate's Huffman code tables over
- * all possible valid and complete Huffman codes, subject to a length limit.
- * Copyright (C) 2007, 2008, 2012 Mark Adler
- * Version 1.4  18 August 2012  Mark Adler
+ * all possible valid and complete prefix codes, subject to a length limit.
+ * Copyright (C) 2007, 2008, 2012, 2018 Mark Adler
+ * Version 1.5  5 August 2018  Mark Adler
  */
 
 /* Version history:
@@ -17,101 +17,107 @@
    1.4  18 Aug 2012  Avoid shifts more than bits in type (caused endless loop!)
                      Clean up comparisons of different types
                      Clean up code indentation
+   1.5   5 Aug 2018  Clean up code style, formatting, and comments
+                     Show all the codes for the maximum, and only the maximum
  */
 
 /*
-   Examine all possible Huffman codes for a given number of symbols and a
-   maximum code length in bits to determine the maximum table size for zilb's
-   inflate.  Only complete Huffman codes are counted.
+   Examine all possible prefix codes for a given number of symbols and a
+   maximum code length in bits to determine the maximum table size for zlib's
+   inflate. Only complete prefix codes are counted.
 
    Two codes are considered distinct if the vectors of the number of codes per
-   length are not identical.  So permutations of the symbol assignments result
+   length are not identical. So permutations of the symbol assignments result
    in the same code for the counting, as do permutations of the assignments of
    the bit values to the codes (i.e. only canonical codes are counted).
 
    We build a code from shorter to longer lengths, determining how many symbols
-   are coded at each length.  At each step, we have how many symbols remain to
+   are coded at each length. At each step, we have how many symbols remain to
    be coded, what the last code length used was, and how many bit patterns of
    that length remain unused. Then we add one to the code length and double the
-   number of unused patterns to graduate to the next code length.  We then
+   number of unused patterns to graduate to the next code length. We then
    assign all portions of the remaining symbols to that code length that
-   preserve the properties of a correct and eventually complete code.  Those
+   preserve the properties of a correct and eventually complete code. Those
    properties are: we cannot use more bit patterns than are available; and when
-   all the symbols are used, there are exactly zero possible bit patterns
-   remaining.
+   all the symbols are used, there are exactly zero possible bit patterns left
+   unused.
 
    The inflate Huffman decoding algorithm uses two-level lookup tables for
-   speed.  There is a single first-level table to decode codes up to root bits
-   in length (root == 9 in the current inflate implementation).  The table
-   has 1 << root entries and is indexed by the next root bits of input.  Codes
-   shorter than root bits have replicated table entries, so that the correct
-   entry is pointed to regardless of the bits that follow the short code.  If
-   the code is longer than root bits, then the table entry points to a second-
-   level table.  The size of that table is determined by the longest code with
-   that root-bit prefix.  If that longest code has length len, then the table
-   has size 1 << (len - root), to index the remaining bits in that set of
-   codes.  Each subsequent root-bit prefix then has its own sub-table.  The
-   total number of table entries required by the code is calculated
-   incrementally as the number of codes at each bit length is populated.  When
-   all of the codes are shorter than root bits, then root is reduced to the
-   longest code length, resulting in a single, smaller, one-level table.
+   speed. There is a single first-level table to decode codes up to root bits
+   in length (root == 9 for literal/length codes and root == 6 for distance
+   codes, in the current inflate implementation). The base table has 1 << root
+   entries and is indexed by the next root bits of input. Codes shorter than
+   root bits have replicated table entries, so that the correct entry is
+   pointed to regardless of the bits that follow the short code. If the code is
+   longer than root bits, then the table entry points to a second-level table.
+   The size of that table is determined by the longest code with that root-bit
+   prefix. If that longest code has length len, then the table has size 1 <<
+   (len - root), to index the remaining bits in that set of codes. Each
+   subsequent root-bit prefix then has its own sub-table. The total number of
+   table entries required by the code is calculated incrementally as the number
+   of codes at each bit length is populated. When all of the codes are shorter
+   than root bits, then root is reduced to the longest code length, resulting
+   in a single, smaller, one-level table.
 
    The inflate algorithm also provides for small values of root (relative to
    the log2 of the number of symbols), where the shortest code has more bits
-   than root.  In that case, root is increased to the length of the shortest
-   code.  This program, by design, does not handle that case, so it is verified
-   that the number of symbols is less than 2^(root + 1).
+   than root. In that case, root is increased to the length of the shortest
+   code. This program, by design, does not handle that case, so it is verified
+   that the number of symbols is less than 1 << (root + 1).
 
    In order to speed up the examination (by about ten orders of magnitude for
    the default arguments), the intermediate states in the build-up of a code
-   are remembered and previously visited branches are pruned.  The memory
+   are remembered and previously visited branches are pruned. The memory
    required for this will increase rapidly with the total number of symbols and
-   the maximum code length in bits.  However this is a very small price to pay
+   the maximum code length in bits. However this is a very small price to pay
    for the vast speedup.
 
-   First, all of the possible Huffman codes are counted, and reachable
+   First, all of the possible prefix codes are counted, and reachable
    intermediate states are noted by a non-zero count in a saved-results array.
    Second, the intermediate states that lead to (root + 1) bit or longer codes
    are used to look at all sub-codes from those junctures for their inflate
-   memory usage.  (The amount of memory used is not affected by the number of
+   memory usage. (The amount of memory used is not affected by the number of
    codes of root bits or less in length.)  Third, the visited states in the
    construction of those sub-codes and the associated calculation of the table
    size is recalled in order to avoid recalculating from the same juncture.
    Beginning the code examination at (root + 1) bit codes, which is enabled by
    identifying the reachable nodes, accounts for about six of the orders of
-   magnitude of improvement for the default arguments.  About another four
-   orders of magnitude come from not revisiting previous states.  Out of
-   approximately 2x10^16 possible Huffman codes, only about 2x10^6 sub-codes
+   magnitude of improvement for the default arguments. About another four
+   orders of magnitude come from not revisiting previous states. Out of
+   approximately 2x10^16 possible prefix codes, only about 2x10^6 sub-codes
    need to be examined to cover all of the possible table memory usage cases
    for the default arguments of 286 symbols limited to 15-bit codes.
 
-   Note that an unsigned long long type is used for counting.  It is quite easy
-   to exceed the capacity of an eight-byte integer with a large number of
-   symbols and a large maximum code length, so multiple-precision arithmetic
-   would need to replace the unsigned long long arithmetic in that case.  This
-   program will abort if an overflow occurs.  The big_t type identifies where
-   the counting takes place.
-
-   An unsigned long long type is also used for calculating the number of
-   possible codes remaining at the maximum length.  This limits the maximum
-   code length to the number of bits in a long long minus the number of bits
-   needed to represent the symbols in a flat code.  The code_t type identifies
-   where the bit pattern counting takes place.
+   Note that the uintmax_t type is used for counting. It is quite easy to
+   exceed the capacity of an eight-byte integer with a large number of symbols
+   and a large maximum code length, so multiple-precision arithmetic would need
+   to replace the integer arithmetic in that case. This program will abort if
+   an overflow occurs. The big_t type identifies where the counting takes
+   place.
+
+   The uintmax_t type is also used for calculating the number of possible codes
+   remaining at the maximum length. This limits the maximum code length to the
+   number of bits in a long long minus the number of bits needed to represent
+   the symbols in a flat code. The code_t type identifies where the bit-pattern
+   counting takes place.
  */
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <stdarg.h>
+#include <stdint.h>
 #include <assert.h>
 
 #define local static
 
-/* special data types */
-typedef unsigned long long big_t;   /* type for code counting */
-typedef unsigned long long code_t;  /* type for bit pattern counting */
-struct tab {                        /* type for been here check */
-    size_t len;         /* length of bit vector in char's */
-    char *vec;          /* allocated bit vector */
+// Special data types.
+typedef uintmax_t big_t;    // type for code counting
+#define PRIbig "ju"         // printf format for big_t
+typedef uintmax_t code_t;   // type for bit pattern counting
+struct tab {                // type for been-here check
+    size_t len;             // allocated length of bit vector in octets
+    char *vec;              // allocated bit vector
 };
 
 /* The array for saving results, num[], is indexed with this triplet:
@@ -126,447 +132,466 @@ struct tab {                        /* type for been here check */
       left: 2..syms - 1, but only the evens (so syms == 8 -> 2, 4, 6)
       len: 1..max - 1 (max == maximum code length in bits)
 
-   syms == 2 is not saved since that immediately leads to a single code.  left
+   syms == 2 is not saved since that immediately leads to a single code. left
    must be even, since it represents the number of available bit patterns at
-   the current length, which is double the number at the previous length.
-   left ends at syms-1 since left == syms immediately results in a single code.
+   the current length, which is double the number at the previous length. left
+   ends at syms-1 since left == syms immediately results in a single code.
    (left > sym is not allowed since that would result in an incomplete code.)
    len is less than max, since the code completes immediately when len == max.
 
-   The offset into the array is calculated for the three indices with the
-   first one (syms) being outermost, and the last one (len) being innermost.
-   We build the array with length max-1 lists for the len index, with syms-3
-   of those for each symbol.  There are totsym-2 of those, with each one
-   varying in length as a function of sym.  See the calculation of index in
-   count() for the index, and the calculation of size in main() for the size
-   of the array.
+   The offset into the array is calculated for the three indices with the first
+   one (syms) being outermost, and the last one (len) being innermost. We build
+   the array with length max-1 lists for the len index, with syms-3 of those
+   for each symbol. There are totsym-2 of those, with each one varying in
+   length as a function of sym. See the calculation of index in map() for the
+   index, and the calculation of size in main() for the size of the array.
 
    For the deflate example of 286 symbols limited to 15-bit codes, the array
-   has 284,284 entries, taking up 2.17 MB for an 8-byte big_t.  More than
-   half of the space allocated for saved results is actually used -- not all
-   possible triplets are reached in the generation of valid Huffman codes.
+   has 284,284 entries, taking up 2.17 MB for an 8-byte big_t. More than half
+   of the space allocated for saved results is actually used -- not all
+   possible triplets are reached in the generation of valid prefix codes.
  */
 
 /* The array for tracking visited states, done[], is itself indexed identically
    to the num[] array as described above for the (syms, left, len) triplet.
    Each element in the array is further indexed by the (mem, rem) doublet,
    where mem is the amount of inflate table space used so far, and rem is the
-   remaining unused entries in the current inflate sub-table.  Each indexed
+   remaining unused entries in the current inflate sub-table. Each indexed
    element is simply one bit indicating whether the state has been visited or
-   not.  Since the ranges for mem and rem are not known a priori, each bit
+   not. Since the ranges for mem and rem are not known a priori, each bit
    vector is of a variable size, and grows as needed to accommodate the visited
-   states.  mem and rem are used to calculate a single index in a triangular
-   array.  Since the range of mem is expected in the default case to be about
+   states. mem and rem are used to calculate a single index in a triangular
+   array. Since the range of mem is expected in the default case to be about
    ten times larger than the range of rem, the array is skewed to reduce the
-   memory usage, with eight times the range for mem than for rem.  See the
-   calculations for offset and bit in beenhere() for the details.
+   memory usage, with eight times the range for mem than for rem. See the
+   calculations for offset and bit in been_here() for the details.
 
    For the deflate example of 286 symbols limited to 15-bit codes, the bit
-   vectors grow to total approximately 21 MB, in addition to the 4.3 MB done[]
-   array itself.
+   vectors grow to total 5.5 MB, in addition to the 4.3 MB done array itself.
  */
 
-/* Globals to avoid propagating constants or constant pointers recursively */
-local int max;          /* maximum allowed bit length for the codes */
-local int root;         /* size of base code table in bits */
-local int large;        /* largest code table so far */
-local size_t size;      /* number of elements in num and done */
-local int *code;        /* number of symbols assigned to each bit length */
-local big_t *num;       /* saved results array for code counting */
-local struct tab *done; /* states already evaluated array */
-
-/* Index function for num[] and done[] */
-#define INDEX(i,j,k) (((size_t)((i-1)>>1)*((i-2)>>1)+(j>>1)-1)*(max-1)+k-1)
-
-/* Free allocated space.  Uses globals code, num, and done. */
-local void cleanup(void)
-{
-    size_t n;
-
-    if (done != NULL) {
-        for (n = 0; n < size; n++)
-            if (done[n].len)
-                free(done[n].vec);
-        free(done);
+// Type for a variable-length, allocated string.
+typedef struct {
+    char *str;          // pointer to allocated string
+    size_t size;        // size of allocation
+    size_t len;         // length of string, not including terminating zero
+} string_t;
+
+// Clear a string_t.
+local void string_clear(string_t *s) {
+    s->str[0] = 0;
+    s->len = 0;
+}
+
+// Initialize a string_t.
+local void string_init(string_t *s) {
+    s->size = 16;
+    s->str = malloc(s->size);
+    assert(s->str != NULL && "out of memory");
+    string_clear(s);
+}
+
+// Release the allocation of a string_t.
+local void string_free(string_t *s) {
+    free(s->str);
+    s->str = NULL;
+    s->size = 0;
+    s->len = 0;
+}
+
+// Save the results of printf with fmt and the subsequent argument list to s.
+// Each call appends to s. The allocated space for s is increased as needed.
+local void string_printf(string_t *s, char *fmt, ...) {
+    va_list ap;
+    va_start(ap, fmt);
+    size_t len = s->len;
+    int ret = vsnprintf(s->str + len, s->size - len, fmt, ap);
+    assert(ret >= 0 && "out of memory");
+    s->len += ret;
+    if (s->size < s->len + 1) {
+        do {
+            s->size <<= 1;
+            assert(s->size != 0 && "overflow");
+        } while (s->size < s->len + 1);
+        s->str = realloc(s->str, s->size);
+        assert(s->str != NULL && "out of memory");
+        vsnprintf(s->str + len, s->size - len, fmt, ap);
     }
-    if (num != NULL)
-        free(num);
-    if (code != NULL)
-        free(code);
+    va_end(ap);
 }
 
-/* Return the number of possible Huffman codes using bit patterns of lengths
-   len through max inclusive, coding syms symbols, with left bit patterns of
-   length len unused -- return -1 if there is an overflow in the counting.
-   Keep a record of previous results in num to prevent repeating the same
-   calculation.  Uses the globals max and num. */
-local big_t count(int syms, int len, int left)
-{
-    big_t sum;          /* number of possible codes from this juncture */
-    big_t got;          /* value returned from count() */
-    int least;          /* least number of syms to use at this juncture */
-    int most;           /* most number of syms to use at this juncture */
-    int use;            /* number of bit patterns to use in next call */
-    size_t index;       /* index of this case in *num */
-
-    /* see if only one possible code */
+// Globals to avoid propagating constants or constant pointers recursively.
+struct {
+    int max;            // maximum allowed bit length for the codes
+    int root;           // size of base code table in bits
+    int large;          // largest code table so far
+    size_t size;        // number of elements in num and done
+    big_t tot;          // total number of codes with maximum tables size
+    string_t out;       // display of subcodes for maximum tables size
+    int *code;          // number of symbols assigned to each bit length
+    big_t *num;         // saved results array for code counting
+    struct tab *done;   // states already evaluated array
+} g;
+
+// Index function for num[] and done[].
+local inline size_t map(int syms, int left, int len) {
+    return ((size_t)((syms - 1) >> 1) * ((syms - 2) >> 1) +
+            (left >> 1) - 1) * (g.max - 1) +
+           len - 1;
+}
+
+// Free allocated space in globals.
+local void cleanup(void) {
+    if (g.done != NULL) {
+        for (size_t n = 0; n < g.size; n++)
+            if (g.done[n].len)
+                free(g.done[n].vec);
+        g.size = 0;
+        free(g.done);   g.done = NULL;
+    }
+    free(g.num);    g.num = NULL;
+    free(g.code);   g.code = NULL;
+    string_free(&g.out);
+}
+
+// Return the number of possible prefix codes using bit patterns of lengths len
+// through max inclusive, coding syms symbols, with left bit patterns of length
+// len unused -- return -1 if there is an overflow in the counting. Keep a
+// record of previous results in num to prevent repeating the same calculation.
+local big_t count(int syms, int left, int len) {
+    // see if only one possible code
     if (syms == left)
         return 1;
 
-    /* note and verify the expected state */
-    assert(syms > left && left > 0 && len < max);
+    // note and verify the expected state
+    assert(syms > left && left > 0 && len < g.max);
 
-    /* see if we've done this one already */
-    index = INDEX(syms, left, len);
-    got = num[index];
+    // see if we've done this one already
+    size_t index = map(syms, left, len);
+    big_t got = g.num[index];
     if (got)
-        return got;         /* we have -- return the saved result */
+        return got;         // we have -- return the saved result
 
-    /* we need to use at least this many bit patterns so that the code won't be
-       incomplete at the next length (more bit patterns than symbols) */
-    least = (left << 1) - syms;
+    // we need to use at least this many bit patterns so that the code won't be
+    // incomplete at the next length (more bit patterns than symbols)
+    int least = (left << 1) - syms;
     if (least < 0)
         least = 0;
 
-    /* we can use at most this many bit patterns, lest there not be enough
-       available for the remaining symbols at the maximum length (if there were
-       no limit to the code length, this would become: most = left - 1) */
-    most = (((code_t)left << (max - len)) - syms) /
-            (((code_t)1 << (max - len)) - 1);
+    // we can use at most this many bit patterns, lest there not be enough
+    // available for the remaining symbols at the maximum length (if there were
+    // no limit to the code length, this would become: most = left - 1)
+    int most = (((code_t)left << (g.max - len)) - syms) /
+               (((code_t)1 << (g.max - len)) - 1);
 
-    /* count all possible codes from this juncture and add them up */
-    sum = 0;
-    for (use = least; use <= most; use++) {
-        got = count(syms - use, len + 1, (left - use) << 1);
+    // count all possible codes from this juncture and add them up
+    big_t sum = 0;
+    for (int use = least; use <= most; use++) {
+        got = count(syms - use, (left - use) << 1, len + 1);
         sum += got;
-        if (got == (big_t)0 - 1 || sum < got)   /* overflow */
-            return (big_t)0 - 1;
+        if (got == (big_t)-1 || sum < got)      // overflow
+            return (big_t)-1;
     }
 
-    /* verify that all recursive calls are productive */
+    // verify that all recursive calls are productive
     assert(sum != 0);
 
-    /* save the result and return it */
-    num[index] = sum;
+    // save the result and return it
+    g.num[index] = sum;
     return sum;
 }
 
-/* Return true if we've been here before, set to true if not.  Set a bit in a
-   bit vector to indicate visiting this state.  Each (syms,len,left) state
-   has a variable size bit vector indexed by (mem,rem).  The bit vector is
-   lengthened if needed to allow setting the (mem,rem) bit. */
-local int beenhere(int syms, int len, int left, int mem, int rem)
-{
-    size_t index;       /* index for this state's bit vector */
-    size_t offset;      /* offset in this state's bit vector */
-    int bit;            /* mask for this state's bit */
-    size_t length;      /* length of the bit vector in bytes */
-    char *vector;       /* new or enlarged bit vector */
-
-    /* point to vector for (syms,left,len), bit in vector for (mem,rem) */
-    index = INDEX(syms, left, len);
-    mem -= 1 << root;
-    offset = (mem >> 3) + rem;
+// Return true if we've been here before, set to true if not. Set a bit in a
+// bit vector to indicate visiting this state. Each (syms,len,left) state has a
+// variable size bit vector indexed by (mem,rem). The bit vector is lengthened
+// as needed to allow setting the (mem,rem) bit.
+local int been_here(int syms, int left, int len, int mem, int rem) {
+    // point to vector for (syms,left,len), bit in vector for (mem,rem)
+    size_t index = map(syms, left, len);
+    mem -= 1 << g.root;             // mem always includes the root table
+    mem >>= 1;                      // mem and rem are always even
+    rem >>= 1;
+    size_t offset = (mem >> 3) + rem;
     offset = ((offset * (offset + 1)) >> 1) + rem;
-    bit = 1 << (mem & 7);
+    int bit = 1 << (mem & 7);
 
-    /* see if we've been here */
-    length = done[index].len;
-    if (offset < length && (done[index].vec[offset] & bit) != 0)
-        return 1;       /* done this! */
+    // see if we've been here
+    size_t length = g.done[index].len;
+    if (offset < length && (g.done[index].vec[offset] & bit) != 0)
+        return 1;       // done this!
 
-    /* we haven't been here before -- set the bit to show we have now */
+    // we haven't been here before -- set the bit to show we have now
 
-    /* see if we need to lengthen the vector in order to set the bit */
+    // see if we need to lengthen the vector in order to set the bit
     if (length <= offset) {
-        /* if we have one already, enlarge it, zero out the appended space */
+        // if we have one already, enlarge it, zero out the appended space
+        char *vector;
         if (length) {
             do {
                 length <<= 1;
             } while (length <= offset);
-            vector = realloc(done[index].vec, length);
-            if (vector != NULL)
-                memset(vector + done[index].len, 0, length - done[index].len);
+            vector = realloc(g.done[index].vec, length);
+            assert(vector != NULL && "out of memory");
+            memset(vector + g.done[index].len, 0, length - g.done[index].len);
         }
 
-        /* otherwise we need to make a new vector and zero it out */
+        // otherwise we need to make a new vector and zero it out
         else {
-            length = 1 << (len - root);
+            length = 16;
             while (length <= offset)
                 length <<= 1;
-            vector = calloc(length, sizeof(char));
-        }
-
-        /* in either case, bail if we can't get the memory */
-        if (vector == NULL) {
-            fputs("abort: unable to allocate enough memory\n", stderr);
-            cleanup();
-            exit(1);
+            vector = calloc(length, 1);
+            assert(vector != NULL && "out of memory");
         }
 
-        /* install the new vector */
-        done[index].len = length;
-        done[index].vec = vector;
+        // install the new vector
+        g.done[index].len = length;
+        g.done[index].vec = vector;
     }
 
-    /* set the bit */
-    done[index].vec[offset] |= bit;
+    // set the bit
+    g.done[index].vec[offset] |= bit;
     return 0;
 }
 
-/* Examine all possible codes from the given node (syms, len, left).  Compute
-   the amount of memory required to build inflate's decoding tables, where the
-   number of code structures used so far is mem, and the number remaining in
-   the current sub-table is rem.  Uses the globals max, code, root, large, and
-   done. */
-local void examine(int syms, int len, int left, int mem, int rem)
-{
-    int least;          /* least number of syms to use at this juncture */
-    int most;           /* most number of syms to use at this juncture */
-    int use;            /* number of bit patterns to use in next call */
-
-    /* see if we have a complete code */
+// Examine all possible codes from the given node (syms, len, left). Compute
+// the amount of memory required to build inflate's decoding tables, where the
+// number of code structures used so far is mem, and the number remaining in
+// the current sub-table is rem.
+local void examine(int syms, int left, int len, int mem, int rem) {
+    // see if we have a complete code
     if (syms == left) {
-        /* set the last code entry */
-        code[len] = left;
+        // set the last code entry
+        g.code[len] = left;
 
-        /* complete computation of memory used by this code */
+        // complete computation of memory used by this code
         while (rem < left) {
             left -= rem;
-            rem = 1 << (len - root);
+            rem = 1 << (len - g.root);
             mem += rem;
         }
         assert(rem == left);
 
-        /* if this is a new maximum, show the entries used and the sub-code */
-        if (mem > large) {
-            large = mem;
-            printf("max %d: ", mem);
-            for (use = root + 1; use <= max; use++)
-                if (code[use])
-                    printf("%d[%d] ", code[use], use);
-            putchar('\n');
-            fflush(stdout);
+        // if this is at the maximum, show the sub-code
+        if (mem >= g.large) {
+            // if this is a new maximum, update the maximum and clear out the
+            // printed sub-codes from the previous maximum
+            if (mem > g.large) {
+                g.large = mem;
+                string_clear(&g.out);
+            }
+
+            // compute the starting state for this sub-code
+            syms = 0;
+            left = 1 << g.max;
+            for (int bits = g.max; bits > g.root; bits--) {
+                syms += g.code[bits];
+                left -= g.code[bits];
+                assert((left & 1) == 0);
+                left >>= 1;
+            }
+
+            // print the starting state and the resulting sub-code to g.out
+            string_printf(&g.out, "<%u, %u, %u>:",
+                          syms, g.root + 1, ((1 << g.root) - left) << 1);
+            for (int bits = g.root + 1; bits <= g.max; bits++)
+                if (g.code[bits])
+                    string_printf(&g.out, " %d[%d]", g.code[bits], bits);
+            string_printf(&g.out, "\n");
         }
 
-        /* remove entries as we drop back down in the recursion */
-        code[len] = 0;
+        // remove entries as we drop back down in the recursion
+        g.code[len] = 0;
         return;
     }
 
-    /* prune the tree if we can */
-    if (beenhere(syms, len, left, mem, rem))
+    // prune the tree if we can
+    if (been_here(syms, left, len, mem, rem))
         return;
 
-    /* we need to use at least this many bit patterns so that the code won't be
-       incomplete at the next length (more bit patterns than symbols) */
-    least = (left << 1) - syms;
+    // we need to use at least this many bit patterns so that the code won't be
+    // incomplete at the next length (more bit patterns than symbols)
+    int least = (left << 1) - syms;
     if (least < 0)
         least = 0;
 
-    /* we can use at most this many bit patterns, lest there not be enough
-       available for the remaining symbols at the maximum length (if there were
-       no limit to the code length, this would become: most = left - 1) */
-    most = (((code_t)left << (max - len)) - syms) /
-            (((code_t)1 << (max - len)) - 1);
+    // we can use at most this many bit patterns, lest there not be enough
+    // available for the remaining symbols at the maximum length (if there were
+    // no limit to the code length, this would become: most = left - 1)
+    int most = (((code_t)left << (g.max - len)) - syms) /
+               (((code_t)1 << (g.max - len)) - 1);
 
-    /* occupy least table spaces, creating new sub-tables as needed */
-    use = least;
+    // occupy least table spaces, creating new sub-tables as needed
+    int use = least;
     while (rem < use) {
         use -= rem;
-        rem = 1 << (len - root);
+        rem = 1 << (len - g.root);
         mem += rem;
     }
     rem -= use;
 
-    /* examine codes from here, updating table space as we go */
+    // examine codes from here, updating table space as we go
     for (use = least; use <= most; use++) {
-        code[len] = use;
-        examine(syms - use, len + 1, (left - use) << 1,
-                mem + (rem ? 1 << (len - root) : 0), rem << 1);
+        g.code[len] = use;
+        examine(syms - use, (left - use) << 1, len + 1,
+                mem + (rem ? 1 << (len - g.root) : 0), rem << 1);
         if (rem == 0) {
-            rem = 1 << (len - root);
+            rem = 1 << (len - g.root);
             mem += rem;
         }
         rem--;
     }
 
-    /* remove entries as we drop back down in the recursion */
-    code[len] = 0;
+    // remove entries as we drop back down in the recursion
+    g.code[len] = 0;
 }
 
-/* Look at all sub-codes starting with root + 1 bits.  Look at only the valid
-   intermediate code states (syms, left, len).  For each completed code,
-   calculate the amount of memory required by inflate to build the decoding
-   tables. Find the maximum amount of memory required and show the code that
-   requires that maximum.  Uses the globals max, root, and num. */
-local void enough(int syms)
-{
-    int n;              /* number of remaing symbols for this node */
-    int left;           /* number of unused bit patterns at this length */
-    size_t index;       /* index of this case in *num */
-
-    /* clear code */
-    for (n = 0; n <= max; n++)
-        code[n] = 0;
-
-    /* look at all (root + 1) bit and longer codes */
-    large = 1 << root;              /* base table */
-    if (root < max)                 /* otherwise, there's only a base table */
-        for (n = 3; n <= syms; n++)
-            for (left = 2; left < n; left += 2)
-            {
-                /* look at all reachable (root + 1) bit nodes, and the
-                   resulting codes (complete at root + 2 or more) */
-                index = INDEX(n, left, root + 1);
-                if (root + 1 < max && num[index])       /* reachable node */
-                    examine(n, root + 1, left, 1 << root, 0);
-
-                /* also look at root bit codes with completions at root + 1
-                   bits (not saved in num, since complete), just in case */
-                if (num[index - 1] && n <= left << 1)
-                    examine((n - left) << 1, root + 1, (n - left) << 1,
-                            1 << root, 0);
+// Look at all sub-codes starting with root + 1 bits. Look at only the valid
+// intermediate code states (syms, left, len). For each completed code,
+// calculate the amount of memory required by inflate to build the decoding
+// tables. Find the maximum amount of memory required and show the codes that
+// require that maximum.
+local void enough(int syms) {
+    // clear code
+    for (int n = 0; n <= g.max; n++)
+        g.code[n] = 0;
+
+    // look at all (root + 1) bit and longer codes
+    string_clear(&g.out);           // empty saved results
+    g.large = 1 << g.root;          // base table
+    if (g.root < g.max)             // otherwise, there's only a base table
+        for (int n = 3; n <= syms; n++)
+            for (int left = 2; left < n; left += 2) {
+                // look at all reachable (root + 1) bit nodes, and the
+                // resulting codes (complete at root + 2 or more)
+                size_t index = map(n, left, g.root + 1);
+                if (g.root + 1 < g.max && g.num[index]) // reachable node
+                    examine(n, left, g.root + 1, 1 << g.root, 0);
+
+                // also look at root bit codes with completions at root + 1
+                // bits (not saved in num, since complete), just in case
+                if (g.num[index - 1] && n <= left << 1)
+                    examine((n - left) << 1, (n - left) << 1, g.root + 1,
+                            1 << g.root, 0);
             }
 
-    /* done */
-    printf("done: maximum of %d table entries\n", large);
+    // done
+    printf("maximum of %d table entries for root = %d\n", g.large, g.root);
+    fputs(g.out.str, stdout);
 }
 
-/*
-   Examine and show the total number of possible Huffman codes for a given
-   maximum number of symbols, initial root table size, and maximum code length
-   in bits -- those are the command arguments in that order.  The default
-   values are 286, 9, and 15 respectively, for the deflate literal/length code.
-   The possible codes are counted for each number of coded symbols from two to
-   the maximum.  The counts for each of those and the total number of codes are
-   shown.  The maximum number of inflate table entires is then calculated
-   across all possible codes.  Each new maximum number of table entries and the
-   associated sub-code (starting at root + 1 == 10 bits) is shown.
-
-   To count and examine Huffman codes that are not length-limited, provide a
-   maximum length equal to the number of symbols minus one.
-
-   For the deflate literal/length code, use "enough".  For the deflate distance
-   code, use "enough 30 6".
-
-   This uses the %llu printf format to print big_t numbers, which assumes that
-   big_t is an unsigned long long.  If the big_t type is changed (for example
-   to a multiple precision type), the method of printing will also need to be
-   updated.
- */
-int main(int argc, char **argv)
-{
-    int syms;           /* total number of symbols to code */
-    int n;              /* number of symbols to code for this run */
-    big_t got;          /* return value of count() */
-    big_t sum;          /* accumulated number of codes over n */
-    code_t word;        /* for counting bits in code_t */
-
-    /* set up globals for cleanup() */
-    code = NULL;
-    num = NULL;
-    done = NULL;
-
-    /* get arguments -- default to the deflate literal/length code */
-    syms = 286;
-    root = 9;
-    max = 15;
+// Examine and show the total number of possible prefix codes for a given
+// maximum number of symbols, initial root table size, and maximum code length
+// in bits -- those are the command arguments in that order. The default values
+// are 286, 9, and 15 respectively, for the deflate literal/length code. The
+// possible codes are counted for each number of coded symbols from two to the
+// maximum. The counts for each of those and the total number of codes are
+// shown. The maximum number of inflate table entries is then calculated across
+// all possible codes. Each new maximum number of table entries and the
+// associated sub-code (starting at root + 1 == 10 bits) is shown.
+//
+// To count and examine prefix codes that are not length-limited, provide a
+// maximum length equal to the number of symbols minus one.
+//
+// For the deflate literal/length code, use "enough". For the deflate distance
+// code, use "enough 30 6".
+int main(int argc, char **argv) {
+    // set up globals for cleanup()
+    g.code = NULL;
+    g.num = NULL;
+    g.done = NULL;
+    string_init(&g.out);
+
+    // get arguments -- default to the deflate literal/length code
+    int syms = 286;
+    g.root = 9;
+    g.max = 15;
     if (argc > 1) {
         syms = atoi(argv[1]);
         if (argc > 2) {
-            root = atoi(argv[2]);
+            g.root = atoi(argv[2]);
             if (argc > 3)
-                max = atoi(argv[3]);
+                g.max = atoi(argv[3]);
         }
     }
-    if (argc > 4 || syms < 2 || root < 1 || max < 1) {
+    if (argc > 4 || syms < 2 || g.root < 1 || g.max < 1) {
         fputs("invalid arguments, need: [sym >= 2 [root >= 1 [max >= 1]]]\n",
               stderr);
         return 1;
     }
 
-    /* if not restricting the code length, the longest is syms - 1 */
-    if (max > syms - 1)
-        max = syms - 1;
+    // if not restricting the code length, the longest is syms - 1
+    if (g.max > syms - 1)
+        g.max = syms - 1;
 
-    /* determine the number of bits in a code_t */
-    for (n = 0, word = 1; word; n++, word <<= 1)
-        ;
+    // determine the number of bits in a code_t
+    int bits = 0;
+    for (code_t word = 1; word; word <<= 1)
+        bits++;
 
-    /* make sure that the calculation of most will not overflow */
-    if (max > n || (code_t)(syms - 2) >= (((code_t)0 - 1) >> (max - 1))) {
+    // make sure that the calculation of most will not overflow
+    if (g.max > bits || (code_t)(syms - 2) >= ((code_t)-1 >> (g.max - 1))) {
         fputs("abort: code length too long for internal types\n", stderr);
         return 1;
     }
 
-    /* reject impossible code requests */
-    if ((code_t)(syms - 1) > ((code_t)1 << max) - 1) {
+    // reject impossible code requests
+    if ((code_t)(syms - 1) > ((code_t)1 << g.max) - 1) {
         fprintf(stderr, "%d symbols cannot be coded in %d bits\n",
-                syms, max);
+                syms, g.max);
         return 1;
     }
 
-    /* allocate code vector */
-    code = calloc(max + 1, sizeof(int));
-    if (code == NULL) {
-        fputs("abort: unable to allocate enough memory\n", stderr);
-        return 1;
-    }
+    // allocate code vector
+    g.code = calloc(g.max + 1, sizeof(int));
+    assert(g.code != NULL && "out of memory");
 
-    /* determine size of saved results array, checking for overflows,
-       allocate and clear the array (set all to zero with calloc()) */
-    if (syms == 2)              /* iff max == 1 */
-        num = NULL;             /* won't be saving any results */
+    // determine size of saved results array, checking for overflows,
+    // allocate and clear the array (set all to zero with calloc())
+    if (syms == 2)              // iff max == 1
+        g.num = NULL;           // won't be saving any results
     else {
-        size = syms >> 1;
-        if (size > ((size_t)0 - 1) / (n = (syms - 1) >> 1) ||
-                (size *= n, size > ((size_t)0 - 1) / (n = max - 1)) ||
-                (size *= n, size > ((size_t)0 - 1) / sizeof(big_t)) ||
-                (num = calloc(size, sizeof(big_t))) == NULL) {
-            fputs("abort: unable to allocate enough memory\n", stderr);
-            cleanup();
-            return 1;
-        }
+        g.size = syms >> 1;
+        int n = (syms - 1) >> 1;
+        assert(g.size <= (size_t)-1 / n && "overflow");
+        g.size *= n;
+        n = g.max - 1;
+        assert(g.size <= (size_t)-1 / n && "overflow");
+        g.size *= n;
+        g.num = calloc(g.size, sizeof(big_t));
+        assert(g.num != NULL && "out of memory");
     }
 
-    /* count possible codes for all numbers of symbols, add up counts */
-    sum = 0;
-    for (n = 2; n <= syms; n++) {
-        got = count(n, 1, 2);
+    // count possible codes for all numbers of symbols, add up counts
+    big_t sum = 0;
+    for (int n = 2; n <= syms; n++) {
+        big_t got = count(n, 2, 1);
         sum += got;
-        if (got == (big_t)0 - 1 || sum < got) {     /* overflow */
-            fputs("abort: can't count that high!\n", stderr);
-            cleanup();
-            return 1;
-        }
-        printf("%llu %d-codes\n", got, n);
+        assert(got != (big_t)-1 && sum >= got && "overflow");
     }
-    printf("%llu total codes for 2 to %d symbols", sum, syms);
-    if (max < syms - 1)
-        printf(" (%d-bit length limit)\n", max);
+    printf("%"PRIbig" total codes for 2 to %d symbols", sum, syms);
+    if (g.max < syms - 1)
+        printf(" (%d-bit length limit)\n", g.max);
     else
         puts(" (no length limit)");
 
-    /* allocate and clear done array for beenhere() */
+    // allocate and clear done array for been_here()
     if (syms == 2)
-        done = NULL;
-    else if (size > ((size_t)0 - 1) / sizeof(struct tab) ||
-             (done = calloc(size, sizeof(struct tab))) == NULL) {
-        fputs("abort: unable to allocate enough memory\n", stderr);
-        cleanup();
-        return 1;
+        g.done = NULL;
+    else {
+        g.done = calloc(g.size, sizeof(struct tab));
+        assert(g.done != NULL && "out of memory");
     }
 
-    /* find and show maximum inflate table usage */
-    if (root > max)                 /* reduce root to max length */
-        root = max;
-    if ((code_t)syms < ((code_t)1 << (root + 1)))
+    // find and show maximum inflate table usage
+    if (g.root > g.max)             // reduce root to max length
+        g.root = g.max;
+    if ((code_t)syms < ((code_t)1 << (g.root + 1)))
         enough(syms);
     else
-        puts("cannot handle minimum code lengths > root");
+        fputs("cannot handle minimum code lengths > root", stderr);
 
-    /* done */
+    // done
     cleanup();
     return 0;
 }
diff --git a/zlib/examples/fitblk.c b/zlib/examples/fitblk.c
index c61de5c..723dc00 100644
--- a/zlib/examples/fitblk.c
+++ b/zlib/examples/fitblk.c
@@ -17,7 +17,7 @@
    data in order to determine how much of that input will compress to
    nearly the requested output block size.  The first pass generates
    enough deflate blocks to produce output to fill the requested
-   output size plus a specfied excess amount (see the EXCESS define
+   output size plus a specified excess amount (see the EXCESS define
    below).  The last deflate block may go quite a bit past that, but
    is discarded.  The second pass decompresses and recompresses just
    the compressed data that fit in the requested plus excess sized
@@ -109,7 +109,7 @@ local int recompress(z_streamp inf, z_streamp def)
         if (ret == Z_MEM_ERROR)
             return ret;
 
-        /* compress what was decompresed until done or no room */
+        /* compress what was decompressed until done or no room */
         def->avail_in = RAWLEN - inf->avail_out;
         def->next_in = raw;
         if (inf->avail_out != 0)
@@ -198,7 +198,7 @@ int main(int argc, char **argv)
     if (ret == Z_MEM_ERROR)
         quit("out of memory");
 
-    /* set up for next reocmpression */
+    /* set up for next recompression */
     ret = inflateReset(&inf);
     assert(ret != Z_STREAM_ERROR);
     ret = deflateReset(&def);
diff --git a/zlib/examples/gun.c b/zlib/examples/gun.c
index be44fa5..bea5497 100644
--- a/zlib/examples/gun.c
+++ b/zlib/examples/gun.c
@@ -43,7 +43,7 @@
    gun will also decompress files made by Unix compress, which uses LZW
    compression.  These files are automatically detected by virtue of their
    magic header bytes.  Since the end of Unix compress stream is marked by the
-   end-of-file, they cannot be concantenated.  If a Unix compress stream is
+   end-of-file, they cannot be concatenated.  If a Unix compress stream is
    encountered in an input file, it is the last stream in that file.
 
    Like gunzip and uncompress, the file attributes of the original compressed
diff --git a/zlib/examples/gzappend.c b/zlib/examples/gzappend.c
index 662dec3..23e93cf 100644
--- a/zlib/examples/gzappend.c
+++ b/zlib/examples/gzappend.c
@@ -33,7 +33,7 @@
  *                      - Add L to constants in lseek() calls
  *                      - Remove some debugging information in error messages
  *                      - Use new data_type definition for zlib 1.2.1
- *                      - Simplfy and unify file operations
+ *                      - Simplify and unify file operations
  *                      - Finish off gzip file in gztack()
  *                      - Use deflatePrime() instead of adding empty blocks
  *                      - Keep gzip file clean on appended file read errors
@@ -54,7 +54,7 @@
    block boundary to facilitate locating and modifying the last block bit at
    the start of the final deflate block.  Also whether using Z_BLOCK or not,
    another required feature of zlib 1.2.x is that inflate() now provides the
-   number of unusued bits in the last input byte used.  gzappend will not work
+   number of unused bits in the last input byte used.  gzappend will not work
    with versions of zlib earlier than 1.2.1.
 
    gzappend first decompresses the gzip file internally, discarding all but
@@ -137,7 +137,7 @@ local void rotate(unsigned char *list, unsigned len, unsigned rot)
     /* do simple left shift by one */
     if (rot == 1) {
         tmp = *list;
-        memcpy(list, list + 1, len - 1);
+        memmove(list, list + 1, len - 1);
         *last = tmp;
         return;
     }
diff --git a/zlib/examples/gzlog.c b/zlib/examples/gzlog.c
index b8c2927..da1b02e 100644
--- a/zlib/examples/gzlog.c
+++ b/zlib/examples/gzlog.c
@@ -1,8 +1,8 @@
 /*
  * gzlog.c
- * Copyright (C) 2004, 2008, 2012, 2016 Mark Adler, all rights reserved
+ * Copyright (C) 2004, 2008, 2012, 2016, 2019 Mark Adler, all rights reserved
  * For conditions of distribution and use, see copyright notice in gzlog.h
- * version 2.2, 14 Aug 2012
+ * version 2.3, 25 May 2019
  */
 
 /*
@@ -212,8 +212,8 @@
      to the appropriate recovery below.  If there is no foo.add file, provide
      a zero data length to the recovery.  In that case, the append recovery
      restores the foo.gz to the previous compressed + uncompressed data state.
-     For the the compress recovery, a missing foo.add file results in foo.gz
-     being restored to the previous compressed-only data state.
+     For the compress recovery, a missing foo.add file results in foo.gz being
+     restored to the previous compressed-only data state.
    - Append recovery:
      - Pick up append at + step above
    - Compress recovery:
@@ -756,12 +756,14 @@ local int log_recover(struct log *log, int op)
                 return -2;
             }
             if ((fd = open(log->path, O_RDONLY, 0)) < 0) {
+                free(data);
                 log_log(log, op, ".add file read failure");
                 return -1;
             }
             ret = (size_t)read(fd, data, len) != len;
             close(fd);
             if (ret) {
+                free(data);
                 log_log(log, op, ".add file read failure");
                 return -1;
             }
diff --git a/zlib/examples/gzlog.h b/zlib/examples/gzlog.h
index 86f0cec..4f05109 100644
--- a/zlib/examples/gzlog.h
+++ b/zlib/examples/gzlog.h
@@ -40,7 +40,7 @@
    its new size at that time.  After each write operation, the log file is a
    valid gzip file that can decompressed to recover what was written.
 
-   The gzlog operations can be interupted at any point due to an application or
+   The gzlog operations can be interrupted at any point due to an application or
    system crash, and the log file will be recovered the next time the log is
    opened with gzlog_open().
  */
diff --git a/zlib/examples/gznorm.c b/zlib/examples/gznorm.c
new file mode 100644
index 0000000..68e0a0f
--- /dev/null
+++ b/zlib/examples/gznorm.c
@@ -0,0 +1,470 @@
+/* gznorm.c -- normalize a gzip stream
+ * Copyright (C) 2018 Mark Adler
+ * For conditions of distribution and use, see copyright notice in zlib.h
+ * Version 1.0  7 Oct 2018  Mark Adler */
+
+// gznorm takes a gzip stream, potentially containing multiple members, and
+// converts it to a gzip stream with a single member. In addition the gzip
+// header is normalized, removing the file name and time stamp, and setting the
+// other header contents (XFL, OS) to fixed values. gznorm does not recompress
+// the data, so it is fast, but no advantage is gained from the history that
+// could be available across member boundaries.
+
+#include <stdio.h>      // fread, fwrite, putc, fflush, ferror, fprintf,
+                        // vsnprintf, stdout, stderr, NULL, FILE
+#include <stdlib.h>     // malloc, free
+#include <string.h>     // strerror
+#include <errno.h>      // errno
+#include <stdarg.h>     // va_list, va_start, va_end
+#include "zlib.h"       // inflateInit2, inflate, inflateReset, inflateEnd,
+                        // z_stream, z_off_t, crc32_combine, Z_NULL, Z_BLOCK,
+                        // Z_OK, Z_STREAM_END, Z_BUF_ERROR, Z_DATA_ERROR,
+                        // Z_MEM_ERROR
+
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__)
+#  include <fcntl.h>
+#  include <io.h>
+#  define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
+#else
+#  define SET_BINARY_MODE(file)
+#endif
+
+#define local static
+
+// printf to an allocated string. Return the string, or NULL if the printf or
+// allocation fails.
+local char *aprintf(char *fmt, ...) {
+    // Get the length of the result of the printf.
+    va_list args;
+    va_start(args, fmt);
+    int len = vsnprintf(NULL, 0, fmt, args);
+    va_end(args);
+    if (len < 0)
+        return NULL;
+
+    // Allocate the required space and printf to it.
+    char *str = malloc(len + 1);
+    if (str == NULL)
+        return NULL;
+    va_start(args, fmt);
+    vsnprintf(str, len + 1, fmt, args);
+    va_end(args);
+    return str;
+}
+
+// Return with an error, putting an allocated error message in *err. Doing an
+// inflateEnd() on an already ended state, or one with state set to Z_NULL, is
+// permitted.
+#define BYE(...) \
+    do { \
+        inflateEnd(&strm); \
+        *err = aprintf(__VA_ARGS__); \
+        return 1; \
+    } while (0)
+
+// Chunk size for buffered reads and for decompression. Twice this many bytes
+// will be allocated on the stack by gzip_normalize(). Must fit in an unsigned.
+#define CHUNK 16384
+
+// Read a gzip stream from in and write an equivalent normalized gzip stream to
+// out. If given no input, an empty gzip stream will be written. If successful,
+// 0 is returned, and *err is set to NULL. On error, 1 is returned, where the
+// details of the error are returned in *err, a pointer to an allocated string.
+//
+// The input may be a stream with multiple gzip members, which is converted to
+// a single gzip member on the output. Each gzip member is decompressed at the
+// level of deflate blocks. This enables clearing the last-block bit, shifting
+// the compressed data to concatenate to the previous member's compressed data,
+// which can end at an arbitrary bit boundary, and identifying stored blocks in
+// order to resynchronize those to byte boundaries. The deflate compressed data
+// is terminated with a 10-bit empty fixed block. If any members on the input
+// end with a 10-bit empty fixed block, then that block is excised from the
+// stream. This avoids appending empty fixed blocks for every normalization,
+// and assures that gzip_normalize applied a second time will not change the
+// input. The pad bits after stored block headers and after the final deflate
+// block are all forced to zeros.
+local int gzip_normalize(FILE *in, FILE *out, char **err) {
+    // initialize the inflate engine to process a gzip member
+    z_stream strm;
+    strm.zalloc = Z_NULL;
+    strm.zfree = Z_NULL;
+    strm.opaque = Z_NULL;
+    strm.avail_in = 0;
+    strm.next_in = Z_NULL;
+    if (inflateInit2(&strm, 15 + 16) != Z_OK)
+        BYE("out of memory");
+
+    // State while processing the input gzip stream.
+    enum {              // BETWEEN -> HEAD -> BLOCK -> TAIL -> BETWEEN -> ...
+        BETWEEN,        // between gzip members (must end in this state)
+        HEAD,           // reading a gzip header
+        BLOCK,          // reading deflate blocks
+        TAIL            // reading a gzip trailer
+    } state = BETWEEN;              // current component being processed
+    unsigned long crc = 0;          // accumulated CRC of uncompressed data
+    unsigned long len = 0;          // accumulated length of uncompressed data
+    unsigned long buf = 0;          // deflate stream bit buffer of num bits
+    int num = 0;                    // number of bits in buf (at bottom)
+
+    // Write a canonical gzip header (no mod time, file name, comment, extra
+    // block, or extra flags, and OS is marked as unknown).
+    fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out);
+
+    // Process the gzip stream from in until reaching the end of the input,
+    // encountering invalid input, or experiencing an i/o error.
+    int more;                       // true if not at the end of the input
+    do {
+        // State inside this loop.
+        unsigned char *put;         // next input buffer location to process
+        int prev;                   // number of bits from previous block in
+                                    // the bit buffer, or -1 if not at the
+                                    // start of a block
+        unsigned long long memb;    // uncompressed length of member
+        size_t tail;                // number of trailer bytes read (0..8)
+        unsigned long part;         // accumulated trailer component
+
+        // Get the next chunk of input from in.
+        unsigned char dat[CHUNK];
+        strm.avail_in = fread(dat, 1, CHUNK, in);
+        if (strm.avail_in == 0)
+            break;
+        more = strm.avail_in == CHUNK;
+        strm.next_in = put = dat;
+
+        // Run that chunk of input through the inflate engine to exhaustion.
+        do {
+            // At this point it is assured that strm.avail_in > 0.
+
+            // Inflate until the end of a gzip component (header, deflate
+            // block, trailer) is reached, or until all of the chunk is
+            // consumed. The resulting decompressed data is discarded, though
+            // the total size of the decompressed data in each member is
+            // tracked, for the calculation of the total CRC.
+            do {
+                // inflate and handle any errors
+                unsigned char scrap[CHUNK];
+                strm.avail_out = CHUNK;
+                strm.next_out = scrap;
+                int ret = inflate(&strm, Z_BLOCK);
+                if (ret == Z_MEM_ERROR)
+                    BYE("out of memory");
+                if (ret == Z_DATA_ERROR)
+                    BYE("input invalid: %s", strm.msg);
+                if (ret != Z_OK && ret != Z_BUF_ERROR && ret != Z_STREAM_END)
+                    BYE("internal error");
+
+                // Update the number of uncompressed bytes generated in this
+                // member. The actual count (not modulo 2^32) is required to
+                // correctly compute the total CRC.
+                unsigned got = CHUNK - strm.avail_out;
+                memb += got;
+                if (memb < got)
+                    BYE("overflow error");
+
+                // Continue to process this chunk until it is consumed, or
+                // until the end of a component (header, deflate block, or
+                // trailer) is reached.
+            } while (strm.avail_out == 0 && (strm.data_type & 0x80) == 0);
+
+            // Since strm.avail_in was > 0 for the inflate call, some input was
+            // just consumed. It is therefore assured that put < strm.next_in.
+
+            // Disposition the consumed component or part of a component.
+            switch (state) {
+                case BETWEEN:
+                    state = HEAD;
+                    // Fall through to HEAD when some or all of the header is
+                    // processed.
+
+                case HEAD:
+                    // Discard the header.
+                    if (strm.data_type & 0x80) {
+                        // End of header reached -- deflate blocks follow.
+                        put = strm.next_in;
+                        prev = num;
+                        memb = 0;
+                        state = BLOCK;
+                    }
+                    break;
+
+                case BLOCK:
+                    // Copy the deflate stream to the output, but with the
+                    // last-block-bit cleared. Re-synchronize stored block
+                    // headers to the output byte boundaries. The bytes at
+                    // put..strm.next_in-1 is the compressed data that has been
+                    // processed and is ready to be copied to the output.
+
+                    // At this point, it is assured that new compressed data is
+                    // available, i.e., put < strm.next_in. If prev is -1, then
+                    // that compressed data starts in the middle of a deflate
+                    // block. If prev is not -1, then the bits in the bit
+                    // buffer, possibly combined with the bits in *put, contain
+                    // the three-bit header of the new deflate block. In that
+                    // case, prev is the number of bits from the previous block
+                    // that remain in the bit buffer. Since num is the number
+                    // of bits in the bit buffer, we have that num - prev is
+                    // the number of bits from the new block currently in the
+                    // bit buffer.
+
+                    // If strm.data_type & 0xc0 is 0x80, then the last byte of
+                    // the available compressed data includes the last bits of
+                    // the end of a deflate block. In that case, that last byte
+                    // also has strm.data_type & 0x1f bits of the next deflate
+                    // block, in the range 0..7. If strm.data_type & 0xc0 is
+                    // 0xc0, then the last byte of the compressed data is the
+                    // end of the deflate stream, followed by strm.data_type &
+                    // 0x1f pad bits, also in the range 0..7.
+
+                    // Set bits to the number of bits not yet consumed from the
+                    // last byte. If we are at the end of the block, bits is
+                    // either the number of bits in the last byte belonging to
+                    // the next block, or the number of pad bits after the
+                    // final block. In either of those cases, bits is in the
+                    // range 0..7.
+                    ;                   // (required due to C syntax oddity)
+                    int bits = strm.data_type & 0x1f;
+
+                    if (prev != -1) {
+                        // We are at the start of a new block. Clear the last
+                        // block bit, and check for special cases. If it is a
+                        // stored block, then emit the header and pad to the
+                        // next byte boundary. If it is a final, empty fixed
+                        // block, then excise it.
+
+                        // Some or all of the three header bits for this block
+                        // may already be in the bit buffer. Load any remaining
+                        // header bits into the bit buffer.
+                        if (num - prev < 3) {
+                            buf += (unsigned long)*put++ << num;
+                            num += 8;
+                        }
+
+                        // Set last to have a 1 in the position of the last
+                        // block bit in the bit buffer.
+                        unsigned long last = (unsigned long)1 << prev;
+
+                        if (((buf >> prev) & 7) == 3) {
+                            // This is a final fixed block. Load at least ten
+                            // bits from this block, including the header, into
+                            // the bit buffer. We already have at least three,
+                            // so at most one more byte needs to be loaded.
+                            if (num - prev < 10) {
+                                if (put == strm.next_in)
+                                    // Need to go get and process more input.
+                                    // We'll end up back here to finish this.
+                                    break;
+                                buf += (unsigned long)*put++ << num;
+                                num += 8;
+                            }
+                            if (((buf >> prev) & 0x3ff) == 3) {
+                                // That final fixed block is empty. Delete it
+                                // to avoid adding an empty block every time a
+                                // gzip stream is normalized.
+                                num = prev;
+                                buf &= last - 1;    // zero the pad bits
+                            }
+                        }
+                        else if (((buf >> prev) & 6) == 0) {
+                            // This is a stored block. Flush to the next
+                            // byte boundary after the three-bit header.
+                            num = (prev + 10) & ~7;
+                            buf &= last - 1;        // zero the pad bits
+                        }
+
+                        // Clear the last block bit.
+                        buf &= ~last;
+
+                        // Write out complete bytes in the bit buffer.
+                        while (num >= 8) {
+                            putc(buf, out);
+                            buf >>= 8;
+                            num -= 8;
+                        }
+
+                        // If no more bytes left to process, then we have
+                        // consumed the byte that had bits from the next block.
+                        if (put == strm.next_in)
+                            bits = 0;
+                    }
+
+                    // We are done handling the deflate block header. Now copy
+                    // all or almost all of the remaining compressed data that
+                    // has been processed so far. Don't copy one byte at the
+                    // end if it contains bits from the next deflate block or
+                    // pad bits at the end of a deflate block.
+
+                    // mix is 1 if we are at the end of a deflate block, and if
+                    // some of the bits in the last byte follow this block. mix
+                    // is 0 if we are in the middle of a deflate block, if the
+                    // deflate block ended on a byte boundary, or if all of the
+                    // compressed data processed so far has been consumed.
+                    int mix = (strm.data_type & 0x80) && bits;
+
+                    // Copy all of the processed compressed data to the output,
+                    // except for the last byte if it contains bits from the
+                    // next deflate block or pad bits at the end of the deflate
+                    // stream. Copy the data after shifting in num bits from
+                    // buf in front of it, leaving num bits from the end of the
+                    // compressed data in buf when done.
+                    unsigned char *end = strm.next_in - mix;
+                    if (put < end) {
+                        if (num)
+                            // Insert num bits from buf before the data being
+                            // copied.
+                            do {
+                                buf += (unsigned)(*put++) << num;
+                                putc(buf, out);
+                                buf >>= 8;
+                            } while (put < end);
+                        else {
+                            // No shifting needed -- write directly.
+                            fwrite(put, 1, end - put, out);
+                            put = end;
+                        }
+                    }
+
+                    // Process the last processed byte if it wasn't written.
+                    if (mix) {
+                        // Load the last byte into the bit buffer.
+                        buf += (unsigned)(*put++) << num;
+                        num += 8;
+
+                        if (strm.data_type & 0x40) {
+                            // We are at the end of the deflate stream and
+                            // there are bits pad bits. Discard the pad bits
+                            // and write a byte to the output, if available.
+                            // Leave the num bits left over in buf to prepend
+                            // to the next deflate stream.
+                            num -= bits;
+                            if (num >= 8) {
+                                putc(buf, out);
+                                num -= 8;
+                                buf >>= 8;
+                            }
+
+                            // Force the pad bits in the bit buffer to zeros.
+                            buf &= ((unsigned long)1 << num) - 1;
+
+                            // Don't need to set prev here since going to TAIL.
+                        }
+                        else
+                            // At the end of an internal deflate block. Leave
+                            // the last byte in the bit buffer to examine on
+                            // the next entry to BLOCK, when more bits from the
+                            // next block will be available.
+                            prev = num - bits;      // number of bits in buffer
+                                                    // from current block
+                    }
+
+                    // Don't have a byte left over, so we are in the middle of
+                    // a deflate block, or the deflate block ended on a byte
+                    // boundary. Set prev appropriately for the next entry into
+                    // BLOCK.
+                    else if (strm.data_type & 0x80)
+                        // The block ended on a byte boundary, so no header
+                        // bits are in the bit buffer.
+                        prev = num;
+                    else
+                        // In the middle of a deflate block, so no header here.
+                        prev = -1;
+
+                    // Check for the end of the deflate stream.
+                    if ((strm.data_type & 0xc0) == 0xc0) {
+                        // That ends the deflate stream on the input side, the
+                        // pad bits were discarded, and any remaining bits from
+                        // the last block in the stream are saved in the bit
+                        // buffer to prepend to the next stream. Process the
+                        // gzip trailer next.
+                        tail = 0;
+                        part = 0;
+                        state = TAIL;
+                    }
+                    break;
+
+                case TAIL:
+                    // Accumulate available trailer bytes to update the total
+                    // CRC and the total uncompressed length.
+                    do {
+                        part = (part >> 8) + ((unsigned long)(*put++) << 24);
+                        tail++;
+                        if (tail == 4) {
+                            // Update the total CRC.
+                            z_off_t len2 = memb;
+                            if (len2 < 0 || (unsigned long long)len2 != memb)
+                                BYE("overflow error");
+                            crc = crc ? crc32_combine(crc, part, len2) : part;
+                            part = 0;
+                        }
+                        else if (tail == 8) {
+                            // Update the total uncompressed length. (It's ok
+                            // if this sum is done modulo 2^32.)
+                            len += part;
+
+                            // At the end of a member. Set up to inflate an
+                            // immediately following gzip member. (If we made
+                            // it this far, then the trailer was valid.)
+                            if (inflateReset(&strm) != Z_OK)
+                                BYE("internal error");
+                            state = BETWEEN;
+                            break;
+                        }
+                    } while (put < strm.next_in);
+                    break;
+            }
+
+            // Process the input buffer until completely consumed.
+        } while (strm.avail_in > 0);
+
+        // Process input until end of file, invalid input, or i/o error.
+    } while (more);
+
+    // Done with the inflate engine.
+    inflateEnd(&strm);
+
+    // Verify the validity of the input.
+    if (state != BETWEEN)
+        BYE("input invalid: incomplete gzip stream");
+
+    // Write the remaining deflate stream bits, followed by a terminating
+    // deflate fixed block.
+    buf += (unsigned long)3 << num;
+    putc(buf, out);
+    putc(buf >> 8, out);
+    if (num > 6)
+        putc(0, out);
+
+    // Write the gzip trailer, which is the CRC and the uncompressed length
+    // modulo 2^32, both in little-endian order.
+    putc(crc, out);
+    putc(crc >> 8, out);
+    putc(crc >> 16, out);
+    putc(crc >> 24, out);
+    putc(len, out);
+    putc(len >> 8, out);
+    putc(len >> 16, out);
+    putc(len >> 24, out);
+    fflush(out);
+
+    // Check for any i/o errors.
+    if (ferror(in) || ferror(out))
+        BYE("i/o error: %s", strerror(errno));
+
+    // All good!
+    *err = NULL;
+    return 0;
+}
+
+// Normalize the gzip stream on stdin, writing the result to stdout.
+int main(void) {
+    // Avoid end-of-line conversions on evil operating systems.
+    SET_BINARY_MODE(stdin);
+    SET_BINARY_MODE(stdout);
+
+    // Normalize from stdin to stdout, returning 1 on error, 0 if ok.
+    char *err;
+    int ret = gzip_normalize(stdin, stdout, &err);
+    if (ret)
+        fprintf(stderr, "gznorm error: %s\n", err);
+    free(err);
+    return ret;
+}
diff --git a/zlib/examples/zlib_how.html b/zlib/examples/zlib_how.html
index 444ff1c..43271b9 100644
--- a/zlib/examples/zlib_how.html
+++ b/zlib/examples/zlib_how.html
@@ -1,10 +1,10 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"
-  "http://www.w3.org/TR/REC-html40/loose.dtd">
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+  "http://www.w3.org/TR/html4/loose.dtd">
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
 <title>zlib Usage Example</title>
-<!--  Copyright (c) 2004, 2005 Mark Adler.  -->
+<!--  Copyright (c) 2004-2023 Mark Adler.  -->
 </head>
 <body bgcolor="#FFFFFF" text="#000000" link="#0000FF" vlink="#00A000">
 <h2 align="center"> zlib Usage Example </h2>
@@ -17,7 +17,7 @@ from an input file to an output file using <tt>deflate()</tt> and <tt>inflate()<
 annotations are interspersed between lines of the code.  So please read between the lines.
 We hope this helps explain some of the intricacies of <em>zlib</em>.
 <p>
-Without further adieu, here is the program <a href="zpipe.c"><tt>zpipe.c</tt></a>:
+Without further ado, here is the program <a href="zpipe.c"><tt>zpipe.c</tt></a>:
 <pre><b>
 /* zpipe.c: example of proper use of zlib's inflate() and deflate()
    Not copyrighted -- provided to the public domain
@@ -155,13 +155,11 @@ before we fall out of the loop at the bottom.
 </b></pre>
 We start off by reading data from the input file.  The number of bytes read is put directly
 into <tt>avail_in</tt>, and a pointer to those bytes is put into <tt>next_in</tt>.  We also
-check to see if end-of-file on the input has been reached.  If we are at the end of file, then <tt>flush</tt> is set to the
+check to see if end-of-file on the input has been reached using feof().
+If we are at the end of file, then <tt>flush</tt> is set to the
 <em>zlib</em> constant <tt>Z_FINISH</tt>, which is later passed to <tt>deflate()</tt> to
-indicate that this is the last chunk of input data to compress.  We need to use <tt>feof()</tt>
-to check for end-of-file as opposed to seeing if fewer than <tt>CHUNK</tt> bytes have been read.  The
-reason is that if the input file length is an exact multiple of <tt>CHUNK</tt>, we will miss
-the fact that we got to the end-of-file, and not know to tell <tt>deflate()</tt> to finish
-up the compressed stream.  If we are not yet at the end of the input, then the <em>zlib</em>
+indicate that this is the last chunk of input data to compress.
+If we are not yet at the end of the input, then the <em>zlib</em>
 constant <tt>Z_NO_FLUSH</tt> will be passed to <tt>deflate</tt> to indicate that we are still
 in the middle of the uncompressed data.
 <p>
@@ -540,6 +538,12 @@ int main(int argc, char **argv)
 }
 </b></pre>
 <hr>
-<i>Copyright (c) 2004, 2005 by Mark Adler<br>Last modified 11 December 2005</i>
+<i>Last modified 24 January 2023<br>
+Copyright &#169; 2004-2023 Mark Adler</i><br>
+<a rel="license" href="http://creativecommons.org/licenses/by-nd/4.0/">
+<img alt="Creative Commons License" style="border-width:0"
+src="https://i.creativecommons.org/l/by-nd/4.0/88x31.png"></a>
+<a rel="license" href="http://creativecommons.org/licenses/by-nd/4.0/">
+Creative Commons Attribution-NoDerivatives 4.0 International License</a>.
 </body>
 </html>
diff --git a/zlib/examples/zran.c b/zlib/examples/zran.c
index 4fec659..d313595 100644
--- a/zlib/examples/zran.c
+++ b/zlib/examples/zran.c
@@ -1,383 +1,503 @@
-/* zran.c -- example of zlib/gzip stream indexing and random access
- * Copyright (C) 2005, 2012 Mark Adler
+/* zran.c -- example of deflate stream indexing and random access
+ * Copyright (C) 2005, 2012, 2018, 2023 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
-   Version 1.1  29 Sep 2012  Mark Adler */
+ * Version 1.4  13 Apr 2023  Mark Adler */
 
 /* Version History:
  1.0  29 May 2005  First version
  1.1  29 Sep 2012  Fix memory reallocation error
+ 1.2  14 Oct 2018  Handle gzip streams with multiple members
+                   Add a header file to facilitate usage in applications
+ 1.3  18 Feb 2023  Permit raw deflate streams as well as zlib and gzip
+                   Permit crossing gzip member boundaries when extracting
+                   Support a size_t size when extracting (was an int)
+                   Do a binary search over the index for an access point
+                   Expose the access point type to enable save and load
+ 1.4  13 Apr 2023  Add a NOPRIME define to not use inflatePrime()
  */
 
-/* Illustrate the use of Z_BLOCK, inflatePrime(), and inflateSetDictionary()
-   for random access of a compressed file.  A file containing a zlib or gzip
-   stream is provided on the command line.  The compressed stream is decoded in
-   its entirety, and an index built with access points about every SPAN bytes
-   in the uncompressed output.  The compressed file is left open, and can then
-   be read randomly, having to decompress on the average SPAN/2 uncompressed
-   bytes before getting to the desired block of data.
-
-   An access point can be created at the start of any deflate block, by saving
-   the starting file offset and bit of that block, and the 32K bytes of
-   uncompressed data that precede that block.  Also the uncompressed offset of
-   that block is saved to provide a referece for locating a desired starting
-   point in the uncompressed stream.  build_index() works by decompressing the
-   input zlib or gzip stream a block at a time, and at the end of each block
-   deciding if enough uncompressed data has gone by to justify the creation of
-   a new access point.  If so, that point is saved in a data structure that
-   grows as needed to accommodate the points.
-
-   To use the index, an offset in the uncompressed data is provided, for which
-   the latest access point at or preceding that offset is located in the index.
-   The input file is positioned to the specified location in the index, and if
-   necessary the first few bits of the compressed data is read from the file.
-   inflate is initialized with those bits and the 32K of uncompressed data, and
-   the decompression then proceeds until the desired offset in the file is
-   reached.  Then the decompression continues to read the desired uncompressed
-   data from the file.
-
-   Another approach would be to generate the index on demand.  In that case,
-   requests for random access reads from the compressed data would try to use
-   the index, but if a read far enough past the end of the index is required,
-   then further index entries would be generated and added.
-
-   There is some fair bit of overhead to starting inflation for the random
-   access, mainly copying the 32K byte dictionary.  So if small pieces of the
-   file are being accessed, it would make sense to implement a cache to hold
-   some lookahead and avoid many calls to extract() for small lengths.
-
-   Another way to build an index would be to use inflateCopy().  That would
-   not be constrained to have access points at block boundaries, but requires
-   more memory per access point, and also cannot be saved to file due to the
-   use of pointers in the state.  The approach here allows for storage of the
-   index in a file.
- */
+// Illustrate the use of Z_BLOCK, inflatePrime(), and inflateSetDictionary()
+// for random access of a compressed file. A file containing a raw deflate
+// stream is provided on the command line. The compressed stream is decoded in
+// its entirety, and an index built with access points about every SPAN bytes
+// in the uncompressed output. The compressed file is left open, and can then
+// be read randomly, having to decompress on the average SPAN/2 uncompressed
+// bytes before getting to the desired block of data.
+//
+// An access point can be created at the start of any deflate block, by saving
+// the starting file offset and bit of that block, and the 32K bytes of
+// uncompressed data that precede that block. Also the uncompressed offset of
+// that block is saved to provide a reference for locating a desired starting
+// point in the uncompressed stream. deflate_index_build() decompresses the
+// input raw deflate stream a block at a time, and at the end of each block
+// decides if enough uncompressed data has gone by to justify the creation of a
+// new access point. If so, that point is saved in a data structure that grows
+// as needed to accommodate the points.
+//
+// To use the index, an offset in the uncompressed data is provided, for which
+// the latest access point at or preceding that offset is located in the index.
+// The input file is positioned to the specified location in the index, and if
+// necessary the first few bits of the compressed data is read from the file.
+// inflate is initialized with those bits and the 32K of uncompressed data, and
+// decompression then proceeds until the desired offset in the file is reached.
+// Then decompression continues to read the requested uncompressed data from
+// the file.
+//
+// There is some fair bit of overhead to starting inflation for the random
+// access, mainly copying the 32K byte dictionary. If small pieces of the file
+// are being accessed, it would make sense to implement a cache to hold some
+// lookahead to avoid many calls to deflate_index_extract() for small lengths.
+//
+// Another way to build an index would be to use inflateCopy(). That would not
+// be constrained to have access points at block boundaries, but would require
+// more memory per access point, and could not be saved to a file due to the
+// use of pointers in the state. The approach here allows for storage of the
+// index in a file.
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <limits.h>
 #include "zlib.h"
+#include "zran.h"
+
+#define WINSIZE 32768U      // sliding window size
+#define CHUNK 16384         // file input buffer size
 
-#define local static
-
-#define SPAN 1048576L       /* desired distance between access points */
-#define WINSIZE 32768U      /* sliding window size */
-#define CHUNK 16384         /* file input buffer size */
-
-/* access point entry */
-struct point {
-    off_t out;          /* corresponding offset in uncompressed data */
-    off_t in;           /* offset in input file of first full byte */
-    int bits;           /* number of bits (1-7) from byte at in - 1, or 0 */
-    unsigned char window[WINSIZE];  /* preceding 32K of uncompressed data */
-};
-
-/* access point list */
-struct access {
-    int have;           /* number of list entries filled in */
-    int size;           /* number of list entries allocated */
-    struct point *list; /* allocated list */
-};
-
-/* Deallocate an index built by build_index() */
-local void free_index(struct access *index)
-{
+// See comments in zran.h.
+void deflate_index_free(struct deflate_index *index) {
     if (index != NULL) {
         free(index->list);
         free(index);
     }
 }
 
-/* Add an entry to the access point list.  If out of memory, deallocate the
-   existing list and return NULL. */
-local struct access *addpoint(struct access *index, int bits,
-    off_t in, off_t out, unsigned left, unsigned char *window)
-{
-    struct point *next;
-
-    /* if list is empty, create it (start with eight points) */
+// Add an access point to the list. If out of memory, deallocate the existing
+// list and return NULL. index->mode is temporarily the allocated number of
+// access points, until it is time for deflate_index_build() to return. Then
+// index->mode is set to the mode of inflation.
+static struct deflate_index *add_point(struct deflate_index *index, int bits,
+                                       off_t in, off_t out, unsigned left,
+                                       unsigned char *window) {
     if (index == NULL) {
-        index = malloc(sizeof(struct access));
-        if (index == NULL) return NULL;
-        index->list = malloc(sizeof(struct point) << 3);
+        // The list is empty. Create it, starting with eight access points.
+        index = malloc(sizeof(struct deflate_index));
+        if (index == NULL)
+            return NULL;
+        index->have = 0;
+        index->mode = 8;
+        index->list = malloc(sizeof(point_t) * index->mode);
         if (index->list == NULL) {
             free(index);
             return NULL;
         }
-        index->size = 8;
-        index->have = 0;
     }
 
-    /* if list is full, make it bigger */
-    else if (index->have == index->size) {
-        index->size <<= 1;
-        next = realloc(index->list, sizeof(struct point) * index->size);
+    else if (index->have == index->mode) {
+        // The list is full. Make it bigger.
+        index->mode <<= 1;
+        point_t *next = realloc(index->list, sizeof(point_t) * index->mode);
         if (next == NULL) {
-            free_index(index);
+            deflate_index_free(index);
             return NULL;
         }
         index->list = next;
     }
 
-    /* fill in entry and increment how many we have */
-    next = index->list + index->have;
-    next->bits = bits;
-    next->in = in;
+    // Fill in the access point and increment how many we have.
+    point_t *next = (point_t *)(index->list) + index->have++;
+    if (index->have < 0) {
+        // Overflowed the int!
+        deflate_index_free(index);
+        return NULL;
+    }
     next->out = out;
+    next->in = in;
+    next->bits = bits;
     if (left)
         memcpy(next->window, window + WINSIZE - left, left);
     if (left < WINSIZE)
         memcpy(next->window + left, window, WINSIZE - left);
-    index->have++;
 
-    /* return list, possibly reallocated */
+    // Return the index, which may have been newly allocated or destroyed.
     return index;
 }
 
-/* Make one entire pass through the compressed stream and build an index, with
-   access points about every span bytes of uncompressed output -- span is
-   chosen to balance the speed of random access against the memory requirements
-   of the list, about 32K bytes per access point.  Note that data after the end
-   of the first zlib or gzip stream in the file is ignored.  build_index()
-   returns the number of access points on success (>= 1), Z_MEM_ERROR for out
-   of memory, Z_DATA_ERROR for an error in the input file, or Z_ERRNO for a
-   file read error.  On success, *built points to the resulting index. */
-local int build_index(FILE *in, off_t span, struct access **built)
-{
-    int ret;
-    off_t totin, totout;        /* our own total counters to avoid 4GB limit */
-    off_t last;                 /* totout value of last access point */
-    struct access *index;       /* access points being generated */
-    z_stream strm;
-    unsigned char input[CHUNK];
-    unsigned char window[WINSIZE];
-
-    /* initialize inflate */
-    strm.zalloc = Z_NULL;
-    strm.zfree = Z_NULL;
-    strm.opaque = Z_NULL;
-    strm.avail_in = 0;
-    strm.next_in = Z_NULL;
-    ret = inflateInit2(&strm, 47);      /* automatic zlib or gzip decoding */
-    if (ret != Z_OK)
-        return ret;
-
-    /* inflate the input, maintain a sliding window, and build an index -- this
-       also validates the integrity of the compressed data using the check
-       information at the end of the gzip or zlib stream */
-    totin = totout = last = 0;
-    index = NULL;               /* will be allocated by first addpoint() */
-    strm.avail_out = 0;
+// Decompression modes. These are the inflateInit2() windowBits parameter.
+#define RAW -15
+#define ZLIB 15
+#define GZIP 31
+
+// See comments in zran.h.
+int deflate_index_build(FILE *in, off_t span, struct deflate_index **built) {
+    // Set up inflation state.
+    z_stream strm = {0};        // inflate engine (gets fired up later)
+    unsigned char buf[CHUNK];   // input buffer
+    unsigned char win[WINSIZE] = {0};   // output sliding window
+    off_t totin = 0;            // total bytes read from input
+    off_t totout = 0;           // total bytes uncompressed
+    int mode = 0;               // mode: RAW, ZLIB, or GZIP (0 => not set yet)
+
+    // Decompress from in, generating access points along the way.
+    int ret;                    // the return value from zlib, or Z_ERRNO
+    off_t last;                 // last access point uncompressed offset
+    struct deflate_index *index = NULL;     // list of access points
     do {
-        /* get some compressed data from input file */
-        strm.avail_in = fread(input, 1, CHUNK, in);
-        if (ferror(in)) {
-            ret = Z_ERRNO;
-            goto build_index_error;
-        }
+        // Assure available input, at least until reaching EOF.
         if (strm.avail_in == 0) {
-            ret = Z_DATA_ERROR;
-            goto build_index_error;
-        }
-        strm.next_in = input;
-
-        /* process all of that, or until end of stream */
-        do {
-            /* reset sliding window if necessary */
-            if (strm.avail_out == 0) {
-                strm.avail_out = WINSIZE;
-                strm.next_out = window;
-            }
-
-            /* inflate until out of input, output, or at end of block --
-               update the total input and output counters */
+            strm.avail_in = fread(buf, 1, sizeof(buf), in);
             totin += strm.avail_in;
-            totout += strm.avail_out;
-            ret = inflate(&strm, Z_BLOCK);      /* return at end of block */
-            totin -= strm.avail_in;
-            totout -= strm.avail_out;
-            if (ret == Z_NEED_DICT)
-                ret = Z_DATA_ERROR;
-            if (ret == Z_MEM_ERROR || ret == Z_DATA_ERROR)
-                goto build_index_error;
-            if (ret == Z_STREAM_END)
+            strm.next_in = buf;
+            if (strm.avail_in < sizeof(buf) && ferror(in)) {
+                ret = Z_ERRNO;
                 break;
+            }
 
-            /* if at end of block, consider adding an index entry (note that if
-               data_type indicates an end-of-block, then all of the
-               uncompressed data from that block has been delivered, and none
-               of the compressed data after that block has been consumed,
-               except for up to seven bits) -- the totout == 0 provides an
-               entry point after the zlib or gzip header, and assures that the
-               index always has at least one access point; we avoid creating an
-               access point after the last block by checking bit 6 of data_type
-             */
-            if ((strm.data_type & 128) && !(strm.data_type & 64) &&
-                (totout == 0 || totout - last > span)) {
-                index = addpoint(index, strm.data_type & 7, totin,
-                                 totout, strm.avail_out, window);
-                if (index == NULL) {
-                    ret = Z_MEM_ERROR;
-                    goto build_index_error;
-                }
-                last = totout;
+            if (mode == 0) {
+                // At the start of the input -- determine the type. Assume raw
+                // if it is neither zlib nor gzip. This could in theory result
+                // in a false positive for zlib, but in practice the fill bits
+                // after a stored block are always zeros, so a raw stream won't
+                // start with an 8 in the low nybble.
+                mode = strm.avail_in == 0 ? RAW :       // empty -- will fail
+                       (strm.next_in[0] & 0xf) == 8 ? ZLIB :
+                       strm.next_in[0] == 0x1f ? GZIP :
+                       /* else */ RAW;
+                ret = inflateInit2(&strm, mode);
+                if (ret != Z_OK)
+                    break;
             }
-        } while (strm.avail_in != 0);
-    } while (ret != Z_STREAM_END);
+        }
 
-    /* clean up and return index (release unused entries in list) */
-    (void)inflateEnd(&strm);
-    index->list = realloc(index->list, sizeof(struct point) * index->have);
-    index->size = index->have;
+        // Assure available output. This rotates the output through, for use as
+        // a sliding window on the uncompressed data.
+        if (strm.avail_out == 0) {
+            strm.avail_out = sizeof(win);
+            strm.next_out = win;
+        }
+
+        if (mode == RAW && index == NULL)
+            // We skip the inflate() call at the start of raw deflate data in
+            // order generate an access point there. Set data_type to imitate
+            // the end of a header.
+            strm.data_type = 0x80;
+        else {
+            // Inflate and update the number of uncompressed bytes.
+            unsigned before = strm.avail_out;
+            ret = inflate(&strm, Z_BLOCK);
+            totout += before - strm.avail_out;
+        }
+
+        if ((strm.data_type & 0xc0) == 0x80 &&
+            (index == NULL || totout - last >= span)) {
+            // We are at the end of a header or a non-last deflate block, so we
+            // can add an access point here. Furthermore, we are either at the
+            // very start for the first access point, or there has been span or
+            // more uncompressed bytes since the last access point, so we want
+            // to add an access point here.
+            index = add_point(index, strm.data_type & 7, totin - strm.avail_in,
+                              totout, strm.avail_out, win);
+            if (index == NULL) {
+                ret = Z_MEM_ERROR;
+                break;
+            }
+            last = totout;
+        }
+
+        if (ret == Z_STREAM_END && mode == GZIP &&
+            (strm.avail_in || ungetc(getc(in), in) != EOF))
+            // There is more input after the end of a gzip member. Reset the
+            // inflate state to read another gzip member. On success, this will
+            // set ret to Z_OK to continue decompressing.
+            ret = inflateReset2(&strm, GZIP);
+
+        // Keep going until Z_STREAM_END or error. If the compressed data ends
+        // prematurely without a file read error, Z_BUF_ERROR is returned.
+    } while (ret == Z_OK);
+    inflateEnd(&strm);
+
+    if (ret != Z_STREAM_END) {
+        // An error was encountered. Discard the index and return a negative
+        // error code.
+        deflate_index_free(index);
+        return ret == Z_NEED_DICT ? Z_DATA_ERROR : ret;
+    }
+
+    // Shrink the index to only the occupied access points and return it.
+    index->mode = mode;
+    index->length = totout;
+    point_t *list = realloc(index->list, sizeof(point_t) * index->have);
+    if (list == NULL) {
+        // Seems like a realloc() to make something smaller should always work,
+        // but just in case.
+        deflate_index_free(index);
+        return Z_MEM_ERROR;
+    }
+    index->list = list;
     *built = index;
-    return index->size;
-
-    /* return error */
-  build_index_error:
-    (void)inflateEnd(&strm);
-    if (index != NULL)
-        free_index(index);
-    return ret;
+    return index->have;
 }
 
-/* Use the index to read len bytes from offset into buf, return bytes read or
-   negative for error (Z_DATA_ERROR or Z_MEM_ERROR).  If data is requested past
-   the end of the uncompressed data, then extract() will return a value less
-   than len, indicating how much as actually read into buf.  This function
-   should not return a data error unless the file was modified since the index
-   was generated.  extract() may also return Z_ERRNO if there is an error on
-   reading or seeking the input file. */
-local int extract(FILE *in, struct access *index, off_t offset,
-                  unsigned char *buf, int len)
-{
-    int ret, skip;
-    z_stream strm;
-    struct point *here;
-    unsigned char input[CHUNK];
-    unsigned char discard[WINSIZE];
+#ifdef NOPRIME
+// Support zlib versions before 1.2.3 (July 2005), or incomplete zlib clones
+// that do not have inflatePrime().
+
+#  define INFLATEPRIME inflatePreface
+
+// Append the low bits bits of value to in[] at bit position *have, updating
+// *have. value must be zero above its low bits bits. bits must be positive.
+// This assumes that any bits above the *have bits in the last byte are zeros.
+// That assumption is preserved on return, as any bits above *have + bits in
+// the last byte written will be set to zeros.
+static inline void append_bits(unsigned value, int bits,
+                               unsigned char *in, int *have) {
+    in += *have >> 3;           // where the first bits from value will go
+    int k = *have & 7;          // the number of bits already there
+    *have += bits;
+    if (k)
+        *in |= value << k;      // write value above the low k bits
+    else
+        *in = value;
+    k = 8 - k;                  // the number of bits just appended
+    while (bits > k) {
+        value >>= k;            // drop the bits appended
+        bits -= k;
+        k = 8;                  // now at a byte boundary
+        *++in = value;
+    }
+}
+
+// Insert enough bits in the form of empty deflate blocks in front of the
+// low bits bits of value, in order to bring the sequence to a byte boundary.
+// Then feed that to inflate(). This does what inflatePrime() does, except that
+// a negative value of bits is not supported. bits must be in 0..16. If the
+// arguments are invalid, Z_STREAM_ERROR is returned. Otherwise the return
+// value from inflate() is returned.
+static int inflatePreface(z_stream *strm, int bits, int value) {
+    // Check input.
+    if (strm == Z_NULL || bits < 0 || bits > 16)
+        return Z_STREAM_ERROR;
+    if (bits == 0)
+        return Z_OK;
+    value &= (2 << (bits - 1)) - 1;
+
+    // An empty dynamic block with an odd number of bits (95). The high bit of
+    // the last byte is unused.
+    static const unsigned char dyn[] = {
+        4, 0xe0, 0x81, 8, 0, 0, 0, 0, 0x20, 0xa8, 0xab, 0x1f
+    };
+    const int dynlen = 95;          // number of bits in the block
+
+    // Build an input buffer for inflate that is a multiple of eight bits in
+    // length, and that ends with the low bits bits of value.
+    unsigned char in[(dynlen + 3 * 10 + 16 + 7) / 8];
+    int have = 0;
+    if (bits & 1) {
+        // Insert an empty dynamic block to get to an odd number of bits, so
+        // when bits bits from value are appended, we are at an even number of
+        // bits.
+        memcpy(in, dyn, sizeof(dyn));
+        have = dynlen;
+    }
+    while ((have + bits) & 7)
+        // Insert empty fixed blocks until appending bits bits would put us on
+        // a byte boundary. This will insert at most three fixed blocks.
+        append_bits(2, 10, in, &have);
+
+    // Append the bits bits from value, which takes us to a byte boundary.
+    append_bits(value, bits, in, &have);
+
+    // Deliver the input to inflate(). There is no output space provided, but
+    // inflate() can't get stuck waiting on output not ingesting all of the
+    // provided input. The reason is that there will be at most 16 bits of
+    // input from value after the empty deflate blocks (which themselves
+    // generate no output). At least ten bits are needed to generate the first
+    // output byte from a fixed block. The last two bytes of the buffer have to
+    // be ingested in order to get ten bits, which is the most that value can
+    // occupy.
+    strm->avail_in = have >> 3;
+    strm->next_in = in;
+    strm->avail_out = 0;
+    strm->next_out = in;                // not used, but can't be NULL
+    return inflate(strm, Z_NO_FLUSH);
+}
 
-    /* proceed only if something reasonable to do */
-    if (len < 0)
+#else
+#  define INFLATEPRIME inflatePrime
+#endif
+
+// See comments in zran.h.
+ptrdiff_t deflate_index_extract(FILE *in, struct deflate_index *index,
+                                off_t offset, unsigned char *buf, size_t len) {
+    // Do a quick sanity check on the index.
+    if (index == NULL || index->have < 1 || index->list[0].out != 0)
+        return Z_STREAM_ERROR;
+
+    // If nothing to extract, return zero bytes extracted.
+    if (len == 0 || offset < 0 || offset >= index->length)
         return 0;
 
-    /* find where in stream to start */
-    here = index->list;
-    ret = index->have;
-    while (--ret && here[1].out <= offset)
-        here++;
-
-    /* initialize file and inflate state to start there */
-    strm.zalloc = Z_NULL;
-    strm.zfree = Z_NULL;
-    strm.opaque = Z_NULL;
-    strm.avail_in = 0;
-    strm.next_in = Z_NULL;
-    ret = inflateInit2(&strm, -15);         /* raw inflate */
+    // Find the access point closest to but not after offset.
+    int lo = -1, hi = index->have;
+    point_t *point = index->list;
+    while (hi - lo > 1) {
+        int mid = (lo + hi) >> 1;
+        if (offset < point[mid].out)
+            hi = mid;
+        else
+            lo = mid;
+    }
+    point += lo;
+
+    // Initialize the input file and prime the inflate engine to start there.
+    int ret = fseeko(in, point->in - (point->bits ? 1 : 0), SEEK_SET);
+    if (ret == -1)
+        return Z_ERRNO;
+    int ch = 0;
+    if (point->bits && (ch = getc(in)) == EOF)
+        return ferror(in) ? Z_ERRNO : Z_BUF_ERROR;
+    z_stream strm = {0};
+    ret = inflateInit2(&strm, RAW);
     if (ret != Z_OK)
         return ret;
-    ret = fseeko(in, here->in - (here->bits ? 1 : 0), SEEK_SET);
-    if (ret == -1)
-        goto extract_ret;
-    if (here->bits) {
-        ret = getc(in);
-        if (ret == -1) {
-            ret = ferror(in) ? Z_ERRNO : Z_DATA_ERROR;
-            goto extract_ret;
-        }
-        (void)inflatePrime(&strm, here->bits, ret >> (8 - here->bits));
-    }
-    (void)inflateSetDictionary(&strm, here->window, WINSIZE);
+    if (point->bits)
+        INFLATEPRIME(&strm, point->bits, ch >> (8 - point->bits));
+    inflateSetDictionary(&strm, point->window, WINSIZE);
 
-    /* skip uncompressed bytes until offset reached, then satisfy request */
-    offset -= here->out;
-    strm.avail_in = 0;
-    skip = 1;                               /* while skipping to offset */
+    // Skip uncompressed bytes until offset reached, then satisfy request.
+    unsigned char input[CHUNK];
+    unsigned char discard[WINSIZE];
+    offset -= point->out;       // number of bytes to skip to get to offset
+    size_t left = len;          // number of bytes left to read after offset
     do {
-        /* define where to put uncompressed data, and how much */
-        if (offset == 0 && skip) {          /* at offset now */
-            strm.avail_out = len;
-            strm.next_out = buf;
-            skip = 0;                       /* only do this once */
-        }
-        if (offset > WINSIZE) {             /* skip WINSIZE bytes */
-            strm.avail_out = WINSIZE;
+        if (offset) {
+            // Discard up to offset uncompressed bytes.
+            strm.avail_out = offset < WINSIZE ? (unsigned)offset : WINSIZE;
             strm.next_out = discard;
-            offset -= WINSIZE;
         }
-        else if (offset != 0) {             /* last skip */
-            strm.avail_out = (unsigned)offset;
-            strm.next_out = discard;
-            offset = 0;
+        else {
+            // Uncompress up to left bytes into buf.
+            strm.avail_out = left < UINT_MAX ? (unsigned)left : UINT_MAX;
+            strm.next_out = buf + len - left;
         }
 
-        /* uncompress until avail_out filled, or end of stream */
-        do {
-            if (strm.avail_in == 0) {
-                strm.avail_in = fread(input, 1, CHUNK, in);
-                if (ferror(in)) {
-                    ret = Z_ERRNO;
-                    goto extract_ret;
-                }
-                if (strm.avail_in == 0) {
-                    ret = Z_DATA_ERROR;
-                    goto extract_ret;
-                }
-                strm.next_in = input;
-            }
-            ret = inflate(&strm, Z_NO_FLUSH);       /* normal inflate */
-            if (ret == Z_NEED_DICT)
-                ret = Z_DATA_ERROR;
-            if (ret == Z_MEM_ERROR || ret == Z_DATA_ERROR)
-                goto extract_ret;
-            if (ret == Z_STREAM_END)
+        // Uncompress, setting got to the number of bytes uncompressed.
+        if (strm.avail_in == 0) {
+            // Assure available input.
+            strm.avail_in = fread(input, 1, CHUNK, in);
+            if (strm.avail_in < CHUNK && ferror(in)) {
+                ret = Z_ERRNO;
                 break;
-        } while (strm.avail_out != 0);
-
-        /* if reach end of stream, then don't keep trying to get more */
-        if (ret == Z_STREAM_END)
-            break;
+            }
+            strm.next_in = input;
+        }
+        unsigned got = strm.avail_out;
+        ret = inflate(&strm, Z_NO_FLUSH);
+        got -= strm.avail_out;
+
+        // Update the appropriate count.
+        if (offset)
+            offset -= got;
+        else
+            left -= got;
+
+        // If we're at the end of a gzip member and there's more to read,
+        // continue to the next gzip member.
+        if (ret == Z_STREAM_END && index->mode == GZIP) {
+            // Discard the gzip trailer.
+            unsigned drop = 8;              // length of gzip trailer
+            if (strm.avail_in >= drop) {
+                strm.avail_in -= drop;
+                strm.next_in += drop;
+            }
+            else {
+                // Read and discard the remainder of the gzip trailer.
+                drop -= strm.avail_in;
+                strm.avail_in = 0;
+                do {
+                    if (getc(in) == EOF)
+                        // The input does not have a complete trailer.
+                        return ferror(in) ? Z_ERRNO : Z_BUF_ERROR;
+                } while (--drop);
+            }
 
-        /* do until offset reached and requested data read, or stream ends */
-    } while (skip);
+            if (strm.avail_in || ungetc(getc(in), in) != EOF) {
+                // There's more after the gzip trailer. Use inflate to skip the
+                // gzip header and resume the raw inflate there.
+                inflateReset2(&strm, GZIP);
+                do {
+                    if (strm.avail_in == 0) {
+                        strm.avail_in = fread(input, 1, CHUNK, in);
+                        if (strm.avail_in < CHUNK && ferror(in)) {
+                            ret = Z_ERRNO;
+                            break;
+                        }
+                        strm.next_in = input;
+                    }
+                    strm.avail_out = WINSIZE;
+                    strm.next_out = discard;
+                    ret = inflate(&strm, Z_BLOCK);  // stop at end of header
+                } while (ret == Z_OK && (strm.data_type & 0x80) == 0);
+                if (ret != Z_OK)
+                    break;
+                inflateReset2(&strm, RAW);
+            }
+        }
 
-    /* compute number of uncompressed bytes read after offset */
-    ret = skip ? 0 : len - strm.avail_out;
+        // Continue until we have the requested data, the deflate data has
+        // ended, or an error is encountered.
+    } while (ret == Z_OK && left);
+    inflateEnd(&strm);
 
-    /* clean up and return bytes read or error */
-  extract_ret:
-    (void)inflateEnd(&strm);
-    return ret;
+    // Return the number of uncompressed bytes read into buf, or the error.
+    return ret == Z_OK || ret == Z_STREAM_END ? len - left : ret;
 }
 
-/* Demonstrate the use of build_index() and extract() by processing the file
-   provided on the command line, and the extracting 16K from about 2/3rds of
-   the way through the uncompressed output, and writing that to stdout. */
-int main(int argc, char **argv)
-{
-    int len;
-    off_t offset;
-    FILE *in;
-    struct access *index = NULL;
-    unsigned char buf[CHUNK];
-
-    /* open input file */
-    if (argc != 2) {
-        fprintf(stderr, "usage: zran file.gz\n");
+#ifdef TEST
+
+#define SPAN 1048576L       // desired distance between access points
+#define LEN 16384           // number of bytes to extract
+
+// Demonstrate the use of deflate_index_build() and deflate_index_extract() by
+// processing the file provided on the command line, and extracting LEN bytes
+// from 2/3rds of the way through the uncompressed output, writing that to
+// stdout. An offset can be provided as the second argument, in which case the
+// data is extracted from there instead.
+int main(int argc, char **argv) {
+    // Open the input file.
+    if (argc < 2 || argc > 3) {
+        fprintf(stderr, "usage: zran file.raw [offset]\n");
         return 1;
     }
-    in = fopen(argv[1], "rb");
+    FILE *in = fopen(argv[1], "rb");
     if (in == NULL) {
         fprintf(stderr, "zran: could not open %s for reading\n", argv[1]);
         return 1;
     }
 
-    /* build index */
-    len = build_index(in, SPAN, &index);
+    // Get optional offset.
+    off_t offset = -1;
+    if (argc == 3) {
+        char *end;
+        offset = strtoll(argv[2], &end, 10);
+        if (*end || offset < 0) {
+            fprintf(stderr, "zran: %s is not a valid offset\n", argv[2]);
+            return 1;
+        }
+    }
+
+    // Build index.
+    struct deflate_index *index = NULL;
+    int len = deflate_index_build(in, SPAN, &index);
     if (len < 0) {
         fclose(in);
         switch (len) {
         case Z_MEM_ERROR:
             fprintf(stderr, "zran: out of memory\n");
             break;
+        case Z_BUF_ERROR:
+            fprintf(stderr, "zran: %s ended prematurely\n", argv[1]);
+            break;
         case Z_DATA_ERROR:
             fprintf(stderr, "zran: compressed data error in %s\n", argv[1]);
             break;
@@ -391,19 +511,23 @@ int main(int argc, char **argv)
     }
     fprintf(stderr, "zran: built index with %d access points\n", len);
 
-    /* use index by reading some bytes from an arbitrary offset */
-    offset = (index->list[index->have - 1].out << 1) / 3;
-    len = extract(in, index, offset, buf, CHUNK);
-    if (len < 0)
+    // Use index by reading some bytes from an arbitrary offset.
+    unsigned char buf[LEN];
+    if (offset == -1)
+        offset = ((index->length + 1) << 1) / 3;
+    ptrdiff_t got = deflate_index_extract(in, index, offset, buf, LEN);
+    if (got < 0)
         fprintf(stderr, "zran: extraction failed: %s error\n",
-                len == Z_MEM_ERROR ? "out of memory" : "input corrupted");
+                got == Z_MEM_ERROR ? "out of memory" : "input corrupted");
     else {
-        fwrite(buf, 1, len, stdout);
-        fprintf(stderr, "zran: extracted %d bytes at %llu\n", len, offset);
+        fwrite(buf, 1, got, stdout);
+        fprintf(stderr, "zran: extracted %ld bytes at %lld\n", got, offset);
     }
 
-    /* clean up and exit */
-    free_index(index);
+    // Clean up and exit.
+    deflate_index_free(index);
     fclose(in);
     return 0;
 }
+
+#endif
diff --git a/zlib/examples/zran.h b/zlib/examples/zran.h
new file mode 100644
index 0000000..ebf780d0
--- /dev/null
+++ b/zlib/examples/zran.h
@@ -0,0 +1,51 @@
+/* zran.h -- example of deflated stream indexing and random access
+ * Copyright (C) 2005, 2012, 2018, 2023 Mark Adler
+ * For conditions of distribution and use, see copyright notice in zlib.h
+ * Version 1.3  18 Feb 2023  Mark Adler */
+
+#include <stdio.h>
+#include "zlib.h"
+
+// Access point.
+typedef struct point {
+    off_t out;          // offset in uncompressed data
+    off_t in;           // offset in compressed file of first full byte
+    int bits;           // 0, or number of bits (1-7) from byte at in-1
+    unsigned char window[32768];    // preceding 32K of uncompressed data
+} point_t;
+
+// Access point list.
+struct deflate_index {
+    int have;           // number of access points in list
+    int mode;           // -15 for raw, 15 for zlib, or 31 for gzip
+    off_t length;       // total length of uncompressed data
+    point_t *list;      // allocated list of access points
+};
+
+// Make one pass through a zlib, gzip, or raw deflate compressed stream and
+// build an index, with access points about every span bytes of uncompressed
+// output. gzip files with multiple members are fully indexed. span should be
+// chosen to balance the speed of random access against the memory requirements
+// of the list, which is about 32K bytes per access point. The return value is
+// the number of access points on success (>= 1), Z_MEM_ERROR for out of
+// memory, Z_BUF_ERROR for a premature end of input, Z_DATA_ERROR for a format
+// or verification error in the input file, or Z_ERRNO for a file read error.
+// On success, *built points to the resulting index.
+int deflate_index_build(FILE *in, off_t span, struct deflate_index **built);
+
+// Use the index to read len bytes from offset into buf. Return the number of
+// bytes read or a negative error code. If data is requested past the end of
+// the uncompressed data, then deflate_index_extract() will return a value less
+// than len, indicating how much was actually read into buf. If given a valid
+// index, this function should not return an error unless the file was modified
+// somehow since the index was generated, given that deflate_index_build() had
+// validated all of the input. If nevertheless there is a failure, Z_BUF_ERROR
+// is returned if the compressed data ends prematurely, Z_DATA_ERROR if the
+// deflate compressed data is not valid, Z_MEM_ERROR if out of memory,
+// Z_STREAM_ERROR if the index is not valid, or Z_ERRNO if there is an error
+// reading or seeking on the input file.
+ptrdiff_t deflate_index_extract(FILE *in, struct deflate_index *index,
+                                off_t offset, unsigned char *buf, size_t len);
+
+// Deallocate an index built by deflate_index_build().
+void deflate_index_free(struct deflate_index *index);
diff --git a/zlib/gzclose.c b/zlib/gzclose.c
index caeb99a..48d6a86 100644
--- a/zlib/gzclose.c
+++ b/zlib/gzclose.c
@@ -8,9 +8,7 @@
 /* gzclose() is in a separate file so that it is linked in only if it is used.
    That way the other gzclose functions can be used instead to avoid linking in
    unneeded compression or decompression routines. */
-int ZEXPORT gzclose(file)
-    gzFile file;
-{
+int ZEXPORT gzclose(gzFile file) {
 #ifndef NO_GZCOMPRESS
     gz_statep state;
 
diff --git a/zlib/gzguts.h b/zlib/gzguts.h
index 6378d46..eba7208 100644
--- a/zlib/gzguts.h
+++ b/zlib/gzguts.h
@@ -1,5 +1,5 @@
 /* gzguts.h -- zlib internal header definitions for gz* operations
- * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler
+ * Copyright (C) 2004-2024 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -7,9 +7,8 @@
 #  ifndef _LARGEFILE_SOURCE
 #    define _LARGEFILE_SOURCE 1
 #  endif
-#  ifdef _FILE_OFFSET_BITS
-#    undef _FILE_OFFSET_BITS
-#  endif
+#  undef _FILE_OFFSET_BITS
+#  undef _TIME_BITS
 #endif
 
 #ifdef HAVE_HIDDEN
@@ -119,8 +118,8 @@
 
 /* gz* functions always use library allocation functions */
 #ifndef STDC
-  extern voidp  malloc OF((uInt size));
-  extern void   free   OF((voidpf ptr));
+  extern voidp  malloc(uInt size);
+  extern void   free(voidpf ptr);
 #endif
 
 /* get errno and strerror definition */
@@ -138,10 +137,10 @@
 
 /* provide prototypes for these when building zlib without LFS */
 #if !defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0
-    ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *));
-    ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int));
-    ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile));
-    ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile));
+    ZEXTERN gzFile ZEXPORT gzopen64(const char *, const char *);
+    ZEXTERN z_off64_t ZEXPORT gzseek64(gzFile, z_off64_t, int);
+    ZEXTERN z_off64_t ZEXPORT gztell64(gzFile);
+    ZEXTERN z_off64_t ZEXPORT gzoffset64(gzFile);
 #endif
 
 /* default memLevel */
@@ -190,6 +189,7 @@ typedef struct {
         /* just for writing */
     int level;              /* compression level */
     int strategy;           /* compression strategy */
+    int reset;              /* true if a reset is pending after a Z_FINISH */
         /* seek request */
     z_off64_t skip;         /* amount to skip (already rewound if backwards) */
     int seek;               /* true if seek request pending */
@@ -202,17 +202,13 @@ typedef struct {
 typedef gz_state FAR *gz_statep;
 
 /* shared functions */
-void ZLIB_INTERNAL gz_error OF((gz_statep, int, const char *));
+void ZLIB_INTERNAL gz_error(gz_statep, int, const char *);
 #if defined UNDER_CE
-char ZLIB_INTERNAL *gz_strwinerror OF((DWORD error));
+char ZLIB_INTERNAL *gz_strwinerror(DWORD error);
 #endif
 
 /* GT_OFF(x), where x is an unsigned value, is true if x > maximum z_off64_t
    value -- needed when comparing unsigned to z_off64_t, which is signed
    (possible z_off64_t types off_t, off64_t, and long are all signed) */
-#ifdef INT_MAX
-#  define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > INT_MAX)
-#else
-unsigned ZLIB_INTERNAL gz_intmax OF((void));
-#  define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax())
-#endif
+unsigned ZLIB_INTERNAL gz_intmax(void);
+#define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax())
diff --git a/zlib/gzlib.c b/zlib/gzlib.c
index 4105e6a..983153c 100644
--- a/zlib/gzlib.c
+++ b/zlib/gzlib.c
@@ -1,11 +1,11 @@
 /* gzlib.c -- zlib functions common to reading and writing gzip files
- * Copyright (C) 2004-2017 Mark Adler
+ * Copyright (C) 2004-2024 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
 #include "gzguts.h"
 
-#if defined(_WIN32) && !defined(__BORLANDC__) && !defined(__MINGW32__)
+#if defined(_WIN32) && !defined(__BORLANDC__)
 #  define LSEEK _lseeki64
 #else
 #if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0
@@ -15,10 +15,6 @@
 #endif
 #endif
 
-/* Local functions */
-local void gz_reset OF((gz_statep));
-local gzFile gz_open OF((const void *, int, const char *));
-
 #if defined UNDER_CE
 
 /* Map the Windows error number in ERROR to a locale-dependent error message
@@ -30,9 +26,7 @@ local gzFile gz_open OF((const void *, int, const char *));
 
    The gz_strwinerror function does not change the current setting of
    GetLastError. */
-char ZLIB_INTERNAL *gz_strwinerror (error)
-     DWORD error;
-{
+char ZLIB_INTERNAL *gz_strwinerror(DWORD error) {
     static char buf[1024];
 
     wchar_t *msgbuf;
@@ -72,15 +66,15 @@ char ZLIB_INTERNAL *gz_strwinerror (error)
 #endif /* UNDER_CE */
 
 /* Reset gzip file state */
-local void gz_reset(state)
-    gz_statep state;
-{
+local void gz_reset(gz_statep state) {
     state->x.have = 0;              /* no output data available */
     if (state->mode == GZ_READ) {   /* for reading ... */
         state->eof = 0;             /* not at end of file */
         state->past = 0;            /* have not read past end yet */
         state->how = LOOK;          /* look for gzip header */
     }
+    else                            /* for writing ... */
+        state->reset = 0;           /* no deflateReset pending */
     state->seek = 0;                /* no seek request pending */
     gz_error(state, Z_OK, NULL);    /* clear error */
     state->x.pos = 0;               /* no uncompressed data yet */
@@ -88,11 +82,7 @@ local void gz_reset(state)
 }
 
 /* Open a gzip file either by name or file descriptor. */
-local gzFile gz_open(path, fd, mode)
-    const void *path;
-    int fd;
-    const char *mode;
-{
+local gzFile gz_open(const void *path, int fd, const char *mode) {
     gz_statep state;
     z_size_t len;
     int oflag;
@@ -267,26 +257,17 @@ local gzFile gz_open(path, fd, mode)
 }
 
 /* -- see zlib.h -- */
-gzFile ZEXPORT gzopen(path, mode)
-    const char *path;
-    const char *mode;
-{
+gzFile ZEXPORT gzopen(const char *path, const char *mode) {
     return gz_open(path, -1, mode);
 }
 
 /* -- see zlib.h -- */
-gzFile ZEXPORT gzopen64(path, mode)
-    const char *path;
-    const char *mode;
-{
+gzFile ZEXPORT gzopen64(const char *path, const char *mode) {
     return gz_open(path, -1, mode);
 }
 
 /* -- see zlib.h -- */
-gzFile ZEXPORT gzdopen(fd, mode)
-    int fd;
-    const char *mode;
-{
+gzFile ZEXPORT gzdopen(int fd, const char *mode) {
     char *path;         /* identifier for error messages */
     gzFile gz;
 
@@ -304,19 +285,13 @@ gzFile ZEXPORT gzdopen(fd, mode)
 
 /* -- see zlib.h -- */
 #ifdef WIDECHAR
-gzFile ZEXPORT gzopen_w(path, mode)
-    const wchar_t *path;
-    const char *mode;
-{
+gzFile ZEXPORT gzopen_w(const wchar_t *path, const char *mode) {
     return gz_open(path, -2, mode);
 }
 #endif
 
 /* -- see zlib.h -- */
-int ZEXPORT gzbuffer(file, size)
-    gzFile file;
-    unsigned size;
-{
+int ZEXPORT gzbuffer(gzFile file, unsigned size) {
     gz_statep state;
 
     /* get internal structure and check integrity */
@@ -333,16 +308,14 @@ int ZEXPORT gzbuffer(file, size)
     /* check and set requested size */
     if ((size << 1) < size)
         return -1;              /* need to be able to double it */
-    if (size < 2)
-        size = 2;               /* need two bytes to check magic header */
+    if (size < 8)
+        size = 8;               /* needed to behave well with flushing */
     state->want = size;
     return 0;
 }
 
 /* -- see zlib.h -- */
-int ZEXPORT gzrewind(file)
-    gzFile file;
-{
+int ZEXPORT gzrewind(gzFile file) {
     gz_statep state;
 
     /* get internal structure */
@@ -363,11 +336,7 @@ int ZEXPORT gzrewind(file)
 }
 
 /* -- see zlib.h -- */
-z_off64_t ZEXPORT gzseek64(file, offset, whence)
-    gzFile file;
-    z_off64_t offset;
-    int whence;
-{
+z_off64_t ZEXPORT gzseek64(gzFile file, z_off64_t offset, int whence) {
     unsigned n;
     z_off64_t ret;
     gz_statep state;
@@ -397,7 +366,7 @@ z_off64_t ZEXPORT gzseek64(file, offset, whence)
     /* if within raw area while reading, just go there */
     if (state->mode == GZ_READ && state->how == COPY &&
             state->x.pos + offset >= 0) {
-        ret = LSEEK(state->fd, offset - state->x.have, SEEK_CUR);
+        ret = LSEEK(state->fd, offset - (z_off64_t)state->x.have, SEEK_CUR);
         if (ret == -1)
             return -1;
         state->x.have = 0;
@@ -440,11 +409,7 @@ z_off64_t ZEXPORT gzseek64(file, offset, whence)
 }
 
 /* -- see zlib.h -- */
-z_off_t ZEXPORT gzseek(file, offset, whence)
-    gzFile file;
-    z_off_t offset;
-    int whence;
-{
+z_off_t ZEXPORT gzseek(gzFile file, z_off_t offset, int whence) {
     z_off64_t ret;
 
     ret = gzseek64(file, (z_off64_t)offset, whence);
@@ -452,9 +417,7 @@ z_off_t ZEXPORT gzseek(file, offset, whence)
 }
 
 /* -- see zlib.h -- */
-z_off64_t ZEXPORT gztell64(file)
-    gzFile file;
-{
+z_off64_t ZEXPORT gztell64(gzFile file) {
     gz_statep state;
 
     /* get internal structure and check integrity */
@@ -469,9 +432,7 @@ z_off64_t ZEXPORT gztell64(file)
 }
 
 /* -- see zlib.h -- */
-z_off_t ZEXPORT gztell(file)
-    gzFile file;
-{
+z_off_t ZEXPORT gztell(gzFile file) {
     z_off64_t ret;
 
     ret = gztell64(file);
@@ -479,9 +440,7 @@ z_off_t ZEXPORT gztell(file)
 }
 
 /* -- see zlib.h -- */
-z_off64_t ZEXPORT gzoffset64(file)
-    gzFile file;
-{
+z_off64_t ZEXPORT gzoffset64(gzFile file) {
     z_off64_t offset;
     gz_statep state;
 
@@ -502,9 +461,7 @@ z_off64_t ZEXPORT gzoffset64(file)
 }
 
 /* -- see zlib.h -- */
-z_off_t ZEXPORT gzoffset(file)
-    gzFile file;
-{
+z_off_t ZEXPORT gzoffset(gzFile file) {
     z_off64_t ret;
 
     ret = gzoffset64(file);
@@ -512,9 +469,7 @@ z_off_t ZEXPORT gzoffset(file)
 }
 
 /* -- see zlib.h -- */
-int ZEXPORT gzeof(file)
-    gzFile file;
-{
+int ZEXPORT gzeof(gzFile file) {
     gz_statep state;
 
     /* get internal structure and check integrity */
@@ -529,10 +484,7 @@ int ZEXPORT gzeof(file)
 }
 
 /* -- see zlib.h -- */
-const char * ZEXPORT gzerror(file, errnum)
-    gzFile file;
-    int *errnum;
-{
+const char * ZEXPORT gzerror(gzFile file, int *errnum) {
     gz_statep state;
 
     /* get internal structure and check integrity */
@@ -550,9 +502,7 @@ const char * ZEXPORT gzerror(file, errnum)
 }
 
 /* -- see zlib.h -- */
-void ZEXPORT gzclearerr(file)
-    gzFile file;
-{
+void ZEXPORT gzclearerr(gzFile file) {
     gz_statep state;
 
     /* get internal structure and check integrity */
@@ -576,11 +526,7 @@ void ZEXPORT gzclearerr(file)
    memory).  Simply save the error message as a static string.  If there is an
    allocation failure constructing the error message, then convert the error to
    out of memory. */
-void ZLIB_INTERNAL gz_error(state, err, msg)
-    gz_statep state;
-    int err;
-    const char *msg;
-{
+void ZLIB_INTERNAL gz_error(gz_statep state, int err, const char *msg) {
     /* free previously allocated message and clear */
     if (state->msg != NULL) {
         if (state->err != Z_MEM_ERROR)
@@ -617,21 +563,20 @@ void ZLIB_INTERNAL gz_error(state, err, msg)
 #endif
 }
 
-#ifndef INT_MAX
 /* portably return maximum value for an int (when limits.h presumed not
    available) -- we need to do this to cover cases where 2's complement not
    used, since C standard permits 1's complement and sign-bit representations,
    otherwise we could just use ((unsigned)-1) >> 1 */
-unsigned ZLIB_INTERNAL gz_intmax()
-{
-    unsigned p, q;
-
-    p = 1;
+unsigned ZLIB_INTERNAL gz_intmax(void) {
+#ifdef INT_MAX
+    return INT_MAX;
+#else
+    unsigned p = 1, q;
     do {
         q = p;
         p <<= 1;
         p++;
     } while (p > q);
     return q >> 1;
-}
 #endif
+}
diff --git a/zlib/gzread.c b/zlib/gzread.c
index 956b91e..4168cbc 100644
--- a/zlib/gzread.c
+++ b/zlib/gzread.c
@@ -1,29 +1,16 @@
 /* gzread.c -- zlib functions for reading gzip files
- * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler
+ * Copyright (C) 2004-2017 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
 #include "gzguts.h"
 
-/* Local functions */
-local int gz_load OF((gz_statep, unsigned char *, unsigned, unsigned *));
-local int gz_avail OF((gz_statep));
-local int gz_look OF((gz_statep));
-local int gz_decomp OF((gz_statep));
-local int gz_fetch OF((gz_statep));
-local int gz_skip OF((gz_statep, z_off64_t));
-local z_size_t gz_read OF((gz_statep, voidp, z_size_t));
-
 /* Use read() to load a buffer -- return -1 on error, otherwise 0.  Read from
    state->fd, and update state->eof, state->err, and state->msg as appropriate.
    This function needs to loop on read(), since read() is not guaranteed to
    read the number of bytes requested, depending on the type of descriptor. */
-local int gz_load(state, buf, len, have)
-    gz_statep state;
-    unsigned char *buf;
-    unsigned len;
-    unsigned *have;
-{
+local int gz_load(gz_statep state, unsigned char *buf, unsigned len,
+                  unsigned *have) {
     int ret;
     unsigned get, max = ((unsigned)-1 >> 2) + 1;
 
@@ -53,9 +40,7 @@ local int gz_load(state, buf, len, have)
    If strm->avail_in != 0, then the current data is moved to the beginning of
    the input buffer, and then the remainder of the buffer is loaded with the
    available data from the input file. */
-local int gz_avail(state)
-    gz_statep state;
-{
+local int gz_avail(gz_statep state) {
     unsigned got;
     z_streamp strm = &(state->strm);
 
@@ -88,9 +73,7 @@ local int gz_avail(state)
    case, all further file reads will be directly to either the output buffer or
    a user buffer.  If decompressing, the inflate state will be initialized.
    gz_look() will return 0 on success or -1 on failure. */
-local int gz_look(state)
-    gz_statep state;
-{
+local int gz_look(gz_statep state) {
     z_streamp strm = &(state->strm);
 
     /* allocate read buffers and inflate memory */
@@ -157,11 +140,9 @@ local int gz_look(state)
        the output buffer is larger than the input buffer, which also assures
        space for gzungetc() */
     state->x.next = state->out;
-    if (strm->avail_in) {
-        memcpy(state->x.next, strm->next_in, strm->avail_in);
-        state->x.have = strm->avail_in;
-        strm->avail_in = 0;
-    }
+    memcpy(state->x.next, strm->next_in, strm->avail_in);
+    state->x.have = strm->avail_in;
+    strm->avail_in = 0;
     state->how = COPY;
     state->direct = 1;
     return 0;
@@ -172,9 +153,7 @@ local int gz_look(state)
    data.  If the gzip stream completes, state->how is reset to LOOK to look for
    the next gzip stream or raw data, once state->x.have is depleted.  Returns 0
    on success, -1 on failure. */
-local int gz_decomp(state)
-    gz_statep state;
-{
+local int gz_decomp(gz_statep state) {
     int ret = Z_OK;
     unsigned had;
     z_streamp strm = &(state->strm);
@@ -226,9 +205,7 @@ local int gz_decomp(state)
    looked for to determine whether to copy or decompress.  Returns -1 on error,
    otherwise 0.  gz_fetch() will leave state->how as COPY or GZIP unless the
    end of the input file has been reached and all data has been processed.  */
-local int gz_fetch(state)
-    gz_statep state;
-{
+local int gz_fetch(gz_statep state) {
     z_streamp strm = &(state->strm);
 
     do {
@@ -256,10 +233,7 @@ local int gz_fetch(state)
 }
 
 /* Skip len uncompressed bytes of output.  Return -1 on error, 0 on success. */
-local int gz_skip(state, len)
-    gz_statep state;
-    z_off64_t len;
-{
+local int gz_skip(gz_statep state, z_off64_t len) {
     unsigned n;
 
     /* skip over len bytes or reach end-of-file, whichever comes first */
@@ -291,11 +265,7 @@ local int gz_skip(state, len)
    input.  Return the number of bytes read.  If zero is returned, either the
    end of file was reached, or there was an error.  state->err must be
    consulted in that case to determine which. */
-local z_size_t gz_read(state, buf, len)
-    gz_statep state;
-    voidp buf;
-    z_size_t len;
-{
+local z_size_t gz_read(gz_statep state, voidp buf, z_size_t len) {
     z_size_t got;
     unsigned n;
 
@@ -314,9 +284,9 @@ local z_size_t gz_read(state, buf, len)
     got = 0;
     do {
         /* set n to the maximum amount of len that fits in an unsigned int */
-        n = -1;
+        n = (unsigned)-1;
         if (n > len)
-            n = len;
+            n = (unsigned)len;
 
         /* first just try copying data from the output buffer */
         if (state->x.have) {
@@ -372,11 +342,7 @@ local z_size_t gz_read(state, buf, len)
 }
 
 /* -- see zlib.h -- */
-int ZEXPORT gzread(file, buf, len)
-    gzFile file;
-    voidp buf;
-    unsigned len;
-{
+int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) {
     gz_statep state;
 
     /* get internal structure */
@@ -397,7 +363,7 @@ int ZEXPORT gzread(file, buf, len)
     }
 
     /* read len or fewer bytes to buf */
-    len = gz_read(state, buf, len);
+    len = (unsigned)gz_read(state, buf, len);
 
     /* check for an error */
     if (len == 0 && state->err != Z_OK && state->err != Z_BUF_ERROR)
@@ -408,12 +374,7 @@ int ZEXPORT gzread(file, buf, len)
 }
 
 /* -- see zlib.h -- */
-z_size_t ZEXPORT gzfread(buf, size, nitems, file)
-    voidp buf;
-    z_size_t size;
-    z_size_t nitems;
-    gzFile file;
-{
+z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems, gzFile file) {
     z_size_t len;
     gz_statep state;
 
@@ -444,10 +405,7 @@ z_size_t ZEXPORT gzfread(buf, size, nitems, file)
 #else
 #  undef gzgetc
 #endif
-int ZEXPORT gzgetc(file)
-    gzFile file;
-{
-    int ret;
+int ZEXPORT gzgetc(gzFile file) {
     unsigned char buf[1];
     gz_statep state;
 
@@ -469,21 +427,15 @@ int ZEXPORT gzgetc(file)
     }
 
     /* nothing there -- try gz_read() */
-    ret = gz_read(state, buf, 1);
-    return ret < 1 ? -1 : buf[0];
+    return gz_read(state, buf, 1) < 1 ? -1 : buf[0];
 }
 
-int ZEXPORT gzgetc_(file)
-gzFile file;
-{
+int ZEXPORT gzgetc_(gzFile file) {
     return gzgetc(file);
 }
 
 /* -- see zlib.h -- */
-int ZEXPORT gzungetc(c, file)
-    int c;
-    gzFile file;
-{
+int ZEXPORT gzungetc(int c, gzFile file) {
     gz_statep state;
 
     /* get internal structure */
@@ -491,6 +443,10 @@ int ZEXPORT gzungetc(c, file)
         return -1;
     state = (gz_statep)file;
 
+    /* in case this was just opened, set up the input buffer */
+    if (state->mode == GZ_READ && state->how == LOOK && state->x.have == 0)
+        (void)gz_look(state);
+
     /* check that we're reading and that there's no (serious) error */
     if (state->mode != GZ_READ ||
         (state->err != Z_OK && state->err != Z_BUF_ERROR))
@@ -540,11 +496,7 @@ int ZEXPORT gzungetc(c, file)
 }
 
 /* -- see zlib.h -- */
-char * ZEXPORT gzgets(file, buf, len)
-    gzFile file;
-    char *buf;
-    int len;
-{
+char * ZEXPORT gzgets(gzFile file, char *buf, int len) {
     unsigned left, n;
     char *str;
     unsigned char *eol;
@@ -604,9 +556,7 @@ char * ZEXPORT gzgets(file, buf, len)
 }
 
 /* -- see zlib.h -- */
-int ZEXPORT gzdirect(file)
-    gzFile file;
-{
+int ZEXPORT gzdirect(gzFile file) {
     gz_statep state;
 
     /* get internal structure */
@@ -624,9 +574,7 @@ int ZEXPORT gzdirect(file)
 }
 
 /* -- see zlib.h -- */
-int ZEXPORT gzclose_r(file)
-    gzFile file;
-{
+int ZEXPORT gzclose_r(gzFile file) {
     int ret, err;
     gz_statep state;
 
diff --git a/zlib/gzwrite.c b/zlib/gzwrite.c
index c7b5651..435b462 100644
--- a/zlib/gzwrite.c
+++ b/zlib/gzwrite.c
@@ -1,22 +1,14 @@
 /* gzwrite.c -- zlib functions for writing gzip files
- * Copyright (C) 2004-2017 Mark Adler
+ * Copyright (C) 2004-2019 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
 #include "gzguts.h"
 
-/* Local functions */
-local int gz_init OF((gz_statep));
-local int gz_comp OF((gz_statep, int));
-local int gz_zero OF((gz_statep, z_off64_t));
-local z_size_t gz_write OF((gz_statep, voidpc, z_size_t));
-
 /* Initialize state for writing a gzip file.  Mark initialization by setting
    state->size to non-zero.  Return -1 on a memory allocation failure, or 0 on
    success. */
-local int gz_init(state)
-    gz_statep state;
-{
+local int gz_init(gz_statep state) {
     int ret;
     z_streamp strm = &(state->strm);
 
@@ -70,10 +62,7 @@ local int gz_init(state)
    deflate() flush value.  If flush is Z_FINISH, then the deflate() state is
    reset to start a new gzip stream.  If gz->direct is true, then simply write
    to the output file without compressing, and ignore flush. */
-local int gz_comp(state, flush)
-    gz_statep state;
-    int flush;
-{
+local int gz_comp(gz_statep state, int flush) {
     int ret, writ;
     unsigned have, put, max = ((unsigned)-1 >> 2) + 1;
     z_streamp strm = &(state->strm);
@@ -97,6 +86,15 @@ local int gz_comp(state, flush)
         return 0;
     }
 
+    /* check for a pending reset */
+    if (state->reset) {
+        /* don't start a new gzip member unless there is data to write */
+        if (strm->avail_in == 0)
+            return 0;
+        deflateReset(strm);
+        state->reset = 0;
+    }
+
     /* run deflate() on provided input until it produces no more output */
     ret = Z_OK;
     do {
@@ -134,7 +132,7 @@ local int gz_comp(state, flush)
 
     /* if that completed a deflate stream, allow another to start */
     if (flush == Z_FINISH)
-        deflateReset(strm);
+        state->reset = 1;
 
     /* all done, no errors */
     return 0;
@@ -142,10 +140,7 @@ local int gz_comp(state, flush)
 
 /* Compress len zeros to output.  Return -1 on a write error or memory
    allocation failure by gz_comp(), or 0 on success. */
-local int gz_zero(state, len)
-    gz_statep state;
-    z_off64_t len;
-{
+local int gz_zero(gz_statep state, z_off64_t len) {
     int first;
     unsigned n;
     z_streamp strm = &(state->strm);
@@ -175,11 +170,7 @@ local int gz_zero(state, len)
 
 /* Write len bytes from buf to file.  Return the number of bytes written.  If
    the returned value is less than len, then there was an error. */
-local z_size_t gz_write(state, buf, len)
-    gz_statep state;
-    voidpc buf;
-    z_size_t len;
-{
+local z_size_t gz_write(gz_statep state, voidpc buf, z_size_t len) {
     z_size_t put = len;
 
     /* if len is zero, avoid unnecessary operations */
@@ -209,7 +200,7 @@ local z_size_t gz_write(state, buf, len)
                               state->in);
             copy = state->size - have;
             if (copy > len)
-                copy = len;
+                copy = (unsigned)len;
             memcpy(state->in + have, buf, copy);
             state->strm.avail_in += copy;
             state->x.pos += copy;
@@ -229,7 +220,7 @@ local z_size_t gz_write(state, buf, len)
         do {
             unsigned n = (unsigned)-1;
             if (n > len)
-                n = len;
+                n = (unsigned)len;
             state->strm.avail_in = n;
             state->x.pos += n;
             if (gz_comp(state, Z_NO_FLUSH) == -1)
@@ -243,11 +234,7 @@ local z_size_t gz_write(state, buf, len)
 }
 
 /* -- see zlib.h -- */
-int ZEXPORT gzwrite(file, buf, len)
-    gzFile file;
-    voidpc buf;
-    unsigned len;
-{
+int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len) {
     gz_statep state;
 
     /* get internal structure */
@@ -271,12 +258,8 @@ int ZEXPORT gzwrite(file, buf, len)
 }
 
 /* -- see zlib.h -- */
-z_size_t ZEXPORT gzfwrite(buf, size, nitems, file)
-    voidpc buf;
-    z_size_t size;
-    z_size_t nitems;
-    gzFile file;
-{
+z_size_t ZEXPORT gzfwrite(voidpc buf, z_size_t size, z_size_t nitems,
+                          gzFile file) {
     z_size_t len;
     gz_statep state;
 
@@ -301,10 +284,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file)
 }
 
 /* -- see zlib.h -- */
-int ZEXPORT gzputc(file, c)
-    gzFile file;
-    int c;
-{
+int ZEXPORT gzputc(gzFile file, int c) {
     unsigned have;
     unsigned char buf[1];
     gz_statep state;
@@ -349,12 +329,8 @@ int ZEXPORT gzputc(file, c)
 }
 
 /* -- see zlib.h -- */
-int ZEXPORT gzputs(file, str)
-    gzFile file;
-    const char *str;
-{
-    int ret;
-    z_size_t len;
+int ZEXPORT gzputs(gzFile file, const char *s) {
+    z_size_t len, put;
     gz_statep state;
 
     /* get internal structure */
@@ -367,17 +343,20 @@ int ZEXPORT gzputs(file, str)
         return -1;
 
     /* write string */
-    len = strlen(str);
-    ret = gz_write(state, str, len);
-    return ret == 0 && len != 0 ? -1 : ret;
+    len = strlen(s);
+    if ((int)len < 0 || (unsigned)len != len) {
+        gz_error(state, Z_STREAM_ERROR, "string length does not fit in int");
+        return -1;
+    }
+    put = gz_write(state, s, len);
+    return put < len ? -1 : (int)len;
 }
 
 #if defined(STDC) || defined(Z_HAVE_STDARG_H)
 #include <stdarg.h>
 
 /* -- see zlib.h -- */
-int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va)
-{
+int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) {
     int len;
     unsigned left;
     char *next;
@@ -441,15 +420,14 @@ int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va)
         strm->avail_in = state->size;
         if (gz_comp(state, Z_NO_FLUSH) == -1)
             return state->err;
-        memcpy(state->in, state->in + state->size, left);
+        memmove(state->in, state->in + state->size, left);
         strm->next_in = state->in;
         strm->avail_in = left;
     }
     return len;
 }
 
-int ZEXPORTVA gzprintf(gzFile file, const char *format, ...)
-{
+int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) {
     va_list va;
     int ret;
 
@@ -462,13 +440,10 @@ int ZEXPORTVA gzprintf(gzFile file, const char *format, ...)
 #else /* !STDC && !Z_HAVE_STDARG_H */
 
 /* -- see zlib.h -- */
-int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10,
-                       a11, a12, a13, a14, a15, a16, a17, a18, a19, a20)
-    gzFile file;
-    const char *format;
-    int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10,
-        a11, a12, a13, a14, a15, a16, a17, a18, a19, a20;
-{
+int ZEXPORTVA gzprintf(gzFile file, const char *format, int a1, int a2, int a3,
+                       int a4, int a5, int a6, int a7, int a8, int a9, int a10,
+                       int a11, int a12, int a13, int a14, int a15, int a16,
+                       int a17, int a18, int a19, int a20) {
     unsigned len, left;
     char *next;
     gz_statep state;
@@ -540,7 +515,7 @@ int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10,
         strm->avail_in = state->size;
         if (gz_comp(state, Z_NO_FLUSH) == -1)
             return state->err;
-        memcpy(state->in, state->in + state->size, left);
+        memmove(state->in, state->in + state->size, left);
         strm->next_in = state->in;
         strm->avail_in = left;
     }
@@ -550,10 +525,7 @@ int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10,
 #endif
 
 /* -- see zlib.h -- */
-int ZEXPORT gzflush(file, flush)
-    gzFile file;
-    int flush;
-{
+int ZEXPORT gzflush(gzFile file, int flush) {
     gz_statep state;
 
     /* get internal structure */
@@ -582,11 +554,7 @@ int ZEXPORT gzflush(file, flush)
 }
 
 /* -- see zlib.h -- */
-int ZEXPORT gzsetparams(file, level, strategy)
-    gzFile file;
-    int level;
-    int strategy;
-{
+int ZEXPORT gzsetparams(gzFile file, int level, int strategy) {
     gz_statep state;
     z_streamp strm;
 
@@ -597,7 +565,7 @@ int ZEXPORT gzsetparams(file, level, strategy)
     strm = &(state->strm);
 
     /* check that we're writing and that there's no error */
-    if (state->mode != GZ_WRITE || state->err != Z_OK)
+    if (state->mode != GZ_WRITE || state->err != Z_OK || state->direct)
         return Z_STREAM_ERROR;
 
     /* if no change is requested, then do nothing */
@@ -624,9 +592,7 @@ int ZEXPORT gzsetparams(file, level, strategy)
 }
 
 /* -- see zlib.h -- */
-int ZEXPORT gzclose_w(file)
-    gzFile file;
-{
+int ZEXPORT gzclose_w(gzFile file) {
     int ret = Z_OK;
     gz_statep state;
 
diff --git a/zlib/infback.c b/zlib/infback.c
index 59679ec..e7b25b3 100644
--- a/zlib/infback.c
+++ b/zlib/infback.c
@@ -1,5 +1,5 @@
 /* infback.c -- inflate using a call-back interface
- * Copyright (C) 1995-2016 Mark Adler
+ * Copyright (C) 1995-2022 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -15,9 +15,6 @@
 #include "inflate.h"
 #include "inffast.h"
 
-/* function prototypes */
-local void fixedtables OF((struct inflate_state FAR *state));
-
 /*
    strm provides memory allocation functions in zalloc and zfree, or
    Z_NULL to use the library memory allocation functions.
@@ -25,13 +22,9 @@ local void fixedtables OF((struct inflate_state FAR *state));
    windowBits is in the range 8..15, and window is a user-supplied
    window and output buffer that is 2**windowBits bytes.
  */
-int ZEXPORT inflateBackInit_(strm, windowBits, window, version, stream_size)
-z_streamp strm;
-int windowBits;
-unsigned char FAR *window;
-const char *version;
-int stream_size;
-{
+int ZEXPORT inflateBackInit_(z_streamp strm, int windowBits,
+                             unsigned char FAR *window, const char *version,
+                             int stream_size) {
     struct inflate_state FAR *state;
 
     if (version == Z_NULL || version[0] != ZLIB_VERSION[0] ||
@@ -66,6 +59,7 @@ int stream_size;
     state->window = window;
     state->wnext = 0;
     state->whave = 0;
+    state->sane = 1;
     return Z_OK;
 }
 
@@ -79,9 +73,7 @@ int stream_size;
    used for threaded applications, since the rewriting of the tables and virgin
    may not be thread-safe.
  */
-local void fixedtables(state)
-struct inflate_state FAR *state;
-{
+local void fixedtables(struct inflate_state FAR *state) {
 #ifdef BUILDFIXED
     static int virgin = 1;
     static code *lenfix, *distfix;
@@ -247,13 +239,8 @@ struct inflate_state FAR *state;
    inflateBack() can also return Z_STREAM_ERROR if the input parameters
    are not correct, i.e. strm is Z_NULL or the state was not initialized.
  */
-int ZEXPORT inflateBack(strm, in, in_desc, out, out_desc)
-z_streamp strm;
-in_func in;
-void FAR *in_desc;
-out_func out;
-void FAR *out_desc;
-{
+int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc,
+                        out_func out, void FAR *out_desc) {
     struct inflate_state FAR *state;
     z_const unsigned char FAR *next;    /* next input */
     unsigned char FAR *put;     /* next output */
@@ -477,6 +464,7 @@ void FAR *out_desc;
             }
             Tracev((stderr, "inflate:       codes ok\n"));
             state->mode = LEN;
+                /* fallthrough */
 
         case LEN:
             /* use inflate_fast() if we have enough input and output */
@@ -604,33 +592,33 @@ void FAR *out_desc;
             break;
 
         case DONE:
-            /* inflate stream terminated properly -- write leftover output */
+            /* inflate stream terminated properly */
             ret = Z_STREAM_END;
-            if (left < state->wsize) {
-                if (out(out_desc, state->window, state->wsize - left))
-                    ret = Z_BUF_ERROR;
-            }
             goto inf_leave;
 
         case BAD:
             ret = Z_DATA_ERROR;
             goto inf_leave;
 
-        default:                /* can't happen, but makes compilers happy */
+        default:
+            /* can't happen, but makes compilers happy */
             ret = Z_STREAM_ERROR;
             goto inf_leave;
         }
 
-    /* Return unused input */
+    /* Write leftover output and return unused input */
   inf_leave:
+    if (left < state->wsize) {
+        if (out(out_desc, state->window, state->wsize - left) &&
+            ret == Z_STREAM_END)
+            ret = Z_BUF_ERROR;
+    }
     strm->next_in = next;
     strm->avail_in = have;
     return ret;
 }
 
-int ZEXPORT inflateBackEnd(strm)
-z_streamp strm;
-{
+int ZEXPORT inflateBackEnd(z_streamp strm) {
     if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0)
         return Z_STREAM_ERROR;
     ZFREE(strm, strm->state);
diff --git a/zlib/inffast.c b/zlib/inffast.c
index 0dbd1db..9354676 100644
--- a/zlib/inffast.c
+++ b/zlib/inffast.c
@@ -47,10 +47,7 @@
       requires strm->avail_out >= 258 for each loop to avoid checking for
       output space.
  */
-void ZLIB_INTERNAL inflate_fast(strm, start)
-z_streamp strm;
-unsigned start;         /* inflate()'s starting value for strm->avail_out */
-{
+void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) {
     struct inflate_state FAR *state;
     z_const unsigned char FAR *in;      /* local strm->next_in */
     z_const unsigned char FAR *last;    /* have enough input while in < last */
@@ -70,7 +67,7 @@ unsigned start;         /* inflate()'s starting value for strm->avail_out */
     code const FAR *dcode;      /* local strm->distcode */
     unsigned lmask;             /* mask for first level of length codes */
     unsigned dmask;             /* mask for first level of distance codes */
-    code here;                  /* retrieved table entry */
+    code const *here;           /* retrieved table entry */
     unsigned op;                /* code bits, operation, extra bits, or */
                                 /*  window position, window bytes to copy */
     unsigned len;               /* match length, unused bytes */
@@ -107,20 +104,20 @@ unsigned start;         /* inflate()'s starting value for strm->avail_out */
             hold += (unsigned long)(*in++) << bits;
             bits += 8;
         }
-        here = lcode[hold & lmask];
+        here = lcode + (hold & lmask);
       dolen:
-        op = (unsigned)(here.bits);
+        op = (unsigned)(here->bits);
         hold >>= op;
         bits -= op;
-        op = (unsigned)(here.op);
+        op = (unsigned)(here->op);
         if (op == 0) {                          /* literal */
-            Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ?
+            Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ?
                     "inflate:         literal '%c'\n" :
-                    "inflate:         literal 0x%02x\n", here.val));
-            *out++ = (unsigned char)(here.val);
+                    "inflate:         literal 0x%02x\n", here->val));
+            *out++ = (unsigned char)(here->val);
         }
         else if (op & 16) {                     /* length base */
-            len = (unsigned)(here.val);
+            len = (unsigned)(here->val);
             op &= 15;                           /* number of extra bits */
             if (op) {
                 if (bits < op) {
@@ -138,14 +135,14 @@ unsigned start;         /* inflate()'s starting value for strm->avail_out */
                 hold += (unsigned long)(*in++) << bits;
                 bits += 8;
             }
-            here = dcode[hold & dmask];
+            here = dcode + (hold & dmask);
           dodist:
-            op = (unsigned)(here.bits);
+            op = (unsigned)(here->bits);
             hold >>= op;
             bits -= op;
-            op = (unsigned)(here.op);
+            op = (unsigned)(here->op);
             if (op & 16) {                      /* distance base */
-                dist = (unsigned)(here.val);
+                dist = (unsigned)(here->val);
                 op &= 15;                       /* number of extra bits */
                 if (bits < op) {
                     hold += (unsigned long)(*in++) << bits;
@@ -264,7 +261,7 @@ unsigned start;         /* inflate()'s starting value for strm->avail_out */
                 }
             }
             else if ((op & 64) == 0) {          /* 2nd level distance code */
-                here = dcode[here.val + (hold & ((1U << op) - 1))];
+                here = dcode + here->val + (hold & ((1U << op) - 1));
                 goto dodist;
             }
             else {
@@ -274,7 +271,7 @@ unsigned start;         /* inflate()'s starting value for strm->avail_out */
             }
         }
         else if ((op & 64) == 0) {              /* 2nd level length code */
-            here = lcode[here.val + (hold & ((1U << op) - 1))];
+            here = lcode + here->val + (hold & ((1U << op) - 1));
             goto dolen;
         }
         else if (op & 32) {                     /* end-of-block */
diff --git a/zlib/inffast.h b/zlib/inffast.h
index e5c1aa4..49c6d15 100644
--- a/zlib/inffast.h
+++ b/zlib/inffast.h
@@ -8,4 +8,4 @@
    subject to change. Applications should only use zlib.h.
  */
 
-void ZLIB_INTERNAL inflate_fast OF((z_streamp strm, unsigned start));
+void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start);
diff --git a/zlib/inflate.c b/zlib/inflate.c
index ac333e8..94ecff0 100644
--- a/zlib/inflate.c
+++ b/zlib/inflate.c
@@ -1,5 +1,5 @@
 /* inflate.c -- zlib decompression
- * Copyright (C) 1995-2016 Mark Adler
+ * Copyright (C) 1995-2022 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -91,20 +91,7 @@
 #  endif
 #endif
 
-/* function prototypes */
-local int inflateStateCheck OF((z_streamp strm));
-local void fixedtables OF((struct inflate_state FAR *state));
-local int updatewindow OF((z_streamp strm, const unsigned char FAR *end,
-                           unsigned copy));
-#ifdef BUILDFIXED
-   void makefixed OF((void));
-#endif
-local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf,
-                              unsigned len));
-
-local int inflateStateCheck(strm)
-z_streamp strm;
-{
+local int inflateStateCheck(z_streamp strm) {
     struct inflate_state FAR *state;
     if (strm == Z_NULL ||
         strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0)
@@ -116,9 +103,7 @@ z_streamp strm;
     return 0;
 }
 
-int ZEXPORT inflateResetKeep(strm)
-z_streamp strm;
-{
+int ZEXPORT inflateResetKeep(z_streamp strm) {
     struct inflate_state FAR *state;
 
     if (inflateStateCheck(strm)) return Z_STREAM_ERROR;
@@ -130,6 +115,7 @@ z_streamp strm;
     state->mode = HEAD;
     state->last = 0;
     state->havedict = 0;
+    state->flags = -1;
     state->dmax = 32768U;
     state->head = Z_NULL;
     state->hold = 0;
@@ -141,9 +127,7 @@ z_streamp strm;
     return Z_OK;
 }
 
-int ZEXPORT inflateReset(strm)
-z_streamp strm;
-{
+int ZEXPORT inflateReset(z_streamp strm) {
     struct inflate_state FAR *state;
 
     if (inflateStateCheck(strm)) return Z_STREAM_ERROR;
@@ -154,10 +138,7 @@ z_streamp strm;
     return inflateResetKeep(strm);
 }
 
-int ZEXPORT inflateReset2(strm, windowBits)
-z_streamp strm;
-int windowBits;
-{
+int ZEXPORT inflateReset2(z_streamp strm, int windowBits) {
     int wrap;
     struct inflate_state FAR *state;
 
@@ -167,6 +148,8 @@ int windowBits;
 
     /* extract wrap request from windowBits parameter */
     if (windowBits < 0) {
+        if (windowBits < -15)
+            return Z_STREAM_ERROR;
         wrap = 0;
         windowBits = -windowBits;
     }
@@ -192,12 +175,8 @@ int windowBits;
     return inflateReset(strm);
 }
 
-int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size)
-z_streamp strm;
-int windowBits;
-const char *version;
-int stream_size;
-{
+int ZEXPORT inflateInit2_(z_streamp strm, int windowBits,
+                          const char *version, int stream_size) {
     int ret;
     struct inflate_state FAR *state;
 
@@ -236,22 +215,17 @@ int stream_size;
     return ret;
 }
 
-int ZEXPORT inflateInit_(strm, version, stream_size)
-z_streamp strm;
-const char *version;
-int stream_size;
-{
+int ZEXPORT inflateInit_(z_streamp strm, const char *version,
+                         int stream_size) {
     return inflateInit2_(strm, DEF_WBITS, version, stream_size);
 }
 
-int ZEXPORT inflatePrime(strm, bits, value)
-z_streamp strm;
-int bits;
-int value;
-{
+int ZEXPORT inflatePrime(z_streamp strm, int bits, int value) {
     struct inflate_state FAR *state;
 
     if (inflateStateCheck(strm)) return Z_STREAM_ERROR;
+    if (bits == 0)
+        return Z_OK;
     state = (struct inflate_state FAR *)strm->state;
     if (bits < 0) {
         state->hold = 0;
@@ -275,9 +249,7 @@ int value;
    used for threaded applications, since the rewriting of the tables and virgin
    may not be thread-safe.
  */
-local void fixedtables(state)
-struct inflate_state FAR *state;
-{
+local void fixedtables(struct inflate_state FAR *state) {
 #ifdef BUILDFIXED
     static int virgin = 1;
     static code *lenfix, *distfix;
@@ -339,7 +311,7 @@ struct inflate_state FAR *state;
 
     a.out > inffixed.h
  */
-void makefixed()
+void makefixed(void)
 {
     unsigned low, size;
     struct inflate_state state;
@@ -393,11 +365,7 @@ void makefixed()
    output will fall in the output data, making match copies simpler and faster.
    The advantage may be dependent on the size of the processor's data caches.
  */
-local int updatewindow(strm, end, copy)
-z_streamp strm;
-const Bytef *end;
-unsigned copy;
-{
+local int updatewindow(z_streamp strm, const Bytef *end, unsigned copy) {
     struct inflate_state FAR *state;
     unsigned dist;
 
@@ -447,10 +415,10 @@ unsigned copy;
 
 /* check function to use adler32() for zlib or crc32() for gzip */
 #ifdef GUNZIP
-#  define UPDATE(check, buf, len) \
+#  define UPDATE_CHECK(check, buf, len) \
     (state->flags ? crc32(check, buf, len) : adler32(check, buf, len))
 #else
-#  define UPDATE(check, buf, len) adler32(check, buf, len)
+#  define UPDATE_CHECK(check, buf, len) adler32(check, buf, len)
 #endif
 
 /* check macros for header crc */
@@ -619,10 +587,7 @@ unsigned copy;
    will return Z_BUF_ERROR if it has not reached the end of the stream.
  */
 
-int ZEXPORT inflate(strm, flush)
-z_streamp strm;
-int flush;
-{
+int ZEXPORT inflate(z_streamp strm, int flush) {
     struct inflate_state FAR *state;
     z_const unsigned char FAR *next;    /* next input */
     unsigned char FAR *put;     /* next output */
@@ -670,7 +635,6 @@ int flush;
                 state->mode = FLAGS;
                 break;
             }
-            state->flags = 0;           /* expect zlib header */
             if (state->head != Z_NULL)
                 state->head->done = -1;
             if (!(state->wrap & 1) ||   /* check if zlib header allowed */
@@ -697,6 +661,7 @@ int flush;
                 break;
             }
             state->dmax = 1U << len;
+            state->flags = 0;               /* indicate zlib header */
             Tracev((stderr, "inflate:   zlib header ok\n"));
             strm->adler = state->check = adler32(0L, Z_NULL, 0);
             state->mode = hold & 0x200 ? DICTID : TYPE;
@@ -722,6 +687,7 @@ int flush;
                 CRC2(state->check, hold);
             INITBITS();
             state->mode = TIME;
+                /* fallthrough */
         case TIME:
             NEEDBITS(32);
             if (state->head != Z_NULL)
@@ -730,6 +696,7 @@ int flush;
                 CRC4(state->check, hold);
             INITBITS();
             state->mode = OS;
+                /* fallthrough */
         case OS:
             NEEDBITS(16);
             if (state->head != Z_NULL) {
@@ -740,6 +707,7 @@ int flush;
                 CRC2(state->check, hold);
             INITBITS();
             state->mode = EXLEN;
+                /* fallthrough */
         case EXLEN:
             if (state->flags & 0x0400) {
                 NEEDBITS(16);
@@ -753,14 +721,16 @@ int flush;
             else if (state->head != Z_NULL)
                 state->head->extra = Z_NULL;
             state->mode = EXTRA;
+                /* fallthrough */
         case EXTRA:
             if (state->flags & 0x0400) {
                 copy = state->length;
                 if (copy > have) copy = have;
                 if (copy) {
                     if (state->head != Z_NULL &&
-                        state->head->extra != Z_NULL) {
-                        len = state->head->extra_len - state->length;
+                        state->head->extra != Z_NULL &&
+                        (len = state->head->extra_len - state->length) <
+                            state->head->extra_max) {
                         zmemcpy(state->head->extra + len, next,
                                 len + copy > state->head->extra_max ?
                                 state->head->extra_max - len : copy);
@@ -775,6 +745,7 @@ int flush;
             }
             state->length = 0;
             state->mode = NAME;
+                /* fallthrough */
         case NAME:
             if (state->flags & 0x0800) {
                 if (have == 0) goto inf_leave;
@@ -796,6 +767,7 @@ int flush;
                 state->head->name = Z_NULL;
             state->length = 0;
             state->mode = COMMENT;
+                /* fallthrough */
         case COMMENT:
             if (state->flags & 0x1000) {
                 if (have == 0) goto inf_leave;
@@ -816,6 +788,7 @@ int flush;
             else if (state->head != Z_NULL)
                 state->head->comment = Z_NULL;
             state->mode = HCRC;
+                /* fallthrough */
         case HCRC:
             if (state->flags & 0x0200) {
                 NEEDBITS(16);
@@ -839,6 +812,7 @@ int flush;
             strm->adler = state->check = ZSWAP32(hold);
             INITBITS();
             state->mode = DICT;
+                /* fallthrough */
         case DICT:
             if (state->havedict == 0) {
                 RESTORE();
@@ -846,8 +820,10 @@ int flush;
             }
             strm->adler = state->check = adler32(0L, Z_NULL, 0);
             state->mode = TYPE;
+                /* fallthrough */
         case TYPE:
             if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave;
+                /* fallthrough */
         case TYPEDO:
             if (state->last) {
                 BYTEBITS();
@@ -898,8 +874,10 @@ int flush;
             INITBITS();
             state->mode = COPY_;
             if (flush == Z_TREES) goto inf_leave;
+                /* fallthrough */
         case COPY_:
             state->mode = COPY;
+                /* fallthrough */
         case COPY:
             copy = state->length;
             if (copy) {
@@ -935,6 +913,7 @@ int flush;
             Tracev((stderr, "inflate:       table sizes ok\n"));
             state->have = 0;
             state->mode = LENLENS;
+                /* fallthrough */
         case LENLENS:
             while (state->have < state->ncode) {
                 NEEDBITS(3);
@@ -956,6 +935,7 @@ int flush;
             Tracev((stderr, "inflate:       code lengths ok\n"));
             state->have = 0;
             state->mode = CODELENS;
+                /* fallthrough */
         case CODELENS:
             while (state->have < state->nlen + state->ndist) {
                 for (;;) {
@@ -1039,8 +1019,10 @@ int flush;
             Tracev((stderr, "inflate:       codes ok\n"));
             state->mode = LEN_;
             if (flush == Z_TREES) goto inf_leave;
+                /* fallthrough */
         case LEN_:
             state->mode = LEN;
+                /* fallthrough */
         case LEN:
             if (have >= 6 && left >= 258) {
                 RESTORE();
@@ -1090,6 +1072,7 @@ int flush;
             }
             state->extra = (unsigned)(here.op) & 15;
             state->mode = LENEXT;
+                /* fallthrough */
         case LENEXT:
             if (state->extra) {
                 NEEDBITS(state->extra);
@@ -1100,6 +1083,7 @@ int flush;
             Tracevv((stderr, "inflate:         length %u\n", state->length));
             state->was = state->length;
             state->mode = DIST;
+                /* fallthrough */
         case DIST:
             for (;;) {
                 here = state->distcode[BITS(state->distbits)];
@@ -1127,6 +1111,7 @@ int flush;
             state->offset = (unsigned)here.val;
             state->extra = (unsigned)(here.op) & 15;
             state->mode = DISTEXT;
+                /* fallthrough */
         case DISTEXT:
             if (state->extra) {
                 NEEDBITS(state->extra);
@@ -1143,6 +1128,7 @@ int flush;
 #endif
             Tracevv((stderr, "inflate:         distance %u\n", state->offset));
             state->mode = MATCH;
+                /* fallthrough */
         case MATCH:
             if (left == 0) goto inf_leave;
             copy = out - left;
@@ -1202,7 +1188,7 @@ int flush;
                 state->total += out;
                 if ((state->wrap & 4) && out)
                     strm->adler = state->check =
-                        UPDATE(state->check, put - out, out);
+                        UPDATE_CHECK(state->check, put - out, out);
                 out = left;
                 if ((state->wrap & 4) && (
 #ifdef GUNZIP
@@ -1218,10 +1204,11 @@ int flush;
             }
 #ifdef GUNZIP
             state->mode = LENGTH;
+                /* fallthrough */
         case LENGTH:
             if (state->wrap && state->flags) {
                 NEEDBITS(32);
-                if (hold != (state->total & 0xffffffffUL)) {
+                if ((state->wrap & 4) && hold != (state->total & 0xffffffff)) {
                     strm->msg = (char *)"incorrect length check";
                     state->mode = BAD;
                     break;
@@ -1231,6 +1218,7 @@ int flush;
             }
 #endif
             state->mode = DONE;
+                /* fallthrough */
         case DONE:
             ret = Z_STREAM_END;
             goto inf_leave;
@@ -1240,6 +1228,7 @@ int flush;
         case MEM:
             return Z_MEM_ERROR;
         case SYNC:
+                /* fallthrough */
         default:
             return Z_STREAM_ERROR;
         }
@@ -1265,7 +1254,7 @@ int flush;
     state->total += out;
     if ((state->wrap & 4) && out)
         strm->adler = state->check =
-            UPDATE(state->check, strm->next_out - out, out);
+            UPDATE_CHECK(state->check, strm->next_out - out, out);
     strm->data_type = (int)state->bits + (state->last ? 64 : 0) +
                       (state->mode == TYPE ? 128 : 0) +
                       (state->mode == LEN_ || state->mode == COPY_ ? 256 : 0);
@@ -1274,9 +1263,7 @@ int flush;
     return ret;
 }
 
-int ZEXPORT inflateEnd(strm)
-z_streamp strm;
-{
+int ZEXPORT inflateEnd(z_streamp strm) {
     struct inflate_state FAR *state;
     if (inflateStateCheck(strm))
         return Z_STREAM_ERROR;
@@ -1288,11 +1275,8 @@ z_streamp strm;
     return Z_OK;
 }
 
-int ZEXPORT inflateGetDictionary(strm, dictionary, dictLength)
-z_streamp strm;
-Bytef *dictionary;
-uInt *dictLength;
-{
+int ZEXPORT inflateGetDictionary(z_streamp strm, Bytef *dictionary,
+                                 uInt *dictLength) {
     struct inflate_state FAR *state;
 
     /* check state */
@@ -1311,11 +1295,8 @@ uInt *dictLength;
     return Z_OK;
 }
 
-int ZEXPORT inflateSetDictionary(strm, dictionary, dictLength)
-z_streamp strm;
-const Bytef *dictionary;
-uInt dictLength;
-{
+int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef *dictionary,
+                                 uInt dictLength) {
     struct inflate_state FAR *state;
     unsigned long dictid;
     int ret;
@@ -1346,10 +1327,7 @@ uInt dictLength;
     return Z_OK;
 }
 
-int ZEXPORT inflateGetHeader(strm, head)
-z_streamp strm;
-gz_headerp head;
-{
+int ZEXPORT inflateGetHeader(z_streamp strm, gz_headerp head) {
     struct inflate_state FAR *state;
 
     /* check state */
@@ -1374,11 +1352,8 @@ gz_headerp head;
    called again with more data and the *have state.  *have is initialized to
    zero for the first call.
  */
-local unsigned syncsearch(have, buf, len)
-unsigned FAR *have;
-const unsigned char FAR *buf;
-unsigned len;
-{
+local unsigned syncsearch(unsigned FAR *have, const unsigned char FAR *buf,
+                          unsigned len) {
     unsigned got;
     unsigned next;
 
@@ -1397,10 +1372,9 @@ unsigned len;
     return next;
 }
 
-int ZEXPORT inflateSync(strm)
-z_streamp strm;
-{
+int ZEXPORT inflateSync(z_streamp strm) {
     unsigned len;               /* number of bytes to look at or looked at */
+    int flags;                  /* temporary to save header status */
     unsigned long in, out;      /* temporary to save total_in and total_out */
     unsigned char buf[4];       /* to restore bit buffer to byte string */
     struct inflate_state FAR *state;
@@ -1413,7 +1387,7 @@ z_streamp strm;
     /* if first time, start search in bit buffer */
     if (state->mode != SYNC) {
         state->mode = SYNC;
-        state->hold <<= state->bits & 7;
+        state->hold >>= state->bits & 7;
         state->bits -= state->bits & 7;
         len = 0;
         while (state->bits >= 8) {
@@ -1433,9 +1407,15 @@ z_streamp strm;
 
     /* return no joy or set up to restart inflate() on a new block */
     if (state->have != 4) return Z_DATA_ERROR;
+    if (state->flags == -1)
+        state->wrap = 0;    /* if no header yet, treat as raw */
+    else
+        state->wrap &= ~4;  /* no point in computing a check value now */
+    flags = state->flags;
     in = strm->total_in;  out = strm->total_out;
     inflateReset(strm);
     strm->total_in = in;  strm->total_out = out;
+    state->flags = flags;
     state->mode = TYPE;
     return Z_OK;
 }
@@ -1448,9 +1428,7 @@ z_streamp strm;
    block. When decompressing, PPP checks that at the end of input packet,
    inflate is waiting for these length bytes.
  */
-int ZEXPORT inflateSyncPoint(strm)
-z_streamp strm;
-{
+int ZEXPORT inflateSyncPoint(z_streamp strm) {
     struct inflate_state FAR *state;
 
     if (inflateStateCheck(strm)) return Z_STREAM_ERROR;
@@ -1458,10 +1436,7 @@ z_streamp strm;
     return state->mode == STORED && state->bits == 0;
 }
 
-int ZEXPORT inflateCopy(dest, source)
-z_streamp dest;
-z_streamp source;
-{
+int ZEXPORT inflateCopy(z_streamp dest, z_streamp source) {
     struct inflate_state FAR *state;
     struct inflate_state FAR *copy;
     unsigned char FAR *window;
@@ -1505,10 +1480,7 @@ z_streamp source;
     return Z_OK;
 }
 
-int ZEXPORT inflateUndermine(strm, subvert)
-z_streamp strm;
-int subvert;
-{
+int ZEXPORT inflateUndermine(z_streamp strm, int subvert) {
     struct inflate_state FAR *state;
 
     if (inflateStateCheck(strm)) return Z_STREAM_ERROR;
@@ -1523,24 +1495,19 @@ int subvert;
 #endif
 }
 
-int ZEXPORT inflateValidate(strm, check)
-z_streamp strm;
-int check;
-{
+int ZEXPORT inflateValidate(z_streamp strm, int check) {
     struct inflate_state FAR *state;
 
     if (inflateStateCheck(strm)) return Z_STREAM_ERROR;
     state = (struct inflate_state FAR *)strm->state;
-    if (check)
+    if (check && state->wrap)
         state->wrap |= 4;
     else
         state->wrap &= ~4;
     return Z_OK;
 }
 
-long ZEXPORT inflateMark(strm)
-z_streamp strm;
-{
+long ZEXPORT inflateMark(z_streamp strm) {
     struct inflate_state FAR *state;
 
     if (inflateStateCheck(strm))
@@ -1551,9 +1518,7 @@ z_streamp strm;
             (state->mode == MATCH ? state->was - state->length : 0));
 }
 
-unsigned long ZEXPORT inflateCodesUsed(strm)
-z_streamp strm;
-{
+unsigned long ZEXPORT inflateCodesUsed(z_streamp strm) {
     struct inflate_state FAR *state;
     if (inflateStateCheck(strm)) return (unsigned long)-1;
     state = (struct inflate_state FAR *)strm->state;
diff --git a/zlib/inflate.h b/zlib/inflate.h
index a46cce6..f127b6b 100644
--- a/zlib/inflate.h
+++ b/zlib/inflate.h
@@ -1,5 +1,5 @@
 /* inflate.h -- internal inflate state definition
- * Copyright (C) 1995-2016 Mark Adler
+ * Copyright (C) 1995-2019 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -86,7 +86,8 @@ struct inflate_state {
     int wrap;                   /* bit 0 true for zlib, bit 1 true for gzip,
                                    bit 2 true to validate check value */
     int havedict;               /* true if dictionary provided */
-    int flags;                  /* gzip header method and flags (0 if zlib) */
+    int flags;                  /* gzip header method and flags, 0 if zlib, or
+                                   -1 if raw or no header yet */
     unsigned dmax;              /* zlib header max distance (INFLATE_STRICT) */
     unsigned long check;        /* protected copy of check value */
     unsigned long total;        /* protected copy of output count */
diff --git a/zlib/inftrees.c b/zlib/inftrees.c
index 2ea08fc..98cfe16 100644
--- a/zlib/inftrees.c
+++ b/zlib/inftrees.c
@@ -1,5 +1,5 @@
 /* inftrees.c -- generate Huffman trees for efficient decoding
- * Copyright (C) 1995-2017 Mark Adler
+ * Copyright (C) 1995-2024 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -9,7 +9,7 @@
 #define MAXBITS 15
 
 const char inflate_copyright[] =
-   " inflate 1.2.11 Copyright 1995-2017 Mark Adler ";
+   " inflate 1.3.1 Copyright 1995-2024 Mark Adler ";
 /*
   If you use the zlib library in a product, an acknowledgment is welcome
   in the documentation of your product. If for some reason you cannot
@@ -29,14 +29,9 @@ const char inflate_copyright[] =
    table index bits.  It will differ if the request is greater than the
    longest code or if it is less than the shortest code.
  */
-int ZLIB_INTERNAL inflate_table(type, lens, codes, table, bits, work)
-codetype type;
-unsigned short FAR *lens;
-unsigned codes;
-code FAR * FAR *table;
-unsigned FAR *bits;
-unsigned short FAR *work;
-{
+int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens,
+                                unsigned codes, code FAR * FAR *table,
+                                unsigned FAR *bits, unsigned short FAR *work) {
     unsigned len;               /* a code's length in bits */
     unsigned sym;               /* index of code symbols */
     unsigned min, max;          /* minimum and maximum code lengths */
@@ -62,7 +57,7 @@ unsigned short FAR *work;
         35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0};
     static const unsigned short lext[31] = { /* Length codes 257..285 extra */
         16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18,
-        19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 77, 202};
+        19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 203, 77};
     static const unsigned short dbase[32] = { /* Distance codes 0..29 base */
         1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193,
         257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145,
diff --git a/zlib/inftrees.h b/zlib/inftrees.h
index baa53a0..396f74b 100644
--- a/zlib/inftrees.h
+++ b/zlib/inftrees.h
@@ -38,11 +38,11 @@ typedef struct {
 /* Maximum size of the dynamic table.  The maximum number of code structures is
    1444, which is the sum of 852 for literal/length codes and 592 for distance
    codes.  These values were found by exhaustive searches using the program
-   examples/enough.c found in the zlib distribtution.  The arguments to that
+   examples/enough.c found in the zlib distribution.  The arguments to that
    program are the number of symbols, the initial root table size, and the
    maximum bit length of a code.  "enough 286 9 15" for literal/length codes
-   returns returns 852, and "enough 30 6 15" for distance codes returns 592.
-   The initial root table size (9 or 6) is found in the fifth argument of the
+   returns 852, and "enough 30 6 15" for distance codes returns 592. The
+   initial root table size (9 or 6) is found in the fifth argument of the
    inflate_table() calls in inflate.c and infback.c.  If the root table size is
    changed, then these maximum sizes would be need to be recalculated and
    updated. */
@@ -57,6 +57,6 @@ typedef enum {
     DISTS
 } codetype;
 
-int ZLIB_INTERNAL inflate_table OF((codetype type, unsigned short FAR *lens,
-                             unsigned codes, code FAR * FAR *table,
-                             unsigned FAR *bits, unsigned short FAR *work));
+int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens,
+                                unsigned codes, code FAR * FAR *table,
+                                unsigned FAR *bits, unsigned short FAR *work);
diff --git a/zlib/make_vms.com b/zlib/make_vms.com
index 65e9d0c..4dc8a89 100644
--- a/zlib/make_vms.com
+++ b/zlib/make_vms.com
@@ -14,9 +14,9 @@ $! 0.02 20061008 Adapt to new Makefile.in
 $! 0.03 20091224 Add support for large file check
 $! 0.04 20100110 Add new gzclose, gzlib, gzread, gzwrite
 $! 0.05 20100221 Exchange zlibdefs.h by zconf.h.in
-$! 0.06 20120111 Fix missing amiss_err, update zconf_h.in, fix new exmples
+$! 0.06 20120111 Fix missing amiss_err, update zconf_h.in, fix new examples
 $!               subdir path, update module search in makefile.in
-$! 0.07 20120115 Triggered by work done by Alexey Chupahin completly redesigned
+$! 0.07 20120115 Triggered by work done by Alexey Chupahin completely redesigned
 $!               shared image creation
 $! 0.08 20120219 Make it work on VAX again, pre-load missing symbols to shared
 $!               image
diff --git a/zlib/old/visual-basic.txt b/zlib/old/visual-basic.txt
index 57efe58..3c8d2a4 100644
--- a/zlib/old/visual-basic.txt
+++ b/zlib/old/visual-basic.txt
@@ -115,7 +115,7 @@ SUCCESS Then
         ReDim Preserve bytaryCpr(lngCprSiz - 1)
         Open strCprPth For Binary Access Write As #1
             Put #1, , bytaryCpr()
-            Put #1, , lngOriSiz 'Add the the original size value to the end
+            Put #1, , lngOriSiz 'Add the original size value to the end
 (last 4 bytes)
         Close #1
     Else
diff --git a/zlib/os400/README400 b/zlib/os400/README400
index 4f98334..30ed5a1 100644
--- a/zlib/os400/README400
+++ b/zlib/os400/README400
@@ -1,9 +1,9 @@
-        ZLIB version 1.2.11 for OS/400 installation instructions
+        ZLIB version 1.3.1 for OS/400 installation instructions
 
 1) Download and unpack the zlib tarball to some IFS directory.
    (i.e.: /path/to/the/zlib/ifs/source/directory)
 
-   If the installed IFS command suppors gzip format, this is straightforward,
+   If the installed IFS command supports gzip format, this is straightforward,
 else you have to unpack first to some directory on a system supporting it,
 then move the whole directory to the IFS via the network (via SMB or FTP).
 
@@ -43,6 +43,6 @@ Notes:  For OS/400 ILE RPG programmers, a /copy member defining the ZLIB
 
         Remember that most foreign textual data are ASCII coded: this
                 implementation does not handle conversion from/to ASCII, so
-                text data code conversions must be done explicitely.
+                text data code conversions must be done explicitly.
 
         Mainly for the reason above, always open zipped files in binary mode.
diff --git a/zlib/os400/bndsrc b/zlib/os400/bndsrc
index 5e6e0a2..9f92bb1 100644
--- a/zlib/os400/bndsrc
+++ b/zlib/os400/bndsrc
@@ -116,4 +116,12 @@ STRPGMEXP PGMLVL(*CURRENT) SIGNATURE('ZLIB')
   EXPORT SYMBOL("inflateValidate")
   EXPORT SYMBOL("uncompress2")
 
+/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/
+/*   Version 1.2.12 additional entry points.                        */
+/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*/
+
+  EXPORT SYMBOL("crc32_combine_gen64")
+  EXPORT SYMBOL("crc32_combine_gen")
+  EXPORT SYMBOL("crc32_combine_op")
+
 ENDPGMEXP
diff --git a/zlib/os400/zlib.inc b/zlib/os400/zlib.inc
index c6aca2c..744729a 100644
--- a/zlib/os400/zlib.inc
+++ b/zlib/os400/zlib.inc
@@ -1,7 +1,7 @@
       *  ZLIB.INC - Interface to the general purpose compression library
       *
       *  ILE RPG400 version by Patrick Monnerat, DATASPHERE.
-      *  Version 1.2.11
+      *  Version 1.3.1
       *
       *
       *  WARNING:
@@ -22,12 +22,12 @@
       *
       *  Versioning information.
       *
-     D ZLIB_VERSION    C                   '1.2.11'
+     D ZLIB_VERSION    C                   '1.3.1'
      D ZLIB_VERNUM     C                   X'12a0'
      D ZLIB_VER_MAJOR  C                   1
-     D ZLIB_VER_MINOR  C                   2
+     D ZLIB_VER_MINOR  C                   3
      D ZLIB_VER_REVISION...
-     D                 C                   11
+     D                 C                   1
      D ZLIB_VER_SUBREVISION...
      D                 C                   0
       *
diff --git a/zlib/qnx/package.qpg b/zlib/qnx/package.qpg
index 31e8e90..4877e0e 100644
--- a/zlib/qnx/package.qpg
+++ b/zlib/qnx/package.qpg
@@ -25,10 +25,10 @@
       <QPG:Files>
          <QPG:Add file="../zconf.h" install="/opt/include/" user="root:sys" permission="644"/>
          <QPG:Add file="../zlib.h" install="/opt/include/" user="root:sys" permission="644"/>
-         <QPG:Add file="../libz.so.1.2.11" install="/opt/lib/" user="root:bin" permission="644"/>
-         <QPG:Add file="libz.so" install="/opt/lib/" component="dev" filetype="symlink" linkto="libz.so.1.2.11"/>
-         <QPG:Add file="libz.so.1" install="/opt/lib/" filetype="symlink" linkto="libz.so.1.2.11"/>
-         <QPG:Add file="../libz.so.1.2.11" install="/opt/lib/" component="slib"/>
+         <QPG:Add file="../libz.so.1.3.1" install="/opt/lib/" user="root:bin" permission="644"/>
+         <QPG:Add file="libz.so" install="/opt/lib/" component="dev" filetype="symlink" linkto="libz.so.1.3.1"/>
+         <QPG:Add file="libz.so.1" install="/opt/lib/" filetype="symlink" linkto="libz.so.1.3.1"/>
+         <QPG:Add file="../libz.so.1.3.1" install="/opt/lib/" component="slib"/>
       </QPG:Files>
 
       <QPG:PackageFilter>
@@ -63,7 +63,7 @@
             </QPM:ProductDescription>
 
             <QPM:ReleaseDescription>
-               <QPM:ReleaseVersion>1.2.11</QPM:ReleaseVersion>
+               <QPM:ReleaseVersion>1.3.1</QPM:ReleaseVersion>
                <QPM:ReleaseUrgency>Medium</QPM:ReleaseUrgency>
                <QPM:ReleaseStability>Stable</QPM:ReleaseStability>
                <QPM:ReleaseNoteMinor></QPM:ReleaseNoteMinor>
diff --git a/zlib/test/example.c b/zlib/test/example.c
index eee17ce..c3521dd 100644
--- a/zlib/test/example.c
+++ b/zlib/test/example.c
@@ -34,37 +34,14 @@ static z_const char hello[] = "hello, hello!";
 static const char dictionary[] = "hello";
 static uLong dictId;    /* Adler32 value of the dictionary */
 
-void test_deflate       OF((Byte *compr, uLong comprLen));
-void test_inflate       OF((Byte *compr, uLong comprLen,
-                            Byte *uncompr, uLong uncomprLen));
-void test_large_deflate OF((Byte *compr, uLong comprLen,
-                            Byte *uncompr, uLong uncomprLen));
-void test_large_inflate OF((Byte *compr, uLong comprLen,
-                            Byte *uncompr, uLong uncomprLen));
-void test_flush         OF((Byte *compr, uLong *comprLen));
-void test_sync          OF((Byte *compr, uLong comprLen,
-                            Byte *uncompr, uLong uncomprLen));
-void test_dict_deflate  OF((Byte *compr, uLong comprLen));
-void test_dict_inflate  OF((Byte *compr, uLong comprLen,
-                            Byte *uncompr, uLong uncomprLen));
-int  main               OF((int argc, char *argv[]));
-
-
 #ifdef Z_SOLO
 
-void *myalloc OF((void *, unsigned, unsigned));
-void myfree OF((void *, void *));
-
-void *myalloc(q, n, m)
-    void *q;
-    unsigned n, m;
-{
+static void *myalloc(void *q, unsigned n, unsigned m) {
     (void)q;
     return calloc(n, m);
 }
 
-void myfree(void *q, void *p)
-{
+static void myfree(void *q, void *p) {
     (void)q;
     free(p);
 }
@@ -77,18 +54,11 @@ static free_func zfree = myfree;
 static alloc_func zalloc = (alloc_func)0;
 static free_func zfree = (free_func)0;
 
-void test_compress      OF((Byte *compr, uLong comprLen,
-                            Byte *uncompr, uLong uncomprLen));
-void test_gzio          OF((const char *fname,
-                            Byte *uncompr, uLong uncomprLen));
-
 /* ===========================================================================
  * Test compress() and uncompress()
  */
-void test_compress(compr, comprLen, uncompr, uncomprLen)
-    Byte *compr, *uncompr;
-    uLong comprLen, uncomprLen;
-{
+static void test_compress(Byte *compr, uLong comprLen, Byte *uncompr,
+                   uLong uncomprLen) {
     int err;
     uLong len = (uLong)strlen(hello)+1;
 
@@ -111,11 +81,7 @@ void test_compress(compr, comprLen, uncompr, uncomprLen)
 /* ===========================================================================
  * Test read/write of .gz files
  */
-void test_gzio(fname, uncompr, uncomprLen)
-    const char *fname; /* compressed file name */
-    Byte *uncompr;
-    uLong uncomprLen;
-{
+static void test_gzio(const char *fname, Byte *uncompr, uLong uncomprLen) {
 #ifdef NO_GZCOMPRESS
     fprintf(stderr, "NO_GZCOMPRESS -- gz* functions cannot compress\n");
 #else
@@ -197,10 +163,7 @@ void test_gzio(fname, uncompr, uncomprLen)
 /* ===========================================================================
  * Test deflate() with small buffers
  */
-void test_deflate(compr, comprLen)
-    Byte *compr;
-    uLong comprLen;
-{
+static void test_deflate(Byte *compr, uLong comprLen) {
     z_stream c_stream; /* compression stream */
     int err;
     uLong len = (uLong)strlen(hello)+1;
@@ -235,10 +198,8 @@ void test_deflate(compr, comprLen)
 /* ===========================================================================
  * Test inflate() with small buffers
  */
-void test_inflate(compr, comprLen, uncompr, uncomprLen)
-    Byte *compr, *uncompr;
-    uLong comprLen, uncomprLen;
-{
+static void test_inflate(Byte *compr, uLong comprLen, Byte *uncompr,
+                  uLong uncomprLen) {
     int err;
     z_stream d_stream; /* decompression stream */
 
@@ -276,10 +237,8 @@ void test_inflate(compr, comprLen, uncompr, uncomprLen)
 /* ===========================================================================
  * Test deflate() with large buffers and dynamic change of compression level
  */
-void test_large_deflate(compr, comprLen, uncompr, uncomprLen)
-    Byte *compr, *uncompr;
-    uLong comprLen, uncomprLen;
-{
+static void test_large_deflate(Byte *compr, uLong comprLen, Byte *uncompr,
+                        uLong uncomprLen) {
     z_stream c_stream; /* compression stream */
     int err;
 
@@ -308,7 +267,7 @@ void test_large_deflate(compr, comprLen, uncompr, uncomprLen)
     /* Feed in already compressed data and switch to no compression: */
     deflateParams(&c_stream, Z_NO_COMPRESSION, Z_DEFAULT_STRATEGY);
     c_stream.next_in = compr;
-    c_stream.avail_in = (uInt)comprLen/2;
+    c_stream.avail_in = (uInt)uncomprLen/2;
     err = deflate(&c_stream, Z_NO_FLUSH);
     CHECK_ERR(err, "deflate");
 
@@ -331,10 +290,8 @@ void test_large_deflate(compr, comprLen, uncompr, uncomprLen)
 /* ===========================================================================
  * Test inflate() with large buffers
  */
-void test_large_inflate(compr, comprLen, uncompr, uncomprLen)
-    Byte *compr, *uncompr;
-    uLong comprLen, uncomprLen;
-{
+static void test_large_inflate(Byte *compr, uLong comprLen, Byte *uncompr,
+                        uLong uncomprLen) {
     int err;
     z_stream d_stream; /* decompression stream */
 
@@ -361,7 +318,7 @@ void test_large_inflate(compr, comprLen, uncompr, uncomprLen)
     err = inflateEnd(&d_stream);
     CHECK_ERR(err, "inflateEnd");
 
-    if (d_stream.total_out != 2*uncomprLen + comprLen/2) {
+    if (d_stream.total_out != 2*uncomprLen + uncomprLen/2) {
         fprintf(stderr, "bad large inflate: %ld\n", d_stream.total_out);
         exit(1);
     } else {
@@ -372,10 +329,7 @@ void test_large_inflate(compr, comprLen, uncompr, uncomprLen)
 /* ===========================================================================
  * Test deflate() with full flush
  */
-void test_flush(compr, comprLen)
-    Byte *compr;
-    uLong *comprLen;
-{
+static void test_flush(Byte *compr, uLong *comprLen) {
     z_stream c_stream; /* compression stream */
     int err;
     uInt len = (uInt)strlen(hello)+1;
@@ -410,10 +364,8 @@ void test_flush(compr, comprLen)
 /* ===========================================================================
  * Test inflateSync()
  */
-void test_sync(compr, comprLen, uncompr, uncomprLen)
-    Byte *compr, *uncompr;
-    uLong comprLen, uncomprLen;
-{
+static void test_sync(Byte *compr, uLong comprLen, Byte *uncompr,
+                      uLong uncomprLen) {
     int err;
     z_stream d_stream; /* decompression stream */
 
@@ -440,9 +392,8 @@ void test_sync(compr, comprLen, uncompr, uncomprLen)
     CHECK_ERR(err, "inflateSync");
 
     err = inflate(&d_stream, Z_FINISH);
-    if (err != Z_DATA_ERROR) {
-        fprintf(stderr, "inflate should report DATA_ERROR\n");
-        /* Because of incorrect adler32 */
+    if (err != Z_STREAM_END) {
+        fprintf(stderr, "inflate should report Z_STREAM_END\n");
         exit(1);
     }
     err = inflateEnd(&d_stream);
@@ -454,10 +405,7 @@ void test_sync(compr, comprLen, uncompr, uncomprLen)
 /* ===========================================================================
  * Test deflate() with preset dictionary
  */
-void test_dict_deflate(compr, comprLen)
-    Byte *compr;
-    uLong comprLen;
-{
+static void test_dict_deflate(Byte *compr, uLong comprLen) {
     z_stream c_stream; /* compression stream */
     int err;
 
@@ -491,10 +439,8 @@ void test_dict_deflate(compr, comprLen)
 /* ===========================================================================
  * Test inflate() with a preset dictionary
  */
-void test_dict_inflate(compr, comprLen, uncompr, uncomprLen)
-    Byte *compr, *uncompr;
-    uLong comprLen, uncomprLen;
-{
+static void test_dict_inflate(Byte *compr, uLong comprLen, Byte *uncompr,
+                       uLong uncomprLen) {
     int err;
     z_stream d_stream; /* decompression stream */
 
@@ -542,13 +488,10 @@ void test_dict_inflate(compr, comprLen, uncompr, uncomprLen)
  * Usage:  example [output.gz  [input.gz]]
  */
 
-int main(argc, argv)
-    int argc;
-    char *argv[];
-{
+int main(int argc, char *argv[]) {
     Byte *compr, *uncompr;
-    uLong comprLen = 10000*sizeof(int); /* don't overflow on MSDOS */
-    uLong uncomprLen = comprLen;
+    uLong uncomprLen = 20000;
+    uLong comprLen = 3 * uncomprLen;
     static const char* myVersion = ZLIB_VERSION;
 
     if (zlibVersion()[0] != myVersion[0]) {
@@ -556,7 +499,8 @@ int main(argc, argv)
         exit(1);
 
     } else if (strcmp(zlibVersion(), ZLIB_VERSION) != 0) {
-        fprintf(stderr, "warning: different zlib version\n");
+        fprintf(stderr, "warning: different zlib version linked: %s\n",
+                zlibVersion());
     }
 
     printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n",
@@ -590,7 +534,7 @@ int main(argc, argv)
 
     test_flush(compr, &comprLen);
     test_sync(compr, comprLen, uncompr, uncomprLen);
-    comprLen = uncomprLen;
+    comprLen = 3 * uncomprLen;
 
     test_dict_deflate(compr, comprLen);
     test_dict_inflate(compr, comprLen, uncompr, uncomprLen);
diff --git a/zlib/test/infcover.c b/zlib/test/infcover.c
index 2be0164..8912c40 100644
--- a/zlib/test/infcover.c
+++ b/zlib/test/infcover.c
@@ -373,7 +373,7 @@ local void cover_support(void)
     mem_setup(&strm);
     strm.avail_in = 0;
     strm.next_in = Z_NULL;
-    ret = inflateInit_(&strm, ZLIB_VERSION - 1, (int)sizeof(z_stream));
+    ret = inflateInit_(&strm, "!", (int)sizeof(z_stream));
                                                 assert(ret == Z_VERSION_ERROR);
     mem_done(&strm, "wrong version");
 
@@ -462,7 +462,8 @@ local unsigned pull(void *desc, unsigned char **buf)
 
 local int push(void *desc, unsigned char *buf, unsigned len)
 {
-    buf += len;
+    (void)buf;
+    (void)len;
     return desc != Z_NULL;      /* force error if desc not null */
 }
 
diff --git a/zlib/test/minigzip.c b/zlib/test/minigzip.c
index e22fb08..134e10e 100644
--- a/zlib/test/minigzip.c
+++ b/zlib/test/minigzip.c
@@ -59,7 +59,7 @@
 
 #if !defined(Z_HAVE_UNISTD_H) && !defined(_LARGEFILE64_SOURCE)
 #ifndef WIN32 /* unlink already in stdio.h for WIN32 */
-  extern int unlink OF((const char *));
+  extern int unlink(const char *);
 #endif
 #endif
 
@@ -149,20 +149,12 @@ static void pwinerror (s)
 #  include <unistd.h>       /* for unlink() */
 #endif
 
-void *myalloc OF((void *, unsigned, unsigned));
-void myfree OF((void *, void *));
-
-void *myalloc(q, n, m)
-    void *q;
-    unsigned n, m;
-{
+static void *myalloc(void *q, unsigned n, unsigned m) {
     (void)q;
     return calloc(n, m);
 }
 
-void myfree(q, p)
-    void *q, *p;
-{
+static void myfree(void *q, void *p) {
     (void)q;
     free(p);
 }
@@ -175,29 +167,7 @@ typedef struct gzFile_s {
     z_stream strm;
 } *gzFile;
 
-gzFile gzopen OF((const char *, const char *));
-gzFile gzdopen OF((int, const char *));
-gzFile gz_open OF((const char *, int, const char *));
-
-gzFile gzopen(path, mode)
-const char *path;
-const char *mode;
-{
-    return gz_open(path, -1, mode);
-}
-
-gzFile gzdopen(fd, mode)
-int fd;
-const char *mode;
-{
-    return gz_open(NULL, fd, mode);
-}
-
-gzFile gz_open(path, fd, mode)
-    const char *path;
-    int fd;
-    const char *mode;
-{
+static gzFile gz_open(const char *path, int fd, const char *mode) {
     gzFile gz;
     int ret;
 
@@ -231,13 +201,15 @@ gzFile gz_open(path, fd, mode)
     return gz;
 }
 
-int gzwrite OF((gzFile, const void *, unsigned));
+static gzFile gzopen(const char *path, const char *mode) {
+    return gz_open(path, -1, mode);
+}
 
-int gzwrite(gz, buf, len)
-    gzFile gz;
-    const void *buf;
-    unsigned len;
-{
+static gzFile gzdopen(int fd, const char *mode) {
+    return gz_open(NULL, fd, mode);
+}
+
+static int gzwrite(gzFile gz, const void *buf, unsigned len) {
     z_stream *strm;
     unsigned char out[BUFLEN];
 
@@ -255,13 +227,7 @@ int gzwrite(gz, buf, len)
     return len;
 }
 
-int gzread OF((gzFile, void *, unsigned));
-
-int gzread(gz, buf, len)
-    gzFile gz;
-    void *buf;
-    unsigned len;
-{
+static int gzread(gzFile gz, void *buf, unsigned len) {
     int ret;
     unsigned got;
     unsigned char in[1];
@@ -292,11 +258,7 @@ int gzread(gz, buf, len)
     return len - strm->avail_out;
 }
 
-int gzclose OF((gzFile));
-
-int gzclose(gz)
-    gzFile gz;
-{
+static int gzclose(gzFile gz) {
     z_stream *strm;
     unsigned char out[BUFLEN];
 
@@ -321,12 +283,7 @@ int gzclose(gz)
     return Z_OK;
 }
 
-const char *gzerror OF((gzFile, int *));
-
-const char *gzerror(gz, err)
-    gzFile gz;
-    int *err;
-{
+static const char *gzerror(gzFile gz, int *err) {
     *err = gz->err;
     return gz->msg;
 }
@@ -335,67 +292,20 @@ const char *gzerror(gz, err)
 
 static char *prog;
 
-void error            OF((const char *msg));
-void gz_compress      OF((FILE   *in, gzFile out));
-#ifdef USE_MMAP
-int  gz_compress_mmap OF((FILE   *in, gzFile out));
-#endif
-void gz_uncompress    OF((gzFile in, FILE   *out));
-void file_compress    OF((char  *file, char *mode));
-void file_uncompress  OF((char  *file));
-int  main             OF((int argc, char *argv[]));
-
 /* ===========================================================================
  * Display error message and exit
  */
-void error(msg)
-    const char *msg;
-{
+static void error(const char *msg) {
     fprintf(stderr, "%s: %s\n", prog, msg);
     exit(1);
 }
 
-/* ===========================================================================
- * Compress input to output then close both files.
- */
-
-void gz_compress(in, out)
-    FILE   *in;
-    gzFile out;
-{
-    local char buf[BUFLEN];
-    int len;
-    int err;
-
-#ifdef USE_MMAP
-    /* Try first compressing with mmap. If mmap fails (minigzip used in a
-     * pipe), use the normal fread loop.
-     */
-    if (gz_compress_mmap(in, out) == Z_OK) return;
-#endif
-    for (;;) {
-        len = (int)fread(buf, 1, sizeof(buf), in);
-        if (ferror(in)) {
-            perror("fread");
-            exit(1);
-        }
-        if (len == 0) break;
-
-        if (gzwrite(out, buf, (unsigned)len) != len) error(gzerror(out, &err));
-    }
-    fclose(in);
-    if (gzclose(out) != Z_OK) error("failed gzclose");
-}
-
 #ifdef USE_MMAP /* MMAP version, Miguel Albrecht <malbrech@eso.org> */
 
 /* Try compressing the input file at once using mmap. Return Z_OK if
- * if success, Z_ERRNO otherwise.
+ * success, Z_ERRNO otherwise.
  */
-int gz_compress_mmap(in, out)
-    FILE   *in;
-    gzFile out;
-{
+static int gz_compress_mmap(FILE *in, gzFile out) {
     int len;
     int err;
     int ifd = fileno(in);
@@ -425,12 +335,38 @@ int gz_compress_mmap(in, out)
 #endif /* USE_MMAP */
 
 /* ===========================================================================
+ * Compress input to output then close both files.
+ */
+
+static void gz_compress(FILE *in, gzFile out) {
+    local char buf[BUFLEN];
+    int len;
+    int err;
+
+#ifdef USE_MMAP
+    /* Try first compressing with mmap. If mmap fails (minigzip used in a
+     * pipe), use the normal fread loop.
+     */
+    if (gz_compress_mmap(in, out) == Z_OK) return;
+#endif
+    for (;;) {
+        len = (int)fread(buf, 1, sizeof(buf), in);
+        if (ferror(in)) {
+            perror("fread");
+            exit(1);
+        }
+        if (len == 0) break;
+
+        if (gzwrite(out, buf, (unsigned)len) != len) error(gzerror(out, &err));
+    }
+    fclose(in);
+    if (gzclose(out) != Z_OK) error("failed gzclose");
+}
+
+/* ===========================================================================
  * Uncompress input to output then close both files.
  */
-void gz_uncompress(in, out)
-    gzFile in;
-    FILE   *out;
-{
+static void gz_uncompress(gzFile in, FILE *out) {
     local char buf[BUFLEN];
     int len;
     int err;
@@ -454,10 +390,7 @@ void gz_uncompress(in, out)
  * Compress the given file: create a corresponding .gz file and remove the
  * original.
  */
-void file_compress(file, mode)
-    char  *file;
-    char  *mode;
-{
+static void file_compress(char *file, char *mode) {
     local char outfile[MAX_NAME_LEN];
     FILE  *in;
     gzFile out;
@@ -493,14 +426,12 @@ void file_compress(file, mode)
 /* ===========================================================================
  * Uncompress the given file and remove the original.
  */
-void file_uncompress(file)
-    char  *file;
-{
+static void file_uncompress(char *file) {
     local char buf[MAX_NAME_LEN];
     char *infile, *outfile;
     FILE  *out;
     gzFile in;
-    unsigned len = strlen(file);
+    z_size_t len = strlen(file);
 
     if (len + strlen(GZ_SUFFIX) >= sizeof(buf)) {
         fprintf(stderr, "%s: filename too long\n", prog);
@@ -553,10 +484,7 @@ void file_uncompress(file)
  *   -1 to -9 : compression level
  */
 
-int main(argc, argv)
-    int argc;
-    char *argv[];
-{
+int main(int argc, char *argv[]) {
     int copyout = 0;
     int uncompr = 0;
     gzFile file;
diff --git a/zlib/treebuild.xml b/zlib/treebuild.xml
index fd75525..930b00b 100644
--- a/zlib/treebuild.xml
+++ b/zlib/treebuild.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" ?>
-<package name="zlib" version="1.2.11">
-    <library name="zlib" dlversion="1.2.11" dlname="z">
+<package name="zlib" version="1.3.1">
+    <library name="zlib" dlversion="1.3.1" dlname="z">
 	<property name="description"> zip compression library </property>
 	<property name="include-target-dir" value="$(@PACKAGE/install-includedir)" />
 
diff --git a/zlib/trees.c b/zlib/trees.c
index d0a09b1..6a523ef 100644
--- a/zlib/trees.c
+++ b/zlib/trees.c
@@ -1,5 +1,5 @@
 /* trees.c -- output deflated data using Huffman coding
- * Copyright (C) 1995-2017 Jean-loup Gailly
+ * Copyright (C) 1995-2024 Jean-loup Gailly
  * detect_data_type() function provided freely by Cosmin Truta, 2006
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
@@ -30,7 +30,7 @@
  *          Addison-Wesley, 1983. ISBN 0-201-06672-6.
  */
 
-/* @(#) $Id: trees.c,v 1.1.1.2 2002/03/11 21:53:27 tromey Exp $ */
+/* @(#) $Id$ */
 
 /* #define GEN_TREES_H */
 
@@ -122,39 +122,116 @@ struct static_tree_desc_s {
     int     max_length;          /* max bit length for the codes */
 };
 
-local const static_tree_desc  static_l_desc =
+#ifdef NO_INIT_GLOBAL_POINTERS
+#  define TCONST
+#else
+#  define TCONST const
+#endif
+
+local TCONST static_tree_desc static_l_desc =
 {static_ltree, extra_lbits, LITERALS+1, L_CODES, MAX_BITS};
 
-local const static_tree_desc  static_d_desc =
+local TCONST static_tree_desc static_d_desc =
 {static_dtree, extra_dbits, 0,          D_CODES, MAX_BITS};
 
-local const static_tree_desc  static_bl_desc =
+local TCONST static_tree_desc static_bl_desc =
 {(const ct_data *)0, extra_blbits, 0,   BL_CODES, MAX_BL_BITS};
 
 /* ===========================================================================
- * Local (static) routines in this file.
+ * Output a short LSB first on the stream.
+ * IN assertion: there is enough room in pendingBuf.
+ */
+#define put_short(s, w) { \
+    put_byte(s, (uch)((w) & 0xff)); \
+    put_byte(s, (uch)((ush)(w) >> 8)); \
+}
+
+/* ===========================================================================
+ * Reverse the first len bits of a code, using straightforward code (a faster
+ * method would use a table)
+ * IN assertion: 1 <= len <= 15
  */
+local unsigned bi_reverse(unsigned code, int len) {
+    register unsigned res = 0;
+    do {
+        res |= code & 1;
+        code >>= 1, res <<= 1;
+    } while (--len > 0);
+    return res >> 1;
+}
 
-local void tr_static_init OF((void));
-local void init_block     OF((deflate_state *s));
-local void pqdownheap     OF((deflate_state *s, ct_data *tree, int k));
-local void gen_bitlen     OF((deflate_state *s, tree_desc *desc));
-local void gen_codes      OF((ct_data *tree, int max_code, ushf *bl_count));
-local void build_tree     OF((deflate_state *s, tree_desc *desc));
-local void scan_tree      OF((deflate_state *s, ct_data *tree, int max_code));
-local void send_tree      OF((deflate_state *s, ct_data *tree, int max_code));
-local int  build_bl_tree  OF((deflate_state *s));
-local void send_all_trees OF((deflate_state *s, int lcodes, int dcodes,
-                              int blcodes));
-local void compress_block OF((deflate_state *s, const ct_data *ltree,
-                              const ct_data *dtree));
-local int  detect_data_type OF((deflate_state *s));
-local unsigned bi_reverse OF((unsigned value, int length));
-local void bi_windup      OF((deflate_state *s));
-local void bi_flush       OF((deflate_state *s));
+/* ===========================================================================
+ * Flush the bit buffer, keeping at most 7 bits in it.
+ */
+local void bi_flush(deflate_state *s) {
+    if (s->bi_valid == 16) {
+        put_short(s, s->bi_buf);
+        s->bi_buf = 0;
+        s->bi_valid = 0;
+    } else if (s->bi_valid >= 8) {
+        put_byte(s, (Byte)s->bi_buf);
+        s->bi_buf >>= 8;
+        s->bi_valid -= 8;
+    }
+}
+
+/* ===========================================================================
+ * Flush the bit buffer and align the output on a byte boundary
+ */
+local void bi_windup(deflate_state *s) {
+    if (s->bi_valid > 8) {
+        put_short(s, s->bi_buf);
+    } else if (s->bi_valid > 0) {
+        put_byte(s, (Byte)s->bi_buf);
+    }
+    s->bi_buf = 0;
+    s->bi_valid = 0;
+#ifdef ZLIB_DEBUG
+    s->bits_sent = (s->bits_sent + 7) & ~7;
+#endif
+}
+
+/* ===========================================================================
+ * Generate the codes for a given tree and bit counts (which need not be
+ * optimal).
+ * IN assertion: the array bl_count contains the bit length statistics for
+ * the given tree and the field len is set for all tree elements.
+ * OUT assertion: the field code is set for all tree elements of non
+ *     zero code length.
+ */
+local void gen_codes(ct_data *tree, int max_code, ushf *bl_count) {
+    ush next_code[MAX_BITS+1]; /* next code value for each bit length */
+    unsigned code = 0;         /* running code value */
+    int bits;                  /* bit index */
+    int n;                     /* code index */
+
+    /* The distribution counts are first used to generate the code values
+     * without bit reversal.
+     */
+    for (bits = 1; bits <= MAX_BITS; bits++) {
+        code = (code + bl_count[bits - 1]) << 1;
+        next_code[bits] = (ush)code;
+    }
+    /* Check that the bit counts in bl_count are consistent. The last code
+     * must be all ones.
+     */
+    Assert (code + bl_count[MAX_BITS] - 1 == (1 << MAX_BITS) - 1,
+            "inconsistent bit counts");
+    Tracev((stderr,"\ngen_codes: max_code %d ", max_code));
+
+    for (n = 0;  n <= max_code; n++) {
+        int len = tree[n].Len;
+        if (len == 0) continue;
+        /* Now reverse the bits */
+        tree[n].Code = (ush)bi_reverse(next_code[len]++, len);
+
+        Tracecv(tree != static_ltree, (stderr,"\nn %3d %c l %2d c %4x (%x) ",
+            n, (isgraph(n) ? n : ' '), len, tree[n].Code, next_code[len] - 1));
+    }
+}
 
 #ifdef GEN_TREES_H
-local void gen_trees_header OF((void));
+local void gen_trees_header(void);
 #endif
 
 #ifndef ZLIB_DEBUG
@@ -168,32 +245,17 @@ local void gen_trees_header OF((void));
 #endif
 
 /* ===========================================================================
- * Output a short LSB first on the stream.
- * IN assertion: there is enough room in pendingBuf.
- */
-#define put_short(s, w) { \
-    put_byte(s, (uch)((w) & 0xff)); \
-    put_byte(s, (uch)((ush)(w) >> 8)); \
-}
-
-/* ===========================================================================
  * Send a value on a given number of bits.
  * IN assertion: length <= 16 and value fits in length bits.
  */
 #ifdef ZLIB_DEBUG
-local void send_bits      OF((deflate_state *s, int value, int length));
-
-local void send_bits(s, value, length)
-    deflate_state *s;
-    int value;  /* value to send */
-    int length; /* number of bits */
-{
+local void send_bits(deflate_state *s, int value, int length) {
     Tracevv((stderr," l %2d v %4x ", length, value));
     Assert(length > 0 && length <= 15, "invalid length");
     s->bits_sent += (ulg)length;
 
     /* If not enough room in bi_buf, use (valid) bits from bi_buf and
-     * (16 - bi_valid) bits from value, leaving (width - (16-bi_valid))
+     * (16 - bi_valid) bits from value, leaving (width - (16 - bi_valid))
      * unused bits in value.
      */
     if (s->bi_valid > (int)Buf_size - length) {
@@ -229,8 +291,7 @@ local void send_bits(s, value, length)
 /* ===========================================================================
  * Initialize the various 'constant' tables.
  */
-local void tr_static_init()
-{
+local void tr_static_init(void) {
 #if defined(GEN_TREES_H) || !defined(STDC)
     static int static_init_done = 0;
     int n;        /* iterates over tree elements */
@@ -256,7 +317,7 @@ local void tr_static_init()
     length = 0;
     for (code = 0; code < LENGTH_CODES-1; code++) {
         base_length[code] = length;
-        for (n = 0; n < (1<<extra_lbits[code]); n++) {
+        for (n = 0; n < (1 << extra_lbits[code]); n++) {
             _length_code[length++] = (uch)code;
         }
     }
@@ -265,13 +326,13 @@ local void tr_static_init()
      * in two different ways: code 284 + 5 bits or code 285, so we
      * overwrite length_code[255] to use the best encoding:
      */
-    _length_code[length-1] = (uch)code;
+    _length_code[length - 1] = (uch)code;
 
     /* Initialize the mapping dist (0..32K) -> dist code (0..29) */
     dist = 0;
     for (code = 0 ; code < 16; code++) {
         base_dist[code] = dist;
-        for (n = 0; n < (1<<extra_dbits[code]); n++) {
+        for (n = 0; n < (1 << extra_dbits[code]); n++) {
             _dist_code[dist++] = (uch)code;
         }
     }
@@ -279,11 +340,11 @@ local void tr_static_init()
     dist >>= 7; /* from now on, all distances are divided by 128 */
     for ( ; code < D_CODES; code++) {
         base_dist[code] = dist << 7;
-        for (n = 0; n < (1<<(extra_dbits[code]-7)); n++) {
+        for (n = 0; n < (1 << (extra_dbits[code] - 7)); n++) {
             _dist_code[256 + dist++] = (uch)code;
         }
     }
-    Assert (dist == 256, "tr_static_init: 256+dist != 512");
+    Assert (dist == 256, "tr_static_init: 256 + dist != 512");
 
     /* Construct the codes of the static literal tree */
     for (bits = 0; bits <= MAX_BITS; bits++) bl_count[bits] = 0;
@@ -312,7 +373,7 @@ local void tr_static_init()
 }
 
 /* ===========================================================================
- * Genererate the file trees.h describing the static trees.
+ * Generate the file trees.h describing the static trees.
  */
 #ifdef GEN_TREES_H
 #  ifndef ZLIB_DEBUG
@@ -321,10 +382,9 @@ local void tr_static_init()
 
 #  define SEPARATOR(i, last, width) \
       ((i) == (last)? "\n};\n\n" :    \
-       ((i) % (width) == (width)-1 ? ",\n" : ", "))
+       ((i) % (width) == (width) - 1 ? ",\n" : ", "))
 
-void gen_trees_header()
-{
+void gen_trees_header(void) {
     FILE *header = fopen("trees.h", "w");
     int i;
 
@@ -374,11 +434,25 @@ void gen_trees_header()
 #endif /* GEN_TREES_H */
 
 /* ===========================================================================
+ * Initialize a new block.
+ */
+local void init_block(deflate_state *s) {
+    int n; /* iterates over tree elements */
+
+    /* Initialize the trees. */
+    for (n = 0; n < L_CODES;  n++) s->dyn_ltree[n].Freq = 0;
+    for (n = 0; n < D_CODES;  n++) s->dyn_dtree[n].Freq = 0;
+    for (n = 0; n < BL_CODES; n++) s->bl_tree[n].Freq = 0;
+
+    s->dyn_ltree[END_BLOCK].Freq = 1;
+    s->opt_len = s->static_len = 0L;
+    s->sym_next = s->matches = 0;
+}
+
+/* ===========================================================================
  * Initialize the tree data structures for a new zlib stream.
  */
-void ZLIB_INTERNAL _tr_init(s)
-    deflate_state *s;
-{
+void ZLIB_INTERNAL _tr_init(deflate_state *s) {
     tr_static_init();
 
     s->l_desc.dyn_tree = s->dyn_ltree;
@@ -401,24 +475,6 @@ void ZLIB_INTERNAL _tr_init(s)
     init_block(s);
 }
 
-/* ===========================================================================
- * Initialize a new block.
- */
-local void init_block(s)
-    deflate_state *s;
-{
-    int n; /* iterates over tree elements */
-
-    /* Initialize the trees. */
-    for (n = 0; n < L_CODES;  n++) s->dyn_ltree[n].Freq = 0;
-    for (n = 0; n < D_CODES;  n++) s->dyn_dtree[n].Freq = 0;
-    for (n = 0; n < BL_CODES; n++) s->bl_tree[n].Freq = 0;
-
-    s->dyn_ltree[END_BLOCK].Freq = 1;
-    s->opt_len = s->static_len = 0L;
-    s->last_lit = s->matches = 0;
-}
-
 #define SMALLEST 1
 /* Index within the heap array of least frequent node in the Huffman tree */
 
@@ -448,17 +504,13 @@ local void init_block(s)
  * when the heap property is re-established (each father smaller than its
  * two sons).
  */
-local void pqdownheap(s, tree, k)
-    deflate_state *s;
-    ct_data *tree;  /* the tree to restore */
-    int k;               /* node to move down */
-{
+local void pqdownheap(deflate_state *s, ct_data *tree, int k) {
     int v = s->heap[k];
     int j = k << 1;  /* left son of k */
     while (j <= s->heap_len) {
         /* Set j to the smallest of the two sons: */
         if (j < s->heap_len &&
-            smaller(tree, s->heap[j+1], s->heap[j], s->depth)) {
+            smaller(tree, s->heap[j + 1], s->heap[j], s->depth)) {
             j++;
         }
         /* Exit if v is smaller than both sons */
@@ -483,10 +535,7 @@ local void pqdownheap(s, tree, k)
  *     The length opt_len is updated; static_len is also updated if stree is
  *     not null.
  */
-local void gen_bitlen(s, desc)
-    deflate_state *s;
-    tree_desc *desc;    /* the tree descriptor */
-{
+local void gen_bitlen(deflate_state *s, tree_desc *desc) {
     ct_data *tree        = desc->dyn_tree;
     int max_code         = desc->max_code;
     const ct_data *stree = desc->stat_desc->static_tree;
@@ -507,7 +556,7 @@ local void gen_bitlen(s, desc)
      */
     tree[s->heap[s->heap_max]].Len = 0; /* root of the heap */
 
-    for (h = s->heap_max+1; h < HEAP_SIZE; h++) {
+    for (h = s->heap_max + 1; h < HEAP_SIZE; h++) {
         n = s->heap[h];
         bits = tree[tree[n].Dad].Len + 1;
         if (bits > max_length) bits = max_length, overflow++;
@@ -518,7 +567,7 @@ local void gen_bitlen(s, desc)
 
         s->bl_count[bits]++;
         xbits = 0;
-        if (n >= base) xbits = extra[n-base];
+        if (n >= base) xbits = extra[n - base];
         f = tree[n].Freq;
         s->opt_len += (ulg)f * (unsigned)(bits + xbits);
         if (stree) s->static_len += (ulg)f * (unsigned)(stree[n].Len + xbits);
@@ -530,10 +579,10 @@ local void gen_bitlen(s, desc)
 
     /* Find the first bit length which could increase: */
     do {
-        bits = max_length-1;
+        bits = max_length - 1;
         while (s->bl_count[bits] == 0) bits--;
-        s->bl_count[bits]--;      /* move one leaf down the tree */
-        s->bl_count[bits+1] += 2; /* move one overflow item as its brother */
+        s->bl_count[bits]--;        /* move one leaf down the tree */
+        s->bl_count[bits + 1] += 2; /* move one overflow item as its brother */
         s->bl_count[max_length]--;
         /* The brother of the overflow item also moves one step up,
          * but this does not affect bl_count[max_length]
@@ -561,48 +610,9 @@ local void gen_bitlen(s, desc)
     }
 }
 
-/* ===========================================================================
- * Generate the codes for a given tree and bit counts (which need not be
- * optimal).
- * IN assertion: the array bl_count contains the bit length statistics for
- * the given tree and the field len is set for all tree elements.
- * OUT assertion: the field code is set for all tree elements of non
- *     zero code length.
- */
-local void gen_codes (tree, max_code, bl_count)
-    ct_data *tree;             /* the tree to decorate */
-    int max_code;              /* largest code with non zero frequency */
-    ushf *bl_count;            /* number of codes at each bit length */
-{
-    ush next_code[MAX_BITS+1]; /* next code value for each bit length */
-    unsigned code = 0;         /* running code value */
-    int bits;                  /* bit index */
-    int n;                     /* code index */
-
-    /* The distribution counts are first used to generate the code values
-     * without bit reversal.
-     */
-    for (bits = 1; bits <= MAX_BITS; bits++) {
-        code = (code + bl_count[bits-1]) << 1;
-        next_code[bits] = (ush)code;
-    }
-    /* Check that the bit counts in bl_count are consistent. The last code
-     * must be all ones.
-     */
-    Assert (code + bl_count[MAX_BITS]-1 == (1<<MAX_BITS)-1,
-            "inconsistent bit counts");
-    Tracev((stderr,"\ngen_codes: max_code %d ", max_code));
-
-    for (n = 0;  n <= max_code; n++) {
-        int len = tree[n].Len;
-        if (len == 0) continue;
-        /* Now reverse the bits */
-        tree[n].Code = (ush)bi_reverse(next_code[len]++, len);
-
-        Tracecv(tree != static_ltree, (stderr,"\nn %3d %c l %2d c %4x (%x) ",
-             n, (isgraph(n) ? n : ' '), len, tree[n].Code, next_code[len]-1));
-    }
-}
+#ifdef DUMP_BL_TREE
+#  include <stdio.h>
+#endif
 
 /* ===========================================================================
  * Construct one Huffman tree and assigns the code bit strings and lengths.
@@ -612,10 +622,7 @@ local void gen_codes (tree, max_code, bl_count)
  *     and corresponding code. The length opt_len is updated; static_len is
  *     also updated if stree is not null. The field max_code is set.
  */
-local void build_tree(s, desc)
-    deflate_state *s;
-    tree_desc *desc; /* the tree descriptor */
-{
+local void build_tree(deflate_state *s, tree_desc *desc) {
     ct_data *tree         = desc->dyn_tree;
     const ct_data *stree  = desc->stat_desc->static_tree;
     int elems             = desc->stat_desc->elems;
@@ -624,7 +631,7 @@ local void build_tree(s, desc)
     int node;          /* new node being created */
 
     /* Construct the initial heap, with least frequent element in
-     * heap[SMALLEST]. The sons of heap[n] are heap[2*n] and heap[2*n+1].
+     * heap[SMALLEST]. The sons of heap[n] are heap[2*n] and heap[2*n + 1].
      * heap[0] is not used.
      */
     s->heap_len = 0, s->heap_max = HEAP_SIZE;
@@ -652,7 +659,7 @@ local void build_tree(s, desc)
     }
     desc->max_code = max_code;
 
-    /* The elements heap[heap_len/2+1 .. heap_len] are leaves of the tree,
+    /* The elements heap[heap_len/2 + 1 .. heap_len] are leaves of the tree,
      * establish sub-heaps of increasing lengths:
      */
     for (n = s->heap_len/2; n >= 1; n--) pqdownheap(s, tree, n);
@@ -700,11 +707,7 @@ local void build_tree(s, desc)
  * Scan a literal or distance tree to determine the frequencies of the codes
  * in the bit length tree.
  */
-local void scan_tree (s, tree, max_code)
-    deflate_state *s;
-    ct_data *tree;   /* the tree to be scanned */
-    int max_code;    /* and its largest code of non zero frequency */
-{
+local void scan_tree(deflate_state *s, ct_data *tree, int max_code) {
     int n;                     /* iterates over all tree elements */
     int prevlen = -1;          /* last emitted length */
     int curlen;                /* length of current code */
@@ -714,10 +717,10 @@ local void scan_tree (s, tree, max_code)
     int min_count = 4;         /* min repeat count */
 
     if (nextlen == 0) max_count = 138, min_count = 3;
-    tree[max_code+1].Len = (ush)0xffff; /* guard */
+    tree[max_code + 1].Len = (ush)0xffff; /* guard */
 
     for (n = 0; n <= max_code; n++) {
-        curlen = nextlen; nextlen = tree[n+1].Len;
+        curlen = nextlen; nextlen = tree[n + 1].Len;
         if (++count < max_count && curlen == nextlen) {
             continue;
         } else if (count < min_count) {
@@ -745,11 +748,7 @@ local void scan_tree (s, tree, max_code)
  * Send a literal or distance tree in compressed form, using the codes in
  * bl_tree.
  */
-local void send_tree (s, tree, max_code)
-    deflate_state *s;
-    ct_data *tree; /* the tree to be scanned */
-    int max_code;       /* and its largest code of non zero frequency */
-{
+local void send_tree(deflate_state *s, ct_data *tree, int max_code) {
     int n;                     /* iterates over all tree elements */
     int prevlen = -1;          /* last emitted length */
     int curlen;                /* length of current code */
@@ -758,11 +757,11 @@ local void send_tree (s, tree, max_code)
     int max_count = 7;         /* max repeat count */
     int min_count = 4;         /* min repeat count */
 
-    /* tree[max_code+1].Len = -1; */  /* guard already set */
+    /* tree[max_code + 1].Len = -1; */  /* guard already set */
     if (nextlen == 0) max_count = 138, min_count = 3;
 
     for (n = 0; n <= max_code; n++) {
-        curlen = nextlen; nextlen = tree[n+1].Len;
+        curlen = nextlen; nextlen = tree[n + 1].Len;
         if (++count < max_count && curlen == nextlen) {
             continue;
         } else if (count < min_count) {
@@ -773,13 +772,13 @@ local void send_tree (s, tree, max_code)
                 send_code(s, curlen, s->bl_tree); count--;
             }
             Assert(count >= 3 && count <= 6, " 3_6?");
-            send_code(s, REP_3_6, s->bl_tree); send_bits(s, count-3, 2);
+            send_code(s, REP_3_6, s->bl_tree); send_bits(s, count - 3, 2);
 
         } else if (count <= 10) {
-            send_code(s, REPZ_3_10, s->bl_tree); send_bits(s, count-3, 3);
+            send_code(s, REPZ_3_10, s->bl_tree); send_bits(s, count - 3, 3);
 
         } else {
-            send_code(s, REPZ_11_138, s->bl_tree); send_bits(s, count-11, 7);
+            send_code(s, REPZ_11_138, s->bl_tree); send_bits(s, count - 11, 7);
         }
         count = 0; prevlen = curlen;
         if (nextlen == 0) {
@@ -796,9 +795,7 @@ local void send_tree (s, tree, max_code)
  * Construct the Huffman tree for the bit lengths and return the index in
  * bl_order of the last bit length code to send.
  */
-local int build_bl_tree(s)
-    deflate_state *s;
-{
+local int build_bl_tree(deflate_state *s) {
     int max_blindex;  /* index of last bit length code of non zero freq */
 
     /* Determine the bit length frequencies for literal and distance trees */
@@ -807,8 +804,8 @@ local int build_bl_tree(s)
 
     /* Build the bit length tree: */
     build_tree(s, (tree_desc *)(&(s->bl_desc)));
-    /* opt_len now includes the length of the tree representations, except
-     * the lengths of the bit lengths codes and the 5+5+4 bits for the counts.
+    /* opt_len now includes the length of the tree representations, except the
+     * lengths of the bit lengths codes and the 5 + 5 + 4 bits for the counts.
      */
 
     /* Determine the number of bit length codes to send. The pkzip format
@@ -819,7 +816,7 @@ local int build_bl_tree(s)
         if (s->bl_tree[bl_order[max_blindex]].Len != 0) break;
     }
     /* Update opt_len to include the bit length tree and counts */
-    s->opt_len += 3*((ulg)max_blindex+1) + 5+5+4;
+    s->opt_len += 3*((ulg)max_blindex + 1) + 5 + 5 + 4;
     Tracev((stderr, "\ndyn trees: dyn %ld, stat %ld",
             s->opt_len, s->static_len));
 
@@ -831,61 +828,54 @@ local int build_bl_tree(s)
  * lengths of the bit length codes, the literal tree and the distance tree.
  * IN assertion: lcodes >= 257, dcodes >= 1, blcodes >= 4.
  */
-local void send_all_trees(s, lcodes, dcodes, blcodes)
-    deflate_state *s;
-    int lcodes, dcodes, blcodes; /* number of codes for each tree */
-{
+local void send_all_trees(deflate_state *s, int lcodes, int dcodes,
+                          int blcodes) {
     int rank;                    /* index in bl_order */
 
     Assert (lcodes >= 257 && dcodes >= 1 && blcodes >= 4, "not enough codes");
     Assert (lcodes <= L_CODES && dcodes <= D_CODES && blcodes <= BL_CODES,
             "too many codes");
     Tracev((stderr, "\nbl counts: "));
-    send_bits(s, lcodes-257, 5); /* not +255 as stated in appnote.txt */
-    send_bits(s, dcodes-1,   5);
-    send_bits(s, blcodes-4,  4); /* not -3 as stated in appnote.txt */
+    send_bits(s, lcodes - 257, 5);  /* not +255 as stated in appnote.txt */
+    send_bits(s, dcodes - 1,   5);
+    send_bits(s, blcodes - 4,  4);  /* not -3 as stated in appnote.txt */
     for (rank = 0; rank < blcodes; rank++) {
         Tracev((stderr, "\nbl code %2d ", bl_order[rank]));
         send_bits(s, s->bl_tree[bl_order[rank]].Len, 3);
     }
     Tracev((stderr, "\nbl tree: sent %ld", s->bits_sent));
 
-    send_tree(s, (ct_data *)s->dyn_ltree, lcodes-1); /* literal tree */
+    send_tree(s, (ct_data *)s->dyn_ltree, lcodes - 1);  /* literal tree */
     Tracev((stderr, "\nlit tree: sent %ld", s->bits_sent));
 
-    send_tree(s, (ct_data *)s->dyn_dtree, dcodes-1); /* distance tree */
+    send_tree(s, (ct_data *)s->dyn_dtree, dcodes - 1);  /* distance tree */
     Tracev((stderr, "\ndist tree: sent %ld", s->bits_sent));
 }
 
 /* ===========================================================================
  * Send a stored block
  */
-void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last)
-    deflate_state *s;
-    charf *buf;       /* input block */
-    ulg stored_len;   /* length of input block */
-    int last;         /* one if this is the last block for a file */
-{
-    send_bits(s, (STORED_BLOCK<<1)+last, 3);    /* send block type */
+void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf,
+                                    ulg stored_len, int last) {
+    send_bits(s, (STORED_BLOCK<<1) + last, 3);  /* send block type */
     bi_windup(s);        /* align on byte boundary */
     put_short(s, (ush)stored_len);
     put_short(s, (ush)~stored_len);
-    zmemcpy(s->pending_buf + s->pending, (Bytef *)buf, stored_len);
+    if (stored_len)
+        zmemcpy(s->pending_buf + s->pending, (Bytef *)buf, stored_len);
     s->pending += stored_len;
 #ifdef ZLIB_DEBUG
     s->compressed_len = (s->compressed_len + 3 + 7) & (ulg)~7L;
     s->compressed_len += (stored_len + 4) << 3;
     s->bits_sent += 2*16;
-    s->bits_sent += stored_len<<3;
+    s->bits_sent += stored_len << 3;
 #endif
 }
 
 /* ===========================================================================
  * Flush the bits in the bit buffer to pending output (leaves at most 7 bits)
  */
-void ZLIB_INTERNAL _tr_flush_bits(s)
-    deflate_state *s;
-{
+void ZLIB_INTERNAL _tr_flush_bits(deflate_state *s) {
     bi_flush(s);
 }
 
@@ -893,9 +883,7 @@ void ZLIB_INTERNAL _tr_flush_bits(s)
  * Send one empty static block to give enough lookahead for inflate.
  * This takes 10 bits, of which 7 may remain in the bit buffer.
  */
-void ZLIB_INTERNAL _tr_align(s)
-    deflate_state *s;
-{
+void ZLIB_INTERNAL _tr_align(deflate_state *s) {
     send_bits(s, STATIC_TREES<<1, 3);
     send_code(s, END_BLOCK, static_ltree);
 #ifdef ZLIB_DEBUG
@@ -905,15 +893,107 @@ void ZLIB_INTERNAL _tr_align(s)
 }
 
 /* ===========================================================================
+ * Send the block data compressed using the given Huffman trees
+ */
+local void compress_block(deflate_state *s, const ct_data *ltree,
+                          const ct_data *dtree) {
+    unsigned dist;      /* distance of matched string */
+    int lc;             /* match length or unmatched char (if dist == 0) */
+    unsigned sx = 0;    /* running index in symbol buffers */
+    unsigned code;      /* the code to send */
+    int extra;          /* number of extra bits to send */
+
+    if (s->sym_next != 0) do {
+#ifdef LIT_MEM
+        dist = s->d_buf[sx];
+        lc = s->l_buf[sx++];
+#else
+        dist = s->sym_buf[sx++] & 0xff;
+        dist += (unsigned)(s->sym_buf[sx++] & 0xff) << 8;
+        lc = s->sym_buf[sx++];
+#endif
+        if (dist == 0) {
+            send_code(s, lc, ltree); /* send a literal byte */
+            Tracecv(isgraph(lc), (stderr," '%c' ", lc));
+        } else {
+            /* Here, lc is the match length - MIN_MATCH */
+            code = _length_code[lc];
+            send_code(s, code + LITERALS + 1, ltree);   /* send length code */
+            extra = extra_lbits[code];
+            if (extra != 0) {
+                lc -= base_length[code];
+                send_bits(s, lc, extra);       /* send the extra length bits */
+            }
+            dist--; /* dist is now the match distance - 1 */
+            code = d_code(dist);
+            Assert (code < D_CODES, "bad d_code");
+
+            send_code(s, code, dtree);       /* send the distance code */
+            extra = extra_dbits[code];
+            if (extra != 0) {
+                dist -= (unsigned)base_dist[code];
+                send_bits(s, dist, extra);   /* send the extra distance bits */
+            }
+        } /* literal or match pair ? */
+
+        /* Check for no overlay of pending_buf on needed symbols */
+#ifdef LIT_MEM
+        Assert(s->pending < 2 * (s->lit_bufsize + sx), "pendingBuf overflow");
+#else
+        Assert(s->pending < s->lit_bufsize + sx, "pendingBuf overflow");
+#endif
+
+    } while (sx < s->sym_next);
+
+    send_code(s, END_BLOCK, ltree);
+}
+
+/* ===========================================================================
+ * Check if the data type is TEXT or BINARY, using the following algorithm:
+ * - TEXT if the two conditions below are satisfied:
+ *    a) There are no non-portable control characters belonging to the
+ *       "block list" (0..6, 14..25, 28..31).
+ *    b) There is at least one printable character belonging to the
+ *       "allow list" (9 {TAB}, 10 {LF}, 13 {CR}, 32..255).
+ * - BINARY otherwise.
+ * - The following partially-portable control characters form a
+ *   "gray list" that is ignored in this detection algorithm:
+ *   (7 {BEL}, 8 {BS}, 11 {VT}, 12 {FF}, 26 {SUB}, 27 {ESC}).
+ * IN assertion: the fields Freq of dyn_ltree are set.
+ */
+local int detect_data_type(deflate_state *s) {
+    /* block_mask is the bit mask of block-listed bytes
+     * set bits 0..6, 14..25, and 28..31
+     * 0xf3ffc07f = binary 11110011111111111100000001111111
+     */
+    unsigned long block_mask = 0xf3ffc07fUL;
+    int n;
+
+    /* Check for non-textual ("block-listed") bytes. */
+    for (n = 0; n <= 31; n++, block_mask >>= 1)
+        if ((block_mask & 1) && (s->dyn_ltree[n].Freq != 0))
+            return Z_BINARY;
+
+    /* Check for textual ("allow-listed") bytes. */
+    if (s->dyn_ltree[9].Freq != 0 || s->dyn_ltree[10].Freq != 0
+            || s->dyn_ltree[13].Freq != 0)
+        return Z_TEXT;
+    for (n = 32; n < LITERALS; n++)
+        if (s->dyn_ltree[n].Freq != 0)
+            return Z_TEXT;
+
+    /* There are no "block-listed" or "allow-listed" bytes:
+     * this stream either is empty or has tolerated ("gray-listed") bytes only.
+     */
+    return Z_BINARY;
+}
+
+/* ===========================================================================
  * Determine the best encoding for the current block: dynamic trees, static
  * trees or store, and write out the encoded block.
  */
-void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last)
-    deflate_state *s;
-    charf *buf;       /* input block, or NULL if too old */
-    ulg stored_len;   /* length of input block */
-    int last;         /* one if this is the last block for a file */
-{
+void ZLIB_INTERNAL _tr_flush_block(deflate_state *s, charf *buf,
+                                   ulg stored_len, int last) {
     ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
     int max_blindex = 0;  /* index of last bit length code of non zero freq */
 
@@ -942,14 +1022,17 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last)
         max_blindex = build_bl_tree(s);
 
         /* Determine the best encoding. Compute the block lengths in bytes. */
-        opt_lenb = (s->opt_len+3+7)>>3;
-        static_lenb = (s->static_len+3+7)>>3;
+        opt_lenb = (s->opt_len + 3 + 7) >> 3;
+        static_lenb = (s->static_len + 3 + 7) >> 3;
 
         Tracev((stderr, "\nopt %lu(%lu) stat %lu(%lu) stored %lu lit %u ",
                 opt_lenb, s->opt_len, static_lenb, s->static_len, stored_len,
-                s->last_lit));
+                s->sym_next / 3));
 
-        if (static_lenb <= opt_lenb) opt_lenb = static_lenb;
+#ifndef FORCE_STATIC
+        if (static_lenb <= opt_lenb || s->strategy == Z_FIXED)
+#endif
+            opt_lenb = static_lenb;
 
     } else {
         Assert(buf != (char*)0, "lost buf");
@@ -959,7 +1042,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last)
 #ifdef FORCE_STORED
     if (buf != (char*)0) { /* force stored block */
 #else
-    if (stored_len+4 <= opt_lenb && buf != (char*)0) {
+    if (stored_len + 4 <= opt_lenb && buf != (char*)0) {
                        /* 4: two words for the lengths */
 #endif
         /* The test buf != NULL is only necessary if LIT_BUFSIZE > WSIZE.
@@ -970,21 +1053,17 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last)
          */
         _tr_stored_block(s, buf, stored_len, last);
 
-#ifdef FORCE_STATIC
-    } else if (static_lenb >= 0) { /* force static trees */
-#else
-    } else if (s->strategy == Z_FIXED || static_lenb == opt_lenb) {
-#endif
-        send_bits(s, (STATIC_TREES<<1)+last, 3);
+    } else if (static_lenb == opt_lenb) {
+        send_bits(s, (STATIC_TREES<<1) + last, 3);
         compress_block(s, (const ct_data *)static_ltree,
                        (const ct_data *)static_dtree);
 #ifdef ZLIB_DEBUG
         s->compressed_len += 3 + s->static_len;
 #endif
     } else {
-        send_bits(s, (DYN_TREES<<1)+last, 3);
-        send_all_trees(s, s->l_desc.max_code+1, s->d_desc.max_code+1,
-                       max_blindex+1);
+        send_bits(s, (DYN_TREES<<1) + last, 3);
+        send_all_trees(s, s->l_desc.max_code + 1, s->d_desc.max_code + 1,
+                       max_blindex + 1);
         compress_block(s, (const ct_data *)s->dyn_ltree,
                        (const ct_data *)s->dyn_dtree);
 #ifdef ZLIB_DEBUG
@@ -1003,21 +1082,23 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last)
         s->compressed_len += 7;  /* align on byte boundary */
 #endif
     }
-    Tracev((stderr,"\ncomprlen %lu(%lu) ", s->compressed_len>>3,
-           s->compressed_len-7*last));
+    Tracev((stderr,"\ncomprlen %lu(%lu) ", s->compressed_len >> 3,
+           s->compressed_len - 7*last));
 }
 
 /* ===========================================================================
  * Save the match info and tally the frequency counts. Return true if
  * the current block must be flushed.
  */
-int ZLIB_INTERNAL _tr_tally (s, dist, lc)
-    deflate_state *s;
-    unsigned dist;  /* distance of matched string */
-    unsigned lc;    /* match length-MIN_MATCH or unmatched char (if dist==0) */
-{
-    s->d_buf[s->last_lit] = (ush)dist;
-    s->l_buf[s->last_lit++] = (uch)lc;
+int ZLIB_INTERNAL _tr_tally(deflate_state *s, unsigned dist, unsigned lc) {
+#ifdef LIT_MEM
+    s->d_buf[s->sym_next] = (ush)dist;
+    s->l_buf[s->sym_next++] = (uch)lc;
+#else
+    s->sym_buf[s->sym_next++] = (uch)dist;
+    s->sym_buf[s->sym_next++] = (uch)(dist >> 8);
+    s->sym_buf[s->sym_next++] = (uch)lc;
+#endif
     if (dist == 0) {
         /* lc is the unmatched char */
         s->dyn_ltree[lc].Freq++;
@@ -1029,175 +1110,8 @@ int ZLIB_INTERNAL _tr_tally (s, dist, lc)
                (ush)lc <= (ush)(MAX_MATCH-MIN_MATCH) &&
                (ush)d_code(dist) < (ush)D_CODES,  "_tr_tally: bad match");
 
-        s->dyn_ltree[_length_code[lc]+LITERALS+1].Freq++;
+        s->dyn_ltree[_length_code[lc] + LITERALS + 1].Freq++;
         s->dyn_dtree[d_code(dist)].Freq++;
     }
-
-#ifdef TRUNCATE_BLOCK
-    /* Try to guess if it is profitable to stop the current block here */
-    if ((s->last_lit & 0x1fff) == 0 && s->level > 2) {
-        /* Compute an upper bound for the compressed length */
-        ulg out_length = (ulg)s->last_lit*8L;
-        ulg in_length = (ulg)((long)s->strstart - s->block_start);
-        int dcode;
-        for (dcode = 0; dcode < D_CODES; dcode++) {
-            out_length += (ulg)s->dyn_dtree[dcode].Freq *
-                (5L+extra_dbits[dcode]);
-        }
-        out_length >>= 3;
-        Tracev((stderr,"\nlast_lit %u, in %ld, out ~%ld(%ld%%) ",
-               s->last_lit, in_length, out_length,
-               100L - out_length*100L/in_length));
-        if (s->matches < s->last_lit/2 && out_length < in_length/2) return 1;
-    }
-#endif
-    return (s->last_lit == s->lit_bufsize-1);
-    /* We avoid equality with lit_bufsize because of wraparound at 64K
-     * on 16 bit machines and because stored blocks are restricted to
-     * 64K-1 bytes.
-     */
-}
-
-/* ===========================================================================
- * Send the block data compressed using the given Huffman trees
- */
-local void compress_block(s, ltree, dtree)
-    deflate_state *s;
-    const ct_data *ltree; /* literal tree */
-    const ct_data *dtree; /* distance tree */
-{
-    unsigned dist;      /* distance of matched string */
-    int lc;             /* match length or unmatched char (if dist == 0) */
-    unsigned lx = 0;    /* running index in l_buf */
-    unsigned code;      /* the code to send */
-    int extra;          /* number of extra bits to send */
-
-    if (s->last_lit != 0) do {
-        dist = s->d_buf[lx];
-        lc = s->l_buf[lx++];
-        if (dist == 0) {
-            send_code(s, lc, ltree); /* send a literal byte */
-            Tracecv(isgraph(lc), (stderr," '%c' ", lc));
-        } else {
-            /* Here, lc is the match length - MIN_MATCH */
-            code = _length_code[lc];
-            send_code(s, code+LITERALS+1, ltree); /* send the length code */
-            extra = extra_lbits[code];
-            if (extra != 0) {
-                lc -= base_length[code];
-                send_bits(s, lc, extra);       /* send the extra length bits */
-            }
-            dist--; /* dist is now the match distance - 1 */
-            code = d_code(dist);
-            Assert (code < D_CODES, "bad d_code");
-
-            send_code(s, code, dtree);       /* send the distance code */
-            extra = extra_dbits[code];
-            if (extra != 0) {
-                dist -= (unsigned)base_dist[code];
-                send_bits(s, dist, extra);   /* send the extra distance bits */
-            }
-        } /* literal or match pair ? */
-
-        /* Check that the overlay between pending_buf and d_buf+l_buf is ok: */
-        Assert((uInt)(s->pending) < s->lit_bufsize + 2*lx,
-               "pendingBuf overflow");
-
-    } while (lx < s->last_lit);
-
-    send_code(s, END_BLOCK, ltree);
-}
-
-/* ===========================================================================
- * Check if the data type is TEXT or BINARY, using the following algorithm:
- * - TEXT if the two conditions below are satisfied:
- *    a) There are no non-portable control characters belonging to the
- *       "black list" (0..6, 14..25, 28..31).
- *    b) There is at least one printable character belonging to the
- *       "white list" (9 {TAB}, 10 {LF}, 13 {CR}, 32..255).
- * - BINARY otherwise.
- * - The following partially-portable control characters form a
- *   "gray list" that is ignored in this detection algorithm:
- *   (7 {BEL}, 8 {BS}, 11 {VT}, 12 {FF}, 26 {SUB}, 27 {ESC}).
- * IN assertion: the fields Freq of dyn_ltree are set.
- */
-local int detect_data_type(s)
-    deflate_state *s;
-{
-    /* black_mask is the bit mask of black-listed bytes
-     * set bits 0..6, 14..25, and 28..31
-     * 0xf3ffc07f = binary 11110011111111111100000001111111
-     */
-    unsigned long black_mask = 0xf3ffc07fUL;
-    int n;
-
-    /* Check for non-textual ("black-listed") bytes. */
-    for (n = 0; n <= 31; n++, black_mask >>= 1)
-        if ((black_mask & 1) && (s->dyn_ltree[n].Freq != 0))
-            return Z_BINARY;
-
-    /* Check for textual ("white-listed") bytes. */
-    if (s->dyn_ltree[9].Freq != 0 || s->dyn_ltree[10].Freq != 0
-            || s->dyn_ltree[13].Freq != 0)
-        return Z_TEXT;
-    for (n = 32; n < LITERALS; n++)
-        if (s->dyn_ltree[n].Freq != 0)
-            return Z_TEXT;
-
-    /* There are no "black-listed" or "white-listed" bytes:
-     * this stream either is empty or has tolerated ("gray-listed") bytes only.
-     */
-    return Z_BINARY;
-}
-
-/* ===========================================================================
- * Reverse the first len bits of a code, using straightforward code (a faster
- * method would use a table)
- * IN assertion: 1 <= len <= 15
- */
-local unsigned bi_reverse(code, len)
-    unsigned code; /* the value to invert */
-    int len;       /* its bit length */
-{
-    register unsigned res = 0;
-    do {
-        res |= code & 1;
-        code >>= 1, res <<= 1;
-    } while (--len > 0);
-    return res >> 1;
-}
-
-/* ===========================================================================
- * Flush the bit buffer, keeping at most 7 bits in it.
- */
-local void bi_flush(s)
-    deflate_state *s;
-{
-    if (s->bi_valid == 16) {
-        put_short(s, s->bi_buf);
-        s->bi_buf = 0;
-        s->bi_valid = 0;
-    } else if (s->bi_valid >= 8) {
-        put_byte(s, (Byte)s->bi_buf);
-        s->bi_buf >>= 8;
-        s->bi_valid -= 8;
-    }
-}
-
-/* ===========================================================================
- * Flush the bit buffer and align the output on a byte boundary
- */
-local void bi_windup(s)
-    deflate_state *s;
-{
-    if (s->bi_valid > 8) {
-        put_short(s, s->bi_buf);
-    } else if (s->bi_valid > 0) {
-        put_byte(s, (Byte)s->bi_buf);
-    }
-    s->bi_buf = 0;
-    s->bi_valid = 0;
-#ifdef ZLIB_DEBUG
-    s->bits_sent = (s->bits_sent+7) & ~7;
-#endif
+    return (s->sym_next == s->sym_end);
 }
diff --git a/zlib/uncompr.c b/zlib/uncompr.c
index a6ef81e..5e25666 100644
--- a/zlib/uncompr.c
+++ b/zlib/uncompr.c
@@ -3,7 +3,7 @@
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
-/* @(#) $Id: uncompr.c,v 1.1.1.2 2002/03/11 21:53:27 tromey Exp $ */
+/* @(#) $Id$ */
 
 #define ZLIB_INTERNAL
 #include "zlib.h"
@@ -24,12 +24,8 @@
    Z_DATA_ERROR if the input data was corrupted, including if the input data is
    an incomplete zlib stream.
 */
-int ZEXPORT uncompress2 (dest, destLen, source, sourceLen)
-    Bytef *dest;
-    uLongf *destLen;
-    const Bytef *source;
-    uLong *sourceLen;
-{
+int ZEXPORT uncompress2(Bytef *dest, uLongf *destLen, const Bytef *source,
+                        uLong *sourceLen) {
     z_stream stream;
     int err;
     const uInt max = (uInt)-1;
@@ -83,11 +79,7 @@ int ZEXPORT uncompress2 (dest, destLen, source, sourceLen)
            err;
 }
 
-int ZEXPORT uncompress (dest, destLen, source, sourceLen)
-    Bytef *dest;
-    uLongf *destLen;
-    const Bytef *source;
-    uLong sourceLen;
-{
+int ZEXPORT uncompress(Bytef *dest, uLongf *destLen, const Bytef *source,
+                       uLong sourceLen) {
     return uncompress2(dest, destLen, source, &sourceLen);
 }
diff --git a/zlib/win32/DLL_FAQ.txt b/zlib/win32/DLL_FAQ.txt
index 12c0090..d8cf5f3 100644
--- a/zlib/win32/DLL_FAQ.txt
+++ b/zlib/win32/DLL_FAQ.txt
@@ -3,7 +3,7 @@
 
 
 This document describes the design, the rationale, and the usage
-of the official DLL build of zlib, named ZLIB1.DLL.  If you have
+of the common DLL build of zlib, named ZLIB1.DLL.  If you have
 general questions about zlib, you should see the file "FAQ" found
 in the zlib distribution, or at the following location:
   http://www.gzip.org/zlib/zlib_faq.html
@@ -11,13 +11,9 @@ in the zlib distribution, or at the following location:
 
  1. What is ZLIB1.DLL, and how can I get it?
 
-  - ZLIB1.DLL is the official build of zlib as a DLL.
+  - ZLIB1.DLL is the common build of zlib as a DLL.
     (Please remark the character '1' in the name.)
 
-    Pointers to a precompiled ZLIB1.DLL can be found in the zlib
-    web site at:
-      http://www.zlib.net/
-
     Applications that link to ZLIB1.DLL can rely on the following
     specification:
 
@@ -379,18 +375,6 @@ in the zlib distribution, or at the following location:
     code.  But you can make your own private DLL build, under a
     different file name, as suggested in the previous answer.
 
-
-17. I made my own ZLIB1.DLL build.  Can I test it for compliance?
-
-  - We prefer that you download the official DLL from the zlib
-    web site.  If you need something peculiar from this DLL, you
-    can send your suggestion to the zlib mailing list.
-
-    However, in case you do rebuild the DLL yourself, you can run
-    it with the test programs found in the DLL distribution.
-    Running these test programs is not a guarantee of compliance,
-    but a failure can imply a detected problem.
-
 **
 
 This document is written and maintained by
diff --git a/zlib/win32/Makefile.bor b/zlib/win32/Makefile.bor
index d152bbb..4495353 100644
--- a/zlib/win32/Makefile.bor
+++ b/zlib/win32/Makefile.bor
@@ -3,7 +3,6 @@
 #
 # Usage:
 #  make -f win32/Makefile.bor
-#  make -f win32/Makefile.bor LOCAL_ZLIB=-DASMV OBJA=match.obj OBJPA=+match.obj
 
 # ------------ Borland C++ ------------
 
diff --git a/zlib/win32/Makefile.gcc b/zlib/win32/Makefile.gcc
index 305be50..081e391 100644
--- a/zlib/win32/Makefile.gcc
+++ b/zlib/win32/Makefile.gcc
@@ -11,10 +11,6 @@
 #
 #   make -fwin32/Makefile.gcc;  make test testdll -fwin32/Makefile.gcc
 #
-# To use the asm code, type:
-#   cp contrib/asm?86/match.S ./match.S
-#   make LOC=-DASMV OBJA=match.o -fwin32/Makefile.gcc
-#
 # To install libz.a, zconf.h and zlib.h in the system directories, type:
 #
 #   make install -fwin32/Makefile.gcc
@@ -38,7 +34,6 @@ IMPLIB    = libz.dll.a
 #
 SHARED_MODE=0
 
-#LOC = -DASMV
 #LOC = -DZLIB_DEBUG -g
 
 PREFIX =
diff --git a/zlib/win32/Makefile.msc b/zlib/win32/Makefile.msc
index 6831882..9c65153 100644
--- a/zlib/win32/Makefile.msc
+++ b/zlib/win32/Makefile.msc
@@ -4,10 +4,6 @@
 # Usage:
 #   nmake -f win32/Makefile.msc                          (standard build)
 #   nmake -f win32/Makefile.msc LOC=-DFOO                (nonstandard build)
-#   nmake -f win32/Makefile.msc LOC="-DASMV -DASMINF" \
-#         OBJA="inffas32.obj match686.obj"               (use ASM code, x86)
-#   nmake -f win32/Makefile.msc AS=ml64 LOC="-DASMV -DASMINF -I." \
-#         OBJA="inffasx64.obj gvmat64.obj inffas8664.obj"  (use ASM code, x64)
 
 # The toplevel directory of the source tree.
 #
diff --git a/zlib/win32/README-WIN32.txt b/zlib/win32/README-WIN32.txt
index df7ab7f..14e6398 100644
--- a/zlib/win32/README-WIN32.txt
+++ b/zlib/win32/README-WIN32.txt
@@ -1,6 +1,6 @@
 ZLIB DATA COMPRESSION LIBRARY
 
-zlib 1.2.11 is a general purpose data compression library.  All the code is
+zlib 1.3.1 is a general purpose data compression library.  All the code is
 thread safe.  The data format used by the zlib library is described by RFCs
 (Request for Comments) 1950 to 1952 in the files
 http://www.ietf.org/rfc/rfc1950.txt (zlib format), rfc1951.txt (deflate format)
@@ -16,13 +16,13 @@ is http://zlib.net/ .  Before reporting a problem, please check this site to
 verify that you have the latest version of zlib; otherwise get the latest
 version and check whether the problem still exists or not.
 
-PLEASE read DLL_FAQ.txt, and the the zlib FAQ http://zlib.net/zlib_faq.html
-before asking for help.
+PLEASE read DLL_FAQ.txt, and the zlib FAQ http://zlib.net/zlib_faq.html before
+asking for help.
 
 
 Manifest:
 
-The package zlib-1.2.11-win32-x86.zip will contain the following files:
+The package zlib-1.3.1-win32-x86.zip will contain the following files:
 
   README-WIN32.txt This document
   ChangeLog        Changes since previous zlib packages
diff --git a/zlib/win32/zlib.def b/zlib/win32/zlib.def
index 0cb3f5d..53c8011 100644
--- a/zlib/win32/zlib.def
+++ b/zlib/win32/zlib.def
@@ -69,6 +69,7 @@ EXPORTS
     gzoffset64

     adler32_combine64

     crc32_combine64

+    crc32_combine_gen64

 ; checksum functions

     adler32

     adler32_z

@@ -76,6 +77,8 @@ EXPORTS
     crc32_z

     adler32_combine

     crc32_combine

+    crc32_combine_gen

+    crc32_combine_op

 ; various hacks, don't look :)

     deflateInit_

     deflateInit2_

@@ -91,3 +94,4 @@ EXPORTS
     inflateCodesUsed

     inflateResetKeep

     deflateResetKeep

+    gzopen_w

diff --git a/zlib/win32/zlib1.rc b/zlib/win32/zlib1.rc
index 234e641..ceb4ee5 100644
--- a/zlib/win32/zlib1.rc
+++ b/zlib/win32/zlib1.rc
@@ -26,7 +26,7 @@ BEGIN
       VALUE "FileDescription",	"zlib data compression library\0"
       VALUE "FileVersion",	ZLIB_VERSION "\0"
       VALUE "InternalName",	"zlib1.dll\0"
-      VALUE "LegalCopyright",	"(C) 1995-2017 Jean-loup Gailly & Mark Adler\0"
+      VALUE "LegalCopyright",	"(C) 1995-2022 Jean-loup Gailly & Mark Adler\0"
       VALUE "OriginalFilename",	"zlib1.dll\0"
       VALUE "ProductName",	"zlib\0"
       VALUE "ProductVersion",	ZLIB_VERSION "\0"
diff --git a/zlib/zconf.h b/zlib/zconf.h
index b399839..62adc8d 100644
--- a/zlib/zconf.h
+++ b/zlib/zconf.h
@@ -1,9 +1,9 @@
 /* zconf.h -- configuration of the zlib compression library
- * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler
+ * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
-/* @(#) $Id: zconf.h,v 1.1.1.2 2002/03/11 21:53:27 tromey Exp $ */
+/* @(#) $Id$ */
 
 #ifndef ZCONF_H
 #define ZCONF_H
@@ -38,6 +38,9 @@
 #  define crc32                 z_crc32
 #  define crc32_combine         z_crc32_combine
 #  define crc32_combine64       z_crc32_combine64
+#  define crc32_combine_gen     z_crc32_combine_gen
+#  define crc32_combine_gen64   z_crc32_combine_gen64
+#  define crc32_combine_op      z_crc32_combine_op
 #  define crc32_z               z_crc32_z
 #  define deflate               z_deflate
 #  define deflateBound          z_deflateBound
@@ -238,7 +241,11 @@
 #endif
 
 #ifdef Z_SOLO
-   typedef unsigned long z_size_t;
+#  ifdef _WIN64
+     typedef unsigned long long z_size_t;
+#  else
+     typedef unsigned long z_size_t;
+#  endif
 #else
 #  define z_longlong long long
 #  if defined(NO_SIZE_T)
@@ -293,14 +300,6 @@
 #  endif
 #endif
 
-#ifndef Z_ARG /* function prototypes for stdarg */
-#  if defined(STDC) || defined(Z_HAVE_STDARG_H)
-#    define Z_ARG(args)  args
-#  else
-#    define Z_ARG(args)  ()
-#  endif
-#endif
-
 /* The following definitions for FAR are needed only for MSDOS mixed
  * model programming (small or medium model with some far allocations).
  * This was tested only with MSC; for other MSDOS compilers you may have
@@ -349,6 +348,9 @@
 #    ifdef FAR
 #      undef FAR
 #    endif
+#    ifndef WIN32_LEAN_AND_MEAN
+#      define WIN32_LEAN_AND_MEAN
+#    endif
 #    include <windows.h>
      /* No need for _export, use ZLIB.DEF instead. */
      /* For complete Windows compatibility, use WINAPI, not __stdcall. */
@@ -467,11 +469,18 @@ typedef uLong FAR uLongf;
 #  undef _LARGEFILE64_SOURCE
 #endif
 
-#if defined(__WATCOMC__) && !defined(Z_HAVE_UNISTD_H)
-#  define Z_HAVE_UNISTD_H
+#ifndef Z_HAVE_UNISTD_H
+#  ifdef __WATCOMC__
+#    define Z_HAVE_UNISTD_H
+#  endif
+#endif
+#ifndef Z_HAVE_UNISTD_H
+#  if defined(_LARGEFILE64_SOURCE) && !defined(_WIN32)
+#    define Z_HAVE_UNISTD_H
+#  endif
 #endif
 #ifndef Z_SOLO
-#  if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
+#  if defined(Z_HAVE_UNISTD_H)
 #    include <unistd.h>         /* for SEEK_*, off_t, and _LFS64_LARGEFILE */
 #    ifdef VMS
 #      include <unixio.h>       /* for off_t */
@@ -507,7 +516,7 @@ typedef uLong FAR uLongf;
 #if !defined(_WIN32) && defined(Z_LARGE64)
 #  define z_off64_t off64_t
 #else
-#  if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO)
+#  if defined(_WIN32) && !defined(__GNUC__)
 #    define z_off64_t __int64
 #  else
 #    define z_off64_t z_off_t
@@ -531,20 +540,4 @@ typedef uLong FAR uLongf;
   #pragma map(inflate_copyright,"INCOPY")
 #endif
 
-/* Solaris 2.5.1 doesn't have vsnprintf */
-#if defined(__sun) && defined(__svr4__)
-#include <sys/feature_tests.h>
-#if _XOPEN_VERSION < 3
-#define NO_vsnprintf
-#endif
-#endif
-
-/* Tru64 UNIX V4.0 neither */
-#if defined(__alpha) && defined(__osf__)
-#include <unistd.h>
-#if _XOPEN_VERSION < 500
-#define NO_vsnprintf
-#endif
-#endif
-
 #endif /* ZCONF_H */
diff --git a/zlib/zconf.h.cmakein b/zlib/zconf.h.cmakein
index a7f24cc..0abe3bc 100644
--- a/zlib/zconf.h.cmakein
+++ b/zlib/zconf.h.cmakein
@@ -1,5 +1,5 @@
 /* zconf.h -- configuration of the zlib compression library
- * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler
+ * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -40,6 +40,9 @@
 #  define crc32                 z_crc32
 #  define crc32_combine         z_crc32_combine
 #  define crc32_combine64       z_crc32_combine64
+#  define crc32_combine_gen     z_crc32_combine_gen
+#  define crc32_combine_gen64   z_crc32_combine_gen64
+#  define crc32_combine_op      z_crc32_combine_op
 #  define crc32_z               z_crc32_z
 #  define deflate               z_deflate
 #  define deflateBound          z_deflateBound
@@ -240,7 +243,11 @@
 #endif
 
 #ifdef Z_SOLO
-   typedef unsigned long z_size_t;
+#  ifdef _WIN64
+     typedef unsigned long long z_size_t;
+#  else
+     typedef unsigned long z_size_t;
+#  endif
 #else
 #  define z_longlong long long
 #  if defined(NO_SIZE_T)
@@ -295,14 +302,6 @@
 #  endif
 #endif
 
-#ifndef Z_ARG /* function prototypes for stdarg */
-#  if defined(STDC) || defined(Z_HAVE_STDARG_H)
-#    define Z_ARG(args)  args
-#  else
-#    define Z_ARG(args)  ()
-#  endif
-#endif
-
 /* The following definitions for FAR are needed only for MSDOS mixed
  * model programming (small or medium model with some far allocations).
  * This was tested only with MSC; for other MSDOS compilers you may have
@@ -351,6 +350,9 @@
 #    ifdef FAR
 #      undef FAR
 #    endif
+#    ifndef WIN32_LEAN_AND_MEAN
+#      define WIN32_LEAN_AND_MEAN
+#    endif
 #    include <windows.h>
      /* No need for _export, use ZLIB.DEF instead. */
      /* For complete Windows compatibility, use WINAPI, not __stdcall. */
@@ -469,11 +471,18 @@ typedef uLong FAR uLongf;
 #  undef _LARGEFILE64_SOURCE
 #endif
 
-#if defined(__WATCOMC__) && !defined(Z_HAVE_UNISTD_H)
-#  define Z_HAVE_UNISTD_H
+#ifndef Z_HAVE_UNISTD_H
+#  ifdef __WATCOMC__
+#    define Z_HAVE_UNISTD_H
+#  endif
+#endif
+#ifndef Z_HAVE_UNISTD_H
+#  if defined(_LARGEFILE64_SOURCE) && !defined(_WIN32)
+#    define Z_HAVE_UNISTD_H
+#  endif
 #endif
 #ifndef Z_SOLO
-#  if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
+#  if defined(Z_HAVE_UNISTD_H)
 #    include <unistd.h>         /* for SEEK_*, off_t, and _LFS64_LARGEFILE */
 #    ifdef VMS
 #      include <unixio.h>       /* for off_t */
@@ -509,7 +518,7 @@ typedef uLong FAR uLongf;
 #if !defined(_WIN32) && defined(Z_LARGE64)
 #  define z_off64_t off64_t
 #else
-#  if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO)
+#  if defined(_WIN32) && !defined(__GNUC__)
 #    define z_off64_t __int64
 #  else
 #    define z_off64_t z_off_t
diff --git a/zlib/zconf.h.in b/zlib/zconf.h.in
index 5e1d68a..62adc8d 100644
--- a/zlib/zconf.h.in
+++ b/zlib/zconf.h.in
@@ -1,5 +1,5 @@
 /* zconf.h -- configuration of the zlib compression library
- * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler
+ * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -38,6 +38,9 @@
 #  define crc32                 z_crc32
 #  define crc32_combine         z_crc32_combine
 #  define crc32_combine64       z_crc32_combine64
+#  define crc32_combine_gen     z_crc32_combine_gen
+#  define crc32_combine_gen64   z_crc32_combine_gen64
+#  define crc32_combine_op      z_crc32_combine_op
 #  define crc32_z               z_crc32_z
 #  define deflate               z_deflate
 #  define deflateBound          z_deflateBound
@@ -238,7 +241,11 @@
 #endif
 
 #ifdef Z_SOLO
-   typedef unsigned long z_size_t;
+#  ifdef _WIN64
+     typedef unsigned long long z_size_t;
+#  else
+     typedef unsigned long z_size_t;
+#  endif
 #else
 #  define z_longlong long long
 #  if defined(NO_SIZE_T)
@@ -293,14 +300,6 @@
 #  endif
 #endif
 
-#ifndef Z_ARG /* function prototypes for stdarg */
-#  if defined(STDC) || defined(Z_HAVE_STDARG_H)
-#    define Z_ARG(args)  args
-#  else
-#    define Z_ARG(args)  ()
-#  endif
-#endif
-
 /* The following definitions for FAR are needed only for MSDOS mixed
  * model programming (small or medium model with some far allocations).
  * This was tested only with MSC; for other MSDOS compilers you may have
@@ -349,6 +348,9 @@
 #    ifdef FAR
 #      undef FAR
 #    endif
+#    ifndef WIN32_LEAN_AND_MEAN
+#      define WIN32_LEAN_AND_MEAN
+#    endif
 #    include <windows.h>
      /* No need for _export, use ZLIB.DEF instead. */
      /* For complete Windows compatibility, use WINAPI, not __stdcall. */
@@ -467,11 +469,18 @@ typedef uLong FAR uLongf;
 #  undef _LARGEFILE64_SOURCE
 #endif
 
-#if defined(__WATCOMC__) && !defined(Z_HAVE_UNISTD_H)
-#  define Z_HAVE_UNISTD_H
+#ifndef Z_HAVE_UNISTD_H
+#  ifdef __WATCOMC__
+#    define Z_HAVE_UNISTD_H
+#  endif
+#endif
+#ifndef Z_HAVE_UNISTD_H
+#  if defined(_LARGEFILE64_SOURCE) && !defined(_WIN32)
+#    define Z_HAVE_UNISTD_H
+#  endif
 #endif
 #ifndef Z_SOLO
-#  if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
+#  if defined(Z_HAVE_UNISTD_H)
 #    include <unistd.h>         /* for SEEK_*, off_t, and _LFS64_LARGEFILE */
 #    ifdef VMS
 #      include <unixio.h>       /* for off_t */
@@ -507,7 +516,7 @@ typedef uLong FAR uLongf;
 #if !defined(_WIN32) && defined(Z_LARGE64)
 #  define z_off64_t off64_t
 #else
-#  if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO)
+#  if defined(_WIN32) && !defined(__GNUC__)
 #    define z_off64_t __int64
 #  else
 #    define z_off64_t z_off_t
diff --git a/zlib/zlib.3 b/zlib/zlib.3
index bda4eb0..c716020 100644
--- a/zlib/zlib.3
+++ b/zlib/zlib.3
@@ -1,4 +1,4 @@
-.TH ZLIB 3 "15 Jan 2017"
+.TH ZLIB 3 "22 Jan 2024"
 .SH NAME
 zlib \- compression/decompression library
 .SH SYNOPSIS
@@ -105,9 +105,9 @@ before asking for help.
 Send questions and/or comments to zlib@gzip.org,
 or (for the Windows DLL version) to Gilles Vollant (info@winimage.com).
 .SH AUTHORS AND LICENSE
-Version 1.2.11
+Version 1.3.1
 .LP
-Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler
+Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler
 .LP
 This software is provided 'as-is', without any express or implied
 warranty.  In no event will the authors be held liable for any damages
diff --git a/zlib/zlib.3.pdf b/zlib/zlib.3.pdf
new file mode 100644
index 0000000..b224532
--- /dev/null
+++ b/zlib/zlib.3.pdf
diff --git a/zlib/zlib.h b/zlib/zlib.h
index 49dfcec..8d4b932 100644
--- a/zlib/zlib.h
+++ b/zlib/zlib.h
@@ -1,7 +1,7 @@
 /* zlib.h -- interface of the 'zlib' general purpose compression library
-  version 1.2.11, January 15th, 2017
+  version 1.3.1, January 22nd, 2024
 
-  Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler
+  Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler
 
   This software is provided 'as-is', without any express or implied
   warranty.  In no event will the authors be held liable for any damages
@@ -37,11 +37,11 @@
 extern "C" {
 #endif
 
-#define ZLIB_VERSION "1.2.11"
-#define ZLIB_VERNUM 0x12b0
+#define ZLIB_VERSION "1.3.1"
+#define ZLIB_VERNUM 0x1310
 #define ZLIB_VER_MAJOR 1
-#define ZLIB_VER_MINOR 2
-#define ZLIB_VER_REVISION 11
+#define ZLIB_VER_MINOR 3
+#define ZLIB_VER_REVISION 1
 #define ZLIB_VER_SUBREVISION 0
 
 /*
@@ -78,8 +78,8 @@ extern "C" {
   even in the case of corrupted input.
 */
 
-typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size));
-typedef void   (*free_func)  OF((voidpf opaque, voidpf address));
+typedef voidpf (*alloc_func)(voidpf opaque, uInt items, uInt size);
+typedef void   (*free_func)(voidpf opaque, voidpf address);
 
 struct internal_state;
 
@@ -217,7 +217,7 @@ typedef gz_header FAR *gz_headerp;
 
                         /* basic functions */
 
-ZEXTERN const char * ZEXPORT zlibVersion OF((void));
+ZEXTERN const char * ZEXPORT zlibVersion(void);
 /* The application can compare zlibVersion and ZLIB_VERSION for consistency.
    If the first character differs, the library code actually used is not
    compatible with the zlib.h header file used by the application.  This check
@@ -225,12 +225,12 @@ ZEXTERN const char * ZEXPORT zlibVersion OF((void));
  */
 
 /*
-ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level));
+ZEXTERN int ZEXPORT deflateInit(z_streamp strm, int level);
 
      Initializes the internal stream state for compression.  The fields
    zalloc, zfree and opaque must be initialized before by the caller.  If
    zalloc and zfree are set to Z_NULL, deflateInit updates them to use default
-   allocation functions.
+   allocation functions.  total_in, total_out, adler, and msg are initialized.
 
      The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9:
    1 gives best speed, 9 gives best compression, 0 gives no compression at all
@@ -247,7 +247,7 @@ ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level));
 */
 
 
-ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush));
+ZEXTERN int ZEXPORT deflate(z_streamp strm, int flush);
 /*
     deflate compresses as much data as possible, and stops when the input
   buffer becomes empty or the output buffer becomes full.  It may introduce
@@ -276,7 +276,7 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush));
   == 0), or after each call of deflate().  If deflate returns Z_OK and with
   zero avail_out, it must be called again after making room in the output
   buffer because there might be more output pending. See deflatePending(),
-  which can be used if desired to determine whether or not there is more ouput
+  which can be used if desired to determine whether or not there is more output
   in that case.
 
     Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to
@@ -320,8 +320,8 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush));
   with the same value of the flush parameter and more output space (updated
   avail_out), until the flush is complete (deflate returns with non-zero
   avail_out).  In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that
-  avail_out is greater than six to avoid repeated flush markers due to
-  avail_out == 0 on return.
+  avail_out is greater than six when the flush marker begins, in order to avoid
+  repeated flush markers upon calling deflate() again when avail_out == 0.
 
     If the parameter flush is set to Z_FINISH, pending input is processed,
   pending output is flushed and deflate returns with Z_STREAM_END if there was
@@ -360,7 +360,7 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush));
 */
 
 
-ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm));
+ZEXTERN int ZEXPORT deflateEnd(z_streamp strm);
 /*
      All dynamically allocated data structures for this stream are freed.
    This function discards any unprocessed input and does not flush any pending
@@ -375,7 +375,7 @@ ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm));
 
 
 /*
-ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm));
+ZEXTERN int ZEXPORT inflateInit(z_streamp strm);
 
      Initializes the internal stream state for decompression.  The fields
    next_in, avail_in, zalloc, zfree and opaque must be initialized before by
@@ -383,7 +383,8 @@ ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm));
    read or consumed.  The allocation of a sliding window will be deferred to
    the first call of inflate (if the decompression does not complete on the
    first call).  If zalloc and zfree are set to Z_NULL, inflateInit updates
-   them to use default allocation functions.
+   them to use default allocation functions.  total_in, total_out, adler, and
+   msg are initialized.
 
      inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough
    memory, Z_VERSION_ERROR if the zlib library version is incompatible with the
@@ -397,7 +398,7 @@ ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm));
 */
 
 
-ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush));
+ZEXTERN int ZEXPORT inflate(z_streamp strm, int flush);
 /*
     inflate decompresses as much data as possible, and stops when the input
   buffer becomes empty or the output buffer becomes full.  It may introduce
@@ -517,7 +518,7 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush));
 */
 
 
-ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm));
+ZEXTERN int ZEXPORT inflateEnd(z_streamp strm);
 /*
      All dynamically allocated data structures for this stream are freed.
    This function discards any unprocessed input and does not flush any pending
@@ -535,16 +536,15 @@ ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm));
 */
 
 /*
-ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm,
-                                     int  level,
-                                     int  method,
-                                     int  windowBits,
-                                     int  memLevel,
-                                     int  strategy));
+ZEXTERN int ZEXPORT deflateInit2(z_streamp strm,
+                                 int level,
+                                 int method,
+                                 int windowBits,
+                                 int memLevel,
+                                 int strategy);
 
      This is another version of deflateInit with more compression options.  The
-   fields next_in, zalloc, zfree and opaque must be initialized before by the
-   caller.
+   fields zalloc, zfree and opaque must be initialized before by the caller.
 
      The method parameter is the compression method.  It must be Z_DEFLATED in
    this version of the library.
@@ -608,9 +608,9 @@ ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm,
    compression: this will be done by deflate().
 */
 
-ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm,
-                                             const Bytef *dictionary,
-                                             uInt  dictLength));
+ZEXTERN int ZEXPORT deflateSetDictionary(z_streamp strm,
+                                         const Bytef *dictionary,
+                                         uInt  dictLength);
 /*
      Initializes the compression dictionary from the given byte sequence
    without producing any compressed output.  When using the zlib format, this
@@ -652,16 +652,16 @@ ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm,
    not perform any compression: this will be done by deflate().
 */
 
-ZEXTERN int ZEXPORT deflateGetDictionary OF((z_streamp strm,
-                                             Bytef *dictionary,
-                                             uInt  *dictLength));
+ZEXTERN int ZEXPORT deflateGetDictionary(z_streamp strm,
+                                         Bytef *dictionary,
+                                         uInt  *dictLength);
 /*
      Returns the sliding dictionary being maintained by deflate.  dictLength is
    set to the number of bytes in the dictionary, and that many bytes are copied
    to dictionary.  dictionary must have enough space, where 32768 bytes is
    always enough.  If deflateGetDictionary() is called with dictionary equal to
    Z_NULL, then only the dictionary length is returned, and nothing is copied.
-   Similary, if dictLength is Z_NULL, then it is not set.
+   Similarly, if dictLength is Z_NULL, then it is not set.
 
      deflateGetDictionary() may return a length less than the window size, even
    when more than the window size in input has been provided. It may return up
@@ -674,8 +674,8 @@ ZEXTERN int ZEXPORT deflateGetDictionary OF((z_streamp strm,
    stream state is inconsistent.
 */
 
-ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest,
-                                    z_streamp source));
+ZEXTERN int ZEXPORT deflateCopy(z_streamp dest,
+                                z_streamp source);
 /*
      Sets the destination stream as a complete copy of the source stream.
 
@@ -692,31 +692,32 @@ ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest,
    destination.
 */
 
-ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm));
+ZEXTERN int ZEXPORT deflateReset(z_streamp strm);
 /*
      This function is equivalent to deflateEnd followed by deflateInit, but
    does not free and reallocate the internal compression state.  The stream
    will leave the compression level and any other attributes that may have been
-   set unchanged.
+   set unchanged.  total_in, total_out, adler, and msg are initialized.
 
      deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
    stream state was inconsistent (such as zalloc or state being Z_NULL).
 */
 
-ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm,
-                                      int level,
-                                      int strategy));
+ZEXTERN int ZEXPORT deflateParams(z_streamp strm,
+                                  int level,
+                                  int strategy);
 /*
      Dynamically update the compression level and compression strategy.  The
    interpretation of level and strategy is as in deflateInit2().  This can be
    used to switch between compression and straight copy of the input data, or
    to switch to a different kind of input data requiring a different strategy.
    If the compression approach (which is a function of the level) or the
-   strategy is changed, and if any input has been consumed in a previous
-   deflate() call, then the input available so far is compressed with the old
-   level and strategy using deflate(strm, Z_BLOCK).  There are three approaches
-   for the compression levels 0, 1..3, and 4..9 respectively.  The new level
-   and strategy will take effect at the next call of deflate().
+   strategy is changed, and if there have been any deflate() calls since the
+   state was initialized or reset, then the input available so far is
+   compressed with the old level and strategy using deflate(strm, Z_BLOCK).
+   There are three approaches for the compression levels 0, 1..3, and 4..9
+   respectively.  The new level and strategy will take effect at the next call
+   of deflate().
 
      If a deflate(strm, Z_BLOCK) is performed by deflateParams(), and it does
    not have enough output space to complete, then the parameter change will not
@@ -729,7 +730,7 @@ ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm,
    Then no more input data should be provided before the deflateParams() call.
    If this is done, the old level and strategy will be applied to the data
    compressed before deflateParams(), and the new level and strategy will be
-   applied to the the data compressed after deflateParams().
+   applied to the data compressed after deflateParams().
 
      deflateParams returns Z_OK on success, Z_STREAM_ERROR if the source stream
    state was inconsistent or if a parameter was invalid, or Z_BUF_ERROR if
@@ -740,11 +741,11 @@ ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm,
    retried with more output space.
 */
 
-ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm,
-                                    int good_length,
-                                    int max_lazy,
-                                    int nice_length,
-                                    int max_chain));
+ZEXTERN int ZEXPORT deflateTune(z_streamp strm,
+                                int good_length,
+                                int max_lazy,
+                                int nice_length,
+                                int max_chain);
 /*
      Fine tune deflate's internal compression parameters.  This should only be
    used by someone who understands the algorithm used by zlib's deflate for
@@ -757,8 +758,8 @@ ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm,
    returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream.
  */
 
-ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm,
-                                       uLong sourceLen));
+ZEXTERN uLong ZEXPORT deflateBound(z_streamp strm,
+                                   uLong sourceLen);
 /*
      deflateBound() returns an upper bound on the compressed size after
    deflation of sourceLen bytes.  It must be called after deflateInit() or
@@ -772,9 +773,9 @@ ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm,
    than Z_FINISH or Z_NO_FLUSH are used.
 */
 
-ZEXTERN int ZEXPORT deflatePending OF((z_streamp strm,
-                                       unsigned *pending,
-                                       int *bits));
+ZEXTERN int ZEXPORT deflatePending(z_streamp strm,
+                                   unsigned *pending,
+                                   int *bits);
 /*
      deflatePending() returns the number of bytes and bits of output that have
    been generated, but not yet provided in the available output.  The bytes not
@@ -787,9 +788,9 @@ ZEXTERN int ZEXPORT deflatePending OF((z_streamp strm,
    stream state was inconsistent.
  */
 
-ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm,
-                                     int bits,
-                                     int value));
+ZEXTERN int ZEXPORT deflatePrime(z_streamp strm,
+                                 int bits,
+                                 int value);
 /*
      deflatePrime() inserts bits in the deflate output stream.  The intent
    is that this function is used to start off the deflate output with the bits
@@ -804,8 +805,8 @@ ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm,
    source stream state was inconsistent.
 */
 
-ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm,
-                                         gz_headerp head));
+ZEXTERN int ZEXPORT deflateSetHeader(z_streamp strm,
+                                     gz_headerp head);
 /*
      deflateSetHeader() provides gzip header information for when a gzip
    stream is requested by deflateInit2().  deflateSetHeader() may be called
@@ -821,16 +822,17 @@ ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm,
    gzip file" and give up.
 
      If deflateSetHeader is not used, the default gzip header has text false,
-   the time set to zero, and os set to 255, with no extra, name, or comment
-   fields.  The gzip header is returned to the default state by deflateReset().
+   the time set to zero, and os set to the current operating system, with no
+   extra, name, or comment fields.  The gzip header is returned to the default
+   state by deflateReset().
 
      deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source
    stream state was inconsistent.
 */
 
 /*
-ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm,
-                                     int  windowBits));
+ZEXTERN int ZEXPORT inflateInit2(z_streamp strm,
+                                 int windowBits);
 
      This is another version of inflateInit with an extra parameter.  The
    fields next_in, avail_in, zalloc, zfree and opaque must be initialized
@@ -865,9 +867,11 @@ ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm,
    detection, or add 16 to decode only the gzip format (the zlib format will
    return a Z_DATA_ERROR).  If a gzip stream is being decoded, strm->adler is a
    CRC-32 instead of an Adler-32.  Unlike the gunzip utility and gzread() (see
-   below), inflate() will not automatically decode concatenated gzip streams.
-   inflate() will return Z_STREAM_END at the end of the gzip stream.  The state
-   would need to be reset to continue decoding a subsequent gzip stream.
+   below), inflate() will *not* automatically decode concatenated gzip members.
+   inflate() will return Z_STREAM_END at the end of the gzip member.  The state
+   would need to be reset to continue decoding a subsequent gzip member.  This
+   *must* be done if there is more data after a gzip member, in order for the
+   decompression to be compliant with the gzip standard (RFC 1952).
 
      inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
    memory, Z_VERSION_ERROR if the zlib library version is incompatible with the
@@ -881,9 +885,9 @@ ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm,
    deferred until inflate() is called.
 */
 
-ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm,
-                                             const Bytef *dictionary,
-                                             uInt  dictLength));
+ZEXTERN int ZEXPORT inflateSetDictionary(z_streamp strm,
+                                         const Bytef *dictionary,
+                                         uInt  dictLength);
 /*
      Initializes the decompression dictionary from the given uncompressed byte
    sequence.  This function must be called immediately after a call of inflate,
@@ -904,22 +908,22 @@ ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm,
    inflate().
 */
 
-ZEXTERN int ZEXPORT inflateGetDictionary OF((z_streamp strm,
-                                             Bytef *dictionary,
-                                             uInt  *dictLength));
+ZEXTERN int ZEXPORT inflateGetDictionary(z_streamp strm,
+                                         Bytef *dictionary,
+                                         uInt  *dictLength);
 /*
      Returns the sliding dictionary being maintained by inflate.  dictLength is
    set to the number of bytes in the dictionary, and that many bytes are copied
    to dictionary.  dictionary must have enough space, where 32768 bytes is
    always enough.  If inflateGetDictionary() is called with dictionary equal to
    Z_NULL, then only the dictionary length is returned, and nothing is copied.
-   Similary, if dictLength is Z_NULL, then it is not set.
+   Similarly, if dictLength is Z_NULL, then it is not set.
 
      inflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the
    stream state is inconsistent.
 */
 
-ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm));
+ZEXTERN int ZEXPORT inflateSync(z_streamp strm);
 /*
      Skips invalid compressed data until a possible full flush point (see above
    for the description of deflate with Z_FULL_FLUSH) can be found, or until all
@@ -932,14 +936,14 @@ ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm));
      inflateSync returns Z_OK if a possible full flush point has been found,
    Z_BUF_ERROR if no more input was provided, Z_DATA_ERROR if no flush point
    has been found, or Z_STREAM_ERROR if the stream structure was inconsistent.
-   In the success case, the application may save the current current value of
-   total_in which indicates where valid compressed data was found.  In the
-   error case, the application may repeatedly call inflateSync, providing more
-   input each time, until success or end of the input data.
+   In the success case, the application may save the current value of total_in
+   which indicates where valid compressed data was found.  In the error case,
+   the application may repeatedly call inflateSync, providing more input each
+   time, until success or end of the input data.
 */
 
-ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest,
-                                    z_streamp source));
+ZEXTERN int ZEXPORT inflateCopy(z_streamp dest,
+                                z_streamp source);
 /*
      Sets the destination stream as a complete copy of the source stream.
 
@@ -954,18 +958,19 @@ ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest,
    destination.
 */
 
-ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm));
+ZEXTERN int ZEXPORT inflateReset(z_streamp strm);
 /*
      This function is equivalent to inflateEnd followed by inflateInit,
    but does not free and reallocate the internal decompression state.  The
    stream will keep attributes that may have been set by inflateInit2.
+   total_in, total_out, adler, and msg are initialized.
 
      inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
    stream state was inconsistent (such as zalloc or state being Z_NULL).
 */
 
-ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm,
-                                      int windowBits));
+ZEXTERN int ZEXPORT inflateReset2(z_streamp strm,
+                                  int windowBits);
 /*
      This function is the same as inflateReset, but it also permits changing
    the wrap and window size requests.  The windowBits parameter is interpreted
@@ -978,9 +983,9 @@ ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm,
    the windowBits parameter is invalid.
 */
 
-ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm,
-                                     int bits,
-                                     int value));
+ZEXTERN int ZEXPORT inflatePrime(z_streamp strm,
+                                 int bits,
+                                 int value);
 /*
      This function inserts bits in the inflate input stream.  The intent is
    that this function is used to start inflating at a bit position in the
@@ -999,7 +1004,7 @@ ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm,
    stream state was inconsistent.
 */
 
-ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm));
+ZEXTERN long ZEXPORT inflateMark(z_streamp strm);
 /*
      This function returns two values, one in the lower 16 bits of the return
    value, and the other in the remaining upper bits, obtained by shifting the
@@ -1027,8 +1032,8 @@ ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm));
    source stream state was inconsistent.
 */
 
-ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm,
-                                         gz_headerp head));
+ZEXTERN int ZEXPORT inflateGetHeader(z_streamp strm,
+                                     gz_headerp head);
 /*
      inflateGetHeader() requests that gzip header information be stored in the
    provided gz_header structure.  inflateGetHeader() may be called after
@@ -1068,8 +1073,8 @@ ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm,
 */
 
 /*
-ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits,
-                                        unsigned char FAR *window));
+ZEXTERN int ZEXPORT inflateBackInit(z_streamp strm, int windowBits,
+                                    unsigned char FAR *window);
 
      Initialize the internal stream state for decompression using inflateBack()
    calls.  The fields zalloc, zfree and opaque in strm must be initialized
@@ -1089,13 +1094,13 @@ ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits,
    the version of the header file.
 */
 
-typedef unsigned (*in_func) OF((void FAR *,
-                                z_const unsigned char FAR * FAR *));
-typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned));
+typedef unsigned (*in_func)(void FAR *,
+                            z_const unsigned char FAR * FAR *);
+typedef int (*out_func)(void FAR *, unsigned char FAR *, unsigned);
 
-ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm,
-                                    in_func in, void FAR *in_desc,
-                                    out_func out, void FAR *out_desc));
+ZEXTERN int ZEXPORT inflateBack(z_streamp strm,
+                                in_func in, void FAR *in_desc,
+                                out_func out, void FAR *out_desc);
 /*
      inflateBack() does a raw inflate with a single call using a call-back
    interface for input and output.  This is potentially more efficient than
@@ -1163,7 +1168,7 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm,
    cannot return Z_OK.
 */
 
-ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm));
+ZEXTERN int ZEXPORT inflateBackEnd(z_streamp strm);
 /*
      All memory allocated by inflateBackInit() is freed.
 
@@ -1171,7 +1176,7 @@ ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm));
    state was inconsistent.
 */
 
-ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void));
+ZEXTERN uLong ZEXPORT zlibCompileFlags(void);
 /* Return flags indicating compile-time options.
 
     Type sizes, two bits each, 00 = 16 bits, 01 = 32, 10 = 64, 11 = other:
@@ -1224,8 +1229,8 @@ ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void));
    you need special options.
 */
 
-ZEXTERN int ZEXPORT compress OF((Bytef *dest,   uLongf *destLen,
-                                 const Bytef *source, uLong sourceLen));
+ZEXTERN int ZEXPORT compress(Bytef *dest,   uLongf *destLen,
+                             const Bytef *source, uLong sourceLen);
 /*
      Compresses the source buffer into the destination buffer.  sourceLen is
    the byte length of the source buffer.  Upon entry, destLen is the total size
@@ -1239,9 +1244,9 @@ ZEXTERN int ZEXPORT compress OF((Bytef *dest,   uLongf *destLen,
    buffer.
 */
 
-ZEXTERN int ZEXPORT compress2 OF((Bytef *dest,   uLongf *destLen,
-                                  const Bytef *source, uLong sourceLen,
-                                  int level));
+ZEXTERN int ZEXPORT compress2(Bytef *dest,   uLongf *destLen,
+                              const Bytef *source, uLong sourceLen,
+                              int level);
 /*
      Compresses the source buffer into the destination buffer.  The level
    parameter has the same meaning as in deflateInit.  sourceLen is the byte
@@ -1255,15 +1260,15 @@ ZEXTERN int ZEXPORT compress2 OF((Bytef *dest,   uLongf *destLen,
    Z_STREAM_ERROR if the level parameter is invalid.
 */
 
-ZEXTERN uLong ZEXPORT compressBound OF((uLong sourceLen));
+ZEXTERN uLong ZEXPORT compressBound(uLong sourceLen);
 /*
      compressBound() returns an upper bound on the compressed size after
    compress() or compress2() on sourceLen bytes.  It would be used before a
    compress() or compress2() call to allocate the destination buffer.
 */
 
-ZEXTERN int ZEXPORT uncompress OF((Bytef *dest,   uLongf *destLen,
-                                   const Bytef *source, uLong sourceLen));
+ZEXTERN int ZEXPORT uncompress(Bytef *dest,   uLongf *destLen,
+                               const Bytef *source, uLong sourceLen);
 /*
      Decompresses the source buffer into the destination buffer.  sourceLen is
    the byte length of the source buffer.  Upon entry, destLen is the total size
@@ -1280,8 +1285,8 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest,   uLongf *destLen,
    buffer with the uncompressed data up to that point.
 */
 
-ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest,   uLongf *destLen,
-                                    const Bytef *source, uLong *sourceLen));
+ZEXTERN int ZEXPORT uncompress2(Bytef *dest,   uLongf *destLen,
+                                const Bytef *source, uLong *sourceLen);
 /*
      Same as uncompress, except that sourceLen is a pointer, where the
    length of the source is *sourceLen.  On return, *sourceLen is the number of
@@ -1300,16 +1305,16 @@ ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest,   uLongf *destLen,
 typedef struct gzFile_s *gzFile;    /* semi-opaque gzip file descriptor */
 
 /*
-ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode));
+ZEXTERN gzFile ZEXPORT gzopen(const char *path, const char *mode);
 
-     Opens a gzip (.gz) file for reading or writing.  The mode parameter is as
-   in fopen ("rb" or "wb") but can also include a compression level ("wb9") or
-   a strategy: 'f' for filtered data as in "wb6f", 'h' for Huffman-only
-   compression as in "wb1h", 'R' for run-length encoding as in "wb1R", or 'F'
-   for fixed code compression as in "wb9F".  (See the description of
-   deflateInit2 for more information about the strategy parameter.)  'T' will
-   request transparent writing or appending with no compression and not using
-   the gzip format.
+     Open the gzip (.gz) file at path for reading and decompressing, or
+   compressing and writing.  The mode parameter is as in fopen ("rb" or "wb")
+   but can also include a compression level ("wb9") or a strategy: 'f' for
+   filtered data as in "wb6f", 'h' for Huffman-only compression as in "wb1h",
+   'R' for run-length encoding as in "wb1R", or 'F' for fixed code compression
+   as in "wb9F".  (See the description of deflateInit2 for more information
+   about the strategy parameter.)  'T' will request transparent writing or
+   appending with no compression and not using the gzip format.
 
      "a" can be used instead of "w" to request that the gzip stream that will
    be written be appended to the file.  "+" will result in an error, since
@@ -1337,11 +1342,11 @@ ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode));
    file could not be opened.
 */
 
-ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode));
+ZEXTERN gzFile ZEXPORT gzdopen(int fd, const char *mode);
 /*
-     gzdopen associates a gzFile with the file descriptor fd.  File descriptors
-   are obtained from calls like open, dup, creat, pipe or fileno (if the file
-   has been previously opened with fopen).  The mode parameter is as in gzopen.
+     Associate a gzFile with the file descriptor fd.  File descriptors are
+   obtained from calls like open, dup, creat, pipe or fileno (if the file has
+   been previously opened with fopen).  The mode parameter is as in gzopen.
 
      The next call of gzclose on the returned gzFile will also close the file
    descriptor fd, just like fclose(fdopen(fd, mode)) closes the file descriptor
@@ -1360,15 +1365,15 @@ ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode));
    will not detect if fd is invalid (unless fd is -1).
 */
 
-ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size));
+ZEXTERN int ZEXPORT gzbuffer(gzFile file, unsigned size);
 /*
-     Set the internal buffer size used by this library's functions.  The
-   default buffer size is 8192 bytes.  This function must be called after
-   gzopen() or gzdopen(), and before any other calls that read or write the
-   file.  The buffer memory allocation is always deferred to the first read or
-   write.  Three times that size in buffer space is allocated.  A larger buffer
-   size of, for example, 64K or 128K bytes will noticeably increase the speed
-   of decompression (reading).
+     Set the internal buffer size used by this library's functions for file to
+   size.  The default buffer size is 8192 bytes.  This function must be called
+   after gzopen() or gzdopen(), and before any other calls that read or write
+   the file.  The buffer memory allocation is always deferred to the first read
+   or write.  Three times that size in buffer space is allocated.  A larger
+   buffer size of, for example, 64K or 128K bytes will noticeably increase the
+   speed of decompression (reading).
 
      The new buffer size also affects the maximum length for gzprintf().
 
@@ -1376,20 +1381,20 @@ ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size));
    too late.
 */
 
-ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy));
+ZEXTERN int ZEXPORT gzsetparams(gzFile file, int level, int strategy);
 /*
-     Dynamically update the compression level or strategy.  See the description
-   of deflateInit2 for the meaning of these parameters.  Previously provided
-   data is flushed before the parameter change.
+     Dynamically update the compression level and strategy for file.  See the
+   description of deflateInit2 for the meaning of these parameters. Previously
+   provided data is flushed before applying the parameter changes.
 
      gzsetparams returns Z_OK if success, Z_STREAM_ERROR if the file was not
    opened for writing, Z_ERRNO if there is an error writing the flushed data,
    or Z_MEM_ERROR if there is a memory allocation error.
 */
 
-ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len));
+ZEXTERN int ZEXPORT gzread(gzFile file, voidp buf, unsigned len);
 /*
-     Reads the given number of uncompressed bytes from the compressed file.  If
+     Read and decompress up to len uncompressed bytes from file into buf.  If
    the input file is not in gzip format, gzread copies the given number of
    bytes into the buffer directly from the file.
 
@@ -1417,14 +1422,14 @@ ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len));
    Z_STREAM_ERROR.
 */
 
-ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems,
-                                     gzFile file));
+ZEXTERN z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems,
+                                 gzFile file);
 /*
-     Read up to nitems items of size size from file to buf, otherwise operating
-   as gzread() does.  This duplicates the interface of stdio's fread(), with
-   size_t request and return types.  If the library defines size_t, then
-   z_size_t is identical to size_t.  If not, then z_size_t is an unsigned
-   integer type that can contain a pointer.
+     Read and decompress up to nitems items of size size from file into buf,
+   otherwise operating as gzread() does.  This duplicates the interface of
+   stdio's fread(), with size_t request and return types.  If the library
+   defines size_t, then z_size_t is identical to size_t.  If not, then z_size_t
+   is an unsigned integer type that can contain a pointer.
 
      gzfread() returns the number of full items read of size size, or zero if
    the end of the file was reached and a full item could not be read, or if
@@ -1435,26 +1440,24 @@ ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems,
 
      In the event that the end of file is reached and only a partial item is
    available at the end, i.e. the remaining uncompressed data length is not a
-   multiple of size, then the final partial item is nevetheless read into buf
+   multiple of size, then the final partial item is nevertheless read into buf
    and the end-of-file flag is set.  The length of the partial item read is not
    provided, but could be inferred from the result of gztell().  This behavior
    is the same as the behavior of fread() implementations in common libraries,
    but it prevents the direct use of gzfread() to read a concurrently written
-   file, reseting and retrying on end-of-file, when size is not 1.
+   file, resetting and retrying on end-of-file, when size is not 1.
 */
 
-ZEXTERN int ZEXPORT gzwrite OF((gzFile file,
-                                voidpc buf, unsigned len));
+ZEXTERN int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len);
 /*
-     Writes the given number of uncompressed bytes into the compressed file.
-   gzwrite returns the number of uncompressed bytes written or 0 in case of
-   error.
+     Compress and write the len uncompressed bytes at buf to file. gzwrite
+   returns the number of uncompressed bytes written or 0 in case of error.
 */
 
-ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size,
-                                      z_size_t nitems, gzFile file));
+ZEXTERN z_size_t ZEXPORT gzfwrite(voidpc buf, z_size_t size,
+                                  z_size_t nitems, gzFile file);
 /*
-     gzfwrite() writes nitems items of size size from buf to file, duplicating
+     Compress and write nitems items of size size from buf to file, duplicating
    the interface of stdio's fwrite(), with size_t request and return types.  If
    the library defines size_t, then z_size_t is identical to size_t.  If not,
    then z_size_t is an unsigned integer type that can contain a pointer.
@@ -1465,61 +1468,62 @@ ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size,
    is returned, and the error state is set to Z_STREAM_ERROR.
 */
 
-ZEXTERN int ZEXPORTVA gzprintf Z_ARG((gzFile file, const char *format, ...));
+ZEXTERN int ZEXPORTVA gzprintf(gzFile file, const char *format, ...);
 /*
-     Converts, formats, and writes the arguments to the compressed file under
-   control of the format string, as in fprintf.  gzprintf returns the number of
+     Convert, format, compress, and write the arguments (...) to file under
+   control of the string format, as in fprintf.  gzprintf returns the number of
    uncompressed bytes actually written, or a negative zlib error code in case
    of error.  The number of uncompressed bytes written is limited to 8191, or
    one less than the buffer size given to gzbuffer().  The caller should assure
    that this limit is not exceeded.  If it is exceeded, then gzprintf() will
    return an error (0) with nothing written.  In this case, there may also be a
    buffer overflow with unpredictable consequences, which is possible only if
-   zlib was compiled with the insecure functions sprintf() or vsprintf()
+   zlib was compiled with the insecure functions sprintf() or vsprintf(),
    because the secure snprintf() or vsnprintf() functions were not available.
    This can be determined using zlibCompileFlags().
 */
 
-ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s));
+ZEXTERN int ZEXPORT gzputs(gzFile file, const char *s);
 /*
-     Writes the given null-terminated string to the compressed file, excluding
+     Compress and write the given null-terminated string s to file, excluding
    the terminating null character.
 
      gzputs returns the number of characters written, or -1 in case of error.
 */
 
-ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len));
+ZEXTERN char * ZEXPORT gzgets(gzFile file, char *buf, int len);
 /*
-     Reads bytes from the compressed file until len-1 characters are read, or a
-   newline character is read and transferred to buf, or an end-of-file
-   condition is encountered.  If any characters are read or if len == 1, the
-   string is terminated with a null character.  If no characters are read due
-   to an end-of-file or len < 1, then the buffer is left untouched.
+     Read and decompress bytes from file into buf, until len-1 characters are
+   read, or until a newline character is read and transferred to buf, or an
+   end-of-file condition is encountered.  If any characters are read or if len
+   is one, the string is terminated with a null character.  If no characters
+   are read due to an end-of-file or len is less than one, then the buffer is
+   left untouched.
 
      gzgets returns buf which is a null-terminated string, or it returns NULL
    for end-of-file or in case of error.  If there was an error, the contents at
    buf are indeterminate.
 */
 
-ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c));
+ZEXTERN int ZEXPORT gzputc(gzFile file, int c);
 /*
-     Writes c, converted to an unsigned char, into the compressed file.  gzputc
+     Compress and write c, converted to an unsigned char, into file.  gzputc
    returns the value that was written, or -1 in case of error.
 */
 
-ZEXTERN int ZEXPORT gzgetc OF((gzFile file));
+ZEXTERN int ZEXPORT gzgetc(gzFile file);
 /*
-     Reads one byte from the compressed file.  gzgetc returns this byte or -1
+     Read and decompress one byte from file.  gzgetc returns this byte or -1
    in case of end of file or error.  This is implemented as a macro for speed.
    As such, it does not do all of the checking the other functions do.  I.e.
    it does not check to see if file is NULL, nor whether the structure file
    points to has been clobbered or not.
 */
 
-ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file));
+ZEXTERN int ZEXPORT gzungetc(int c, gzFile file);
 /*
-     Push one character back onto the stream to be read as the first character
-   on the next read.  At least one character of push-back is allowed.
+     Push c back onto the stream for file to be read as the first character on
+   the next read.  At least one character of push-back is always allowed.
    gzungetc() returns the character pushed, or -1 on failure.  gzungetc() will
    fail if c is -1, and may fail if a character has been pushed but not read
    yet.  If gzungetc is used immediately after gzopen or gzdopen, at least the
@@ -1528,11 +1532,11 @@ ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file));
    gzseek() or gzrewind().
 */
 
-ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush));
+ZEXTERN int ZEXPORT gzflush(gzFile file, int flush);
 /*
-     Flushes all pending output into the compressed file.  The parameter flush
-   is as in the deflate() function.  The return value is the zlib error number
-   (see function gzerror below).  gzflush is only permitted when writing.
+     Flush all pending output to file.  The parameter flush is as in the
+   deflate() function.  The return value is the zlib error number (see function
+   gzerror below).  gzflush is only permitted when writing.
 
      If the flush parameter is Z_FINISH, the remaining data is written and the
    gzip stream is completed in the output.  If gzwrite() is called again, a new
@@ -1544,11 +1548,11 @@ ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush));
 */
 
 /*
-ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file,
-                                   z_off_t offset, int whence));
+ZEXTERN z_off_t ZEXPORT gzseek(gzFile file,
+                               z_off_t offset, int whence);
 
-     Sets the starting position for the next gzread or gzwrite on the given
-   compressed file.  The offset represents a number of bytes in the
+     Set the starting position to offset relative to whence for the next gzread
+   or gzwrite on file.  The offset represents a number of bytes in the
    uncompressed data stream.  The whence parameter is defined as in lseek(2);
    the value SEEK_END is not supported.
 
@@ -1563,52 +1567,52 @@ ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file,
    would be before the current position.
 */
 
-ZEXTERN int ZEXPORT    gzrewind OF((gzFile file));
+ZEXTERN int ZEXPORT    gzrewind(gzFile file);
 /*
-     Rewinds the given file. This function is supported only for reading.
+     Rewind file. This function is supported only for reading.
 
-     gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET)
+     gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET).
 */
 
 /*
-ZEXTERN z_off_t ZEXPORT    gztell OF((gzFile file));
+ZEXTERN z_off_t ZEXPORT    gztell(gzFile file);
 
-     Returns the starting position for the next gzread or gzwrite on the given
-   compressed file.  This position represents a number of bytes in the
-   uncompressed data stream, and is zero when starting, even if appending or
-   reading a gzip stream from the middle of a file using gzdopen().
+     Return the starting position for the next gzread or gzwrite on file.
+   This position represents a number of bytes in the uncompressed data stream,
+   and is zero when starting, even if appending or reading a gzip stream from
+   the middle of a file using gzdopen().
 
      gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR)
 */
 
 /*
-ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile file));
+ZEXTERN z_off_t ZEXPORT gzoffset(gzFile file);
 
-     Returns the current offset in the file being read or written.  This offset
-   includes the count of bytes that precede the gzip stream, for example when
-   appending or when using gzdopen() for reading.  When reading, the offset
-   does not include as yet unused buffered input.  This information can be used
-   for a progress indicator.  On error, gzoffset() returns -1.
+     Return the current compressed (actual) read or write offset of file.  This
+   offset includes the count of bytes that precede the gzip stream, for example
+   when appending or when using gzdopen() for reading.  When reading, the
+   offset does not include as yet unused buffered input.  This information can
+   be used for a progress indicator.  On error, gzoffset() returns -1.
 */
 
-ZEXTERN int ZEXPORT gzeof OF((gzFile file));
+ZEXTERN int ZEXPORT gzeof(gzFile file);
 /*
-     Returns true (1) if the end-of-file indicator has been set while reading,
-   false (0) otherwise.  Note that the end-of-file indicator is set only if the
-   read tried to go past the end of the input, but came up short.  Therefore,
-   just like feof(), gzeof() may return false even if there is no more data to
-   read, in the event that the last read request was for the exact number of
-   bytes remaining in the input file.  This will happen if the input file size
-   is an exact multiple of the buffer size.
+     Return true (1) if the end-of-file indicator for file has been set while
+   reading, false (0) otherwise.  Note that the end-of-file indicator is set
+   only if the read tried to go past the end of the input, but came up short.
+   Therefore, just like feof(), gzeof() may return false even if there is no
+   more data to read, in the event that the last read request was for the exact
+   number of bytes remaining in the input file.  This will happen if the input
+   file size is an exact multiple of the buffer size.
 
      If gzeof() returns true, then the read functions will return no more data,
    unless the end-of-file indicator is reset by gzclearerr() and the input file
    has grown since the previous end of file was detected.
 */
 
-ZEXTERN int ZEXPORT gzdirect OF((gzFile file));
+ZEXTERN int ZEXPORT gzdirect(gzFile file);
 /*
-     Returns true (1) if file is being copied directly while reading, or false
+     Return true (1) if file is being copied directly while reading, or false
    (0) if file is a gzip stream being decompressed.
 
      If the input file is empty, gzdirect() will return true, since the input
@@ -1627,10 +1631,10 @@ ZEXTERN int ZEXPORT gzdirect OF((gzFile file));
    gzip file reading and decompression, which may not be desired.)
 */
 
-ZEXTERN int ZEXPORT    gzclose OF((gzFile file));
+ZEXTERN int ZEXPORT    gzclose(gzFile file);
 /*
-     Flushes all pending output if necessary, closes the compressed file and
-   deallocates the (de)compression state.  Note that once file is closed, you
+     Flush all pending output for file, if necessary, close file and
+   deallocate the (de)compression state.  Note that once file is closed, you
    cannot call gzerror with file, since its structures have been deallocated.
    gzclose must not be called more than once on the same file, just as free
    must not be called more than once on the same allocation.
@@ -1640,8 +1644,8 @@ ZEXTERN int ZEXPORT    gzclose OF((gzFile file));
    last read ended in the middle of a gzip stream, or Z_OK on success.
 */
 
-ZEXTERN int ZEXPORT gzclose_r OF((gzFile file));
-ZEXTERN int ZEXPORT gzclose_w OF((gzFile file));
+ZEXTERN int ZEXPORT gzclose_r(gzFile file);
+ZEXTERN int ZEXPORT gzclose_w(gzFile file);
 /*
      Same as gzclose(), but gzclose_r() is only for use when reading, and
    gzclose_w() is only for use when writing or appending.  The advantage to
@@ -1652,12 +1656,12 @@ ZEXTERN int ZEXPORT gzclose_w OF((gzFile file));
    zlib library.
 */
 
-ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum));
+ZEXTERN const char * ZEXPORT gzerror(gzFile file, int *errnum);
 /*
-     Returns the error message for the last error which occurred on the given
-   compressed file.  errnum is set to zlib error number.  If an error occurred
-   in the file system and not in the compression library, errnum is set to
-   Z_ERRNO and the application may consult errno to get the exact error code.
+     Return the error message for the last error which occurred on file.
+   errnum is set to zlib error number.  If an error occurred in the file system
+   and not in the compression library, errnum is set to Z_ERRNO and the
+   application may consult errno to get the exact error code.
 
      The application must not modify the returned string.  Future calls to
    this function may invalidate the previously returned string.  If file is
@@ -1668,9 +1672,9 @@ ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum));
    functions above that do not distinguish those cases in their return values.
 */
 
-ZEXTERN void ZEXPORT gzclearerr OF((gzFile file));
+ZEXTERN void ZEXPORT gzclearerr(gzFile file);
 /*
-     Clears the error and end-of-file flags for file.  This is analogous to the
+     Clear the error and end-of-file flags for file.  This is analogous to the
    clearerr() function in stdio.  This is useful for continuing to read a gzip
    file that is being written concurrently.
 */
@@ -1685,11 +1689,12 @@ ZEXTERN void ZEXPORT gzclearerr OF((gzFile file));
    library.
 */
 
-ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len));
+ZEXTERN uLong ZEXPORT adler32(uLong adler, const Bytef *buf, uInt len);
 /*
      Update a running Adler-32 checksum with the bytes buf[0..len-1] and
-   return the updated checksum.  If buf is Z_NULL, this function returns the
-   required initial value for the checksum.
+   return the updated checksum. An Adler-32 value is in the range of a 32-bit
+   unsigned integer. If buf is Z_NULL, this function returns the required
+   initial value for the checksum.
 
      An Adler-32 checksum is almost as reliable as a CRC-32 but can be computed
    much faster.
@@ -1704,15 +1709,15 @@ ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len));
      if (adler != original_adler) error();
 */
 
-ZEXTERN uLong ZEXPORT adler32_z OF((uLong adler, const Bytef *buf,
-                                    z_size_t len));
+ZEXTERN uLong ZEXPORT adler32_z(uLong adler, const Bytef *buf,
+                                z_size_t len);
 /*
      Same as adler32(), but with a size_t length.
 */
 
 /*
-ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2,
-                                          z_off_t len2));
+ZEXTERN uLong ZEXPORT adler32_combine(uLong adler1, uLong adler2,
+                                      z_off_t len2);
 
      Combine two Adler-32 checksums into one.  For two sequences of bytes, seq1
    and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for
@@ -1722,12 +1727,13 @@ ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2,
    negative, the result has no meaning or utility.
 */
 
-ZEXTERN uLong ZEXPORT crc32   OF((uLong crc, const Bytef *buf, uInt len));
+ZEXTERN uLong ZEXPORT crc32(uLong crc, const Bytef *buf, uInt len);
 /*
      Update a running CRC-32 with the bytes buf[0..len-1] and return the
-   updated CRC-32.  If buf is Z_NULL, this function returns the required
-   initial value for the crc.  Pre- and post-conditioning (one's complement) is
-   performed within this function so it shouldn't be done by the application.
+   updated CRC-32. A CRC-32 value is in the range of a 32-bit unsigned integer.
+   If buf is Z_NULL, this function returns the required initial value for the
+   crc. Pre- and post-conditioning (one's complement) is performed within this
+   function so it shouldn't be done by the application.
 
    Usage example:
 
@@ -1739,20 +1745,34 @@ ZEXTERN uLong ZEXPORT crc32   OF((uLong crc, const Bytef *buf, uInt len));
      if (crc != original_crc) error();
 */
 
-ZEXTERN uLong ZEXPORT crc32_z OF((uLong adler, const Bytef *buf,
-                                  z_size_t len));
+ZEXTERN uLong ZEXPORT crc32_z(uLong crc, const Bytef *buf,
+                              z_size_t len);
 /*
      Same as crc32(), but with a size_t length.
 */
 
 /*
-ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2));
+ZEXTERN uLong ZEXPORT crc32_combine(uLong crc1, uLong crc2, z_off_t len2);
 
      Combine two CRC-32 check values into one.  For two sequences of bytes,
    seq1 and seq2 with lengths len1 and len2, CRC-32 check values were
    calculated for each, crc1 and crc2.  crc32_combine() returns the CRC-32
    check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and
-   len2.
+   len2. len2 must be non-negative.
+*/
+
+/*
+ZEXTERN uLong ZEXPORT crc32_combine_gen(z_off_t len2);
+
+     Return the operator corresponding to length len2, to be used with
+   crc32_combine_op(). len2 must be non-negative.
+*/
+
+ZEXTERN uLong ZEXPORT crc32_combine_op(uLong crc1, uLong crc2, uLong op);
+/*
+     Give the same result as crc32_combine(), using op in place of len2. op is
+   is generated from len2 by crc32_combine_gen(). This will be faster than
+   crc32_combine() if the generated op is used more than once.
 */
 
 
@@ -1761,20 +1781,20 @@ ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2));
 /* deflateInit and inflateInit are macros to allow checking the zlib version
  * and the compiler's view of z_stream:
  */
-ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level,
-                                     const char *version, int stream_size));
-ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm,
-                                     const char *version, int stream_size));
-ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int  level, int  method,
-                                      int windowBits, int memLevel,
-                                      int strategy, const char *version,
-                                      int stream_size));
-ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int  windowBits,
-                                      const char *version, int stream_size));
-ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits,
-                                         unsigned char FAR *window,
-                                         const char *version,
-                                         int stream_size));
+ZEXTERN int ZEXPORT deflateInit_(z_streamp strm, int level,
+                                 const char *version, int stream_size);
+ZEXTERN int ZEXPORT inflateInit_(z_streamp strm,
+                                 const char *version, int stream_size);
+ZEXTERN int ZEXPORT deflateInit2_(z_streamp strm, int  level, int  method,
+                                  int windowBits, int memLevel,
+                                  int strategy, const char *version,
+                                  int stream_size);
+ZEXTERN int ZEXPORT inflateInit2_(z_streamp strm, int  windowBits,
+                                  const char *version, int stream_size);
+ZEXTERN int ZEXPORT inflateBackInit_(z_streamp strm, int windowBits,
+                                     unsigned char FAR *window,
+                                     const char *version,
+                                     int stream_size);
 #ifdef Z_PREFIX_SET
 #  define z_deflateInit(strm, level) \
           deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream))
@@ -1819,7 +1839,7 @@ struct gzFile_s {
     unsigned char *next;
     z_off64_t pos;
 };
-ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file));  /* backward compatibility */
+ZEXTERN int ZEXPORT gzgetc_(gzFile file);       /* backward compatibility */
 #ifdef Z_PREFIX_SET
 #  undef z_gzgetc
 #  define z_gzgetc(g) \
@@ -1836,12 +1856,13 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file));  /* backward compatibility */
  * without large file support, _LFS64_LARGEFILE must also be true
  */
 #ifdef Z_LARGE64
-   ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *));
-   ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int));
-   ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile));
-   ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile));
-   ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off64_t));
-   ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t));
+   ZEXTERN gzFile ZEXPORT gzopen64(const char *, const char *);
+   ZEXTERN z_off64_t ZEXPORT gzseek64(gzFile, z_off64_t, int);
+   ZEXTERN z_off64_t ZEXPORT gztell64(gzFile);
+   ZEXTERN z_off64_t ZEXPORT gzoffset64(gzFile);
+   ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off64_t);
+   ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off64_t);
+   ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off64_t);
 #endif
 
 #if !defined(ZLIB_INTERNAL) && defined(Z_WANT64)
@@ -1852,6 +1873,7 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file));  /* backward compatibility */
 #    define z_gzoffset z_gzoffset64
 #    define z_adler32_combine z_adler32_combine64
 #    define z_crc32_combine z_crc32_combine64
+#    define z_crc32_combine_gen z_crc32_combine_gen64
 #  else
 #    define gzopen gzopen64
 #    define gzseek gzseek64
@@ -1859,49 +1881,53 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file));  /* backward compatibility */
 #    define gzoffset gzoffset64
 #    define adler32_combine adler32_combine64
 #    define crc32_combine crc32_combine64
+#    define crc32_combine_gen crc32_combine_gen64
 #  endif
 #  ifndef Z_LARGE64
-     ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *));
-     ZEXTERN z_off_t ZEXPORT gzseek64 OF((gzFile, z_off_t, int));
-     ZEXTERN z_off_t ZEXPORT gztell64 OF((gzFile));
-     ZEXTERN z_off_t ZEXPORT gzoffset64 OF((gzFile));
-     ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t));
-     ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t));
+     ZEXTERN gzFile ZEXPORT gzopen64(const char *, const char *);
+     ZEXTERN z_off_t ZEXPORT gzseek64(gzFile, z_off_t, int);
+     ZEXTERN z_off_t ZEXPORT gztell64(gzFile);
+     ZEXTERN z_off_t ZEXPORT gzoffset64(gzFile);
+     ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off_t);
+     ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off_t);
+     ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off_t);
 #  endif
 #else
-   ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *));
-   ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile, z_off_t, int));
-   ZEXTERN z_off_t ZEXPORT gztell OF((gzFile));
-   ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile));
-   ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t));
-   ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t));
+   ZEXTERN gzFile ZEXPORT gzopen(const char *, const char *);
+   ZEXTERN z_off_t ZEXPORT gzseek(gzFile, z_off_t, int);
+   ZEXTERN z_off_t ZEXPORT gztell(gzFile);
+   ZEXTERN z_off_t ZEXPORT gzoffset(gzFile);
+   ZEXTERN uLong ZEXPORT adler32_combine(uLong, uLong, z_off_t);
+   ZEXTERN uLong ZEXPORT crc32_combine(uLong, uLong, z_off_t);
+   ZEXTERN uLong ZEXPORT crc32_combine_gen(z_off_t);
 #endif
 
 #else /* Z_SOLO */
 
-   ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t));
-   ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t));
+   ZEXTERN uLong ZEXPORT adler32_combine(uLong, uLong, z_off_t);
+   ZEXTERN uLong ZEXPORT crc32_combine(uLong, uLong, z_off_t);
+   ZEXTERN uLong ZEXPORT crc32_combine_gen(z_off_t);
 
 #endif /* !Z_SOLO */
 
 /* undocumented functions */
-ZEXTERN const char   * ZEXPORT zError           OF((int));
-ZEXTERN int            ZEXPORT inflateSyncPoint OF((z_streamp));
-ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table    OF((void));
-ZEXTERN int            ZEXPORT inflateUndermine OF((z_streamp, int));
-ZEXTERN int            ZEXPORT inflateValidate OF((z_streamp, int));
-ZEXTERN unsigned long  ZEXPORT inflateCodesUsed OF ((z_streamp));
-ZEXTERN int            ZEXPORT inflateResetKeep OF((z_streamp));
-ZEXTERN int            ZEXPORT deflateResetKeep OF((z_streamp));
+ZEXTERN const char   * ZEXPORT zError(int);
+ZEXTERN int            ZEXPORT inflateSyncPoint(z_streamp);
+ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table(void);
+ZEXTERN int            ZEXPORT inflateUndermine(z_streamp, int);
+ZEXTERN int            ZEXPORT inflateValidate(z_streamp, int);
+ZEXTERN unsigned long  ZEXPORT inflateCodesUsed(z_streamp);
+ZEXTERN int            ZEXPORT inflateResetKeep(z_streamp);
+ZEXTERN int            ZEXPORT deflateResetKeep(z_streamp);
 #if defined(_WIN32) && !defined(Z_SOLO)
-ZEXTERN gzFile         ZEXPORT gzopen_w OF((const wchar_t *path,
-                                            const char *mode));
+ZEXTERN gzFile         ZEXPORT gzopen_w(const wchar_t *path,
+                                        const char *mode);
 #endif
 #if defined(STDC) || defined(Z_HAVE_STDARG_H)
 #  ifndef Z_SOLO
-ZEXTERN int            ZEXPORTVA gzvprintf Z_ARG((gzFile file,
-                                                  const char *format,
-                                                  va_list va));
+ZEXTERN int            ZEXPORTVA gzvprintf(gzFile file,
+                                           const char *format,
+                                           va_list va);
 #  endif
 #endif
 
diff --git a/zlib/zlib.map b/zlib/zlib.map
index 40fa9db..31544f2 100644
--- a/zlib/zlib.map
+++ b/zlib/zlib.map
@@ -1,94 +1,100 @@
-ZLIB_1.2.0 {

-  global:

-    compressBound;

-    deflateBound;

-    inflateBack;

-    inflateBackEnd;

-    inflateBackInit_;

-    inflateCopy;

-  local:

-    deflate_copyright;

-    inflate_copyright;

-    inflate_fast;

-    inflate_table;

-    zcalloc;

-    zcfree;

-    z_errmsg;

-    gz_error;

-    gz_intmax;

-    _*;

-};

-

-ZLIB_1.2.0.2 {

-    gzclearerr;

-    gzungetc;

-    zlibCompileFlags;

-} ZLIB_1.2.0;

-

-ZLIB_1.2.0.8 {

-    deflatePrime;

-} ZLIB_1.2.0.2;

-

-ZLIB_1.2.2 {

-    adler32_combine;

-    crc32_combine;

-    deflateSetHeader;

-    inflateGetHeader;

-} ZLIB_1.2.0.8;

-

-ZLIB_1.2.2.3 {

-    deflateTune;

-    gzdirect;

-} ZLIB_1.2.2;

-

-ZLIB_1.2.2.4 {

-    inflatePrime;

-} ZLIB_1.2.2.3;

-

-ZLIB_1.2.3.3 {

-    adler32_combine64;

-    crc32_combine64;

-    gzopen64;

-    gzseek64;

-    gztell64;

-    inflateUndermine;

-} ZLIB_1.2.2.4;

-

-ZLIB_1.2.3.4 {

-    inflateReset2;

-    inflateMark;

-} ZLIB_1.2.3.3;

-

-ZLIB_1.2.3.5 {

-    gzbuffer;

-    gzoffset;

-    gzoffset64;

-    gzclose_r;

-    gzclose_w;

-} ZLIB_1.2.3.4;

-

-ZLIB_1.2.5.1 {

-    deflatePending;

-} ZLIB_1.2.3.5;

-

-ZLIB_1.2.5.2 {

-    deflateResetKeep;

-    gzgetc_;

-    inflateResetKeep;

-} ZLIB_1.2.5.1;

-

-ZLIB_1.2.7.1 {

-    inflateGetDictionary;

-    gzvprintf;

-} ZLIB_1.2.5.2;

-

-ZLIB_1.2.9 {

-    inflateCodesUsed;

-    inflateValidate;

-    uncompress2;

-    gzfread;

-    gzfwrite;

-    deflateGetDictionary;

-    adler32_z;

-    crc32_z;

-} ZLIB_1.2.7.1;

+ZLIB_1.2.0 {
+  global:
+    compressBound;
+    deflateBound;
+    inflateBack;
+    inflateBackEnd;
+    inflateBackInit_;
+    inflateCopy;
+  local:
+    deflate_copyright;
+    inflate_copyright;
+    inflate_fast;
+    inflate_table;
+    zcalloc;
+    zcfree;
+    z_errmsg;
+    gz_error;
+    gz_intmax;
+    _*;
+};
+
+ZLIB_1.2.0.2 {
+    gzclearerr;
+    gzungetc;
+    zlibCompileFlags;
+} ZLIB_1.2.0;
+
+ZLIB_1.2.0.8 {
+    deflatePrime;
+} ZLIB_1.2.0.2;
+
+ZLIB_1.2.2 {
+    adler32_combine;
+    crc32_combine;
+    deflateSetHeader;
+    inflateGetHeader;
+} ZLIB_1.2.0.8;
+
+ZLIB_1.2.2.3 {
+    deflateTune;
+    gzdirect;
+} ZLIB_1.2.2;
+
+ZLIB_1.2.2.4 {
+    inflatePrime;
+} ZLIB_1.2.2.3;
+
+ZLIB_1.2.3.3 {
+    adler32_combine64;
+    crc32_combine64;
+    gzopen64;
+    gzseek64;
+    gztell64;
+    inflateUndermine;
+} ZLIB_1.2.2.4;
+
+ZLIB_1.2.3.4 {
+    inflateReset2;
+    inflateMark;
+} ZLIB_1.2.3.3;
+
+ZLIB_1.2.3.5 {
+    gzbuffer;
+    gzoffset;
+    gzoffset64;
+    gzclose_r;
+    gzclose_w;
+} ZLIB_1.2.3.4;
+
+ZLIB_1.2.5.1 {
+    deflatePending;
+} ZLIB_1.2.3.5;
+
+ZLIB_1.2.5.2 {
+    deflateResetKeep;
+    gzgetc_;
+    inflateResetKeep;
+} ZLIB_1.2.5.1;
+
+ZLIB_1.2.7.1 {
+    inflateGetDictionary;
+    gzvprintf;
+} ZLIB_1.2.5.2;
+
+ZLIB_1.2.9 {
+    inflateCodesUsed;
+    inflateValidate;
+    uncompress2;
+    gzfread;
+    gzfwrite;
+    deflateGetDictionary;
+    adler32_z;
+    crc32_z;
+} ZLIB_1.2.7.1;
+
+ZLIB_1.2.12 {
+	crc32_combine_gen;
+	crc32_combine_gen64;
+	crc32_combine_op;
+} ZLIB_1.2.9;
diff --git a/zlib/zutil.c b/zlib/zutil.c
index 08787c6..b1c5d2d 100644
--- a/zlib/zutil.c
+++ b/zlib/zutil.c
@@ -3,7 +3,7 @@
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
-/* @(#) $Id: zutil.c,v 1.1.1.2 2002/03/11 21:53:27 tromey Exp $ */
+/* @(#) $Id$ */
 
 #include "zutil.h"
 #ifndef Z_SOLO
@@ -24,13 +24,11 @@ z_const char * const z_errmsg[10] = {
 };
 
 
-const char * ZEXPORT zlibVersion()
-{
+const char * ZEXPORT zlibVersion(void) {
     return ZLIB_VERSION;
 }
 
-uLong ZEXPORT zlibCompileFlags()
-{
+uLong ZEXPORT zlibCompileFlags(void) {
     uLong flags;
 
     flags = 0;
@@ -61,9 +59,11 @@ uLong ZEXPORT zlibCompileFlags()
 #ifdef ZLIB_DEBUG
     flags += 1 << 8;
 #endif
+    /*
 #if defined(ASMV) || defined(ASMINF)
     flags += 1 << 9;
 #endif
+     */
 #ifdef ZLIB_WINAPI
     flags += 1 << 10;
 #endif
@@ -119,9 +119,7 @@ uLong ZEXPORT zlibCompileFlags()
 #  endif
 int ZLIB_INTERNAL z_verbose = verbose;
 
-void ZLIB_INTERNAL z_error (m)
-    char *m;
-{
+void ZLIB_INTERNAL z_error(char *m) {
     fprintf(stderr, "%s\n", m);
     exit(1);
 }
@@ -130,14 +128,12 @@ void ZLIB_INTERNAL z_error (m)
 /* exported to allow conversion of error code to string for compress() and
  * uncompress()
  */
-const char * ZEXPORT zError(err)
-    int err;
-{
+const char * ZEXPORT zError(int err) {
     return ERR_MSG(err);
 }
 
-#if defined(_WIN32_WCE)
-    /* The Microsoft C Run-Time Library for Windows CE doesn't have
+#if defined(_WIN32_WCE) && _WIN32_WCE < 0x800
+    /* The older Microsoft C Run-Time Library for Windows CE doesn't have
      * errno.  We define it as a global variable to simplify porting.
      * Its value is always 0 and should not be used.
      */
@@ -146,22 +142,14 @@ const char * ZEXPORT zError(err)
 
 #ifndef HAVE_MEMCPY
 
-void ZLIB_INTERNAL zmemcpy(dest, source, len)
-    Bytef* dest;
-    const Bytef* source;
-    uInt  len;
-{
+void ZLIB_INTERNAL zmemcpy(Bytef* dest, const Bytef* source, uInt len) {
     if (len == 0) return;
     do {
         *dest++ = *source++; /* ??? to be unrolled */
     } while (--len != 0);
 }
 
-int ZLIB_INTERNAL zmemcmp(s1, s2, len)
-    const Bytef* s1;
-    const Bytef* s2;
-    uInt  len;
-{
+int ZLIB_INTERNAL zmemcmp(const Bytef* s1, const Bytef* s2, uInt len) {
     uInt j;
 
     for (j = 0; j < len; j++) {
@@ -170,10 +158,7 @@ int ZLIB_INTERNAL zmemcmp(s1, s2, len)
     return 0;
 }
 
-void ZLIB_INTERNAL zmemzero(dest, len)
-    Bytef* dest;
-    uInt  len;
-{
+void ZLIB_INTERNAL zmemzero(Bytef* dest, uInt len) {
     if (len == 0) return;
     do {
         *dest++ = 0;  /* ??? to be unrolled */
@@ -214,8 +199,7 @@ local ptr_table table[MAX_PTR];
  * a protected system like OS/2. Use Microsoft C instead.
  */
 
-voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size)
-{
+voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, unsigned size) {
     voidpf buf;
     ulg bsize = (ulg)items*size;
 
@@ -240,8 +224,7 @@ voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size)
     return buf;
 }
 
-void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr)
-{
+void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) {
     int n;
 
     (void)opaque;
@@ -277,14 +260,12 @@ void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr)
 #  define _hfree   hfree
 #endif
 
-voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, uInt items, uInt size)
-{
+voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, uInt items, uInt size) {
     (void)opaque;
     return _halloc((long)items, size);
 }
 
-void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr)
-{
+void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) {
     (void)opaque;
     _hfree(ptr);
 }
@@ -297,25 +278,18 @@ void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr)
 #ifndef MY_ZCALLOC /* Any system without a special alloc function */
 
 #ifndef STDC
-extern voidp  malloc OF((uInt size));
-extern voidp  calloc OF((uInt items, uInt size));
-extern void   free   OF((voidpf ptr));
+extern voidp malloc(uInt size);
+extern voidp calloc(uInt items, uInt size);
+extern void free(voidpf ptr);
 #endif
 
-voidpf ZLIB_INTERNAL zcalloc (opaque, items, size)
-    voidpf opaque;
-    unsigned items;
-    unsigned size;
-{
+voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, unsigned size) {
     (void)opaque;
     return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) :
                               (voidpf)calloc(items, size);
 }
 
-void ZLIB_INTERNAL zcfree (opaque, ptr)
-    voidpf opaque;
-    voidpf ptr;
-{
+void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) {
     (void)opaque;
     free(ptr);
 }
diff --git a/zlib/zutil.h b/zlib/zutil.h
index 4b596ad..48dd7fe 100644
--- a/zlib/zutil.h
+++ b/zlib/zutil.h
@@ -1,5 +1,5 @@
 /* zutil.h -- internal interface and configuration of the compression library
- * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler
+ * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -8,7 +8,7 @@
    subject to change. Applications should only use zlib.h.
  */
 
-/* @(#) $Id: zutil.h,v 1.1.1.2 2002/03/11 21:53:28 tromey Exp $ */
+/* @(#) $Id$ */
 
 #ifndef ZUTIL_H
 #define ZUTIL_H
@@ -29,10 +29,6 @@
 #  include <stdlib.h>
 #endif
 
-#ifdef Z_SOLO
-   typedef long ptrdiff_t;  /* guess -- will be caught if guess is wrong */
-#endif
-
 #ifndef local
 #  define local static
 #endif
@@ -46,10 +42,21 @@ typedef unsigned short ush;
 typedef ush FAR ushf;
 typedef unsigned long  ulg;
 
+#if !defined(Z_U8) && !defined(Z_SOLO) && defined(STDC)
+#  include <limits.h>
+#  if (ULONG_MAX == 0xffffffffffffffff)
+#    define Z_U8 unsigned long
+#  elif (ULLONG_MAX == 0xffffffffffffffff)
+#    define Z_U8 unsigned long long
+#  elif (UINT_MAX == 0xffffffffffffffff)
+#    define Z_U8 unsigned
+#  endif
+#endif
+
 extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
 /* (size given to avoid silly warnings with Visual C++) */
 
-#define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)]
+#define ERR_MSG(err) z_errmsg[(err) < -6 || (err) > 2 ? 9 : 2 - (err)]
 
 #define ERR_RETURN(strm,err) \
   return (strm->msg = ERR_MSG(err), (err))
@@ -130,17 +137,8 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
 #  endif
 #endif
 
-#if defined(MACOS) || defined(TARGET_OS_MAC)
+#if defined(MACOS)
 #  define OS_CODE  7
-#  ifndef Z_SOLO
-#    if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os
-#      include <unix.h> /* for fdopen */
-#    else
-#      ifndef fdopen
-#        define fdopen(fd,mode) NULL /* No fdopen() */
-#      endif
-#    endif
-#  endif
 #endif
 
 #ifdef __acorn
@@ -163,22 +161,6 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
 #  define OS_CODE 19
 #endif
 
-#if defined(_BEOS_) || defined(RISCOS)
-#  define fdopen(fd,mode) NULL /* No fdopen() */
-#endif
-
-#if (defined(_MSC_VER) && (_MSC_VER > 600)) && !defined __INTERIX
-#  if defined(_WIN32_WCE)
-#    define fdopen(fd,mode) NULL /* No fdopen() */
-#    ifndef _PTRDIFF_T_DEFINED
-       typedef int ptrdiff_t;
-#      define _PTRDIFF_T_DEFINED
-#    endif
-#  else
-#    define fdopen(fd,type)  _fdopen(fd,type)
-#  endif
-#endif
-
 #if defined(__BORLANDC__) && !defined(MSDOS)
   #pragma warn -8004
   #pragma warn -8008
@@ -188,8 +170,9 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
 /* provide prototypes for these when building zlib without LFS */
 #if !defined(_WIN32) && \
     (!defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0)
-    ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t));
-    ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t));
+    ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off_t);
+    ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off_t);
+    ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off_t);
 #endif
 
         /* common defaults */
@@ -228,16 +211,16 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
 #    define zmemzero(dest, len) memset(dest, 0, len)
 #  endif
 #else
-   void ZLIB_INTERNAL zmemcpy OF((Bytef* dest, const Bytef* source, uInt len));
-   int ZLIB_INTERNAL zmemcmp OF((const Bytef* s1, const Bytef* s2, uInt len));
-   void ZLIB_INTERNAL zmemzero OF((Bytef* dest, uInt len));
+   void ZLIB_INTERNAL zmemcpy(Bytef* dest, const Bytef* source, uInt len);
+   int ZLIB_INTERNAL zmemcmp(const Bytef* s1, const Bytef* s2, uInt len);
+   void ZLIB_INTERNAL zmemzero(Bytef* dest, uInt len);
 #endif
 
 /* Diagnostic functions */
 #ifdef ZLIB_DEBUG
 #  include <stdio.h>
    extern int ZLIB_INTERNAL z_verbose;
-   extern void ZLIB_INTERNAL z_error OF((char *m));
+   extern void ZLIB_INTERNAL z_error(char *m);
 #  define Assert(cond,msg) {if(!(cond)) z_error(msg);}
 #  define Trace(x) {if (z_verbose>=0) fprintf x ;}
 #  define Tracev(x) {if (z_verbose>0) fprintf x ;}
@@ -254,9 +237,9 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
 #endif
 
 #ifndef Z_SOLO
-   voidpf ZLIB_INTERNAL zcalloc OF((voidpf opaque, unsigned items,
-                                    unsigned size));
-   void ZLIB_INTERNAL zcfree  OF((voidpf opaque, voidpf ptr));
+   voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items,
+                                unsigned size);
+   void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr);
 #endif
 
 #define ZALLOC(strm, items, size) \